Use cache for authentication backends

The commit introduces the following API incompatibilities:

In ejabberd_auth.erl:
* dirty_get_registered_users/0 is renamed to get_users/0
* get_vh_registered_users/1 is renamed to get_users/1
* get_vh_registered_users/2 is renamed to get_users/2
* get_vh_registered_users_number/1 is renamed to count_users/1
* get_vh_registered_users_number/2 is renamed to count_users/2

In ejabberd_auth callbacks
* plain_password_required/0 is replaced by plain_password_required/1
  where the argument is a virtual host
* store_type/0 is replaced by store_type/1 where the argument is
  a virtual host
* set_password/3 is now an optional callback
* remove_user/3 callback is no longer needed
* remove_user/2 now should return `ok | {error, atom()}`
* is_user_exists/2 now must only be implemented for backends
  with `external` store type
* check_password/6 is no longer needed
* check_password/4 now must only be implemented for backends
  with `external` store type
* try_register/3 is now an optional callback and should return
  `ok | {error, atom()}`
* dirty_get_registered_users/0 is no longer needed
* get_vh_registered_users/1 is no longer needed
* get_vh_registered_users/2 is renamed to get_users/2
* get_vh_registered_users_number/1 is no longer needed
* get_vh_registered_users_number/2 is renamed to count_users/2
* get_password_s/2 is no longer needed
* get_password/2 now must only be implemented for backends with
  `plain` or `scram` store type

Additionally, the commit introduces two new callbacks:
* use_cache/1 where the argument is a virtual host
* cache_nodes/1 where the argument is a virtual host

New options are also introduced: `auth_use_cache`, `auth_cache_missed`,
`auth_cache_life_time` and `auth_cache_size`.

1 files changed, 64 insertions, 384 deletions

diff --git a/src/ejabberd_auth_sql.erl b/src/ejabberd_auth_sql.erl
index 8514b9cf1..d682634f0 100644
--- a/src/ejabberd_auth_sql.erl
+++ b/src/ejabberd_auth_sql.erl
@@ -31,14 +31,9 @@
 
 -behaviour(ejabberd_auth).
 
--export([start/1, stop/1, set_password/3, check_password/4,
-	 check_password/6, try_register/3,
-	 dirty_get_registered_users/0, get_vh_registered_users/1,
-	 get_vh_registered_users/2,
-	 get_vh_registered_users_number/1,
-	 get_vh_registered_users_number/2, get_password/2,
-	 get_password_s/2, is_user_exists/2, remove_user/2,
-	 remove_user/3, store_type/0, plain_password_required/0,
+-export([start/1, stop/1, set_password/3, try_register/3,
+	 get_users/2, count_users/2, get_password/2,
+	 remove_user/2, store_type/1, plain_password_required/1,
 	 convert_to_scram/1]).
 
 -include("ejabberd.hrl").
@@ -54,397 +49,82 @@ start(_Host) -> ok.
 
 stop(_Host) -> ok.
 
-plain_password_required() ->
-    case is_scrammed() of
-      false -> false;
-      true -> true
-    end.
-
-store_type() ->
-    case is_scrammed() of
-      false -> plain; %% allows: PLAIN DIGEST-MD5 SCRAM
-      true -> scram %% allows: PLAIN SCRAM
-    end.
+plain_password_required(Server) ->
+    store_type(Server) == scram.
 
-%% @spec (User, AuthzId, Server, Password) -> true | false | {error, Error}
-check_password(User, AuthzId, Server, Password) ->
-    if AuthzId /= <<>> andalso AuthzId /= User ->
-        false;
-    true ->
-        LServer = jid:nameprep(Server),
-        LUser = jid:nodeprep(User),
-    if (LUser == error) or (LServer == error) ->
-            false;
-       (LUser == <<>>) or (LServer == <<>>) ->
-            false;
-       true ->
-            case is_scrammed() of
-                true ->
-                    try sql_queries:get_password_scram(LServer, LUser) of
-                        {selected,
-                         [{StoredKey, ServerKey, Salt, IterationCount}]} ->
-                            Scram =
-                                #scram{storedkey = StoredKey,
-                                       serverkey = ServerKey,
-                                       salt = Salt,
-                                       iterationcount = IterationCount},
-                            is_password_scram_valid_stored(Password, Scram, LUser, LServer);
-                        {selected, []} ->
-                            false; %% Account does not exist
-                        {error, _Error} ->
-                            false %% Typical error is that table doesn't exist
-                    catch
-                        _:_ ->
-                            false %% Typical error is database not accessible
-                    end;
-                false ->
-                    try sql_queries:get_password(LServer, LUser) of
-                        {selected, [{Password}]} ->
-                            Password /= <<"">>;
-                        {selected, [{_Password2}]} ->
-                            false; %% Password is not correct
-                        {selected, []} ->
-                            false; %% Account does not exist
-                        {error, _Error} ->
-                            false %% Typical error is that table doesn't exist
-                    catch
-                        _:_ ->
-                            false %% Typical error is database not accessible
-                    end
-            end
-        end
-    end.
+store_type(Server) ->
+    ejabberd_auth:password_format(Server).
 
-%% @spec (User, AuthzId, Server, Password, Digest, DigestGen) -> true | false | {error, Error}
-check_password(User, AuthzId, Server, Password, Digest,
-	       DigestGen) ->
-    if AuthzId /= <<>> andalso AuthzId /= User ->
-        false;
-    true ->
-        LServer = jid:nameprep(Server),
-        LUser = jid:nodeprep(User),
-    if (LUser == error) or (LServer == error) ->
-            false;
-       (LUser == <<>>) or (LServer == <<>>) ->
-            false;
-       true ->
-            case is_scrammed() of
-                false ->
-                    try sql_queries:get_password(LServer, LUser) of
-                        %% Account exists, check if password is valid
-                        {selected, [{Passwd}]} ->
-                            DigRes = if Digest /= <<"">> ->
-                                             Digest == DigestGen(Passwd);
-                                        true -> false
-                                     end,
-                            if DigRes -> true;
-                               true -> (Passwd == Password) and (Password /= <<"">>)
-                            end;
-                        {selected, []} ->
-                            false; %% Account does not exist
-                        {error, _Error} ->
-                            false %% Typical error is that table doesn't exist
-                    catch
-                        _:_ ->
-                            false %% Typical error is database not accessible
-                    end;
-                true ->
-                    false
-            end
-        end
-    end.
-
-%% @spec (User::string(), Server::string(), Password::string()) ->
-%%       ok | {error, invalid_jid}
 set_password(User, Server, Password) ->
-    LServer = jid:nameprep(Server),
-    LUser = jid:nodeprep(User),
-    LPassword = jid:resourceprep(Password),
-    if (LUser == error) or (LServer == error) ->
-            {error, invalid_jid};
-       (LUser == <<>>) or (LServer == <<>>) ->
-            {error, invalid_jid};
-       LPassword == error ->
-	   {error, invalid_password};
-       true ->
-            case is_scrammed() of
-                true ->
-                    Scram = password_to_scram(Password),
-                    case catch sql_queries:set_password_scram_t(
-                                 LServer,
-                                 LUser,
-                                 Scram#scram.storedkey,
-                                 Scram#scram.serverkey,
-                                 Scram#scram.salt,
-                                 Scram#scram.iterationcount
-                                )
-                        of
-                        {atomic, ok} -> ok;
-                        Other -> {error, Other}
-                    end;
-                false ->
-                    case catch sql_queries:set_password_t(LServer,
-                                                           LUser, Password)
-                        of
-                        {atomic, ok} -> ok;
-                        Other -> {error, Other}
-                    end
-            end
+    Res = if is_record(Password, scram) ->
+		  sql_queries:set_password_scram_t(
+		    Server, User,
+		    Password#scram.storedkey, Password#scram.serverkey,
+		    Password#scram.salt, Password#scram.iterationcount);
+	     true ->
+		  sql_queries:set_password_t(Server, User, Password)
+	  end,
+    case Res of
+	{atomic, _} ->
+	    ok;
+	{aborted, Reason} ->
+	    ?ERROR_MSG("failed to write to SQL table: ~p", [Reason]),
+	    {error, db_failure}
     end.
 
-%% @spec (User, Server, Password) -> {atomic, ok} | {atomic, exists} | {error, invalid_jid}
 try_register(User, Server, Password) ->
-    LServer = jid:nameprep(Server),
-    LUser = jid:nodeprep(User),
-    LPassword = jid:resourceprep(Password),
-    if (LUser == error) or (LServer == error) ->
-            {error, invalid_jid};
-       (LUser == <<>>) or (LServer == <<>>) ->
-            {error, invalid_jid};
-       LPassword == error and not is_record(Password, scram) ->
-	   {error, invalid_password};
-       true ->
-            case is_scrammed() of
-                true ->
-                    Scram = case is_record(Password, scram) of
-                        true -> Password;
-                        false -> password_to_scram(Password)
-                    end,
-                    case catch sql_queries:add_user_scram(
-                                 LServer,
-                                 LUser,
-                                 Scram#scram.storedkey,
-                                 Scram#scram.serverkey,
-                                 Scram#scram.salt,
-                                 Scram#scram.iterationcount
-                                ) of
-                        {updated, 1} -> {atomic, ok};
-                        _ -> {atomic, exists}
-                    end;
-                false ->
-                    case catch sql_queries:add_user(LServer, LUser,
-                                                     Password) of
-                        {updated, 1} -> {atomic, ok};
-                        _ -> {atomic, exists}
-                    end
-            end
+    Res = if is_record(Password, scram) ->
+		  sql_queries:add_user_scram(
+		    Server, User,
+		    Password#scram.storedkey, Password#scram.serverkey,
+		    Password#scram.salt, Password#scram.iterationcount);
+	     true ->
+		  sql_queries:add_user(Server, User, Password)
+	  end,
+    case Res of
+	{updated, 1} -> ok;
+	_ -> {error, exists}
     end.
 
-dirty_get_registered_users() ->
-    Servers = ejabberd_config:get_vh_by_auth_method(sql),
-    lists:flatmap(fun (Server) ->
-			  get_vh_registered_users(Server)
-		  end,
-		  Servers).
-
-get_vh_registered_users(Server) ->
-    case jid:nameprep(Server) of
-        error -> [];
-        <<>> -> [];
-        LServer ->
-            case catch sql_queries:list_users(LServer) of
-                {selected, Res} ->
-                    [{U, LServer} || {U} <- Res];
-                _ -> []
-            end
+get_users(Server, Opts) ->
+    case sql_queries:list_users(Server, Opts) of
+	{selected, Res} ->
+	    [{U, Server} || {U} <- Res];
+	_ -> []
     end.
 
-get_vh_registered_users(Server, Opts) ->
-    case jid:nameprep(Server) of
-        error -> [];
-        <<>> -> [];
-        LServer ->
-            case catch sql_queries:list_users(LServer, Opts) of
-                {selected, Res} ->
-                    [{U, LServer} || {U} <- Res];
-                _ -> []
-            end
-    end.
-
-get_vh_registered_users_number(Server) ->
-    case jid:nameprep(Server) of
-        error -> 0;
-        <<>> -> 0;
-        LServer ->
-            case catch sql_queries:users_number(LServer) of
-                {selected, [{Res}]} ->
-                    Res;
-                _ -> 0
-            end
-    end.
-
-get_vh_registered_users_number(Server, Opts) ->
-    case jid:nameprep(Server) of
-        error -> 0;
-        <<>> -> 0;
-        LServer ->
-            case catch sql_queries:users_number(LServer, Opts) of
-                {selected, [{Res}]} ->
-                    Res;
-                _Other -> 0
-            end
+count_users(Server, Opts) ->
+    case sql_queries:users_number(Server, Opts) of
+	{selected, [{Res}]} ->
+	    Res;
+	_Other -> 0
     end.
 
 get_password(User, Server) ->
-    LServer = jid:nameprep(Server),
-    LUser = jid:nodeprep(User),
-    if (LUser == error) or (LServer == error) ->
-            false;
-       (LUser == <<>>) or (LServer == <<>>) ->
-            false;
-       true ->
-            case is_scrammed() of
-                true ->
-                    case catch sql_queries:get_password_scram(
-                                 LServer, LUser) of
-                        {selected,
-                         [{StoredKey, ServerKey, Salt, IterationCount}]} ->
-                            {misc:decode_base64(StoredKey),
-                             misc:decode_base64(ServerKey),
-                             misc:decode_base64(Salt),
-                             IterationCount};
-                        _ -> false
-                    end;
-                false ->
-                    case catch sql_queries:get_password(LServer, LUser)
-                        of
-                        {selected, [{Password}]} -> Password;
-                        _ -> false
-                    end
-            end
-    end.
-
-get_password_s(User, Server) ->
-    LServer = jid:nameprep(Server),
-    LUser = jid:nodeprep(User),
-    if (LUser == error) or (LServer == error) ->
-            <<"">>;
-       (LUser == <<>>) or (LServer == <<>>) ->
-            <<"">>;
-       true ->
-            case is_scrammed() of
-                false ->
-                    case catch sql_queries:get_password(LServer, LUser) of
-                        {selected, [{Password}]} -> Password;
-                        _ -> <<"">>
-                    end;
-                true -> <<"">>
-            end
+    case sql_queries:get_password_scram(Server, User) of
+	{selected, [{Password, <<>>, <<>>, 0}]} ->
+	    {ok, Password};
+	{selected, [{StoredKey, ServerKey, Salt, IterationCount}]} ->
+	    {ok, #scram{storedkey = StoredKey,
+			serverkey = ServerKey,
+			salt = Salt,
+			iterationcount = IterationCount}};
+	{selected, []} ->
+	    error;
+	Err ->
+	    ?ERROR_MSG("Failed to read password for user ~s@~s: ~p",
+		       [User, Server, Err]),
+	    error
     end.
 
-%% @spec (User, Server) -> true | false | {error, Error}
-is_user_exists(User, Server) ->
-    LServer = jid:nameprep(Server),
-    LUser = jid:nodeprep(User),
-    if (LUser == error) or (LServer == error) ->
-            false;
-       (LUser == <<>>) or (LServer == <<>>) ->
-            false;
-       true ->
-	  try sql_queries:get_password(LServer, LUser) of
-	    {selected, [{_Password}]} ->
-		true; %% Account exists
-	    {selected, []} ->
-		false; %% Account does not exist
-	    {error, Error} -> {error, Error}
-	  catch
-	    _:B -> {error, B}
-	  end
-    end.
-
-%% @spec (User, Server) -> ok | error
-%% @doc Remove user.
-%% Note: it may return ok even if there was some problem removing the user.
 remove_user(User, Server) ->
-    LServer = jid:nameprep(Server),
-    LUser = jid:nodeprep(User),
-    if (LUser == error) or (LServer == error) ->
-            error;
-       (LUser == <<>>) or (LServer == <<>>) ->
-            error;
-       true ->
-	  catch sql_queries:del_user(LServer, LUser),
-	  ok
-    end.
-
-%% @spec (User, Server, Password) -> ok | error | not_exists | not_allowed
-%% @doc Remove user if the provided password is correct.
-remove_user(User, Server, Password) ->
-    LServer = jid:nameprep(Server),
-    LUser = jid:nodeprep(User),
-    if (LUser == error) or (LServer == error) ->
-            error;
-       (LUser == <<>>) or (LServer == <<>>) ->
-            error;
-       true ->
-            case is_scrammed() of
-                true ->
-                    case check_password(User, <<"">>, Server, Password) of
-                        true ->
-                            remove_user(User, Server),
-                            ok;
-                        false -> not_allowed
-                    end;
-                false ->
-                    F = fun () ->
-                                Result = sql_queries:del_user_return_password(
-                                           LServer, LUser, Password),
-                                case Result of
-                                    {selected, [{Password}]} -> ok;
-                                    {selected, []} -> not_exists;
-                                    _ -> not_allowed
-                                end
-                        end,
-                    {atomic, Result} = sql_queries:sql_transaction(
-                                         LServer, F),
-                    Result
-            end
-    end.
-
-%%%
-%%% SCRAM
-%%%
-
-is_scrammed() ->
-    scram == ejabberd_auth:password_format(?MYNAME).
-
-password_to_scram(Password) ->
-    password_to_scram(Password,
-		      ?SCRAM_DEFAULT_ITERATION_COUNT).
-
-password_to_scram(Password, IterationCount) ->
-    Salt = randoms:bytes(?SALT_LENGTH),
-    SaltedPassword = scram:salted_password(Password, Salt,
-					   IterationCount),
-    StoredKey =
-	scram:stored_key(scram:client_key(SaltedPassword)),
-    ServerKey = scram:server_key(SaltedPassword),
-    #scram{storedkey = misc:encode_base64(StoredKey),
-	   serverkey = misc:encode_base64(ServerKey),
-	   salt = misc:encode_base64(Salt),
-	   iterationcount = IterationCount}.
-
-is_password_scram_valid_stored(Pass, {scram,Pass,<<>>,<<>>,0}, LUser, LServer) ->
-    ?INFO_MSG("Apparently, SQL auth method and scram password formatting are "
-	"enabled, but the password of user '~s' in the 'users' table is not "
-	"scrammed. You may want to execute this command: "
-	"ejabberdctl convert_to_scram ~s", [LUser, LServer]),
-    false;
-is_password_scram_valid_stored(Password, Scram, _, _) ->
-    is_password_scram_valid(Password, Scram).
-
-is_password_scram_valid(Password, Scram) ->
-    case jid:resourceprep(Password) of
-	error ->
-	    false;
-	_ ->
-	    IterationCount = Scram#scram.iterationcount,
-	    Salt = misc:decode_base64(Scram#scram.salt),
-	    SaltedPassword = scram:salted_password(Password, Salt,
-						   IterationCount),
-	    StoredKey =
-		scram:stored_key(scram:client_key(SaltedPassword)),
-	    misc:decode_base64(Scram#scram.storedkey) == StoredKey
+    case sql_queries:del_user(Server, User) of
+	{updated, _} ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to delete user ~s@~s: ~p",
+		       [User, Server, Err]),
+	    {error, db_failure}
     end.
 
 -define(BATCH_SIZE, 1000).
@@ -485,7 +165,7 @@ convert_to_scram(Server) ->
 						     "password of user ~s@~s",
 						     [LUser, LServer]);
 					      _ ->
-						  Scram = password_to_scram(Password),
+						  Scram = ejabberd_auth:password_to_scram(Password),
 						  set_password_scram_t(
 						    LUser,
 						    Scram#scram.storedkey,