From f9a8b9ff314d8de829f2a70ea807a86ea8f2d8ef Mon Sep 17 00:00:00 2001 From: Badlop Date: Fri, 7 Aug 2009 09:52:45 +0000 Subject: Add support for XEP-227 import/export, requires exmpp to use it (EJAB-993) SVN Revision: 2439 --- src/ejabberd_piefxis.erl | 690 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 690 insertions(+) create mode 100644 src/ejabberd_piefxis.erl (limited to 'src/ejabberd_piefxis.erl') diff --git a/src/ejabberd_piefxis.erl b/src/ejabberd_piefxis.erl new file mode 100644 index 00000000..1c085383 --- /dev/null +++ b/src/ejabberd_piefxis.erl @@ -0,0 +1,690 @@ +%%%---------------------------------------------------------------------- +%%% File : ejabberd_piefxis.erl +%%% Author : Pablo Polvorin, Vidal Santiago Martinez +%%% Purpose : XEP-0227: Portable Import/Export Format for XMPP-IM Servers +%%% Created : 17 Jul 2008 by Pablo Polvorin +%%% +%%% +%%% ejabberd, Copyright (C) 2002-2009 ProcessOne +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License +%%% along with this program; if not, write to the Free Software +%%% Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +%%% 02111-1307 USA +%%% +%%%---------------------------------------------------------------------- + +%%% Not implemented: +%%% - write mod_piefxis with ejabberdctl commands +%%% - Export from mod_offline_odbc.erl +%%% - Export from mod_private_odbc.erl +%%% - XEP-227: 6. Security Considerations +%%% - Other schemas of XInclude are not tested, and may not be imported correctly. +%%% - If a host has many users, split that host in XML files with 50 users each. + +%%%% Headers + +-module(ejabberd_piefxis). + +-export([import_file/1, export_server/1, export_host/2]). + +-record(parsing_state, {parser, host, dir}). + +-include("ejabberd.hrl"). + +%%-include_lib("exmpp/include/exmpp.hrl"). +%%-include_lib("exmpp/include/exmpp_client.hrl"). +%% Copied from exmpp header files: +-define(NS_ROSTER, "jabber:iq:roster"). +-define(NS_VCARD, "vcard-temp"). +-type(xmlname() :: atom() | string()). +-record(xmlcdata, { + cdata = <<>> :: binary() + }). +-type(xmlcdata() :: #xmlcdata{}). +-record(xmlattr, { + ns = undefined :: xmlname() | undefined, + name :: xmlname(), + value :: binary() + }). +-type(xmlattr() :: #xmlattr{}). +-record(xmlel, { + ns = undefined :: xmlname() | undefined, + declared_ns = [] :: [{xmlname(), string() | none}], + name :: xmlname(), + attrs = [] :: [xmlattr()], + children = [] :: [#xmlel{} | xmlcdata()] | undefined + }). +-record(iq, { + kind :: request | response, + type :: get | set | result | error, + id :: binary() | undefined, + ns :: xmlname() | undefined, + payload :: #xmlel{} | undefined, + error :: #xmlel{} | undefined, + lang :: binary() | undefined, + iq_ns :: xmlname() | undefined + }). +-record(xmlendtag, { + ns = undefined :: xmlname() | undefined, + name :: xmlname() + }). + + +%% Copied from mod_private.erl +-record(private_storage, {usns, xml}). + +%%-define(ERROR_MSG(M,Args),io:format(M,Args)). +%%-define(INFO_MSG(M,Args),ok). + +-define(CHUNK_SIZE,1024*20). %20k + +-define(BTL, binary_to_list). +-define(LTB, list_to_binary). + +-define(NS_XINCLUDE, 'http://www.w3.org/2001/XInclude'). + +%%%================================== + +%%%% Import file + +import_file(FileName) -> + _ = #xmlattr{}, %% this stupid line is only to prevent compilation warning about "recod xmlattr is unused" + import_file(FileName, 2). + +import_file(FileName, RootDepth) -> + try_start_exmpp(), + Dir = filename:dirname(FileName), + {ok, IO} = try_open_file(FileName), + Parser = exmpp_xml:start_parser([{max_size,infinity}, + {root_depth, RootDepth}, + {emit_endtag,true}]), + read_chunks(IO, #parsing_state{parser=Parser, dir=Dir}), + file:close(IO), + exmpp_xml:stop_parser(Parser). + +try_start_exmpp() -> + try exmpp:start() + catch + error:{already_started, exmpp} -> ok; + error:undef -> throw({error, exmpp_not_installed}) + end. + +try_open_file(FileName) -> + case file:open(FileName,[read,binary]) of + {ok, IO} -> {ok, IO}; + {error, enoent} -> throw({error, {file_not_found, FileName}}) + end. + +%%File could be large.. we read it in chunks +read_chunks(IO,State) -> + case file:read(IO,?CHUNK_SIZE) of + {ok,Chunk} -> + NewState = process_chunk(Chunk,State), + read_chunks(IO,NewState); + eof -> + ok + end. + +process_chunk(Chunk,S =#parsing_state{parser=Parser}) -> + case exmpp_xml:parse(Parser,Chunk) of + continue -> + S; + XMLElements -> + process_elements(XMLElements,S) + end. + +%%%================================== +%%%% Process Elements + +process_elements(Elements,State) -> + lists:foldl(fun process_element/2,State,Elements). + +%%%================================== +%%%% Process Element + +process_element(El=#xmlel{name=user, ns=_XMLNS}, + State=#parsing_state{host=Host}) -> + case add_user(El,Host) of + {error, _Other} -> error; + _ -> ok + end, + State; + +process_element(H=#xmlel{name=host},State) -> + State#parsing_state{host=?BTL(exmpp_xml:get_attribute(H,"jid",none))}; + +process_element(#xmlel{name='server-data'},State) -> + State; + +process_element(El=#xmlel{name=include, ns=?NS_XINCLUDE}, State=#parsing_state{dir=Dir}) -> + case exmpp_xml:get_attribute(El, href, none) of + none -> + ok; + HrefB -> + Href = binary_to_list(HrefB), + %%?INFO_MSG("Parse also this file: ~n~p", [Href]), + FileName = filename:join([Dir, Href]), + import_file(FileName, 1), + Href + end, + State; + +process_element(#xmlcdata{cdata = _CData},State) -> + State; + +process_element(#xmlendtag{ns = _NS, name='server-data'},State) -> + State; + +process_element(#xmlendtag{ns = _NS, name=_Name},State) -> + State; + +process_element(El,State) -> + io:format("Warning!: unknown element found: ~p ~n",[El]), + State. + +%%%================================== +%%%% Add user + +add_user(El, Domain) -> + User = exmpp_xml:get_attribute(El,name,none), + Password = exmpp_xml:get_attribute(El,password,none), + add_user(El, Domain, ?BTL(User), ?BTL(Password)). + +%% @spec El = XML element +%% Domain = String with a domain name +%% User = String with an user name +%% Password = String with an user password +%% @ret ok | {atomic, exists} | {error, not_allowed} +%% @doc Add a new user to the database. +%% If user already exists, it will be only updated. +add_user(El, Domain, User, Password) -> + case create_user(User,Password,Domain) of + ok -> + ok = exmpp_xml:foreach( + fun(_,Child) -> + populate_user(User,Domain,Child) + end, + El), + ok; + {atomic, exists} -> + ?INFO_MSG("User ~p@~p already exists, using stored profile...~n", + [User, Domain]), + io:format(""), + ok = exmpp_xml:foreach( + fun(_,Child) -> + populate_user(User,Domain,Child) + end, + El); + {error, Other} -> + ?ERROR_MSG("Error adding user ~s@~s: ~p~n", [User, Domain, Other]) + end. + +%% @spec User = String with User name +%% Password = String with a Password value +%% Domain = Stirng with a Domain name +%% @ret ok | {atomic, exists} | {error, not_allowed} +%% @doc Create a new user +create_user(User,Password,Domain) -> + case ejabberd_auth:try_register(User,Domain,Password) of + {atomic,ok} -> ok; + {atomic, exists} -> {atomic, exists}; + {error, not_allowed} -> {error, not_allowed}; + Other -> {error, Other} + end. + +%%%================================== +%%%% Populate user + +%% @spec User = String +%% Domain = String +%% El = XML element +%% @ret ok | {error, not_found} +%% +%% @doc Add a new user from a XML file with a roster list. +%% +%% Example of a file: +%% +%% +%% +%% +%% +%% +%% Friends +%% +%% +%% +%% +%% + +populate_user(User,Domain,El=#xmlel{name='query', ns='jabber:iq:roster'}) -> + io:format("Trying to add/update roster list...",[]), + case loaded_module(Domain,[mod_roster_odbc,mod_roster]) of + {ok, M} -> + case M:set_items(User, Domain, exmpp_xml:xmlel_to_xmlelement(El)) of + {atomic, ok} -> + io:format(" DONE.~n",[]), + ok; + _ -> + io:format(" ERROR.~n",[]), + ?ERROR_MSG("Error trying to add a new user: ~s ~n", + [exmpp_xml:document_to_list(El)]), + {error, not_found} + end; + E -> io:format(" ERROR: ~p~n",[E]), + ?ERROR_MSG("No modules loaded [mod_roster, mod_roster_odbc] ~s ~n", + [exmpp_xml:document_to_list(El)]), + {error, not_found} + end; + + +%% @spec User = String with the user name +%% Domain = String with a domain name +%% El = Sub XML element with vCard tags values +%% @ret ok | {error, not_found} +%% @doc Read vcards from the XML and send it to the server +%% +%% Example: +%% +%% +%% +%% +%% +%% Admin +%% +%% +%% +%% + +populate_user(User,Domain,El=#xmlel{name='vCard', ns='vcard-temp'}) -> + io:format("Trying to add/update vCards...",[]), + case loaded_module(Domain,[mod_vcard,mod_vcard_odbc]) of + {ok, M} -> FullUser = jid_to_old_jid(exmpp_jid:make(User, Domain)), + IQ = iq_to_old_iq(#iq{type = set, payload = El}), + case M:process_sm_iq(FullUser, FullUser , IQ) of + {error,_Err} -> + io:format(" ERROR.~n",[]), + ?ERROR_MSG("Error processing vcard ~s : ~p ~n", + [exmpp_xml:document_to_list(El), _Err]); + _ -> + io:format(" DONE.~n",[]), ok + end; + _ -> + io:format(" ERROR.~n",[]), + ?ERROR_MSG("No modules loaded [mod_vcard, mod_vcard_odbc] ~s ~n", + [exmpp_xml:document_to_list(El)]), + {error, not_found} + end; + +%% @spec User = String with the user name +%% Domain = String with a domain name +%% El = Sub XML element with offline messages values +%% @ret ok | {error, not_found} +%% @doc Read off-line message from the XML and send it to the server + +populate_user(User,Domain,El=#xmlel{name='offline-messages'}) -> + io:format("Trying to add/update offline-messages...",[]), + case loaded_module(Domain, [mod_offline, mod_offline_odbc]) of + {ok, M} -> + ok = exmpp_xml:foreach( + fun (_Element, {xmlcdata, _}) -> + ok; + (_Element, Child) -> + From = exmpp_xml:get_attribute(Child,from,none), + FullFrom = jid_to_old_jid(exmpp_jid:parse(From)), + FullUser = jid_to_old_jid(exmpp_jid:make(User, + Domain)), + OldChild = exmpp_xml:xmlel_to_xmlelement(Child), + _R = M:store_packet(FullFrom, FullUser, OldChild) + end, El), io:format(" DONE.~n",[]); + _ -> + io:format(" ERROR.~n",[]), + ?ERROR_MSG("No modules loaded [mod_offline, mod_offline_odbc] ~s ~n", + [exmpp_xml:document_to_list(El)]), + {error, not_found} + end; + +%% @spec User = String with the user name +%% Domain = String with a domain name +%% El = Sub XML element with private storage values +%% @ret ok | {error, not_found} +%% @doc Private storage parsing + +populate_user(User,Domain,El=#xmlel{name='query', ns='jabber:iq:private'}) -> + io:format("Trying to add/update private storage...",[]), + case loaded_module(Domain,[mod_private_odbc,mod_private]) of + {ok, M} -> + FullUser = jid_to_old_jid(exmpp_jid:make(User, Domain)), + IQ = iq_to_old_iq(#iq{type = set, + ns = 'jabber:iq:private', + kind = request, + iq_ns = 'jabberd:client', + payload = El}), + case M:process_sm_iq(FullUser, FullUser, IQ ) of + {error, _Err} -> + io:format(" ERROR.~n",[]), + ?ERROR_MSG("Error processing private storage ~s : ~p ~n", + [exmpp_xml:document_to_list(El), _Err]); + _ -> io:format(" DONE.~n",[]), ok + end; + _ -> + io:format(" ERROR.~n",[]), + ?ERROR_MSG("No modules loaded [mod_private, mod_private_odbc] ~s ~n", + [exmpp_xml:document_to_list(El)]), + {error, not_found} + end; + +populate_user(_User, _Domain, #xmlcdata{cdata = _CData}) -> + ok; + +populate_user(_User, _Domain, _El) -> + ok. + +%%%================================== +%%%% Utilities + +loaded_module(Domain,Options) -> + LoadedModules = gen_mod:loaded_modules(Domain), + case lists:filter(fun(Module) -> + lists:member(Module, LoadedModules) + end, Options) of + [M|_] -> {ok, M}; + [] -> {error,not_found} + end. + +jid_to_old_jid(Jid) -> + {jid, to_list(exmpp_jid:node_as_list(Jid)), + to_list(exmpp_jid:domain_as_list(Jid)), + to_list(exmpp_jid:resource_as_list(Jid)), + to_list(exmpp_jid:prep_node_as_list(Jid)), + to_list(exmpp_jid:prep_domain_as_list(Jid)), + to_list(exmpp_jid:prep_resource_as_list(Jid))}. + +iq_to_old_iq(#iq{id = ID, type = Type, lang = Lang, ns= NS, payload = El }) -> + {iq, to_list(ID), Type, to_list(NS), to_list(Lang), + exmpp_xml:xmlel_to_xmlelement(El)}. + +to_list(L) when is_list(L) -> L; +to_list(B) when is_binary(B) -> binary_to_list(B); +to_list(undefined) -> ""; +to_list(B) when is_atom(B) -> atom_to_list(B). + +%%%================================== + +%%%% Export server + +%% @spec (Dir::string()) -> ok +export_server(Dir) -> + try_start_exmpp(), + + FnT = make_filename_template(), + DFn = make_main_basefilename(Dir, FnT), + + {ok, Fd} = file_open(DFn), + print(Fd, make_piefxis_xml_head()), + print(Fd, make_piefxis_server_head()), + + Hosts = ?MYHOSTS, + FilesAndHosts = [{make_host_filename(FnT, Host), Host} || Host <- Hosts], + [print(Fd, make_xinclude(FnH)) || {FnH, _Host} <- FilesAndHosts], + + print(Fd, make_piefxis_server_tail()), + print(Fd, make_piefxis_xml_tail()), + file_close(Fd), + + [export_host(Dir, FnH, Host) || {FnH, Host} <- FilesAndHosts], + + ok. + +%%%================================== +%%%% Export host + +%% @spec (Dir::string(), Host::string()) -> ok +export_host(Dir, Host) -> + try_start_exmpp(), + FnT = make_filename_template(), + FnH = make_host_filename(FnT, Host), + export_host(Dir, FnH, Host). + +%% @spec (Dir::string(), Fn::string(), Host::string()) -> ok +export_host(Dir, FnH, Host) -> + + DFn = make_host_basefilename(Dir, FnH), + + {ok, Fd} = file_open(DFn), + print(Fd, make_piefxis_xml_head()), + print(Fd, make_piefxis_host_head(Host)), + + Users = ejabberd_auth:get_vh_registered_users(Host), + [export_user(Fd, Username, Host) || {Username, _Host} <- Users], + + print(Fd, make_piefxis_host_tail()), + print(Fd, make_piefxis_xml_tail()), + file_close(Fd). + +%%%================================== +%%%% PIEFXIS formatting + +%% @spec () -> string() +make_piefxis_xml_head() -> + "". + +%% @spec () -> string() +make_piefxis_xml_tail() -> + "". + +%% @spec () -> string() +make_piefxis_server_head() -> + "". + +%% @spec () -> string() +make_piefxis_server_tail() -> + "". + +%% @spec (Host::string()) -> string() +make_piefxis_host_head(Host) -> + NSString = + " xmlns='http://www.xmpp.org/extensions/xep-0227.html#ns'" + " xmlns:xi='http://www.w3.org/2001/XInclude'", + io_lib:format("", [NSString, Host]). + +%% @spec () -> string() +make_piefxis_host_tail() -> + "". + +%% @spec (Fn::string()) -> string() +make_xinclude(Fn) -> + Base = filename:basename(Fn), + io_lib:format("", [Base]). + +%%%================================== +%%%% Export user + +%% @spec (Fd, Username::string(), Host::string()) -> ok +%% extraer su informacion e imprimirla +export_user(Fd, Username, Host) -> + UserString = extract_user(Username, Host), + print(Fd, UserString). + +%% @spec (Username::string(), Host::string()) -> string() +extract_user(Username, Host) -> + Password = ejabberd_auth:get_password_s(Username, Host), + UserInfo = [extract_user_info(InfoName, Username, Host) || InfoName <- [roster, offline, private, vcard]], + UserInfoString = lists:flatten(UserInfo), + io_lib:format("~s", [Username, Password, UserInfoString]). + +%% @spec (InfoName::atom(), Username::string(), Host::string()) -> string() +extract_user_info(roster, Username, Host) -> + case loaded_module(Host,[mod_roster_odbc,mod_roster]) of + {ok, M} -> + From = To = jlib:make_jid(Username, Host, ""), + SubelGet = {xmlelement, "query", [{"xmlns",?NS_ROSTER}], []}, + %%IQGet = #iq{type=get, xmlns=?NS_ROSTER, payload=SubelGet}, % this is for 3.0.0 version + IQGet = {iq, "", get, ?NS_ROSTER, "" , SubelGet}, + Res = M:process_local_iq(From, To, IQGet), + %%[El] = Res#iq.payload, % this is for 3.0.0 version + {iq, _, result, _, _, Els} = Res, + case Els of + [El] -> exmpp_xml:document_to_list(El); + [] -> "" + end; + _E -> + "" + end; + +extract_user_info(offline, Username, Host) -> + case loaded_module(Host,[mod_offline,mod_offline_odbc]) of + {ok, mod_offline} -> + Els = mnesia_pop_offline_messages([], Username, Host), + case Els of + [] -> ""; + Els -> + OfEl = {xmlelement, "offline-messages", [], Els}, + exmpp_xml:document_to_list(OfEl) + end; + {ok, mod_offline_odbc} -> + ""; + _E -> + "" + end; + +extract_user_info(private, Username, Host) -> + case loaded_module(Host,[mod_private,mod_private_odbc]) of + {ok, mod_private} -> + get_user_private_mnesia(Username, Host); + {ok, mod_private_odbc} -> + ""; + _E -> + "" + end; + +extract_user_info(vcard, Username, Host) -> + case loaded_module(Host,[mod_vcard, mod_vcard_odbc, mod_vcard_odbc]) of + {ok, M} -> + From = To = jlib:make_jid(Username, Host, ""), + SubelGet = {xmlelement, "vCard", [{"xmlns",?NS_VCARD}], []}, + %%IQGet = #iq{type=get, xmlns=?NS_VCARD, payload=SubelGet}, % this is for 3.0.0 version + IQGet = {iq, "", get, ?NS_VCARD, "" , SubelGet}, + Res = M:process_sm_iq(From, To, IQGet), + %%[El] = Res#iq.payload, % this is for 3.0.0 version + {iq, _, result, _, _, Els} = Res, + case Els of + [El] -> exmpp_xml:document_to_list(El); + [] -> "" + end; + _E -> + "" + end. + +%%%================================== +%%%% Interface with ejabberd offline storage + +%% Copied from mod_offline.erl and customized +-record(offline_msg, {us, timestamp, expire, from, to, packet}). +mnesia_pop_offline_messages(Ls, User, Server) -> + LUser = jlib:nodeprep(User), + LServer = jlib:nameprep(Server), + US = {LUser, LServer}, + F = fun() -> + Rs = mnesia:wread({offline_msg, US}), + %%mnesia:delete({offline_msg, US}), + Rs + end, + case mnesia:transaction(F) of + {atomic, Rs} -> + TS = now(), + Ls ++ lists:map( + fun(R) -> + {xmlelement, Name, Attrs, Els} = R#offline_msg.packet, + FromString = jlib:jid_to_string(R#offline_msg.from), + Attrs2 = lists:keystore("from", 1, Attrs, {"from", FromString}), + Attrs3 = lists:keystore("xmlns", 1, Attrs2, {"xmlns", "jabber:client"}), + {xmlelement, Name, Attrs3, + Els ++ + [jlib:timestamp_to_xml( + calendar:now_to_universal_time( + R#offline_msg.timestamp))]} + end, + lists:filter( + fun(R) -> + case R#offline_msg.expire of + never -> + true; + TimeStamp -> + TS < TimeStamp + end + end, + lists:keysort(#offline_msg.timestamp, Rs))); + _ -> + Ls + end. + +%%%================================== +%%%% Interface with ejabberd private storage + +get_user_private_mnesia(Username, Host) -> + ListNsEl = mnesia:dirty_select(private_storage, + [{#private_storage{usns={Username, Host, '$1'}, xml = '$2'}, + [], ['$$']}]), + Els = [exmpp_xml:document_to_list(El) || [_Ns, El] <- ListNsEl], + case lists:flatten(Els) of + "" -> ""; + ElsString -> + io_lib:format("~s", [ElsString]) + end. + +%%%================================== +%%%% Disk file access + +%% @spec () -> string() +make_filename_template() -> + {{Year, Month, Day}, {Hour, Minute, Second}} = calendar:local_time(), + lists:flatten( + io_lib:format("~4..0w~2..0w~2..0w-~2..0w~2..0w~2..0w", + [Year, Month, Day, Hour, Minute, Second])). + +%% @spec (Dir::string(), FnT::string()) -> string() +make_main_basefilename(Dir, FnT) -> + Filename2 = filename:flatten([FnT, ".xml"]), + filename:join([Dir, Filename2]). + +%% @spec (FnT::string(), Host::string()) -> FnH::string() +%% FnH = FnT + _ + Host2 + Extension +%% Host2 = Host with any . replaced by _ +%% Example: ("20080804-231550", "jabber.example.org") -> "20080804-231550_jabber_example_org.xml" +make_host_filename(FnT, Host) -> + Host2 = string:join(string:tokens(Host, "."), "_"), + filename:flatten([FnT, "_", Host2, ".xml"]). + +make_host_basefilename(Dir, FnT) -> + filename:join([Dir, FnT]). + +%% @spec (Fn::string()) -> {ok, Fd} +file_open(Fn) -> + file:open(Fn, [write]). + +%% @spec (Fd) -> ok +file_close(Fd) -> + file:close(Fd). + +%% @spec (Fd, String::string()) -> ok +print(Fd, String) -> + io:format(Fd, String, []). + +%%%================================== + +%%% vim: set filetype=erlang tabstop=8 foldmarker=%%%%,%%%= foldmethod=marker: -- cgit v1.2.3