* contrib/extract_translations/extract_translations.erl: Use

Gettext PO for translators, export to ejabberd MSG (EJAB-468)
* contrib/extract_translations/prepare-translation.sh: Likewise
* doc/guide.tex: Likewise
* doc/guide.html: Likewise
* src/Makefile.in: New option 'make translations'
* src/msgs/ejabberd.pot: Template translation file
* src/msgs/*.po: Generated from old MSG files
* src/msgs/*.msg: Automatic exported from PO files

SVN Revision: 1527

2 files changed, 261 insertions, 32 deletions

diff --git a/contrib/extract_translations/extract_translations.erl b/contrib/extract_translations/extract_translations.erl
index 5c727e78..1f38cd08 100644
--- a/contrib/extract_translations/extract_translations.erl
+++ b/contrib/extract_translations/extract_translations.erl
@@ -20,9 +20,14 @@
 
 start() ->
     ets:new(translations, [named_table, public]),
+    ets:new(translations_obsolete, [named_table, public]),
     ets:new(files, [named_table, public]),
     ets:new(vars, [named_table, public]),
     case init:get_plain_arguments() of
+	["-srcmsg2po", Dir, File] ->
+	    print_po_header(File),
+	    Status = process(Dir, File, srcmsg2po),
+	    halt(Status);
 	["-unused", Dir, File] ->
 	    Status = process(Dir, File, unused),
 	    halt(Status);
@@ -50,7 +55,11 @@ process(Dir, File, Used) ->
 		unused ->
 		    ets:foldl(fun({Key, _}, _) ->
 				      io:format("~p~n", [Key])
-			      end, ok, translations); 
+			      end, ok, translations);
+		srcmsg2po ->
+		    ets:foldl(fun({Key, Trans}, _) ->
+				      print_translation_obsolete(Key, Trans)
+			      end, ok, translations_obsolete);
 		_ ->
 		    ok
 	    end,
@@ -74,46 +83,52 @@ parse_form(Dir, File, Form, Used) ->
 	{call,
 	 _,
 	 {remote, _, {atom, _, translate}, {atom, _, translate}},
-	 [_, {string, _, Str}]
+	 [_, {string, Line, Str}]
 	} ->
-	    process_string(Dir, File, Str, Used);
+	    process_string(Dir, File, Line, Str, Used);
 	{call,
 	 _,
 	 {remote, _, {atom, _, translate}, {atom, _, translate}},
 	 [_, {var, _, Name}]
 	} ->
 	    case ets:lookup(vars, Name) of
-		[{_Name, Value}] ->
-		    process_string(Dir, File, Value, Used);
+		[{_Name, Value, Line}] ->
+		    process_string(Dir, File, Line, Value, Used);
 		_ ->
 		    ok
 	    end;
 	{match,
 	 _,
 	 {var, _, Name},
-	 {string, _, Value}
+	 {string, Line, Value}
 	} ->
-	    ets:insert(vars, {Name, Value});
+	    ets:insert(vars, {Name, Value, Line});
 	L when is_list(L) ->
 	    lists:foreach(
 	      fun(F) ->
-		    parse_form(Dir, File, F, Used)
+		      parse_form(Dir, File, F, Used)
 	      end, L);
 	T when is_tuple(T) ->
 	    lists:foreach(
 	      fun(F) ->
-		    parse_form(Dir, File, F, Used)
+		      parse_form(Dir, File, F, Used)
 	      end, tuple_to_list(T));
 	_ ->
 	    ok
     end.
 	    
-process_string(_Dir, File, Str, Used) ->
+process_string(_Dir, _File, _Line, "", _Used) ->
+    ok;
+
+process_string(_Dir, File, Line, Str, Used) ->
     case {ets:lookup(translations, Str), Used} of
 	{[{_Key, _Trans}], unused} ->
 	    ets:delete(translations, Str);
 	{[{_Key, _Trans}], used} ->
 	    ok;
+	{[{_Key, Trans}], srcmsg2po} ->
+	    ets:delete(translations_obsolete, Str),
+	    print_translation(File, Line, Str, Trans);
 	{_, used} ->
 	    case ets:lookup(files, File) of
 		[{_}] ->
@@ -127,6 +142,15 @@ process_string(_Dir, File, Str, Used) ->
 		_ -> io:format("{~p, \"\"}.~n", [Str])
 	    end,
 	    ets:insert(translations, {Str, ""});
+	{_, srcmsg2po} ->
+	    case ets:lookup(files, File) of
+		[{_}] ->
+		    ok;
+		_ ->
+		    ets:insert(files, {File})
+	    end,
+	    ets:insert(translations, {Str, ""}),
+	    print_translation(File, Line, Str, "");
 	_ ->
 	    ok
     end.
@@ -140,7 +164,8 @@ load_file(File) ->
 			"" ->
 			    ok;
 			_ ->
-			    ets:insert(translations, {Orig, Trans})
+			    ets:insert(translations, {Orig, Trans}),
+			    ets:insert(translations_obsolete, {Orig, Trans})
 		    end
 	      end, Terms);
 	Err ->
@@ -191,3 +216,77 @@ print_usage() ->
       "  extract_translations . ./msgs/ru.msg~n"
      ).
 
+
+%%%
+%%% Gettext
+%%%
+
+print_po_header(File) ->
+    MsgProps = get_msg_header_props(File),
+    {Language, [LastT | AddT]} = prepare_props(MsgProps),
+    application:load(ejabberd),
+    {ok, Version} = application:get_key(ejabberd, vsn),
+    print_po_header(Version, Language, LastT, AddT).
+
+get_msg_header_props(File) ->
+    {ok, F} = file:open(File, [read]),
+    Lines = get_msg_header_props(F, []),
+    file:close(F),
+    Lines.
+
+get_msg_header_props(F, Lines) ->
+    String = io:get_line(F, ""),
+    case io_lib:fread("% ", String) of
+	{ok, [], RemString} ->
+	    case io_lib:fread("~s", RemString) of
+		{ok, [Key], Value} when Value /= "\n" ->
+		    %% The first character in Value is a blankspace:
+		    %% And the last characters are 'slash n'
+		    ValueClean = string:substr(Value, 2, string:len(Value)-2),
+		    get_msg_header_props(F, Lines ++ [{Key, ValueClean}]);
+		_ ->
+		    get_msg_header_props(F, Lines)
+	    end;
+	_ ->
+	    Lines
+    end.
+
+prepare_props(MsgProps) ->
+    Language = proplists:get_value("Language:", MsgProps),
+    Authors = proplists:get_all_values("Author:", MsgProps),
+    {Language, Authors}.
+
+print_po_header(Version, Language, LastTranslator, AdditionalTranslatorsList) ->
+    AdditionalTranslatorsString = build_additional_translators(AdditionalTranslatorsList),
+    HeaderString =
+	"msgid \"\"\n"
+	"msgstr \"\"\n"
+	"\"Project-Id-Version: " ++ Version ++ "\\n\"\n"
+	++ "\"X-Language: " ++ Language ++ "\\n\"\n"
+	"\"Last-Translator: " ++ LastTranslator ++ "\\n\"\n"
+	++ AdditionalTranslatorsString ++
+	"\"MIME-Version: 1.0\\n\"\n"
+	"\"Content-Type: text/plain; charset=UTF-8\\n\"\n"
+	"\"Content-Transfer-Encoding: 8bit\\n\"\n",
+    io:format("~s~n", [HeaderString]).
+
+build_additional_translators(List) ->
+    lists:foldl(
+      fun(T, Str) ->
+	      Str ++ "\"X-Additional-Translator: " ++ T ++ "\\n\"\n"
+      end,
+      "",
+      List).
+
+print_translation(File, Line, Str, StrT) ->
+    {ok, StrQ, _} = regexp:gsub(Str, "\"", "\\\""),
+    {ok, StrTQ, _} = regexp:gsub(StrT, "\"", "\\\""),
+    io:format("#: ~s:~p~nmsgid \"~s\"~nmsgstr \"~s\"~n~n", [File, Line, StrQ, StrTQ]).
+
+print_translation_obsolete(Str, StrT) ->
+    File = "unknown.erl",
+    Line = 1,
+    {ok, StrQ, _} = regexp:gsub(Str, "\"", "\\\""),
+    {ok, StrTQ, _} = regexp:gsub(StrT, "\"", "\\\""),
+    io:format("#: ~s:~p~n#~~ msgid \"~s\"~n#~~ msgstr \"~s\"~n~n", [File, Line, StrQ, StrTQ]).
+
diff --git a/contrib/extract_translations/prepare-translation.sh b/contrib/extract_translations/prepare-translation.sh
index 2fd895bf..d1c435fb 100755
--- a/contrib/extract_translations/prepare-translation.sh
+++ b/contrib/extract_translations/prepare-translation.sh
@@ -8,10 +8,10 @@ prepare_dirs ()
 	# Where is Erlang binary
 	ERL=`which erl`
 
-	EJA_DIR=`pwd`/../..
+	EJA_DIR=`pwd`/..
 	EXTRACT_DIR=$EJA_DIR/contrib/extract_translations/
-	EXTRACT_ERL=extract_translations.erl
-	EXTRACT_BEAM=extract_translations.beam
+	EXTRACT_ERL=$EXTRACT_DIR/extract_translations.erl
+	EXTRACT_BEAM=$EXTRACT_DIR/extract_translations.beam
 	SRC_DIR=$EJA_DIR/src
 	MSGS_DIR=$SRC_DIR/msgs
 
@@ -22,9 +22,7 @@ prepare_dirs ()
 
 	if !([[ -x $EXTRACT_BEAM ]])
 	then 
-	echo -n "Compiling extract_translations.erl: "
 	sh -c "cd $EXTRACT_DIR; $ERL -compile $EXTRACT_ERL"
-	echo "ok"
 	fi
 }
 
@@ -140,6 +138,116 @@ find_unused_full ()
 	cd ..
 }
 
+extract_lang_srcmsg2po ()
+{
+	LANG_CODE=$1
+	MSGS_PATH=$MSGS_DIR/$LANG_CODE.msg
+	PO_PATH=$MSGS_DIR/$LANG_CODE.po
+
+	$ERL -pa $EXTRACT_DIR -pa $SRC_DIR -noinput -noshell -s extract_translations -s init stop -extra -srcmsg2po . $MSGS_PATH >$PO_PATH.1
+	sed -e 's/ \[\]$/ \"\"/g;' $PO_PATH.1 > $PO_PATH.2
+	msguniq --sort-by-file $PO_PATH.2 --output-file=$PO_PATH
+	
+	rm $PO_PATH.*
+}
+
+extract_lang_src2pot ()
+{
+	LANG_CODE=ejabberd
+	MSGS_PATH=$MSGS_DIR/$LANG_CODE.msg
+	POT_PATH=$MSGS_DIR/$LANG_CODE.pot
+
+	echo -n "" >$MSGS_PATH
+	echo "% Language: Language Name" >>$MSGS_PATH
+	echo "% Author: Translator name and contact method" >>$MSGS_PATH
+	echo "" >>$MSGS_PATH
+
+	cd $SRC_DIR
+	$ERL -pa $EXTRACT_DIR -pa $SRC_DIR -noinput -noshell -s extract_translations -s init stop -extra -srcmsg2po . $MSGS_PATH >$POT_PATH.1
+	sed -e 's/ \[\]$/ \"\"/g;' $POT_PATH.1 > $POT_PATH.2
+	msguniq --sort-by-file $POT_PATH.2 --output-file=$POT_PATH
+
+	rm $POT_PATH.*
+	rm $MSGS_PATH
+}
+
+extract_lang_popot2po ()
+{
+	LANG_CODE=$1
+	PO_PATH=$MSGS_DIR/$LANG_CODE.po
+	POT_PATH=$MSGS_DIR/ejabberd.pot
+
+	msgmerge $PO_PATH $POT_PATH >$PO_PATH.translate 2>/dev/null
+	mv $PO_PATH.translate $PO_PATH 
+}
+
+extract_lang_po2msg ()
+{
+	LANG_CODE=$1
+	PO_PATH=$LANG_CODE.po
+	MS_PATH=$PO_PATH.ms
+	MSGID_PATH=$PO_PATH.msgid
+	MSGSTR_PATH=$PO_PATH.msgstr
+	MSGS_PATH=$LANG_CODE.msg
+
+	cd $MSGS_DIR
+
+	# Check PO has correct ~
+	# Let's convert to C format so we can use msgfmt
+	PO_TEMP=$LANG_CODE.po.temp
+	cat $PO_PATH | sed 's/%/perc/g' | sed 's/~/%/g' | sed 's/#:.*/#, c-format/g' >$PO_TEMP
+	msgfmt $PO_TEMP --check-format
+	result=$?
+	rm $PO_TEMP
+	if [ $result -ne 0 ] ; then
+		exit 1
+	fi
+
+	msgattrib $PO_PATH --translated --no-fuzzy --no-obsolete --no-location --no-wrap | grep "^msg" | tail --lines=+3 >$MS_PATH
+	grep "^msgid" $PO_PATH.ms | sed 's/^msgid //g' >$MSGID_PATH
+	grep "^msgstr" $PO_PATH.ms | sed 's/^msgstr //g' >$MSGSTR_PATH
+	paste $MSGID_PATH $MSGSTR_PATH --delimiter=, | awk '{print "{" $0 "}."}' | sort -g >$MSGS_PATH
+
+	rm $MS_PATH
+	rm $MSGID_PATH
+	rm $MSGSTR_PATH
+}
+
+extract_lang_updateall ()
+{
+	echo "Generating POT"
+	extract_lang_src2pot 
+
+	cd $MSGS_DIR
+	echo ""
+	echo -e "File Missing Language     Last translator"
+	echo -e "---- ------- --------     ---------------"
+	for i in *.msg; do 
+                LANG_CODE=${i%.msg}
+		echo -n $LANG_CODE | awk '{printf "%-6s", $1 }'
+
+		# Convert old MSG file to PO
+		PO=$LANG_CODE.po
+		[ -f $PO ] || extract_lang_srcmsg2po $LANG_CODE
+
+		extract_lang_popot2po $LANG_CODE
+		extract_lang_po2msg $LANG_CODE
+
+		MISSING=`msgfmt --statistics $PO 2>&1 | awk '{printf "%5s", $4 }'`
+		echo -n " $MISSING"
+
+		LANGUAGE=`grep "Language:" $PO | sed 's/\"X-Language: //g' | sed 's/\\\\n\"//g' | awk '{printf "%-12s", $1}'`
+		echo -n " $LANGUAGE"
+
+		LASTAUTH=`grep "Last-Translator" $PO | sed 's/\"Last-Translator: //g' | sed 's/\\\\n\"//g'`
+		echo " $LASTAUTH"
+	done
+	echo ""
+	rm messages.mo
+
+	cd ..
+}
+
 translation_instructions ()
 {
 	echo ""
@@ -158,34 +266,56 @@ translation_instructions ()
 	echo "  $MSGS_PATH"
 }
 
+prepare_dirs
 case "$1" in
-	-help)
-		echo "Options:"
-		echo "  -langall"
-		echo "  -lang LANGUAGE_FILE"
-		echo ""
-		echo "Example:"
-		echo "  ./prepare-translation.sh -lang es.msg"
-		exit 0
-		;;
 	-lang)
 		LANGU=$2
-		prepare_dirs
 		extract_lang $LANGU
 		shift
 		shift
 		;;
 	-langall)
-		prepare_dirs
 		extract_lang_all
 		shift
 		;;
-	*)
-		echo "unknown option: '$1 $2'"
+	-srcmsg2po)
+		LANG_CODE=$2
+		extract_lang_srcmsg2po $LANG_CODE
+		shift
+		shift
+		;;
+	-popot2po)
+		LANG_CODE=$2
+		extract_lang_popot2po $LANG_CODE
 		shift
 		shift
 		;;
+	-src2pot)
+		extract_lang_src2pot 
+		shift
+		;;
+	-po2msg)
+		LANG_CODE=$2
+		extract_lang_po2msg $LANG_CODE
+		shift
+		shift
+		;;
+	-updateall)
+		extract_lang_updateall
+		shift
+		;;
+	*)
+		echo "Options:"
+		echo "  -langall"
+		echo "  -lang LANGUAGE_FILE"
+		echo "  -srcmsg2po LANGUAGE   Construct .msg file using source code to PO file"
+		echo "  -src2pot              Generate template POT file from source code"
+		echo "  -popot2po LANGUAGE    Update PO file with template POT file"
+		echo "  -po2msg LANGUAGE      Export PO file to MSG file"
+		echo "  -updateall            Generate POT and update all PO"
+		echo ""
+		echo "Example:"
+		echo "  ./prepare-translation.sh -lang es.msg"
+		exit 0
+		;;
 esac
-
-echo ""
-echo "End."