Index: netwerk/base/src/nsStandardURL.cpp =================================================================== RCS file: /cvsroot/mozilla/netwerk/base/src/nsStandardURL.cpp,v retrieving revision 1.82.4.12 diff -u -9 -p -r1.82.4.12 nsStandardURL.cpp --- netwerk/base/src/nsStandardURL.cpp 7 Aug 2008 21:24:18 -0000 1.82.4.12 +++ netwerk/base/src/nsStandardURL.cpp 3 Jun 2009 09:48:29 -0000 @@ -52,25 +52,23 @@ #include "nsIPrefBranch2.h" #include "nsIIDNService.h" #include "nsNetUtil.h" #include "prlog.h" #include "nsAutoPtr.h" static NS_DEFINE_CID(kThisImplCID, NS_THIS_STANDARDURL_IMPL_CID); static NS_DEFINE_CID(kStandardURLCID, NS_STANDARDURL_CID); -nsIIDNService *nsStandardURL::gIDN = nsnull; +nsIIDNService_MOZILLA_1_8_BRANCH *nsStandardURL::gIDN = nsnull; nsICharsetConverterManager *nsStandardURL::gCharsetMgr = nsnull; PRBool nsStandardURL::gInitialized = PR_FALSE; PRBool nsStandardURL::gEscapeUTF8 = PR_TRUE; PRBool nsStandardURL::gAlwaysEncodeInUTF8 = PR_TRUE; -PRBool nsStandardURL::gShowPunycode = PR_FALSE; -nsIPrefBranch *nsStandardURL::gIDNWhitelistPrefBranch = nsnull; #if defined(PR_LOGGING) // // setenv NSPR_LOG_MODULES nsStandardURL:5 // static PRLogModuleInfo *gStandardURLLog; #endif #define LOG(args) PR_LOG(gStandardURLLog, PR_LOG_DEBUG, args) #define LOG_ENABLED() PR_LOG_TEST(gStandardURLLog, PR_LOG_DEBUG) @@ -131,20 +129,18 @@ end: } //---------------------------------------------------------------------------- // nsStandardURL::nsPrefObserver //---------------------------------------------------------------------------- #define NS_NET_PREF_ESCAPEUTF8 "network.standard-url.escape-utf8" #define NS_NET_PREF_ENABLEIDN "network.enableIDN" #define NS_NET_PREF_ALWAYSENCODEINUTF8 "network.standard-url.encode-utf8" -#define NS_NET_PREF_SHOWPUNYCODE "network.IDN_show_punycode" -#define NS_NET_PREF_IDNWHITELIST "network.IDN.whitelist." NS_IMPL_ISUPPORTS1(nsStandardURL::nsPrefObserver, nsIObserver) NS_IMETHODIMP nsStandardURL:: nsPrefObserver::Observe(nsISupports *subject, const char *topic, const PRUnichar *data) { if (!strcmp(topic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) { @@ -298,38 +294,28 @@ nsStandardURL::~nsStandardURL() void nsStandardURL::InitGlobalObjects() { nsCOMPtr prefBranch( do_GetService(NS_PREFSERVICE_CONTRACTID) ); if (prefBranch) { nsCOMPtr obs( new nsPrefObserver() ); prefBranch->AddObserver(NS_NET_PREF_ESCAPEUTF8, obs.get(), PR_FALSE); prefBranch->AddObserver(NS_NET_PREF_ALWAYSENCODEINUTF8, obs.get(), PR_FALSE); prefBranch->AddObserver(NS_NET_PREF_ENABLEIDN, obs.get(), PR_FALSE); - prefBranch->AddObserver(NS_NET_PREF_SHOWPUNYCODE, obs.get(), PR_FALSE); PrefsChanged(prefBranch, nsnull); - - nsCOMPtr prefs = do_QueryInterface(prefBranch); - if (prefs) { - nsCOMPtr branch; - if (NS_SUCCEEDED(prefs->GetBranch( NS_NET_PREF_IDNWHITELIST, - getter_AddRefs(branch) ))) - NS_ADDREF(gIDNWhitelistPrefBranch = branch); - } } } void nsStandardURL::ShutdownGlobalObjects() { NS_IF_RELEASE(gIDN); NS_IF_RELEASE(gCharsetMgr); - NS_IF_RELEASE(gIDNWhitelistPrefBranch); } //---------------------------------------------------------------------------- // nsStandardURL //---------------------------------------------------------------------------- void nsStandardURL::Clear() { @@ -378,45 +364,35 @@ nsStandardURL::EscapeIPv6(const char *ho return PR_FALSE; } PRBool nsStandardURL::NormalizeIDN(const nsCSubstring &host, nsCString &result) { // If host is ACE, then convert to UTF-8. Else, if host is already UTF-8, // then make sure it is normalized per IDN. - // this function returns PR_TRUE iff it writes something to |result|. + // this function returns PR_TRUE if normalization succeeds. // NOTE: As a side-effect this function sets mHostEncoding. While it would // be nice to avoid side-effects in this function, the implementation of // this function is already somewhat bound to the behavior of the // callsites. Anyways, this function exists to avoid code duplication, so // side-effects abound :-/ NS_ASSERTION(mHostEncoding == eEncoding_ASCII, "unexpected default encoding"); - if (IsASCII(host)) { - PRBool isACE; - if (gIDN && - NS_SUCCEEDED(gIDN->IsACE(host, &isACE)) && isACE && - NS_SUCCEEDED(ACEtoDisplayIDN(host, result))) { + PRBool isASCII; + if (gIDN && + NS_SUCCEEDED(gIDN->ConvertToDisplayIDN(host, &isASCII, result))) { + if (!isASCII) mHostEncoding = eEncoding_UTF8; - return PR_TRUE; - } - } - else { - mHostEncoding = eEncoding_UTF8; - if (gIDN && NS_SUCCEEDED(UTF8toDisplayIDN(host, result))) { - // normalization could result in an ASCII only hostname - if (IsASCII(result)) - mHostEncoding = eEncoding_ASCII; - return PR_TRUE; - } + + return PR_TRUE; } result.Truncate(); return PR_FALSE; } void nsStandardURL::CoalescePath(netCoalesceFlags coalesceFlag, char *path) { @@ -819,98 +795,40 @@ nsStandardURL::PrefsChanged(nsIPrefBranc LOG(("nsStandardURL::PrefsChanged [pref=%s]\n", pref)); #define PREF_CHANGED(p) ((pref == nsnull) || !strcmp(pref, p)) #define GOT_PREF(p, b) (NS_SUCCEEDED(prefs->GetBoolPref(p, &b))) if (PREF_CHANGED(NS_NET_PREF_ENABLEIDN)) { NS_IF_RELEASE(gIDN); if (GOT_PREF(NS_NET_PREF_ENABLEIDN, val) && val) { // initialize IDN - nsCOMPtr serv(do_GetService(NS_IDNSERVICE_CONTRACTID)); + nsCOMPtr + serv(do_GetService(NS_IDNSERVICE_CONTRACTID)); if (serv) NS_ADDREF(gIDN = serv.get()); } LOG(("IDN support %s\n", gIDN ? "enabled" : "disabled")); } if (PREF_CHANGED(NS_NET_PREF_ESCAPEUTF8)) { if (GOT_PREF(NS_NET_PREF_ESCAPEUTF8, val)) gEscapeUTF8 = val; LOG(("escape UTF-8 %s\n", gEscapeUTF8 ? "enabled" : "disabled")); } if (PREF_CHANGED(NS_NET_PREF_ALWAYSENCODEINUTF8)) { if (GOT_PREF(NS_NET_PREF_ALWAYSENCODEINUTF8, val)) gAlwaysEncodeInUTF8 = val; LOG(("encode in UTF-8 %s\n", gAlwaysEncodeInUTF8 ? "enabled" : "disabled")); } - - if (PREF_CHANGED(NS_NET_PREF_SHOWPUNYCODE)) { - if (GOT_PREF(NS_NET_PREF_SHOWPUNYCODE, val)) - gShowPunycode = val; - LOG(("show punycode %s\n", gShowPunycode ? "enabled" : "disabled")); - } #undef PREF_CHANGED #undef GOT_PREF } - -/* static */ nsresult -nsStandardURL::ACEtoDisplayIDN(const nsCSubstring &host, nsCString &result) -{ - if (gShowPunycode || !IsInWhitelist(host)) { - result = host; - return NS_OK; - } - - return gIDN->ConvertACEtoUTF8(host, result); -} - -/* static */ nsresult -nsStandardURL::UTF8toDisplayIDN(const nsCSubstring &host, nsCString &result) -{ - // We have to normalize the hostname before testing against the domain - // whitelist. See bug 315411. - - nsCAutoString temp; - if (gShowPunycode || NS_FAILED(gIDN->Normalize(host, temp))) - return gIDN->ConvertUTF8toACE(host, result); - - PRBool isACE = PR_FALSE; - gIDN->IsACE(temp, &isACE); - - // If host is converted to ACE by the normalizer, then the host may contain - // unsafe characters. See bug 283016, bug 301694, and bug 309311. - - if (!isACE && !IsInWhitelist(temp)) - return gIDN->ConvertUTF8toACE(temp, result); - - result = temp; - return NS_OK; -} - -/* static */ PRBool -nsStandardURL::IsInWhitelist(const nsCSubstring &host) -{ - PRInt32 pos; - PRBool safe; - - // XXX This code uses strings inefficiently. - - if (gIDNWhitelistPrefBranch && - (pos = nsCAutoString(host).RFind(".")) != kNotFound && - NS_SUCCEEDED(gIDNWhitelistPrefBranch-> - GetBoolPref(nsCAutoString(Substring(host, pos + 1)).get(), - &safe))) - return safe; - - return PR_FALSE; -} - //---------------------------------------------------------------------------- // nsStandardURL::nsISupports //---------------------------------------------------------------------------- NS_IMPL_ADDREF(nsStandardURL) NS_IMPL_RELEASE(nsStandardURL) NS_INTERFACE_MAP_BEGIN(nsStandardURL) NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIStandardURL) Index: netwerk/base/src/nsStandardURL.h =================================================================== RCS file: /cvsroot/mozilla/netwerk/base/src/nsStandardURL.h,v retrieving revision 1.28.4.1 diff -u -9 -p -r1.28.4.1 nsStandardURL.h --- netwerk/base/src/nsStandardURL.h 13 Sep 2005 18:23:14 -0000 1.28.4.1 +++ netwerk/base/src/nsStandardURL.h 3 Jun 2009 09:48:29 -0000 @@ -49,19 +49,19 @@ #include "nsIURLParser.h" #include "nsIUnicodeEncoder.h" #include "nsIObserver.h" #include "nsIIOService.h" #include "nsCOMPtr.h" #include "nsURLHelper.h" class nsIBinaryInputStream; class nsIBinaryOutputStream; -class nsIIDNService; +class nsIIDNService_MOZILLA_1_8_BRANCH; class nsICharsetConverterManager; class nsIPrefBranch; //----------------------------------------------------------------------------- // standard URL implementation //----------------------------------------------------------------------------- class nsStandardURL : public nsIFileURL , public nsIStandardURL @@ -213,23 +213,18 @@ private: void ShiftFromQuery(PRInt32 diff) { mQuery.mPos += diff; ShiftFromRef(diff); } void ShiftFromRef(PRInt32 diff) { mRef.mPos += diff; } // fastload helper functions nsresult ReadSegment(nsIBinaryInputStream *, URLSegment &); nsresult WriteSegment(nsIBinaryOutputStream *, const URLSegment &); static void PrefsChanged(nsIPrefBranch *prefs, const char *pref); - // IDN routines - static nsresult ACEtoDisplayIDN(const nsCSubstring &in, nsCString &out); - static nsresult UTF8toDisplayIDN(const nsCSubstring &in, nsCString &out); - static PRBool IsInWhitelist(const nsCSubstring &host); - // mSpec contains the normalized version of the URL spec (UTF-8 encoded). nsCString mSpec; PRInt32 mDefaultPort; PRInt32 mPort; // url parts (relative to mSpec) URLSegment mScheme; URLSegment mAuthority; URLSegment mUsername; @@ -262,25 +257,23 @@ private: PRUint32 mHostEncoding : 2; // eEncoding_xxx PRUint32 mSpecEncoding : 2; // eEncoding_xxx PRUint32 mURLType : 2; // nsIStandardURL::URLTYPE_xxx PRUint32 mMutable : 1; // nsIStandardURL::mutable PRUint32 mSupportsFileURL : 1; // QI to nsIFileURL? // global objects. don't use COMPtr as its destructor will cause a // coredump if we leak it. - static nsIIDNService *gIDN; + static nsIIDNService_MOZILLA_1_8_BRANCH *gIDN; static nsICharsetConverterManager *gCharsetMgr; static PRBool gInitialized; static PRBool gEscapeUTF8; static PRBool gAlwaysEncodeInUTF8; - static PRBool gShowPunycode; - static nsIPrefBranch *gIDNWhitelistPrefBranch; }; #define NS_THIS_STANDARDURL_IMPL_CID \ { /* b8e3e97b-1ccd-4b45-af5a-79596770f5d7 */ \ 0xb8e3e97b, \ 0x1ccd, \ 0x4b45, \ {0xaf, 0x5a, 0x79, 0x59, 0x67, 0x70, 0xf5, 0xd7} \ } Index: netwerk/dns/public/nsIIDNService.idl =================================================================== RCS file: /cvsroot/mozilla/netwerk/dns/public/nsIIDNService.idl,v retrieving revision 1.4 diff -u -9 -p -r1.4 nsIIDNService.idl --- netwerk/dns/public/nsIIDNService.idl 3 Apr 2004 07:32:18 -0000 1.4 +++ netwerk/dns/public/nsIIDNService.idl 3 Jun 2009 09:48:29 -0000 @@ -18,19 +18,21 @@ * Portions created by the Initial Developer are Copyright (C) 2001 * the Initial Developer. All Rights Reserved. * * Contributor(s): bobj@netscape.com, * brendan@mozilla.org, * darin@netscape.com, * ftang@netscape.com, * gagan@netscape.com, * nhotta@netscape.com, - * william.tan@i-dns.net + * william.tan@i-dns.net, + * dwitte@stanford.edu, + * smontagu@smontagu.org * * * Alternatively, the contents of this file may be used under the terms of * either the GNU General Public License Version 2 or later (the "GPL"), or * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), * in which case the provisions of the GPL or the LGPL are applicable instead * of those above. If you wish to allow use of your version of this file only * under the terms of either the GPL or the LGPL, and not to allow others to * use your version of this file under the terms of the MPL, indicate your @@ -83,9 +85,21 @@ interface nsIIDNService : nsISupports */ boolean isACE(in ACString input); /** * Performs the unicode normalization needed for hostnames in IDN, * for callers that want early normalization. */ AUTF8String normalize(in AUTF8String input); }; + +[scriptable, uuid(7be196fc-82fd-40d8-9d8c-6421faddeee9)] +interface nsIIDNService_MOZILLA_1_8_BRANCH : nsIIDNService +{ + /** + * Normalizes and converts a host to UTF-8 if the host is in the IDN + * whitelist, otherwise converts it to ACE. This is useful for display + * purposes and to ensure an encoding consistent with nsIURI::GetHost(). + * If the result is ASCII or ACE encoded, |isASCII| will be true. + */ + AUTF8String convertToDisplayIDN(in AUTF8String input, out boolean isASCII); +}; \ No newline at end of file Index: netwerk/dns/src/nsIDNService.cpp =================================================================== RCS file: /cvsroot/mozilla/netwerk/dns/src/nsIDNService.cpp,v retrieving revision 1.28.2.1 diff -u -9 -p -r1.28.2.1 nsIDNService.cpp --- netwerk/dns/src/nsIDNService.cpp 27 Aug 2008 07:36:26 -0000 1.28.2.1 +++ netwerk/dns/src/nsIDNService.cpp 3 Jun 2009 09:48:29 -0000 @@ -1,10 +1,10 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* ***** BEGIN LICENSE BLOCK ***** * Version: MPL 1.1/GPL 2.0/LGPL 2.1 * * The contents of this file are subject to the Mozilla Public License Version * 1.1 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.mozilla.org/MPL/ * * Software distributed under the License is distributed on an "AS IS" basis, @@ -51,43 +51,50 @@ //----------------------------------------------------------------------------- // RFC 1034 - 3.1. Name space specifications and terminology static const PRUint32 kMaxDNSNodeLen = 63; //----------------------------------------------------------------------------- #define NS_NET_PREF_IDNTESTBED "network.IDN_testbed" #define NS_NET_PREF_IDNPREFIX "network.IDN_prefix" #define NS_NET_PREF_IDNBLACKLIST "network.IDN.blacklist_chars" +#define NS_NET_PREF_SHOWPUNYCODE "network.IDN_show_punycode" +#define NS_NET_PREF_IDNWHITELIST "network.IDN.whitelist." inline PRBool isOnlySafeChars(const nsAFlatString& in, const nsAFlatString& blacklist) { return (blacklist.IsEmpty() || in.FindCharInSet(blacklist) == kNotFound); } //----------------------------------------------------------------------------- // nsIDNService //----------------------------------------------------------------------------- /* Implementation file */ NS_IMPL_THREADSAFE_ISUPPORTS3(nsIDNService, - nsIIDNService, + nsIIDNService_MOZILLA_1_8_BRANCH, nsIObserver, nsISupportsWeakReference) nsresult nsIDNService::Init() { - nsCOMPtr prefInternal(do_GetService(NS_PREFSERVICE_CONTRACTID)); + nsCOMPtr prefs(do_GetService(NS_PREFSERVICE_CONTRACTID)); + if (prefs) + prefs->GetBranch(NS_NET_PREF_IDNWHITELIST, getter_AddRefs(mIDNWhitelistPrefBranch)); + + nsCOMPtr prefInternal(do_QueryInterface(prefs)); if (prefInternal) { prefInternal->AddObserver(NS_NET_PREF_IDNTESTBED, this, PR_TRUE); prefInternal->AddObserver(NS_NET_PREF_IDNPREFIX, this, PR_TRUE); prefInternal->AddObserver(NS_NET_PREF_IDNBLACKLIST, this, PR_TRUE); + prefInternal->AddObserver(NS_NET_PREF_SHOWPUNYCODE, this, PR_TRUE); prefsChanged(prefInternal, nsnull); } return NS_OK; } NS_IMETHODIMP nsIDNService::Observe(nsISupports *aSubject, const char *aTopic, const PRUnichar *aData) { @@ -116,18 +123,23 @@ void nsIDNService::prefsChanged(nsIPrefB nsCOMPtr blacklist; nsresult rv = prefBranch->GetComplexValue(NS_NET_PREF_IDNBLACKLIST, NS_GET_IID(nsISupportsString), getter_AddRefs(blacklist)); if (NS_SUCCEEDED(rv)) blacklist->ToString(getter_Copies(mIDNBlacklist)); else mIDNBlacklist.Truncate(); } + if (!pref || NS_LITERAL_STRING(NS_NET_PREF_SHOWPUNYCODE).Equals(pref)) { + PRBool val; + if (NS_SUCCEEDED(prefBranch->GetBoolPref(NS_NET_PREF_SHOWPUNYCODE, &val))) + mShowPunycode = val; + } } nsIDNService::nsIDNService() { nsresult rv; // initialize to the official prefix (RFC 3490 "5. ACE prefix") const char kIDNSPrefix[] = "xn--"; strcpy(mACEPrefix, kIDNSPrefix); @@ -148,18 +160,23 @@ nsIDNService::~nsIDNService() idn_nameprep_destroy(mNamePrepHandle); } /* ACString ConvertUTF8toACE (in AUTF8String input); */ NS_IMETHODIMP nsIDNService::ConvertUTF8toACE(const nsACString & input, nsACString & ace) { // protect against bogus input NS_ENSURE_TRUE(IsUTF8(input), NS_ERROR_UNEXPECTED); + return UTF8toACE(input, ace, PR_TRUE); +} + +nsresult nsIDNService::UTF8toACE(const nsACString & input, nsACString & ace, PRBool allowUnassigned) +{ nsresult rv; NS_ConvertUTF8toUCS2 ustr(input); // map ideographic period to ASCII period etc. normalizeFullStops(ustr); PRUint32 len, offset; len = 0; @@ -169,45 +186,53 @@ NS_IMETHODIMP nsIDNService::ConvertUTF8t nsAString::const_iterator start, end; ustr.BeginReading(start); ustr.EndReading(end); ace.Truncate(); // encode nodes if non ASCII while (start != end) { len++; if (*start++ == (PRUnichar)'.') { - rv = stringPrepAndACE(Substring(ustr, offset, len - 1), encodedBuf); + rv = stringPrepAndACE(Substring(ustr, offset, len - 1), encodedBuf, + allowUnassigned); NS_ENSURE_SUCCESS(rv, rv); ace.Append(encodedBuf); ace.Append('.'); offset += len; len = 0; } } // add extra node for multilingual test bed if (mMultilingualTestBed) ace.AppendLiteral("mltbd."); // encode the last node if non ASCII if (len) { - rv = stringPrepAndACE(Substring(ustr, offset, len), encodedBuf); + rv = stringPrepAndACE(Substring(ustr, offset, len), encodedBuf, + allowUnassigned); NS_ENSURE_SUCCESS(rv, rv); ace.Append(encodedBuf); } return NS_OK; } /* [noscript] string ConvertACEtoUTF8 (in string input); */ NS_IMETHODIMP nsIDNService::ConvertACEtoUTF8(const nsACString & input, nsACString & _retval) { + return ACEtoUTF8(input, _retval, PR_TRUE); +} + +nsresult nsIDNService::ACEtoUTF8(const nsACString & input, nsACString & _retval, + PRBool allowUnassigned) +{ // RFC 3490 - 4.2 ToUnicode // ToUnicode never fails. If any step fails, then the original input // sequence is returned immediately in that step. if (!IsASCII(input)) { _retval.Assign(input); return NS_OK; } @@ -217,32 +242,34 @@ NS_IMETHODIMP nsIDNService::ConvertACEto nsACString::const_iterator start, end; input.BeginReading(start); input.EndReading(end); _retval.Truncate(); // loop and decode nodes while (start != end) { len++; if (*start++ == '.') { - if (NS_FAILED(decodeACE(Substring(input, offset, len - 1), decodedBuf))) { + if (NS_FAILED(decodeACE(Substring(input, offset, len - 1), decodedBuf, + allowUnassigned))) { _retval.Assign(input); return NS_OK; } _retval.Append(decodedBuf); _retval.Append('.'); offset += len; len = 0; } } // decode the last node if (len) { - if (NS_FAILED(decodeACE(Substring(input, offset, len), decodedBuf))) + if (NS_FAILED(decodeACE(Substring(input, offset, len), decodedBuf, + allowUnassigned))) _retval.Assign(input); else _retval.Append(decodedBuf); } return NS_OK; } /* boolean encodedInACE (in ACString input); */ @@ -266,30 +293,100 @@ NS_IMETHODIMP nsIDNService::IsACE(const NS_IMETHODIMP nsIDNService::Normalize(const nsACString & input, nsACString & output) { // protect against bogus input NS_ENSURE_TRUE(IsUTF8(input), NS_ERROR_UNEXPECTED); NS_ConvertUTF8toUTF16 inUTF16(input); normalizeFullStops(inUTF16); - nsAutoString outUTF16; - nsresult rv = stringPrep(inUTF16, outUTF16); - if (NS_FAILED(rv)) - return rv; + // pass the domain name to stringprep label by label + nsAutoString outUTF16, outLabel; + + PRUint32 len = 0, offset = 0; + nsresult rv; + nsAString::const_iterator start, end; + inUTF16.BeginReading(start); + inUTF16.EndReading(end); + + while (start != end) { + len++; + if (*start++ == PRUnichar('.')) { + rv = stringPrep(Substring(inUTF16, offset, len - 1), outLabel, PR_TRUE); + NS_ENSURE_SUCCESS(rv, rv); + + outUTF16.Append(outLabel); + outUTF16.Append(PRUnichar('.')); + offset += len; + len = 0; + } + } + if (len) { + rv = stringPrep(Substring(inUTF16, offset, len), outLabel, PR_TRUE); + NS_ENSURE_SUCCESS(rv, rv); + + outUTF16.Append(outLabel); + } CopyUTF16toUTF8(outUTF16, output); if (!isOnlySafeChars(outUTF16, mIDNBlacklist)) return ConvertUTF8toACE(output, output); return NS_OK; } +NS_IMETHODIMP nsIDNService::ConvertToDisplayIDN(const nsACString & input, PRBool * _isASCII, nsACString & _retval) +{ + // If host is ACE, then convert to UTF-8 if the host is in the IDN whitelist. + // Else, if host is already UTF-8, then make sure it is normalized per IDN. + + nsresult rv; + + if (IsASCII(input)) { + // first, canonicalize the host to lowercase, for whitelist lookup + _retval = input; + ToLowerCase(_retval); + + PRBool isACE; + IsACE(_retval, &isACE); + + if (isACE && !mShowPunycode && isInWhitelist(_retval)) { + // ACEtoUTF8() can't fail, but might return the original ACE string + nsCAutoString temp(_retval); + ACEtoUTF8(temp, _retval, PR_FALSE); + *_isASCII = IsASCII(_retval); + } else { + *_isASCII = PR_TRUE; + } + } else { + if (mShowPunycode && NS_SUCCEEDED(ConvertUTF8toACE(input, _retval))) { + *_isASCII = PR_TRUE; + return NS_OK; + } + + // We have to normalize the hostname before testing against the domain + // whitelist. See bug 315411. + rv = Normalize(input, _retval); + if (NS_FAILED(rv)) return rv; + + // normalization could result in an ASCII-only hostname. alternatively, if + // the host is converted to ACE by the normalizer, then the host may contain + // unsafe characters, so leave it ACE encoded. see bug 283016, bug 301694, and bug 309311. + *_isASCII = IsASCII(_retval); + if (!*_isASCII && !isInWhitelist(_retval)) { + *_isASCII = PR_TRUE; + return ConvertUTF8toACE(_retval, _retval); + } + } + + return NS_OK; +} + //----------------------------------------------------------------------------- static void utf16ToUcs4(const nsAString& in, PRUint32 *out, PRUint32 outBufLen, PRUint32 *outLen) { PRUint32 i = 0; nsAString::const_iterator start, end; in.BeginReading(start); in.EndReading(end); @@ -404,19 +501,24 @@ static nsresult encodeToRACE(const char* // 3) Prohibit -- Check for any characters that are not allowed in the // output. If any are found, return an error. This is described in section // 5. // // 4) Check bidi -- Possibly check for right-to-left characters, and if any // are found, make sure that the whole string satisfies the requirements // for bidirectional strings. If the string does not satisfy the requirements // for bidirectional strings, return an error. This is described in section 6. // -nsresult nsIDNService::stringPrep(const nsAString& in, nsAString& out) +// 5) Check unassigned code points -- If allowUnassigned is false, check for +// any unassigned Unicode points and if any are found return an error. +// This is described in section 7. +// +nsresult nsIDNService::stringPrep(const nsAString& in, nsAString& out, + PRBool allowUnassigned) { if (!mNamePrepHandle || !mNormalizer) return NS_ERROR_FAILURE; nsresult rv = NS_OK; PRUint32 ucs4Buf[kMaxDNSNodeLen + 1]; PRUint32 ucs4Len; utf16ToUcs4(in, ucs4Buf, kMaxDNSNodeLen, &ucs4Len); @@ -446,50 +548,59 @@ nsresult nsIDNService::stringPrep(const if (idn_err != idn_success || found) return NS_ERROR_FAILURE; // check bidi idn_err = idn_nameprep_isvalidbidi(mNamePrepHandle, (const PRUint32 *) ucs4Buf, &found); if (idn_err != idn_success || found) return NS_ERROR_FAILURE; + if (!allowUnassigned) { + // check unassigned code points + idn_err = idn_nameprep_isunassigned(mNamePrepHandle, + (const PRUint32 *) ucs4Buf, &found); + if (idn_err != idn_success || found) + return NS_ERROR_FAILURE; + } + // set the result string out.Assign(normlizedStr); return rv; } nsresult nsIDNService::encodeToACE(const nsAString& in, nsACString& out) { // RACE encode is supported for existing testing environment if (!strcmp("bq--", mACEPrefix)) return encodeToRACE(mACEPrefix, in, out); // use punycoce return punycode(mACEPrefix, in, out); } -nsresult nsIDNService::stringPrepAndACE(const nsAString& in, nsACString& out) +nsresult nsIDNService::stringPrepAndACE(const nsAString& in, nsACString& out, + PRBool allowUnassigned) { nsresult rv = NS_OK; out.Truncate(); if (in.Length() > kMaxDNSNodeLen) { NS_ERROR("IDN node too large"); return NS_ERROR_FAILURE; } if (IsASCII(in)) CopyUCS2toASCII(in, out); else { nsAutoString strPrep; - rv = stringPrep(in, strPrep); + rv = stringPrep(in, strPrep, allowUnassigned); if (NS_SUCCEEDED(rv)) { if (IsASCII(strPrep)) CopyUCS2toASCII(strPrep, out); else rv = encodeToACE(strPrep, out); } } if (out.Length() > kMaxDNSNodeLen) { @@ -522,19 +633,20 @@ void nsIDNService::normalizeFullStops(ns break; default: break; } start++; index++; } } -nsresult nsIDNService::decodeACE(const nsACString& in, nsACString& out) +nsresult nsIDNService::decodeACE(const nsACString& in, nsACString& out, + PRBool allowUnassigned) { PRBool isAce; IsACE(in, &isAce); if (!isAce) { out.Assign(in); return NS_OK; } // RFC 3490 - 4.2 ToUnicode @@ -558,17 +670,39 @@ nsresult nsIDNService::decodeACE(const n nsAutoString utf16; ucs4toUtf16(output, utf16); delete [] output; if (!isOnlySafeChars(utf16, mIDNBlacklist)) return NS_ERROR_FAILURE; CopyUTF16toUTF8(utf16, out); // Validation: encode back to ACE and compare the strings nsCAutoString ace; - nsresult rv = ConvertUTF8toACE(out, ace); + nsresult rv = UTF8toACE(out, ace, allowUnassigned); NS_ENSURE_SUCCESS(rv, rv); if (!ace.Equals(in, nsCaseInsensitiveCStringComparator())) return NS_ERROR_FAILURE; return NS_OK; } + +PRBool nsIDNService::isInWhitelist(const nsACString &host) +{ + if (mIDNWhitelistPrefBranch) { + nsCAutoString tld(host); + // make sure the host is ACE for lookup and check that there are no + // unassigned codepoints + if (!IsASCII(tld) && NS_FAILED(UTF8toACE(tld, tld, PR_FALSE))) { + return PR_FALSE; + } + + tld.Trim("."); + PRInt32 pos = tld.RFind("."); + + PRBool safe; + if (pos != kNotFound && + NS_SUCCEEDED(mIDNWhitelistPrefBranch->GetBoolPref(tld.get() + pos + 1, &safe))) + return safe; + } + + return PR_FALSE; +} Index: netwerk/dns/src/nsIDNService.h =================================================================== RCS file: /cvsroot/mozilla/netwerk/dns/src/nsIDNService.h,v retrieving revision 1.8 diff -u -9 -p -r1.8 nsIDNService.h --- netwerk/dns/src/nsIDNService.h 22 Jul 2005 15:07:33 -0000 1.8 +++ netwerk/dns/src/nsIDNService.h 3 Jun 2009 09:48:30 -0000 @@ -1,10 +1,10 @@ -/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* ***** BEGIN LICENSE BLOCK ***** * Version: MPL 1.1/GPL 2.0/LGPL 2.1 * * The contents of this file are subject to the Mozilla Public License Version * 1.1 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.mozilla.org/MPL/ * * Software distributed under the License is distributed on an "AS IS" basis, @@ -49,39 +49,50 @@ class nsIPrefBranch; //----------------------------------------------------------------------------- // nsIDNService //----------------------------------------------------------------------------- #define kACEPrefixLen 4 -class nsIDNService : public nsIIDNService, +class nsIDNService : public nsIIDNService_MOZILLA_1_8_BRANCH, public nsIObserver, public nsSupportsWeakReference { public: NS_DECL_ISUPPORTS NS_DECL_NSIIDNSERVICE + NS_DECL_NSIIDNSERVICE_MOZILLA_1_8_BRANCH NS_DECL_NSIOBSERVER nsIDNService(); virtual ~nsIDNService(); nsresult Init(); private: void normalizeFullStops(nsAString& s); - nsresult stringPrepAndACE(const nsAString& in, nsACString& out); + nsresult stringPrepAndACE(const nsAString& in, nsACString& out, + PRBool allowUnassigned); nsresult encodeToACE(const nsAString& in, nsACString& out); - nsresult stringPrep(const nsAString& in, nsAString& out); - nsresult decodeACE(const nsACString& in, nsACString& out); + nsresult stringPrep(const nsAString& in, nsAString& out, + PRBool allowUnassigned); + nsresult decodeACE(const nsACString& in, nsACString& out, + PRBool allowUnassigned); + nsresult UTF8toACE(const nsACString& in, nsACString& out, + PRBool allowUnassigned); + nsresult ACEtoUTF8(const nsACString& in, nsACString& out, + PRBool allowUnassigned); + PRBool isInWhitelist(const nsACString &host); void prefsChanged(nsIPrefBranch *prefBranch, const PRUnichar *pref); - PRBool mMultilingualTestBed; // if true generates extra node for mulitlingual testbed + PRBool mMultilingualTestBed; // if true generates extra node for multilingual testbed idn_nameprep_t mNamePrepHandle; nsCOMPtr mNormalizer; char mACEPrefix[kACEPrefixLen+1]; nsXPIDLString mIDNBlacklist; + PRBool mShowPunycode; + nsCOMPtr mIDNWhitelistPrefBranch; }; #endif // nsIDNService_h__ Index: netwerk/test/Makefile.in =================================================================== RCS file: /cvsroot/mozilla/netwerk/test/Makefile.in,v retrieving revision 1.85.2.7 diff -u -9 -p -r1.85.2.7 Makefile.in --- netwerk/test/Makefile.in 27 Jun 2006 20:27:29 -0000 1.85.2.7 +++ netwerk/test/Makefile.in 3 Jun 2009 09:48:31 -0000 @@ -97,20 +97,23 @@ _UNIT_FILES = unit/test_all.sh \ unit/head.js \ unit/head_http_server.js \ unit/tail.js \ unit/test_protocolproxyservice.js \ unit/test_http_headers.js \ unit/test_cookie_header.js \ unit/test_parse_content_type.js \ unit/test_event_sink.js \ unit/test_content_sniffer.js \ + unit/test_idnservice.js \ unit/test_bug331825.js \ unit/test_bug336501.js \ + unit/test_bug427957.js \ + unit/test_bug479413.js \ $(NULL) libs:: $(_UNIT_FILES) $(INSTALL) $^ $(DIST)/bin/necko_unit_tests check:: $(RUN_TEST_PROGRAM) $(DIST)/bin/necko_unit_tests/test_all.sh _RES_FILES = urlparse.dat \ urlparse_unx.dat \ Index: netwerk/test/unit/test_bug427957.js =================================================================== RCS file: netwerk/test/unit/test_bug427957.js diff -N netwerk/test/unit/test_bug427957.js --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ netwerk/test/unit/test_bug427957.js 3 Jun 2009 09:48:31 -0000 @@ -0,0 +1,100 @@ +/** + * Test for Bidi restrictions on IDNs from RFC 3454 + */ + +var Cc = Components.classes; +var Ci = Components.interfaces; +var idnService; + +function expected_pass(inputIDN) +{ + var isASCII = {}; + var displayIDN = idnService.convertToDisplayIDN(inputIDN, isASCII); + do_check_eq(displayIDN, inputIDN); +} + +function expected_fail(inputIDN) +{ + var isASCII = {}; + var displayIDN = ""; + + try { + displayIDN = idnService.convertToDisplayIDN(inputIDN, isASCII); + } + catch(e) {} + + do_check_neq(displayIDN, inputIDN); +} + +function run_test() { + // add an IDN whitelist pref + var pbi = Cc["@mozilla.org/preferences-service;1"] + .getService(Ci.nsIPrefBranch2); + pbi.setBoolPref("network.IDN.whitelist.com", true); + + idnService = Cc["@mozilla.org/network/idn-service;1"] + .getService(Ci.nsIIDNService_MOZILLA_1_8_BRANCH); + /* + * In any profile that specifies bidirectional character handling, all + * three of the following requirements MUST be met: + * + * 1) The characters in section 5.8 MUST be prohibited. + */ + + // 0340; COMBINING GRAVE TONE MARK + expected_fail("foo\u0340bar.com"); + // 0341; COMBINING ACUTE TONE MARK + expected_fail("foo\u0341bar.com"); + // 200E; LEFT-TO-RIGHT MARK + expected_fail("foo\200ebar.com"); + // 200F; RIGHT-TO-LEFT MARK + // Note: this is an RTL IDN so that it doesn't fail test 2) below + expected_fail("\u200f\u0645\u062B\u0627\u0644.\u0622\u0632\u0645\u0627\u06CC\u0634\u06CC"); + // 202A; LEFT-TO-RIGHT EMBEDDING + expected_fail("foo\u202abar.com"); + // 202B; RIGHT-TO-LEFT EMBEDDING + expected_fail("foo\u202bbar.com"); + // 202C; POP DIRECTIONAL FORMATTING + expected_fail("foo\u202cbar.com"); + // 202D; LEFT-TO-RIGHT OVERRIDE + expected_fail("foo\u202dbar.com"); + // 202E; RIGHT-TO-LEFT OVERRIDE + expected_fail("foo\u202ebar.com"); + // 206A; INHIBIT SYMMETRIC SWAPPING + expected_fail("foo\u206abar.com"); + // 206B; ACTIVATE SYMMETRIC SWAPPING + expected_fail("foo\u206bbar.com"); + // 206C; INHIBIT ARABIC FORM SHAPING + expected_fail("foo\u206cbar.com"); + // 206D; ACTIVATE ARABIC FORM SHAPING + expected_fail("foo\u206dbar.com"); + // 206E; NATIONAL DIGIT SHAPES + expected_fail("foo\u206ebar.com"); + // 206F; NOMINAL DIGIT SHAPES + expected_fail("foo\u206fbar.com"); + + /* + * 2) If a string contains any RandALCat character, the string MUST NOT + * contain any LCat character. + */ + + // www.מיץpetel.com is invalid + expected_fail("www.\u05DE\u05D9\u05E5petel.com"); + // But www.מיץפטל.com is fine because the ltr and rtl characters are in + // different labels + expected_pass("www.\u05DE\u05D9\u05E5\u05E4\u05D8\u05DC.com"); + + /* + * 3) If a string contains any RandALCat character, a RandALCat + * character MUST be the first character of the string, and a + * RandALCat character MUST be the last character of the string. + */ + + // www.1מיץ.com is invalid + expected_fail("www.1\u05DE\u05D9\u05E5.com"); + // www.מיץ1.com is invalid + expected_fail("www.\u05DE\u05D9\u05E51.com"); + // But www.מיץ1פטל.com is fine + expected_pass("www.\u05DE\u05D9\u05E51\u05E4\u05D8\u05DC.com"); +} + Index: netwerk/test/unit/test_bug479413.js =================================================================== RCS file: netwerk/test/unit/test_bug479413.js diff -N netwerk/test/unit/test_bug479413.js --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ netwerk/test/unit/test_bug479413.js 3 Jun 2009 09:48:31 -0000 @@ -0,0 +1,60 @@ +/** + * Test for unassigned code points in IDNs (RFC 3454 section 7) + */ + +const Cc = Components.classes; +const Ci = Components.interfaces; +var idnService; + +function expected_pass(inputIDN) +{ + var isASCII = {}; + var displayIDN = idnService.convertToDisplayIDN(inputIDN, isASCII); + do_check_eq(displayIDN, inputIDN); +} + +function expected_fail(inputIDN) +{ + var isASCII = {}; + var displayIDN = ""; + + try { + displayIDN = idnService.convertToDisplayIDN(inputIDN, isASCII); + } + catch(e) {} + + do_check_neq(displayIDN, inputIDN); +} + +function run_test() { + // add an IDN whitelist pref + var pbi = Cc["@mozilla.org/preferences-service;1"] + .getService(Ci.nsIPrefBranch2); + var whitelistPref = "network.IDN.whitelist.com"; + + pbi.setBoolPref(whitelistPref, true); + + idnService = Cc["@mozilla.org/network/idn-service;1"] + .getService(Ci.nsIIDNService_MOZILLA_1_8_BRANCH); + + // assigned code point + expected_pass("foo\u0101bar.com"); + + // assigned code point in punycode. Should *fail* because the URL will be + // converted to Unicode for display + expected_fail("xn--foobar-5za.com"); + + // unassigned code point + expected_fail("foo\u3040bar.com"); + + // unassigned code point in punycode. Should *pass* because the URL will not + // be converted to Unicode + expected_pass("xn--foobar-533e.com"); + + // code point assigned since Unicode 3.0 + // XXX This test will unexpectedly pass when we update to IDNAbis + expected_fail("foo\u0370bar.com"); + + // reset the pref + pbi.clearUserPref(whitelistPref); +} Index: netwerk/test/unit/test_idnservice.js =================================================================== RCS file: netwerk/test/unit/test_idnservice.js diff -N netwerk/test/unit/test_idnservice.js --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ netwerk/test/unit/test_idnservice.js 3 Jun 2009 09:48:31 -0000 @@ -0,0 +1,48 @@ +// Tests nsIIDNService + +var reference = [ + // The 3rd element indicates whether the second element + // is ACE-encoded + ["asciihost", "asciihost", false], + ["b\u00FCcher", "xn--bcher-kva", true] + ]; + +function run_test() { + var idnService = Components.classes["@mozilla.org/network/idn-service;1"] + .getService(Components.interfaces.nsIIDNService_MOZILLA_1_8_BRANCH); + + for (var i = 0; i < reference.length; ++i) { + dump("Testing " + reference[i] + "\n"); + // We test the following: + // - Converting UTF-8 to ACE and back gives us the expected answer + // - Converting the ASCII string UTF-8 -> ACE leaves the string unchanged + // - isACE returns true when we expect it to (third array elem true) + do_check_eq(idnService.convertUTF8toACE(reference[i][0]), reference[i][1]); + do_check_eq(idnService.convertUTF8toACE(reference[i][1]), reference[i][1]); + do_check_eq(idnService.convertACEtoUTF8(reference[i][1]), reference[i][0]); + do_check_eq(idnService.isACE(reference[i][1]), reference[i][2]); + } + + // add an IDN whitelist pref + var pbi = Components.classes["@mozilla.org/preferences-service;1"] + .getService(Components.interfaces.nsIPrefBranch2); + pbi.setBoolPref("network.IDN.whitelist.es", true); + + // check convertToDisplayIDN against the whitelist + var isASCII = {}; + do_check_eq(idnService.convertToDisplayIDN("b\u00FCcher.es", isASCII), "b\u00FCcher.es"); + do_check_eq(isASCII.value, false); + do_check_eq(idnService.convertToDisplayIDN("xn--bcher-kva.es", isASCII), "b\u00FCcher.es"); + do_check_eq(isASCII.value, false); + do_check_eq(idnService.convertToDisplayIDN("b\u00FCcher.uk", isASCII), "xn--bcher-kva.uk"); + do_check_eq(isASCII.value, true); + do_check_eq(idnService.convertToDisplayIDN("xn--bcher-kva.uk", isASCII), "xn--bcher-kva.uk"); + do_check_eq(isASCII.value, true); + + // check ACE TLD's are handled by the whitelist + pbi.setBoolPref("network.IDN.whitelist.xn--k-dha", true); + do_check_eq(idnService.convertToDisplayIDN("test.\u00FCk", isASCII), "test.\u00FCk"); + do_check_eq(isASCII.value, false); + do_check_eq(idnService.convertToDisplayIDN("test.xn--k-dha", isASCII), "test.\u00FCk"); + do_check_eq(isASCII.value, false); +}