Optional support for IDNAs with PEAR Net_IDNA2

Signed-off-by: Edward Z. Yang <ezyang@mit.edu>

INSTALL

@@ -26,6 +26,10 @@ These optional extensions can enhance the capabilities of HTML Purifier:
     * bcmath : Used for unit conversion and imagecrash protection
     * tidy   : Used for pretty-printing HTML
 
+These optional libraries can enhance the capabilities of HTML Purifier:
+
+    * CSSTidy : Clean CSS stylesheets using %Core.ExtractStyleBlocks
+    * Net_IDNA2 (PEAR) : IRI support using %Core.EnableIDNA
 
 ---------------------------------------------------------------------------
 2.  Reconnaissance

NEWS

@@ -25,6 +25,9 @@ NEWS ( CHANGELOG and HISTORY )                                     HTMLPurifier
   %HTML.SafeIframe and %URI.SafeIframeRegexp.  Thanks Bradley M. Froehle
   <brad.froehle@gmail.com> for submitting an initial version of the patch.
 ! The Forms module now works properly for transitional doctypes.
+! Added support for internationalized domain names. You need the PEAR
+  Net_IDNA2 module to be in your path; if it is installed, ensure the
+  class can be loaded and then set %Core.EnableIDNA to true.
 - Color keywords are now case insensitive.  Thanks Yzmir Ramirez
   <yramirez-htmlpurifier@adicio.com> for reporting.
 - Explicitly initialize anonModule variable to null.

configdoc/usage.xml

@@ -347,6 +347,11 @@
    <line>30</line>
   </file>
  </directive>
+ <directive id="Core.EnableIDNA">
+  <file name="HTMLPurifier/AttrDef/URI/Host.php">
+   <line>67</line>
+  </file>
+ </directive>
  <directive id="Attr.DefaultTextDir">
   <file name="HTMLPurifier/AttrTransform/BdoDir.php">
    <line>13</line>

library/HTMLPurifier/AttrDef/URI/Host.php

@@ -44,9 +44,8 @@ class HTMLPurifier_AttrDef_URI_Host extends HTMLPurifier_AttrDef
 
         // A regular domain name.
 
-        // This breaks I18N domain names, but we don't have proper IRI support,
-        // so force users to insert Punycode. If there's complaining we'll
-        // try to fix things into an international friendly form.
+        // This doesn't match I18N domain names, but we don't have proper IRI support,
+        // so force users to insert Punycode.
 
         // The productions describing this are:
         $a   = '[a-z]';     // alpha
@@ -57,10 +56,44 @@ class HTMLPurifier_AttrDef_URI_Host extends HTMLPurifier_AttrDef
         // toplabel    = alpha | alpha *( alphanum | "-" ) alphanum
         $toplabel      = "$a($and*$an)?";
         // hostname    = *( domainlabel "." ) toplabel [ "." ]
-        $match = preg_match("/^($domainlabel\.)*$toplabel\.?$/i", $string);
-        if (!$match) return false;
+        if (preg_match("/^($domainlabel\.)*$toplabel\.?$/i", $string)) {
+            return $string;
+        }
 
-        return $string;
+        // If we have Net_IDNA2 support, we can support IRIs by
+        // punycoding them. (This is the most portable thing to do,
+        // since otherwise we have to assume browsers support
+
+        if ($config->get('Core.EnableIDNA')) {
+            $idna = new Net_IDNA2(array('encoding' => 'utf8', 'overlong' => false, 'strict' => true));
+            // we need to encode each period separately
+            $parts = explode('.', $string);
+            try {
+                $new_parts = array();
+                foreach ($parts as $part) {
+                    $encodable = false;
+                    for ($i = 0, $c = strlen($part); $i < $c; $i++) {
+                        if (ord($part[$i]) > 0x7a) {
+                            $encodable = true;
+                            break;
+                        }
+                    }
+                    if (!$encodable) {
+                        $new_parts[] = $part;
+                    } else {
+                        $new_parts[] = $idna->encode($part);
+                    }
+                }
+                $string = implode('.', $new_parts);
+                if (preg_match("/^($domainlabel\.)*$toplabel\.?$/i", $string)) {
+                    return $string;
+                }
+            } catch (Exception $e) {
+                // XXX error reporting
+            }
+        }
+
+        return false;
     }
 
 }

library/HTMLPurifier/ConfigSchema/schema/Core.EnableIDNA.txt

@@ -0,0 +1,9 @@
+Core.EnableIDNA
+TYPE: bool
+DEFAULT: false
+VERSION: 4.4.0
+--DESCRIPTION--
+Allows international domain names in URLs.  This configuration option
+requires the PEAR Net_IDNA2 module to be installed.  It operates by
+punycoding any internationalized host names for maximum portability.
+--# vim: et sw=4 sts=4

maintenance/generate-standalone.php

@@ -145,7 +145,6 @@ make_dir_standalone('HTMLPurifier/Filter');
 make_dir_standalone('HTMLPurifier/Printer');
 make_file_standalone('HTMLPurifier/Printer.php');
 make_file_standalone('HTMLPurifier/Lexer/PH5P.php');
-make_file_standalone('HTMLPurifier/Lexer/PEARSax3.php');
 
 echo ' done!' . PHP_EOL;
 

test-settings.sample.php

@@ -69,4 +69,8 @@ $phpv = false;
 // to true (or, if it's not in the include path, to its install directory).
 $GLOBALS['HTMLPurifierTest']['PEAR'] = false;
 
+// If PEAR is enabled, what PEAR tests should be run? (Note: you will
+// need to ensure these libraries are installed)
+$GLOBALS['HTMLPurifierTest']['Net_IDNA2'] = true;
+
 // vim: et sw=4 sts=4

tests/HTMLPurifier/AttrDef/URI/HostTest.php

@@ -35,6 +35,17 @@ class HTMLPurifier_AttrDef_URI_HostTest extends HTMLPurifier_AttrDefHarness
         $this->assertDef('f1.top');
         $this->assertDef('f-.top', false);
 
+        $this->assertDef("\xE4\xB8\xAD\xE6\x96\x87.com.cn", false);
+
+    }
+
+    function testIDNA() {
+        if (!$GLOBALS['HTMLPurifierTest']['Net_IDNA2']) {
+            return false;
+        }
+        $this->config->set('Core.EnableIDNA', true);
+        $this->assertDef("\xE4\xB8\xAD\xE6\x96\x87.com.cn", "xn--fiq228c.com.cn");
+        $this->assertDef("\xe2\x80\x85.com", false); // rejected
     }
 
 }