Add CDATA support to the Lexers, as well as give PEARSax3 entity replacement.

git-svn-id: http://htmlpurifier.org/svnroot/htmlpurifier/trunk@106 48356398-32a2-884e-a903-53898d9a118a

library/HTMLPurifier/Lexer.php

@@ -15,20 +15,26 @@ require_once 'HTMLPurifier/Token.php';
  * recommended, as we adhere to a subset of the specification for optimization
  * reasons.
  * 
- * This class cannot be directly instantiated, but you may use create() to
+ * This class should not be directly instantiated, but you may use create() to
  * retrieve a default copy of the lexer.
  * 
+ * @note The unit tests will instantiate this class for testing purposes, as
+ *       many of the utility functions require a class to be instantiated.
+ *       Be careful when porting this class to PHP 5.
+ * 
+ * @par
+ * 
  * @note
  * We use tokens rather than create a DOM representation because DOM would:
  * 
- * @note
+ * @par
  *  -# Require more processing power to create,
  *  -# Require recursion to iterate,
  *  -# Must be compatible with PHP 5's DOM (otherwise duplication),
  *  -# Has the entire document structure (html and body not needed), and
  *  -# Has unknown readability improvement.
  * 
- * @note
+ * @par
  * What the last item means is that the functions for manipulating tokens are
  * already fairly compact, and when well-commented, more abstraction may not
  * be needed.
@@ -85,14 +91,53 @@ class HTMLPurifier_Lexer
         return $lexer;
     }
     
+    /**
+     * Decimal to parsed string conversion table for special entities.
+     * @protected
+     */
+    var $_special_dec2str =
+            array(
+                    34 => '"',
+                    38 => '&',
+                    39 => "'",
+                    60 => '<',
+                    62 => '>'
+            );
+    
+    /**
+     * Stripped entity names to decimal conversion table for special entities.
+     * @protected
+     */
+    var $_special_ent2dec =
+            array(
+                    'quot' => 34,
+                    'amp'  => 38,
+                    'lt'   => 60,
+                    'gt'   => 62
+            );
+    
+    /**
+     * Most common entity to raw value conversion table for special entities.
+     * @protected
+     */
+    var $_special_entity2str =
+            array(
+                    '&quot;' => '"',
+                    '&amp;'  => '&',
+                    '&lt;'   => '<',
+                    '&gt;'   => '>',
+                    '&#39;'  => "'",
+                    '&#039;' => "'",
+                    '&#x27;' => "'"
+            );
     
     /**
      * Callback regex string for parsing entities.
      * @protected
-     */
+     */                             
     var $_substituteEntitiesRegex =
-        //       1. hex          2. dec  3. string
+'/&(?:[#]x([a-fA-F0-9]+)|[#]0*(\d+)|([A-Za-z]+));?/';
-        '/&[#](?:x([a-fA-F0-9]+)|0*(\d+)|([A-Za-z]+));?/';
+//     1. hex             2. dec      3. string
     
     /**
      * Substitutes non-special entities with their parsed equivalents. Since
@@ -107,8 +152,9 @@ class HTMLPurifier_Lexer
         // it will try to detect missing semicolons, but don't rely on it
         return preg_replace_callback(
             $this->_substituteEntitiesRegex,
-            array('HTMLPurifier_Lexer_DirectLex', 'nonSpecialEntityCallback'),
+            array($this, 'nonSpecialEntityCallback'),
-            $string);
+            $string
+            );
     }
     
     /**
@@ -116,7 +162,7 @@ class HTMLPurifier_Lexer
      * 
      * @warning Though this is public in order to let the callback happen,
      *          calling it directly is not recommended.
-     * @param $matches  PCRE-style matches array, with 0 the entire match, and
+     * @param $matches  PCRE matches array, with 0 the entire match, and
      *                  either index 1, 2 or 3 set with a hex value, dec value,
      *                  or string (respectively).
      * @returns Replacement string.
@@ -135,7 +181,7 @@ class HTMLPurifier_Lexer
             if (isset($this->_special_ent2dec[$matches[3]])) return $entity;
             if (!$this->_entity_lookup) {
                 require_once 'HTMLPurifier/EntityLookup.php';
-                $this->_entity_lookup = EntityLookup::instance();
+                $this->_entity_lookup = HTMLPurifier_EntityLookup::instance();
             }
             if (isset($this->_entity_lookup->table[$matches[3]])) {
                 return $this->_entity_lookup->table[$matches[3]];
@@ -145,8 +191,41 @@ class HTMLPurifier_Lexer
         }
     }
     
+    /**
+     * Contains a copy of the EntityLookup table.
+     * @protected
+     */
     var $_entity_lookup;
     
+    /**
+     * Translates CDATA sections into regular sections (through escaping).
+     * 
+     * @protected
+     * @param $string HTML string to process.
+     * @returns HTML with CDATA sections escaped.
+     */
+    function escapeCDATA($string) {
+        return preg_replace_callback(
+            '/<!\[CDATA\[(.+?)\]\]>/',
+            array('HTMLPurifier_Lexer', 'CDATACallback'),
+            $string
+        );
+    }
+    
+    /**
+     * Callback function for escapeCDATA() that does the work.
+     * 
+     * @warning Though this is public in order to let the callback happen,
+     *          calling it directly is not recommended.
+     * @params $matches PCRE matches array, with index 0 the entire match
+     *                  and 1 the inside of the CDATA section.
+     * @returns Escaped internals of the CDATA section.
+     */
+    function CDATACallback($matches) {
+        // not exactly sure why the character set is needed, but whatever
+        return htmlspecialchars($matches[1], ENT_COMPAT, 'UTF-8');
+    }
+    
 }
 
 ?>

library/HTMLPurifier/Lexer/DOMLex.php

@@ -27,8 +27,14 @@ class HTMLPurifier_Lexer_DOMLex extends HTMLPurifier_Lexer
     
     public function tokenizeHTML($string) {
         $doc = new DOMDocument();
+        
         // preprocess string
         $string = '<html><body><div>'.$string.'</div></body></html>';
+        
+        // replace and escape the CDATA sections, since parsing under HTML
+        // mode won't get 'em.
+        $string = $this->escapeCDATA($string);
+        
         @$doc->loadHTML($string); // mute all errors, handle it transparently
         return $this->tokenizeDOM(
             $doc->childNodes->item(1)-> // html
@@ -55,7 +61,8 @@ class HTMLPurifier_Lexer_DOMLex extends HTMLPurifier_Lexer
         if ( !($node instanceof DOMElement) ) {
             if ($node instanceof DOMComment) {
                 $tokens[] = new HTMLPurifier_Token_Comment($node->data);
-            } elseif ($node instanceof DOMText) {
+            } elseif ($node instanceof DOMText ||
+                      $node instanceof DOMCharacterData) {
                 $tokens[] = new HTMLPurifier_Token_Text($node->data);
             }
             // quite possibly, the object wasn't handled, that's fine

library/HTMLPurifier/Lexer/DirectLex.php

@@ -61,43 +61,6 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
      */
     var $_whitespace = "\x20\x09\x0D\x0A";
     
-    /**
-     * Decimal to parsed string conversion table for special entities.
-     * @protected
-     */
-    var $_special_dec2str = array(
-            34 => '"', // quote
-            38 => '&', // ampersand            
-            39 => "'", // apostrophe           
-            60 => '<', // less than sign
-            62 => '>'  // greater than sign
-        );
-    
-    /**
-     * Stripped entity names to decimal conversion table for special entities.
-     * @protected
-     */
-    var $_special_ent2dec = array(
-            'quot' => 34,
-            'amp'  => 38,
-            'lt'   => 60,
-            'gt'   => 62,
-        );
-    
-    /**
-     * Most common entity to raw value conversion table for special entities.
-     * @protected
-     */
-    var $_special_entity2str = array(
-            '&quot;' => '"',
-            '&amp;'  => '&',
-            '&lt;'   => '<',
-            '&gt;'   => '>',
-            '&#39;'  => "'",
-            '&#039;' => "'",
-            '&#x27;' => "'",
-        );
-    
     /**
      * Substitutes only special entities with their parsed equivalents.
      * 
@@ -153,6 +116,9 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
         $inside_tag = false; // whether or not we're parsing the inside of a tag
         $array = array(); // result array
         
+        // escape CDATA
+        $string = $this->escapeCDATA($string);
+        
         // expand entities THAT AREN'T THE BIG FIVE
         $string = $this->substituteNonSpecialEntities($string);
         

library/HTMLPurifier/Lexer/PEARSax3.php

@@ -29,6 +29,8 @@ class HTMLPurifier_Lexer_PEARSax3 extends HTMLPurifier_Lexer
     var $tokens = array();
     
     function tokenizeHTML($html) {
+        $html = $this->escapeCDATA($html);
+        $html = $this->substituteNonSpecialEntities($html);
         $parser=& new XML_HTMLSax3();
         $parser->set_object($this);
         $parser->set_element_handler('openHandler','closeHandler');
@@ -79,9 +81,14 @@ class HTMLPurifier_Lexer_PEARSax3 extends HTMLPurifier_Lexer
      * Escaped text handler, interface is defined by PEAR package.
      */
     function escapeHandler(&$parser, $data) {
-        if (strpos($data, '-') === 0) {
+        if (strpos($data, '--') === 0) {
             $this->tokens[] = new HTMLPurifier_Token_Comment($data);
         }
+        // CDATA is handled elsewhere, but if it was handled here:
+        //if (strpos($data, '[CDATA[') === 0) {
+        //    $this->tokens[] = new HTMLPurifier_Token_Text(
+        //        substr($data, 7, strlen($data) - 9) );
+        //}
         return true;
     }
     

tests/HTMLPurifier/Lexer/DirectLexTest.php

@@ -11,6 +11,13 @@ class HTMLPurifier_Lexer_DirectLexTest extends UnitTestCase
         $this->DirectLex = new HTMLPurifier_Lexer_DirectLex();
     }
     
+    function test_specialEntityCallback() {
+        $HP =& $this->DirectLex;
+        
+        $this->assertIdentical("'",$HP->specialEntityCallback(
+            array(''', null, '39', null) ));
+    }
+    
     function test_parseData() {
         $HP =& $this->DirectLex;
         
@@ -29,13 +36,6 @@ class HTMLPurifier_Lexer_DirectLexTest extends UnitTestCase
         $this->assertIdentical('-', $HP->parseData('-'));
     }
     
-    function test_specialEntityCallback() {
-        $HP =& $this->DirectLex;
-        
-        $this->assertIdentical("'",$HP->specialEntityCallback(
-            array(''', null, '39', null) ));
-    }
-    
     // internals testing
     function test_parseAttributeString() {
         

tests/HTMLPurifier/LexerTest.php

@@ -6,11 +6,14 @@ require_once 'HTMLPurifier/Lexer/PEARSax3.php';
 class HTMLPurifier_LexerTest extends UnitTestCase
 {
     
+    var $Lexer;
     var $DirectLex, $PEARSax3, $DOMLex;
     var $_entity_lookup;
     var $_has_dom;
     
     function setUp() {
+        $this->Lexer     = new HTMLPurifier_Lexer();
+        
         $this->DirectLex = new HTMLPurifier_Lexer_DirectLex();
         $this->PEARSax3  = new HTMLPurifier_Lexer_PEARSax3();
         
@@ -24,6 +27,14 @@ class HTMLPurifier_LexerTest extends UnitTestCase
         
     }
     
+    function test_substituteNonSpecialEntities() {
+        $char_theta = $this->_entity_lookup->table['theta'];
+        $this->assertIdentical($char_theta,
+            $this->Lexer->substituteNonSpecialEntities('θ') );
+        $this->assertIdentical('"',
+            $this->Lexer->substituteNonSpecialEntities('"') );
+    }
+    
     function test_tokenizeHTML() {
         
         $input = array();
@@ -156,40 +167,70 @@ class HTMLPurifier_LexerTest extends UnitTestCase
         $expect[12] = array( new HTMLPurifier_Token_Text('"') );
         $sax_expect[12] = false; // choked!
         
-        // DOM and SAX choke on this
+        // CDATA sections!
-        //$char_circ = $this->_entity_lookup->table['circ'];
+        $input[13] = '<![CDATA[You <b>can&#39;t</b> get me!]]>';
-        //$input[13] = '&circ;';
+        $expect[13] = array( new HTMLPurifier_Token_Text(
-        //$expect[13] = array( new HTMLPurifier_Token_Text($char_circ) );
+            'You <b>can&#39;t</b> get me!' // raw
+            ) );
+        $sax_expect[13] = array( // SAX has a seperate call for each entity
+                new HTMLPurifier_Token_Text('You '),
+                new HTMLPurifier_Token_Text('<'),
+                new HTMLPurifier_Token_Text('b'),
+                new HTMLPurifier_Token_Text('>'),
+                new HTMLPurifier_Token_Text('can'),
+                new HTMLPurifier_Token_Text('&'),
+                new HTMLPurifier_Token_Text('#39;t'),
+                new HTMLPurifier_Token_Text('<'),
+                new HTMLPurifier_Token_Text('/b'),
+                new HTMLPurifier_Token_Text('>'),
+                new HTMLPurifier_Token_Text(' get me!')
+            );
+        
+        $char_theta = $this->_entity_lookup->table['theta'];
+        $char_rarr  = $this->_entity_lookup->table['rarr'];
+        
+        // test entity replacement
+        $input[14] = '&theta;';
+        $expect[14] = array( new HTMLPurifier_Token_Text($char_theta) );
+        
+        // test that entities aren't replaced in CDATA sections
+        $input[15] = '&theta; <![CDATA[&rarr;]]>';
+        $expect[15] = array( new HTMLPurifier_Token_Text($char_theta . ' &rarr;') );
+        $sax_expect[15] = array(
+                new HTMLPurifier_Token_Text($char_theta . ' '),
+                new HTMLPurifier_Token_Text('&'),
+                new HTMLPurifier_Token_Text('rarr;')
+            );
         
         foreach($input as $i => $discard) {
             $result = $this->DirectLex->tokenizeHTML($input[$i]);
-            $this->assertEqual($expect[$i], $result, 'Test '.$i.': %s');
+            $this->assertEqual($expect[$i], $result, 'DirectLexTest '.$i.': %s');
             paintIf($result, $expect[$i] != $result);
             
             // assert unless I say otherwise
             $sax_result = $this->PEARSax3->tokenizeHTML($input[$i]);
             if (!isset($sax_expect[$i])) {
                 // by default, assert with normal result
-                $this->assertEqual($expect[$i], $sax_result, 'Test '.$i.': %s');
+                $this->assertEqual($expect[$i], $sax_result, 'PEARSax3Test '.$i.': %s');
                 paintIf($sax_result, $expect[$i] != $sax_result);
             } elseif ($sax_expect[$i] === false) {
                 // assertions were turned off, optionally dump
                 // paintIf($sax_expect, $i == NUMBER);
             } else {
                 // match with a custom SAX result array
-                $this->assertEqual($sax_expect[$i], $sax_result, 'Test '.$i.': %s');
+                $this->assertEqual($sax_expect[$i], $sax_result, 'PEARSax3Test (custom) '.$i.': %s');
                 paintIf($sax_result, $sax_expect[$i] != $sax_result);
             }
             if ($this->_has_dom) {
                 $dom_result = $this->DOMLex->tokenizeHTML($input[$i]);
                 // same structure as SAX
                 if (!isset($dom_expect[$i])) {
-                    $this->assertEqual($expect[$i], $dom_result, 'Test '.$i.': %s');
+                    $this->assertEqual($expect[$i], $dom_result, 'DOMLexTest '.$i.': %s');
                     paintIf($dom_result, $expect[$i] != $dom_result);
                 } elseif ($dom_expect[$i] === false) {
                     // paintIf($dom_result, $i == NUMBER);
                 } else {
-                    $this->assertEqual($dom_expect[$i], $dom_result, 'Test '.$i.': %s');
+                    $this->assertEqual($dom_expect[$i], $dom_result, 'DOMLexTest (custom) '.$i.': %s');
                     paintIf($dom_result, $dom_expect[$i] != $dom_result);
                 }
             }