Scratch PI or JASP from spec. Add extra comments about MarkupFragment's behavior to incoming data. More Todos for our home-brew lexer, and also fix empty tag support in the SAX parer adapter. Add test framework for marking SAX to be tested with the regular result, or a specialized result.

git-svn-id: http://htmlpurifier.org/svnroot/html_purifier/trunk@22 48356398-32a2-884e-a903-53898d9a118a

HTML_Lexer.php

@@ -1,11 +1,14 @@
 <?php
 
 /*
-Lexes SGML style documents, aka HTML, XML, XHMTML, you name it.
+Forgivingly lexes SGML style documents, aka HTML, XML, XHMTML, you name it.
 
 TODO:
- * Validate element names and attributes for correct composition
  * Reread the XML spec and make sure I got everything right
+ * Add support for CDATA sections
+ * Have comments output with the leading and trailing --s
+ * Optimize and benchmark
+ * Check MF_Text behavior: shouldn't the info in there be raw (entities parsed?)
 
 */
 
@@ -193,6 +196,7 @@ class HTML_Lexer
 }
 
 // uses the PEAR class XML_HTMLSax3 to parse XML
+//   only shares the tokenizeHTML() function
 class HTML_Lexer_Sax extends HTML_Lexer
 {
     
@@ -209,12 +213,22 @@ class HTML_Lexer_Sax extends HTML_Lexer
         return $this->tokens;
     }
     
-    function openHandler(&$parser, $name, $attrs) {
+    function openHandler(&$parser, $name, $attrs, $closed) {
-        $this->tokens[] = new MF_StartTag($name, $attrs);
+        if ($closed) {
+            $this->tokens[] = new MF_EmptyTag($name, $attrs);
+        } else {
+            $this->tokens[] = new MF_StartTag($name, $attrs);
+        }
         return true;
     }
     
     function closeHandler(&$parser, $name) {
+        // HTMLSax3 seems to always send empty tags an extra close tag
+        // check and ignore if you see it:
+        // [TESTME] to make sure it doesn't overreach
+        if (is_a($this->tokens[count($this->tokens)-1], 'MF_EmptyTag')) {
+            return true;
+        }
         $this->tokens[] = new MF_EndTag($name);
         return true;
     }

MarkupFragment.php

@@ -9,7 +9,8 @@ class MF_Tag extends MF
 {
     var $name;
     function MF_Tag($name) {
-        $this->name = strtolower($name);
+        $this->name = strtolower($name); // for some reason, the SAX parser
+                                         // uses uppercase. Investigate?
     }
 }
 
@@ -31,6 +32,7 @@ class MF_Text extends MF
     var $data;
     function MF_Text($data) {
         $this->data = trim($data); // fairly certain trimming it's okay
+                                   // but it's not default SAX behavior
     }
     function append($mf_text) {
         return new MF_Text($this->data . $mf_text->data);

docs/spec.txt

@@ -20,7 +20,7 @@ of little classes:
 * Comment(text)                 is escapeHandler (has leading -)
 * CharacterData(text)           is escapeHandler (has leading [)
 
-Ignorable (although we probably want to output them raw):
+Ignorable/not being implemented (although we probably want to output them raw):
 * ProcessingInstructions(text)  is piHandler
 * JavaOrASPInstructions(text)   is jaspHandler
 

tests/HTML_Lexer.php

@@ -1,5 +1,9 @@
 <?php
 
+/* TODO
+    * Benchmark the SAX parser with my homemade one
+ */
+
 class TestCase_HTML_Lexer extends UnitTestCase
 {
     
@@ -23,16 +27,20 @@ class TestCase_HTML_Lexer extends UnitTestCase
     
     function test_tokenizeHTML() {
         
-        $input[] = '';
+        $input = array();
-        $expect[] = array();
+        $expect = array();
+        $sax_expect = array();
         
-        $input[] = 'This is regular text.';
+        $input[0] = '';
-        $expect[] = array(
+        $expect[0] = array();
+        
+        $input[1] = 'This is regular text.';
+        $expect[1] = array(
             new MF_Text('This is regular text.')
             );
         
-        $input[] = 'This is <b>bold</b> text';
+        $input[2] = 'This is <b>bold</b> text';
-        $expect[] = array(
+        $expect[2] = array(
             new MF_Text('This is ')
            ,new MF_StartTag('b', array())
            ,new MF_Text('bold')
@@ -40,8 +48,8 @@ class TestCase_HTML_Lexer extends UnitTestCase
            ,new MF_Text(' text')
             );
         
-        $input[] = '<DIV>Totally rad dude. <b>asdf</b></div>';
+        $input[3] = '<DIV>Totally rad dude. <b>asdf</b></div>';
-        $expect[] = array(
+        $expect[3] = array(
             new MF_StartTag('DIV', array())
            ,new MF_Text('Totally rad dude. ')
            ,new MF_StartTag('b', array())
@@ -50,8 +58,8 @@ class TestCase_HTML_Lexer extends UnitTestCase
            ,new MF_EndTag('div')
             );
         
-        $input[] = '<asdf></asdf><d></d><poOloka><poolasdf><ds></asdf></ASDF>';
+        $input[4] = '<asdf></asdf><d></d><poOloka><poolasdf><ds></asdf></ASDF>';
-        $expect[] = array(
+        $expect[4] = array(
             new MF_StartTag('asdf')
            ,new MF_EndTag('asdf')
            ,new MF_StartTag('d')
@@ -63,8 +71,8 @@ class TestCase_HTML_Lexer extends UnitTestCase
            ,new MF_EndTag('ASDF')
             );
         
-        $input[] = '<a'."\t".'href="foobar.php"'."\n".'title="foo!">Link to <b id="asdf">foobar</b></a>';
+        $input[5] = '<a'."\t".'href="foobar.php"'."\n".'title="foo!">Link to <b id="asdf">foobar</b></a>';
-        $expect[] = array(
+        $expect[5] = array(
             new MF_StartTag('a',array('href'=>'foobar.php','title'=>'foo!'))
            ,new MF_Text('Link to ')
            ,new MF_StartTag('b',array('id'=>'asdf'))
@@ -73,32 +81,56 @@ class TestCase_HTML_Lexer extends UnitTestCase
            ,new MF_EndTag('a')
             );
         
-        $input[] = '<br />';
+        $input[6] = '<br />';
-        $expect[] = array(
+        $expect[6] = array(
             new MF_EmptyTag('br')
             );
         
-        $input[] = '<!-- Comment --> <!-- not so well formed --->';
+        // [INVALID] [RECOVERABLE]
-        $expect[] = array(
+        $input[7] = '<!-- Comment --> <!-- not so well formed --->';
+        $expect[7] = array(
             new MF_Comment(' Comment ')
            ,new MF_Text(' ')
            ,new MF_Comment(' not so well formed -')
             );
+        $sax_expect[7] = false; // we need to figure out proper comment output
         
-        $input[] = '<a href=""';
+        // [INVALID]
-        $expect[] = array(
+        $input[8] = '<a href=""';
+        $expect[8] = array(
             new MF_Text('<a href=""')
             );
+        // SAX parses it into a tag
+        $sax_expect[8] = array(
+            new MF_StartTag('a', array('href'=>''))
+            ); 
         
-        $size = count($input);
+        $input[9] = '&lt;b&gt;';
-        for($i = 0; $i < $size; $i++) {
+        $expect[9] = array(
+            new MF_Text('&lt;b&gt;')
+            );
+        // however, we may want to change both styles
+        // into parsed: '<b>'. SAX has an option for this
+        
+        foreach($input as $i => $discard) {
             $result = $this->HTML_Lexer->tokenizeHTML($input[$i]);
             $this->assertEqual($expect[$i], $result);
             paintIf($result, $expect[$i] != $result);
             
-            // since I didn't write the parser, I can't define its behavior
+            // assert unless I say otherwise
-            // however, make sure that the class runs without any errors
+            $sax_result = $this->HTML_Lexer_Sax->tokenizeHTML($input[$i]);
-            $exp_result = $this->HTML_Lexer_Sax->tokenizeHTML($input[$i]);
+            if (!isset($sax_expect[$i])) {
+                // by default, assert with normal result
+                $this->assertEqual($expect[$i], $sax_result);
+                paintIf($sax_result, $expect[$i] != $sax_result);
+            } elseif ($sax_expect[$i] === false) {
+                // assertions were turned off, optionally dump
+                // paintIf($sax_expect, $i == NUMBER);
+            } else {
+                // match with a custom SAX result array
+                $this->assertEqual($sax_expect[$i], $sax_result);
+                paintIf($sax_result, $sax_expect[$i] != $sax_result);
+            }
         }
         
     }