mirrors/htmlpurifier
0
0
Fork 0
mirror of https://github.com/ezyang/htmlpurifier.git synced 2025-01-03 13:21:51 +00:00
Code Releases Activity

- Implemented special entity conversion.

Browse Source 
- Optimized and documented DirectLex.
- Rearranged test cases.

git-svn-id: http://htmlpurifier.org/svnroot/htmlpurifier/trunk@100 48356398-32a2-884e-a903-53898d9a118a
This commit is contained in:
Edward Z. Yang 2006-07-23 18:13:04 +00:00
parent dc24215874
commit 2fa1161d3d
2 changed files with 226 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

270
library/HTMLPurifier/Lexer/DirectLex.php
View File

@ -1,74 +1,172 @@
<?php <?php
/*
TODO:
* Reread the XML spec and make sure I got everything right
* Add support for CDATA sections
* Have comments output with the leading and trailing --s
* Optimize and benchmark
* Check MF_Text behavior: shouldn't the info in there be raw (entities parsed?)
*/
require_once 'HTMLPurifier/Lexer.php'; require_once 'HTMLPurifier/Lexer.php';
/**
* Our in-house implementation of a parser.
*
* A pure PHP parser, DirectLex has absolutely no dependencies, making
* it a reasonably good default for PHP4. Written with efficiency in mind,
* it can be four times faster than HTMLPurifier_Lexer_PEARSax3, although it
* pales in comparison to HTMLPurifier_Lexer_DOMLex. It will support UTF-8
* completely eventually.
*
* @todo Implement non-special string entity conversion.
* @todo Reread XML spec and document differences.
* @todo Add support for CDATA sections.
* @todo Determine correct behavior in outputting comment data. (preserve dashes?)
* @todo Optimize main function tokenizeHTML().
*/
class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
{ {
// does this version of PHP support utf8 as entity function charset? /**
var $_entity_utf8; * Parses special entities into the proper characters.
*
function HTMLPurifier_Lexer() { * This string will translate escaped versions of the special characters
$this->_entity_utf8 = version_compare(PHP_VERSION, '5', '>='); * into the correct ones.
} *
* @warning
// this is QUITE a knotty problem * You should be able to treat the output of this function as
// * completely parsed, but that's only because all other entities should
// The main trouble is that, even while assuming UTF-8 is what we're * have been handled previously in substituteNonSpecialEntities()
// using, we've got to deal with HTML entities (like &mdash;) *
// Not even sure if the PHP 5 decoding function does that. Plus, * @param $string String character data to be parsed.
// SimpleTest doesn't use UTF-8! * @returns Parsed character data.
// */
// However, we MUST parse everything possible, because once you get
// to the HTML generator, it will escape everything possible (although
// that may not be correct, and we should be using htmlspecialchars() ).
//
// Nevertheless, strictly XML speaking, we cannot assume any character
// entities are defined except the htmlspecialchars() ones, so leaving
// the entities inside HERE is not acceptable. (plus, htmlspecialchars
// might convert them anyway). So EVERYTHING must get parsed.
//
// We may need to roll our own character entity lookup table. It's only
// about 250, fortunantely, the decimal/hex ones map cleanly to UTF-8.
function parseData($string) { function parseData($string) {
// we may want to let the user do a different char encoding,
// although there is NO REASON why they shouldn't be able
// to convert it to UTF-8 before they pass it to us
// no support for less than PHP 4.3 // subtracts amps that cannot possibly be escaped
if ($this->_entity_utf8) { $num_amp = substr_count($string, '&') - substr_count($string, '& ') -
// PHP 5+, UTF-8 is nicely supported ($string[strlen($string)-1] === '&' ? 1 : 0);
return @html_entity_decode($string, ENT_QUOTES, 'UTF-8');
} else { if (!$num_amp) return $string; // abort if no entities
// PHP 4, do compat stuff $num_esc_amp = substr_count($string, '&amp;');
$string = html_entity_decode($string, ENT_QUOTES, 'ISO-8859-1'); $string = strtr($string, $this->_special_entity2str);
// get the numeric UTF-8 stuff
$string = preg_replace('/&#(\d+);/me', "chr(\\1)", $string); // code duplication for sake of optimization, see above
$string = preg_replace('/&#x([a-f0-9]+);/mei',"chr(0x\\1)",$string); $num_amp_2 = substr_count($string, '&') - substr_count($string, '& ') -
// get the stringy UTF-8 stuff ($string[strlen($string)-1] === '&' ? 1 : 0);
if ($num_amp_2 <= $num_esc_amp) return $string;
// hmm... now we have some uncommon entities. Use the callback.
$string = $this->substituteSpecialEntities($string);
return $string; return $string;
} }
/**
* Whitespace characters for str(c)spn.
* @protected
*/
var $_whitespace = "\x20\x09\x0D\x0A";
/**
* Decimal to parsed string conversion table for special entities.
* @protected
*/
var $_special_dec2str = array(
34 => '"', // quote
38 => '&', // ampersand
39 => "'", // apostrophe
60 => '<', // less than sign
62 => '>' // greater than sign
);
/**
* Stripped entity names to decimal conversion table for special entities.
* @protected
*/
var $_special_ent2dec = array(
'quot' => 34,
'amp' => 38,
'lt' => 60,
'gt' => 62,
);
/**
* Most common entity to raw value conversion table for special entities.
* @protected
*/
var $_special_entity2str = array(
'&quot;' => '"',
'&amp;' => '&',
'&lt;' => '<',
'&gt;' => '>',
'&#39;' => "'",
'&#039;' => "'",
'&#x27;' => "'",
);
/**
* Callback regex string for parsing entities.
* @protected
*/
var $_substituteEntitiesRegex =
// 1. hex 2. dec 3. string
'/&[#](?:x([a-fA-F0-9]+)|0*(\d+)|([A-Za-z]+));?/';
/**
* Substitutes non-special entities with their parsed equivalents.
*/
function substituteNonSpecialEntities($string) {
// it will try to detect missing semicolons, but don't rely on it
return preg_replace_callback(
$this->_substituteEntitiesRegex,
array('HTMLPurifier_Lexer_DirectLex', 'nonSpecialEntityCallback'),
$string);
} }
function nextQuote($string, $offset = 0) { /**
$next = strcspn($string, '"\'', $offset) + $offset; * Callback function for substituteNonSpecialEntities() that does the work.
return strlen($string) == $next ? false : $next; */
function nonSpecialEntityCallback($matches) {
// replaces all but big five
$entity = $matches[0];
$is_num = (@$matches[0][1] === '#');
if ($is_num) {
$is_hex = (@$entity[2] === 'x');
$int = $is_hex ? hexdec($matches[1]) : (int) $matches[2];
if (isset($this->_special_dec2str[$int])) return $entity;
return chr($int);
} else {
if (isset($this->_special_ent2dec[$matches[3]])) return $entity;
// translate $matches[3]
}
} }
function nextWhiteSpace($string, $offset = 0) { /**
$next = strcspn($string, "\x20\x09\x0D\x0A", $offset) + $offset; * Substitutes only special entities with their parsed equivalents.
return strlen($string) == $next ? false : $next; *
* We try to avoid calling this function because otherwise, it would have
* to be called a lot (for every parsed section).
*/
function substituteSpecialEntities($string) {
return preg_replace_callback(
$this->_substituteEntitiesRegex,
array('HTMLPurifier_Lexer_DirectLex', 'specialEntityCallback'),
$string);
}
/**
* Callback function for substituteSpecialEntities() that does the work.
*
* This callback is very similar to nonSpecialEntityCallback().
*/
function specialEntityCallback($matches) {
$entity = $matches[0];
$is_num = (@$matches[0][1] === '#');
if ($is_num) {
$is_hex = (@$entity[2] === 'x');
$int = $is_hex ? hexdec($matches[1]) : (int) $matches[2];
return isset($this->_special_dec2str[$int]) ?
$this->_special_dec2str[$int] :
$entity;
} else {
return isset($this->_special_ent2dec[$matches[3]]) ?
$this->_special_ent2dec[$matches[3]] :
$entity;
}
} }
function tokenizeHTML($string) { function tokenizeHTML($string) {
@ -81,6 +179,9 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
$inside_tag = false; // whether or not we're parsing the inside of a tag $inside_tag = false; // whether or not we're parsing the inside of a tag
$array = array(); // result array $array = array(); // result array
// expand entities THAT AREN'T THE BIG FIVE
$string = $this->substituteNonSpecialEntities($string);
// infinite loop protection // infinite loop protection
// has to be pretty big, since html docs can be big // has to be pretty big, since html docs can be big
// we're allow two hundred thousand tags... more than enough? // we're allow two hundred thousand tags... more than enough?
@ -104,11 +205,10 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
// We are not inside tag and there still is another tag to parse // We are not inside tag and there still is another tag to parse
$array[] = new $array[] = new
HTMLPurifier_Token_Text( HTMLPurifier_Token_Text(
html_entity_decode( $this->parseData(
substr( substr(
$string, $cursor, $position_next_lt - $cursor $string, $cursor, $position_next_lt - $cursor
), )
ENT_QUOTES
) )
); );
$cursor = $position_next_lt + 1; $cursor = $position_next_lt + 1;
@ -121,28 +221,28 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
// Create Text of rest of string // Create Text of rest of string
$array[] = new $array[] = new
HTMLPurifier_Token_Text( HTMLPurifier_Token_Text(
html_entity_decode( $this->parseData(
substr( substr(
$string, $cursor $string, $cursor
), )
ENT_QUOTES
) )
); );
break; break;
} elseif ($inside_tag && $position_next_gt !== false) { } elseif ($inside_tag && $position_next_gt !== false) {
// We are in tag and it is well formed // We are in tag and it is well formed
// Grab the internals of the tag // Grab the internals of the tag
$segment = substr($string, $cursor, $position_next_gt-$cursor); $strlen_segment = $position_next_gt - $cursor;
$segment = substr($string, $cursor, $strlen_segment);
// Check if it's a comment // Check if it's a comment
if ( if (
substr($segment, 0, 3) == '!--' && substr($segment, 0, 3) == '!--' &&
substr($segment,strlen($segment)-2,2) == '--' substr($segment, $strlen_segment-2, 2) == '--'
) { ) {
$array[] = new $array[] = new
HTMLPurifier_Token_Comment( HTMLPurifier_Token_Comment(
substr( substr(
$segment, 3, strlen($segment) - 5 $segment, 3, $strlen_segment - 5
) )
); );
$inside_tag = false; $inside_tag = false;
@ -164,14 +264,16 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
// trailing slash. Remember, we could have a tag like <br>, so // trailing slash. Remember, we could have a tag like <br>, so
// any later token processing scripts must convert improperly // any later token processing scripts must convert improperly
// classified EmptyTags from StartTags. // classified EmptyTags from StartTags.
$is_self_closing= (strpos($segment,'/') === strlen($segment)-1); $is_self_closing= (strpos($segment,'/') === $strlen_segment-1);
if ($is_self_closing) { if ($is_self_closing) {
$segment = substr($segment, 0, strlen($segment) - 1); $strlen_segment--;
$segment = substr($segment, 0, $strlen_segment);
} }
// Check if there are any attributes // Check if there are any attributes
$position_first_space = $this->nextWhiteSpace($segment); $position_first_space = strcspn($segment, $this->_whitespace);
if ($position_first_space === false) {
if ($position_first_space >= $strlen_segment) {
if ($is_self_closing) { if ($is_self_closing) {
$array[] = new HTMLPurifier_Token_Empty($segment); $array[] = new HTMLPurifier_Token_Empty($segment);
} else { } else {
@ -191,7 +293,7 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
) )
); );
if ($attribute_string) { if ($attribute_string) {
$attributes = $this->tokenizeAttributeString( $attributes = $this->parseAttributeString(
$attribute_string $attribute_string
); );
} else { } else {
@ -210,9 +312,8 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
$array[] = new $array[] = new
HTMLPurifier_Token_Text( HTMLPurifier_Token_Text(
'<' . '<' .
html_entity_decode( $this->parseData(
substr($string, $cursor), substr($string, $cursor)
ENT_QUOTES
) )
); );
break; break;
@ -222,7 +323,13 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
return $array; return $array;
} }
function tokenizeAttributeString($string) { /**
* Takes the inside of an HTML tag and makes an assoc array of attributes.
*
* @param $string Inside of tag excluding name.
* @return Assoc array of attributes.
*/
function parseAttributeString($string) {
$string = (string) $string; // quick typecast $string = (string) $string; // quick typecast
if ($string == '') return array(); // no attributes if ($string == '') return array(); // no attributes
@ -281,17 +388,14 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
break; break;
} }
$cursor += ($value = strspn($string, "\x20\x09\x0D\x0A", $cursor)); $cursor += ($value = strspn($string, $this->_whitespace, $cursor));
$position_next_space = $this->nextWhiteSpace($string, $cursor);
$position_next_equal = strpos($string, '=', $cursor);
// grab the key // grab the key
$key_begin = $cursor; //we're currently at the start of the key $key_begin = $cursor; //we're currently at the start of the key
// scroll past all characters that are the key (not whitespace or =) // scroll past all characters that are the key (not whitespace or =)
$cursor += strcspn($string, "\x20\x09\x0D\x0A=", $cursor); $cursor += strcspn($string, $this->_whitespace . '=', $cursor);
$key_end = $cursor; // now at the end of the key $key_end = $cursor; // now at the end of the key
@ -300,7 +404,7 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
if (!$key) continue; // empty key if (!$key) continue; // empty key
// scroll past all whitespace // scroll past all whitespace
$cursor += strspn($string, "\x20\x09\x0D\x0A", $cursor); $cursor += strspn($string, $this->_whitespace, $cursor);
if ($cursor >= $size) { if ($cursor >= $size) {
$array[$key] = $key; $array[$key] = $key;
@ -315,7 +419,7 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
// key="value" // key="value"
$cursor++; $cursor++;
$cursor += strspn($string, "\x20\x09\x0D\x0A", $cursor); $cursor += strspn($string, $this->_whitespace, $cursor);
// we might be in front of a quote right now // we might be in front of a quote right now
@ -330,12 +434,12 @@ class HTMLPurifier_Lexer_DirectLex extends HTMLPurifier_Lexer
} else { } else {
// it's not quoted, end bound is whitespace // it's not quoted, end bound is whitespace
$value_begin = $cursor; $value_begin = $cursor;
$cursor += strcspn($string, "\x20\x09\x0D\x0A", $cursor); $cursor += strcspn($string, $this->_whitespace, $cursor);
$value_end = $cursor; $value_end = $cursor;
} }
$value = substr($string, $value_begin, $value_end - $value_begin); $value = substr($string, $value_begin, $value_end - $value_begin);
$array[$key] = $value; $array[$key] = $this->parseData($value);
$cursor++; $cursor++;
} else { } else {

66
tests/HTMLPurifier/Lexer/DirectLexTest.php
View File

@ -11,66 +11,76 @@ class HTMLPurifier_Lexer_DirectLexTest extends UnitTestCase
$this->DirectLex = new HTMLPurifier_Lexer_DirectLex(); $this->DirectLex = new HTMLPurifier_Lexer_DirectLex();
} }
function test_nextWhiteSpace() {
$HP =& $this->DirectLex;
$this->assertIdentical(false, $HP->nextWhiteSpace('asdf'));
$this->assertIdentical(0, $HP->nextWhiteSpace(' asdf'));
$this->assertIdentical(0, $HP->nextWhiteSpace("\nasdf"));
$this->assertIdentical(1, $HP->nextWhiteSpace("a\tsdf"));
$this->assertIdentical(4, $HP->nextWhiteSpace("asdf\r"));
$this->assertIdentical(2, $HP->nextWhiteSpace("as\t\r\nasdf as"));
$this->assertIdentical(3, $HP->nextWhiteSpace('a a ', 2));
}
function test_parseData() { function test_parseData() {
$HP =& $this->DirectLex; $HP =& $this->DirectLex;
$this->assertIdentical('asdf', $HP->parseData('asdf')); $this->assertIdentical('asdf', $HP->parseData('asdf'));
$this->assertIdentical('&', $HP->parseData('&amp;')); $this->assertIdentical('&', $HP->parseData('&amp;'));
$this->assertIdentical('"', $HP->parseData('&quot;')); $this->assertIdentical('"', $HP->parseData('&quot;'));
$this->assertIdentical("'", $HP->parseData('&#039;')); $this->assertIdentical("'", $HP->parseData('&#039;'));
$this->assertIdentical('-', $HP->parseData('&#x2D;')); $this->assertIdentical("'", $HP->parseData('&#39;'));
// UTF-8 needed!!! $this->assertIdentical('&&&', $HP->parseData('&amp;&amp;&amp;'));
$this->assertIdentical('&&', $HP->parseData('&amp;&')); // [INVALID]
$this->assertIdentical('Procter & Gamble',
$HP->parseData('Procter & Gamble')); // [INVALID]
// This is not special, thus not converted. Test of fault tolerance,
// realistically speaking, this should never happen
$this->assertIdentical('&#x2D;', $HP->parseData('&#x2D;'));
}
function test_specialEntityCallback() {
$HP =& $this->DirectLex;
$this->assertIdentical("'",$HP->specialEntityCallback(
array('&#39;', null, '39', null) ));
} }
// internals testing // internals testing
function test_tokenizeAttributeString() { function test_parseAttributeString() {
$input[0] = 'href="asdf" boom="assdf"'; $input[0] = 'href="about:blank" rel="nofollow"';
$expect[0] = array('href'=>'asdf', 'boom'=>'assdf'); $expect[0] = array('href'=>'about:blank', 'rel'=>'nofollow');
$input[1] = "href='r'"; $input[1] = "href='about:blank'";
$expect[1] = array('href'=>'r'); $expect[1] = array('href'=>'about:blank');
// note that the single quotes aren't /really/ escaped
$input[2] = 'onclick="javascript:alert(\'asdf\');"'; $input[2] = 'onclick="javascript:alert(\'asdf\');"';
$expect[2] = array('onclick' => "javascript:alert('asdf');"); $expect[2] = array('onclick' => "javascript:alert('asdf');");
$input[3] = 'selected'; $input[3] = 'selected';
$expect[3] = array('selected'=>'selected'); $expect[3] = array('selected'=>'selected');
$input[4] = '="asdf"'; // [INVALID]
$input[4] = '="nokey"';
$expect[4] = array(); $expect[4] = array();
$input[5] = 'missile=launch'; // [SIMPLE]
$expect[5] = array('missile' => 'launch'); $input[5] = 'color=blue';
$expect[5] = array('color' => 'blue');
$input[6] = 'href="foo'; // [INVALID]
$expect[6] = array('href' => 'foo'); $input[6] = 'href="about:blank';
$expect[6] = array('href' => 'about:blank');
// [INVALID]
$input[7] = '"='; $input[7] = '"=';
$expect[7] = array('"' => ''); $expect[7] = array('"' => '');
// we ought to get array()
$input[8] = 'href ="about:blank"rel ="nofollow"'; $input[8] = 'href ="about:blank"rel ="nofollow"';
$expect[8] = array('href' => 'about:blank', 'rel' => 'nofollow'); $expect[8] = array('href' => 'about:blank', 'rel' => 'nofollow');
$input[9] = 'foo bar'; $input[9] = 'two bool';
$expect[9] = array('foo' => 'foo', 'bar' => 'bar'); $expect[9] = array('two' => 'two', 'bool' => 'bool');
$input[10] = 'foo="bar" blue'; $input[10] = 'name="input" selected';
$expect[10] = array('foo' => 'bar', 'blue' => 'blue'); $expect[10] = array('name' => 'input', 'selected' => 'selected');
$size = count($input); $size = count($input);
for($i = 0; $i < $size; $i++) { for($i = 0; $i < $size; $i++) {
$result = $this->DirectLex->tokenizeAttributeString($input[$i]); $result = $this->DirectLex->parseAttributeString($input[$i]);
$this->assertEqual($expect[$i], $result, 'Test ' . $i . ': %s'); $this->assertEqual($expect[$i], $result, 'Test ' . $i . ': %s');
paintIf($result, $expect[$i] != $result); paintIf($result, $expect[$i] != $result);
} }