[3.1.1] Implement %URI.SecureMunge and %URI.SecureMungeSecretKey, thanks Chris!

- URIFilter->prepare can return false in order to abort loading of the filter
- Implemented post URI filtering. Set member variable $post to true to set a URIFilter as such.

git-svn-id: http://htmlpurifier.org/svnroot/htmlpurifier/trunk@1772 48356398-32a2-884e-a903-53898d9a118a

NEWS

@@ -12,6 +12,11 @@ NEWS ( CHANGELOG and HISTORY )                                     HTMLPurifier
 3.1.1, unknown release date
 ! More robust imagecrash protection with height/width CSS with %CSS.MaxImgLength,
   and height/width HTML with %HTML.MaxImgLength.
+! %URI.SecureMunge for secure URI munging (as opposed to %URI.Munge). Be sure
+  to set %URI.SecureMungeSecretKey when using this directive. Thanks Chris
+  for sponsoring this feature.
+! Implemented post URI filtering. Set member variable $post to true to set
+  a URIFilter as such.
 - Disable percent height/width attributes for img
 - AttrValidator operations are now atomic; updates to attributes are not
   manifest in token until end of operations. This prevents naughty internal
@@ -52,6 +57,7 @@ NEWS ( CHANGELOG and HISTORY )                                     HTMLPurifier
   the Printers for HTML Purifier related tasks.
 . HTML/CSS Printers must be primed with prepareGenerator($gen_config), otherwise
   fatal errors will ensue.
+. URIFilter->prepare can return false in order to abort loading of the filter
 
 3.1.0, released 2008-05-18
 # Unnecessary references to objects (vestiges of PHP4) removed from method

configdoc/usage.xml

@@ -195,7 +195,7 @@
  </directive>
  <directive id="URI.Host">
   <file name="HTMLPurifier/URIDefinition.php">
-   <line>57</line>
+   <line>63</line>
   </file>
   <file name="HTMLPurifier/URIFilter/DisableExternal.php">
    <line>8</line>
@@ -203,12 +203,12 @@
  </directive>
  <directive id="URI.Base">
   <file name="HTMLPurifier/URIDefinition.php">
-   <line>58</line>
+   <line>64</line>
   </file>
  </directive>
  <directive id="URI.DefaultScheme">
   <file name="HTMLPurifier/URIDefinition.php">
-   <line>65</line>
+   <line>71</line>
   </file>
  </directive>
  <directive id="URI.AllowedSchemes">
@@ -228,7 +228,7 @@
  </directive>
  <directive id="URI.Munge">
   <file name="HTMLPurifier/AttrDef/URI.php">
-   <line>68</line>
+   <line>72</line>
   </file>
  </directive>
  <directive id="Core.ColorKeywords">
@@ -371,4 +371,14 @@
    <line>8</line>
   </file>
  </directive>
+ <directive id="URI.SecureMunge">
+  <file name="HTMLPurifier/URIFilter/SecureMunge.php">
+   <line>9</line>
+  </file>
+ </directive>
+ <directive id="URI.SecureMungeSecretKey">
+  <file name="HTMLPurifier/URIFilter/SecureMunge.php">
+   <line>10</line>
+  </file>
+ </directive>
 </usage>

docs/enduser-uri-filter.html

@@ -130,30 +130,26 @@
 </p>
 
 <p>
-  Let's suppose I wanted to write a filter that de-internationalized domain
-  names by converting them to <a href="http://en.wikipedia.org/wiki/Punycode">Punycode</a>.
-  Assuming that <code>punycode_encode($input)</code> converts <code>$input</code> to
-  Punycode and returns <code>false</code> on failure:
+  Let's suppose I wanted to write a filter that converted links with a
+  custom <code>image</code> scheme to its corresponding real path on
+  our website:
 </p>
 
-<pre>class HTMLPurifier_URIFilter_ConvertIDNToPunycode extends HTMLPurifier_URIFilter
+<pre>class HTMLPurifier_URIFilter_TransformImageScheme extends HTMLPurifier_URIFilter
 {
-    public $name = 'ConvertIDNToPunycode';
+    public $name = 'TransformImageScheme';
     public function filter(&$uri, $config, $context) {
-        if (is_null($uri->host)) return true;
-        if ($uri->host == utf8_decode($uri->host)) {
-            // is ASCII, abort
-            return true;
-        }
-        $host = punycode_encode($uri->host);
-        if ($host === false) return false;
-        $uri->host = $host;
+        if ($uri->scheme !== 'image') return true;
+        $img_name = $uri->path;
+        // Overwrite the previous URI object
+        $uri = new HTMLPurifier_URI('http', null, null, null, '/img/' . $img_name . '.png', null, null);
         return true;
     }
 }</pre>
 
 <p>
-  Notice I did not <code>return $uri;</code>.
+  Notice I did not <code>return $uri;</code>. This filter would turn
+  <code>image:Foo</code> into <code>/img/Foo.png</code>.
 </p>
 
 <h2>Activating your filter</h2>
@@ -186,6 +182,25 @@ $uri->registerFilter(new HTMLPurifier_URIFilter_<strong>NameOfFilter</strong>())
   is set to true.
 </p>
 
+<h2>Post-filter</h2>
+
+<p>
+    Remember our TransformImageScheme filter? That filter acted before we had
+    performed scheme validation; otherwise, the URI would have been filtered
+    out when it was discovered that there was no image scheme. Well, a post-filter
+    is run after scheme specific validation, so it's ideal for bulk
+    post-processing of URIs, including munging. To specify a URI as a post-filter,
+    set the <code>$post</code> member variable to TRUE.
+</p>
+
+<pre>class HTMLPurifier_URIFilter_MyPostFilter extends HTMLPurifier_URIFilter
+{
+    public $name = 'MyPostFilter';
+    public $post = true;
+    // ... extra code here
+}
+</pre>
+
 <h2>Examples</h2>
 
 <p>

library/HTMLPurifier.includes.php

@@ -178,6 +178,7 @@ require 'HTMLPurifier/URIFilter/DisableExternal.php';
 require 'HTMLPurifier/URIFilter/DisableExternalResources.php';
 require 'HTMLPurifier/URIFilter/HostBlacklist.php';
 require 'HTMLPurifier/URIFilter/MakeAbsolute.php';
+require 'HTMLPurifier/URIFilter/SecureMunge.php';
 require 'HTMLPurifier/URIScheme/ftp.php';
 require 'HTMLPurifier/URIScheme/http.php';
 require 'HTMLPurifier/URIScheme/https.php';

library/HTMLPurifier.safe-includes.php

@@ -172,6 +172,7 @@ require_once $__dir . '/HTMLPurifier/URIFilter/DisableExternal.php';
 require_once $__dir . '/HTMLPurifier/URIFilter/DisableExternalResources.php';
 require_once $__dir . '/HTMLPurifier/URIFilter/HostBlacklist.php';
 require_once $__dir . '/HTMLPurifier/URIFilter/MakeAbsolute.php';
+require_once $__dir . '/HTMLPurifier/URIFilter/SecureMunge.php';
 require_once $__dir . '/HTMLPurifier/URIScheme/ftp.php';
 require_once $__dir . '/HTMLPurifier/URIScheme/http.php';
 require_once $__dir . '/HTMLPurifier/URIScheme/https.php';

library/HTMLPurifier/AttrDef/URI.php

@@ -50,6 +50,10 @@ class HTMLPurifier_AttrDef_URI extends HTMLPurifier_AttrDef
             $result = $scheme_obj->validate($uri, $config, $context);
             if (!$result) break;
             
+            // Post chained filtering
+            $result = $uri_def->postFilter($uri, $config, $context);
+            if (!$result) break;
+            
             // survived gauntlet
             $ok = true;
             

library/HTMLPurifier/ConfigSchema/schema/URI.SecureMunge.txt

@@ -0,0 +1,33 @@
+URI.SecureMunge
+TYPE: string/null
+VERSION: 3.1.1
+DEFAULT: NULL
+--DESCRIPTION--
+<p>
+    Like %URI.Munge, this directive munges browsable external resources
+    into another URI redirection service. %URI.SecureMunge accepts a URI
+    with a %s located where the original URI should be substituted in,
+    and %t located where the secure checksum should be provided.
+    However, this directive affords
+    an additional level of protection by generating a secure checksum from
+    the URI as well as a secret key provided by %URI.SecureMungeSecretKey.
+    Any redirector script can check this key by using:
+</p>
+
+<pre>$checksum === sha1($secret_key . ':' . $url)</pre>
+
+<p>
+    If the output is TRUE, the redirector script should accept the URI.
+</p>
+
+<p>
+    Please note that it would still be possible for an attacker to procure
+    secure hashes en-mass by abusing your website's Preview feature or the
+    like, but this service affords an additional level of protection
+    that should be combined with website blacklisting.
+</p>
+
+<p>
+    <strong>This is a post-filter.</strong> This filter may conflict with other
+    post-filters that deal with external links.
+</p>

library/HTMLPurifier/ConfigSchema/schema/URI.SecureMungeSecretKey.txt

@@ -0,0 +1,11 @@
+URI.SecureMungeSecretKey
+TYPE: string/null
+VERSION: 3.1.1
+DEFAULT: NULL
+--DESCRIPTION--
+<p>
+    This is the secret key used in conjunction with %URI.SecureMunge. Your
+    redirector script needs to know about this key, and no one else should
+    know about this key. Please see the above
+    directive for more details.
+</p>

library/HTMLPurifier/URIDefinition.php

@@ -5,6 +5,7 @@ class HTMLPurifier_URIDefinition extends HTMLPurifier_Definition
     
     public $type = 'URI';
     protected $filters = array();
+    protected $postFilters = array();
     protected $registeredFilters = array();
     
     /**
@@ -34,9 +35,14 @@ class HTMLPurifier_URIDefinition extends HTMLPurifier_Definition
     }
     
     public function addFilter($filter, $config) {
-        $filter->prepare($config);
+        $r = $filter->prepare($config);
+        if ($r === false) return; // null is ok, for backwards compat
+        if ($filter->post) {
+            $this->postFilters[$filter->name] = $filter;
+        } else {
             $this->filters[$filter->name] = $filter;
         }
+    }
     
     protected function doSetup($config) {
         $this->setupMemberVariables($config);
@@ -66,8 +72,16 @@ class HTMLPurifier_URIDefinition extends HTMLPurifier_Definition
     }
     
     public function filter(&$uri, $config, $context) {
-        foreach ($this->filters as $name => $x) {
-            $result = $this->filters[$name]->filter($uri, $config, $context);
+        foreach ($this->filters as $name => $f) {
+            $result = $f->filter($uri, $config, $context);
+            if (!$result) return false;
+        }
+        return true;
+    }
+    
+    public function postFilter(&$uri, $config, $context) {
+        foreach ($this->postFilters as $name => $f) {
+            $result = $f->filter($uri, $config, $context);
             if (!$result) return false;
         }
         return true;

library/HTMLPurifier/URIFilter.php

@@ -19,10 +19,15 @@ abstract class HTMLPurifier_URIFilter
      */
     public $name;
     
+    /**
+     * True if this filter should be run after scheme validation.
+     */
+    public $post = false;
+    
     /**
      * Performs initialization for the filter
      */
-    public function prepare($config) {}
+    public function prepare($config) {return true;}
     
     /**
      * Filter a URI object

library/HTMLPurifier/URIFilter/HostBlacklist.php

@@ -6,6 +6,7 @@ class HTMLPurifier_URIFilter_HostBlacklist extends HTMLPurifier_URIFilter
     protected $blacklist = array();
     public function prepare($config) {
         $this->blacklist = $config->get('URI', 'HostBlacklist');
+        return true;
     }
     public function filter(&$uri, $config, $context) {
         foreach($this->blacklist as $blacklisted_host_fragment) {

library/HTMLPurifier/URIFilter/MakeAbsolute.php

@@ -11,14 +11,15 @@ class HTMLPurifier_URIFilter_MakeAbsolute extends HTMLPurifier_URIFilter
         $def = $config->getDefinition('URI');
         $this->base = $def->base;
         if (is_null($this->base)) {
-            trigger_error('URI.MakeAbsolute is being ignored due to lack of value for URI.Base configuration', E_USER_ERROR);
-            return;
+            trigger_error('URI.MakeAbsolute is being ignored due to lack of value for URI.Base configuration', E_USER_WARNING);
+            return false;
         }
         $this->base->fragment = null; // fragment is invalid for base URI
         $stack = explode('/', $this->base->path);
         array_pop($stack); // discard last segment
         $stack = $this->_collapseStack($stack); // do pre-parsing
         $this->basePathStack = $stack;
+        return true;
     }
     public function filter(&$uri, $config, $context) {
         if (is_null($this->base)) return true; // abort early

library/HTMLPurifier/URIFilter/SecureMunge.php

@@ -0,0 +1,32 @@
+<?php
+
+class HTMLPurifier_URIFilter_SecureMunge extends HTMLPurifier_URIFilter
+{
+    public $name = 'SecureMunge';
+    public $post = true;
+    private $target, $secretKey, $parser;
+    public function prepare($config) {
+        $this->target = $config->get('URI', 'SecureMunge');
+        $this->secretKey = $config->get('URI', 'SecureMungeSecretKey');
+        $this->parser = new HTMLPurifier_URIParser();
+        if (!$this->secretKey) {
+            trigger_error('URI.SecureMunge is being ignored due to lack of value for URI.SecureMungeSecretKey', E_USER_WARNING);
+            return false;
+        }
+        return true;
+    }
+    public function filter(&$uri, $config, $context) {
+        if (!$this->target || !$this->secretKey) return true;
+        $scheme_obj = $uri->getSchemeObj($config, $context);
+        if (!$scheme_obj) return true; // ignore unknown schemes, maybe another postfilter did it
+        if (is_null($uri->host) || empty($scheme_obj->browsable)) {
+            return true;
+        }
+        $string = $uri->toString();
+        $checksum = sha1($this->secretKey . ':' . $string);
+        $new_uri = str_replace('%s', rawurlencode($string), $this->target);
+        $new_uri = str_replace('%t', $checksum, $new_uri);
+        $uri = $this->parser->parse($new_uri); // overwrite
+        return true;
+    }
+}

tests/HTMLPurifier/AttrDef/URITest.php

@@ -83,6 +83,8 @@ class HTMLPurifier_AttrDef_URITest extends HTMLPurifier_AttrDefHarness
         $uri_def = new HTMLPurifier_URIDefinitionMock();
         $uri_def->expectOnce('filter', array($uri, '*', '*'));
         $uri_def->setReturnValue('filter', true, array($uri, '*', '*'));
+        $uri_def->expectOnce('postFilter', array($uri, '*', '*'));
+        $uri_def->setReturnValue('postFilter', true, array($uri, '*', '*'));
         $uri_def->setup = true;
         
         // Since definitions are no longer passed by reference, we need

tests/HTMLPurifier/URIDefinitionTest.php

@@ -3,14 +3,16 @@
 class HTMLPurifier_URIDefinitionTest extends HTMLPurifier_URIHarness
 {
     
-    protected function createFilterMock($expect = true, $result = true) {
+    protected function createFilterMock($expect = true, $result = true, $post = false, $setup = true) {
         static $i = 0;
         generate_mock_once('HTMLPurifier_URIFilter');
         $mock = new HTMLPurifier_URIFilterMock();
         if ($expect) $mock->expectOnce('filter');
         else $mock->expectNever('filter');
         $mock->setReturnValue('filter', $result);
+        $mock->setReturnValue('prepare', $setup);
         $mock->name = $i++;
+        $mock->post = $post;
         return $mock;
     }
     

tests/HTMLPurifier/URIFilter/SecureMungeTest.php

@@ -0,0 +1,49 @@
+<?php
+
+class HTMLPurifier_URIFilter_SecureMungeTest extends HTMLPurifier_URIFilterHarness
+{
+    
+    function setUp() {
+        parent::setUp();
+        $this->filter = new HTMLPurifier_URIFilter_SecureMunge();
+        $this->setSecureMunge();
+        $this->setSecretKey();
+    }
+    
+    function setSecureMunge($uri = '/redirect.php?url=%s&checksum=%t') {
+        $this->config->set('URI', 'SecureMunge', $uri);
+    }
+    
+    function setSecretKey($key = 'secret') {
+        $this->config->set('URI', 'SecureMungeSecretKey', $key);
+    }
+    
+    function testPreserve() {
+        $this->assertFiltering('/local');
+    }
+    
+    function testStandardMunge() {
+        $this->assertFiltering('http://google.com', '/redirect.php?url=http%3A%2F%2Fgoogle.com&checksum=0072e2f817fd2844825def74e54443debecf0892');
+    }
+    
+    function testIgnoreUnknownSchemes() {
+        // This should be integration tested as well to be false
+        $this->assertFiltering('javascript:', true);
+    }
+    
+    function testIgnoreUnbrowsableSchemes() {
+        $this->assertFiltering('news:', true);
+    }
+    
+    function testMungeToDirectory() {
+        $this->setSecureMunge('/links/%s/%t');
+        $this->assertFiltering('http://google.com', '/links/http%3A%2F%2Fgoogle.com/0072e2f817fd2844825def74e54443debecf0892');
+    }
+    
+    function testErrorNoSecretKey() {
+        $this->setSecretKey(null);
+        $this->expectError('URI.SecureMunge is being ignored due to lack of value for URI.SecureMungeSecretKey');
+        $this->assertFiltering('http://google.com');
+    }
+    
+}