Include html5lib for BeautifulSoup

BeautifulSoup needs lxml or html5, have included html5lib.

Also latest BeautifulSoup 4.1.3

html5lib/tests/test_encoding.py

@@ -0,0 +1,54 @@
+import os
+import unittest
+from support import html5lib_test_files, TestData, test_dir
+
+from html5lib import HTMLParser, inputstream
+
+import re, unittest
+
+class Html5EncodingTestCase(unittest.TestCase):
+    def test_codec_name(self):
+        self.assertEquals(inputstream.codecName("utf-8"), "utf-8")
+        self.assertEquals(inputstream.codecName("utf8"), "utf-8")
+        self.assertEquals(inputstream.codecName("  utf8  "), "utf-8")
+        self.assertEquals(inputstream.codecName("ISO_8859--1"), "windows-1252")
+
+def buildTestSuite():
+    for filename in html5lib_test_files("encoding"):
+        test_name = os.path.basename(filename).replace('.dat',''). \
+            replace('-','')
+        tests = TestData(filename, "data")
+        for idx, test in enumerate(tests):
+            def encodingTest(self, data=test['data'], 
+                             encoding=test['encoding']):
+                p = HTMLParser()
+                t = p.parse(data, useChardet=False)
+                
+                errorMessage = ("Input:\n%s\nExpected:\n%s\nRecieved\n%s\n"%
+                                (data, repr(encoding.lower()), 
+                                 repr(p.tokenizer.stream.charEncoding)))
+                self.assertEquals(encoding.lower(),
+                                  p.tokenizer.stream.charEncoding[0], 
+                                  errorMessage)
+            setattr(Html5EncodingTestCase, 'test_%s_%d' % (test_name, idx+1),
+                encodingTest)
+
+    try:
+        import chardet
+        def test_chardet(self):
+            data = open(os.path.join(test_dir, "encoding" , "chardet", "test_big5.txt")).read()
+            encoding = inputstream.HTMLInputStream(data).charEncoding
+            assert encoding[0].lower() == "big5"
+        setattr(Html5EncodingTestCase, 'test_chardet', test_chardet)
+    except ImportError:
+        print "chardet not found, skipping chardet tests"
+        
+
+    return unittest.defaultTestLoader.loadTestsFromName(__name__)
+
+def main():
+    buildTestSuite()
+    unittest.main()
+
+if __name__ == "__main__":
+    main()