Include html5lib for BeautifulSoup

BeautifulSoup needs lxml or html5, have included html5lib. Also latest BeautifulSoup 4.1.3
2026-05-02 17:59:28 +01:00 · 2012-09-06 10:47:07 +12:00
parent 7aac60cecd
commit b8c5782765
136 changed files with 87265 additions and 2428 deletions
--- a/html5lib/tests/tokenizertotree.py
+++ b/html5lib/tests/tokenizertotree.py
@@ -0,0 +1,64 @@
+import sys
+import os
+import json
+import re
+
+import html5lib
+import support
+import test_parser
+import test_tokenizer
+
+p = html5lib.HTMLParser()
+
+unnamespaceExpected = re.compile(r"^(\|\s*)<html ([^>]+)>", re.M).sub
+
+def main(out_path):
+    if not os.path.exists(out_path):
+        sys.stderr.write("Path %s does not exist"%out_path)
+        sys.exit(1)
+
+    for filename in support.html5lib_test_files('tokenizer', '*.test'):
+        run_file(filename, out_path)
+
+def run_file(filename, out_path):
+    try:
+        tests_data = json.load(file(filename))
+    except ValueError:
+        sys.stderr.write("Failed to load %s\n"%filename)
+        return
+    name = os.path.splitext(os.path.split(filename)[1])[0]
+    output_file = open(os.path.join(out_path, "tokenizer_%s.dat"%name), "w")
+
+    if 'tests' in tests_data:
+        for test_data in tests_data['tests']:
+            if 'initialStates' not in test_data:
+                test_data["initialStates"] = ["Data state"]
+                
+            for initial_state in test_data["initialStates"]:
+                if initial_state != "Data state":
+                    #don't support this yet
+                    continue
+                test = make_test(test_data)
+                output_file.write(test)
+
+    output_file.close()
+
+def make_test(test_data):
+    if 'doubleEscaped' in test_data:
+        test_data = test_tokenizer.unescape_test(test_data)
+
+    rv = []
+    rv.append("#data")
+    rv.append(test_data["input"].encode("utf8"))
+    rv.append("#errors")
+    tree = p.parse(test_data["input"])
+    output = p.tree.testSerializer(tree)
+    output  = "\n".join(("| "+ line[3:]) if line.startswith("|  ") else line
+                        for line in output.split("\n"))
+    output = unnamespaceExpected(r"\1<\2>", output)
+    rv.append(output.encode("utf8"))
+    rv.append("")
+    return "\n".join(rv)
+
+if __name__ == "__main__":
+    main(sys.argv[1])