Merged in a lot of bugfixes and improvements from doskir, whatcd and rutracker from cohena, specify cache dir & size, brink skin updates, better searching fix for new artists

.gitignore

@@ -1,7 +1,10 @@
+
 # Compiled source #
 ###################
 *.pyc
 *.py~
+*.pyproj
+*.sln
 
 # Logs and databases #
 ######################
@@ -19,3 +22,32 @@ cache/*
 ehthumbs.db
 Icon?
 Thumbs.db
+
+#ignore thumbnails created by windows
+Thumbs.db
+#Ignore files build by Visual Studio
+*.obj
+*.exe
+*.pdb
+*.user
+*.aps
+*.pch
+*.vspscc
+*_i.c
+*_p.c
+*.ncb
+*.suo
+*.tlb
+*.tlh
+*.bak
+*.cache
+*.ilk
+*.log
+[Bb]in
+[Dd]ebug*/
+*.lib
+*.sbr
+obj/
+[Rr]elease*/
+_ReSharper*/
+[Tt]est[Rr]esult*

Headphones.py

@@ -109,6 +109,9 @@ def main():
         
     if headphones.DAEMON:
         headphones.daemonize()
+    
+    #configure the connection to the musicbrainz database
+    headphones.mb.startmb()
 
     # Force the http port if neccessary
     if args.port:

bs4/__init__.py

@@ -0,0 +1,359 @@
+"""Beautiful Soup
+Elixir and Tonic
+"The Screen-Scraper's Friend"
+http://www.crummy.com/software/BeautifulSoup/
+
+Beautiful Soup uses a pluggable XML or HTML parser to parse a
+(possibly invalid) document into a tree representation. Beautiful Soup
+provides provides methods and Pythonic idioms that make it easy to
+navigate, search, and modify the parse tree.
+
+Beautiful Soup works with Python 2.6 and up. It works better if lxml
+and/or html5lib is installed.
+
+For more than you ever wanted to know about Beautiful Soup, see the
+documentation:
+http://www.crummy.com/software/BeautifulSoup/bs4/doc/
+"""
+
+__author__ = "Leonard Richardson (leonardr@segfault.org)"
+__version__ = "4.1.3"
+__copyright__ = "Copyright (c) 2004-2012 Leonard Richardson"
+__license__ = "MIT"
+
+__all__ = ['BeautifulSoup']
+
+import re
+import warnings
+
+from .builder import builder_registry
+from .dammit import UnicodeDammit
+from .element import (
+    CData,
+    Comment,
+    DEFAULT_OUTPUT_ENCODING,
+    Declaration,
+    Doctype,
+    NavigableString,
+    PageElement,
+    ProcessingInstruction,
+    ResultSet,
+    SoupStrainer,
+    Tag,
+    )
+
+# The very first thing we do is give a useful error if someone is
+# running this code under Python 3 without converting it.
+syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
+
+class BeautifulSoup(Tag):
+    """
+    This class defines the basic interface called by the tree builders.
+
+    These methods will be called by the parser:
+      reset()
+      feed(markup)
+
+    The tree builder may call these methods from its feed() implementation:
+      handle_starttag(name, attrs) # See note about return value
+      handle_endtag(name)
+      handle_data(data) # Appends to the current data node
+      endData(containerClass=NavigableString) # Ends the current data node
+
+    No matter how complicated the underlying parser is, you should be
+    able to build a tree using 'start tag' events, 'end tag' events,
+    'data' events, and "done with data" events.
+
+    If you encounter an empty-element tag (aka a self-closing tag,
+    like HTML's <br> tag), call handle_starttag and then
+    handle_endtag.
+    """
+    ROOT_TAG_NAME = u'[document]'
+
+    # If the end-user gives no indication which tree builder they
+    # want, look for one with these features.
+    DEFAULT_BUILDER_FEATURES = ['html', 'fast']
+
+    # Used when determining whether a text node is all whitespace and
+    # can be replaced with a single space. A text node that contains
+    # fancy Unicode spaces (usually non-breaking) should be left
+    # alone.
+    STRIP_ASCII_SPACES = {9: None, 10: None, 12: None, 13: None, 32: None, }
+
+    def __init__(self, markup="", features=None, builder=None,
+                 parse_only=None, from_encoding=None, **kwargs):
+        """The Soup object is initialized as the 'root tag', and the
+        provided markup (which can be a string or a file-like object)
+        is fed into the underlying parser."""
+
+        if 'convertEntities' in kwargs:
+            warnings.warn(
+                "BS4 does not respect the convertEntities argument to the "
+                "BeautifulSoup constructor. Entities are always converted "
+                "to Unicode characters.")
+
+        if 'markupMassage' in kwargs:
+            del kwargs['markupMassage']
+            warnings.warn(
+                "BS4 does not respect the markupMassage argument to the "
+                "BeautifulSoup constructor. The tree builder is responsible "
+                "for any necessary markup massage.")
+
+        if 'smartQuotesTo' in kwargs:
+            del kwargs['smartQuotesTo']
+            warnings.warn(
+                "BS4 does not respect the smartQuotesTo argument to the "
+                "BeautifulSoup constructor. Smart quotes are always converted "
+                "to Unicode characters.")
+
+        if 'selfClosingTags' in kwargs:
+            del kwargs['selfClosingTags']
+            warnings.warn(
+                "BS4 does not respect the selfClosingTags argument to the "
+                "BeautifulSoup constructor. The tree builder is responsible "
+                "for understanding self-closing tags.")
+
+        if 'isHTML' in kwargs:
+            del kwargs['isHTML']
+            warnings.warn(
+                "BS4 does not respect the isHTML argument to the "
+                "BeautifulSoup constructor. You can pass in features='html' "
+                "or features='xml' to get a builder capable of handling "
+                "one or the other.")
+
+        def deprecated_argument(old_name, new_name):
+            if old_name in kwargs:
+                warnings.warn(
+                    'The "%s" argument to the BeautifulSoup constructor '
+                    'has been renamed to "%s."' % (old_name, new_name))
+                value = kwargs[old_name]
+                del kwargs[old_name]
+                return value
+            return None
+
+        parse_only = parse_only or deprecated_argument(
+            "parseOnlyThese", "parse_only")
+
+        from_encoding = from_encoding or deprecated_argument(
+            "fromEncoding", "from_encoding")
+
+        if len(kwargs) > 0:
+            arg = kwargs.keys().pop()
+            raise TypeError(
+                "__init__() got an unexpected keyword argument '%s'" % arg)
+
+        if builder is None:
+            if isinstance(features, basestring):
+                features = [features]
+            if features is None or len(features) == 0:
+                features = self.DEFAULT_BUILDER_FEATURES
+            builder_class = builder_registry.lookup(*features)
+            if builder_class is None:
+                raise FeatureNotFound(
+                    "Couldn't find a tree builder with the features you "
+                    "requested: %s. Do you need to install a parser library?"
+                    % ",".join(features))
+            builder = builder_class()
+        self.builder = builder
+        self.is_xml = builder.is_xml
+        self.builder.soup = self
+
+        self.parse_only = parse_only
+
+        self.reset()
+
+        if hasattr(markup, 'read'):        # It's a file-type object.
+            markup = markup.read()
+        (self.markup, self.original_encoding, self.declared_html_encoding,
+         self.contains_replacement_characters) = (
+            self.builder.prepare_markup(markup, from_encoding))
+
+        try:
+            self._feed()
+        except StopParsing:
+            pass
+
+        # Clear out the markup and remove the builder's circular
+        # reference to this object.
+        self.markup = None
+        self.builder.soup = None
+
+    def _feed(self):
+        # Convert the document to Unicode.
+        self.builder.reset()
+
+        self.builder.feed(self.markup)
+        # Close out any unfinished strings and close all the open tags.
+        self.endData()
+        while self.currentTag.name != self.ROOT_TAG_NAME:
+            self.popTag()
+
+    def reset(self):
+        Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
+        self.hidden = 1
+        self.builder.reset()
+        self.currentData = []
+        self.currentTag = None
+        self.tagStack = []
+        self.pushTag(self)
+
+    def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
+        """Create a new tag associated with this soup."""
+        return Tag(None, self.builder, name, namespace, nsprefix, attrs)
+
+    def new_string(self, s):
+        """Create a new NavigableString associated with this soup."""
+        navigable = NavigableString(s)
+        navigable.setup()
+        return navigable
+
+    def insert_before(self, successor):
+        raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
+
+    def insert_after(self, successor):
+        raise NotImplementedError("BeautifulSoup objects don't support insert_after().")
+
+    def popTag(self):
+        tag = self.tagStack.pop()
+        #print "Pop", tag.name
+        if self.tagStack:
+            self.currentTag = self.tagStack[-1]
+        return self.currentTag
+
+    def pushTag(self, tag):
+        #print "Push", tag.name
+        if self.currentTag:
+            self.currentTag.contents.append(tag)
+        self.tagStack.append(tag)
+        self.currentTag = self.tagStack[-1]
+
+    def endData(self, containerClass=NavigableString):
+        if self.currentData:
+            currentData = u''.join(self.currentData)
+            if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and
+                not set([tag.name for tag in self.tagStack]).intersection(
+                    self.builder.preserve_whitespace_tags)):
+                if '\n' in currentData:
+                    currentData = '\n'
+                else:
+                    currentData = ' '
+            self.currentData = []
+            if self.parse_only and len(self.tagStack) <= 1 and \
+                   (not self.parse_only.text or \
+                    not self.parse_only.search(currentData)):
+                return
+            o = containerClass(currentData)
+            self.object_was_parsed(o)
+
+    def object_was_parsed(self, o):
+        """Add an object to the parse tree."""
+        o.setup(self.currentTag, self.previous_element)
+        if self.previous_element:
+            self.previous_element.next_element = o
+        self.previous_element = o
+        self.currentTag.contents.append(o)
+
+    def _popToTag(self, name, nsprefix=None, inclusivePop=True):
+        """Pops the tag stack up to and including the most recent
+        instance of the given tag. If inclusivePop is false, pops the tag
+        stack up to but *not* including the most recent instqance of
+        the given tag."""
+        #print "Popping to %s" % name
+        if name == self.ROOT_TAG_NAME:
+            return
+
+        numPops = 0
+        mostRecentTag = None
+
+        for i in range(len(self.tagStack) - 1, 0, -1):
+            if (name == self.tagStack[i].name
+                and nsprefix == self.tagStack[i].prefix):
+                numPops = len(self.tagStack) - i
+                break
+        if not inclusivePop:
+            numPops = numPops - 1
+
+        for i in range(0, numPops):
+            mostRecentTag = self.popTag()
+        return mostRecentTag
+
+    def handle_starttag(self, name, namespace, nsprefix, attrs):
+        """Push a start tag on to the stack.
+
+        If this method returns None, the tag was rejected by the
+        SoupStrainer. You should proceed as if the tag had not occured
+        in the document. For instance, if this was a self-closing tag,
+        don't call handle_endtag.
+        """
+
+        # print "Start tag %s: %s" % (name, attrs)
+        self.endData()
+
+        if (self.parse_only and len(self.tagStack) <= 1
+            and (self.parse_only.text
+                 or not self.parse_only.search_tag(name, attrs))):
+            return None
+
+        tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
+                  self.currentTag, self.previous_element)
+        if tag is None:
+            return tag
+        if self.previous_element:
+            self.previous_element.next_element = tag
+        self.previous_element = tag
+        self.pushTag(tag)
+        return tag
+
+    def handle_endtag(self, name, nsprefix=None):
+        #print "End tag: " + name
+        self.endData()
+        self._popToTag(name, nsprefix)
+
+    def handle_data(self, data):
+        self.currentData.append(data)
+
+    def decode(self, pretty_print=False,
+               eventual_encoding=DEFAULT_OUTPUT_ENCODING,
+               formatter="minimal"):
+        """Returns a string or Unicode representation of this document.
+        To get Unicode, pass None for encoding."""
+
+        if self.is_xml:
+            # Print the XML declaration
+            encoding_part = ''
+            if eventual_encoding != None:
+                encoding_part = ' encoding="%s"' % eventual_encoding
+            prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
+        else:
+            prefix = u''
+        if not pretty_print:
+            indent_level = None
+        else:
+            indent_level = 0
+        return prefix + super(BeautifulSoup, self).decode(
+            indent_level, eventual_encoding, formatter)
+
+class BeautifulStoneSoup(BeautifulSoup):
+    """Deprecated interface to an XML parser."""
+
+    def __init__(self, *args, **kwargs):
+        kwargs['features'] = 'xml'
+        warnings.warn(
+            'The BeautifulStoneSoup class is deprecated. Instead of using '
+            'it, pass features="xml" into the BeautifulSoup constructor.')
+        super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
+
+
+class StopParsing(Exception):
+    pass
+
+
+class FeatureNotFound(ValueError):
+    pass
+
+
+#By default, act as an HTML pretty-printer.
+if __name__ == '__main__':
+    import sys
+    soup = BeautifulSoup(sys.stdin)
+    print soup.prettify()

bs4/builder/__init__.py

@@ -0,0 +1,316 @@
+from collections import defaultdict
+import itertools
+import sys
+from bs4.element import (
+    CharsetMetaAttributeValue,
+    ContentMetaAttributeValue,
+    whitespace_re
+    )
+
+__all__ = [
+    'HTMLTreeBuilder',
+    'SAXTreeBuilder',
+    'TreeBuilder',
+    'TreeBuilderRegistry',
+    ]
+
+# Some useful features for a TreeBuilder to have.
+FAST = 'fast'
+PERMISSIVE = 'permissive'
+STRICT = 'strict'
+XML = 'xml'
+HTML = 'html'
+HTML_5 = 'html5'
+
+
+class TreeBuilderRegistry(object):
+
+    def __init__(self):
+        self.builders_for_feature = defaultdict(list)
+        self.builders = []
+
+    def register(self, treebuilder_class):
+        """Register a treebuilder based on its advertised features."""
+        for feature in treebuilder_class.features:
+            self.builders_for_feature[feature].insert(0, treebuilder_class)
+        self.builders.insert(0, treebuilder_class)
+
+    def lookup(self, *features):
+        if len(self.builders) == 0:
+            # There are no builders at all.
+            return None
+
+        if len(features) == 0:
+            # They didn't ask for any features. Give them the most
+            # recently registered builder.
+            return self.builders[0]
+
+        # Go down the list of features in order, and eliminate any builders
+        # that don't match every feature.
+        features = list(features)
+        features.reverse()
+        candidates = None
+        candidate_set = None
+        while len(features) > 0:
+            feature = features.pop()
+            we_have_the_feature = self.builders_for_feature.get(feature, [])
+            if len(we_have_the_feature) > 0:
+                if candidates is None:
+                    candidates = we_have_the_feature
+                    candidate_set = set(candidates)
+                else:
+                    # Eliminate any candidates that don't have this feature.
+                    candidate_set = candidate_set.intersection(
+                        set(we_have_the_feature))
+
+        # The only valid candidates are the ones in candidate_set.
+        # Go through the original list of candidates and pick the first one
+        # that's in candidate_set.
+        if candidate_set is None:
+            return None
+        for candidate in candidates:
+            if candidate in candidate_set:
+                return candidate
+        return None
+
+# The BeautifulSoup class will take feature lists from developers and use them
+# to look up builders in this registry.
+builder_registry = TreeBuilderRegistry()
+
+class TreeBuilder(object):
+    """Turn a document into a Beautiful Soup object tree."""
+
+    features = []
+
+    is_xml = False
+    preserve_whitespace_tags = set()
+    empty_element_tags = None # A tag will be considered an empty-element
+                              # tag when and only when it has no contents.
+
+    # A value for these tag/attribute combinations is a space- or
+    # comma-separated list of CDATA, rather than a single CDATA.
+    cdata_list_attributes = {}
+
+
+    def __init__(self):
+        self.soup = None
+
+    def reset(self):
+        pass
+
+    def can_be_empty_element(self, tag_name):
+        """Might a tag with this name be an empty-element tag?
+
+        The final markup may or may not actually present this tag as
+        self-closing.
+
+        For instance: an HTMLBuilder does not consider a <p> tag to be
+        an empty-element tag (it's not in
+        HTMLBuilder.empty_element_tags). This means an empty <p> tag
+        will be presented as "<p></p>", not "<p />".
+
+        The default implementation has no opinion about which tags are
+        empty-element tags, so a tag will be presented as an
+        empty-element tag if and only if it has no contents.
+        "<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will
+        be left alone.
+        """
+        if self.empty_element_tags is None:
+            return True
+        return tag_name in self.empty_element_tags
+
+    def feed(self, markup):
+        raise NotImplementedError()
+
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        return markup, None, None, False
+
+    def test_fragment_to_document(self, fragment):
+        """Wrap an HTML fragment to make it look like a document.
+
+        Different parsers do this differently. For instance, lxml
+        introduces an empty <head> tag, and html5lib
+        doesn't. Abstracting this away lets us write simple tests
+        which run HTML fragments through the parser and compare the
+        results against other HTML fragments.
+
+        This method should not be used outside of tests.
+        """
+        return fragment
+
+    def set_up_substitutions(self, tag):
+        return False
+
+    def _replace_cdata_list_attribute_values(self, tag_name, attrs):
+        """Replaces class="foo bar" with class=["foo", "bar"]
+
+        Modifies its input in place.
+        """
+        if self.cdata_list_attributes:
+            universal = self.cdata_list_attributes.get('*', [])
+            tag_specific = self.cdata_list_attributes.get(
+                tag_name.lower(), [])
+            for cdata_list_attr in itertools.chain(universal, tag_specific):
+                if cdata_list_attr in dict(attrs):
+                    # Basically, we have a "class" attribute whose
+                    # value is a whitespace-separated list of CSS
+                    # classes. Split it into a list.
+                    value = attrs[cdata_list_attr]
+                    if isinstance(value, basestring):
+                        values = whitespace_re.split(value)
+                    else:
+                        # html5lib sometimes calls setAttributes twice
+                        # for the same tag when rearranging the parse
+                        # tree. On the second call the attribute value
+                        # here is already a list.  If this happens,
+                        # leave the value alone rather than trying to
+                        # split it again.
+                        values = value
+                    attrs[cdata_list_attr] = values
+        return attrs
+
+class SAXTreeBuilder(TreeBuilder):
+    """A Beautiful Soup treebuilder that listens for SAX events."""
+
+    def feed(self, markup):
+        raise NotImplementedError()
+
+    def close(self):
+        pass
+
+    def startElement(self, name, attrs):
+        attrs = dict((key[1], value) for key, value in list(attrs.items()))
+        #print "Start %s, %r" % (name, attrs)
+        self.soup.handle_starttag(name, attrs)
+
+    def endElement(self, name):
+        #print "End %s" % name
+        self.soup.handle_endtag(name)
+
+    def startElementNS(self, nsTuple, nodeName, attrs):
+        # Throw away (ns, nodeName) for now.
+        self.startElement(nodeName, attrs)
+
+    def endElementNS(self, nsTuple, nodeName):
+        # Throw away (ns, nodeName) for now.
+        self.endElement(nodeName)
+        #handler.endElementNS((ns, node.nodeName), node.nodeName)
+
+    def startPrefixMapping(self, prefix, nodeValue):
+        # Ignore the prefix for now.
+        pass
+
+    def endPrefixMapping(self, prefix):
+        # Ignore the prefix for now.
+        # handler.endPrefixMapping(prefix)
+        pass
+
+    def characters(self, content):
+        self.soup.handle_data(content)
+
+    def startDocument(self):
+        pass
+
+    def endDocument(self):
+        pass
+
+
+class HTMLTreeBuilder(TreeBuilder):
+    """This TreeBuilder knows facts about HTML.
+
+    Such as which tags are empty-element tags.
+    """
+
+    preserve_whitespace_tags = set(['pre', 'textarea'])
+    empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
+                              'spacer', 'link', 'frame', 'base'])
+
+    # The HTML standard defines these attributes as containing a
+    # space-separated list of values, not a single value. That is,
+    # class="foo bar" means that the 'class' attribute has two values,
+    # 'foo' and 'bar', not the single value 'foo bar'.  When we
+    # encounter one of these attributes, we will parse its value into
+    # a list of values if possible. Upon output, the list will be
+    # converted back into a string.
+    cdata_list_attributes = {
+        "*" : ['class', 'accesskey', 'dropzone'],
+        "a" : ['rel', 'rev'],
+        "link" :  ['rel', 'rev'],
+        "td" : ["headers"],
+        "th" : ["headers"],
+        "td" : ["headers"],
+        "form" : ["accept-charset"],
+        "object" : ["archive"],
+
+        # These are HTML5 specific, as are *.accesskey and *.dropzone above.
+        "area" : ["rel"],
+        "icon" : ["sizes"],
+        "iframe" : ["sandbox"],
+        "output" : ["for"],
+        }
+
+    def set_up_substitutions(self, tag):
+        # We are only interested in <meta> tags
+        if tag.name != 'meta':
+            return False
+
+        http_equiv = tag.get('http-equiv')
+        content = tag.get('content')
+        charset = tag.get('charset')
+
+        # We are interested in <meta> tags that say what encoding the
+        # document was originally in. This means HTML 5-style <meta>
+        # tags that provide the "charset" attribute. It also means
+        # HTML 4-style <meta> tags that provide the "content"
+        # attribute and have "http-equiv" set to "content-type".
+        #
+        # In both cases we will replace the value of the appropriate
+        # attribute with a standin object that can take on any
+        # encoding.
+        meta_encoding = None
+        if charset is not None:
+            # HTML 5 style:
+            # <meta charset="utf8">
+            meta_encoding = charset
+            tag['charset'] = CharsetMetaAttributeValue(charset)
+
+        elif (content is not None and http_equiv is not None
+              and http_equiv.lower() == 'content-type'):
+            # HTML 4 style:
+            # <meta http-equiv="content-type" content="text/html; charset=utf8">
+            tag['content'] = ContentMetaAttributeValue(content)
+
+        return (meta_encoding is not None)
+
+def register_treebuilders_from(module):
+    """Copy TreeBuilders from the given module into this module."""
+    # I'm fairly sure this is not the best way to do this.
+    this_module = sys.modules['bs4.builder']
+    for name in module.__all__:
+        obj = getattr(module, name)
+
+        if issubclass(obj, TreeBuilder):
+            setattr(this_module, name, obj)
+            this_module.__all__.append(name)
+            # Register the builder while we're at it.
+            this_module.builder_registry.register(obj)
+
+# Builders are registered in reverse order of priority, so that custom
+# builder registrations will take precedence. In general, we want lxml
+# to take precedence over html5lib, because it's faster. And we only
+# want to use HTMLParser as a last result.
+from . import _htmlparser
+register_treebuilders_from(_htmlparser)
+try:
+    from . import _html5lib
+    register_treebuilders_from(_html5lib)
+except ImportError:
+    # They don't have html5lib installed.
+    pass
+try:
+    from . import _lxml
+    register_treebuilders_from(_lxml)
+except ImportError:
+    # They don't have lxml installed.
+    pass

bs4/builder/_html5lib.py

@@ -0,0 +1,222 @@
+__all__ = [
+    'HTML5TreeBuilder',
+    ]
+
+import warnings
+from bs4.builder import (
+    PERMISSIVE,
+    HTML,
+    HTML_5,
+    HTMLTreeBuilder,
+    )
+from bs4.element import NamespacedAttribute
+import html5lib
+from html5lib.constants import namespaces
+from bs4.element import (
+    Comment,
+    Doctype,
+    NavigableString,
+    Tag,
+    )
+
+class HTML5TreeBuilder(HTMLTreeBuilder):
+    """Use html5lib to build a tree."""
+
+    features = ['html5lib', PERMISSIVE, HTML_5, HTML]
+
+    def prepare_markup(self, markup, user_specified_encoding):
+        # Store the user-specified encoding for use later on.
+        self.user_specified_encoding = user_specified_encoding
+        return markup, None, None, False
+
+    # These methods are defined by Beautiful Soup.
+    def feed(self, markup):
+        if self.soup.parse_only is not None:
+            warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
+        parser = html5lib.HTMLParser(tree=self.create_treebuilder)
+        doc = parser.parse(markup, encoding=self.user_specified_encoding)
+
+        # Set the character encoding detected by the tokenizer.
+        if isinstance(markup, unicode):
+            # We need to special-case this because html5lib sets
+            # charEncoding to UTF-8 if it gets Unicode input.
+            doc.original_encoding = None
+        else:
+            doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
+
+    def create_treebuilder(self, namespaceHTMLElements):
+        self.underlying_builder = TreeBuilderForHtml5lib(
+            self.soup, namespaceHTMLElements)
+        return self.underlying_builder
+
+    def test_fragment_to_document(self, fragment):
+        """See `TreeBuilder`."""
+        return u'<html><head></head><body>%s</body></html>' % fragment
+
+
+class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
+
+    def __init__(self, soup, namespaceHTMLElements):
+        self.soup = soup
+        super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
+
+    def documentClass(self):
+        self.soup.reset()
+        return Element(self.soup, self.soup, None)
+
+    def insertDoctype(self, token):
+        name = token["name"]
+        publicId = token["publicId"]
+        systemId = token["systemId"]
+
+        doctype = Doctype.for_name_and_ids(name, publicId, systemId)
+        self.soup.object_was_parsed(doctype)
+
+    def elementClass(self, name, namespace):
+        tag = self.soup.new_tag(name, namespace)
+        return Element(tag, self.soup, namespace)
+
+    def commentClass(self, data):
+        return TextNode(Comment(data), self.soup)
+
+    def fragmentClass(self):
+        self.soup = BeautifulSoup("")
+        self.soup.name = "[document_fragment]"
+        return Element(self.soup, self.soup, None)
+
+    def appendChild(self, node):
+        # XXX This code is not covered by the BS4 tests.
+        self.soup.append(node.element)
+
+    def getDocument(self):
+        return self.soup
+
+    def getFragment(self):
+        return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
+
+class AttrList(object):
+    def __init__(self, element):
+        self.element = element
+        self.attrs = dict(self.element.attrs)
+    def __iter__(self):
+        return list(self.attrs.items()).__iter__()
+    def __setitem__(self, name, value):
+        "set attr", name, value
+        self.element[name] = value
+    def items(self):
+        return list(self.attrs.items())
+    def keys(self):
+        return list(self.attrs.keys())
+    def __len__(self):
+        return len(self.attrs)
+    def __getitem__(self, name):
+        return self.attrs[name]
+    def __contains__(self, name):
+        return name in list(self.attrs.keys())
+
+
+class Element(html5lib.treebuilders._base.Node):
+    def __init__(self, element, soup, namespace):
+        html5lib.treebuilders._base.Node.__init__(self, element.name)
+        self.element = element
+        self.soup = soup
+        self.namespace = namespace
+
+    def appendChild(self, node):
+        if (node.element.__class__ == NavigableString and self.element.contents
+            and self.element.contents[-1].__class__ == NavigableString):
+            # Concatenate new text onto old text node
+            # XXX This has O(n^2) performance, for input like
+            # "a</a>a</a>a</a>..."
+            old_element = self.element.contents[-1]
+            new_element = self.soup.new_string(old_element + node.element)
+            old_element.replace_with(new_element)
+        else:
+            self.element.append(node.element)
+            node.parent = self
+
+    def getAttributes(self):
+        return AttrList(self.element)
+
+    def setAttributes(self, attributes):
+        if attributes is not None and len(attributes) > 0:
+
+            converted_attributes = []
+            for name, value in list(attributes.items()):
+                if isinstance(name, tuple):
+                    new_name = NamespacedAttribute(*name)
+                    del attributes[name]
+                    attributes[new_name] = value
+
+            self.soup.builder._replace_cdata_list_attribute_values(
+                self.name, attributes)
+            for name, value in attributes.items():
+                self.element[name] = value
+
+            # The attributes may contain variables that need substitution.
+            # Call set_up_substitutions manually.
+            #
+            # The Tag constructor called this method when the Tag was created,
+            # but we just set/changed the attributes, so call it again.
+            self.soup.builder.set_up_substitutions(self.element)
+    attributes = property(getAttributes, setAttributes)
+
+    def insertText(self, data, insertBefore=None):
+        text = TextNode(self.soup.new_string(data), self.soup)
+        if insertBefore:
+            self.insertBefore(text, insertBefore)
+        else:
+            self.appendChild(text)
+
+    def insertBefore(self, node, refNode):
+        index = self.element.index(refNode.element)
+        if (node.element.__class__ == NavigableString and self.element.contents
+            and self.element.contents[index-1].__class__ == NavigableString):
+            # (See comments in appendChild)
+            old_node = self.element.contents[index-1]
+            new_str = self.soup.new_string(old_node + node.element)
+            old_node.replace_with(new_str)
+        else:
+            self.element.insert(index, node.element)
+            node.parent = self
+
+    def removeChild(self, node):
+        node.element.extract()
+
+    def reparentChildren(self, newParent):
+        while self.element.contents:
+            child = self.element.contents[0]
+            child.extract()
+            if isinstance(child, Tag):
+                newParent.appendChild(
+                    Element(child, self.soup, namespaces["html"]))
+            else:
+                newParent.appendChild(
+                    TextNode(child, self.soup))
+
+    def cloneNode(self):
+        tag = self.soup.new_tag(self.element.name, self.namespace)
+        node = Element(tag, self.soup, self.namespace)
+        for key,value in self.attributes:
+            node.attributes[key] = value
+        return node
+
+    def hasContent(self):
+        return self.element.contents
+
+    def getNameTuple(self):
+        if self.namespace == None:
+            return namespaces["html"], self.name
+        else:
+            return self.namespace, self.name
+
+    nameTuple = property(getNameTuple)
+
+class TextNode(Element):
+    def __init__(self, element, soup):
+        html5lib.treebuilders._base.Node.__init__(self, None)
+        self.element = element
+        self.soup = soup
+
+    def cloneNode(self):
+        raise NotImplementedError

bs4/builder/_htmlparser.py

@@ -0,0 +1,244 @@
+"""Use the HTMLParser library to parse HTML files that aren't too bad."""
+
+__all__ = [
+    'HTMLParserTreeBuilder',
+    ]
+
+from HTMLParser import (
+    HTMLParser,
+    HTMLParseError,
+    )
+import sys
+import warnings
+
+# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
+# argument, which we'd like to set to False. Unfortunately,
+# http://bugs.python.org/issue13273 makes strict=True a better bet
+# before Python 3.2.3.
+#
+# At the end of this file, we monkeypatch HTMLParser so that
+# strict=True works well on Python 3.2.2.
+major, minor, release = sys.version_info[:3]
+CONSTRUCTOR_TAKES_STRICT = (
+    major > 3
+    or (major == 3 and minor > 2)
+    or (major == 3 and minor == 2 and release >= 3))
+
+from bs4.element import (
+    CData,
+    Comment,
+    Declaration,
+    Doctype,
+    ProcessingInstruction,
+    )
+from bs4.dammit import EntitySubstitution, UnicodeDammit
+
+from bs4.builder import (
+    HTML,
+    HTMLTreeBuilder,
+    STRICT,
+    )
+
+
+HTMLPARSER = 'html.parser'
+
+class BeautifulSoupHTMLParser(HTMLParser):
+    def handle_starttag(self, name, attrs):
+        # XXX namespace
+        self.soup.handle_starttag(name, None, None, dict(attrs))
+
+    def handle_endtag(self, name):
+        self.soup.handle_endtag(name)
+
+    def handle_data(self, data):
+        self.soup.handle_data(data)
+
+    def handle_charref(self, name):
+        # XXX workaround for a bug in HTMLParser. Remove this once
+        # it's fixed.
+        if name.startswith('x'):
+            real_name = int(name.lstrip('x'), 16)
+        else:
+            real_name = int(name)
+
+        try:
+            data = unichr(real_name)
+        except (ValueError, OverflowError), e:
+            data = u"\N{REPLACEMENT CHARACTER}"
+
+        self.handle_data(data)
+
+    def handle_entityref(self, name):
+        character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
+        if character is not None:
+            data = character
+        else:
+            data = "&%s;" % name
+        self.handle_data(data)
+
+    def handle_comment(self, data):
+        self.soup.endData()
+        self.soup.handle_data(data)
+        self.soup.endData(Comment)
+
+    def handle_decl(self, data):
+        self.soup.endData()
+        if data.startswith("DOCTYPE "):
+            data = data[len("DOCTYPE "):]
+        self.soup.handle_data(data)
+        self.soup.endData(Doctype)
+
+    def unknown_decl(self, data):
+        if data.upper().startswith('CDATA['):
+            cls = CData
+            data = data[len('CDATA['):]
+        else:
+            cls = Declaration
+        self.soup.endData()
+        self.soup.handle_data(data)
+        self.soup.endData(cls)
+
+    def handle_pi(self, data):
+        self.soup.endData()
+        if data.endswith("?") and data.lower().startswith("xml"):
+            # "An XHTML processing instruction using the trailing '?'
+            # will cause the '?' to be included in data." - HTMLParser
+            # docs.
+            #
+            # Strip the question mark so we don't end up with two
+            # question marks.
+            data = data[:-1]
+        self.soup.handle_data(data)
+        self.soup.endData(ProcessingInstruction)
+
+
+class HTMLParserTreeBuilder(HTMLTreeBuilder):
+
+    is_xml = False
+    features = [HTML, STRICT, HTMLPARSER]
+
+    def __init__(self, *args, **kwargs):
+        if CONSTRUCTOR_TAKES_STRICT:
+            kwargs['strict'] = False
+        self.parser_args = (args, kwargs)
+
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        """
+        :return: A 4-tuple (markup, original encoding, encoding
+        declared within markup, whether any characters had to be
+        replaced with REPLACEMENT CHARACTER).
+        """
+        if isinstance(markup, unicode):
+            return markup, None, None, False
+
+        try_encodings = [user_specified_encoding, document_declared_encoding]
+        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
+        return (dammit.markup, dammit.original_encoding,
+                dammit.declared_html_encoding,
+                dammit.contains_replacement_characters)
+
+    def feed(self, markup):
+        args, kwargs = self.parser_args
+        parser = BeautifulSoupHTMLParser(*args, **kwargs)
+        parser.soup = self.soup
+        try:
+            parser.feed(markup)
+        except HTMLParseError, e:
+            warnings.warn(RuntimeWarning(
+                "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
+            raise e
+
+# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
+# 3.2.3 code. This ensures they don't treat markup like <p></p> as a
+# string.
+#
+# XXX This code can be removed once most Python 3 users are on 3.2.3.
+if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
+    import re
+    attrfind_tolerant = re.compile(
+        r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
+        r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
+    HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant
+
+    locatestarttagend = re.compile(r"""
+  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
+  (?:\s+                             # whitespace before attribute name
+    (?:[a-zA-Z_][-.:a-zA-Z0-9_]*     # attribute name
+      (?:\s*=\s*                     # value indicator
+        (?:'[^']*'                   # LITA-enclosed value
+          |\"[^\"]*\"                # LIT-enclosed value
+          |[^'\">\s]+                # bare value
+         )
+       )?
+     )
+   )*
+  \s*                                # trailing whitespace
+""", re.VERBOSE)
+    BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
+
+    from html.parser import tagfind, attrfind
+
+    def parse_starttag(self, i):
+        self.__starttag_text = None
+        endpos = self.check_for_whole_start_tag(i)
+        if endpos < 0:
+            return endpos
+        rawdata = self.rawdata
+        self.__starttag_text = rawdata[i:endpos]
+
+        # Now parse the data between i+1 and j into a tag and attrs
+        attrs = []
+        match = tagfind.match(rawdata, i+1)
+        assert match, 'unexpected call to parse_starttag()'
+        k = match.end()
+        self.lasttag = tag = rawdata[i+1:k].lower()
+        while k < endpos:
+            if self.strict:
+                m = attrfind.match(rawdata, k)
+            else:
+                m = attrfind_tolerant.match(rawdata, k)
+            if not m:
+                break
+            attrname, rest, attrvalue = m.group(1, 2, 3)
+            if not rest:
+                attrvalue = None
+            elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
+                 attrvalue[:1] == '"' == attrvalue[-1:]:
+                attrvalue = attrvalue[1:-1]
+            if attrvalue:
+                attrvalue = self.unescape(attrvalue)
+            attrs.append((attrname.lower(), attrvalue))
+            k = m.end()
+
+        end = rawdata[k:endpos].strip()
+        if end not in (">", "/>"):
+            lineno, offset = self.getpos()
+            if "\n" in self.__starttag_text:
+                lineno = lineno + self.__starttag_text.count("\n")
+                offset = len(self.__starttag_text) \
+                         - self.__starttag_text.rfind("\n")
+            else:
+                offset = offset + len(self.__starttag_text)
+            if self.strict:
+                self.error("junk characters in start tag: %r"
+                           % (rawdata[k:endpos][:20],))
+            self.handle_data(rawdata[i:endpos])
+            return endpos
+        if end.endswith('/>'):
+            # XHTML-style empty tag: <span attr="value" />
+            self.handle_startendtag(tag, attrs)
+        else:
+            self.handle_starttag(tag, attrs)
+            if tag in self.CDATA_CONTENT_ELEMENTS:
+                self.set_cdata_mode(tag)
+        return endpos
+
+    def set_cdata_mode(self, elem):
+        self.cdata_elem = elem.lower()
+        self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
+
+    BeautifulSoupHTMLParser.parse_starttag = parse_starttag
+    BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
+
+    CONSTRUCTOR_TAKES_STRICT = True

bs4/builder/_lxml.py

@@ -0,0 +1,199 @@
+__all__ = [
+    'LXMLTreeBuilderForXML',
+    'LXMLTreeBuilder',
+    ]
+
+from StringIO import StringIO
+import collections
+from lxml import etree
+from bs4.element import Comment, Doctype, NamespacedAttribute
+from bs4.builder import (
+    FAST,
+    HTML,
+    HTMLTreeBuilder,
+    PERMISSIVE,
+    TreeBuilder,
+    XML)
+from bs4.dammit import UnicodeDammit
+
+LXML = 'lxml'
+
+class LXMLTreeBuilderForXML(TreeBuilder):
+    DEFAULT_PARSER_CLASS = etree.XMLParser
+
+    is_xml = True
+
+    # Well, it's permissive by XML parser standards.
+    features = [LXML, XML, FAST, PERMISSIVE]
+
+    CHUNK_SIZE = 512
+
+    @property
+    def default_parser(self):
+        # This can either return a parser object or a class, which
+        # will be instantiated with default arguments.
+        return etree.XMLParser(target=self, strip_cdata=False, recover=True)
+
+    def __init__(self, parser=None, empty_element_tags=None):
+        if empty_element_tags is not None:
+            self.empty_element_tags = set(empty_element_tags)
+        if parser is None:
+            # Use the default parser.
+            parser = self.default_parser
+        if isinstance(parser, collections.Callable):
+            # Instantiate the parser with default arguments
+            parser = parser(target=self, strip_cdata=False)
+        self.parser = parser
+        self.soup = None
+        self.nsmaps = None
+
+    def _getNsTag(self, tag):
+        # Split the namespace URL out of a fully-qualified lxml tag
+        # name. Copied from lxml's src/lxml/sax.py.
+        if tag[0] == '{':
+            return tuple(tag[1:].split('}', 1))
+        else:
+            return (None, tag)
+
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        """
+        :return: A 3-tuple (markup, original encoding, encoding
+        declared within markup).
+        """
+        if isinstance(markup, unicode):
+            return markup, None, None, False
+
+        try_encodings = [user_specified_encoding, document_declared_encoding]
+        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
+        return (dammit.markup, dammit.original_encoding,
+                dammit.declared_html_encoding,
+                dammit.contains_replacement_characters)
+
+    def feed(self, markup):
+        if isinstance(markup, basestring):
+            markup = StringIO(markup)
+        # Call feed() at least once, even if the markup is empty,
+        # or the parser won't be initialized.
+        data = markup.read(self.CHUNK_SIZE)
+        self.parser.feed(data)
+        while data != '':
+            # Now call feed() on the rest of the data, chunk by chunk.
+            data = markup.read(self.CHUNK_SIZE)
+            if data != '':
+                self.parser.feed(data)
+        self.parser.close()
+
+    def close(self):
+        self.nsmaps = None
+
+    def start(self, name, attrs, nsmap={}):
+        # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
+        attrs = dict(attrs)
+
+        nsprefix = None
+        # Invert each namespace map as it comes in.
+        if len(nsmap) == 0 and self.nsmaps != None:
+            # There are no new namespaces for this tag, but namespaces
+            # are in play, so we need a separate tag stack to know
+            # when they end.
+            self.nsmaps.append(None)
+        elif len(nsmap) > 0:
+            # A new namespace mapping has come into play.
+            if self.nsmaps is None:
+                self.nsmaps = []
+            inverted_nsmap = dict((value, key) for key, value in nsmap.items())
+            self.nsmaps.append(inverted_nsmap)
+            # Also treat the namespace mapping as a set of attributes on the
+            # tag, so we can recreate it later.
+            attrs = attrs.copy()
+            for prefix, namespace in nsmap.items():
+                attribute = NamespacedAttribute(
+                    "xmlns", prefix, "http://www.w3.org/2000/xmlns/")
+                attrs[attribute] = namespace
+
+        if self.nsmaps is not None and len(self.nsmaps) > 0:
+            # Namespaces are in play. Find any attributes that came in
+            # from lxml with namespaces attached to their names, and
+            # turn then into NamespacedAttribute objects.
+            new_attrs = {}
+            for attr, value in attrs.items():
+                namespace, attr = self._getNsTag(attr)
+                if namespace is None:
+                    new_attrs[attr] = value
+                else:
+                    nsprefix = self._prefix_for_namespace(namespace)
+                    attr = NamespacedAttribute(nsprefix, attr, namespace)
+                    new_attrs[attr] = value
+            attrs = new_attrs
+
+        namespace, name = self._getNsTag(name)
+        nsprefix = self._prefix_for_namespace(namespace)
+        self.soup.handle_starttag(name, namespace, nsprefix, attrs)
+
+    def _prefix_for_namespace(self, namespace):
+        """Find the currently active prefix for the given namespace."""
+        if namespace is None:
+            return None
+        for inverted_nsmap in reversed(self.nsmaps):
+            if inverted_nsmap is not None and namespace in inverted_nsmap:
+                return inverted_nsmap[namespace]
+
+    def end(self, name):
+        self.soup.endData()
+        completed_tag = self.soup.tagStack[-1]
+        namespace, name = self._getNsTag(name)
+        nsprefix = None
+        if namespace is not None:
+            for inverted_nsmap in reversed(self.nsmaps):
+                if inverted_nsmap is not None and namespace in inverted_nsmap:
+                    nsprefix = inverted_nsmap[namespace]
+                    break
+        self.soup.handle_endtag(name, nsprefix)
+        if self.nsmaps != None:
+            # This tag, or one of its parents, introduced a namespace
+            # mapping, so pop it off the stack.
+            self.nsmaps.pop()
+            if len(self.nsmaps) == 0:
+                # Namespaces are no longer in play, so don't bother keeping
+                # track of the namespace stack.
+                self.nsmaps = None
+
+    def pi(self, target, data):
+        pass
+
+    def data(self, content):
+        self.soup.handle_data(content)
+
+    def doctype(self, name, pubid, system):
+        self.soup.endData()
+        doctype = Doctype.for_name_and_ids(name, pubid, system)
+        self.soup.object_was_parsed(doctype)
+
+    def comment(self, content):
+        "Handle comments as Comment objects."
+        self.soup.endData()
+        self.soup.handle_data(content)
+        self.soup.endData(Comment)
+
+    def test_fragment_to_document(self, fragment):
+        """See `TreeBuilder`."""
+        return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
+
+
+class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
+
+    features = [LXML, HTML, FAST, PERMISSIVE]
+    is_xml = False
+
+    @property
+    def default_parser(self):
+        return etree.HTMLParser
+
+    def feed(self, markup):
+        self.parser.feed(markup)
+        self.parser.close()
+
+    def test_fragment_to_document(self, fragment):
+        """See `TreeBuilder`."""
+        return u'<html><body>%s</body></html>' % fragment

bs4/dammit.py

@@ -0,0 +1,803 @@
+# -*- coding: utf-8 -*-
+"""Beautiful Soup bonus library: Unicode, Dammit
+
+This class forces XML data into a standard format (usually to UTF-8 or
+Unicode).  It is heavily based on code from Mark Pilgrim's Universal
+Feed Parser. It does not rewrite the XML or HTML to reflect a new
+encoding; that's the tree builder's job.
+"""
+
+import codecs
+from htmlentitydefs import codepoint2name
+import re
+import logging
+
+# Import a library to autodetect character encodings.
+chardet_type = None
+try:
+    # First try the fast C implementation.
+    #  PyPI package: cchardet
+    import cchardet
+    def chardet_dammit(s):
+        return cchardet.detect(s)['encoding']
+except ImportError:
+    try:
+        # Fall back to the pure Python implementation
+        #  Debian package: python-chardet
+        #  PyPI package: chardet
+        import chardet
+        def chardet_dammit(s):
+            return chardet.detect(s)['encoding']
+        #import chardet.constants
+        #chardet.constants._debug = 1
+    except ImportError:
+        # No chardet available.
+        def chardet_dammit(s):
+            return None
+
+# Available from http://cjkpython.i18n.org/.
+try:
+    import iconv_codec
+except ImportError:
+    pass
+
+xml_encoding_re = re.compile(
+    '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
+html_meta_re = re.compile(
+    '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
+
+class EntitySubstitution(object):
+
+    """Substitute XML or HTML entities for the corresponding characters."""
+
+    def _populate_class_variables():
+        lookup = {}
+        reverse_lookup = {}
+        characters_for_re = []
+        for codepoint, name in list(codepoint2name.items()):
+            character = unichr(codepoint)
+            if codepoint != 34:
+                # There's no point in turning the quotation mark into
+                # &quot;, unless it happens within an attribute value, which
+                # is handled elsewhere.
+                characters_for_re.append(character)
+                lookup[character] = name
+            # But we do want to turn &quot; into the quotation mark.
+            reverse_lookup[name] = character
+        re_definition = "[%s]" % "".join(characters_for_re)
+        return lookup, reverse_lookup, re.compile(re_definition)
+    (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
+     CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
+
+    CHARACTER_TO_XML_ENTITY = {
+        "'": "apos",
+        '"': "quot",
+        "&": "amp",
+        "<": "lt",
+        ">": "gt",
+        }
+
+    BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
+                                           "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
+                                           ")")
+
+    @classmethod
+    def _substitute_html_entity(cls, matchobj):
+        entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
+        return "&%s;" % entity
+
+    @classmethod
+    def _substitute_xml_entity(cls, matchobj):
+        """Used with a regular expression to substitute the
+        appropriate XML entity for an XML special character."""
+        entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
+        return "&%s;" % entity
+
+    @classmethod
+    def quoted_attribute_value(self, value):
+        """Make a value into a quoted XML attribute, possibly escaping it.
+
+         Most strings will be quoted using double quotes.
+
+          Bob's Bar -> "Bob's Bar"
+
+         If a string contains double quotes, it will be quoted using
+         single quotes.
+
+          Welcome to "my bar" -> 'Welcome to "my bar"'
+
+         If a string contains both single and double quotes, the
+         double quotes will be escaped, and the string will be quoted
+         using double quotes.
+
+          Welcome to "Bob's Bar" -> "Welcome to &quot;Bob's bar&quot;
+        """
+        quote_with = '"'
+        if '"' in value:
+            if "'" in value:
+                # The string contains both single and double
+                # quotes.  Turn the double quotes into
+                # entities. We quote the double quotes rather than
+                # the single quotes because the entity name is
+                # "&quot;" whether this is HTML or XML.  If we
+                # quoted the single quotes, we'd have to decide
+                # between &apos; and &squot;.
+                replace_with = "&quot;"
+                value = value.replace('"', replace_with)
+            else:
+                # There are double quotes but no single quotes.
+                # We can use single quotes to quote the attribute.
+                quote_with = "'"
+        return quote_with + value + quote_with
+
+    @classmethod
+    def substitute_xml(cls, value, make_quoted_attribute=False):
+        """Substitute XML entities for special XML characters.
+
+        :param value: A string to be substituted. The less-than sign will
+          become &lt;, the greater-than sign will become &gt;, and any
+          ampersands that are not part of an entity defition will
+          become &amp;.
+
+        :param make_quoted_attribute: If True, then the string will be
+         quoted, as befits an attribute value.
+        """
+        # Escape angle brackets, and ampersands that aren't part of
+        # entities.
+        value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
+            cls._substitute_xml_entity, value)
+
+        if make_quoted_attribute:
+            value = cls.quoted_attribute_value(value)
+        return value
+
+    @classmethod
+    def substitute_html(cls, s):
+        """Replace certain Unicode characters with named HTML entities.
+
+        This differs from data.encode(encoding, 'xmlcharrefreplace')
+        in that the goal is to make the result more readable (to those
+        with ASCII displays) rather than to recover from
+        errors. There's absolutely nothing wrong with a UTF-8 string
+        containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
+        character with "&eacute;" will make it more readable to some
+        people.
+        """
+        return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
+            cls._substitute_html_entity, s)
+
+
+class UnicodeDammit:
+    """A class for detecting the encoding of a *ML document and
+    converting it to a Unicode string. If the source encoding is
+    windows-1252, can replace MS smart quotes with their HTML or XML
+    equivalents."""
+
+    # This dictionary maps commonly seen values for "charset" in HTML
+    # meta tags to the corresponding Python codec names. It only covers
+    # values that aren't in Python's aliases and can't be determined
+    # by the heuristics in find_codec.
+    CHARSET_ALIASES = {"macintosh": "mac-roman",
+                       "x-sjis": "shift-jis"}
+
+    ENCODINGS_WITH_SMART_QUOTES = [
+        "windows-1252",
+        "iso-8859-1",
+        "iso-8859-2",
+        ]
+
+    def __init__(self, markup, override_encodings=[],
+                 smart_quotes_to=None, is_html=False):
+        self.declared_html_encoding = None
+        self.smart_quotes_to = smart_quotes_to
+        self.tried_encodings = []
+        self.contains_replacement_characters = False
+
+        if markup == '' or isinstance(markup, unicode):
+            self.markup = markup
+            self.unicode_markup = unicode(markup)
+            self.original_encoding = None
+            return
+
+        new_markup, document_encoding, sniffed_encoding = \
+            self._detectEncoding(markup, is_html)
+        self.markup = new_markup
+
+        u = None
+        if new_markup != markup:
+            # _detectEncoding modified the markup, then converted it to
+            # Unicode and then to UTF-8. So convert it from UTF-8.
+            u = self._convert_from("utf8")
+            self.original_encoding = sniffed_encoding
+
+        if not u:
+            for proposed_encoding in (
+                override_encodings + [document_encoding, sniffed_encoding]):
+                if proposed_encoding is not None:
+                    u = self._convert_from(proposed_encoding)
+                    if u:
+                        break
+
+        # If no luck and we have auto-detection library, try that:
+        if not u and not isinstance(self.markup, unicode):
+            u = self._convert_from(chardet_dammit(self.markup))
+
+        # As a last resort, try utf-8 and windows-1252:
+        if not u:
+            for proposed_encoding in ("utf-8", "windows-1252"):
+                u = self._convert_from(proposed_encoding)
+                if u:
+                    break
+
+        # As an absolute last resort, try the encodings again with
+        # character replacement.
+        if not u:
+            for proposed_encoding in (
+                override_encodings + [
+                    document_encoding, sniffed_encoding, "utf-8", "windows-1252"]):
+                if proposed_encoding != "ascii":
+                    u = self._convert_from(proposed_encoding, "replace")
+                if u is not None:
+                    logging.warning(
+                            "Some characters could not be decoded, and were "
+                            "replaced with REPLACEMENT CHARACTER.")
+                    self.contains_replacement_characters = True
+                    break
+
+        # We could at this point force it to ASCII, but that would
+        # destroy so much data that I think giving up is better
+        self.unicode_markup = u
+        if not u:
+            self.original_encoding = None
+
+    def _sub_ms_char(self, match):
+        """Changes a MS smart quote character to an XML or HTML
+        entity, or an ASCII character."""
+        orig = match.group(1)
+        if self.smart_quotes_to == 'ascii':
+            sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
+        else:
+            sub = self.MS_CHARS.get(orig)
+            if type(sub) == tuple:
+                if self.smart_quotes_to == 'xml':
+                    sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
+                else:
+                    sub = '&'.encode() + sub[0].encode() + ';'.encode()
+            else:
+                sub = sub.encode()
+        return sub
+
+    def _convert_from(self, proposed, errors="strict"):
+        proposed = self.find_codec(proposed)
+        if not proposed or (proposed, errors) in self.tried_encodings:
+            return None
+        self.tried_encodings.append((proposed, errors))
+        markup = self.markup
+
+        # Convert smart quotes to HTML if coming from an encoding
+        # that might have them.
+        if (self.smart_quotes_to is not None
+            and proposed.lower() in self.ENCODINGS_WITH_SMART_QUOTES):
+            smart_quotes_re = b"([\x80-\x9f])"
+            smart_quotes_compiled = re.compile(smart_quotes_re)
+            markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
+
+        try:
+            #print "Trying to convert document to %s (errors=%s)" % (
+            #    proposed, errors)
+            u = self._to_unicode(markup, proposed, errors)
+            self.markup = u
+            self.original_encoding = proposed
+        except Exception as e:
+            #print "That didn't work!"
+            #print e
+            return None
+        #print "Correct encoding: %s" % proposed
+        return self.markup
+
+    def _to_unicode(self, data, encoding, errors="strict"):
+        '''Given a string and its encoding, decodes the string into Unicode.
+        %encoding is a string recognized by encodings.aliases'''
+
+        # strip Byte Order Mark (if present)
+        if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
+               and (data[2:4] != '\x00\x00'):
+            encoding = 'utf-16be'
+            data = data[2:]
+        elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
+                 and (data[2:4] != '\x00\x00'):
+            encoding = 'utf-16le'
+            data = data[2:]
+        elif data[:3] == '\xef\xbb\xbf':
+            encoding = 'utf-8'
+            data = data[3:]
+        elif data[:4] == '\x00\x00\xfe\xff':
+            encoding = 'utf-32be'
+            data = data[4:]
+        elif data[:4] == '\xff\xfe\x00\x00':
+            encoding = 'utf-32le'
+            data = data[4:]
+        newdata = unicode(data, encoding, errors)
+        return newdata
+
+    def _detectEncoding(self, xml_data, is_html=False):
+        """Given a document, tries to detect its XML encoding."""
+        xml_encoding = sniffed_xml_encoding = None
+        try:
+            if xml_data[:4] == b'\x4c\x6f\xa7\x94':
+                # EBCDIC
+                xml_data = self._ebcdic_to_ascii(xml_data)
+            elif xml_data[:4] == b'\x00\x3c\x00\x3f':
+                # UTF-16BE
+                sniffed_xml_encoding = 'utf-16be'
+                xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
+            elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xfe\xff') \
+                     and (xml_data[2:4] != b'\x00\x00'):
+                # UTF-16BE with BOM
+                sniffed_xml_encoding = 'utf-16be'
+                xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
+            elif xml_data[:4] == b'\x3c\x00\x3f\x00':
+                # UTF-16LE
+                sniffed_xml_encoding = 'utf-16le'
+                xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
+            elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xff\xfe') and \
+                     (xml_data[2:4] != b'\x00\x00'):
+                # UTF-16LE with BOM
+                sniffed_xml_encoding = 'utf-16le'
+                xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
+            elif xml_data[:4] == b'\x00\x00\x00\x3c':
+                # UTF-32BE
+                sniffed_xml_encoding = 'utf-32be'
+                xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
+            elif xml_data[:4] == b'\x3c\x00\x00\x00':
+                # UTF-32LE
+                sniffed_xml_encoding = 'utf-32le'
+                xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
+            elif xml_data[:4] == b'\x00\x00\xfe\xff':
+                # UTF-32BE with BOM
+                sniffed_xml_encoding = 'utf-32be'
+                xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
+            elif xml_data[:4] == b'\xff\xfe\x00\x00':
+                # UTF-32LE with BOM
+                sniffed_xml_encoding = 'utf-32le'
+                xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
+            elif xml_data[:3] == b'\xef\xbb\xbf':
+                # UTF-8 with BOM
+                sniffed_xml_encoding = 'utf-8'
+                xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
+            else:
+                sniffed_xml_encoding = 'ascii'
+                pass
+        except:
+            xml_encoding_match = None
+        xml_encoding_match = xml_encoding_re.match(xml_data)
+        if not xml_encoding_match and is_html:
+            xml_encoding_match = html_meta_re.search(xml_data)
+        if xml_encoding_match is not None:
+            xml_encoding = xml_encoding_match.groups()[0].decode(
+                'ascii').lower()
+            if is_html:
+                self.declared_html_encoding = xml_encoding
+            if sniffed_xml_encoding and \
+               (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
+                                 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
+                                 'utf-16', 'utf-32', 'utf_16', 'utf_32',
+                                 'utf16', 'u16')):
+                xml_encoding = sniffed_xml_encoding
+        return xml_data, xml_encoding, sniffed_xml_encoding
+
+    def find_codec(self, charset):
+        return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
+               or (charset and self._codec(charset.replace("-", ""))) \
+               or (charset and self._codec(charset.replace("-", "_"))) \
+               or charset
+
+    def _codec(self, charset):
+        if not charset:
+            return charset
+        codec = None
+        try:
+            codecs.lookup(charset)
+            codec = charset
+        except (LookupError, ValueError):
+            pass
+        return codec
+
+    EBCDIC_TO_ASCII_MAP = None
+
+    def _ebcdic_to_ascii(self, s):
+        c = self.__class__
+        if not c.EBCDIC_TO_ASCII_MAP:
+            emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
+                    16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
+                    128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
+                    144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
+                    32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
+                    38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
+                    45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
+                    186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
+                    195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
+                    201,202,106,107,108,109,110,111,112,113,114,203,204,205,
+                    206,207,208,209,126,115,116,117,118,119,120,121,122,210,
+                    211,212,213,214,215,216,217,218,219,220,221,222,223,224,
+                    225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
+                    73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
+                    82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
+                    90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
+                    250,251,252,253,254,255)
+            import string
+            c.EBCDIC_TO_ASCII_MAP = string.maketrans(
+            ''.join(map(chr, list(range(256)))), ''.join(map(chr, emap)))
+        return s.translate(c.EBCDIC_TO_ASCII_MAP)
+
+    # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
+    MS_CHARS = {b'\x80': ('euro', '20AC'),
+                b'\x81': ' ',
+                b'\x82': ('sbquo', '201A'),
+                b'\x83': ('fnof', '192'),
+                b'\x84': ('bdquo', '201E'),
+                b'\x85': ('hellip', '2026'),
+                b'\x86': ('dagger', '2020'),
+                b'\x87': ('Dagger', '2021'),
+                b'\x88': ('circ', '2C6'),
+                b'\x89': ('permil', '2030'),
+                b'\x8A': ('Scaron', '160'),
+                b'\x8B': ('lsaquo', '2039'),
+                b'\x8C': ('OElig', '152'),
+                b'\x8D': '?',
+                b'\x8E': ('#x17D', '17D'),
+                b'\x8F': '?',
+                b'\x90': '?',
+                b'\x91': ('lsquo', '2018'),
+                b'\x92': ('rsquo', '2019'),
+                b'\x93': ('ldquo', '201C'),
+                b'\x94': ('rdquo', '201D'),
+                b'\x95': ('bull', '2022'),
+                b'\x96': ('ndash', '2013'),
+                b'\x97': ('mdash', '2014'),
+                b'\x98': ('tilde', '2DC'),
+                b'\x99': ('trade', '2122'),
+                b'\x9a': ('scaron', '161'),
+                b'\x9b': ('rsaquo', '203A'),
+                b'\x9c': ('oelig', '153'),
+                b'\x9d': '?',
+                b'\x9e': ('#x17E', '17E'),
+                b'\x9f': ('Yuml', ''),}
+
+    # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
+    # horrors like stripping diacritical marks to turn á into a, but also
+    # contains non-horrors like turning “ into ".
+    MS_CHARS_TO_ASCII = {
+        b'\x80' : 'EUR',
+        b'\x81' : ' ',
+        b'\x82' : ',',
+        b'\x83' : 'f',
+        b'\x84' : ',,',
+        b'\x85' : '...',
+        b'\x86' : '+',
+        b'\x87' : '++',
+        b'\x88' : '^',
+        b'\x89' : '%',
+        b'\x8a' : 'S',
+        b'\x8b' : '<',
+        b'\x8c' : 'OE',
+        b'\x8d' : '?',
+        b'\x8e' : 'Z',
+        b'\x8f' : '?',
+        b'\x90' : '?',
+        b'\x91' : "'",
+        b'\x92' : "'",
+        b'\x93' : '"',
+        b'\x94' : '"',
+        b'\x95' : '*',
+        b'\x96' : '-',
+        b'\x97' : '--',
+        b'\x98' : '~',
+        b'\x99' : '(TM)',
+        b'\x9a' : 's',
+        b'\x9b' : '>',
+        b'\x9c' : 'oe',
+        b'\x9d' : '?',
+        b'\x9e' : 'z',
+        b'\x9f' : 'Y',
+        b'\xa0' : ' ',
+        b'\xa1' : '!',
+        b'\xa2' : 'c',
+        b'\xa3' : 'GBP',
+        b'\xa4' : '$', #This approximation is especially parochial--this is the
+                       #generic currency symbol.
+        b'\xa5' : 'YEN',
+        b'\xa6' : '|',
+        b'\xa7' : 'S',
+        b'\xa8' : '..',
+        b'\xa9' : '',
+        b'\xaa' : '(th)',
+        b'\xab' : '<<',
+        b'\xac' : '!',
+        b'\xad' : ' ',
+        b'\xae' : '(R)',
+        b'\xaf' : '-',
+        b'\xb0' : 'o',
+        b'\xb1' : '+-',
+        b'\xb2' : '2',
+        b'\xb3' : '3',
+        b'\xb4' : ("'", 'acute'),
+        b'\xb5' : 'u',
+        b'\xb6' : 'P',
+        b'\xb7' : '*',
+        b'\xb8' : ',',
+        b'\xb9' : '1',
+        b'\xba' : '(th)',
+        b'\xbb' : '>>',
+        b'\xbc' : '1/4',
+        b'\xbd' : '1/2',
+        b'\xbe' : '3/4',
+        b'\xbf' : '?',
+        b'\xc0' : 'A',
+        b'\xc1' : 'A',
+        b'\xc2' : 'A',
+        b'\xc3' : 'A',
+        b'\xc4' : 'A',
+        b'\xc5' : 'A',
+        b'\xc6' : 'AE',
+        b'\xc7' : 'C',
+        b'\xc8' : 'E',
+        b'\xc9' : 'E',
+        b'\xca' : 'E',
+        b'\xcb' : 'E',
+        b'\xcc' : 'I',
+        b'\xcd' : 'I',
+        b'\xce' : 'I',
+        b'\xcf' : 'I',
+        b'\xd0' : 'D',
+        b'\xd1' : 'N',
+        b'\xd2' : 'O',
+        b'\xd3' : 'O',
+        b'\xd4' : 'O',
+        b'\xd5' : 'O',
+        b'\xd6' : 'O',
+        b'\xd7' : '*',
+        b'\xd8' : 'O',
+        b'\xd9' : 'U',
+        b'\xda' : 'U',
+        b'\xdb' : 'U',
+        b'\xdc' : 'U',
+        b'\xdd' : 'Y',
+        b'\xde' : 'b',
+        b'\xdf' : 'B',
+        b'\xe0' : 'a',
+        b'\xe1' : 'a',
+        b'\xe2' : 'a',
+        b'\xe3' : 'a',
+        b'\xe4' : 'a',
+        b'\xe5' : 'a',
+        b'\xe6' : 'ae',
+        b'\xe7' : 'c',
+        b'\xe8' : 'e',
+        b'\xe9' : 'e',
+        b'\xea' : 'e',
+        b'\xeb' : 'e',
+        b'\xec' : 'i',
+        b'\xed' : 'i',
+        b'\xee' : 'i',
+        b'\xef' : 'i',
+        b'\xf0' : 'o',
+        b'\xf1' : 'n',
+        b'\xf2' : 'o',
+        b'\xf3' : 'o',
+        b'\xf4' : 'o',
+        b'\xf5' : 'o',
+        b'\xf6' : 'o',
+        b'\xf7' : '/',
+        b'\xf8' : 'o',
+        b'\xf9' : 'u',
+        b'\xfa' : 'u',
+        b'\xfb' : 'u',
+        b'\xfc' : 'u',
+        b'\xfd' : 'y',
+        b'\xfe' : 'b',
+        b'\xff' : 'y',
+        }
+
+    # A map used when removing rogue Windows-1252/ISO-8859-1
+    # characters in otherwise UTF-8 documents.
+    #
+    # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
+    # Windows-1252.
+    WINDOWS_1252_TO_UTF8 = {
+        0x80 : b'\xe2\x82\xac', # €
+        0x82 : b'\xe2\x80\x9a', # ‚
+        0x83 : b'\xc6\x92',     # ƒ
+        0x84 : b'\xe2\x80\x9e', # „
+        0x85 : b'\xe2\x80\xa6', # …
+        0x86 : b'\xe2\x80\xa0', # †
+        0x87 : b'\xe2\x80\xa1', # ‡
+        0x88 : b'\xcb\x86',     # ˆ
+        0x89 : b'\xe2\x80\xb0', # ‰
+        0x8a : b'\xc5\xa0',     # Š
+        0x8b : b'\xe2\x80\xb9', # ‹
+        0x8c : b'\xc5\x92',     # Œ
+        0x8e : b'\xc5\xbd',     # Ž
+        0x91 : b'\xe2\x80\x98', # ‘
+        0x92 : b'\xe2\x80\x99', # ’
+        0x93 : b'\xe2\x80\x9c', # “
+        0x94 : b'\xe2\x80\x9d', # ”
+        0x95 : b'\xe2\x80\xa2', # •
+        0x96 : b'\xe2\x80\x93', # –
+        0x97 : b'\xe2\x80\x94', # —
+        0x98 : b'\xcb\x9c',     # ˜
+        0x99 : b'\xe2\x84\xa2', # ™
+        0x9a : b'\xc5\xa1',     # š
+        0x9b : b'\xe2\x80\xba', # ›
+        0x9c : b'\xc5\x93',     # œ
+        0x9e : b'\xc5\xbe',     # ž
+        0x9f : b'\xc5\xb8',     # Ÿ
+        0xa0 : b'\xc2\xa0',     #  
+        0xa1 : b'\xc2\xa1',     # ¡
+        0xa2 : b'\xc2\xa2',     # ¢
+        0xa3 : b'\xc2\xa3',     # £
+        0xa4 : b'\xc2\xa4',     # ¤
+        0xa5 : b'\xc2\xa5',     # ¥
+        0xa6 : b'\xc2\xa6',     # ¦
+        0xa7 : b'\xc2\xa7',     # §
+        0xa8 : b'\xc2\xa8',     # ¨
+        0xa9 : b'\xc2\xa9',     # ©
+        0xaa : b'\xc2\xaa',     # ª
+        0xab : b'\xc2\xab',     # «
+        0xac : b'\xc2\xac',     # ¬
+        0xad : b'\xc2\xad',     # ­
+        0xae : b'\xc2\xae',     # ®
+        0xaf : b'\xc2\xaf',     # ¯
+        0xb0 : b'\xc2\xb0',     # °
+        0xb1 : b'\xc2\xb1',     # ±
+        0xb2 : b'\xc2\xb2',     # ²
+        0xb3 : b'\xc2\xb3',     # ³
+        0xb4 : b'\xc2\xb4',     # ´
+        0xb5 : b'\xc2\xb5',     # µ
+        0xb6 : b'\xc2\xb6',     # ¶
+        0xb7 : b'\xc2\xb7',     # ·
+        0xb8 : b'\xc2\xb8',     # ¸
+        0xb9 : b'\xc2\xb9',     # ¹
+        0xba : b'\xc2\xba',     # º
+        0xbb : b'\xc2\xbb',     # »
+        0xbc : b'\xc2\xbc',     # ¼
+        0xbd : b'\xc2\xbd',     # ½
+        0xbe : b'\xc2\xbe',     # ¾
+        0xbf : b'\xc2\xbf',     # ¿
+        0xc0 : b'\xc3\x80',     # À
+        0xc1 : b'\xc3\x81',     # Á
+        0xc2 : b'\xc3\x82',     # Â
+        0xc3 : b'\xc3\x83',     # Ã
+        0xc4 : b'\xc3\x84',     # Ä
+        0xc5 : b'\xc3\x85',     # Å
+        0xc6 : b'\xc3\x86',     # Æ
+        0xc7 : b'\xc3\x87',     # Ç
+        0xc8 : b'\xc3\x88',     # È
+        0xc9 : b'\xc3\x89',     # É
+        0xca : b'\xc3\x8a',     # Ê
+        0xcb : b'\xc3\x8b',     # Ë
+        0xcc : b'\xc3\x8c',     # Ì
+        0xcd : b'\xc3\x8d',     # Í
+        0xce : b'\xc3\x8e',     # Î
+        0xcf : b'\xc3\x8f',     # Ï
+        0xd0 : b'\xc3\x90',     # Ð
+        0xd1 : b'\xc3\x91',     # Ñ
+        0xd2 : b'\xc3\x92',     # Ò
+        0xd3 : b'\xc3\x93',     # Ó
+        0xd4 : b'\xc3\x94',     # Ô
+        0xd5 : b'\xc3\x95',     # Õ
+        0xd6 : b'\xc3\x96',     # Ö
+        0xd7 : b'\xc3\x97',     # ×
+        0xd8 : b'\xc3\x98',     # Ø
+        0xd9 : b'\xc3\x99',     # Ù
+        0xda : b'\xc3\x9a',     # Ú
+        0xdb : b'\xc3\x9b',     # Û
+        0xdc : b'\xc3\x9c',     # Ü
+        0xdd : b'\xc3\x9d',     # Ý
+        0xde : b'\xc3\x9e',     # Þ
+        0xdf : b'\xc3\x9f',     # ß
+        0xe0 : b'\xc3\xa0',     # à
+        0xe1 : b'\xa1',     # á
+        0xe2 : b'\xc3\xa2',     # â
+        0xe3 : b'\xc3\xa3',     # ã
+        0xe4 : b'\xc3\xa4',     # ä
+        0xe5 : b'\xc3\xa5',     # å
+        0xe6 : b'\xc3\xa6',     # æ
+        0xe7 : b'\xc3\xa7',     # ç
+        0xe8 : b'\xc3\xa8',     # è
+        0xe9 : b'\xc3\xa9',     # é
+        0xea : b'\xc3\xaa',     # ê
+        0xeb : b'\xc3\xab',     # ë
+        0xec : b'\xc3\xac',     # ì
+        0xed : b'\xc3\xad',     # í
+        0xee : b'\xc3\xae',     # î
+        0xef : b'\xc3\xaf',     # ï
+        0xf0 : b'\xc3\xb0',     # ð
+        0xf1 : b'\xc3\xb1',     # ñ
+        0xf2 : b'\xc3\xb2',     # ò
+        0xf3 : b'\xc3\xb3',     # ó
+        0xf4 : b'\xc3\xb4',     # ô
+        0xf5 : b'\xc3\xb5',     # õ
+        0xf6 : b'\xc3\xb6',     # ö
+        0xf7 : b'\xc3\xb7',     # ÷
+        0xf8 : b'\xc3\xb8',     # ø
+        0xf9 : b'\xc3\xb9',     # ù
+        0xfa : b'\xc3\xba',     # ú
+        0xfb : b'\xc3\xbb',     # û
+        0xfc : b'\xc3\xbc',     # ü
+        0xfd : b'\xc3\xbd',     # ý
+        0xfe : b'\xc3\xbe',     # þ
+        }
+
+    MULTIBYTE_MARKERS_AND_SIZES = [
+        (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
+        (0xe0, 0xef, 3), # 3-byte characters start with E0-EF
+        (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
+        ]
+
+    FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
+    LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
+
+    @classmethod
+    def detwingle(cls, in_bytes, main_encoding="utf8",
+                  embedded_encoding="windows-1252"):
+        """Fix characters from one encoding embedded in some other encoding.
+
+        Currently the only situation supported is Windows-1252 (or its
+        subset ISO-8859-1), embedded in UTF-8.
+
+        The input must be a bytestring. If you've already converted
+        the document to Unicode, you're too late.
+
+        The output is a bytestring in which `embedded_encoding`
+        characters have been converted to their `main_encoding`
+        equivalents.
+        """
+        if embedded_encoding.replace('_', '-').lower() not in (
+            'windows-1252', 'windows_1252'):
+            raise NotImplementedError(
+                "Windows-1252 and ISO-8859-1 are the only currently supported "
+                "embedded encodings.")
+
+        if main_encoding.lower() not in ('utf8', 'utf-8'):
+            raise NotImplementedError(
+                "UTF-8 is the only currently supported main encoding.")
+
+        byte_chunks = []
+
+        chunk_start = 0
+        pos = 0
+        while pos < len(in_bytes):
+            byte = in_bytes[pos]
+            if not isinstance(byte, int):
+                # Python 2.x
+                byte = ord(byte)
+            if (byte >= cls.FIRST_MULTIBYTE_MARKER
+                and byte <= cls.LAST_MULTIBYTE_MARKER):
+                # This is the start of a UTF-8 multibyte character. Skip
+                # to the end.
+                for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
+                    if byte >= start and byte <= end:
+                        pos += size
+                        break
+            elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
+                # We found a Windows-1252 character!
+                # Save the string up to this point as a chunk.
+                byte_chunks.append(in_bytes[chunk_start:pos])
+
+                # Now translate the Windows-1252 character into UTF-8
+                # and add it as another, one-byte chunk.
+                byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
+                pos += 1
+                chunk_start = pos
+            else:
+                # Go on to the next character.
+                pos += 1
+        if chunk_start == 0:
+            # The string is unchanged.
+            return in_bytes
+        else:
+            # Store the final chunk.
+            byte_chunks.append(in_bytes[chunk_start:])
+        return b''.join(byte_chunks)
+

bs4/testing.py

@@ -0,0 +1,537 @@
+"""Helper classes for tests."""
+
+import copy
+import functools
+import unittest
+from unittest import TestCase
+from bs4 import BeautifulSoup
+from bs4.element import (
+    CharsetMetaAttributeValue,
+    Comment,
+    ContentMetaAttributeValue,
+    Doctype,
+    SoupStrainer,
+)
+
+from bs4.builder import HTMLParserTreeBuilder
+default_builder = HTMLParserTreeBuilder
+
+
+class SoupTest(unittest.TestCase):
+
+    @property
+    def default_builder(self):
+        return default_builder()
+
+    def soup(self, markup, **kwargs):
+        """Build a Beautiful Soup object from markup."""
+        builder = kwargs.pop('builder', self.default_builder)
+        return BeautifulSoup(markup, builder=builder, **kwargs)
+
+    def document_for(self, markup):
+        """Turn an HTML fragment into a document.
+
+        The details depend on the builder.
+        """
+        return self.default_builder.test_fragment_to_document(markup)
+
+    def assertSoupEquals(self, to_parse, compare_parsed_to=None):
+        builder = self.default_builder
+        obj = BeautifulSoup(to_parse, builder=builder)
+        if compare_parsed_to is None:
+            compare_parsed_to = to_parse
+
+        self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
+
+
+class HTMLTreeBuilderSmokeTest(object):
+
+    """A basic test of a treebuilder's competence.
+
+    Any HTML treebuilder, present or future, should be able to pass
+    these tests. With invalid markup, there's room for interpretation,
+    and different parsers can handle it differently. But with the
+    markup in these tests, there's not much room for interpretation.
+    """
+
+    def assertDoctypeHandled(self, doctype_fragment):
+        """Assert that a given doctype string is handled correctly."""
+        doctype_str, soup = self._document_with_doctype(doctype_fragment)
+
+        # Make sure a Doctype object was created.
+        doctype = soup.contents[0]
+        self.assertEqual(doctype.__class__, Doctype)
+        self.assertEqual(doctype, doctype_fragment)
+        self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)
+
+        # Make sure that the doctype was correctly associated with the
+        # parse tree and that the rest of the document parsed.
+        self.assertEqual(soup.p.contents[0], 'foo')
+
+    def _document_with_doctype(self, doctype_fragment):
+        """Generate and parse a document with the given doctype."""
+        doctype = '<!DOCTYPE %s>' % doctype_fragment
+        markup = doctype + '\n<p>foo</p>'
+        soup = self.soup(markup)
+        return doctype, soup
+
+    def test_normal_doctypes(self):
+        """Make sure normal, everyday HTML doctypes are handled correctly."""
+        self.assertDoctypeHandled("html")
+        self.assertDoctypeHandled(
+            'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')
+
+    def test_public_doctype_with_url(self):
+        doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
+        self.assertDoctypeHandled(doctype)
+
+    def test_system_doctype(self):
+        self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"')
+
+    def test_namespaced_system_doctype(self):
+        # We can handle a namespaced doctype with a system ID.
+        self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"')
+
+    def test_namespaced_public_doctype(self):
+        # Test a namespaced doctype with a public id.
+        self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"')
+
+    def test_real_xhtml_document(self):
+        """A real XHTML document should come out more or less the same as it went in."""
+        markup = b"""<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>Hello.</title></head>
+<body>Goodbye.</body>
+</html>"""
+        soup = self.soup(markup)
+        self.assertEqual(
+            soup.encode("utf-8").replace(b"\n", b""),
+            markup.replace(b"\n", b""))
+
+    def test_deepcopy(self):
+        """Make sure you can copy the tree builder.
+
+        This is important because the builder is part of a
+        BeautifulSoup object, and we want to be able to copy that.
+        """
+        copy.deepcopy(self.default_builder)
+
+    def test_p_tag_is_never_empty_element(self):
+        """A <p> tag is never designated as an empty-element tag.
+
+        Even if the markup shows it as an empty-element tag, it
+        shouldn't be presented that way.
+        """
+        soup = self.soup("<p/>")
+        self.assertFalse(soup.p.is_empty_element)
+        self.assertEqual(str(soup.p), "<p></p>")
+
+    def test_unclosed_tags_get_closed(self):
+        """A tag that's not closed by the end of the document should be closed.
+
+        This applies to all tags except empty-element tags.
+        """
+        self.assertSoupEquals("<p>", "<p></p>")
+        self.assertSoupEquals("<b>", "<b></b>")
+
+        self.assertSoupEquals("<br>", "<br/>")
+
+    def test_br_is_always_empty_element_tag(self):
+        """A <br> tag is designated as an empty-element tag.
+
+        Some parsers treat <br></br> as one <br/> tag, some parsers as
+        two tags, but it should always be an empty-element tag.
+        """
+        soup = self.soup("<br></br>")
+        self.assertTrue(soup.br.is_empty_element)
+        self.assertEqual(str(soup.br), "<br/>")
+
+    def test_nested_formatting_elements(self):
+        self.assertSoupEquals("<em><em></em></em>")
+
+    def test_comment(self):
+        # Comments are represented as Comment objects.
+        markup = "<p>foo<!--foobar-->baz</p>"
+        self.assertSoupEquals(markup)
+
+        soup = self.soup(markup)
+        comment = soup.find(text="foobar")
+        self.assertEqual(comment.__class__, Comment)
+
+    def test_preserved_whitespace_in_pre_and_textarea(self):
+        """Whitespace must be preserved in <pre> and <textarea> tags."""
+        self.assertSoupEquals("<pre>   </pre>")
+        self.assertSoupEquals("<textarea> woo  </textarea>")
+
+    def test_nested_inline_elements(self):
+        """Inline elements can be nested indefinitely."""
+        b_tag = "<b>Inside a B tag</b>"
+        self.assertSoupEquals(b_tag)
+
+        nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
+        self.assertSoupEquals(nested_b_tag)
+
+        double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
+        self.assertSoupEquals(nested_b_tag)
+
+    def test_nested_block_level_elements(self):
+        """Block elements can be nested."""
+        soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
+        blockquote = soup.blockquote
+        self.assertEqual(blockquote.p.b.string, 'Foo')
+        self.assertEqual(blockquote.b.string, 'Foo')
+
+    def test_correctly_nested_tables(self):
+        """One table can go inside another one."""
+        markup = ('<table id="1">'
+                  '<tr>'
+                  "<td>Here's another table:"
+                  '<table id="2">'
+                  '<tr><td>foo</td></tr>'
+                  '</table></td>')
+
+        self.assertSoupEquals(
+            markup,
+            '<table id="1"><tr><td>Here\'s another table:'
+            '<table id="2"><tr><td>foo</td></tr></table>'
+            '</td></tr></table>')
+
+        self.assertSoupEquals(
+            "<table><thead><tr><td>Foo</td></tr></thead>"
+            "<tbody><tr><td>Bar</td></tr></tbody>"
+            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
+
+    def test_deeply_nested_multivalued_attribute(self):
+        # html5lib can set the attributes of the same tag many times
+        # as it rearranges the tree. This has caused problems with
+        # multivalued attributes.
+        markup = '<table><div><div class="css"></div></div></table>'
+        soup = self.soup(markup)
+        self.assertEqual(["css"], soup.div.div['class'])
+
+    def test_angle_brackets_in_attribute_values_are_escaped(self):
+        self.assertSoupEquals('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')
+
+    def test_entities_in_attributes_converted_to_unicode(self):
+        expect = u'<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
+        self.assertSoupEquals('<p id="pi&#241;ata"></p>', expect)
+        self.assertSoupEquals('<p id="pi&#xf1;ata"></p>', expect)
+        self.assertSoupEquals('<p id="pi&ntilde;ata"></p>', expect)
+
+    def test_entities_in_text_converted_to_unicode(self):
+        expect = u'<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
+        self.assertSoupEquals("<p>pi&#241;ata</p>", expect)
+        self.assertSoupEquals("<p>pi&#xf1;ata</p>", expect)
+        self.assertSoupEquals("<p>pi&ntilde;ata</p>", expect)
+
+    def test_quot_entity_converted_to_quotation_mark(self):
+        self.assertSoupEquals("<p>I said &quot;good day!&quot;</p>",
+                              '<p>I said "good day!"</p>')
+
+    def test_out_of_range_entity(self):
+        expect = u"\N{REPLACEMENT CHARACTER}"
+        self.assertSoupEquals("&#10000000000000;", expect)
+        self.assertSoupEquals("&#x10000000000000;", expect)
+        self.assertSoupEquals("&#1000000000;", expect)
+
+    def test_basic_namespaces(self):
+        """Parsers don't need to *understand* namespaces, but at the
+        very least they should not choke on namespaces or lose
+        data."""
+
+        markup = b'<html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>'
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode())
+        html = soup.html
+        self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
+        self.assertEqual(
+            'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
+        self.assertEqual(
+            'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])
+
+    def test_multivalued_attribute_value_becomes_list(self):
+        markup = b'<a class="foo bar">'
+        soup = self.soup(markup)
+        self.assertEqual(['foo', 'bar'], soup.a['class'])
+
+    #
+    # Generally speaking, tests below this point are more tests of
+    # Beautiful Soup than tests of the tree builders. But parsers are
+    # weird, so we run these tests separately for every tree builder
+    # to detect any differences between them.
+    #
+
+    def test_soupstrainer(self):
+        """Parsers should be able to work with SoupStrainers."""
+        strainer = SoupStrainer("b")
+        soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>",
+                         parse_only=strainer)
+        self.assertEqual(soup.decode(), "<b>bold</b>")
+
+    def test_single_quote_attribute_values_become_double_quotes(self):
+        self.assertSoupEquals("<foo attr='bar'></foo>",
+                              '<foo attr="bar"></foo>')
+
+    def test_attribute_values_with_nested_quotes_are_left_alone(self):
+        text = """<foo attr='bar "brawls" happen'>a</foo>"""
+        self.assertSoupEquals(text)
+
+    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
+        text = """<foo attr='bar "brawls" happen'>a</foo>"""
+        soup = self.soup(text)
+        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
+        self.assertSoupEquals(
+            soup.foo.decode(),
+            """<foo attr="Brawls happen at &quot;Bob\'s Bar&quot;">a</foo>""")
+
+    def test_ampersand_in_attribute_value_gets_escaped(self):
+        self.assertSoupEquals('<this is="really messed up & stuff"></this>',
+                              '<this is="really messed up &amp; stuff"></this>')
+
+        self.assertSoupEquals(
+            '<a href="http://example.org?a=1&b=2;3">foo</a>',
+            '<a href="http://example.org?a=1&amp;b=2;3">foo</a>')
+
+    def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
+        self.assertSoupEquals('<a href="http://example.org?a=1&amp;b=2;3"></a>')
+
+    def test_entities_in_strings_converted_during_parsing(self):
+        # Both XML and HTML entities are converted to Unicode characters
+        # during parsing.
+        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
+        expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
+        self.assertSoupEquals(text, expected)
+
+    def test_smart_quotes_converted_on_the_way_in(self):
+        # Microsoft smart quotes are converted to Unicode characters during
+        # parsing.
+        quote = b"<p>\x91Foo\x92</p>"
+        soup = self.soup(quote)
+        self.assertEqual(
+            soup.p.string,
+            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
+
+    def test_non_breaking_spaces_converted_on_the_way_in(self):
+        soup = self.soup("<a>&nbsp;&nbsp;</a>")
+        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
+
+    def test_entities_converted_on_the_way_out(self):
+        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
+        expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
+        soup = self.soup(text)
+        self.assertEqual(soup.p.encode("utf-8"), expected)
+
+    def test_real_iso_latin_document(self):
+        # Smoke test of interrelated functionality, using an
+        # easy-to-understand document.
+
+        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
+        unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
+
+        # That's because we're going to encode it into ISO-Latin-1, and use
+        # that to test.
+        iso_latin_html = unicode_html.encode("iso-8859-1")
+
+        # Parse the ISO-Latin-1 HTML.
+        soup = self.soup(iso_latin_html)
+        # Encode it to UTF-8.
+        result = soup.encode("utf-8")
+
+        # What do we expect the result to look like? Well, it would
+        # look like unicode_html, except that the META tag would say
+        # UTF-8 instead of ISO-Latin-1.
+        expected = unicode_html.replace("ISO-Latin-1", "utf-8")
+
+        # And, of course, it would be in UTF-8, not Unicode.
+        expected = expected.encode("utf-8")
+
+        # Ta-da!
+        self.assertEqual(result, expected)
+
+    def test_real_shift_jis_document(self):
+        # Smoke test to make sure the parser can handle a document in
+        # Shift-JIS encoding, without choking.
+        shift_jis_html = (
+            b'<html><head></head><body><pre>'
+            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
+            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
+            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
+            b'</pre></body></html>')
+        unicode_html = shift_jis_html.decode("shift-jis")
+        soup = self.soup(unicode_html)
+
+        # Make sure the parse tree is correctly encoded to various
+        # encodings.
+        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
+        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
+
+    def test_real_hebrew_document(self):
+        # A real-world test to make sure we can convert ISO-8859-9 (a
+        # Hebrew encoding) to UTF-8.
+        hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
+        soup = self.soup(
+            hebrew_document, from_encoding="iso8859-8")
+        self.assertEqual(soup.original_encoding, 'iso8859-8')
+        self.assertEqual(
+            soup.encode('utf-8'),
+            hebrew_document.decode("iso8859-8").encode("utf-8"))
+
+    def test_meta_tag_reflects_current_encoding(self):
+        # Here's the <meta> tag saying that a document is
+        # encoded in Shift-JIS.
+        meta_tag = ('<meta content="text/html; charset=x-sjis" '
+                    'http-equiv="Content-type"/>')
+
+        # Here's a document incorporating that meta tag.
+        shift_jis_html = (
+            '<html><head>\n%s\n'
+            '<meta http-equiv="Content-language" content="ja"/>'
+            '</head><body>Shift-JIS markup goes here.') % meta_tag
+        soup = self.soup(shift_jis_html)
+
+        # Parse the document, and the charset is seemingly unaffected.
+        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
+        content = parsed_meta['content']
+        self.assertEqual('text/html; charset=x-sjis', content)
+
+        # But that value is actually a ContentMetaAttributeValue object.
+        self.assertTrue(isinstance(content, ContentMetaAttributeValue))
+
+        # And it will take on a value that reflects its current
+        # encoding.
+        self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
+
+        # For the rest of the story, see TestSubstitutions in
+        # test_tree.py.
+
+    def test_html5_style_meta_tag_reflects_current_encoding(self):
+        # Here's the <meta> tag saying that a document is
+        # encoded in Shift-JIS.
+        meta_tag = ('<meta id="encoding" charset="x-sjis" />')
+
+        # Here's a document incorporating that meta tag.
+        shift_jis_html = (
+            '<html><head>\n%s\n'
+            '<meta http-equiv="Content-language" content="ja"/>'
+            '</head><body>Shift-JIS markup goes here.') % meta_tag
+        soup = self.soup(shift_jis_html)
+
+        # Parse the document, and the charset is seemingly unaffected.
+        parsed_meta = soup.find('meta', id="encoding")
+        charset = parsed_meta['charset']
+        self.assertEqual('x-sjis', charset)
+
+        # But that value is actually a CharsetMetaAttributeValue object.
+        self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
+
+        # And it will take on a value that reflects its current
+        # encoding.
+        self.assertEqual('utf8', charset.encode("utf8"))
+
+    def test_tag_with_no_attributes_can_have_attributes_added(self):
+        data = self.soup("<a>text</a>")
+        data.a['foo'] = 'bar'
+        self.assertEqual('<a foo="bar">text</a>', data.a.decode())
+
+class XMLTreeBuilderSmokeTest(object):
+
+    def test_docstring_generated(self):
+        soup = self.soup("<root/>")
+        self.assertEqual(
+            soup.encode(), b'<?xml version="1.0" encoding="utf-8"?>\n<root/>')
+
+    def test_real_xhtml_document(self):
+        """A real XHTML document should come out *exactly* the same as it went in."""
+        markup = b"""<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>Hello.</title></head>
+<body>Goodbye.</body>
+</html>"""
+        soup = self.soup(markup)
+        self.assertEqual(
+            soup.encode("utf-8"), markup)
+
+    def test_popping_namespaced_tag(self):
+        markup = '<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>'
+        soup = self.soup(markup)
+        self.assertEqual(
+            unicode(soup.rss), markup)
+
+    def test_docstring_includes_correct_encoding(self):
+        soup = self.soup("<root/>")
+        self.assertEqual(
+            soup.encode("latin1"),
+            b'<?xml version="1.0" encoding="latin1"?>\n<root/>')
+
+    def test_large_xml_document(self):
+        """A large XML document should come out the same as it went in."""
+        markup = (b'<?xml version="1.0" encoding="utf-8"?>\n<root>'
+                  + b'0' * (2**12)
+                  + b'</root>')
+        soup = self.soup(markup)
+        self.assertEqual(soup.encode("utf-8"), markup)
+
+
+    def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
+        self.assertSoupEquals("<p>", "<p/>")
+        self.assertSoupEquals("<p>foo</p>")
+
+    def test_namespaces_are_preserved(self):
+        markup = '<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>'
+        soup = self.soup(markup)
+        root = soup.root
+        self.assertEqual("http://example.com/", root['xmlns:a'])
+        self.assertEqual("http://example.net/", root['xmlns:b'])
+
+    def test_closing_namespaced_tag(self):
+        markup = '<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>'
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.p), markup)
+
+    def test_namespaced_attributes(self):
+        markup = '<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>'
+        soup = self.soup(markup)
+        self.assertEqual(unicode(soup.foo), markup)
+
+class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
+    """Smoke test for a tree builder that supports HTML5."""
+
+    def test_real_xhtml_document(self):
+        # Since XHTML is not HTML5, HTML5 parsers are not tested to handle
+        # XHTML documents in any particular way.
+        pass
+
+    def test_html_tags_have_namespace(self):
+        markup = "<a>"
+        soup = self.soup(markup)
+        self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)
+
+    def test_svg_tags_have_namespace(self):
+        markup = '<svg><circle/></svg>'
+        soup = self.soup(markup)
+        namespace = "http://www.w3.org/2000/svg"
+        self.assertEqual(namespace, soup.svg.namespace)
+        self.assertEqual(namespace, soup.circle.namespace)
+
+
+    def test_mathml_tags_have_namespace(self):
+        markup = '<math><msqrt>5</msqrt></math>'
+        soup = self.soup(markup)
+        namespace = 'http://www.w3.org/1998/Math/MathML'
+        self.assertEqual(namespace, soup.math.namespace)
+        self.assertEqual(namespace, soup.msqrt.namespace)
+
+
+def skipIf(condition, reason):
+   def nothing(test, *args, **kwargs):
+       return None
+
+   def decorator(test_item):
+       if condition:
+           return nothing
+       else:
+           return test_item
+
+   return decorator

data/interfaces/brink/artist.html

@@ -8,22 +8,23 @@
 <%def name="body()">
 	<script>
 		$(document).ready(function(){
-			$('#menu_link_getextra').click(function() {
-				$('#dialog').dialog();
+			$('#getExtras').click(function() {
+				$('#extras-dialog').dialog();
 				return false;
 			});
-			$('#menu_link_modifyextra').click(function() {
-				$('#dialog').dialog();
+			$('#modifyExtras').click(function() {
+				$('#extras-dialog').dialog();
 			});
 		});
 	</script>
 
-	<div id="dialog" title="Choose Which Extras to Fetch" style="display:none" class="configtable">
+	<div id="extras-dialog" title="Choose Which Extras to Fetch" style="display:none" class="configtable">
 		<form action="getExtras" method="get" class="ajax">
 			<input type="hidden" name="ArtistID" value="${artist['ArtistID']}">
 			<input type="hidden" name="newstyle" value="true">
 			%for extra in extras:
-				<input type="checkbox" id="${extra}" name="${extra}" value="1" ${extras[extra]} />${string.capwords(extra)}<br>
+				<input type="checkbox" id="${extra}" name="${extra}" value="1" ${extras[extra]} />
+					<label for="${extra}">${string.capwords(extra)}</label><br />
 			%endfor
 			<br>
 			<input id="submit" type="submit" value="Fetch Extras">
@@ -41,9 +42,9 @@
 			%endif
 			%if artist['IncludeExtras']:
 				<li><a href="removeExtras?ArtistID=${artist['ArtistID']}">Remove Extras</a></li>
-				<li><a class="menu_link_edit" href="#">Modify Extras</a></li>
+				<li><a id="modifyExtras" href="#">Modify Extras</a></li>
 			%else:	
-				<li><a id="menu_link_getextra" href="#">Get Extras</a></li>
+				<li><a id="getExtras" href="#">Get Extras</a></li>
 			%endif
 		</ul>
 		<ul id="nav-view">
@@ -74,12 +75,13 @@
 		</select>
 		<input type="submit" value="GO">
 	</p>
+	<small><span class="wsr Tag"></span> Click CTRL + LMOUSE on albums to select them in grid view.</small>
 	<div id="gridView">
 	%for album in albums:
 		<%
 		%>
 		<div class="image-container">
-			<div class="image-box ${album['Status']}">
+			<div class="image-box">
 				<div class="image-tag ${album['Status']}"></div>
 				<img />
 				<div class="image-actions">
@@ -98,7 +100,7 @@
 			
 			<div class="image-info">
 				<b style="display: none;">${artist['ArtistName']}</b>
-				<span><a href="albumPage?AlbumID=${album['AlbumID']}" title="${album['AlbumID']}">${album['AlbumTitle']}</a></span>
+				<span><a class="image-url" href="albumPage?AlbumID=${album['AlbumID']}" title="${album['AlbumID']}">${album['AlbumTitle']}</a></span>
 			</div>
 		</div>
 	%endfor
@@ -152,16 +154,7 @@
 				<td id="albumart">
 				<div class="album-art-small">
 					<a href="albumPage?AlbumID=${album['AlbumID']}" title="${album['AlbumID']}">
-						<div class="status">
-							%if album['Status'] == 'Skipped':
-								<a class="wsr" href="queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}" title="Mark Wanted"><span>O</span></a>
-							%elif album['Status'] == 'Wanted':
-								<a class="wsr" href="unqueueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}" title="Unmark Wanted"><span>N</span></a>
-							%else:
-								<a class="wsr" href="queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}" title="Retry"><span>*</span></a>
-								<a class="wsr" href="queueAlbum?AlbumID=${album['AlbumID']}&ArtistID=${album['ArtistID']}&new=True" title="Try new"><span>J</span></a>
-							%endif
-						</div>
+						<img />
 					</a>
 				</div>
 				</td>
@@ -194,6 +187,19 @@
 <%def name="javascriptIncludes()">
 	<script>
 	$(document).ready(function(){
+		$(".image-box").click(function(evt) {
+			if (evt.ctrlKey)
+			{
+				var input = $(this).find('.image-select');
+				if( $(input).is(':checked') ){
+					$(input).attr('checked', false);
+				}
+				else{
+					$(input).attr('checked', true);
+				}
+			}
+		});
+		
 		//Load Art
 		getAlbumArt();
 		getArtistArt();
@@ -257,6 +263,14 @@
 				getAlbumInfo(artistname,albumname,element,2);
 			});
 		});
+		$("table#album_table").each(function() {
+			$(this).fadeIn("slow", function(){
+				var element = $(this).find("img");
+				var artistname = $(".artist-art img").attr("alt");
+				var albumname = $(this).find("#albumname").text();
+				getAlbumInfo(artistname,albumname,element,2);
+			});
+		});
 	}
 	</script>
 </%def>

data/interfaces/brink/base.html

@@ -42,6 +42,7 @@
 	
 	<!--Global Script-->
 	<script src="interfaces/brink/js/script.js"></script>
+	<script src="interfaces/brink/js/plugins.js"></script>
 	
 	<!--Datatables-->
 	<script type="text/javascript" src="/js/libs/jquery.dataTables.min.js"></script>
@@ -61,11 +62,17 @@
 	<script type="text/javascript">
 		$(document).ready(function(){
 	% if not headphones.CURRENT_VERSION:
-			$("#commits-behind").fadeIn('slow').text("${headphones.COMMITS_BEHIND}");
+			var noty_id = noty({
+				text:'<span class="wsr">V</span> <strong>Headpones Update Available!</strong><br />A new version is ready to be installed.',
+				layout: 'bottomRight',
+				type:'information
+			});
 	% elif headphones.CURRENT_VERSION != headphones.LATEST_VERSION and headphones.INSTALL_TYPE != 'win':
-			$("#commits-behind").fadeIn('slow').text("(${headphones.COMMITS_BEHIND})");	
-	% else: 
-			$("#commits-behind").hide()
+			var noty_id = noty({
+				text:'<span class="wsr">V</span> You are ${headphones.COMMITS_BEHIND} commits behind',
+				layout: 'bottomRight',
+				type:'information'
+			});
 	% endif
 		});
 	</script>
@@ -128,8 +135,7 @@
 					<a href="forceSearch"><span class="wsr">I</span> Wanted Albums</a>
 					<a href="forceUpdate"><span class="wsr">U</span> Active Artists</a>
 					<a href="forcePostProcess"><span class="wsr">J</span> Post-Process</a>
-					<a href="checkGithub" onclick="javascript:return confirm('Do you wish to Update Headphones now?')"><span class="wsr">V</span> Update
-						<span id="commits-behind"></span>
+					<a href="update" onclick="javascript:return confirm('Do you wish to Update Headphones now?')"><span class="wsr">V</span> Update
 					</a>
 				</div>
 				<div class="btn right">

data/interfaces/brink/config.html

@@ -1,6 +1,7 @@
 <%inherit file="base.html"/>
 <%!
 	import headphones
+	import string
 %>
 
 <%def name="headIncludes()">
@@ -77,7 +78,7 @@
 						</tr>
 						<tr>
 							<td>
-								<label for="launch_browser">Launch browser:</label><br />
+								<label for="launch_browser">Launch browser:</label>
 								<br /><small>Enabling this feature will open Headphones at startup.</small>
 							</td>
 							<td>
@@ -111,15 +112,19 @@
 						<!--API-->
 							<tr>
 								<td><h2>API:</h2>
-									<small><label for="api_switch">Enabled/Disabled</label></small></td>
-								<td><input class="switch" type="checkbox" name="api_switch" id="api_enabled" value="1" ${config['api_enabled']} /></td>
+									<small><label for="api_enabled">Enabled/Disabled</label></small></td>
+								<td><input class="switch" type="checkbox" name="api_enabled" id="api_enabled" value="1" ${config['api_enabled']} /></td>
 							</tr>
-							<tr id="api_switch">
+							<tr id="api_enabled_switch">
 								<td>
-									<small>Visit <a href="https://github.com/rembo10/headphones/tree/" target="_blank"><span>D</span>repo</a> for more information.</small>
-								</td>
-								<td>
-									<input type="text" name="api_key" value="${config['api_key']}" size="30"><input class="submit" type="button" value="Generate" id="generate_api">
+									<table>
+										<tr>
+											<td colspan="2">
+											<small>Current API:<strong id="api_key">${config['api_key']}</strong></small>
+											<input class="submit" type="button" value="Generate" id="generate_api" />
+											</td>
+										</tr>
+									</table>
 								</td>
 							</tr>
 						<!--end API-->
@@ -289,123 +294,146 @@
 				<small>"We provide you with a automated search service to locate binary files/secgments that can be found on the public access network called Usenet."</small>
 				
 				<!--NZB Matrix-->
+				<div class="group">
 				<table id="nzbmatrix">
 					<tr>
+						<td><input class="switch" type="checkbox" name="nzbmatrix" value="1" ${config['use_nzbmatrix']} /></td>
 						<td>
 							<h2><a href="http://www.nzbmatrix.com" target="_blank"><span class="wsr">D</span>NZBMatrix</a></h2>
-							<br /><small><label for="nzbmatrix_switch">Enabled/Disabled</label></small>
+							<small><label for="nzbmatrix">Enabled/Disabled</label></small>
 						</td>
-						<td><input class="switch" type="checkbox" name="nzbmatrix_switch" value="1" ${config['use_nzbmatrix']} /></td>
-					</tr>
-				</table>
-				<table id="nzbmatrix_switch">
-					<tr>
-						<td><label for="nzbmatrix_username">Username:</label></td>
-							<td><input type="text" name="nzbmatrix_username" value="${config['nzbmatrix_user']}" size="30" maxlength="40" /></td>
-					</tr>
-					<tr>
-						<td><label for="nzbmatrix_apikey">API:</label></td>
-							<td><input type="text" name="nzbmatrix_apikey" value="${config['nzbmatrix_api']}" size="36" maxlength="40" /></td>
 					</tr>
 				</table>
+				<div class="group" id="nzbmatrix_switch">
+					<table>
+						<tr>
+							<td><label for="nzbmatrix_username">Username:</label></td>
+								<td><input type="text" name="nzbmatrix_username" value="${config['nzbmatrix_user']}" size="30" maxlength="40" /></td>
+						</tr>
+						<tr>
+							<td><label for="nzbmatrix_apikey">API:</label></td>
+								<td><input type="text" name="nzbmatrix_apikey" value="${config['nzbmatrix_api']}" size="36" maxlength="40" /></td>
+						</tr>
+					</table>
+				</div>
+				</div>
 				<!--end NZB Matrix-->
 
 				<!--Newznab-->
-				<table id="newznab" class="configtable">
-					<tr>
-						<td>
-							<h2><a href="http://www.newznab.com/" target="_blank"><span class="wsr">D</span>Newznab</a></h2>
-							<br /><small><label for="newznab">Enabled/Disabled</label></small>
-						</td>
-						<td><input class="switch" type="checkbox" name="newznab" value="1" ${config['use_newznab']} /></td>
-					</tr>
-					<tr>
-						<div id="newznab">
-							<div class="config" id="newznab1">
-								<div class="row">
-									<label>Newznab Host</label>
-									<input type="text" name="newznab_host" value="${config['newznab_host']}" size="30">
-									<br /><small>e.g. http://nzb.su</small>
-								</div>
-								<div class="row">
-									<label>Newznab API</label>
-									<input type="text" name="newznab_apikey" value="${config['newznab_api']}" size="36">
-								</div>
-								<div class="row checkbox">
-									<input id="newznab_enabled" type="checkbox" name="newznab_enabled" onclick="initConfigCheckbox($(this));" value="1" ${config['newznab_enabled']} />
-									<label>Enabled</label>
-								</div>
-							</div>
-							<%
-							newznab_number = 1
-							%>
+				<div class="group">
+					<table class="configtable">
+						<tr>
+							<td><input class="switch" type="checkbox" name="newznab" value="1" ${config['use_newznab']} /></td>
+							<td>
+								<h2><a href="http://www.newznab.com/" target="_blank"><span class="wsr">D</span>Newznab Providers</a></h2>
+								<small><label for="newznab">Enabled/Disabled</label></small>
+							</td>
+						</tr>
+					</table>
+					<div class="group" id="newznab_switch">
+						<div>
+							<table id="newznab1">
+								<tr><td>
+										<label>Host:</label>
+										<input type="text" name="newznab_host" value="${config['newznab_host']}" size="30">
+										<br /><small>e.g. http://nzb.su</small>
+									</td>
+								</tr>
+								<tr>
+									<td>
+										<label>API</label>
+										<input type="text" name="newznab_apikey" value="${config['newznab_api']}" size="36">
+									</td>
+								</tr>
+								<tr>
+									<td>
+										<input id="newznab_enabled" type="checkbox" name="newznab_enabled" value="1" ${config['newznab_enabled']} />
+										<label>Enabled</label>
+									</td>
+								</tr>
+								<tr>
+									<td><input class="newznab-edit-extras" type="button" value ="Edit Extras"/></td>
+								</tr>
+							</table>
+						</div>
+						<div id="newznab-dialog">
 							%for newznab in config['extra_newznabs']:
-							<div class="config" id="newznab${newznab_number}">
-								<div class="row">
-								<label>Newznab Host:</label>
-								<input type="text" name="newznab_host${newznab_number}" value="${newznab[0]}" size="30">
-								</div>
-								<div class="row">
-								<label>Newznab API:</label>
-								<input type="text" name="newznab_api${newznab_number}" value="${newznab[1]}" size="36">
-								</div>
-								<div class="row checkbox">
-								<input id="newznab_enabled" type="checkbox" name="newznab_enabled${newznab_number}" value="1" ${newznab_enabled} /><label>Enabled</label>
-								</div>
-								<div class="row">
-								<input type="button" class="remove" id="newznab${newznab_number}" value="Remove ${newznab[0]}">
-								</div>
-							</div>
-							<%
-								newznab_number += 1
-							%>
+							<table id="newznab${newznab_number}">
+								<tr>
+									<td>
+										<label>Newznab Host:</label>
+										<input type="text" name="newznab_host${newznab_number}" value="${newznab[0]}" size="30">
+									</td>
+								</tr>
+								<tr>
+									<td>
+										<label>Newznab API:</label>
+										<input type="text" name="newznab_api${newznab_number}" value="${newznab[1]}" size="36">
+									</td>
+								</tr>
+								<tr>
+									<td>
+										<input id="newznab_enabled" type="checkbox" name="newznab_enabled${newznab_number}" value="1" ${newznab_enabled} /><label>Enabled/Disabled</label>
+									</td>
+								</tr>
+								<tr>
+									<td>
+										<input type="button" class="remove" id="newznab${newznab_number}" value="Remove ${newznab[0]}">
+									</td>
+								</tr>
+							</table>
 							%endfor
 							<input type="button" value="Add Newznab" class="add_newznab" id="add_newznab" />
 						</div>
-					</tr>
-				</table>
+					</div>
+				</div>
 				<!--end Newznab-->
 
 				<!--NZBs.org-->
-				<table>
-					<tr>
-						<td>
-							<h2><a href="http://nzbs.org/" target="_blank"><span class="wsr">D</span>Nzbs</a></h2>
-							<small></small>
-						</td>
-						<td><input class="switch" type="checkbox" name="nzbsorg" value="1" ${config['use_nzbsorg']} /><small><label for="nzbsorg"></label></small></td>
-					</tr>
-				</table>
-				<table id="nzbsorg">
-					<tr>
-						<td><label for="nzbsorg_hash">API Key:</label></td>
-						<td><input type="text" name="nzbsorg_hash" value="${config['nzbsorg_hash']}" size="30" maxlength="40"></td>
-					</tr>
-				</table>
+				<div class="group">
+					<table>
+						<tr>
+							<td><input class="switch" type="checkbox" name="nzbsorg" value="1" ${config['use_nzbsorg']} /></td>
+							<td><h2><a href="http://nzbs.org/" target="_blank"><span class="wsr">D</span>Nzbs.org</a></h2>
+								<small><label for="nzbsorg"></label></small>
+							</td>
+						</tr>
+					</table>
+					<div class="group" id="nzbsorg_switch">
+						<table>
+							<tr>
+								<td><label for="nzbsorg_hash">API Key:</label></td>
+								<td><input type="text" name="nzbsorg_hash" value="${config['nzbsorg_hash']}" size="30" maxlength="40"></td>
+							</tr>
+						</table>
+					</div>
+				</div>
 				<!--end NZBs.org-->
 				
 				<!--Newzbin-->
+				<div class="group">
 				<table>
 					<tr>
+						<td><input class="switch" type="checkbox" name="newzbin" value="1" ${config['use_newzbin']} /></td>
 						<td>
 							<h2><a href="http://www.newsbin.com/" target="_blank"><span class="wsr">D</span>Newzbin</a></h2>
 							<small><label for="newzbin">Enabled/Disabled</label></small>
 						</td>
-						<td>
-							<input class="switch" type="checkbox" name="newzbin" value="1" ${config['use_newzbin']} />
-						</td>
-					</tr>
-				</table>
-				<table id="newsbin" class="configtable">
-					<tr>
-						<td><label for="newzbin_uid">UID:</label></td>
-						<td><input type="text" name="newzbin_uid" value="${config['newzbin_uid']}" size="30" maxlength="40" /></td>
-					</tr>
-					<tr>
-						<td><label for="newzbin_password">Password:</label></td>
-						<td><input type="text" name="newzbin_password" value="${config['newzbin_pass']}" size="36" maxlength="40" /></td>
 					</tr>
 				</table>
+				<div class="group" id="newzbin_switch">
+					<table>
+						<tr>
+							<td><label for="newzbin_uid">UID:</label></td>
+							<td><input type="text" name="newzbin_uid" value="${config['newzbin_uid']}" size="30" maxlength="40" /></td>
+						</tr>
+						<tr>
+							<td><label for="newzbin_password">Password:</label></td>
+							<td><input type="text" name="newzbin_password" value="${config['newzbin_pass']}" size="36" maxlength="40" /></td>
+						</tr>
+					</table>
+				</div>
+				</div>
 				<!--end Newzbin-->
 					
 				<!--Torrents-->
@@ -559,6 +587,15 @@
 								<input type="text" name="log_dir" value="${config['log_dir']}" size="40" />
 							</td>
 						</tr><!--end Logs-->
+						<!--Cache-->
+						<tr>
+							<td>
+								<label for="cache_dir">Cache Directory:</label>
+							</td>
+							<td>
+								<input type="text" name="cache_dir" value="${config['cache_dir']}" size="40" />
+							</td>
+						</tr><!--end Cache-->
 					</table>
 				</div><!--end Miscellaneous-->
 				
@@ -569,7 +606,7 @@
 					<div><!--Encoder Options-->
 						<table id="encoderoptions">
 							<tr>
-								<td><h2>Encoder Options</h2></td>
+								<td colspan="2"><h2>Encoder Options</h2></td>
 							</tr>
 							<tr>
 								<td><label for="encoderlossless">Only re-encode lossless files (.flac)</label></td>
@@ -702,15 +739,17 @@
 					<h1><span class="wsr">W</span> Notifications</h1>
 					<small></small>
 					<!--Prowl-->
+					<div class="group">
 					<table>
 						<tr>
+							<td><input class="switch" type="checkbox" name="prowl_enabled" id="prowl" value="1" ${config['prowl_enabled']} /></td>
 							<td><h2>Prowl Options</h2>
 								<small><label for="prowl_enabled">Enabled/Disabled</label></small>
 							</td>
-							<td><input class="switch" type="checkbox" name="prowl_enabled" id="prowl" value="1" ${config['prowl_enabled']} /></td>
 						</tr>
 					</table>
-					<table id="prowloptions">
+					<div class="group" id="prowl_enabled_switch">
+					<table>
 						
 						<tr>
 							<td><label for="prowl_keys">API key:</label></td>
@@ -723,17 +762,22 @@
 							<tr><td><label for="prowl_priority">Priority (-2,-1,0,1 or 2):</label></td>
 							<td><input type="text" name="prowl_priority" value="${config['prowl_priority']}" size="2"></td>
 						</tr>
-					</table><!--end Prowl-->
+					</table>
+					</div>
+					</div>
+					<!--end Prowl-->
 					<!--XBMC-->
+					<div class="group">
 					<table>
 						<tr>
+							<td><input class="switch" type="checkbox" name="xbmc_enabled" id="xbmc" value="1" ${config['xbmc_enabled']} /></td>
 							<td><h2>XBMC</h2>
 								<small><label for="xbmc_enabled">Enabled/Disabled</label></small>
 							</td>
-							<td><input class="switch" type="checkbox" name="xbmc_enabled" id="xbmc" value="1" ${config['xbmc_enabled']} /></td>
 						</tr>
 					</table>
-					<table id="xbmcoptions">
+					<div class="group" id="xbmc_enabled_switch">
+					<table>
 						<tr>
 							<td><label for="xbmc_host">XBMC Host:</label>
 								<small>e.g. http://localhost:8080. Separate hosts with commas</small>
@@ -757,19 +801,24 @@
 							<td><label>Send Notification to XBMC:</label></td>
 							<td><input type="checkbox" name="xbmc_notify" value="1" ${config['xbmc_notify']} /></td>
 						</tr>
-					</table><!--end XBMC-->
+					</table>
+					</div>
+					</div>
+					<!--end XBMC-->
 					
 					<!--NMA Options-->
+					<div class="group">
 					<table>
 						<tr>
+							<td><input class="switch" type="checkbox" name="nma_enabled" id="nma" value="1" ${config['nma_enabled']} /></td>
 							<td>
 								<h2>Notify My Android</h2>
 								<small><label for="nma_enabled">Enabled/Disabled</label></small>
 							</td>
-							<td><input class="switch" type="checkbox" name="nma_enabled" id="nma" value="1" ${config['nma_enabled']} /></td>
 						</tr>
 					</table>
-					<table id="nmaoptions">
+					<div class="group" id="nma_enabled_switch">
+					<table>
 						<tr>
 							<td><label for="nma_apikey">NotifyMyAndroid API Key:</label>
 								<small>Separate multiple api keys with ",".</small>
@@ -780,6 +829,11 @@
 							<td><a href="https://play.google.com/store/apps/details?id=com.usk.app.notifymyandroid">Get NotifyMyAndroid</a></td>
 						</tr>
 						
+						<tr>
+							<td><label for="nma_onsnatch">Notify on snatch?</label></td>
+								<td><input type="checkbox" name="nma_onsnatch" value="1" ${config['nma_onsnatch']} /></td>
+						</tr>
+						
 						<!--Priority-->
 						<tr>
 							<td><label for="nma_priority">Priority:</label></td>
@@ -809,10 +863,18 @@
 							</td>
 						</tr><!--end Priority-->
 					</table><!--end NMA Options-->
+					</div>
 				</div><!--end Notifications-->
+				
 				<div id="NAS">
 					<h2>Synology NAS</h2>
-					<input type="checkbox" name="synoindex_enabled" id="synoindex" value="1" ${config['synoindex_enabled']} /><label>Enable Synoindex</label>
+					<table>
+						<tr>
+							<td>
+								<input type="checkbox" name="synoindex_enabled" id="synoindex" value="1" ${config['synoindex_enabled']} /><label>Enable Synoindex</label>
+							</td>
+						</tr>
+					</table>
 				</div>
 				<!--Mirror-->
 				<div id="Mirror">
@@ -885,20 +947,26 @@
 <%def name="javascriptIncludes()">
 	<script>
 		$(document).ready(function(){
+			$('#api_key').click(function(){ $('#api_key').select() });
+			$("#generate_api").click(function(){
+				$.get('generateAPI', 
+					function(data){
+						if (data.error != undefined) {
+							alert(data.error);
+							return;
+						}
+						$('#api_key').val(data);
+				});
+			});
+		
+			$("#newznab-dialog").hide();
+			$(".newznab-edit-extras").click(function(){
+				$("#newznab-dialog").dialog();
+			});
 			$(".submitForm").click(function(){
 				$("#config").submit();
 			});
 			//Newznab
-			function newznab(){
-				if( $('input[name="newznab"]').is(":checked") ){
-					$("table#newznab-group").show();
-					enit();
-				}
-				else {
-					$("table#newznab-group").hide();
-					enit();
-				}
-			}
 			var deletedNewznabs = 0;
 			$(".remove").click(function() {
 				$(this).parent().parent().remove();
@@ -906,17 +974,31 @@
 			});
 			
 			$("#add_newznab").click(function() {
-				var intId = $("#newznab_providers > div").size() + deletedNewznabs + 1;
-				var formfields = $("<div class=\"config\" id=\"newznab" + intId + "\"><div class=\"row\"><label>Newznab Host</label><input type=\"text\" name=\"newznab_host" + intId + "\" size=\"30\"></div><div class=\"row\"><label>Newznab API</label><input type=\"text\" name=\"newznab_api" + intId + "\" size=\"36\"></div><div class=\"row checkbox\"><input type=\"checkbox\" name=\"newznab_enabled" + intId + "\" value=\"1\" checked /><label>Enabled</label></div>");
-				var removeButton = $("<div class=\"row\"><input type=\"button\" class=\"remove\" value=\"Remove\" /></div>");
+				var intId = $("#newznab-dialog > table").size() + deletedNewznabs + 1;
+				alert(intId);
+				var rowStart = "<tr><td>";
+				var rowEnd = "</td></tr>";
+				var formfields = $("<table id='newznab" + intId + "'>" +
+								rowStart +
+								"<label>Newznab Host</label>" +
+								"<input type='text' name='newznab_host" + intId + "' size='30'>" +
+								rowEnd +
+								rowStart +
+										"<label>Newznab API</label>" +
+										"<input type='text' name='newznab_api" + intId + "' size='36'>" +
+								rowEnd +
+								rowStart +
+									"<input type='checkbox' name='newznab_enabled" + intId + "' value='1' checked />" + 
+									"<label>Enabled</label>" + 
+								rowEnd);
+				var removeButton = $(rowStart + "<input type='button' class='remove' value='Remove' />" + rowEnd);
 				removeButton.click(function() {
 				$(this).parent().remove();
-					deletedNewznabs = deletedNewznabs + 1;
-				
+					deletedNewznabs = deletedNewznabs - 1;
 				});
 				formfields.append(removeButton);
-				formfields.append("</div>");
-				$("#add_newznab").after(formfields);
+				formfields.append("</table>");
+				$("#add_newznab").before(formfields);
 			});
 		
 			//Mirrors

data/interfaces/brink/css/style.css

@@ -159,6 +159,7 @@ ul#nav li a{
 	color: rgba(0,0,0,0.9);
 	text-shadow: 1px 1px rgba(0,145,255,0.2);
 }
+
 /* *
  * Sub navigation
  */
@@ -223,6 +224,11 @@ ul#nav-sub > li:hover > a {
 	border-bottom: 2px solid rgb(0,145,255);
 	cursor: pointer;
 }
+ul#nav-sub > li > a.active {
+	color: rgb(0,145,255);
+	border-bottom: 2px solid rgb(0,145,255);
+	cursor: pointer;
+}
  
 ul#content-container{
 	overflow: hidden;
@@ -248,11 +254,12 @@ input, textarea, select, option {
 	appearance:none;
 	-moz-appearance:none; /* Firefox */
 	-webkit-appearance:none; /* Safari and Chrome */
+	-0-appearance:none; /* Safari and Chrome */
 	padding-left: 5px;
 	padding-right: 5px;
 	font-size: .8em;
 	color: rgba(255,255,255,0.6);
-	line-height: 25px;
+	line-height: 2em;
 	font-weight: bold;
 	margin: 0;
 }
@@ -279,6 +286,8 @@ input[type="button"],
 input[type="submit"]{
 	background-color: rgba(0,0,0,.1);
 	border: 1px solid rgba(0,0,0,.3);
+	height: 2.3em;
+	display: inline;
 }
 select:hover,
 input[type="button"]:hover, 
@@ -288,6 +297,7 @@ input[type="submit"]:hover{
 }
 select option{
 	background: rgb(31,31,31);
+	border: 0;
 }
 select option:hover{
 	background: rgb(255,255,255);
@@ -335,8 +345,17 @@ input[type="checkbox"]:checked:after{
 /*config*/
 div#config_wrapper { margin: auto; width: 640px;}
 div#config_wrapper div table { width: 100%; border-collapse:separate; border-spacing: 10px; }
-div#config_wrapper div table tr td:first-child{ width: 240px; }
+div#config_wrapper div table tr td:first-child{ width: 50%; }
 div#config_wrapper div table tr td{ vertical-align: middle; }
+div.group {
+	padding: 5px;
+	background-color: rgba(0,0,0,0.1);
+	border-top: 1px solid rgba(0,0,0,0.1);
+	border-left: 1px solid rgba(0,0,0,0.1);
+	border-bottom: 1px solid rgba(255,255,255,0.1);
+	border-right: 1px solid rgba(255,255,255,0.1);
+	display: block;
+}
 /*end config*/
 
 /* Artist Table Style*/
@@ -424,35 +443,38 @@ table#album_table td#albumart { vertical-align: middle; text-align: left; }
 	height: 126px;
 	width: 126px;
 	position: relative;
-	box-shadow: 0 0 6px #000;
-	-moz-box-shadow: 0 0 6px #000;
-	-webkit-box-shadow: 0 0 6px #000;
-	overflow: hidden;
 }
-.image-container  .image-box img { 
+.image-container .image-box img { 
 	width: 100%;
-	min-height: 100%;
+	height: 100%;
 	position: absolute;
 	top: 0px;
 	left: 0px;
 	z-index: 1;
 	border: 0;
+	box-shadow: 0px 0px 10px rgb(0,0,0);
+	-moz-box-shadow: 0px 0px 10px rgb(0,0,0);
+	-webkit-box-shadow: 0px 0px 10px rgb(0,0,0);
+	-o-box-shadow: 0px 0px 10px rgb(0,0,0);
 }
 
-.image-container  input {
+.image-container .image-box input {
 	position: absolute;
 	top: 0px;
-	z-index: 2;
 	width: 100%;
 	height: 100%;
-	text-align: right; 
+	text-align: center;
 }
-.image-container  input:checked {
-	content: "";
-	background: rgba(0,0,0,0.1);
-	box-shadow: 0 0 10px rgb(0,145,255);
-	-moz-box-shadow: 0 0 10px rgb(0,145,255);
-	-webkit-box-shadow: 0 0 10px rgb(0,145,255);
+.image-container .image-box input:checked {
+	box-shadow: 0px 0px 10px rgb(0,145,255);
+	-moz-box-shadow: 0px 0px 10px rgb(0,145,255);
+	-webkit-box-shadow: 0px 0px 10px rgb(0,145,255);
+	-0-box-shadow: 0px 0px 10px rgb(0,145,255);
+}
+.image-container .image-box .image-count {
+	width: 100%;
+	height: 100%;
+	font-size: 10em;
 }
 .image-container .image-actions {
 	display: none;

data/interfaces/brink/extras.html

@@ -2,12 +2,24 @@
 <%def name="body()">
 	<div class="table_wrapper">
 		<h1>Suggestions</h1>
-		<div class="cloudtag">
-			<ul id="cloud">
 			%for artist in cloudlist:
-				<li><a href="addArtist?artistid=${artist['ArtistID']}" class="tag${artist['Count']}">${artist['ArtistName']}</a></li>
+				<div class="image-container">
+					<div class="image-box">
+						<div class="image-count">${artist['Count']}</div>
+						<img />
+						<div class="image-actions">
+							%if artist['Status'] == 'Loading':
+								<a href="addArtist?artistid=${artist['ArtistID']}"><span class="wsr Like"></span></a>
+							%else:
+								${havetracks}/${totaltracks}
+							%endif	
+						</div>
+					</div>
+					<div class="image-info">
+						<b title="${artist['ArtistID']}"><a href="artistPage?ArtistID=${artist['ArtistID']}">${artist['ArtistName']}</a></b>
+					</div>
+				</div>
 			%endfor
-			</ul>
-		</div>
+			
 	</div>
 </%def>

data/interfaces/brink/js/configscript.js

@@ -43,15 +43,15 @@ $(document).ready(function(){
 	$( '.switch' ).each(function(){
 		//set Label name to current active state
 		var labelName = 'label[for="' + $( this ).attr( "name" ) + '"]';
-		var switchId = $( this ).attr( "name" );
+		var switchId = $( this ).attr( "name" ) + "_switch";
 		if ( $( this ).is(':checked') ){
 			$( labelName ).text( 'Enabled' );
-			$("#" + switchId ).show();
+			$("#" + switchId ).slideDown();
 			enit();
 		}
 		else {
 			$( labelName ).text( 'Disabled' );
-			$("#" + switchId ).hide();
+			$("#" + switchId ).slideUp();
 			enit();
 		}
 		//Change label
@@ -59,12 +59,12 @@ $(document).ready(function(){
 			if ( $( this ).is( ':checked' ) ){
 				
 				$( labelName ).text( 'Enabled' );
-				$("#" + switchId ).show();
+				$("#" + switchId ).slideDown();
 				enit();
 			}
 			else{
 				$( labelName ).text( 'Disabled' );
-				$("#" + switchId ).hide();
+				$("#" + switchId ).slideUp();
 				enit();
 			}
 		});

data/interfaces/brink/js/script.js

@@ -15,7 +15,7 @@ function getArtistInfo(name,imgElem,size,artistID) {
 			} else {
 				if ( data.artist === undefined || imageUrl == "" || imageUrl == undefined ) {
 					var imageLarge = "#";
-					var imageUrl = "interfaces/brink/images/no-cover-artist.png";
+					var imageUrl = "interfaces/brink/images/no-artist-art.png";
 				} else {
 					var artist = data.artist.mbid;
 					var artistBio = data.artist.bio.summary;
@@ -42,7 +42,7 @@ function getArtistInfo(name,imgElem,size,artistID) {
 				}
 				if ( data.artist === undefined || imageUrl == "" ) {
 					var imageLarge = "#";
-					var imageUrl = "interfaces/brink/images/no-cover-artist.png";
+					var imageUrl = "interfaces/brink/images/no-artist-art.png";
 				} else {
 					var artist = data.artist.name;
 					var artistBio = data.artist.bio.summary;
@@ -101,7 +101,7 @@ function getAlbumInfo(name, album, elem,size) {
 					$(elem).css("background", "url("+ imageUrl+")");
 				} 
 				$(elem).css("background", "url("+ imageUrl+") center top no-repeat");
-				$(elem).wrap('<a href="'+ imageLarge +'" rel="dialog" title="' + name + '"></a>');
+				//$(elem).wrap('<a href="'+ imageLarge +'" rel="dialog" title="' + name + '"></a>');
 			}
 		});
 	}

data/interfaces/brink/manage.html

@@ -7,7 +7,7 @@
 <%def name="body()">
 	<div id="nav-sub-container">
 		<ul id="nav-sub">
-			<li>Manage: </li>
+			<li><a class="active" href="manage">Manage</a></li>
 			<li><a href="manageAlbums">Albums</a></li>
 			<li><a href="manageArtists">Artists</a></li>
 			%if not headphones.ADD_ARTISTS:

data/interfaces/brink/managealbums.html

@@ -6,10 +6,17 @@
 
 
 <%def name="body()">
-	<div class="table_wrapper">
-	<div id="manageheader" class="title">
-		<h1 class="clearfix"><img src="interfaces/default/images/icon_manage.png" alt="manage"/>Manage Albums</h1>
+	<div id="nav-sub-container">
+		<ul id="nav-sub">
+			<li><a href="manage">Manage</a></li>
+			<li><a class="active" href="manageAlbums">Albums</a></li>
+			<li><a href="manageArtists">Artists</a></li>
+			%if not headphones.ADD_ARTISTS:
+			<li><a href="manageNew">New Artists</a></li>
+			%endif
+		</ul>
 	</div>
+	<div class="table_wrapper">
 	<form action="markAlbums" method="get" id="markAlbums">
 	<div id="markalbum">Mark selected albums as 
 		<select name="action" onChange="doAjaxCall('markAlbums',$(this),'table',true);" data-error="You didn't select any albums">

data/interfaces/brink/manageartists.html

@@ -1,8 +1,19 @@
 <%inherit file="base.html" />
+<%!
+	from headphones import db
+	import headphones
+%>
 
 <%def name="body()">
 	<div id="nav-sub-container">
-		<h1>Manage Artists<h1>
+		<ul id="nav-sub">
+			<li><a href="manage">Manage</a></li>
+			<li><a href="manageAlbums">Albums</a></li>
+			<li><a class="active" href="manageArtists">Artists</a></li>
+			%if not headphones.ADD_ARTISTS:
+			<li><a href="manageNew">New Artists</a></li>
+			%endif
+		</ul>
 	</div>
 	<form action="markArtists" method="get">
 	<p class="indented">
@@ -52,10 +63,12 @@
 					<span class="wsr Clock"></span>
 				%elif artist['Status'] == 'Loading':
 					<span class="wsr Loading loader"></span>
-				%else: 
+				%elif artist['Status'] == 'Active':
 					<span class="wsr Approved"></span>
+				%else: 
+					<span class="wsr Alert"></span>
 				%endif
-				
+				${artist['Status']}
 				</td>	
 				<td id="name"><span title="${artist['ArtistSortName']}"></span><a href="artistPage?ArtistID=${artist['ArtistID']}">${artist['ArtistName']}</a></td>
 				<td id="album"><span title="${releasedate}"></span><a href="albumPage?AlbumID=${artist['AlbumID']}">${albumdisplay}</a></td>

data/interfaces/classic/config.html

@@ -342,6 +342,8 @@
 						</select>
 						<br><br>
                     	<h3>Log Directory:</h3><input type="text" name="log_dir" value="${config['log_dir']}" size="50">
+						<br><br>
+                    	<h3>Cache Directory:</h3><input type="text" name="cache_dir" value="${config['cache_dir']}" size="50">
                 </td>
             </tr>
             <tr>

data/interfaces/default/album.html

@@ -51,7 +51,7 @@
 	<div class="table_wrapper">
 		<div id="albumheader" class="clearfix">	
 			<div id="albumImg">
-				<img height="200" alt="" class="albumArt">
+				<img height="200" alt="" class="albumArt" src="/artwork/album/${album['AlbumID']}">
 			</div>
 			
 			<h1><a href="http://musicbrainz.org/release-group/${album['AlbumID']}">${album['AlbumTitle']}</a></h1>
@@ -151,17 +151,6 @@
 <%def name="javascriptIncludes()">
 	<script src="js/libs/jquery.dataTables.min.js"></script>
 	<script>
-	
-	function getAlbumArt() {
-		var id = "${album['AlbumID']}";
-		var name = "${album['AlbumTitle']}";
-		var image = $("div#albumImg img");
-		if ( !image.hasClass('done') ) {
-			image.addClass('done');
-			getArtwork(image,id,name,'album');
-		}
-	}
-	
 	function getAlbumInfo() {
 		var id = "${album['AlbumID']}";
 		var elem = $("#albumInfo");
@@ -192,7 +181,6 @@
 	
 	$(document).ready(function()	{
 		getAlbumInfo();
-		getAlbumArt();
 		initThisPage();
 	});
 

data/interfaces/default/artist.html

@@ -40,7 +40,7 @@
 <%def name="body()">
 	<div id="artistheader" class="clearfix">
 		<div id="artistImg">
-			<img id="artistImage" class="albumArt" alt="" />
+			<img id="artistImage" class="albumArt" alt="" src="artwork/artist/${artist['ArtistID']}"/>
 		</div>
 		<h1>	
 			%if artist['Status'] == 'Loading':	
@@ -122,7 +122,7 @@
 			%>
 			<tr class="grade${grade}">
 				<td id="select"><input type="checkbox" name="${album['AlbumID']}" class="checkbox" /></td>
-				<td id="albumart"><img class="albumArt" id="${album['AlbumID']}" height="64" width="64"></td>
+				<td id="albumart"><img class="albumArt" id="${album['AlbumID']}" src="/artwork/thumbs/album/${album['AlbumID']}" height="64" width="64"></td>
 				<td id="albumname"><a href="albumPage?AlbumID=${album['AlbumID']}">${album['AlbumTitle']}</a></td>
 				<td id="reldate">${album['ReleaseDate']}</td>
 				<td id="type">${album['Type']}</td>
@@ -161,28 +161,6 @@
 	<script src="js/libs/jquery.dataTables.min.js"></script>
 	
 	<script>
-	
-	function getArtistArt() {
-		var id = "${artist['ArtistID']}";
-		var name = "${artist['ArtistName']}";
-		var image = $("div#artistImg img#artistImage");
-		if ( !image.hasClass('done') ) {
-			image.addClass('done');
-			getArtwork(image,id,name,'artist');
-		}
-	}
-	
-	function getAlbumArt() {
-		$("table#album_table tr td#albumart").each(function(){	
-			var id = $(this).children('img').attr('id');
-			var image = $(this).children('img');
-			if ( !image.hasClass('done') ) {
-				image.addClass('done');
-				getThumb(image,id,'album');
-			}
-		});
-	}
-	
 	function getArtistBio() {
 		var id = "${artist['ArtistID']}";
 		var elem = $("#artistBio");
@@ -202,8 +180,6 @@
 		%if artist['Status'] == 'Loading':
 			showMsg("Getting artist information",true);
 		%endif
-		getArtistArt();
-		getAlbumArt();
 		$('#album_table').dataTable({
 			"bDestroy": true,
 			"aoColumns": [

data/interfaces/default/base.html

@@ -36,7 +36,7 @@
             </div>
             % elif headphones.CURRENT_VERSION != headphones.LATEST_VERSION and headphones.INSTALL_TYPE != 'win':
             <div id="updatebar">
-                A <a href="http://github.com/rembo10/headphones/compare/${headphones.CURRENT_VERSION}...${headphones.LATEST_VERSION}"> newer version</a> is available. You're ${headphones.COMMITS_BEHIND} commits behind. <a href="update">Update</a> or <a href="#" onclick="$('#updatebar').slideUp('slow');">Close</a>
+                A <a href="https://github.com/AdeHub/headphones/compare/${headphones.CURRENT_VERSION}...${headphones.LATEST_VERSION}"> newer version</a> is available. You're ${headphones.COMMITS_BEHIND} commits behind. <a href="update">Update</a> or <a href="#" onclick="$('#updatebar').slideUp('slow');">Close</a>
             </div>
 			% endif
 	

data/interfaces/default/config.html

@@ -1,4 +1,4 @@
-m<%inherit file="base.html"/>
+<%inherit file="base.html"/>
 <%!
 	import headphones
 	import string
@@ -302,6 +302,32 @@ m<%inherit file="base.html"/>
                                 <input type="text" name="waffles_passkey" value="${config['waffles_passkey']}" size="36">
                             </div>
                         </div>
+                        <div class="row checkbox">
+                            <input id="userutracker" type="checkbox" name="rutracker" onclick="initConfigCheckbox($(this));" value="1" ${config['use_rutracker']} /><label>rutracker.org</label>
+                        </div>
+                        <div class="config">
+                            <div class="row">
+                                <label>rutracker User Name: </label>
+                                <input type="text" name="rutracker_user" value="${config['rutracker_user']}" size="36">
+                            </div>
+                            <div class="row">
+                                <label>rutracker Password: </label>
+                                <input type="password" name="rutracker_password" value="${config['rutracker_password']}" size="36">
+			    </div>
+			</div>
+                        <div class="row checkbox">    
+			    <input id="usewhatcd" type="checkbox" name="whatcd" onclick="initConfigCheckbox($(this));" value="1" ${config['use_whatcd']} /><label>What.cd</label>
+                        </div>
+                        <div class="config">
+                            <div class="row">
+                                <label>What.cd Username: </label>
+                                <input type="text" name="whatcd_username" value="${config['whatcd_username']}" size="36">
+                            </div>
+                            <div class="row">
+                                <label>What.cd Password: </label>
+                                <input type="password" name="whatcd_password" value="${config['whatcd_password']}" size="36">
+                            </div>
+                        </div>
                     </fieldset>
                     
                 </td>
@@ -538,6 +564,10 @@ m<%inherit file="base.html"/>
 		        				<input type="checkbox" name="autowant_all" value="1" ${config['autowant_all']} /><label>Automatically Mark All Albums as Wanted</label>
 		        			</div>
 		        		</div>                		
+                        <div class="row">
+							<label>Cache Size (in MB):</label>
+							<input type="text" name="cache_sizemb" value="${config['cache_sizemb']}" size="7">
+						</div> 
 		        	</fieldset>
 					<fieldset>
 						<legend>Interface</legend>
@@ -560,6 +590,10 @@ m<%inherit file="base.html"/>
 							<label>Log Directory:</label>
 							<input type="text" name="log_dir" value="${config['log_dir']}" size="50">
 						</div>
+						<div class="row">
+							<label>Cache Directory:</label>
+							<input type="text" name="cache_dir" value="${config['cache_dir']}" size="50">
+						</div>
 					</fieldset>
 						            	
                                         
@@ -926,6 +960,8 @@ m<%inherit file="base.html"/>
 		initConfigCheckbox("#usenewzbin");
 		initConfigCheckbox("#usenzbsorg");
 		initConfigCheckbox("#usewaffles");
+		initConfigCheckbox("#userutracker");
+		initConfigCheckbox("#usewhatcd");
 		initConfigCheckbox("#useblackhole");
 		initConfigCheckbox("#useapi");
 	}

data/interfaces/default/css/data_table.css

@@ -12,16 +12,16 @@
 	position: absolute;
 	top: 50%;
 	left: 50%;
-	width: 20px;
 	height: 30px;
 	margin-left: -125px;
 	margin-top: -15px;
 	padding: 14px 0 2px 0;
-	border: 1px solid #ddd;
+	border: 2px solid #C8C8C8;
 	text-align: center;
 	color: #999;
 	font-size: 14px;
-	background-color: white;
+	color: #323232;
+	background-color: #E1E1E1;
 }
 
 .dataTables_length {
@@ -340,8 +340,7 @@ td.details {
 	line-height: 22px;
 }
 
-.paging_full_numbers span.paginate_button,
- 	.paging_full_numbers span.paginate_active {
+.paginate_button,.paginate_active {
 	background: none repeat scroll 0 0 #F3F3F3;
 	   border-radius: 5px 5px 5px 5px;
 	   margin: 0 0 0 5px;
@@ -352,11 +351,11 @@ td.details {
 	*cursor: hand;
 }
 
-.paging_full_numbers span.paginate_button:hover {
+.paginate_button:hover {
 	background-color: #e2e2e2;
 }
 
-.paging_full_numbers span.paginate_active {
+.paginate_active {
 	background-color: #4183C4;
 	color: #FFF;
 }

data/interfaces/default/css/style.css

@@ -871,17 +871,24 @@ div#artistheader h2 a {
 #artist_table th#name {
   min-width: 200px;
   text-align: left;
+  width:200px;
 }
 #artist_table th#album {
   min-width: 300px;
   text-align: left;
 }
+#artist_table th#albumart,
+#artist_table th#status{
+    width:50px;
+}
+
 #artist_table th#status,
 #artist_table th#albumart,
 #artist_table th#lastupdated {
   min-width: 50px;
   text-align: left;
 }
+
 #artist_table th#have {
   text-align: center;
 }
@@ -896,6 +903,7 @@ div#artistheader h2 a {
   text-align: left;
   vertical-align: middle;
 }
+
 #artist_table td#album {
   min-width: 300px;
   text-align: left;

data/interfaces/default/history.html

@@ -45,6 +45,8 @@
 					fileid = 'nzb'
 				if item['URL'].find('torrent') != -1:
 					fileid = 'torrent'
+				if item['URL'].find('rutracker') != -1:
+					fileid = 'torrent'
 			%>
 			<tr class="grade${grade}">
 				<td id="dateadded">${item['DateAdded']}</td>

data/interfaces/default/index.html

@@ -15,51 +15,6 @@
 			</tr>
 		</thead>
 		<tbody>
-		%for artist in artists:
-			<%
-				totaltracks = artist['TotalTracks']
-				havetracks = artist['HaveTracks']
-				if not havetracks:
-					havetracks = 0
-				try:
-					percent = (havetracks*100.0)/totaltracks
-					if percent > 100:
-						percent = 100
-				except (ZeroDivisionError, TypeError):
-					percent = 0
-					totaltracks = '?'
-				
-				if artist['ReleaseDate'] and artist['LatestAlbum']:
-					releasedate = artist['ReleaseDate']
-					albumdisplay = '<i>%s</i> (%s)' % (artist['LatestAlbum'], artist['ReleaseDate'])
-					if releasedate > helpers.today():
-						grade = 'A'
-					else:
-						grade = 'Z'
-				elif artist['LatestAlbum']:
-					releasedate = ''
-					grade = 'Z'
-					albumdisplay = '<i>%s</i>' % artist['LatestAlbum']
-				else:
-					releasedate = ''
-					grade = 'Z'
-					albumdisplay = '<i>None</i>'
-				
-				if artist['Status'] == 'Paused':
-					grade = 'X'
-
-				if artist['Status'] == 'Loading':
-					grade = 'L'
-				
-			%>
-			<tr class="grade${grade}">
-				<td id="albumart"><div id="artistImg"><img class="albumArt" alt="" id="${artist['ArtistID']}"/></div></td>
-				<td id="name"><span title="${artist['ArtistSortName']}"></span><a href="artistPage?ArtistID=${artist['ArtistID']}">${artist['ArtistName']}</a></td>
-				<td id="status">${artist['Status']}</td>
-				<td id="album"><span title="${releasedate}"></span><a href="albumPage?AlbumID=${artist['AlbumID']}">${albumdisplay}</a></td>
-				<td id="have"><span title="${percent}"></span><div class="progress-container"><div style="width:${percent}%"><div class="havetracks">${havetracks}/${totaltracks}</div></div></div></td>
-			</tr>
-		%endfor
 		</tbody>
 	</table>
 </%def>
@@ -71,36 +26,116 @@
 <%def name="javascriptIncludes()">
 	<script src="js/libs/jquery.dataTables.min.js"></script>
 	<script>
-	function getArtistArt() {
-		$("table#artist_table tr td#albumart #artistImg").each(function(){	
-			var id = $(this).children('img').attr('id');
-			var image = $(this).children('img');
-			if ( !image.hasClass('done') ) {
-				image.addClass('done');
-				getThumb(image,id,'artist');
-			}
-		});
-	}
 	function initThisPage() {
-		getArtistArt();
 		$('#artist_table').dataTable({
 			"bDestroy": true,
 			"aoColumnDefs": [
-			      { 'bSortable': false, 'aTargets': [ 0 ] }
-			],
-			"aoColumns": [
-				null,
-				{ "sType": "title-string"},
-				null,
-				{ "sType": "title-string"},
-				{ "sType": "title-numeric"}
-			],
+			      { 
+                    "bSortable": false, 
+                    "aTargets": [ 0 ],
+                    "mData":"ArtistID",
+                    "mRender": function ( data, type, full ) {
+                        return '<div id="artistImg"><img class="albumArt" alt="" id="'+ data + '" src="/artwork/thumbs/artist/' + data + '"/></div>';
+                    }
+                   },
+                   {
+                   "aTargets":[1],
+                   "mDataProp":"ArtistSortName",
+                   "mRender":function (data,type,full) {
+                        
+                        return '<span title="' + full['ArtistID'] + '"></span><a href="artistPage?ArtistID=' + full['ArtistID'] + '">' + data + '</a>'
+                        }
+                   },
+                   {"aTargets":[2],"mDataProp":"Status"},
+                   {
+                   "aTargets":[3],
+                   "mDataProp":"LatestAlbum",
+                   "mRender":function(data,type,full){
+                        artist = full;
+                        if (artist['ReleaseDate']  && artist['LatestAlbum'])
+                        {
+                            releasedate = artist['ReleaseDate'];
+                            albumdisplay = '<i>' + artist['LatestAlbum'] + '</i> (' + artist['ReleaseDate'] + ')';
+                        }
+                        else if(artist['LatestAlbum'])
+                        {
+                            releasedate = '';
+                            albumdisplay = '<i>' + artist['LatestAlbum'] + '</i>';
+                        }
+                        else
+                        {
+                            releasedate = '';
+                            albumdisplay = '<i>None</i>';
+                        }
+                        if (artist['ReleaseInFuture'] === 'True')
+                        {
+                            grade = 'gradeA';
+                        }
+                        else
+                        {
+                            grade = 'gradeZ';
+                        }
+                        artist['Grade'] = grade;
+                        return '<span title="' + releasedate + '"></span><a href="albumPage?AlbumID=' + full['AlbumID'] + '">' + albumdisplay + '</a>'
+                   }
+                   
+                   
+                   },
+                   {
+                   "aTargets":[4],
+                   "mDataProp":"HaveTracks",
+                   "mRender":function(data,type,full){
+                        if(full['TotalTracks'] > 0)
+                        {
+                            percent = (full['HaveTracks']*100.0)/full['TotalTracks']
+                            if(percent > 100){
+                                percent = 100;
+                            }
+                        }
+                        else
+                        {
+                            full['TotalTracks'] = '?';
+                            percent = 0;
+                        }                      
+                        
+                        return '<span title="' + percent + '"></span><div class="progress-container"><div style="width:' + percent + '%"><div class="havetracks">' + full['HaveTracks'] + '/' + full['TotalTracks'] + '</div></div></div>';
+                   
+                   }},
+                   
+                 ],
+			
 			"oLanguage": {
-				"sSearch": ""},
+				"sSearch": "",
+                "sLengthMenu":"Show _MENU_ artists per page",
+					"sInfo":"Showing _START_ to _END_ of _TOTAL_ artists",
+					"sInfoEmpty":"Showing 0 to 0 of 0 artists",
+					"sInfoFiltered":"(filtered from _MAX_ total artists)"
+                    },
 			"bStateSave": true,
 			"iDisplayLength": 50,
-			"sPaginationType": "full_numbers"				
-		});
+			"sPaginationType": "full_numbers",
+            "bProcessing": true,
+            "bServerSide": true,
+            "sAjaxSource": 'getArtists.json',
+            "fnRowCallback": function( nRow, aData, iDisplayIndex, iDisplayIndexFull ){
+                                    $('td', nRow).closest('tr').addClass(aData['Grade'])
+                                    nRow.children[0].id = 'albumart';
+                                    nRow.children[1].id = 'name';
+                                    nRow.children[2].id = 'status'
+                                    nRow.children[3].id = 'album'
+                                    nRow.children[4].id = 'have'
+                                    return nRow;
+                                    },
+            "fnServerData": function ( sSource, aoData, fnCallback ) {
+                /* Add some extra data to the sender */
+                $.getJSON( sSource, aoData, function (json) {    fnCallback(json)                }  )
+                },
+            "fnInitComplete": function(oSettings, json) 
+            {
+            }});
+
+            
+
 		resetFilters("artist or album");
 	}
 	

data/interfaces/default/logs.html

@@ -16,23 +16,20 @@ lossless<%inherit file="base.html"/>
 			</tr>
 		</thead>
 		<tbody>
-		%for line in lineList:
-			<%
-				timestamp, message, level, threadname = line
-				
-				if level == 'WARNING' or level == 'ERROR':
-					grade = 'X'
-				else:
-					grade = 'Z'
-			%>
-				<tr class="grade${grade}">
-					<td id="timestamp">${timestamp}</td>
-					<td id="level">${level}</td>
-					<td id="message">${message}</td>
-				</tr>
-		%endfor
 		</tbody>
 	</table>
+	<br>
+	<div align="center">Refresh rate: 
+	<select id="refreshrate" onchange="setRefresh()">
+	<option value="0" selected="selected">No Refresh</option>
+	<option value="5">5 Seconds</option>
+	<option value="15">15 Seconds</option>
+	<option value="30">30 Seconds</option>
+	<option value="60">60 Seconds</option>
+	<option value="300">5 Minutes</option>
+	<option value="600">10 Minutes</option>
+	</select></div>
+	
 </%def>
 
 <%def name="headIncludes()">
@@ -42,22 +39,58 @@ lossless<%inherit file="base.html"/>
 <%def name="javascriptIncludes()">
 	<script src="js/libs/jquery.dataTables.min.js"></script>
 	<script>
-	$(document).ready(function()
-	{
-		$('#log_table').dataTable(
-			{
-				"oLanguage": {
+$(document).ready(function() {
+	$('#log_table').dataTable( {
+		"bProcessing": true,
+        "bServerSide": true,
+		"sAjaxSource": 'getLog',
+        "sPaginationType": "full_numbers",
+        "bStateSave": true,
+        "oLanguage": {
+                    "sSearch":"",
 					"sLengthMenu":"Show _MENU_ lines per page",
 					"sEmptyTable": "No log information available",
 					"sInfo":"Showing _START_ to _END_ of _TOTAL_ lines",
 					"sInfoEmpty":"Showing 0 to 0 of 0 lines",
 					"sInfoFiltered":"(filtered from _MAX_ total lines)"},
-				"bStateSave": true,
-				"iDisplayLength": 100,
-				"sPaginationType": "full_numbers",
-				"aaSorting": []
-
-			});
-	});
+        "fnRowCallback": function (nRow, aData, iDisplayIndex, iDisplayIndexFull) {
+                                if (aData[1] === "WARNING" || aData[1] === "ERROR")
+                                {
+                                    $('td', nRow).closest('tr').addClass("gradeX");
+                                }
+                                else
+                                {
+                                    $('td', nRow).closest('tr').addClass("gradeZ");
+                                }
+                                
+                                
+                                return nRow;
+                                },
+        "fnServerData": function ( sSource, aoData, fnCallback ) {
+			/* Add some extra data to the sender */
+			$.getJSON( sSource, aoData, function (json) { 
+				fnCallback(json)
+			} );
+		}
+	} );
+} );
+	</script>
+	<script>
+		var timer; 
+		function setRefresh()
+		{
+			refreshrate = document.getElementById('refreshrate');
+			if(refreshrate != null)
+			{
+				if(timer)
+				{
+					clearInterval(timer);
+				}
+				if(refreshrate.value != 0)
+				{
+					timer = setInterval("$('#log_table').dataTable().fnDraw()",1000*refreshrate.value);
+				}
+			}
+		}
 	</script>
 </%def>

data/interfaces/default/manage.html

@@ -97,6 +97,7 @@
 				<a href="#" onclick="doAjaxCall('forceUpdate',$(this))" data-success="Update active artists successful" data-error="Error forcing update artists"><span class="ui-icon ui-icon-heart"></span>Force Update Active Artists</a>
 				<a href="#" onclick="doAjaxCall('forcePostProcess',$(this))" data-success="Post-Processor is being loaded" data-error="Error during Post-Processing"><span class="ui-icon ui-icon-wrench"></span>Force Post-Process Albums in Download Folder</a>
 				<a href="#" onclick="doAjaxCall('checkGithub',$(this))" data-success="Checking for update successful" data-error="Error checking for update"><span class="ui-icon ui-icon-refresh"></span>Check for Headphones Updates</a>
+				<a href="#" onclick="doAjaxCall('deleteEmptyArtists',$(this))" data-success="Empty Artists deleted" data-error="Error deleting empty artists"><span class="ui-icon ui-icon-trash"></span>Delete empty Artists</a>
 			</div>
 		</fieldset>
 		

data/interfaces/default/manageartists.html

@@ -66,7 +66,7 @@
 			%>
 			<tr class="grade${grade}">
 				<td id="select"><input type="checkbox" name="${artist['ArtistID']}" class="checkbox" /></td>
-				<td id="albumart"><div id="artistImg"><img class="albumArt" id="${artist['ArtistID']}" height="50" width="50"></div></td>
+				<td id="albumart"><div id="artistImg"><img class="albumArt" id="${artist['ArtistID']}" src="/artwork/thumbs/artist/${artist['ArtistID']}" height="50" width="50"></div></td>
 				<td id="name"><span title="${artist['ArtistSortName']}"></span><a href="artistPage?ArtistID=${artist['ArtistID']}">${artist['ArtistName']}</a></td>
 				<td id="status">${artist['Status']}</td>	
 				<td id="album"><span title="${releasedate}"></span><a href="albumPage?AlbumID=${artist['AlbumID']}">${albumdisplay}</a></td>
@@ -86,19 +86,7 @@
 <%def name="javascriptIncludes()">
 	<script src="js/libs/jquery.dataTables.min.js"></script>
 	<script>
-	function getArtistArt() {
-		$("table#artist_table tr td#albumart #artistImg").each(function(){	
-			var id = $(this).children('img').attr('id');
-			var image = $(this).children('img');
-			if ( !image.hasClass('done') ) {
-				image.addClass('done');
-				getThumb(image,id,'artist');
-			}
-		});
-	}
-
 	function initThisPage() {
-		getArtistArt();
 		$('#artist_table').dataTable({	
 			"bDestroy":true,		
 			"aoColumns": [

data/interfaces/default/upcoming.html

@@ -40,7 +40,7 @@
 		%for album in wanted:
 			<tr class="gradeZ">
 				<td id="select"><input type="checkbox" name="${album['AlbumID']}" class="checkbox" /></th>
-				<td id="albumart"><img title="${album['AlbumID']}" height="64" width="64"></td>
+				<td id="albumart"><img title="${album['AlbumID']}" height="64" width="64" src="/artwork/thumbs/album/${album['AlbumID']}"></td>
 				<td id="artistname"><a href="artistPage?ArtistID=${album['ArtistID']}">${album['ArtistName']}</a></td>
 				<td id="albumname"><a href="albumPage?AlbumID=${album['AlbumID']}">${album['AlbumTitle']}</a></td>
 				<td id="reldate">${album['ReleaseDate']}</td>
@@ -70,7 +70,7 @@
 		<tbody>
 		%for album in upcoming:
 			<tr class="gradeZ">
-				<td id="albumart"><img title="${album['AlbumID']}" height="64" width="64"></td>
+				<td id="albumart"><img title="${album['AlbumID']}" height="64" width="64" src="/artwork/thumbs/album/${album['AlbumID']}"></td>
 				<td id="artistname"><a href="artistPage?ArtistID=${album['ArtistID']}">${album['ArtistName']}</a></td>
 				<td id="albumname"><a href="albumPage?AlbumID=${album['AlbumID']}">${album['AlbumTitle']}</a></td>
 				<td id="reldate">${album['ReleaseDate']}</td>
@@ -90,19 +90,8 @@
 <%def name="javascriptIncludes()">
 	<script src="js/libs/jquery.dataTables.min.js"></script>
 	<script>
-	
-	function getAlbumArt() {
-		$("td#albumart img").each(function(){	
-			var id = $(this).attr('title');
-			var image = $(this);
-			if ( !image.hasClass('done') ) {
-				image.addClass('done');
-				getThumb(image,id,'album');
-			}
-		});
-	}
 	function initThisPage() {
-		getAlbumArt();
+		
 		$('#wanted_table').dataTable({
 			"bDestroy":true,
 			"bFilter": false,

data/interfaces/remix/config.html

@@ -342,6 +342,8 @@
 						</select>
 						<br><br>
                     	<h3>Log Directory:</h3><input type="text" name="log_dir" value="${config['log_dir']}" size="50">
+						<br><br>
+                    	<h3>Cache Directory:</h3><input type="text" name="cache_dir" value="${config['cache_dir']}" size="50">
                 </td>
             </tr>
             <tr>

data/js/libs/jquery.dataTables.min.js

@@ -1,648 +1,155 @@
 /*
-* File:        jquery.dataTables.min.js
-* Version:     1.8.1
-* Author:      Allan Jardine (www.sprymedia.co.uk)
-* Info:        www.datatables.net
-* 
-* Copyright 2008-2011 Allan Jardine, all rights reserved.
-*
-* This source file is free software, under either the GPL v2 license or a
-* BSD style license, as supplied with this software.
-* 
-* This source file is distributed in the hope that it will be useful, but 
-* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 
-* or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
-*/
-(function (i, wa, p) {
-    i.fn.dataTableSettings = [];
-    var D = i.fn.dataTableSettings; i.fn.dataTableExt = {};
-    var o = i.fn.dataTableExt;
-    o.sVersion = "1.8.1";
-    o.sErrMode = "alert";
-    o.iApiIndex = 0; o.oApi = {};
-    o.afnFiltering = [];
-    o.aoFeatures = [];
-    o.ofnSearch = {};
-    o.afnSortData = [];
-    o.oStdClasses = {
-        sPagePrevEnabled: "paginate_enabled_previous",
-        sPagePrevDisabled: "paginate_disabled_previous",
-        sPageNextEnabled: "paginate_enabled_next",
-        sPageNextDisabled: "paginate_disabled_next",
-        sPageJUINext: "", sPageJUIPrev: "",
-        sPageButton: "paginate_button",
-        sPageButtonActive: "paginate_active",
-        sPageButtonStaticDisabled: "paginate_button paginate_button_disabled",
-        sPageFirst: "first",
-        sPagePrevious: "previous",
-        sPageNext: "next",
-        sPageLast: "last",
-        sStripOdd: "odd",
-        sStripEven: "even",
-        sRowEmpty: "dataTables_empty",
-        sWrapper: "dataTables_wrapper",
-        sFilter: "dataTables_filter",
-        sInfo: "dataTables_info",
-        sPaging: "dataTables_paginate paging_",
-        sLength: "dataTables_length",
-        sProcessing: "dataTables_processing",
-        sSortAsc: "sorting_asc",
-        sSortDesc: "sorting_desc",
-        sSortable: "sorting",
-        sSortableAsc: "sorting_asc_disabled",
-        sSortableDesc: "sorting_desc_disabled",
-        sSortableNone: "sorting_disabled",
-        sSortColumn: "sorting_",
-        sSortJUIAsc: "",
-        sSortJUIDesc: "",
-        sSortJUI: "",
-        sSortJUIAscAllowed: "",
-        sSortJUIDescAllowed: "", sSortJUIWrapper: "", sSortIcon: "", sScrollWrapper: "dataTables_scroll",
-        sScrollHead: "dataTables_scrollHead",
-        sScrollHeadInner: "dataTables_scrollHeadInner",
-        sScrollBody: "dataTables_scrollBody",
-        sScrollFoot: "dataTables_scrollFoot",
-        sScrollFootInner: "dataTables_scrollFootInner",
-        sFooterTH: ""
-    }; o.oJUIClasses = { sPagePrevEnabled: "fg-button ui-button ui-state-default ui-corner-left",
-        sPagePrevDisabled: "fg-button ui-button ui-state-default ui-corner-left ui-state-disabled",
-        sPageNextEnabled: "fg-button ui-button ui-state-default ui-corner-right",
-        sPageNextDisabled: "fg-button ui-button ui-state-default ui-corner-right ui-state-disabled",
-        sPageJUINext: "ui-icon ui-icon-circle-arrow-e",
-        sPageJUIPrev: "ui-icon ui-icon-circle-arrow-w",
-        sPageButton: "fg-button ui-button ui-state-default",
-        sPageButtonActive: "fg-button ui-button ui-state-default ui-state-disabled",
-        sPageButtonStaticDisabled: "fg-button ui-button ui-state-default ui-state-disabled",
-        sPageFirst: "first ui-corner-tl ui-corner-bl",
-        sPagePrevious: "previous",
-        sPageNext: "next",
-        sPageLast: "last ui-corner-tr ui-corner-br",
-        sStripOdd: "odd",
-        sStripEven: "even",
-        sRowEmpty: "dataTables_empty",
-        sWrapper: "dataTables_wrapper",
-        sFilter: "dataTables_filter",
-        sInfo: "dataTables_info",
-        sPaging: "dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",
-        sLength: "dataTables_length",
-        sProcessing: "dataTables_processing",
-        sSortAsc: "ui-state-default",
-        sSortDesc: "ui-state-default",
-        sSortable: "ui-state-default",
-        sSortableAsc: "ui-state-default",
-        sSortableDesc: "ui-state-default",
-        sSortableNone: "ui-state-default",
-        sSortColumn: "sorting_",
-        sSortJUIAsc: "css_right ui-icon ui-icon-triangle-1-n",
-        sSortJUIDesc: "css_right ui-icon ui-icon-triangle-1-s",
-        sSortJUI: "css_right ui-icon ui-icon-carat-2-n-s",
-        sSortJUIAscAllowed: "css_right ui-icon ui-icon-carat-1-n",
-        sSortJUIDescAllowed: "css_right ui-icon ui-icon-carat-1-s",
-        sSortJUIWrapper: "DataTables_sort_wrapper",
-        sSortIcon: "DataTables_sort_icon",
-        sScrollWrapper: "dataTables_scroll",
-        sScrollHead: "dataTables_scrollHead ui-state-default",
-        sScrollHeadInner: "dataTables_scrollHeadInner",
-        sScrollBody: "dataTables_scrollBody",
-        sScrollFoot: "dataTables_scrollFoot ui-state-default",
-        sScrollFootInner: "dataTables_scrollFootInner",
-        sFooterTH: "ui-state-default"
-    }; o.oPagination = { two_button: { fnInit: function (g, l, r) {
-        var s, w, y; if (g.bJUI) {
-            s = p.createElement("a"); w = p.createElement("a"); y = p.createElement("span"); y.className = g.oClasses.sPageJUINext; w.appendChild(y); y = p.createElement("span"); y.className = g.oClasses.sPageJUIPrev;
-            s.appendChild(y)
-        }
-        else { s = p.createElement("div"); w = p.createElement("div") } s.className = g.oClasses.sPagePrevDisabled; w.className = g.oClasses.sPageNextDisabled; s.title = g.oLanguage.oPaginate.sPrevious; w.title = g.oLanguage.oPaginate.sNext; l.appendChild(s); l.appendChild(w); i(s).bind("click.DT", function () { g.oApi._fnPageChange(g, "previous") && r(g) }); i(w).bind("click.DT", function () { g.oApi._fnPageChange(g, "next") && r(g) }); i(s).bind("selectstart.DT", function () { return false }); i(w).bind("selectstart.DT", function () { return false });
-
-        if (g.sTableId !== "" && typeof g.aanFeatures.p == "undefined") {
-            l.setAttribute("id", g.sTableId + "_paginate");
-            s.setAttribute("id", g.sTableId + "_previous");
-            w.setAttribute("id", g.sTableId + "_next")
-        }
-    }, fnUpdate: function (g) {
-        if (g.aanFeatures.p) for (var l = g.aanFeatures.p, r = 0, s = l.length; r < s; r++)
-            if (l[r].childNodes.length !== 0) {
-                l[r].childNodes[0].className = g._iDisplayStart === 0 ? g.oClasses.sPagePrevDisabled : g.oClasses.sPagePrevEnabled; l[r].childNodes[1].className = g.fnDisplayEnd() == g.fnRecordsDisplay() ? g.oClasses.sPageNextDisabled :
-g.oClasses.sPageNextEnabled
-            }
-    }
-    }, iFullNumbersShowPages: 5, full_numbers: { fnInit: function (g, l, r) {
-        var s = p.createElement("span"), w = p.createElement("span"), y = p.createElement("span"), G = p.createElement("span"), x = p.createElement("span"); s.innerHTML = g.oLanguage.oPaginate.sFirst; w.innerHTML = g.oLanguage.oPaginate.sPrevious; G.innerHTML = g.oLanguage.oPaginate.sNext; x.innerHTML = g.oLanguage.oPaginate.sLast; var v = g.oClasses; s.className = v.sPageButton + " " + v.sPageFirst; w.className = v.sPageButton + " " + v.sPagePrevious; G.className =
-v.sPageButton + " " + v.sPageNext; x.className = v.sPageButton + " " + v.sPageLast; l.appendChild(s); l.appendChild(w); l.appendChild(y); l.appendChild(G); l.appendChild(x); i(s).bind("click.DT", function () { g.oApi._fnPageChange(g, "first") && r(g) }); i(w).bind("click.DT", function () { g.oApi._fnPageChange(g, "previous") && r(g) }); i(G).bind("click.DT", function () { g.oApi._fnPageChange(g, "next") && r(g) }); i(x).bind("click.DT", function () { g.oApi._fnPageChange(g, "last") && r(g) }); i("span", l).bind("mousedown.DT", function () { return false }).bind("selectstart.DT",
-
-function () { return false }); if (g.sTableId !== "" && typeof g.aanFeatures.p == "undefined") { l.setAttribute("id", g.sTableId + "_paginate"); s.setAttribute("id", g.sTableId + "_first"); w.setAttribute("id", g.sTableId + "_previous"); G.setAttribute("id", g.sTableId + "_next"); x.setAttribute("id", g.sTableId + "_last") }
-    }, fnUpdate: function (g, l) {
-        if (g.aanFeatures.p) {
-            var r = o.oPagination.iFullNumbersShowPages, s = Math.floor(r / 2), w = Math.ceil(g.fnRecordsDisplay() / g._iDisplayLength), y = Math.ceil(g._iDisplayStart / g._iDisplayLength) + 1, G =
-"", x, v = g.oClasses; if (w < r) { s = 1; x = w } else if (y <= s) { s = 1; x = r } else if (y >= w - s) { s = w - r + 1; x = w } else { s = y - Math.ceil(r / 2) + 1; x = s + r - 1 } for (r = s; r <= x; r++) G += y != r ? '<span class="' + v.sPageButton + '">' + r + "</span>" : '<span class="' + v.sPageButtonActive + '">' + r + "</span>"; x = g.aanFeatures.p; var z, Y = function (L) { g._iDisplayStart = (this.innerHTML * 1 - 1) * g._iDisplayLength; l(g); L.preventDefault() }, V = function () { return false }; r = 0; for (s = x.length; r < s; r++) if (x[r].childNodes.length !== 0) {
-                z = i("span:eq(2)", x[r]); z.html(G); i("span", z).bind("click.DT",
-Y).bind("mousedown.DT", V).bind("selectstart.DT", V); z = x[r].getElementsByTagName("span"); z = [z[0], z[1], z[z.length - 2], z[z.length - 1]]; i(z).removeClass(v.sPageButton + " " + v.sPageButtonActive + " " + v.sPageButtonStaticDisabled); if (y == 1) { z[0].className += " " + v.sPageButtonStaticDisabled; z[1].className += " " + v.sPageButtonStaticDisabled } else { z[0].className += " " + v.sPageButton; z[1].className += " " + v.sPageButton } if (w === 0 || y == w || g._iDisplayLength == -1) {
-                    z[2].className += " " + v.sPageButtonStaticDisabled; z[3].className += " " +
-v.sPageButtonStaticDisabled
-                } else { z[2].className += " " + v.sPageButton; z[3].className += " " + v.sPageButton }
-            }
-        }
-    }
-    }
-    }; o.oSort = { "string-asc": function (g, l) { if (typeof g != "string") g = ""; if (typeof l != "string") l = ""; g = g.toLowerCase(); l = l.toLowerCase(); return g < l ? -1 : g > l ? 1 : 0 }, "string-desc": function (g, l) { if (typeof g != "string") g = ""; if (typeof l != "string") l = ""; g = g.toLowerCase(); l = l.toLowerCase(); return g < l ? 1 : g > l ? -1 : 0 }, "html-asc": function (g, l) {
-        g = g.replace(/<.*?>/g, "").toLowerCase(); l = l.replace(/<.*?>/g, "").toLowerCase(); return g <
-l ? -1 : g > l ? 1 : 0
-    }, "html-desc": function (g, l) { g = g.replace(/<.*?>/g, "").toLowerCase(); l = l.replace(/<.*?>/g, "").toLowerCase(); return g < l ? 1 : g > l ? -1 : 0 }, "date-asc": function (g, l) { g = Date.parse(g); l = Date.parse(l); if (isNaN(g) || g === "") g = Date.parse("01/01/1970 00:00:00"); if (isNaN(l) || l === "") l = Date.parse("01/01/1970 00:00:00"); return g - l }, "date-desc": function (g, l) {
-        g = Date.parse(g); l = Date.parse(l); if (isNaN(g) || g === "") g = Date.parse("01/01/1970 00:00:00"); if (isNaN(l) || l === "") l = Date.parse("01/01/1970 00:00:00"); return l -
-g
-    }, "numeric-asc": function (g, l) { return (g == "-" || g === "" ? 0 : g * 1) - (l == "-" || l === "" ? 0 : l * 1) }, "numeric-desc": function (g, l) { return (l == "-" || l === "" ? 0 : l * 1) - (g == "-" || g === "" ? 0 : g * 1) }
-    }; o.aTypes = [function (g) { if (typeof g == "number") return "numeric"; else if (typeof g != "string") return null; var l, r = false; l = g.charAt(0); if ("0123456789-".indexOf(l) == -1) return null; for (var s = 1; s < g.length; s++) { l = g.charAt(s); if ("0123456789.".indexOf(l) == -1) return null; if (l == ".") { if (r) return null; r = true } } return "numeric" }, function (g) {
-        var l = Date.parse(g);
-
-        if (l !== null && !isNaN(l) || typeof g == "string" && g.length === 0) return "date"; return null
-    }, function (g) { if (typeof g == "string" && g.indexOf("<") != -1 && g.indexOf(">") != -1) return "html"; return null } ]; o.fnVersionCheck = function (g) { var l = function (x, v) { for (; x.length < v; ) x += "0"; return x }, r = o.sVersion.split("."); g = g.split("."); for (var s = "", w = "", y = 0, G = g.length; y < G; y++) { s += l(r[y], 3); w += l(g[y], 3) } return parseInt(s, 10) >= parseInt(w, 10) }; o._oExternConfig = { iNextUnique: 0 }; i.fn.dataTable = function (g) {
-        function l() {
-            this.fnRecordsTotal =
-function () { return this.oFeatures.bServerSide ? parseInt(this._iRecordsTotal, 10) : this.aiDisplayMaster.length }; this.fnRecordsDisplay = function () { return this.oFeatures.bServerSide ? parseInt(this._iRecordsDisplay, 10) : this.aiDisplay.length }; this.fnDisplayEnd = function () { return this.oFeatures.bServerSide ? this.oFeatures.bPaginate === false || this._iDisplayLength == -1 ? this._iDisplayStart + this.aiDisplay.length : Math.min(this._iDisplayStart + this._iDisplayLength, this._iRecordsDisplay) : this._iDisplayEnd }; this.sInstance =
-this.oInstance = null; this.oFeatures = { bPaginate: true, bLengthChange: true, bFilter: true, bSort: true, bInfo: true, bAutoWidth: true, bProcessing: false, bSortClasses: true, bStateSave: false, bServerSide: false, bDeferRender: false }; this.oScroll = { sX: "", sXInner: "", sY: "", bCollapse: false, bInfinite: false, iLoadGap: 100, iBarWidth: 0, bAutoCss: true }; this.aanFeatures = [];
-            this.oLanguage = { sProcessing: "Processing...", sLengthMenu: "Show _MENU_ artists per page", sZeroRecords: "No matching records found", sEmptyTable: "",
-                sLoadingRecords: "Loading...", sInfo: "Showing _START_ to _END_ of _TOTAL_ artists", sInfoEmpty: "Showing 0 to 0 of 0 artists", sInfoFiltered: "(filtered from _MAX_ total artists)", sInfoPostFix: "", sSearch: "Filter:", sUrl: "", oPaginate: { sFirst: "First", sPrevious: "Previous", sNext: "Next", sLast: "Last" }, fnInfoCallback: null
-            }; this.aoData = []; this.aiDisplay = []; this.aiDisplayMaster = []; this.aoColumns = []; this.aoHeader = []; this.aoFooter = []; this.iNextId = 0; this.asDataSearch = []; this.oPreviousSearch = { sSearch: "", bRegex: false,
-                bSmart: true
-            }; this.aoPreSearchCols = []; this.aaSorting = [[0, "asc", 0]]; this.aaSortingFixed = null; this.asStripClasses = []; this.asDestoryStrips = []; this.sDestroyWidth = 0; this.fnFooterCallback = this.fnHeaderCallback = this.fnRowCallback = null; this.aoDrawCallback = []; this.fnInitComplete = this.fnPreDrawCallback = null; this.sTableId = ""; this.nTableWrapper = this.nTBody = this.nTFoot = this.nTHead = this.nTable = null; this.bInitialised = this.bDeferLoading = false; this.aoOpenRows = []; this.sDom = "lfrtip"; this.sPaginationType = "two_button";
-            this.iCookieDuration = 7200; this.sCookiePrefix = "SpryMedia_DataTables_"; this.fnCookieCallback = null; this.aoStateSave = []; this.aoStateLoad = []; this.sAjaxSource = this.oLoadedState = null; this.sAjaxDataProp = "aaData"; this.bAjaxDataGet = true; this.jqXHR = null; this.fnServerData = function (a, b, c, d) { d.jqXHR = i.ajax({ url: a, data: b, success: c, dataType: "json", cache: false, error: function (f, e) { e == "parsererror" && alert("DataTables warning: JSON data from server could not be parsed. This is caused by a JSON formatting error.") } }) };
-            this.fnFormatNumber = function (a) { if (a < 1E3) return a; else { var b = a + ""; a = b.split(""); var c = ""; b = b.length; for (var d = 0; d < b; d++) { if (d % 3 === 0 && d !== 0) c = "," + c; c = a[b - d - 1] + c } } return c }; this.aLengthMenu = [10, 25, 50, 100]; this.bDrawing = this.iDraw = 0; this.iDrawError = -1; this._iDisplayLength = 10; this._iDisplayStart = 0; this._iDisplayEnd = 10; this._iRecordsDisplay = this._iRecordsTotal = 0; this.bJUI = false; this.oClasses = o.oStdClasses; this.bSortCellsTop = this.bSorted = this.bFiltered = false; this.oInit = null
-        } function r(a) {
-            return function () {
-                var b =
-[A(this[o.iApiIndex])].concat(Array.prototype.slice.call(arguments)); return o.oApi[a].apply(this, b)
-            }
-        } function s(a) {
-            var b, c, d = a.iInitDisplayStart; if (a.bInitialised === false) setTimeout(function () { s(a) }, 200); else {
-                xa(a); V(a); L(a, a.aoHeader); a.nTFoot && L(a, a.aoFooter);
-                K(a, true); a.oFeatures.bAutoWidth && ea(a);
-                b = 0; for (c = a.aoColumns.length; b < c; b++)
-                    if (a.aoColumns[b].sWidth !== null) a.aoColumns[b].nTh.style.width = u(a.aoColumns[b].sWidth);
-                if (a.oFeatures.bSort) R(a);
-                else if (a.oFeatures.bFilter) M(a, a.oPreviousSearch);
-                else { a.aiDisplay = a.aiDisplayMaster.slice(); E(a); C(a) } if (a.sAjaxSource !== null && !a.oFeatures.bServerSide) a.fnServerData.call(a.oInstance, a.sAjaxSource, [], function (f) { var e = f; if (a.sAjaxDataProp !== "") e = Z(a.sAjaxDataProp)(f); for (b = 0; b < e.length; b++) v(a, e[b]); a.iInitDisplayStart = d; if (a.oFeatures.bSort) R(a); else { a.aiDisplay = a.aiDisplayMaster.slice(); E(a); C(a) } K(a, false); w(a, f) }, a); else if (!a.oFeatures.bServerSide) { K(a, false); w(a) }
-            }
-        } function w(a, b) {
-            a._bInitComplete = true; if (typeof a.fnInitComplete == "function") typeof b !=
-"undefined" ? a.fnInitComplete.call(a.oInstance, a, b) : a.fnInitComplete.call(a.oInstance, a)
-        } function y(a, b, c) {
-            n(a.oLanguage, b, "sProcessing"); n(a.oLanguage, b, "sLengthMenu"); n(a.oLanguage, b, "sEmptyTable"); n(a.oLanguage, b, "sLoadingRecords"); n(a.oLanguage, b, "sZeroRecords"); n(a.oLanguage, b, "sInfo"); n(a.oLanguage, b, "sInfoEmpty"); n(a.oLanguage, b, "sInfoFiltered"); n(a.oLanguage, b, "sInfoPostFix"); n(a.oLanguage, b, "sSearch"); if (typeof b.oPaginate != "undefined") {
-                n(a.oLanguage.oPaginate, b.oPaginate, "sFirst"); n(a.oLanguage.oPaginate,
-b.oPaginate, "sPrevious"); n(a.oLanguage.oPaginate, b.oPaginate, "sNext"); n(a.oLanguage.oPaginate, b.oPaginate, "sLast")
-            } typeof b.sEmptyTable == "undefined" && typeof b.sZeroRecords != "undefined" && n(a.oLanguage, b, "sZeroRecords", "sEmptyTable"); typeof b.sLoadingRecords == "undefined" && typeof b.sZeroRecords != "undefined" && n(a.oLanguage, b, "sZeroRecords", "sLoadingRecords"); c && s(a)
-        } function G(a, b) {
-            var c = a.aoColumns.length; b = { sType: null, _bAutoType: true, bVisible: true, bSearchable: true, bSortable: true, asSorting: ["asc", "desc"],
-                sSortingClass: a.oClasses.sSortable, sSortingClassJUI: a.oClasses.sSortJUI, sTitle: b ? b.innerHTML : "", sName: "", sWidth: null, sWidthOrig: null, sClass: null, fnRender: null, bUseRendered: true, iDataSort: c, mDataProp: c, fnGetData: null, fnSetData: null, sSortDataType: "std", sDefaultContent: null, sContentPadding: "", nTh: b ? b : p.createElement("th"), nTf: null
-            }; a.aoColumns.push(b); if (typeof a.aoPreSearchCols[c] == "undefined" || a.aoPreSearchCols[c] === null) a.aoPreSearchCols[c] = { sSearch: "", bRegex: false, bSmart: true }; else {
-                if (typeof a.aoPreSearchCols[c].bRegex ==
-"undefined") a.aoPreSearchCols[c].bRegex = true; if (typeof a.aoPreSearchCols[c].bSmart == "undefined") a.aoPreSearchCols[c].bSmart = true
-            } x(a, c, null)
-        } function x(a, b, c) {
-            b = a.aoColumns[b]; if (typeof c != "undefined" && c !== null) {
-                if (typeof c.sType != "undefined") { b.sType = c.sType; b._bAutoType = false } n(b, c, "bVisible"); n(b, c, "bSearchable"); n(b, c, "bSortable"); n(b, c, "sTitle"); n(b, c, "sName"); n(b, c, "sWidth"); n(b, c, "sWidth", "sWidthOrig"); n(b, c, "sClass"); n(b, c, "fnRender"); n(b, c, "bUseRendered"); n(b, c, "iDataSort"); n(b, c, "mDataProp");
-                n(b, c, "asSorting"); n(b, c, "sSortDataType"); n(b, c, "sDefaultContent"); n(b, c, "sContentPadding")
-            } b.fnGetData = Z(b.mDataProp); b.fnSetData = ya(b.mDataProp); if (!a.oFeatures.bSort) b.bSortable = false; if (!b.bSortable || i.inArray("asc", b.asSorting) == -1 && i.inArray("desc", b.asSorting) == -1) { b.sSortingClass = a.oClasses.sSortableNone; b.sSortingClassJUI = "" } else if (b.bSortable || i.inArray("asc", b.asSorting) == -1 && i.inArray("desc", b.asSorting) == -1) { b.sSortingClass = a.oClasses.sSortable; b.sSortingClassJUI = a.oClasses.sSortJUI } else if (i.inArray("asc",
-b.asSorting) != -1 && i.inArray("desc", b.asSorting) == -1) { b.sSortingClass = a.oClasses.sSortableAsc; b.sSortingClassJUI = a.oClasses.sSortJUIAscAllowed } else if (i.inArray("asc", b.asSorting) == -1 && i.inArray("desc", b.asSorting) != -1) { b.sSortingClass = a.oClasses.sSortableDesc; b.sSortingClassJUI = a.oClasses.sSortJUIDescAllowed }
-        } function v(a, b) {
-            var c; c = typeof b.length == "number" ? b.slice() : i.extend(true, {}, b); b = a.aoData.length; var d = { nTr: null, _iId: a.iNextId++, _aData: c, _anHidden: [], _sRowStripe: "" }; a.aoData.push(d); for (var f,
-e = 0, h = a.aoColumns.length; e < h; e++) { c = a.aoColumns[e]; typeof c.fnRender == "function" && c.bUseRendered && c.mDataProp !== null && N(a, b, e, c.fnRender({ iDataRow: b, iDataColumn: e, aData: d._aData, oSettings: a })); if (c._bAutoType && c.sType != "string") { f = H(a, b, e, "type"); if (f !== null && f !== "") { f = fa(f); if (c.sType === null) c.sType = f; else if (c.sType != f) c.sType = "string" } } } a.aiDisplayMaster.push(b); a.oFeatures.bDeferRender || z(a, b); return b
-        } function z(a, b) {
-            var c = a.aoData[b], d; if (c.nTr === null) {
-                c.nTr = p.createElement("tr"); typeof c._aData.DT_RowId !=
-"undefined" && c.nTr.setAttribute("id", c._aData.DT_RowId); typeof c._aData.DT_RowClass != "undefined" && i(c.nTr).addClass(c._aData.DT_RowClass); for (var f = 0, e = a.aoColumns.length; f < e; f++) {
-                    var h = a.aoColumns[f]; d = p.createElement("td"); d.innerHTML = typeof h.fnRender == "function" && (!h.bUseRendered || h.mDataProp === null) ? h.fnRender({ iDataRow: b, iDataColumn: f, aData: c._aData, oSettings: a }) : H(a, b, f, "display"); if (h.sClass !== null) d.className = h.sClass; if (h.bVisible) { c.nTr.appendChild(d); c._anHidden[f] = null } else c._anHidden[f] =
-d
-                }
-            }
-        } function Y(a) {
-            var b, c, d, f, e, h, j, k, m; if (a.bDeferLoading || a.sAjaxSource === null) { j = a.nTBody.childNodes; b = 0; for (c = j.length; b < c; b++) if (j[b].nodeName.toUpperCase() == "TR") { k = a.aoData.length; a.aoData.push({ nTr: j[b], _iId: a.iNextId++, _aData: [], _anHidden: [], _sRowStripe: "" }); a.aiDisplayMaster.push(k); h = j[b].childNodes; d = e = 0; for (f = h.length; d < f; d++) { m = h[d].nodeName.toUpperCase(); if (m == "TD" || m == "TH") { N(a, k, e, i.trim(h[d].innerHTML)); e++ } } } } j = $(a); h = []; b = 0; for (c = j.length; b < c; b++) {
-                d = 0; for (f = j[b].childNodes.length; d <
-f; d++) { e = j[b].childNodes[d]; m = e.nodeName.toUpperCase(); if (m == "TD" || m == "TH") h.push(e) }
-            } h.length != j.length * a.aoColumns.length && J(a, 1, "Unexpected number of TD elements. Expected " + j.length * a.aoColumns.length + " and got " + h.length + ". DataTables does not support rowspan / colspan in the table body, and there must be one cell for each row/column combination."); d = 0; for (f = a.aoColumns.length; d < f; d++) {
-                if (a.aoColumns[d].sTitle === null) a.aoColumns[d].sTitle = a.aoColumns[d].nTh.innerHTML; j = a.aoColumns[d]._bAutoType;
-                m = typeof a.aoColumns[d].fnRender == "function"; e = a.aoColumns[d].sClass !== null; k = a.aoColumns[d].bVisible; var t, q; if (j || m || e || !k) {
-                    b = 0; for (c = a.aoData.length; b < c; b++) {
-                        t = h[b * f + d]; if (j && a.aoColumns[d].sType != "string") { q = H(a, b, d, "type"); if (q !== "") { q = fa(q); if (a.aoColumns[d].sType === null) a.aoColumns[d].sType = q; else if (a.aoColumns[d].sType != q) a.aoColumns[d].sType = "string" } } if (m) {
-                            q = a.aoColumns[d].fnRender({ iDataRow: b, iDataColumn: d, aData: a.aoData[b]._aData, oSettings: a }); t.innerHTML = q; a.aoColumns[d].bUseRendered &&
-N(a, b, d, q)
-                        } if (e) t.className += " " + a.aoColumns[d].sClass; if (k) a.aoData[b]._anHidden[d] = null; else { a.aoData[b]._anHidden[d] = t; t.parentNode.removeChild(t) }
-                    }
-                }
-            }
-        } function V(a) {
-            var b, c, d; a.nTHead.getElementsByTagName("tr"); if (a.nTHead.getElementsByTagName("th").length !== 0) { b = 0; for (d = a.aoColumns.length; b < d; b++) { c = a.aoColumns[b].nTh; a.aoColumns[b].sClass !== null && i(c).addClass(a.aoColumns[b].sClass); if (a.aoColumns[b].sTitle != c.innerHTML) c.innerHTML = a.aoColumns[b].sTitle } } else {
-                var f = p.createElement("tr"); b = 0;
-                for (d = a.aoColumns.length; b < d; b++) { c = a.aoColumns[b].nTh; c.innerHTML = a.aoColumns[b].sTitle; a.aoColumns[b].sClass !== null && i(c).addClass(a.aoColumns[b].sClass); f.appendChild(c) } i(a.nTHead).html("")[0].appendChild(f); W(a.aoHeader, a.nTHead)
-            } if (a.bJUI) { b = 0; for (d = a.aoColumns.length; b < d; b++) { c = a.aoColumns[b].nTh; f = p.createElement("div"); f.className = a.oClasses.sSortJUIWrapper; i(c).contents().appendTo(f); var e = p.createElement("span"); e.className = a.oClasses.sSortIcon; f.appendChild(e); c.appendChild(f) } } d = function () {
-                this.onselectstart =
-function () { return false }; return false
-            }; if (a.oFeatures.bSort) for (b = 0; b < a.aoColumns.length; b++) if (a.aoColumns[b].bSortable !== false) { ga(a, a.aoColumns[b].nTh, b); i(a.aoColumns[b].nTh).bind("mousedown.DT", d) } else i(a.aoColumns[b].nTh).addClass(a.oClasses.sSortableNone); a.oClasses.sFooterTH !== "" && i(">tr>th", a.nTFoot).addClass(a.oClasses.sFooterTH); if (a.nTFoot !== null) { c = S(a, null, a.aoFooter); b = 0; for (d = a.aoColumns.length; b < d; b++) if (typeof c[b] != "undefined") a.aoColumns[b].nTf = c[b] }
-        } function L(a, b, c) {
-            var d, f,
-e, h = [], j = [], k = a.aoColumns.length; if (typeof c == "undefined") c = false; d = 0; for (f = b.length; d < f; d++) { h[d] = b[d].slice(); h[d].nTr = b[d].nTr; for (e = k - 1; e >= 0; e--) !a.aoColumns[e].bVisible && !c && h[d].splice(e, 1); j.push([]) } d = 0; for (f = h.length; d < f; d++) {
-                if (h[d].nTr) { a = 0; for (e = h[d].nTr.childNodes.length; a < e; a++) h[d].nTr.removeChild(h[d].nTr.childNodes[0]) } e = 0; for (b = h[d].length; e < b; e++) {
-                    k = c = 1; if (typeof j[d][e] == "undefined") {
-                        h[d].nTr.appendChild(h[d][e].cell); for (j[d][e] = 1; typeof h[d + c] != "undefined" && h[d][e].cell == h[d +
-c][e].cell; ) { j[d + c][e] = 1; c++ } for (; typeof h[d][e + k] != "undefined" && h[d][e].cell == h[d][e + k].cell; ) { for (a = 0; a < c; a++) j[d + a][e + k] = 1; k++ } h[d][e].cell.setAttribute("rowspan", c); h[d][e].cell.setAttribute("colspan", k)
-                    }
-                }
-            }
-        } function C(a) {
-            var b, c, d = [], f = 0, e = false; b = a.asStripClasses.length; c = a.aoOpenRows.length; if (!(a.fnPreDrawCallback !== null && a.fnPreDrawCallback.call(a.oInstance, a) === false)) {
-                a.bDrawing = true; if (typeof a.iInitDisplayStart != "undefined" && a.iInitDisplayStart != -1) {
-                    a._iDisplayStart = a.oFeatures.bServerSide ?
-a.iInitDisplayStart : a.iInitDisplayStart >= a.fnRecordsDisplay() ? 0 : a.iInitDisplayStart; a.iInitDisplayStart = -1; E(a)
-                } if (a.bDeferLoading) { a.bDeferLoading = false; a.iDraw++ } else if (a.oFeatures.bServerSide) { if (!a.bDestroying && !za(a)) return } else a.iDraw++; if (a.aiDisplay.length !== 0) {
-                    var h = a._iDisplayStart, j = a._iDisplayEnd; if (a.oFeatures.bServerSide) { h = 0; j = a.aoData.length } for (h = h; h < j; h++) {
-                        var k = a.aoData[a.aiDisplay[h]]; k.nTr === null && z(a, a.aiDisplay[h]); var m = k.nTr; if (b !== 0) {
-                            var t = a.asStripClasses[f % b]; if (k._sRowStripe !=
-t) { i(m).removeClass(k._sRowStripe).addClass(t); k._sRowStripe = t }
-                        } if (typeof a.fnRowCallback == "function") { m = a.fnRowCallback.call(a.oInstance, m, a.aoData[a.aiDisplay[h]]._aData, f, h); if (!m && !e) { J(a, 0, "A node was not returned by fnRowCallback"); e = true } } d.push(m); f++; if (c !== 0) for (k = 0; k < c; k++) m == a.aoOpenRows[k].nParent && d.push(a.aoOpenRows[k].nTr)
-                    }
-                } else {
-                    d[0] = p.createElement("tr"); if (typeof a.asStripClasses[0] != "undefined") d[0].className = a.asStripClasses[0]; e = a.oLanguage.sZeroRecords.replace("_MAX_", a.fnFormatNumber(a.fnRecordsTotal()));
-                    if (a.iDraw == 1 && a.sAjaxSource !== null && !a.oFeatures.bServerSide) e = a.oLanguage.sLoadingRecords; else if (typeof a.oLanguage.sEmptyTable != "undefined" && a.fnRecordsTotal() === 0) e = a.oLanguage.sEmptyTable; b = p.createElement("td"); b.setAttribute("valign", "top"); b.colSpan = X(a); b.className = a.oClasses.sRowEmpty; b.innerHTML = e; d[f].appendChild(b)
-                } typeof a.fnHeaderCallback == "function" && a.fnHeaderCallback.call(a.oInstance, i(">tr", a.nTHead)[0], aa(a), a._iDisplayStart, a.fnDisplayEnd(), a.aiDisplay); typeof a.fnFooterCallback ==
-"function" && a.fnFooterCallback.call(a.oInstance, i(">tr", a.nTFoot)[0], aa(a), a._iDisplayStart, a.fnDisplayEnd(), a.aiDisplay); f = p.createDocumentFragment(); b = p.createDocumentFragment(); if (a.nTBody) { e = a.nTBody.parentNode; b.appendChild(a.nTBody); if (!a.oScroll.bInfinite || !a._bInitComplete || a.bSorted || a.bFiltered) { c = a.nTBody.childNodes; for (b = c.length - 1; b >= 0; b--) c[b].parentNode.removeChild(c[b]) } b = 0; for (c = d.length; b < c; b++) f.appendChild(d[b]); a.nTBody.appendChild(f); e !== null && e.appendChild(a.nTBody) } for (b = a.aoDrawCallback.length -
-1; b >= 0; b--) a.aoDrawCallback[b].fn.call(a.oInstance, a); a.bSorted = false; a.bFiltered = false; a.bDrawing = false; if (a.oFeatures.bServerSide) { K(a, false); typeof a._bInitComplete == "undefined" && w(a) }
-            }
-        } function ba(a) { if (a.oFeatures.bSort) R(a, a.oPreviousSearch); else if (a.oFeatures.bFilter) M(a, a.oPreviousSearch); else { E(a); C(a) } } function za(a) {
-            if (a.bAjaxDataGet) {
-                K(a, true); var b = a.aoColumns.length, c = [], d, f; a.iDraw++; c.push({ name: "sEcho", value: a.iDraw }); c.push({ name: "iColumns", value: b }); c.push({ name: "sColumns", value: ha(a) });
-                c.push({ name: "iDisplayStart", value: a._iDisplayStart }); c.push({ name: "iDisplayLength", value: a.oFeatures.bPaginate !== false ? a._iDisplayLength : -1 }); for (f = 0; f < b; f++) { d = a.aoColumns[f].mDataProp; c.push({ name: "mDataProp_" + f, value: typeof d == "function" ? "function" : d }) } if (a.oFeatures.bFilter !== false) {
-                    c.push({ name: "sSearch", value: a.oPreviousSearch.sSearch }); c.push({ name: "bRegex", value: a.oPreviousSearch.bRegex }); for (f = 0; f < b; f++) {
-                        c.push({ name: "sSearch_" + f, value: a.aoPreSearchCols[f].sSearch }); c.push({ name: "bRegex_" +
-f, value: a.aoPreSearchCols[f].bRegex
-                        }); c.push({ name: "bSearchable_" + f, value: a.aoColumns[f].bSearchable })
-                    }
-                } if (a.oFeatures.bSort !== false) {
-                    d = a.aaSortingFixed !== null ? a.aaSortingFixed.length : 0; var e = a.aaSorting.length; c.push({ name: "iSortingCols", value: d + e }); for (f = 0; f < d; f++) { c.push({ name: "iSortCol_" + f, value: a.aaSortingFixed[f][0] }); c.push({ name: "sSortDir_" + f, value: a.aaSortingFixed[f][1] }) } for (f = 0; f < e; f++) { c.push({ name: "iSortCol_" + (f + d), value: a.aaSorting[f][0] }); c.push({ name: "sSortDir_" + (f + d), value: a.aaSorting[f][1] }) } for (f =
-0; f < b; f++) c.push({ name: "bSortable_" + f, value: a.aoColumns[f].bSortable })
-                } a.fnServerData.call(a.oInstance, a.sAjaxSource, c, function (h) { Aa(a, h) }, a); return false
-            } else return true
-        } function Aa(a, b) {
-            if (typeof b.sEcho != "undefined") if (b.sEcho * 1 < a.iDraw) return; else a.iDraw = b.sEcho * 1; if (!a.oScroll.bInfinite || a.oScroll.bInfinite && (a.bSorted || a.bFiltered)) ia(a); a._iRecordsTotal = b.iTotalRecords; a._iRecordsDisplay = b.iTotalDisplayRecords; var c = ha(a); if (c = typeof b.sColumns != "undefined" && c !== "" && b.sColumns != c) var d =
-Ba(a, b.sColumns); b = Z(a.sAjaxDataProp)(b); for (var f = 0, e = b.length; f < e; f++) if (c) { for (var h = [], j = 0, k = a.aoColumns.length; j < k; j++) h.push(b[f][d[j]]); v(a, h) } else v(a, b[f]); a.aiDisplay = a.aiDisplayMaster.slice(); a.bAjaxDataGet = false; C(a); a.bAjaxDataGet = true; K(a, false)
-        } function xa(a) {
-            var b = p.createElement("div"); a.nTable.parentNode.insertBefore(b, a.nTable); a.nTableWrapper = p.createElement("div"); a.nTableWrapper.className = a.oClasses.sWrapper; a.sTableId !== "" && a.nTableWrapper.setAttribute("id", a.sTableId + "_wrapper");
-            a.nTableReinsertBefore = a.nTable.nextSibling; for (var c = a.nTableWrapper, d = a.sDom.split(""), f, e, h, j, k, m, t, q = 0; q < d.length; q++) {
-                e = 0; h = d[q]; if (h == "<") {
-                    j = p.createElement("div"); k = d[q + 1]; if (k == "'" || k == '"') {
-                        m = ""; for (t = 2; d[q + t] != k; ) { m += d[q + t]; t++ } if (m == "H") m = "fg-toolbar ui-toolbar ui-widget-header ui-corner-tl ui-corner-tr ui-helper-clearfix"; else if (m == "F") m = "fg-toolbar ui-toolbar ui-widget-header ui-corner-bl ui-corner-br ui-helper-clearfix"; if (m.indexOf(".") != -1) {
-                            k = m.split("."); j.setAttribute("id", k[0].substr(1,
-k[0].length - 1)); j.className = k[1]
-                        } else if (m.charAt(0) == "#") j.setAttribute("id", m.substr(1, m.length - 1)); else j.className = m; q += t
-                    } c.appendChild(j); c = j
-                } else if (h == ">") c = c.parentNode; else if (h == "l" && a.oFeatures.bPaginate && a.oFeatures.bLengthChange) { f = Ca(a); e = 1 } else if (h == "f" && a.oFeatures.bFilter) { f = Da(a); e = 1 } else if (h == "r" && a.oFeatures.bProcessing) { f = Ea(a); e = 1 } else if (h == "t") { f = Fa(a); e = 1 } else if (h == "i" && a.oFeatures.bInfo) { f = Ga(a); e = 1 } else if (h == "p" && a.oFeatures.bPaginate) { f = Ha(a); e = 1 } else if (o.aoFeatures.length !==
-0) { j = o.aoFeatures; t = 0; for (k = j.length; t < k; t++) if (h == j[t].cFeature) { if (f = j[t].fnInit(a)) e = 1; break } } if (e == 1 && f !== null) { if (typeof a.aanFeatures[h] != "object") a.aanFeatures[h] = []; a.aanFeatures[h].push(f); c.appendChild(f) }
-            } b.parentNode.replaceChild(a.nTableWrapper, b)
-        } function Fa(a) {
-            if (a.oScroll.sX === "" && a.oScroll.sY === "") return a.nTable; var b = p.createElement("div"), c = p.createElement("div"), d = p.createElement("div"), f = p.createElement("div"), e = p.createElement("div"), h = p.createElement("div"), j = a.nTable.cloneNode(false),
-k = a.nTable.cloneNode(false), m = a.nTable.getElementsByTagName("thead")[0], t = a.nTable.getElementsByTagName("tfoot").length === 0 ? null : a.nTable.getElementsByTagName("tfoot")[0], q = typeof g.bJQueryUI != "undefined" && g.bJQueryUI ? o.oJUIClasses : o.oStdClasses; c.appendChild(d); e.appendChild(h); f.appendChild(a.nTable); b.appendChild(c); b.appendChild(f); d.appendChild(j); j.appendChild(m); if (t !== null) { b.appendChild(e); h.appendChild(k); k.appendChild(t) } b.className = q.sScrollWrapper; c.className = q.sScrollHead; d.className =
-q.sScrollHeadInner; f.className = q.sScrollBody; e.className = q.sScrollFoot; h.className = q.sScrollFootInner; if (a.oScroll.bAutoCss) { c.style.overflow = "hidden"; c.style.position = "relative"; e.style.overflow = "hidden"; f.style.overflow = "auto" } c.style.border = "0"; c.style.width = "100%"; e.style.border = "0"; d.style.width = "150%"; j.removeAttribute("id"); j.style.marginLeft = "0"; a.nTable.style.marginLeft = "0"; if (t !== null) { k.removeAttribute("id"); k.style.marginLeft = "0" } d = i(">caption", a.nTable); h = 0; for (k = d.length; h < k; h++) j.appendChild(d[h]);
-            if (a.oScroll.sX !== "") { c.style.width = u(a.oScroll.sX); f.style.width = u(a.oScroll.sX); if (t !== null) e.style.width = u(a.oScroll.sX); i(f).scroll(function () { c.scrollLeft = this.scrollLeft; if (t !== null) e.scrollLeft = this.scrollLeft }) } if (a.oScroll.sY !== "") f.style.height = u(a.oScroll.sY); a.aoDrawCallback.push({ fn: Ia, sName: "scrolling" }); a.oScroll.bInfinite && i(f).scroll(function () {
-                if (!a.bDrawing) if (i(this).scrollTop() + i(this).height() > i(a.nTable).height() - a.oScroll.iLoadGap) if (a.fnDisplayEnd() < a.fnRecordsDisplay()) {
-                    ja(a,
-"next"); E(a); C(a)
-                }
-            }); a.nScrollHead = c; a.nScrollFoot = e; return b
-        } function Ia(a) {
-            var b = a.nScrollHead.getElementsByTagName("div")[0], c = b.getElementsByTagName("table")[0], d = a.nTable.parentNode, f, e, h, j, k, m, t, q, I = []; h = a.nTable.getElementsByTagName("thead"); h.length > 0 && a.nTable.removeChild(h[0]); if (a.nTFoot !== null) { k = a.nTable.getElementsByTagName("tfoot"); k.length > 0 && a.nTable.removeChild(k[0]) } h = a.nTHead.cloneNode(true); a.nTable.insertBefore(h, a.nTable.childNodes[0]); if (a.nTFoot !== null) {
-                k = a.nTFoot.cloneNode(true);
-                a.nTable.insertBefore(k, a.nTable.childNodes[1])
-            } if (a.oScroll.sX === "") { d.style.width = "100%"; b.parentNode.style.width = "100%" } var O = S(a, h); f = 0; for (e = O.length; f < e; f++) { t = Ja(a, f); O[f].style.width = a.aoColumns[t].sWidth } a.nTFoot !== null && P(function (B) { B.style.width = "" }, k.getElementsByTagName("tr")); f = i(a.nTable).outerWidth(); if (a.oScroll.sX === "") { a.nTable.style.width = "100%"; if (i.browser.msie && i.browser.version <= 7) a.nTable.style.width = u(i(a.nTable).outerWidth() - a.oScroll.iBarWidth) } else if (a.oScroll.sXInner !==
-"") a.nTable.style.width = u(a.oScroll.sXInner); else if (f == i(d).width() && i(d).height() < i(a.nTable).height()) { a.nTable.style.width = u(f - a.oScroll.iBarWidth); if (i(a.nTable).outerWidth() > f - a.oScroll.iBarWidth) a.nTable.style.width = u(f) } else a.nTable.style.width = u(f); f = i(a.nTable).outerWidth(); if (a.oScroll.sX === "") { d.style.width = u(f + a.oScroll.iBarWidth); b.parentNode.style.width = u(f + a.oScroll.iBarWidth) } e = a.nTHead.getElementsByTagName("tr"); h = h.getElementsByTagName("tr"); P(function (B, F) {
-    m = B.style; m.paddingTop =
-"0"; m.paddingBottom = "0"; m.borderTopWidth = "0"; m.borderBottomWidth = "0"; m.height = 0; q = i(B).width(); F.style.width = u(q); I.push(q)
-}, h, e); i(h).height(0); if (a.nTFoot !== null) { j = k.getElementsByTagName("tr"); k = a.nTFoot.getElementsByTagName("tr"); P(function (B, F) { m = B.style; m.paddingTop = "0"; m.paddingBottom = "0"; m.borderTopWidth = "0"; m.borderBottomWidth = "0"; m.height = 0; q = i(B).width(); F.style.width = u(q); I.push(q) }, j, k); i(j).height(0) } P(function (B) { B.innerHTML = ""; B.style.width = u(I.shift()) }, h); a.nTFoot !== null && P(function (B) {
-    B.innerHTML =
-""; B.style.width = u(I.shift())
-}, j); if (i(a.nTable).outerWidth() < f) if (a.oScroll.sX === "") J(a, 1, "The table cannot fit into the current element which will cause column misalignment. It is suggested that you enable x-scrolling or increase the width the table has in which to be drawn"); else a.oScroll.sXInner !== "" && J(a, 1, "The table cannot fit into the current element which will cause column misalignment. It is suggested that you increase the sScrollXInner property to allow it to draw in a larger area, or simply remove that parameter to allow automatic calculation");
-            if (a.oScroll.sY === "") if (i.browser.msie && i.browser.version <= 7) d.style.height = u(a.nTable.offsetHeight + a.oScroll.iBarWidth); if (a.oScroll.sY !== "" && a.oScroll.bCollapse) { d.style.height = u(a.oScroll.sY); j = a.oScroll.sX !== "" && a.nTable.offsetWidth > d.offsetWidth ? a.oScroll.iBarWidth : 0; if (a.nTable.offsetHeight < d.offsetHeight) d.style.height = u(i(a.nTable).height() + j) } j = i(a.nTable).outerWidth(); c.style.width = u(j); b.style.width = u(j + a.oScroll.iBarWidth); if (a.nTFoot !== null) {
-                b = a.nScrollFoot.getElementsByTagName("div")[0];
-                c = b.getElementsByTagName("table")[0]; b.style.width = u(a.nTable.offsetWidth + a.oScroll.iBarWidth); c.style.width = u(a.nTable.offsetWidth)
-            } if (a.bSorted || a.bFiltered) d.scrollTop = 0
-        }
-        // sFilter
-        function ca(a) { if (a.oFeatures.bAutoWidth === false) return false; ea(a); for (var b = 0, c = a.aoColumns.length; b < c; b++) a.aoColumns[b].nTh.style.width = a.aoColumns[b].sWidth } function Da(a) {
-            var b = a.oLanguage.sSearch;
-            b = b.indexOf("_INPUT_") !== -1 ? b.replace("_INPUT_", '<input type="text" />') : b === "" ? '<input type="text" />' : b + ' <input type="text" />';
-            var c = p.createElement("div");
-            c.className = a.oClasses.sFilter;
-            c.innerHTML = "<div>" + b + "</div>";
-            a.sTableId !== "" && typeof a.aanFeatures.f == "undefined" && c.setAttribute("id", a.sTableId + "_filter");
-            b = i("input", c);
-            b.val(a.oPreviousSearch.sSearch.replace('"', "&quot;"));
-            b.bind("keyup.DT", function () {
-                for (var d = a.aanFeatures.f, f = 0, e = d.length;
-                f < e; f++)
-                    d[f] != this.parentNode && i("input", d[f]).val(this.value);
-                this.value != a.oPreviousSearch.sSearch && M(a, {
-                    sSearch: this.value, bRegex: a.oPreviousSearch.bRegex, bSmart: a.oPreviousSearch.bSmart
-                })
-            });
-            b.bind("keypress.DT", function (d) {
-                if (d.keyCode == 13) return false
-            });
-            return c
-
-        } function M(a, b, c) { Ka(a, b.sSearch, c, b.bRegex, b.bSmart); for (b = 0; b < a.aoPreSearchCols.length; b++) La(a, a.aoPreSearchCols[b].sSearch, b, a.aoPreSearchCols[b].bRegex, a.aoPreSearchCols[b].bSmart); o.afnFiltering.length !== 0 && Ma(a); a.bFiltered = true; a._iDisplayStart = 0; E(a); C(a); ka(a, 0) } function Ma(a) {
-            for (var b = o.afnFiltering, c = 0, d = b.length; c < d; c++) for (var f = 0, e = 0, h = a.aiDisplay.length; e < h; e++) {
-                var j = a.aiDisplay[e - f];
-                if (!b[c](a, da(a, j, "filter"),
-                j)) {
-                    a.aiDisplay.splice(e - f, 1); f++
-                }
-            }
-        } function La(a, b, c, d, f) { if (b !== "") { var e = 0; b = la(b, d, f); for (d = a.aiDisplay.length - 1; d >= 0; d--) { f = ma(H(a, a.aiDisplay[d], c, "filter"), a.aoColumns[c].sType); if (!b.test(f)) { a.aiDisplay.splice(d, 1); e++ } } } } function Ka(a, b, c, d, f) {
-            var e = la(b, d, f); if (typeof c == "undefined" || c === null) c = 0; if (o.afnFiltering.length !== 0) c = 1; if (b.length <= 0) { a.aiDisplay.splice(0, a.aiDisplay.length); a.aiDisplay = a.aiDisplayMaster.slice() } else if (a.aiDisplay.length == a.aiDisplayMaster.length || a.oPreviousSearch.sSearch.length >
-b.length || c == 1 || b.indexOf(a.oPreviousSearch.sSearch) !== 0) { a.aiDisplay.splice(0, a.aiDisplay.length); ka(a, 1); for (c = 0; c < a.aiDisplayMaster.length; c++) e.test(a.asDataSearch[c]) && a.aiDisplay.push(a.aiDisplayMaster[c]) } else { var h = 0; for (c = 0; c < a.asDataSearch.length; c++) if (!e.test(a.asDataSearch[c])) { a.aiDisplay.splice(c - h, 1); h++ } } a.oPreviousSearch.sSearch = b; a.oPreviousSearch.bRegex = d; a.oPreviousSearch.bSmart = f
-        } function ka(a, b) {
-            a.asDataSearch.splice(0, a.asDataSearch.length); b = typeof b != "undefined" && b == 1 ? a.aiDisplayMaster :
-a.aiDisplay; for (var c = 0, d = b.length; c < d; c++) a.asDataSearch[c] = na(a, da(a, b[c], "filter"))
-        } function na(a, b) { var c = ""; if (typeof a.__nTmpFilter == "undefined") a.__nTmpFilter = p.createElement("div"); for (var d = a.__nTmpFilter, f = 0, e = a.aoColumns.length; f < e; f++) if (a.aoColumns[f].bSearchable) c += ma(b[f], a.aoColumns[f].sType) + "  "; if (c.indexOf("&") !== -1) { d.innerHTML = c; c = d.textContent ? d.textContent : d.innerText; c = c.replace(/\n/g, " ").replace(/\r/g, "") } return c } function la(a, b, c) {
-            if (c) {
-                a = b ? a.split(" ") : oa(a).split(" ");
-                a = "^(?=.*?" + a.join(")(?=.*?") + ").*$"; return new RegExp(a, "i")
-            } else { a = b ? a : oa(a); return new RegExp(a, "i") }
-        } function ma(a, b) { if (typeof o.ofnSearch[b] == "function") return o.ofnSearch[b](a); else if (b == "html") return a.replace(/\n/g, " ").replace(/<.*?>/g, ""); else if (typeof a == "string") return a.replace(/\n/g, " "); else if (a === null) return ""; return a } function R(a, b) {
-            var c, d, f, e, h = [], j = [], k = o.oSort; d = a.aoData; var m = a.aoColumns; if (!a.oFeatures.bServerSide && (a.aaSorting.length !== 0 || a.aaSortingFixed !== null)) {
-                h = a.aaSortingFixed !==
-null ? a.aaSortingFixed.concat(a.aaSorting) : a.aaSorting.slice(); for (c = 0; c < h.length; c++) { var t = h[c][0]; f = pa(a, t); e = a.aoColumns[t].sSortDataType; if (typeof o.afnSortData[e] != "undefined") { var q = o.afnSortData[e](a, t, f); f = 0; for (e = d.length; f < e; f++) N(a, f, t, q[f]) } } c = 0; for (d = a.aiDisplayMaster.length; c < d; c++) j[a.aiDisplayMaster[c]] = c; var I = h.length; a.aiDisplayMaster.sort(function (O, B) {
-    var F, qa; for (c = 0; c < I; c++) {
-        F = m[h[c][0]].iDataSort; qa = m[F].sType; F = k[(qa ? qa : "string") + "-" + h[c][1]](H(a, O, F, "sort"), H(a, B, F, "sort"));
-        if (F !== 0) return F
-    } return k["numeric-asc"](j[O], j[B])
-})
-            } if ((typeof b == "undefined" || b) && !a.oFeatures.bDeferRender) T(a); a.bSorted = true; if (a.oFeatures.bFilter) M(a, a.oPreviousSearch, 1); else { a.aiDisplay = a.aiDisplayMaster.slice(); a._iDisplayStart = 0; E(a); C(a) }
-        } function ga(a, b, c, d) {
-            i(b).bind("click.DT", function (f) {
-                if (a.aoColumns[c].bSortable !== false) {
-                    var e = function () {
-                        var h, j; if (f.shiftKey) {
-                            for (var k = false, m = 0; m < a.aaSorting.length; m++) if (a.aaSorting[m][0] == c) {
-                                k = true; h = a.aaSorting[m][0]; j = a.aaSorting[m][2] +
-1; if (typeof a.aoColumns[h].asSorting[j] == "undefined") a.aaSorting.splice(m, 1); else { a.aaSorting[m][1] = a.aoColumns[h].asSorting[j]; a.aaSorting[m][2] = j } break
-                            } k === false && a.aaSorting.push([c, a.aoColumns[c].asSorting[0], 0])
-                        } else if (a.aaSorting.length == 1 && a.aaSorting[0][0] == c) { h = a.aaSorting[0][0]; j = a.aaSorting[0][2] + 1; if (typeof a.aoColumns[h].asSorting[j] == "undefined") j = 0; a.aaSorting[0][1] = a.aoColumns[h].asSorting[j]; a.aaSorting[0][2] = j } else {
-                            a.aaSorting.splice(0, a.aaSorting.length); a.aaSorting.push([c, a.aoColumns[c].asSorting[0],
-0])
-                        } R(a)
-                    }; if (a.oFeatures.bProcessing) { K(a, true); setTimeout(function () { e(); a.oFeatures.bServerSide || K(a, false) }, 0) } else e(); typeof d == "function" && d(a)
-                }
-            })
-        } function T(a) {
-            var b, c, d, f, e, h = a.aoColumns.length, j = a.oClasses; for (b = 0; b < h; b++) a.aoColumns[b].bSortable && i(a.aoColumns[b].nTh).removeClass(j.sSortAsc + " " + j.sSortDesc + " " + a.aoColumns[b].sSortingClass); f = a.aaSortingFixed !== null ? a.aaSortingFixed.concat(a.aaSorting) : a.aaSorting.slice(); for (b = 0; b < a.aoColumns.length; b++) if (a.aoColumns[b].bSortable) {
-                e = a.aoColumns[b].sSortingClass;
-                d = -1; for (c = 0; c < f.length; c++) if (f[c][0] == b) { e = f[c][1] == "asc" ? j.sSortAsc : j.sSortDesc; d = c; break } i(a.aoColumns[b].nTh).addClass(e); if (a.bJUI) { c = i("span", a.aoColumns[b].nTh); c.removeClass(j.sSortJUIAsc + " " + j.sSortJUIDesc + " " + j.sSortJUI + " " + j.sSortJUIAscAllowed + " " + j.sSortJUIDescAllowed); c.addClass(d == -1 ? a.aoColumns[b].sSortingClassJUI : f[d][1] == "asc" ? j.sSortJUIAsc : j.sSortJUIDesc) }
-            } else i(a.aoColumns[b].nTh).addClass(a.aoColumns[b].sSortingClass); e = j.sSortColumn; if (a.oFeatures.bSort && a.oFeatures.bSortClasses) {
-                d =
-Q(a); if (a.oFeatures.bDeferRender) i(d).removeClass(e + "1 " + e + "2 " + e + "3"); else if (d.length >= h) for (b = 0; b < h; b++) if (d[b].className.indexOf(e + "1") != -1) { c = 0; for (a = d.length / h; c < a; c++) d[h * c + b].className = i.trim(d[h * c + b].className.replace(e + "1", "")) } else if (d[b].className.indexOf(e + "2") != -1) { c = 0; for (a = d.length / h; c < a; c++) d[h * c + b].className = i.trim(d[h * c + b].className.replace(e + "2", "")) } else if (d[b].className.indexOf(e + "3") != -1) {
-                    c = 0; for (a = d.length / h; c < a; c++) d[h * c + b].className = i.trim(d[h * c + b].className.replace(" " +
-e + "3", ""))
-                } j = 1; var k; for (b = 0; b < f.length; b++) { k = parseInt(f[b][0], 10); c = 0; for (a = d.length / h; c < a; c++) d[h * c + k].className += " " + e + j; j < 3 && j++ }
-            }
-        } function Ha(a) { if (a.oScroll.bInfinite) return null; var b = p.createElement("div"); b.className = a.oClasses.sPaging + a.sPaginationType; o.oPagination[a.sPaginationType].fnInit(a, b, function (c) { E(c); C(c) }); typeof a.aanFeatures.p == "undefined" && a.aoDrawCallback.push({ fn: function (c) { o.oPagination[c.sPaginationType].fnUpdate(c, function (d) { E(d); C(d) }) }, sName: "pagination" }); return b }
-        function ja(a, b) {
-            var c = a._iDisplayStart; if (b == "first") a._iDisplayStart = 0; else if (b == "previous") { a._iDisplayStart = a._iDisplayLength >= 0 ? a._iDisplayStart - a._iDisplayLength : 0; if (a._iDisplayStart < 0) a._iDisplayStart = 0 } else if (b == "next") if (a._iDisplayLength >= 0) { if (a._iDisplayStart + a._iDisplayLength < a.fnRecordsDisplay()) a._iDisplayStart += a._iDisplayLength } else a._iDisplayStart = 0; else if (b == "last") if (a._iDisplayLength >= 0) { b = parseInt((a.fnRecordsDisplay() - 1) / a._iDisplayLength, 10) + 1; a._iDisplayStart = (b - 1) * a._iDisplayLength } else a._iDisplayStart =
-0; else J(a, 0, "Unknown paging action: " + b); return c != a._iDisplayStart
-        } function Ga(a) { var b = p.createElement("div"); b.className = a.oClasses.sInfo; if (typeof a.aanFeatures.i == "undefined") { a.aoDrawCallback.push({ fn: Na, sName: "information" }); a.sTableId !== "" && b.setAttribute("id", a.sTableId + "_info") } return b } function Na(a) {
-            if (!(!a.oFeatures.bInfo || a.aanFeatures.i.length === 0)) {
-                var b = a._iDisplayStart + 1, c = a.fnDisplayEnd(), d = a.fnRecordsTotal(), f = a.fnRecordsDisplay(), e = a.fnFormatNumber(b), h = a.fnFormatNumber(c), j =
-a.fnFormatNumber(d), k = a.fnFormatNumber(f); if (a.oScroll.bInfinite) e = a.fnFormatNumber(1); e = a.fnRecordsDisplay() === 0 && a.fnRecordsDisplay() == a.fnRecordsTotal() ? a.oLanguage.sInfoEmpty + a.oLanguage.sInfoPostFix : a.fnRecordsDisplay() === 0 ? a.oLanguage.sInfoEmpty + " " + a.oLanguage.sInfoFiltered.replace("_MAX_", j) + a.oLanguage.sInfoPostFix : a.fnRecordsDisplay() == a.fnRecordsTotal() ? a.oLanguage.sInfo.replace("_START_", e).replace("_END_", h).replace("_TOTAL_", k) + a.oLanguage.sInfoPostFix : a.oLanguage.sInfo.replace("_START_",
-e).replace("_END_", h).replace("_TOTAL_", k) + " " + a.oLanguage.sInfoFiltered.replace("_MAX_", a.fnFormatNumber(a.fnRecordsTotal())) + a.oLanguage.sInfoPostFix; if (a.oLanguage.fnInfoCallback !== null) e = a.oLanguage.fnInfoCallback(a, b, c, d, f, e); a = a.aanFeatures.i; b = 0; for (c = a.length; b < c; b++) i(a[b]).html(e)
-            }
-        } function Ca(a) {
-            if (a.oScroll.bInfinite) return null; var b = '<select size="1" ' + (a.sTableId === "" ? "" : 'name="' + a.sTableId + '_length"') + ">", c, d; if (a.aLengthMenu.length == 2 && typeof a.aLengthMenu[0] == "object" && typeof a.aLengthMenu[1] ==
-"object") { c = 0; for (d = a.aLengthMenu[0].length; c < d; c++) b += '<option value="' + a.aLengthMenu[0][c] + '">' + a.aLengthMenu[1][c] + "</option>" } else { c = 0; for (d = a.aLengthMenu.length; c < d; c++) b += '<option value="' + a.aLengthMenu[c] + '">' + a.aLengthMenu[c] + "</option>" } b += "</select>"; var f = p.createElement("div"); a.sTableId !== "" && typeof a.aanFeatures.l == "undefined" && f.setAttribute("id", a.sTableId + "_length"); f.className = a.oClasses.sLength; f.innerHTML = "<label>" + a.oLanguage.sLengthMenu.replace("_MENU_", b) + "</label>"; i('select option[value="' +
-a._iDisplayLength + '"]', f).attr("selected", true); i("select", f).bind("change.DT", function () { var e = i(this).val(), h = a.aanFeatures.l; c = 0; for (d = h.length; c < d; c++) h[c] != this.parentNode && i("select", h[c]).val(e); a._iDisplayLength = parseInt(e, 10); E(a); if (a.fnDisplayEnd() == a.fnRecordsDisplay()) { a._iDisplayStart = a.fnDisplayEnd() - a._iDisplayLength; if (a._iDisplayStart < 0) a._iDisplayStart = 0 } if (a._iDisplayLength == -1) a._iDisplayStart = 0; C(a) }); return f
-        } function Ea(a) {
-            var b = p.createElement("div"); a.sTableId !== "" && typeof a.aanFeatures.r ==
-"undefined" && b.setAttribute("id", a.sTableId + "_processing"); b.innerHTML = a.oLanguage.sProcessing; b.className = a.oClasses.sProcessing; a.nTable.parentNode.insertBefore(b, a.nTable); return b
-        } function K(a, b) { if (a.oFeatures.bProcessing) { a = a.aanFeatures.r; for (var c = 0, d = a.length; c < d; c++) a[c].style.visibility = b ? "visible" : "hidden" } } function Ja(a, b) { for (var c = -1, d = 0; d < a.aoColumns.length; d++) { a.aoColumns[d].bVisible === true && c++; if (c == b) return d } return null } function pa(a, b) {
-            for (var c = -1, d = 0; d < a.aoColumns.length; d++) {
-                a.aoColumns[d].bVisible ===
-true && c++; if (d == b) return a.aoColumns[d].bVisible === true ? c : null
-            } return null
-        } function U(a, b) { var c, d; c = a._iDisplayStart; for (d = a._iDisplayEnd; c < d; c++) if (a.aoData[a.aiDisplay[c]].nTr == b) return a.aiDisplay[c]; c = 0; for (d = a.aoData.length; c < d; c++) if (a.aoData[c].nTr == b) return c; return null } function X(a) { for (var b = 0, c = 0; c < a.aoColumns.length; c++) a.aoColumns[c].bVisible === true && b++; return b } function E(a) {
-            a._iDisplayEnd = a.oFeatures.bPaginate === false ? a.aiDisplay.length : a._iDisplayStart + a._iDisplayLength > a.aiDisplay.length ||
-a._iDisplayLength == -1 ? a.aiDisplay.length : a._iDisplayStart + a._iDisplayLength
-        } function Oa(a, b) { if (!a || a === null || a === "") return 0; if (typeof b == "undefined") b = p.getElementsByTagName("body")[0]; var c = p.createElement("div"); c.style.width = u(a); b.appendChild(c); a = c.offsetWidth; b.removeChild(c); return a } function ea(a) {
-            var b = 0, c, d = 0, f = a.aoColumns.length, e, h = i("th", a.nTHead); for (e = 0; e < f; e++) if (a.aoColumns[e].bVisible) {
-                d++; if (a.aoColumns[e].sWidth !== null) {
-                    c = Oa(a.aoColumns[e].sWidthOrig, a.nTable.parentNode); if (c !==
-null) a.aoColumns[e].sWidth = u(c); b++
-                }
-            } if (f == h.length && b === 0 && d == f && a.oScroll.sX === "" && a.oScroll.sY === "") for (e = 0; e < a.aoColumns.length; e++) { c = i(h[e]).width(); if (c !== null) a.aoColumns[e].sWidth = u(c) } else {
-                b = a.nTable.cloneNode(false); e = a.nTHead.cloneNode(true); d = p.createElement("tbody"); c = p.createElement("tr"); b.removeAttribute("id"); b.appendChild(e); if (a.nTFoot !== null) { b.appendChild(a.nTFoot.cloneNode(true)); P(function (k) { k.style.width = "" }, b.getElementsByTagName("tr")) } b.appendChild(d); d.appendChild(c);
-                d = i("thead th", b); if (d.length === 0) d = i("tbody tr:eq(0)>td", b); h = S(a, e); for (e = d = 0; e < f; e++) { var j = a.aoColumns[e]; if (j.bVisible && j.sWidthOrig !== null && j.sWidthOrig !== "") h[e - d].style.width = u(j.sWidthOrig); else if (j.bVisible) h[e - d].style.width = ""; else d++ } for (e = 0; e < f; e++) if (a.aoColumns[e].bVisible) { d = Pa(a, e); if (d !== null) { d = d.cloneNode(true); if (a.aoColumns[e].sContentPadding !== "") d.innerHTML += a.aoColumns[e].sContentPadding; c.appendChild(d) } } f = a.nTable.parentNode; f.appendChild(b); if (a.oScroll.sX !== "" && a.oScroll.sXInner !==
-"") b.style.width = u(a.oScroll.sXInner); else if (a.oScroll.sX !== "") { b.style.width = ""; if (i(b).width() < f.offsetWidth) b.style.width = u(f.offsetWidth) } else if (a.oScroll.sY !== "") b.style.width = u(f.offsetWidth); b.style.visibility = "hidden"; Qa(a, b); f = i("tbody tr:eq(0)", b).children(); if (f.length === 0) f = S(a, i("thead", b)[0]); if (a.oScroll.sX !== "") {
-                    for (e = d = c = 0; e < a.aoColumns.length; e++) if (a.aoColumns[e].bVisible) {
-                        c += a.aoColumns[e].sWidthOrig === null ? i(f[d]).outerWidth() : parseInt(a.aoColumns[e].sWidth.replace("px", ""),
-10) + (i(f[d]).outerWidth() - i(f[d]).width()); d++
-                    } b.style.width = u(c); a.nTable.style.width = u(c)
-                } for (e = d = 0; e < a.aoColumns.length; e++) if (a.aoColumns[e].bVisible) { c = i(f[d]).width(); if (c !== null && c > 0) a.aoColumns[e].sWidth = u(c); d++ } a.nTable.style.width = u(i(b).outerWidth()); b.parentNode.removeChild(b)
-            }
-        } function Qa(a, b) { if (a.oScroll.sX === "" && a.oScroll.sY !== "") { i(b).width(); b.style.width = u(i(b).outerWidth() - a.oScroll.iBarWidth) } else if (a.oScroll.sX !== "") b.style.width = u(i(b).outerWidth()) } function Pa(a, b) {
-            var c =
-Ra(a, b); if (c < 0) return null; if (a.aoData[c].nTr === null) { var d = p.createElement("td"); d.innerHTML = H(a, c, b, ""); return d } return Q(a, c)[b]
-        } function Ra(a, b) { for (var c = -1, d = -1, f = 0; f < a.aoData.length; f++) { var e = H(a, f, b, "display") + ""; e = e.replace(/<.*?>/g, ""); if (e.length > c) { c = e.length; d = f } } return d } function u(a) { if (a === null) return "0px"; if (typeof a == "number") { if (a < 0) return "0px"; return a + "px" } var b = a.charCodeAt(a.length - 1); if (b < 48 || b > 57) return a; return a + "px" } function Va(a, b) {
-            if (a.length != b.length) return 1; for (var c =
-0; c < a.length; c++) if (a[c] != b[c]) return 2; return 0
-        } function fa(a) { for (var b = o.aTypes, c = b.length, d = 0; d < c; d++) { var f = b[d](a); if (f !== null) return f } return "string" } function A(a) { for (var b = 0; b < D.length; b++) if (D[b].nTable == a) return D[b]; return null } function aa(a) { for (var b = [], c = a.aoData.length, d = 0; d < c; d++) b.push(a.aoData[d]._aData); return b } function $(a) { for (var b = [], c = 0, d = a.aoData.length; c < d; c++) a.aoData[c].nTr !== null && b.push(a.aoData[c].nTr); return b } function Q(a, b) {
-            var c = [], d, f, e, h, j; f = 0; var k = a.aoData.length;
-            if (typeof b != "undefined") { f = b; k = b + 1 } for (f = f; f < k; f++) { j = a.aoData[f]; if (j.nTr !== null) { b = []; e = 0; for (h = j.nTr.childNodes.length; e < h; e++) { d = j.nTr.childNodes[e].nodeName.toLowerCase(); if (d == "td" || d == "th") b.push(j.nTr.childNodes[e]) } e = d = 0; for (h = a.aoColumns.length; e < h; e++) if (a.aoColumns[e].bVisible) c.push(b[e - d]); else { c.push(j._anHidden[e]); d++ } } } return c
-        } function oa(a) { return a.replace(new RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^)", "g"), "\\$1") } function ra(a, b) {
-            for (var c = -1, d =
-0, f = a.length; d < f; d++) if (a[d] == b) c = d; else a[d] > b && a[d]--; c != -1 && a.splice(c, 1)
-        } function Ba(a, b) { b = b.split(","); for (var c = [], d = 0, f = a.aoColumns.length; d < f; d++) for (var e = 0; e < f; e++) if (a.aoColumns[d].sName == b[e]) { c.push(e); break } return c } function ha(a) { for (var b = "", c = 0, d = a.aoColumns.length; c < d; c++) b += a.aoColumns[c].sName + ","; if (b.length == d) return ""; return b.slice(0, -1) } function J(a, b, c) {
-            a = a.sTableId === "" ? "DataTables warning: " + c : "DataTables warning (table id = '" + a.sTableId + "'): " + c; if (b === 0) if (o.sErrMode ==
-"alert") alert(a); else throw a; else typeof console != "undefined" && typeof console.log != "undefined" && console.log(a)
-        } function ia(a) { a.aoData.splice(0, a.aoData.length); a.aiDisplayMaster.splice(0, a.aiDisplayMaster.length); a.aiDisplay.splice(0, a.aiDisplay.length); E(a) } function sa(a) {
-            if (!(!a.oFeatures.bStateSave || typeof a.bDestroying != "undefined")) {
-                var b, c, d, f = "{"; f += '"iCreate":' + (new Date).getTime() + ","; f += '"iStart":' + (a.oScroll.bInfinite ? 0 : a._iDisplayStart) + ","; f += '"iEnd":' + (a.oScroll.bInfinite ? a._iDisplayLength :
-a._iDisplayEnd) + ","; f += '"iLength":' + a._iDisplayLength + ","; f += '"sFilter":"' + encodeURIComponent(a.oPreviousSearch.sSearch) + '",'; f += '"sFilterEsc":' + !a.oPreviousSearch.bRegex + ","; f += '"aaSorting":[ '; for (b = 0; b < a.aaSorting.length; b++) f += "[" + a.aaSorting[b][0] + ',"' + a.aaSorting[b][1] + '"],'; f = f.substring(0, f.length - 1); f += "],"; f += '"aaSearchCols":[ '; for (b = 0; b < a.aoPreSearchCols.length; b++) f += '["' + encodeURIComponent(a.aoPreSearchCols[b].sSearch) + '",' + !a.aoPreSearchCols[b].bRegex + "],"; f = f.substring(0, f.length -
-1); f += "],"; f += '"abVisCols":[ '; for (b = 0; b < a.aoColumns.length; b++) f += a.aoColumns[b].bVisible + ","; f = f.substring(0, f.length - 1); f += "]"; b = 0; for (c = a.aoStateSave.length; b < c; b++) { d = a.aoStateSave[b].fn(a, f); if (d !== "") f = d } f += "}"; Sa(a.sCookiePrefix + a.sInstance, f, a.iCookieDuration, a.sCookiePrefix, a.fnCookieCallback)
-            }
-        } function Ta(a, b) {
-            if (a.oFeatures.bStateSave) {
-                var c, d, f; d = ta(a.sCookiePrefix + a.sInstance); if (d !== null && d !== "") {
-                    try { c = typeof i.parseJSON == "function" ? i.parseJSON(d.replace(/'/g, '"')) : eval("(" + d + ")") } catch (e) { return } d =
-0; for (f = a.aoStateLoad.length; d < f; d++) if (!a.aoStateLoad[d].fn(a, c)) return; a.oLoadedState = i.extend(true, {}, c); a._iDisplayStart = c.iStart; a.iInitDisplayStart = c.iStart; a._iDisplayEnd = c.iEnd; a._iDisplayLength = c.iLength; a.oPreviousSearch.sSearch = decodeURIComponent(c.sFilter); a.aaSorting = c.aaSorting.slice(); a.saved_aaSorting = c.aaSorting.slice(); if (typeof c.sFilterEsc != "undefined") a.oPreviousSearch.bRegex = !c.sFilterEsc; if (typeof c.aaSearchCols != "undefined") for (d = 0; d < c.aaSearchCols.length; d++) a.aoPreSearchCols[d] =
-{ sSearch: decodeURIComponent(c.aaSearchCols[d][0]), bRegex: !c.aaSearchCols[d][1] }; if (typeof c.abVisCols != "undefined") { b.saved_aoColumns = []; for (d = 0; d < c.abVisCols.length; d++) { b.saved_aoColumns[d] = {}; b.saved_aoColumns[d].bVisible = c.abVisCols[d] } }
-                }
-            }
-        } function Sa(a, b, c, d, f) {
-            var e = new Date; e.setTime(e.getTime() + c * 1E3); c = wa.location.pathname.split("/"); a = a + "_" + c.pop().replace(/[\/:]/g, "").toLowerCase(); var h; if (f !== null) {
-                h = typeof i.parseJSON == "function" ? i.parseJSON(b) : eval("(" + b + ")"); b = f(a, h, e.toGMTString(),
-c.join("/") + "/")
-            } else b = a + "=" + encodeURIComponent(b) + "; expires=" + e.toGMTString() + "; path=" + c.join("/") + "/"; f = ""; e = 9999999999999; if ((ta(a) !== null ? p.cookie.length : b.length + p.cookie.length) + 10 > 4096) {
-                a = p.cookie.split(";"); for (var j = 0, k = a.length; j < k; j++) if (a[j].indexOf(d) != -1) { var m = a[j].split("="); try { h = eval("(" + decodeURIComponent(m[1]) + ")") } catch (t) { continue } if (typeof h.iCreate != "undefined" && h.iCreate < e) { f = m[0]; e = h.iCreate } } if (f !== "") p.cookie = f + "=; expires=Thu, 01-Jan-1970 00:00:01 GMT; path=" + c.join("/") +
-"/"
-            } p.cookie = b
-        } function ta(a) { var b = wa.location.pathname.split("/"); a = a + "_" + b[b.length - 1].replace(/[\/:]/g, "").toLowerCase() + "="; b = p.cookie.split(";"); for (var c = 0; c < b.length; c++) { for (var d = b[c]; d.charAt(0) == " "; ) d = d.substring(1, d.length); if (d.indexOf(a) === 0) return decodeURIComponent(d.substring(a.length, d.length)) } return null } function W(a, b) {
-            b = b.getElementsByTagName("tr"); var c, d, f, e, h, j, k, m, t = function (O, B, F) { for (; typeof O[B][F] != "undefined"; ) F++; return F }; a.splice(0, a.length); d = 0; for (j = b.length; d <
-j; d++) a.push([]); d = 0; for (j = b.length; d < j; d++) { f = 0; for (k = b[d].childNodes.length; f < k; f++) { c = b[d].childNodes[f]; if (c.nodeName.toUpperCase() == "TD" || c.nodeName.toUpperCase() == "TH") { var q = c.getAttribute("colspan") * 1, I = c.getAttribute("rowspan") * 1; q = !q || q === 0 || q === 1 ? 1 : q; I = !I || I === 0 || I === 1 ? 1 : I; m = t(a, d, 0); for (h = 0; h < q; h++) for (e = 0; e < I; e++) { a[d + e][m + h] = { cell: c, unique: q == 1 ? true : false }; a[d + e].nTr = b[d] } } } }
-        } function S(a, b, c) {
-            var d = []; if (typeof c == "undefined") { c = a.aoHeader; if (typeof b != "undefined") { c = []; W(c, b) } } b = 0;
-            for (var f = c.length; b < f; b++) for (var e = 0, h = c[b].length; e < h; e++) if (c[b][e].unique && (typeof d[e] == "undefined" || !a.bSortCellsTop)) d[e] = c[b][e].cell; return d
-        } function Ua() {
-            var a = p.createElement("p"), b = a.style; b.width = "100%"; b.height = "200px"; var c = p.createElement("div"); b = c.style; b.position = "absolute"; b.top = "0px"; b.left = "0px"; b.visibility = "hidden"; b.width = "200px"; b.height = "150px"; b.overflow = "hidden"; c.appendChild(a); p.body.appendChild(c); b = a.offsetWidth; c.style.overflow = "scroll"; a = a.offsetWidth; if (b == a) a =
-c.clientWidth; p.body.removeChild(c); return b - a
-        } function P(a, b, c) { for (var d = 0, f = b.length; d < f; d++) for (var e = 0, h = b[d].childNodes.length; e < h; e++) if (b[d].childNodes[e].nodeType == 1) typeof c != "undefined" ? a(b[d].childNodes[e], c[d].childNodes[e]) : a(b[d].childNodes[e]) } function n(a, b, c, d) { if (typeof d == "undefined") d = c; if (typeof b[c] != "undefined") a[d] = b[c] } function da(a, b, c) { for (var d = [], f = 0, e = a.aoColumns.length; f < e; f++) d.push(H(a, b, f, c)); return d } function H(a, b, c, d) {
-            var f = a.aoColumns[c]; if ((c = f.fnGetData(a.aoData[b]._aData)) ===
-undefined) { if (a.iDrawError != a.iDraw && f.sDefaultContent === null) { J(a, 0, "Requested unknown parameter '" + f.mDataProp + "' from the data source for row " + b); a.iDrawError = a.iDraw } return f.sDefaultContent } if (c === null && f.sDefaultContent !== null) c = f.sDefaultContent; if (d == "display" && c === null) return ""; return c
-        } function N(a, b, c, d) { a.aoColumns[c].fnSetData(a.aoData[b]._aData, d) } function Z(a) {
-            if (a === null) return function () { return null }; else if (typeof a == "function") return function (c) { return a(c) }; else if (typeof a ==
-"string" && a.indexOf(".") != -1) { var b = a.split("."); return b.length == 2 ? function (c) { return c[b[0]][b[1]] } : b.length == 3 ? function (c) { return c[b[0]][b[1]][b[2]] } : function (c) { for (var d = 0, f = b.length; d < f; d++) c = c[b[d]]; return c } } else return function (c) { return c[a] }
-        } function ya(a) {
-            if (a === null) return function () { }; else if (typeof a == "function") return function (c, d) { return a(c, d) }; else if (typeof a == "string" && a.indexOf(".") != -1) {
-                var b = a.split("."); return b.length == 2 ? function (c, d) { c[b[0]][b[1]] = d } : b.length == 3 ? function (c,
-d) { c[b[0]][b[1]][b[2]] = d } : function (c, d) { for (var f = 0, e = b.length - 1; f < e; f++) c = c[b[f]]; c[b[b.length - 1]] = d }
-            } else return function (c, d) { c[a] = d }
-        } this.oApi = {}; this.fnDraw = function (a) { var b = A(this[o.iApiIndex]); if (typeof a != "undefined" && a === false) { E(b); C(b) } else ba(b) }; this.fnFilter = function (a, b, c, d, f) {
-            var e = A(this[o.iApiIndex]); if (e.oFeatures.bFilter) {
-                if (typeof c == "undefined") c = false; if (typeof d == "undefined") d = true; if (typeof f == "undefined") f = true; if (typeof b == "undefined" || b === null) {
-                    M(e, { sSearch: a, bRegex: c,
-                        bSmart: d
-                    }, 1); if (f && typeof e.aanFeatures.f != "undefined") { b = e.aanFeatures.f; c = 0; for (d = b.length; c < d; c++) i("input", b[c]).val(a) }
-                } else { e.aoPreSearchCols[b].sSearch = a; e.aoPreSearchCols[b].bRegex = c; e.aoPreSearchCols[b].bSmart = d; M(e, e.oPreviousSearch, 1) }
-            }
-        }; this.fnSettings = function () { return A(this[o.iApiIndex]) }; this.fnVersionCheck = o.fnVersionCheck; this.fnSort = function (a) { var b = A(this[o.iApiIndex]); b.aaSorting = a; R(b) }; this.fnSortListener = function (a, b, c) { ga(A(this[o.iApiIndex]), a, b, c) }; this.fnAddData = function (a,
-b) { if (a.length === 0) return []; var c = [], d, f = A(this[o.iApiIndex]); if (typeof a[0] == "object") for (var e = 0; e < a.length; e++) { d = v(f, a[e]); if (d == -1) return c; c.push(d) } else { d = v(f, a); if (d == -1) return c; c.push(d) } f.aiDisplay = f.aiDisplayMaster.slice(); if (typeof b == "undefined" || b) ba(f); return c }; this.fnDeleteRow = function (a, b, c) {
-    var d = A(this[o.iApiIndex]); a = typeof a == "object" ? U(d, a) : a; var f = d.aoData.splice(a, 1), e = i.inArray(a, d.aiDisplay); d.asDataSearch.splice(e, 1); ra(d.aiDisplayMaster, a); ra(d.aiDisplay, a); typeof b ==
-"function" && b.call(this, d, f); if (d._iDisplayStart >= d.aiDisplay.length) { d._iDisplayStart -= d._iDisplayLength; if (d._iDisplayStart < 0) d._iDisplayStart = 0 } if (typeof c == "undefined" || c) { E(d); C(d) } return f
-}; this.fnClearTable = function (a) { var b = A(this[o.iApiIndex]); ia(b); if (typeof a == "undefined" || a) C(b) }; this.fnOpen = function (a, b, c) {
-    var d = A(this[o.iApiIndex]); this.fnClose(a); var f = p.createElement("tr"), e = p.createElement("td"); f.appendChild(e); e.className = c; e.colSpan = X(d); if (typeof b.jquery != "undefined" || typeof b ==
-"object") e.appendChild(b); else e.innerHTML = b; b = i("tr", d.nTBody); i.inArray(a, b) != -1 && i(f).insertAfter(a); d.aoOpenRows.push({ nTr: f, nParent: a }); return f
-}; this.fnClose = function (a) { for (var b = A(this[o.iApiIndex]), c = 0; c < b.aoOpenRows.length; c++) if (b.aoOpenRows[c].nParent == a) { (a = b.aoOpenRows[c].nTr.parentNode) && a.removeChild(b.aoOpenRows[c].nTr); b.aoOpenRows.splice(c, 1); return 0 } return 1 }; this.fnGetData = function (a, b) {
-    var c = A(this[o.iApiIndex]); if (typeof a != "undefined") {
-        a = typeof a == "object" ? U(c, a) : a; if (typeof b !=
-"undefined") return H(c, a, b, ""); return typeof c.aoData[a] != "undefined" ? c.aoData[a]._aData : null
-    } return aa(c)
-}; this.fnGetNodes = function (a) { var b = A(this[o.iApiIndex]); if (typeof a != "undefined") return typeof b.aoData[a] != "undefined" ? b.aoData[a].nTr : null; return $(b) }; this.fnGetPosition = function (a) { var b = A(this[o.iApiIndex]), c = a.nodeName.toUpperCase(); if (c == "TR") return U(b, a); else if (c == "TD" || c == "TH") { c = U(b, a.parentNode); for (var d = Q(b, c), f = 0; f < b.aoColumns.length; f++) if (d[f] == a) return [c, pa(b, f), f] } return null };
-        this.fnUpdate = function (a, b, c, d, f) {
-            var e = A(this[o.iApiIndex]); b = typeof b == "object" ? U(e, b) : b; if (i.isArray(a) && typeof a == "object") { e.aoData[b]._aData = a.slice(); for (c = 0; c < e.aoColumns.length; c++) this.fnUpdate(H(e, b, c), b, c, false, false) } else if (typeof a == "object") { e.aoData[b]._aData = i.extend(true, {}, a); for (c = 0; c < e.aoColumns.length; c++) this.fnUpdate(H(e, b, c), b, c, false, false) } else {
-                a = a; N(e, b, c, a); if (e.aoColumns[c].fnRender !== null) {
-                    a = e.aoColumns[c].fnRender({ iDataRow: b, iDataColumn: c, aData: e.aoData[b]._aData,
-                        oSettings: e
-                    }); e.aoColumns[c].bUseRendered && N(e, b, c, a)
-                } if (e.aoData[b].nTr !== null) Q(e, b)[c].innerHTML = a
-            } c = i.inArray(b, e.aiDisplay); e.asDataSearch[c] = na(e, da(e, b, "filter")); if (typeof f == "undefined" || f) ca(e); if (typeof d == "undefined" || d) ba(e); return 0
-        }; this.fnSetColumnVis = function (a, b, c) {
-            var d = A(this[o.iApiIndex]), f, e; e = d.aoColumns.length; var h, j; if (d.aoColumns[a].bVisible != b) {
-                if (b) {
-                    for (f = j = 0; f < a; f++) d.aoColumns[f].bVisible && j++; j = j >= X(d); if (!j) for (f = a; f < e; f++) if (d.aoColumns[f].bVisible) { h = f; break } f = 0;
-                    for (e = d.aoData.length; f < e; f++) if (d.aoData[f].nTr !== null) j ? d.aoData[f].nTr.appendChild(d.aoData[f]._anHidden[a]) : d.aoData[f].nTr.insertBefore(d.aoData[f]._anHidden[a], Q(d, f)[h])
-                } else { f = 0; for (e = d.aoData.length; f < e; f++) if (d.aoData[f].nTr !== null) { h = Q(d, f)[a]; d.aoData[f]._anHidden[a] = h; h.parentNode.removeChild(h) } } d.aoColumns[a].bVisible = b; L(d, d.aoHeader); d.nTFoot && L(d, d.aoFooter); f = 0; for (e = d.aoOpenRows.length; f < e; f++) d.aoOpenRows[f].nTr.colSpan = X(d); if (typeof c == "undefined" || c) { ca(d); C(d) } sa(d)
-            }
-        }; this.fnPageChange =
-function (a, b) { var c = A(this[o.iApiIndex]); ja(c, a); E(c); if (typeof b == "undefined" || b) C(c) }; this.fnDestroy = function () {
-    var a = A(this[o.iApiIndex]), b = a.nTableWrapper.parentNode, c = a.nTBody, d, f; a.bDestroying = true; d = 0; for (f = a.aoColumns.length; d < f; d++) a.aoColumns[d].bVisible === false && this.fnSetColumnVis(d, true); i(a.nTableWrapper).find("*").andSelf().unbind(".DT"); i("tbody>tr>td." + a.oClasses.sRowEmpty, a.nTable).parent().remove(); if (a.nTable != a.nTHead.parentNode) { i(">thead", a.nTable).remove(); a.nTable.appendChild(a.nTHead) } if (a.nTFoot &&
-a.nTable != a.nTFoot.parentNode) { i(">tfoot", a.nTable).remove(); a.nTable.appendChild(a.nTFoot) } a.nTable.parentNode.removeChild(a.nTable); i(a.nTableWrapper).remove(); a.aaSorting = []; a.aaSortingFixed = []; T(a); i($(a)).removeClass(a.asStripClasses.join(" ")); if (a.bJUI) {
-        i("th", a.nTHead).removeClass([o.oStdClasses.sSortable, o.oJUIClasses.sSortableAsc, o.oJUIClasses.sSortableDesc, o.oJUIClasses.sSortableNone].join(" ")); i("th span." + o.oJUIClasses.sSortIcon, a.nTHead).remove(); i("th", a.nTHead).each(function () {
-            var e =
-i("div." + o.oJUIClasses.sSortJUIWrapper, this), h = e.contents(); i(this).append(h); e.remove()
-        })
-    } else i("th", a.nTHead).removeClass([o.oStdClasses.sSortable, o.oStdClasses.sSortableAsc, o.oStdClasses.sSortableDesc, o.oStdClasses.sSortableNone].join(" ")); a.nTableReinsertBefore ? b.insertBefore(a.nTable, a.nTableReinsertBefore) : b.appendChild(a.nTable); d = 0; for (f = a.aoData.length; d < f; d++) a.aoData[d].nTr !== null && c.appendChild(a.aoData[d].nTr); if (a.oFeatures.bAutoWidth === true) a.nTable.style.width = u(a.sDestroyWidth);
-    i(">tr:even", c).addClass(a.asDestoryStrips[0]); i(">tr:odd", c).addClass(a.asDestoryStrips[1]); d = 0; for (f = D.length; d < f; d++) D[d] == a && D.splice(d, 1); a = null
-};
-        this.fnAdjustColumnSizing = function (a) { var b = A(this[o.iApiIndex]); ca(b); if (typeof a == "undefined" || a) this.fnDraw(false); else if (b.oScroll.sX !== "" || b.oScroll.sY !== "") this.oApi._fnScrollDraw(b) }; for (var ua in o.oApi) if (ua) this[ua] = r(ua); this.oApi._fnExternApiFunc = r; this.oApi._fnInitalise = s; this.oApi._fnInitComplete = w; this.oApi._fnLanguageProcess = y; this.oApi._fnAddColumn =
-G;
-        this.oApi._fnColumnOptions = x; this.oApi._fnAddData = v; this.oApi._fnCreateTr = z; this.oApi._fnGatherData = Y; this.oApi._fnBuildHead = V; this.oApi._fnDrawHead = L; this.oApi._fnDraw = C; this.oApi._fnReDraw = ba; this.oApi._fnAjaxUpdate = za; this.oApi._fnAjaxUpdateDraw = Aa; this.oApi._fnAddOptionsHtml = xa; this.oApi._fnFeatureHtmlTable = Fa; this.oApi._fnScrollDraw = Ia; this.oApi._fnAjustColumnSizing = ca; this.oApi._fnFeatureHtmlFilter = Da; this.oApi._fnFilterComplete = M; this.oApi._fnFilterCustom = Ma; this.oApi._fnFilterColumn = La;
-        this.oApi._fnFilter = Ka; this.oApi._fnBuildSearchArray = ka; this.oApi._fnBuildSearchRow = na; this.oApi._fnFilterCreateSearch = la; this.oApi._fnDataToSearch = ma; this.oApi._fnSort = R; this.oApi._fnSortAttachListener = ga; this.oApi._fnSortingClasses = T; this.oApi._fnFeatureHtmlPaginate = Ha; this.oApi._fnPageChange = ja; this.oApi._fnFeatureHtmlInfo = Ga; this.oApi._fnUpdateInfo = Na; this.oApi._fnFeatureHtmlLength = Ca; this.oApi._fnFeatureHtmlProcessing = Ea; this.oApi._fnProcessingDisplay = K; this.oApi._fnVisibleToColumnIndex = Ja; this.oApi._fnColumnIndexToVisible =
-pa;
-        this.oApi._fnNodeToDataIndex = U;
-        this.oApi._fnVisbleColumns = X; this.oApi._fnCalculateEnd = E; this.oApi._fnConvertToWidth = Oa; this.oApi._fnCalculateColumnWidths = ea; this.oApi._fnScrollingWidthAdjust = Qa; this.oApi._fnGetWidestNode = Pa; this.oApi._fnGetMaxLenString = Ra; this.oApi._fnStringToCss = u; this.oApi._fnArrayCmp = Va; this.oApi._fnDetectType = fa; this.oApi._fnSettingsFromNode = A; this.oApi._fnGetDataMaster = aa; this.oApi._fnGetTrNodes = $; this.oApi._fnGetTdNodes = Q; this.oApi._fnEscapeRegex = oa; this.oApi._fnDeleteIndex =
-ra;
-        this.oApi._fnReOrderIndex = Ba; 
-        this.oApi._fnColumnOrdering = ha; 
-        this.oApi._fnLog = J;
-        this.oApi._fnClearTable = ia;
-        this.oApi._fnSaveState = sa;
-        this.oApi._fnLoadState = Ta;
-        this.oApi._fnCreateCookie = Sa;
-        this.oApi._fnReadCookie = ta; 
-        this.oApi._fnDetectHeader = W; this.oApi._fnGetUniqueThs = S; this.oApi._fnScrollBarWidth = Ua; this.oApi._fnApplyToChildren = P; this.oApi._fnMap = n; this.oApi._fnGetRowData = da; this.oApi._fnGetCellData = H; this.oApi._fnSetCellData = N; this.oApi._fnGetObjectDataFn = Z; this.oApi._fnSetObjectDataFn = ya; var va =
-this;
-        return this.each(function () {
-            var a = 0, b, c, d, f; a = 0; for (b = D.length; a < b; a++) {
-                if (D[a].nTable == this) if (typeof g == "undefined" || typeof g.bRetrieve != "undefined" && g.bRetrieve === true) return D[a].oInstance; else if (typeof g.bDestroy != "undefined" && g.bDestroy === true) { D[a].oInstance.fnDestroy(); break } else {
-                    J(D[a], 0, "Cannot reinitialise DataTable.\n\nTo retrieve the DataTables object for this table, please pass either no arguments to the dataTable() function, or set bRetrieve to true. Alternatively, to destory the old table and create a new one, set bDestroy to true (note that a lot of changes to the configuration can be made through the API which is usually much faster).");
-                    return
-                } if (D[a].sTableId !== "" && D[a].sTableId == this.getAttribute("id")) { D.splice(a, 1); break }
-            } var e = new l; D.push(e); var h = false, j = false; a = this.getAttribute("id"); if (a !== null) { e.sTableId = a; e.sInstance = a } else e.sInstance = o._oExternConfig.iNextUnique++; if (this.nodeName.toLowerCase() != "table") J(e, 0, "Attempted to initialise DataTables on a node which is not a table: " + this.nodeName); else {
-                e.nTable = this; e.oInstance = va.length == 1 ? va : i(this).dataTable(); e.oApi = va.oApi; e.sDestroyWidth = i(this).width(); if (typeof g !=
-"undefined" && g !== null) {
-                    e.oInit = g; n(e.oFeatures, g, "bPaginate");
-                    n(e.oFeatures, g, "bLengthChange");
-                    n(e.oFeatures, g, "bFilter");
-                    n(e.oFeatures, g, "bSort");
-                    n(e.oFeatures, g, "bInfo");
-                    n(e.oFeatures, g, "bProcessing");
-                    n(e.oFeatures, g, "bAutoWidth");
-                    n(e.oFeatures, g, "bSortClasses");
-                    n(e.oFeatures, g, "bServerSide");
-                    n(e.oFeatures, g, "bDeferRender");
-                    n(e.oScroll, g, "sScrollX", "sX");
-                    n(e.oScroll, g, "sScrollXInner", "sXInner");
-                    n(e.oScroll, g, "sScrollY", "sY");
-                    n(e.oScroll, g, "bScrollCollapse", "bCollapse");
-                    n(e.oScroll, g, "bScrollInfinite", "bInfinite");
-                    n(e.oScroll, g, "iScrollLoadGap", "iLoadGap");
-                    n(e.oScroll, g, "bScrollAutoCss", "bAutoCss");
-                    n(e, g, "asStripClasses");
-                    n(e, g, "fnPreDrawCallback");
-                    n(e, g, "fnRowCallback"); n(e, g, "fnHeaderCallback"); n(e, g, "fnFooterCallback"); n(e, g, "fnCookieCallback"); n(e, g, "fnInitComplete"); n(e, g, "fnServerData"); n(e, g, "fnFormatNumber"); n(e, g, "aaSorting"); n(e, g, "aaSortingFixed"); n(e, g, "aLengthMenu"); n(e, g, "sPaginationType"); n(e, g, "sAjaxSource"); n(e, g, "sAjaxDataProp"); n(e, g, "iCookieDuration"); n(e, g, "sCookiePrefix");
-                    n(e, g, "sDom");
-                    n(e, g, "bSortCellsTop");
-                    n(e, g, "oSearch", "oPreviousSearch");
-                    n(e, g, "aoSearchCols", "aoPreSearchCols");
-                    n(e, g, "iDisplayLength", "_iDisplayLength");
-                    n(e, g, "bJQueryUI", "bJUI");
-                    n(e.oLanguage, g, "fnInfoCallback");
-                    typeof g.fnDrawCallback == "function" && e.aoDrawCallback.push({
-                        fn: g.fnDrawCallback, sName: "user"
-                    });
-                    typeof g.fnStateSaveCallback == "function" && e.aoStateSave.push({
-                        fn: g.fnStateSaveCallback, sName: "user"
-                    });
-                    typeof g.fnStateLoadCallback == "function" && e.aoStateLoad.push({
-                        fn: g.fnStateLoadCallback, sName: "user"
-                    });
-                    if (e.oFeatures.bServerSide && e.oFeatures.bSort && e.oFeatures.bSortClasses) e.aoDrawCallback.push({ fn: T, sName: "server_side_sort_classes" }); else e.oFeatures.bDeferRender && e.aoDrawCallback.push({ fn: T, sName: "defer_sort_classes" }); if (typeof g.bJQueryUI != "undefined" && g.bJQueryUI) { e.oClasses = o.oJUIClasses; if (typeof g.sDom == "undefined") e.sDom = '<"H"lfr>t<"F"ip>' } if (e.oScroll.sX !== "" || e.oScroll.sY !== "") e.oScroll.iBarWidth = Ua(); if (typeof g.iDisplayStart != "undefined" && typeof e.iInitDisplayStart == "undefined") {
-                        e.iInitDisplayStart = g.iDisplayStart; e._iDisplayStart = g.iDisplayStart
-                    } if (typeof g.bStateSave != "undefined") { e.oFeatures.bStateSave = g.bStateSave; Ta(e, g); e.aoDrawCallback.push({ fn: sa, sName: "state_save" }) } if (typeof g.iDeferLoading != "undefined") { e.bDeferLoading = true; e._iRecordsTotal = g.iDeferLoading; e._iRecordsDisplay = g.iDeferLoading } if (typeof g.aaData != "undefined") j = true; if (typeof g != "undefined" && typeof g.aoData != "undefined") g.aoColumns = g.aoData; if (typeof g.oLanguage != "undefined") if (typeof g.oLanguage.sUrl != "undefined" && g.oLanguage.sUrl !==
-"") { e.oLanguage.sUrl = g.oLanguage.sUrl; i.getJSON(e.oLanguage.sUrl, null, function (t) { y(e, t, true) }); h = true } else y(e, g.oLanguage, false)
-                } else g = {}; if (typeof g.asStripClasses == "undefined") { e.asStripClasses.push(e.oClasses.sStripOdd); e.asStripClasses.push(e.oClasses.sStripEven) } c = false; d = i(">tbody>tr", this); a = 0; for (b = e.asStripClasses.length; a < b; a++) if (d.filter(":lt(2)").hasClass(e.asStripClasses[a])) { c = true; break } if (c) {
-                    e.asDestoryStrips = ["", ""]; if (i(d[0]).hasClass(e.oClasses.sStripOdd)) e.asDestoryStrips[0] +=
-e.oClasses.sStripOdd + " "; if (i(d[0]).hasClass(e.oClasses.sStripEven)) e.asDestoryStrips[0] += e.oClasses.sStripEven; if (i(d[1]).hasClass(e.oClasses.sStripOdd)) e.asDestoryStrips[1] += e.oClasses.sStripOdd + " "; if (i(d[1]).hasClass(e.oClasses.sStripEven)) e.asDestoryStrips[1] += e.oClasses.sStripEven; d.removeClass(e.asStripClasses.join(" "))
-                } c = []; var k; a = this.getElementsByTagName("thead"); if (a.length !== 0) { W(e.aoHeader, a[0]); c = S(e) } if (typeof g.aoColumns == "undefined") { k = []; a = 0; for (b = c.length; a < b; a++) k.push(null) } else k =
-g.aoColumns; a = 0; for (b = k.length; a < b; a++) { if (typeof g.saved_aoColumns != "undefined" && g.saved_aoColumns.length == b) { if (k[a] === null) k[a] = {}; k[a].bVisible = g.saved_aoColumns[a].bVisible } G(e, c ? c[a] : null) } if (typeof g.aoColumnDefs != "undefined") for (a = g.aoColumnDefs.length - 1; a >= 0; a--) {
-                    var m = g.aoColumnDefs[a].aTargets; i.isArray(m) || J(e, 1, "aTargets must be an array of targets, not a " + typeof m); c = 0; for (d = m.length; c < d; c++) if (typeof m[c] == "number" && m[c] >= 0) { for (; e.aoColumns.length <= m[c]; ) G(e); x(e, m[c], g.aoColumnDefs[a]) } else if (typeof m[c] ==
-"number" && m[c] < 0) x(e, e.aoColumns.length + m[c], g.aoColumnDefs[a]); else if (typeof m[c] == "string") { b = 0; for (f = e.aoColumns.length; b < f; b++) if (m[c] == "_all" || i(e.aoColumns[b].nTh).hasClass(m[c])) x(e, b, g.aoColumnDefs[a]) }
-                } if (typeof k != "undefined") { a = 0; for (b = k.length; a < b; a++) x(e, a, k[a]) } a = 0; for (b = e.aaSorting.length; a < b; a++) {
-                    if (e.aaSorting[a][0] >= e.aoColumns.length)
-                        e.aaSorting[a][0] = 0; k = e.aoColumns[e.aaSorting[a][0]];
-                    if (typeof e.aaSorting[a][2] == "undefined") e.aaSorting[a][2] = 0;
-                    if (typeof g.aaSorting == "undefined" && typeof e.saved_aaSorting == "undefined")
-                        e.aaSorting[a][1] = k.asSorting[0];
-                    c = 0; for (d = k.asSorting.length;
-                   c < d; c++) if (e.aaSorting[a][1] == k.asSorting[c]) {
-                        e.aaSorting[a][2] = c; break
-                    }
-                } T(e); a = i(">thead", this); if (a.length === 0) {
-                    a = [p.createElement("thead")]; this.appendChild(a[0])
-                } e.nTHead = a[0]; a = i(">tbody", this);
-                if (a.length === 0) {
-                    a = [p.createElement("tbody")];
-                    this.appendChild(a[0])
-                } e.nTBody = a[0]; a = i(">tfoot", this);
-                if (a.length > 0) { e.nTFoot = a[0]; W(e.aoFooter, e.nTFoot) }
-                if (j)
-                    for (a = 0; a < g.aaData.length; a++) v(e, g.aaData[a]);
-                else Y(e); e.aiDisplay = e.aiDisplayMaster.slice(); e.bInitialised = true; h === false && s(e)
-            }
-        })
-    }
-})(jQuery, window, document);
+ * File:        jquery.dataTables.min.js
+ * Version:     1.9.4
+ * Author:      Allan Jardine (www.sprymedia.co.uk)
+ * Info:        www.datatables.net
+ * 
+ * Copyright 2008-2012 Allan Jardine, all rights reserved.
+ *
+ * This source file is free software, under either the GPL v2 license or a
+ * BSD style license, available at:
+ *   http://datatables.net/license_gpl2
+ *   http://datatables.net/license_bsd
+ * 
+ * This source file is distributed in the hope that it will be useful, but 
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
+ */
+(function(X,l,n){var L=function(h){var j=function(e){function o(a,b){var c=j.defaults.columns,d=a.aoColumns.length,c=h.extend({},j.models.oColumn,c,{sSortingClass:a.oClasses.sSortable,sSortingClassJUI:a.oClasses.sSortJUI,nTh:b?b:l.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.oDefaults:d});a.aoColumns.push(c);if(a.aoPreSearchCols[d]===n||null===a.aoPreSearchCols[d])a.aoPreSearchCols[d]=h.extend({},j.models.oSearch);else if(c=a.aoPreSearchCols[d],
+c.bRegex===n&&(c.bRegex=!0),c.bSmart===n&&(c.bSmart=!0),c.bCaseInsensitive===n)c.bCaseInsensitive=!0;m(a,d,null)}function m(a,b,c){var d=a.aoColumns[b];c!==n&&null!==c&&(c.mDataProp&&!c.mData&&(c.mData=c.mDataProp),c.sType!==n&&(d.sType=c.sType,d._bAutoType=!1),h.extend(d,c),p(d,c,"sWidth","sWidthOrig"),c.iDataSort!==n&&(d.aDataSort=[c.iDataSort]),p(d,c,"aDataSort"));var i=d.mRender?Q(d.mRender):null,f=Q(d.mData);d.fnGetData=function(a,b){var c=f(a,b);return d.mRender&&b&&""!==b?i(c,b,a):c};d.fnSetData=
+L(d.mData);a.oFeatures.bSort||(d.bSortable=!1);!d.bSortable||-1==h.inArray("asc",d.asSorting)&&-1==h.inArray("desc",d.asSorting)?(d.sSortingClass=a.oClasses.sSortableNone,d.sSortingClassJUI=""):-1==h.inArray("asc",d.asSorting)&&-1==h.inArray("desc",d.asSorting)?(d.sSortingClass=a.oClasses.sSortable,d.sSortingClassJUI=a.oClasses.sSortJUI):-1!=h.inArray("asc",d.asSorting)&&-1==h.inArray("desc",d.asSorting)?(d.sSortingClass=a.oClasses.sSortableAsc,d.sSortingClassJUI=a.oClasses.sSortJUIAscAllowed):-1==
+h.inArray("asc",d.asSorting)&&-1!=h.inArray("desc",d.asSorting)&&(d.sSortingClass=a.oClasses.sSortableDesc,d.sSortingClassJUI=a.oClasses.sSortJUIDescAllowed)}function k(a){if(!1===a.oFeatures.bAutoWidth)return!1;da(a);for(var b=0,c=a.aoColumns.length;b<c;b++)a.aoColumns[b].nTh.style.width=a.aoColumns[b].sWidth}function G(a,b){var c=r(a,"bVisible");return"number"===typeof c[b]?c[b]:null}function R(a,b){var c=r(a,"bVisible"),c=h.inArray(b,c);return-1!==c?c:null}function t(a){return r(a,"bVisible").length}
+function r(a,b){var c=[];h.map(a.aoColumns,function(a,i){a[b]&&c.push(i)});return c}function B(a){for(var b=j.ext.aTypes,c=b.length,d=0;d<c;d++){var i=b[d](a);if(null!==i)return i}return"string"}function u(a,b){for(var c=b.split(","),d=[],i=0,f=a.aoColumns.length;i<f;i++)for(var g=0;g<f;g++)if(a.aoColumns[i].sName==c[g]){d.push(g);break}return d}function M(a){for(var b="",c=0,d=a.aoColumns.length;c<d;c++)b+=a.aoColumns[c].sName+",";return b.length==d?"":b.slice(0,-1)}function ta(a,b,c,d){var i,f,
+g,e,w;if(b)for(i=b.length-1;0<=i;i--){var j=b[i].aTargets;h.isArray(j)||D(a,1,"aTargets must be an array of targets, not a "+typeof j);f=0;for(g=j.length;f<g;f++)if("number"===typeof j[f]&&0<=j[f]){for(;a.aoColumns.length<=j[f];)o(a);d(j[f],b[i])}else if("number"===typeof j[f]&&0>j[f])d(a.aoColumns.length+j[f],b[i]);else if("string"===typeof j[f]){e=0;for(w=a.aoColumns.length;e<w;e++)("_all"==j[f]||h(a.aoColumns[e].nTh).hasClass(j[f]))&&d(e,b[i])}}if(c){i=0;for(a=c.length;i<a;i++)d(i,c[i])}}function H(a,
+b){var c;c=h.isArray(b)?b.slice():h.extend(!0,{},b);var d=a.aoData.length,i=h.extend(!0,{},j.models.oRow);i._aData=c;a.aoData.push(i);for(var f,i=0,g=a.aoColumns.length;i<g;i++)c=a.aoColumns[i],"function"===typeof c.fnRender&&c.bUseRendered&&null!==c.mData?F(a,d,i,S(a,d,i)):F(a,d,i,v(a,d,i)),c._bAutoType&&"string"!=c.sType&&(f=v(a,d,i,"type"),null!==f&&""!==f&&(f=B(f),null===c.sType?c.sType=f:c.sType!=f&&"html"!=c.sType&&(c.sType="string")));a.aiDisplayMaster.push(d);a.oFeatures.bDeferRender||ea(a,
+d);return d}function ua(a){var b,c,d,i,f,g,e;if(a.bDeferLoading||null===a.sAjaxSource)for(b=a.nTBody.firstChild;b;){if("TR"==b.nodeName.toUpperCase()){c=a.aoData.length;b._DT_RowIndex=c;a.aoData.push(h.extend(!0,{},j.models.oRow,{nTr:b}));a.aiDisplayMaster.push(c);f=b.firstChild;for(d=0;f;){g=f.nodeName.toUpperCase();if("TD"==g||"TH"==g)F(a,c,d,h.trim(f.innerHTML)),d++;f=f.nextSibling}}b=b.nextSibling}i=T(a);d=[];b=0;for(c=i.length;b<c;b++)for(f=i[b].firstChild;f;)g=f.nodeName.toUpperCase(),("TD"==
+g||"TH"==g)&&d.push(f),f=f.nextSibling;c=0;for(i=a.aoColumns.length;c<i;c++){e=a.aoColumns[c];null===e.sTitle&&(e.sTitle=e.nTh.innerHTML);var w=e._bAutoType,o="function"===typeof e.fnRender,k=null!==e.sClass,n=e.bVisible,m,p;if(w||o||k||!n){g=0;for(b=a.aoData.length;g<b;g++)f=a.aoData[g],m=d[g*i+c],w&&"string"!=e.sType&&(p=v(a,g,c,"type"),""!==p&&(p=B(p),null===e.sType?e.sType=p:e.sType!=p&&"html"!=e.sType&&(e.sType="string"))),e.mRender?m.innerHTML=v(a,g,c,"display"):e.mData!==c&&(m.innerHTML=v(a,
+g,c,"display")),o&&(p=S(a,g,c),m.innerHTML=p,e.bUseRendered&&F(a,g,c,p)),k&&(m.className+=" "+e.sClass),n?f._anHidden[c]=null:(f._anHidden[c]=m,m.parentNode.removeChild(m)),e.fnCreatedCell&&e.fnCreatedCell.call(a.oInstance,m,v(a,g,c,"display"),f._aData,g,c)}}if(0!==a.aoRowCreatedCallback.length){b=0;for(c=a.aoData.length;b<c;b++)f=a.aoData[b],A(a,"aoRowCreatedCallback",null,[f.nTr,f._aData,b])}}function I(a,b){return b._DT_RowIndex!==n?b._DT_RowIndex:null}function fa(a,b,c){for(var b=J(a,b),d=0,a=
+a.aoColumns.length;d<a;d++)if(b[d]===c)return d;return-1}function Y(a,b,c,d){for(var i=[],f=0,g=d.length;f<g;f++)i.push(v(a,b,d[f],c));return i}function v(a,b,c,d){var i=a.aoColumns[c];if((c=i.fnGetData(a.aoData[b]._aData,d))===n)return a.iDrawError!=a.iDraw&&null===i.sDefaultContent&&(D(a,0,"Requested unknown parameter "+("function"==typeof i.mData?"{mData function}":"'"+i.mData+"'")+" from the data source for row "+b),a.iDrawError=a.iDraw),i.sDefaultContent;if(null===c&&null!==i.sDefaultContent)c=
+i.sDefaultContent;else if("function"===typeof c)return c();return"display"==d&&null===c?"":c}function F(a,b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d)}function Q(a){if(null===a)return function(){return null};if("function"===typeof a)return function(b,d,i){return a(b,d,i)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("["))){var b=function(a,d,i){var f=i.split("."),g;if(""!==i){var e=0;for(g=f.length;e<g;e++){if(i=f[e].match(U)){f[e]=f[e].replace(U,"");""!==f[e]&&(a=a[f[e]]);
+g=[];f.splice(0,e+1);for(var f=f.join("."),e=0,h=a.length;e<h;e++)g.push(b(a[e],d,f));a=i[0].substring(1,i[0].length-1);a=""===a?g:g.join(a);break}if(null===a||a[f[e]]===n)return n;a=a[f[e]]}}return a};return function(c,d){return b(c,d,a)}}return function(b){return b[a]}}function L(a){if(null===a)return function(){};if("function"===typeof a)return function(b,d){a(b,"set",d)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("["))){var b=function(a,d,i){var i=i.split("."),f,g,e=0;for(g=
+i.length-1;e<g;e++){if(f=i[e].match(U)){i[e]=i[e].replace(U,"");a[i[e]]=[];f=i.slice();f.splice(0,e+1);g=f.join(".");for(var h=0,j=d.length;h<j;h++)f={},b(f,d[h],g),a[i[e]].push(f);return}if(null===a[i[e]]||a[i[e]]===n)a[i[e]]={};a=a[i[e]]}a[i[i.length-1].replace(U,"")]=d};return function(c,d){return b(c,d,a)}}return function(b,d){b[a]=d}}function Z(a){for(var b=[],c=a.aoData.length,d=0;d<c;d++)b.push(a.aoData[d]._aData);return b}function ga(a){a.aoData.splice(0,a.aoData.length);a.aiDisplayMaster.splice(0,
+a.aiDisplayMaster.length);a.aiDisplay.splice(0,a.aiDisplay.length);y(a)}function ha(a,b){for(var c=-1,d=0,i=a.length;d<i;d++)a[d]==b?c=d:a[d]>b&&a[d]--; -1!=c&&a.splice(c,1)}function S(a,b,c){var d=a.aoColumns[c];return d.fnRender({iDataRow:b,iDataColumn:c,oSettings:a,aData:a.aoData[b]._aData,mDataProp:d.mData},v(a,b,c,"display"))}function ea(a,b){var c=a.aoData[b],d;if(null===c.nTr){c.nTr=l.createElement("tr");c.nTr._DT_RowIndex=b;c._aData.DT_RowId&&(c.nTr.id=c._aData.DT_RowId);c._aData.DT_RowClass&&
+(c.nTr.className=c._aData.DT_RowClass);for(var i=0,f=a.aoColumns.length;i<f;i++){var g=a.aoColumns[i];d=l.createElement(g.sCellType);d.innerHTML="function"===typeof g.fnRender&&(!g.bUseRendered||null===g.mData)?S(a,b,i):v(a,b,i,"display");null!==g.sClass&&(d.className=g.sClass);g.bVisible?(c.nTr.appendChild(d),c._anHidden[i]=null):c._anHidden[i]=d;g.fnCreatedCell&&g.fnCreatedCell.call(a.oInstance,d,v(a,b,i,"display"),c._aData,b,i)}A(a,"aoRowCreatedCallback",null,[c.nTr,c._aData,b])}}function va(a){var b,
+c,d;if(0!==h("th, td",a.nTHead).length){b=0;for(d=a.aoColumns.length;b<d;b++)if(c=a.aoColumns[b].nTh,c.setAttribute("role","columnheader"),a.aoColumns[b].bSortable&&(c.setAttribute("tabindex",a.iTabIndex),c.setAttribute("aria-controls",a.sTableId)),null!==a.aoColumns[b].sClass&&h(c).addClass(a.aoColumns[b].sClass),a.aoColumns[b].sTitle!=c.innerHTML)c.innerHTML=a.aoColumns[b].sTitle}else{var i=l.createElement("tr");b=0;for(d=a.aoColumns.length;b<d;b++)c=a.aoColumns[b].nTh,c.innerHTML=a.aoColumns[b].sTitle,
+c.setAttribute("tabindex","0"),null!==a.aoColumns[b].sClass&&h(c).addClass(a.aoColumns[b].sClass),i.appendChild(c);h(a.nTHead).html("")[0].appendChild(i);V(a.aoHeader,a.nTHead)}h(a.nTHead).children("tr").attr("role","row");if(a.bJUI){b=0;for(d=a.aoColumns.length;b<d;b++){c=a.aoColumns[b].nTh;i=l.createElement("div");i.className=a.oClasses.sSortJUIWrapper;h(c).contents().appendTo(i);var f=l.createElement("span");f.className=a.oClasses.sSortIcon;i.appendChild(f);c.appendChild(i)}}if(a.oFeatures.bSort)for(b=
+0;b<a.aoColumns.length;b++)!1!==a.aoColumns[b].bSortable?ia(a,a.aoColumns[b].nTh,b):h(a.aoColumns[b].nTh).addClass(a.oClasses.sSortableNone);""!==a.oClasses.sFooterTH&&h(a.nTFoot).children("tr").children("th").addClass(a.oClasses.sFooterTH);if(null!==a.nTFoot){c=N(a,null,a.aoFooter);b=0;for(d=a.aoColumns.length;b<d;b++)c[b]&&(a.aoColumns[b].nTf=c[b],a.aoColumns[b].sClass&&h(c[b]).addClass(a.aoColumns[b].sClass))}}function W(a,b,c){var d,i,f,g=[],e=[],h=a.aoColumns.length,j;c===n&&(c=!1);d=0;for(i=
+b.length;d<i;d++){g[d]=b[d].slice();g[d].nTr=b[d].nTr;for(f=h-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[d].splice(f,1);e.push([])}d=0;for(i=g.length;d<i;d++){if(a=g[d].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[d].length;f<b;f++)if(j=h=1,e[d][f]===n){a.appendChild(g[d][f].cell);for(e[d][f]=1;g[d+h]!==n&&g[d][f].cell==g[d+h][f].cell;)e[d+h][f]=1,h++;for(;g[d][f+j]!==n&&g[d][f].cell==g[d][f+j].cell;){for(c=0;c<h;c++)e[d+c][f+j]=1;j++}g[d][f].cell.rowSpan=h;g[d][f].cell.colSpan=j}}}function x(a){var b=
+A(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))E(a,!1);else{var c,d,b=[],i=0,f=a.asStripeClasses.length;c=a.aoOpenRows.length;a.bDrawing=!0;a.iInitDisplayStart!==n&&-1!=a.iInitDisplayStart&&(a._iDisplayStart=a.oFeatures.bServerSide?a.iInitDisplayStart:a.iInitDisplayStart>=a.fnRecordsDisplay()?0:a.iInitDisplayStart,a.iInitDisplayStart=-1,y(a));if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++;else if(a.oFeatures.bServerSide){if(!a.bDestroying&&!wa(a))return}else a.iDraw++;if(0!==a.aiDisplay.length){var g=
+a._iDisplayStart;d=a._iDisplayEnd;a.oFeatures.bServerSide&&(g=0,d=a.aoData.length);for(;g<d;g++){var e=a.aoData[a.aiDisplay[g]];null===e.nTr&&ea(a,a.aiDisplay[g]);var j=e.nTr;if(0!==f){var o=a.asStripeClasses[i%f];e._sRowStripe!=o&&(h(j).removeClass(e._sRowStripe).addClass(o),e._sRowStripe=o)}A(a,"aoRowCallback",null,[j,a.aoData[a.aiDisplay[g]]._aData,i,g]);b.push(j);i++;if(0!==c)for(e=0;e<c;e++)if(j==a.aoOpenRows[e].nParent){b.push(a.aoOpenRows[e].nTr);break}}}else b[0]=l.createElement("tr"),a.asStripeClasses[0]&&
+(b[0].className=a.asStripeClasses[0]),c=a.oLanguage,f=c.sZeroRecords,1==a.iDraw&&null!==a.sAjaxSource&&!a.oFeatures.bServerSide?f=c.sLoadingRecords:c.sEmptyTable&&0===a.fnRecordsTotal()&&(f=c.sEmptyTable),c=l.createElement("td"),c.setAttribute("valign","top"),c.colSpan=t(a),c.className=a.oClasses.sRowEmpty,c.innerHTML=ja(a,f),b[i].appendChild(c);A(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Z(a),a._iDisplayStart,a.fnDisplayEnd(),a.aiDisplay]);A(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],
+Z(a),a._iDisplayStart,a.fnDisplayEnd(),a.aiDisplay]);i=l.createDocumentFragment();c=l.createDocumentFragment();if(a.nTBody){f=a.nTBody.parentNode;c.appendChild(a.nTBody);if(!a.oScroll.bInfinite||!a._bInitComplete||a.bSorted||a.bFiltered)for(;c=a.nTBody.firstChild;)a.nTBody.removeChild(c);c=0;for(d=b.length;c<d;c++)i.appendChild(b[c]);a.nTBody.appendChild(i);null!==f&&f.appendChild(a.nTBody)}A(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1;a.oFeatures.bServerSide&&(E(a,!1),
+a._bInitComplete||$(a))}}function aa(a){a.oFeatures.bSort?O(a,a.oPreviousSearch):a.oFeatures.bFilter?K(a,a.oPreviousSearch):(y(a),x(a))}function xa(a){var b=h("<div></div>")[0];a.nTable.parentNode.insertBefore(b,a.nTable);a.nTableWrapper=h('<div id="'+a.sTableId+'_wrapper" class="'+a.oClasses.sWrapper+'" role="grid"></div>')[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var c=a.nTableWrapper,d=a.sDom.split(""),i,f,g,e,w,o,k,m=0;m<d.length;m++){f=0;g=d[m];if("<"==g){e=h("<div></div>")[0];w=d[m+
+1];if("'"==w||'"'==w){o="";for(k=2;d[m+k]!=w;)o+=d[m+k],k++;"H"==o?o=a.oClasses.sJUIHeader:"F"==o&&(o=a.oClasses.sJUIFooter);-1!=o.indexOf(".")?(w=o.split("."),e.id=w[0].substr(1,w[0].length-1),e.className=w[1]):"#"==o.charAt(0)?e.id=o.substr(1,o.length-1):e.className=o;m+=k}c.appendChild(e);c=e}else if(">"==g)c=c.parentNode;else if("l"==g&&a.oFeatures.bPaginate&&a.oFeatures.bLengthChange)i=ya(a),f=1;else if("f"==g&&a.oFeatures.bFilter)i=za(a),f=1;else if("r"==g&&a.oFeatures.bProcessing)i=Aa(a),f=
+1;else if("t"==g)i=Ba(a),f=1;else if("i"==g&&a.oFeatures.bInfo)i=Ca(a),f=1;else if("p"==g&&a.oFeatures.bPaginate)i=Da(a),f=1;else if(0!==j.ext.aoFeatures.length){e=j.ext.aoFeatures;k=0;for(w=e.length;k<w;k++)if(g==e[k].cFeature){(i=e[k].fnInit(a))&&(f=1);break}}1==f&&null!==i&&("object"!==typeof a.aanFeatures[g]&&(a.aanFeatures[g]=[]),a.aanFeatures[g].push(i),c.appendChild(i))}b.parentNode.replaceChild(a.nTableWrapper,b)}function V(a,b){var c=h(b).children("tr"),d,i,f,g,e,j,o,k,m,p;a.splice(0,a.length);
+f=0;for(j=c.length;f<j;f++)a.push([]);f=0;for(j=c.length;f<j;f++){d=c[f];for(i=d.firstChild;i;){if("TD"==i.nodeName.toUpperCase()||"TH"==i.nodeName.toUpperCase()){k=1*i.getAttribute("colspan");m=1*i.getAttribute("rowspan");k=!k||0===k||1===k?1:k;m=!m||0===m||1===m?1:m;g=0;for(e=a[f];e[g];)g++;o=g;p=1===k?!0:!1;for(e=0;e<k;e++)for(g=0;g<m;g++)a[f+g][o+e]={cell:i,unique:p},a[f+g].nTr=d}i=i.nextSibling}}}function N(a,b,c){var d=[];c||(c=a.aoHeader,b&&(c=[],V(c,b)));for(var b=0,i=c.length;b<i;b++)for(var f=
+0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!d[f]||!a.bSortCellsTop))d[f]=c[b][f].cell;return d}function wa(a){if(a.bAjaxDataGet){a.iDraw++;E(a,!0);var b=Ea(a);ka(a,b);a.fnServerData.call(a.oInstance,a.sAjaxSource,b,function(b){Fa(a,b)},a);return!1}return!0}function Ea(a){var b=a.aoColumns.length,c=[],d,i,f,g;c.push({name:"sEcho",value:a.iDraw});c.push({name:"iColumns",value:b});c.push({name:"sColumns",value:M(a)});c.push({name:"iDisplayStart",value:a._iDisplayStart});c.push({name:"iDisplayLength",
+value:!1!==a.oFeatures.bPaginate?a._iDisplayLength:-1});for(f=0;f<b;f++)d=a.aoColumns[f].mData,c.push({name:"mDataProp_"+f,value:"function"===typeof d?"function":d});if(!1!==a.oFeatures.bFilter){c.push({name:"sSearch",value:a.oPreviousSearch.sSearch});c.push({name:"bRegex",value:a.oPreviousSearch.bRegex});for(f=0;f<b;f++)c.push({name:"sSearch_"+f,value:a.aoPreSearchCols[f].sSearch}),c.push({name:"bRegex_"+f,value:a.aoPreSearchCols[f].bRegex}),c.push({name:"bSearchable_"+f,value:a.aoColumns[f].bSearchable})}if(!1!==
+a.oFeatures.bSort){var e=0;d=null!==a.aaSortingFixed?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(f=0;f<d.length;f++){i=a.aoColumns[d[f][0]].aDataSort;for(g=0;g<i.length;g++)c.push({name:"iSortCol_"+e,value:i[g]}),c.push({name:"sSortDir_"+e,value:d[f][1]}),e++}c.push({name:"iSortingCols",value:e});for(f=0;f<b;f++)c.push({name:"bSortable_"+f,value:a.aoColumns[f].bSortable})}return c}function ka(a,b){A(a,"aoServerParams","serverParams",[b])}function Fa(a,b){if(b.sEcho!==n){if(1*b.sEcho<
+a.iDraw)return;a.iDraw=1*b.sEcho}(!a.oScroll.bInfinite||a.oScroll.bInfinite&&(a.bSorted||a.bFiltered))&&ga(a);a._iRecordsTotal=parseInt(b.iTotalRecords,10);a._iRecordsDisplay=parseInt(b.iTotalDisplayRecords,10);var c=M(a),c=b.sColumns!==n&&""!==c&&b.sColumns!=c,d;c&&(d=u(a,b.sColumns));for(var i=Q(a.sAjaxDataProp)(b),f=0,g=i.length;f<g;f++)if(c){for(var e=[],h=0,j=a.aoColumns.length;h<j;h++)e.push(i[f][d[h]]);H(a,e)}else H(a,i[f]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;x(a);a.bAjaxDataGet=
+!0;E(a,!1)}function za(a){var b=a.oPreviousSearch,c=a.oLanguage.sSearch,c=-1!==c.indexOf("_INPUT_")?c.replace("_INPUT_",'<input type="text" />'):""===c?'<input type="text" />':c+' <input type="text" />',d=l.createElement("div");d.className=a.oClasses.sFilter;d.innerHTML="<label>"+c+"</label>";a.aanFeatures.f||(d.id=a.sTableId+"_filter");c=h('input[type="text"]',d);d._DT_Input=c[0];c.val(b.sSearch.replace('"',"&quot;"));c.bind("keyup.DT",function(){for(var c=a.aanFeatures.f,d=this.value===""?"":this.value,
+g=0,e=c.length;g<e;g++)c[g]!=h(this).parents("div.dataTables_filter")[0]&&h(c[g]._DT_Input).val(d);d!=b.sSearch&&K(a,{sSearch:d,bRegex:b.bRegex,bSmart:b.bSmart,bCaseInsensitive:b.bCaseInsensitive})});c.attr("aria-controls",a.sTableId).bind("keypress.DT",function(a){if(a.keyCode==13)return false});return d}function K(a,b,c){var d=a.oPreviousSearch,i=a.aoPreSearchCols,f=function(a){d.sSearch=a.sSearch;d.bRegex=a.bRegex;d.bSmart=a.bSmart;d.bCaseInsensitive=a.bCaseInsensitive};if(a.oFeatures.bServerSide)f(b);
+else{Ga(a,b.sSearch,c,b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<a.aoPreSearchCols.length;b++)Ha(a,i[b].sSearch,b,i[b].bRegex,i[b].bSmart,i[b].bCaseInsensitive);Ia(a)}a.bFiltered=!0;h(a.oInstance).trigger("filter",a);a._iDisplayStart=0;y(a);x(a);la(a,0)}function Ia(a){for(var b=j.ext.afnFiltering,c=r(a,"bSearchable"),d=0,i=b.length;d<i;d++)for(var f=0,g=0,e=a.aiDisplay.length;g<e;g++){var h=a.aiDisplay[g-f];b[d](a,Y(a,h,"filter",c),h)||(a.aiDisplay.splice(g-f,1),f++)}}function Ha(a,b,c,
+d,i,f){if(""!==b)for(var g=0,b=ma(b,d,i,f),d=a.aiDisplay.length-1;0<=d;d--)i=Ja(v(a,a.aiDisplay[d],c,"filter"),a.aoColumns[c].sType),b.test(i)||(a.aiDisplay.splice(d,1),g++)}function Ga(a,b,c,d,i,f){d=ma(b,d,i,f);i=a.oPreviousSearch;c||(c=0);0!==j.ext.afnFiltering.length&&(c=1);if(0>=b.length)a.aiDisplay.splice(0,a.aiDisplay.length),a.aiDisplay=a.aiDisplayMaster.slice();else if(a.aiDisplay.length==a.aiDisplayMaster.length||i.sSearch.length>b.length||1==c||0!==b.indexOf(i.sSearch)){a.aiDisplay.splice(0,
+a.aiDisplay.length);la(a,1);for(b=0;b<a.aiDisplayMaster.length;b++)d.test(a.asDataSearch[b])&&a.aiDisplay.push(a.aiDisplayMaster[b])}else for(b=c=0;b<a.asDataSearch.length;b++)d.test(a.asDataSearch[b])||(a.aiDisplay.splice(b-c,1),c++)}function la(a,b){if(!a.oFeatures.bServerSide){a.asDataSearch=[];for(var c=r(a,"bSearchable"),d=1===b?a.aiDisplayMaster:a.aiDisplay,i=0,f=d.length;i<f;i++)a.asDataSearch[i]=na(a,Y(a,d[i],"filter",c))}}function na(a,b){var c=b.join("  ");-1!==c.indexOf("&")&&(c=h("<div>").html(c).text());
+return c.replace(/[\n\r]/g," ")}function ma(a,b,c,d){if(c)return a=b?a.split(" "):oa(a).split(" "),a="^(?=.*?"+a.join(")(?=.*?")+").*$",RegExp(a,d?"i":"");a=b?a:oa(a);return RegExp(a,d?"i":"")}function Ja(a,b){return"function"===typeof j.ext.ofnSearch[b]?j.ext.ofnSearch[b](a):null===a?"":"html"==b?a.replace(/[\r\n]/g," ").replace(/<.*?>/g,""):"string"===typeof a?a.replace(/[\r\n]/g," "):a}function oa(a){return a.replace(RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),
+"\\$1")}function Ca(a){var b=l.createElement("div");b.className=a.oClasses.sInfo;a.aanFeatures.i||(a.aoDrawCallback.push({fn:Ka,sName:"information"}),b.id=a.sTableId+"_info");a.nTable.setAttribute("aria-describedby",a.sTableId+"_info");return b}function Ka(a){if(a.oFeatures.bInfo&&0!==a.aanFeatures.i.length){var b=a.oLanguage,c=a._iDisplayStart+1,d=a.fnDisplayEnd(),i=a.fnRecordsTotal(),f=a.fnRecordsDisplay(),g;g=0===f?b.sInfoEmpty:b.sInfo;f!=i&&(g+=" "+b.sInfoFiltered);g+=b.sInfoPostFix;g=ja(a,g);
+null!==b.fnInfoCallback&&(g=b.fnInfoCallback.call(a.oInstance,a,c,d,i,f,g));a=a.aanFeatures.i;b=0;for(c=a.length;b<c;b++)h(a[b]).html(g)}}function ja(a,b){var c=a.fnFormatNumber(a._iDisplayStart+1),d=a.fnDisplayEnd(),d=a.fnFormatNumber(d),i=a.fnRecordsDisplay(),i=a.fnFormatNumber(i),f=a.fnRecordsTotal(),f=a.fnFormatNumber(f);a.oScroll.bInfinite&&(c=a.fnFormatNumber(1));return b.replace(/_START_/g,c).replace(/_END_/g,d).replace(/_TOTAL_/g,i).replace(/_MAX_/g,f)}function ba(a){var b,c,d=a.iInitDisplayStart;
+if(!1===a.bInitialised)setTimeout(function(){ba(a)},200);else{xa(a);va(a);W(a,a.aoHeader);a.nTFoot&&W(a,a.aoFooter);E(a,!0);a.oFeatures.bAutoWidth&&da(a);b=0;for(c=a.aoColumns.length;b<c;b++)null!==a.aoColumns[b].sWidth&&(a.aoColumns[b].nTh.style.width=q(a.aoColumns[b].sWidth));a.oFeatures.bSort?O(a):a.oFeatures.bFilter?K(a,a.oPreviousSearch):(a.aiDisplay=a.aiDisplayMaster.slice(),y(a),x(a));null!==a.sAjaxSource&&!a.oFeatures.bServerSide?(c=[],ka(a,c),a.fnServerData.call(a.oInstance,a.sAjaxSource,
+c,function(c){var f=a.sAjaxDataProp!==""?Q(a.sAjaxDataProp)(c):c;for(b=0;b<f.length;b++)H(a,f[b]);a.iInitDisplayStart=d;if(a.oFeatures.bSort)O(a);else{a.aiDisplay=a.aiDisplayMaster.slice();y(a);x(a)}E(a,false);$(a,c)},a)):a.oFeatures.bServerSide||(E(a,!1),$(a))}}function $(a,b){a._bInitComplete=!0;A(a,"aoInitComplete","init",[a,b])}function pa(a){var b=j.defaults.oLanguage;!a.sEmptyTable&&(a.sZeroRecords&&"No data available in table"===b.sEmptyTable)&&p(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&
+(a.sZeroRecords&&"Loading..."===b.sLoadingRecords)&&p(a,a,"sZeroRecords","sLoadingRecords")}function ya(a){if(a.oScroll.bInfinite)return null;var b='<select size="1" '+('name="'+a.sTableId+'_length"')+">",c,d,i=a.aLengthMenu;if(2==i.length&&"object"===typeof i[0]&&"object"===typeof i[1]){c=0;for(d=i[0].length;c<d;c++)b+='<option value="'+i[0][c]+'">'+i[1][c]+"</option>"}else{c=0;for(d=i.length;c<d;c++)b+='<option value="'+i[c]+'">'+i[c]+"</option>"}b+="</select>";i=l.createElement("div");a.aanFeatures.l||
+(i.id=a.sTableId+"_length");i.className=a.oClasses.sLength;i.innerHTML="<label>"+a.oLanguage.sLengthMenu.replace("_MENU_",b)+"</label>";h('select option[value="'+a._iDisplayLength+'"]',i).attr("selected",!0);h("select",i).bind("change.DT",function(){var b=h(this).val(),i=a.aanFeatures.l;c=0;for(d=i.length;c<d;c++)i[c]!=this.parentNode&&h("select",i[c]).val(b);a._iDisplayLength=parseInt(b,10);y(a);if(a.fnDisplayEnd()==a.fnRecordsDisplay()){a._iDisplayStart=a.fnDisplayEnd()-a._iDisplayLength;if(a._iDisplayStart<
+0)a._iDisplayStart=0}if(a._iDisplayLength==-1)a._iDisplayStart=0;x(a)});h("select",i).attr("aria-controls",a.sTableId);return i}function y(a){a._iDisplayEnd=!1===a.oFeatures.bPaginate?a.aiDisplay.length:a._iDisplayStart+a._iDisplayLength>a.aiDisplay.length||-1==a._iDisplayLength?a.aiDisplay.length:a._iDisplayStart+a._iDisplayLength}function Da(a){if(a.oScroll.bInfinite)return null;var b=l.createElement("div");b.className=a.oClasses.sPaging+a.sPaginationType;j.ext.oPagination[a.sPaginationType].fnInit(a,
+b,function(a){y(a);x(a)});a.aanFeatures.p||a.aoDrawCallback.push({fn:function(a){j.ext.oPagination[a.sPaginationType].fnUpdate(a,function(a){y(a);x(a)})},sName:"pagination"});return b}function qa(a,b){var c=a._iDisplayStart;if("number"===typeof b)a._iDisplayStart=b*a._iDisplayLength,a._iDisplayStart>a.fnRecordsDisplay()&&(a._iDisplayStart=0);else if("first"==b)a._iDisplayStart=0;else if("previous"==b)a._iDisplayStart=0<=a._iDisplayLength?a._iDisplayStart-a._iDisplayLength:0,0>a._iDisplayStart&&(a._iDisplayStart=
+0);else if("next"==b)0<=a._iDisplayLength?a._iDisplayStart+a._iDisplayLength<a.fnRecordsDisplay()&&(a._iDisplayStart+=a._iDisplayLength):a._iDisplayStart=0;else if("last"==b)if(0<=a._iDisplayLength){var d=parseInt((a.fnRecordsDisplay()-1)/a._iDisplayLength,10)+1;a._iDisplayStart=(d-1)*a._iDisplayLength}else a._iDisplayStart=0;else D(a,0,"Unknown paging action: "+b);h(a.oInstance).trigger("page",a);return c!=a._iDisplayStart}function Aa(a){var b=l.createElement("div");a.aanFeatures.r||(b.id=a.sTableId+
+"_processing");b.innerHTML=a.oLanguage.sProcessing;b.className=a.oClasses.sProcessing;a.nTable.parentNode.insertBefore(b,a.nTable);return b}function E(a,b){if(a.oFeatures.bProcessing)for(var c=a.aanFeatures.r,d=0,i=c.length;d<i;d++)c[d].style.visibility=b?"visible":"hidden";h(a.oInstance).trigger("processing",[a,b])}function Ba(a){if(""===a.oScroll.sX&&""===a.oScroll.sY)return a.nTable;var b=l.createElement("div"),c=l.createElement("div"),d=l.createElement("div"),i=l.createElement("div"),f=l.createElement("div"),
+g=l.createElement("div"),e=a.nTable.cloneNode(!1),j=a.nTable.cloneNode(!1),o=a.nTable.getElementsByTagName("thead")[0],k=0===a.nTable.getElementsByTagName("tfoot").length?null:a.nTable.getElementsByTagName("tfoot")[0],m=a.oClasses;c.appendChild(d);f.appendChild(g);i.appendChild(a.nTable);b.appendChild(c);b.appendChild(i);d.appendChild(e);e.appendChild(o);null!==k&&(b.appendChild(f),g.appendChild(j),j.appendChild(k));b.className=m.sScrollWrapper;c.className=m.sScrollHead;d.className=m.sScrollHeadInner;
+i.className=m.sScrollBody;f.className=m.sScrollFoot;g.className=m.sScrollFootInner;a.oScroll.bAutoCss&&(c.style.overflow="hidden",c.style.position="relative",f.style.overflow="hidden",i.style.overflow="auto");c.style.border="0";c.style.width="100%";f.style.border="0";d.style.width=""!==a.oScroll.sXInner?a.oScroll.sXInner:"100%";e.removeAttribute("id");e.style.marginLeft="0";a.nTable.style.marginLeft="0";null!==k&&(j.removeAttribute("id"),j.style.marginLeft="0");d=h(a.nTable).children("caption");0<
+d.length&&(d=d[0],"top"===d._captionSide?e.appendChild(d):"bottom"===d._captionSide&&k&&j.appendChild(d));""!==a.oScroll.sX&&(c.style.width=q(a.oScroll.sX),i.style.width=q(a.oScroll.sX),null!==k&&(f.style.width=q(a.oScroll.sX)),h(i).scroll(function(){c.scrollLeft=this.scrollLeft;if(k!==null)f.scrollLeft=this.scrollLeft}));""!==a.oScroll.sY&&(i.style.height=q(a.oScroll.sY));a.aoDrawCallback.push({fn:La,sName:"scrolling"});a.oScroll.bInfinite&&h(i).scroll(function(){if(!a.bDrawing&&h(this).scrollTop()!==
+0&&h(this).scrollTop()+h(this).height()>h(a.nTable).height()-a.oScroll.iLoadGap&&a.fnDisplayEnd()<a.fnRecordsDisplay()){qa(a,"next");y(a);x(a)}});a.nScrollHead=c;a.nScrollFoot=f;return b}function La(a){var b=a.nScrollHead.getElementsByTagName("div")[0],c=b.getElementsByTagName("table")[0],d=a.nTable.parentNode,i,f,g,e,j,o,k,m,p=[],n=[],l=null!==a.nTFoot?a.nScrollFoot.getElementsByTagName("div")[0]:null,R=null!==a.nTFoot?l.getElementsByTagName("table")[0]:null,r=a.oBrowser.bScrollOversize,s=function(a){k=
+a.style;k.paddingTop="0";k.paddingBottom="0";k.borderTopWidth="0";k.borderBottomWidth="0";k.height=0};h(a.nTable).children("thead, tfoot").remove();i=h(a.nTHead).clone()[0];a.nTable.insertBefore(i,a.nTable.childNodes[0]);g=a.nTHead.getElementsByTagName("tr");e=i.getElementsByTagName("tr");null!==a.nTFoot&&(j=h(a.nTFoot).clone()[0],a.nTable.insertBefore(j,a.nTable.childNodes[1]),o=a.nTFoot.getElementsByTagName("tr"),j=j.getElementsByTagName("tr"));""===a.oScroll.sX&&(d.style.width="100%",b.parentNode.style.width=
+"100%");var t=N(a,i);i=0;for(f=t.length;i<f;i++)m=G(a,i),t[i].style.width=a.aoColumns[m].sWidth;null!==a.nTFoot&&C(function(a){a.style.width=""},j);a.oScroll.bCollapse&&""!==a.oScroll.sY&&(d.style.height=d.offsetHeight+a.nTHead.offsetHeight+"px");i=h(a.nTable).outerWidth();if(""===a.oScroll.sX){if(a.nTable.style.width="100%",r&&(h("tbody",d).height()>d.offsetHeight||"scroll"==h(d).css("overflow-y")))a.nTable.style.width=q(h(a.nTable).outerWidth()-a.oScroll.iBarWidth)}else""!==a.oScroll.sXInner?a.nTable.style.width=
+q(a.oScroll.sXInner):i==h(d).width()&&h(d).height()<h(a.nTable).height()?(a.nTable.style.width=q(i-a.oScroll.iBarWidth),h(a.nTable).outerWidth()>i-a.oScroll.iBarWidth&&(a.nTable.style.width=q(i))):a.nTable.style.width=q(i);i=h(a.nTable).outerWidth();C(s,e);C(function(a){p.push(q(h(a).width()))},e);C(function(a,b){a.style.width=p[b]},g);h(e).height(0);null!==a.nTFoot&&(C(s,j),C(function(a){n.push(q(h(a).width()))},j),C(function(a,b){a.style.width=n[b]},o),h(j).height(0));C(function(a,b){a.innerHTML=
+"";a.style.width=p[b]},e);null!==a.nTFoot&&C(function(a,b){a.innerHTML="";a.style.width=n[b]},j);if(h(a.nTable).outerWidth()<i){g=d.scrollHeight>d.offsetHeight||"scroll"==h(d).css("overflow-y")?i+a.oScroll.iBarWidth:i;if(r&&(d.scrollHeight>d.offsetHeight||"scroll"==h(d).css("overflow-y")))a.nTable.style.width=q(g-a.oScroll.iBarWidth);d.style.width=q(g);a.nScrollHead.style.width=q(g);null!==a.nTFoot&&(a.nScrollFoot.style.width=q(g));""===a.oScroll.sX?D(a,1,"The table cannot fit into the current element which will cause column misalignment. The table has been drawn at its minimum possible width."):
+""!==a.oScroll.sXInner&&D(a,1,"The table cannot fit into the current element which will cause column misalignment. Increase the sScrollXInner value or remove it to allow automatic calculation")}else d.style.width=q("100%"),a.nScrollHead.style.width=q("100%"),null!==a.nTFoot&&(a.nScrollFoot.style.width=q("100%"));""===a.oScroll.sY&&r&&(d.style.height=q(a.nTable.offsetHeight+a.oScroll.iBarWidth));""!==a.oScroll.sY&&a.oScroll.bCollapse&&(d.style.height=q(a.oScroll.sY),r=""!==a.oScroll.sX&&a.nTable.offsetWidth>
+d.offsetWidth?a.oScroll.iBarWidth:0,a.nTable.offsetHeight<d.offsetHeight&&(d.style.height=q(a.nTable.offsetHeight+r)));r=h(a.nTable).outerWidth();c.style.width=q(r);b.style.width=q(r);c=h(a.nTable).height()>d.clientHeight||"scroll"==h(d).css("overflow-y");b.style.paddingRight=c?a.oScroll.iBarWidth+"px":"0px";null!==a.nTFoot&&(R.style.width=q(r),l.style.width=q(r),l.style.paddingRight=c?a.oScroll.iBarWidth+"px":"0px");h(d).scroll();if(a.bSorted||a.bFiltered)d.scrollTop=0}function C(a,b,c){for(var d=
+0,i=0,f=b.length,g,e;i<f;){g=b[i].firstChild;for(e=c?c[i].firstChild:null;g;)1===g.nodeType&&(c?a(g,e,d):a(g,d),d++),g=g.nextSibling,e=c?e.nextSibling:null;i++}}function Ma(a,b){if(!a||null===a||""===a)return 0;b||(b=l.body);var c,d=l.createElement("div");d.style.width=q(a);b.appendChild(d);c=d.offsetWidth;b.removeChild(d);return c}function da(a){var b=0,c,d=0,i=a.aoColumns.length,f,e,j=h("th",a.nTHead),o=a.nTable.getAttribute("width");e=a.nTable.parentNode;for(f=0;f<i;f++)a.aoColumns[f].bVisible&&
+(d++,null!==a.aoColumns[f].sWidth&&(c=Ma(a.aoColumns[f].sWidthOrig,e),null!==c&&(a.aoColumns[f].sWidth=q(c)),b++));if(i==j.length&&0===b&&d==i&&""===a.oScroll.sX&&""===a.oScroll.sY)for(f=0;f<a.aoColumns.length;f++)c=h(j[f]).width(),null!==c&&(a.aoColumns[f].sWidth=q(c));else{b=a.nTable.cloneNode(!1);f=a.nTHead.cloneNode(!0);d=l.createElement("tbody");c=l.createElement("tr");b.removeAttribute("id");b.appendChild(f);null!==a.nTFoot&&(b.appendChild(a.nTFoot.cloneNode(!0)),C(function(a){a.style.width=
+""},b.getElementsByTagName("tr")));b.appendChild(d);d.appendChild(c);d=h("thead th",b);0===d.length&&(d=h("tbody tr:eq(0)>td",b));j=N(a,f);for(f=d=0;f<i;f++){var k=a.aoColumns[f];k.bVisible&&null!==k.sWidthOrig&&""!==k.sWidthOrig?j[f-d].style.width=q(k.sWidthOrig):k.bVisible?j[f-d].style.width="":d++}for(f=0;f<i;f++)a.aoColumns[f].bVisible&&(d=Na(a,f),null!==d&&(d=d.cloneNode(!0),""!==a.aoColumns[f].sContentPadding&&(d.innerHTML+=a.aoColumns[f].sContentPadding),c.appendChild(d)));e.appendChild(b);
+""!==a.oScroll.sX&&""!==a.oScroll.sXInner?b.style.width=q(a.oScroll.sXInner):""!==a.oScroll.sX?(b.style.width="",h(b).width()<e.offsetWidth&&(b.style.width=q(e.offsetWidth))):""!==a.oScroll.sY?b.style.width=q(e.offsetWidth):o&&(b.style.width=q(o));b.style.visibility="hidden";Oa(a,b);i=h("tbody tr:eq(0)",b).children();0===i.length&&(i=N(a,h("thead",b)[0]));if(""!==a.oScroll.sX){for(f=d=e=0;f<a.aoColumns.length;f++)a.aoColumns[f].bVisible&&(e=null===a.aoColumns[f].sWidthOrig?e+h(i[d]).outerWidth():
+e+(parseInt(a.aoColumns[f].sWidth.replace("px",""),10)+(h(i[d]).outerWidth()-h(i[d]).width())),d++);b.style.width=q(e);a.nTable.style.width=q(e)}for(f=d=0;f<a.aoColumns.length;f++)a.aoColumns[f].bVisible&&(e=h(i[d]).width(),null!==e&&0<e&&(a.aoColumns[f].sWidth=q(e)),d++);i=h(b).css("width");a.nTable.style.width=-1!==i.indexOf("%")?i:q(h(b).outerWidth());b.parentNode.removeChild(b)}o&&(a.nTable.style.width=q(o))}function Oa(a,b){""===a.oScroll.sX&&""!==a.oScroll.sY?(h(b).width(),b.style.width=q(h(b).outerWidth()-
+a.oScroll.iBarWidth)):""!==a.oScroll.sX&&(b.style.width=q(h(b).outerWidth()))}function Na(a,b){var c=Pa(a,b);if(0>c)return null;if(null===a.aoData[c].nTr){var d=l.createElement("td");d.innerHTML=v(a,c,b,"");return d}return J(a,c)[b]}function Pa(a,b){for(var c=-1,d=-1,i=0;i<a.aoData.length;i++){var e=v(a,i,b,"display")+"",e=e.replace(/<.*?>/g,"");e.length>c&&(c=e.length,d=i)}return d}function q(a){if(null===a)return"0px";if("number"==typeof a)return 0>a?"0px":a+"px";var b=a.charCodeAt(a.length-1);
+return 48>b||57<b?a:a+"px"}function Qa(){var a=l.createElement("p"),b=a.style;b.width="100%";b.height="200px";b.padding="0px";var c=l.createElement("div"),b=c.style;b.position="absolute";b.top="0px";b.left="0px";b.visibility="hidden";b.width="200px";b.height="150px";b.padding="0px";b.overflow="hidden";c.appendChild(a);l.body.appendChild(c);b=a.offsetWidth;c.style.overflow="scroll";a=a.offsetWidth;b==a&&(a=c.clientWidth);l.body.removeChild(c);return b-a}function O(a,b){var c,d,i,e,g,k,o=[],m=[],p=
+j.ext.oSort,l=a.aoData,q=a.aoColumns,G=a.oLanguage.oAria;if(!a.oFeatures.bServerSide&&(0!==a.aaSorting.length||null!==a.aaSortingFixed)){o=null!==a.aaSortingFixed?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(c=0;c<o.length;c++)if(d=o[c][0],i=R(a,d),e=a.aoColumns[d].sSortDataType,j.ext.afnSortData[e])if(g=j.ext.afnSortData[e].call(a.oInstance,a,d,i),g.length===l.length){i=0;for(e=l.length;i<e;i++)F(a,i,d,g[i])}else D(a,0,"Returned data sort array (col "+d+") is the wrong length");c=
+0;for(d=a.aiDisplayMaster.length;c<d;c++)m[a.aiDisplayMaster[c]]=c;var r=o.length,s;c=0;for(d=l.length;c<d;c++)for(i=0;i<r;i++){s=q[o[i][0]].aDataSort;g=0;for(k=s.length;g<k;g++)e=q[s[g]].sType,e=p[(e?e:"string")+"-pre"],l[c]._aSortData[s[g]]=e?e(v(a,c,s[g],"sort")):v(a,c,s[g],"sort")}a.aiDisplayMaster.sort(function(a,b){var c,d,e,i,f;for(c=0;c<r;c++){f=q[o[c][0]].aDataSort;d=0;for(e=f.length;d<e;d++)if(i=q[f[d]].sType,i=p[(i?i:"string")+"-"+o[c][1]](l[a]._aSortData[f[d]],l[b]._aSortData[f[d]]),0!==
+i)return i}return p["numeric-asc"](m[a],m[b])})}(b===n||b)&&!a.oFeatures.bDeferRender&&P(a);c=0;for(d=a.aoColumns.length;c<d;c++)e=q[c].sTitle.replace(/<.*?>/g,""),i=q[c].nTh,i.removeAttribute("aria-sort"),i.removeAttribute("aria-label"),q[c].bSortable?0<o.length&&o[0][0]==c?(i.setAttribute("aria-sort","asc"==o[0][1]?"ascending":"descending"),i.setAttribute("aria-label",e+("asc"==(q[c].asSorting[o[0][2]+1]?q[c].asSorting[o[0][2]+1]:q[c].asSorting[0])?G.sSortAscending:G.sSortDescending))):i.setAttribute("aria-label",
+e+("asc"==q[c].asSorting[0]?G.sSortAscending:G.sSortDescending)):i.setAttribute("aria-label",e);a.bSorted=!0;h(a.oInstance).trigger("sort",a);a.oFeatures.bFilter?K(a,a.oPreviousSearch,1):(a.aiDisplay=a.aiDisplayMaster.slice(),a._iDisplayStart=0,y(a),x(a))}function ia(a,b,c,d){Ra(b,{},function(b){if(!1!==a.aoColumns[c].bSortable){var e=function(){var d,e;if(b.shiftKey){for(var f=!1,h=0;h<a.aaSorting.length;h++)if(a.aaSorting[h][0]==c){f=!0;d=a.aaSorting[h][0];e=a.aaSorting[h][2]+1;a.aoColumns[d].asSorting[e]?
+(a.aaSorting[h][1]=a.aoColumns[d].asSorting[e],a.aaSorting[h][2]=e):a.aaSorting.splice(h,1);break}!1===f&&a.aaSorting.push([c,a.aoColumns[c].asSorting[0],0])}else 1==a.aaSorting.length&&a.aaSorting[0][0]==c?(d=a.aaSorting[0][0],e=a.aaSorting[0][2]+1,a.aoColumns[d].asSorting[e]||(e=0),a.aaSorting[0][1]=a.aoColumns[d].asSorting[e],a.aaSorting[0][2]=e):(a.aaSorting.splice(0,a.aaSorting.length),a.aaSorting.push([c,a.aoColumns[c].asSorting[0],0]));O(a)};a.oFeatures.bProcessing?(E(a,!0),setTimeout(function(){e();
+a.oFeatures.bServerSide||E(a,!1)},0)):e();"function"==typeof d&&d(a)}})}function P(a){var b,c,d,e,f,g=a.aoColumns.length,j=a.oClasses;for(b=0;b<g;b++)a.aoColumns[b].bSortable&&h(a.aoColumns[b].nTh).removeClass(j.sSortAsc+" "+j.sSortDesc+" "+a.aoColumns[b].sSortingClass);c=null!==a.aaSortingFixed?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(b=0;b<a.aoColumns.length;b++)if(a.aoColumns[b].bSortable){f=a.aoColumns[b].sSortingClass;e=-1;for(d=0;d<c.length;d++)if(c[d][0]==b){f="asc"==c[d][1]?
+j.sSortAsc:j.sSortDesc;e=d;break}h(a.aoColumns[b].nTh).addClass(f);a.bJUI&&(f=h("span."+j.sSortIcon,a.aoColumns[b].nTh),f.removeClass(j.sSortJUIAsc+" "+j.sSortJUIDesc+" "+j.sSortJUI+" "+j.sSortJUIAscAllowed+" "+j.sSortJUIDescAllowed),f.addClass(-1==e?a.aoColumns[b].sSortingClassJUI:"asc"==c[e][1]?j.sSortJUIAsc:j.sSortJUIDesc))}else h(a.aoColumns[b].nTh).addClass(a.aoColumns[b].sSortingClass);f=j.sSortColumn;if(a.oFeatures.bSort&&a.oFeatures.bSortClasses){a=J(a);e=[];for(b=0;b<g;b++)e.push("");b=0;
+for(d=1;b<c.length;b++)j=parseInt(c[b][0],10),e[j]=f+d,3>d&&d++;f=RegExp(f+"[123]");var o;b=0;for(c=a.length;b<c;b++)j=b%g,d=a[b].className,o=e[j],j=d.replace(f,o),j!=d?a[b].className=h.trim(j):0<o.length&&-1==d.indexOf(o)&&(a[b].className=d+" "+o)}}function ra(a){if(a.oFeatures.bStateSave&&!a.bDestroying){var b,c;b=a.oScroll.bInfinite;var d={iCreate:(new Date).getTime(),iStart:b?0:a._iDisplayStart,iEnd:b?a._iDisplayLength:a._iDisplayEnd,iLength:a._iDisplayLength,aaSorting:h.extend(!0,[],a.aaSorting),
+oSearch:h.extend(!0,{},a.oPreviousSearch),aoSearchCols:h.extend(!0,[],a.aoPreSearchCols),abVisCols:[]};b=0;for(c=a.aoColumns.length;b<c;b++)d.abVisCols.push(a.aoColumns[b].bVisible);A(a,"aoStateSaveParams","stateSaveParams",[a,d]);a.fnStateSave.call(a.oInstance,a,d)}}function Sa(a,b){if(a.oFeatures.bStateSave){var c=a.fnStateLoad.call(a.oInstance,a);if(c){var d=A(a,"aoStateLoadParams","stateLoadParams",[a,c]);if(-1===h.inArray(!1,d)){a.oLoadedState=h.extend(!0,{},c);a._iDisplayStart=c.iStart;a.iInitDisplayStart=
+c.iStart;a._iDisplayEnd=c.iEnd;a._iDisplayLength=c.iLength;a.aaSorting=c.aaSorting.slice();a.saved_aaSorting=c.aaSorting.slice();h.extend(a.oPreviousSearch,c.oSearch);h.extend(!0,a.aoPreSearchCols,c.aoSearchCols);b.saved_aoColumns=[];for(d=0;d<c.abVisCols.length;d++)b.saved_aoColumns[d]={},b.saved_aoColumns[d].bVisible=c.abVisCols[d];A(a,"aoStateLoaded","stateLoaded",[a,c])}}}}function s(a){for(var b=0;b<j.settings.length;b++)if(j.settings[b].nTable===a)return j.settings[b];return null}function T(a){for(var b=
+[],a=a.aoData,c=0,d=a.length;c<d;c++)null!==a[c].nTr&&b.push(a[c].nTr);return b}function J(a,b){var c=[],d,e,f,g,h,j;e=0;var o=a.aoData.length;b!==n&&(e=b,o=b+1);for(f=e;f<o;f++)if(j=a.aoData[f],null!==j.nTr){e=[];for(d=j.nTr.firstChild;d;)g=d.nodeName.toLowerCase(),("td"==g||"th"==g)&&e.push(d),d=d.nextSibling;g=d=0;for(h=a.aoColumns.length;g<h;g++)a.aoColumns[g].bVisible?c.push(e[g-d]):(c.push(j._anHidden[g]),d++)}return c}function D(a,b,c){a=null===a?"DataTables warning: "+c:"DataTables warning (table id = '"+
+a.sTableId+"'): "+c;if(0===b)if("alert"==j.ext.sErrMode)alert(a);else throw Error(a);else X.console&&console.log&&console.log(a)}function p(a,b,c,d){d===n&&(d=c);b[c]!==n&&(a[d]=b[c])}function Ta(a,b){var c,d;for(d in b)b.hasOwnProperty(d)&&(c=b[d],"object"===typeof e[d]&&null!==c&&!1===h.isArray(c)?h.extend(!0,a[d],c):a[d]=c);return a}function Ra(a,b,c){h(a).bind("click.DT",b,function(b){a.blur();c(b)}).bind("keypress.DT",b,function(a){13===a.which&&c(a)}).bind("selectstart.DT",function(){return!1})}
+function z(a,b,c,d){c&&a[b].push({fn:c,sName:d})}function A(a,b,c,d){for(var b=a[b],e=[],f=b.length-1;0<=f;f--)e.push(b[f].fn.apply(a.oInstance,d));null!==c&&h(a.oInstance).trigger(c,d);return e}function Ua(a){var b=h('<div style="position:absolute; top:0; left:0; height:1px; width:1px; overflow:hidden"><div style="position:absolute; top:1px; left:1px; width:100px; overflow:scroll;"><div id="DT_BrowserTest" style="width:100%; height:10px;"></div></div></div>')[0];l.body.appendChild(b);a.oBrowser.bScrollOversize=
+100===h("#DT_BrowserTest",b)[0].offsetWidth?!0:!1;l.body.removeChild(b)}function Va(a){return function(){var b=[s(this[j.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return j.ext.oApi[a].apply(this,b)}}var U=/\[.*?\]$/,Wa=X.JSON?JSON.stringify:function(a){var b=typeof a;if("object"!==b||null===a)return"string"===b&&(a='"'+a+'"'),a+"";var c,d,e=[],f=h.isArray(a);for(c in a)d=a[c],b=typeof d,"string"===b?d='"'+d+'"':"object"===b&&null!==d&&(d=Wa(d)),e.push((f?"":'"'+c+'":')+d);return(f?
+"[":"{")+e+(f?"]":"}")};this.$=function(a,b){var c,d,e=[],f;d=s(this[j.ext.iApiIndex]);var g=d.aoData,o=d.aiDisplay,k=d.aiDisplayMaster;b||(b={});b=h.extend({},{filter:"none",order:"current",page:"all"},b);if("current"==b.page){c=d._iDisplayStart;for(d=d.fnDisplayEnd();c<d;c++)(f=g[o[c]].nTr)&&e.push(f)}else if("current"==b.order&&"none"==b.filter){c=0;for(d=k.length;c<d;c++)(f=g[k[c]].nTr)&&e.push(f)}else if("current"==b.order&&"applied"==b.filter){c=0;for(d=o.length;c<d;c++)(f=g[o[c]].nTr)&&e.push(f)}else if("original"==
+b.order&&"none"==b.filter){c=0;for(d=g.length;c<d;c++)(f=g[c].nTr)&&e.push(f)}else if("original"==b.order&&"applied"==b.filter){c=0;for(d=g.length;c<d;c++)f=g[c].nTr,-1!==h.inArray(c,o)&&f&&e.push(f)}else D(d,1,"Unknown selection options");e=h(e);c=e.filter(a);e=e.find(a);return h([].concat(h.makeArray(c),h.makeArray(e)))};this._=function(a,b){var c=[],d,e,f=this.$(a,b);d=0;for(e=f.length;d<e;d++)c.push(this.fnGetData(f[d]));return c};this.fnAddData=function(a,b){if(0===a.length)return[];var c=[],
+d,e=s(this[j.ext.iApiIndex]);if("object"===typeof a[0]&&null!==a[0])for(var f=0;f<a.length;f++){d=H(e,a[f]);if(-1==d)return c;c.push(d)}else{d=H(e,a);if(-1==d)return c;c.push(d)}e.aiDisplay=e.aiDisplayMaster.slice();(b===n||b)&&aa(e);return c};this.fnAdjustColumnSizing=function(a){var b=s(this[j.ext.iApiIndex]);k(b);a===n||a?this.fnDraw(!1):(""!==b.oScroll.sX||""!==b.oScroll.sY)&&this.oApi._fnScrollDraw(b)};this.fnClearTable=function(a){var b=s(this[j.ext.iApiIndex]);ga(b);(a===n||a)&&x(b)};this.fnClose=
+function(a){for(var b=s(this[j.ext.iApiIndex]),c=0;c<b.aoOpenRows.length;c++)if(b.aoOpenRows[c].nParent==a)return(a=b.aoOpenRows[c].nTr.parentNode)&&a.removeChild(b.aoOpenRows[c].nTr),b.aoOpenRows.splice(c,1),0;return 1};this.fnDeleteRow=function(a,b,c){var d=s(this[j.ext.iApiIndex]),e,f,a="object"===typeof a?I(d,a):a,g=d.aoData.splice(a,1);e=0;for(f=d.aoData.length;e<f;e++)null!==d.aoData[e].nTr&&(d.aoData[e].nTr._DT_RowIndex=e);e=h.inArray(a,d.aiDisplay);d.asDataSearch.splice(e,1);ha(d.aiDisplayMaster,
+a);ha(d.aiDisplay,a);"function"===typeof b&&b.call(this,d,g);d._iDisplayStart>=d.fnRecordsDisplay()&&(d._iDisplayStart-=d._iDisplayLength,0>d._iDisplayStart&&(d._iDisplayStart=0));if(c===n||c)y(d),x(d);return g};this.fnDestroy=function(a){var b=s(this[j.ext.iApiIndex]),c=b.nTableWrapper.parentNode,d=b.nTBody,i,f,a=a===n?!1:a;b.bDestroying=!0;A(b,"aoDestroyCallback","destroy",[b]);if(!a){i=0;for(f=b.aoColumns.length;i<f;i++)!1===b.aoColumns[i].bVisible&&this.fnSetColumnVis(i,!0)}h(b.nTableWrapper).find("*").andSelf().unbind(".DT");
+h("tbody>tr>td."+b.oClasses.sRowEmpty,b.nTable).parent().remove();b.nTable!=b.nTHead.parentNode&&(h(b.nTable).children("thead").remove(),b.nTable.appendChild(b.nTHead));b.nTFoot&&b.nTable!=b.nTFoot.parentNode&&(h(b.nTable).children("tfoot").remove(),b.nTable.appendChild(b.nTFoot));b.nTable.parentNode.removeChild(b.nTable);h(b.nTableWrapper).remove();b.aaSorting=[];b.aaSortingFixed=[];P(b);h(T(b)).removeClass(b.asStripeClasses.join(" "));h("th, td",b.nTHead).removeClass([b.oClasses.sSortable,b.oClasses.sSortableAsc,
+b.oClasses.sSortableDesc,b.oClasses.sSortableNone].join(" "));b.bJUI&&(h("th span."+b.oClasses.sSortIcon+", td span."+b.oClasses.sSortIcon,b.nTHead).remove(),h("th, td",b.nTHead).each(function(){var a=h("div."+b.oClasses.sSortJUIWrapper,this),c=a.contents();h(this).append(c);a.remove()}));!a&&b.nTableReinsertBefore?c.insertBefore(b.nTable,b.nTableReinsertBefore):a||c.appendChild(b.nTable);i=0;for(f=b.aoData.length;i<f;i++)null!==b.aoData[i].nTr&&d.appendChild(b.aoData[i].nTr);!0===b.oFeatures.bAutoWidth&&
+(b.nTable.style.width=q(b.sDestroyWidth));if(f=b.asDestroyStripes.length){a=h(d).children("tr");for(i=0;i<f;i++)a.filter(":nth-child("+f+"n + "+i+")").addClass(b.asDestroyStripes[i])}i=0;for(f=j.settings.length;i<f;i++)j.settings[i]==b&&j.settings.splice(i,1);e=b=null};this.fnDraw=function(a){var b=s(this[j.ext.iApiIndex]);!1===a?(y(b),x(b)):aa(b)};this.fnFilter=function(a,b,c,d,e,f){var g=s(this[j.ext.iApiIndex]);if(g.oFeatures.bFilter){if(c===n||null===c)c=!1;if(d===n||null===d)d=!0;if(e===n||null===
+e)e=!0;if(f===n||null===f)f=!0;if(b===n||null===b){if(K(g,{sSearch:a+"",bRegex:c,bSmart:d,bCaseInsensitive:f},1),e&&g.aanFeatures.f){b=g.aanFeatures.f;c=0;for(d=b.length;c<d;c++)try{b[c]._DT_Input!=l.activeElement&&h(b[c]._DT_Input).val(a)}catch(o){h(b[c]._DT_Input).val(a)}}}else h.extend(g.aoPreSearchCols[b],{sSearch:a+"",bRegex:c,bSmart:d,bCaseInsensitive:f}),K(g,g.oPreviousSearch,1)}};this.fnGetData=function(a,b){var c=s(this[j.ext.iApiIndex]);if(a!==n){var d=a;if("object"===typeof a){var e=a.nodeName.toLowerCase();
+"tr"===e?d=I(c,a):"td"===e&&(d=I(c,a.parentNode),b=fa(c,d,a))}return b!==n?v(c,d,b,""):c.aoData[d]!==n?c.aoData[d]._aData:null}return Z(c)};this.fnGetNodes=function(a){var b=s(this[j.ext.iApiIndex]);return a!==n?b.aoData[a]!==n?b.aoData[a].nTr:null:T(b)};this.fnGetPosition=function(a){var b=s(this[j.ext.iApiIndex]),c=a.nodeName.toUpperCase();return"TR"==c?I(b,a):"TD"==c||"TH"==c?(c=I(b,a.parentNode),a=fa(b,c,a),[c,R(b,a),a]):null};this.fnIsOpen=function(a){for(var b=s(this[j.ext.iApiIndex]),c=0;c<
+b.aoOpenRows.length;c++)if(b.aoOpenRows[c].nParent==a)return!0;return!1};this.fnOpen=function(a,b,c){var d=s(this[j.ext.iApiIndex]),e=T(d);if(-1!==h.inArray(a,e)){this.fnClose(a);var e=l.createElement("tr"),f=l.createElement("td");e.appendChild(f);f.className=c;f.colSpan=t(d);"string"===typeof b?f.innerHTML=b:h(f).html(b);b=h("tr",d.nTBody);-1!=h.inArray(a,b)&&h(e).insertAfter(a);d.aoOpenRows.push({nTr:e,nParent:a});return e}};this.fnPageChange=function(a,b){var c=s(this[j.ext.iApiIndex]);qa(c,a);
+y(c);(b===n||b)&&x(c)};this.fnSetColumnVis=function(a,b,c){var d=s(this[j.ext.iApiIndex]),e,f,g=d.aoColumns,h=d.aoData,o,m;if(g[a].bVisible!=b){if(b){for(e=f=0;e<a;e++)g[e].bVisible&&f++;m=f>=t(d);if(!m)for(e=a;e<g.length;e++)if(g[e].bVisible){o=e;break}e=0;for(f=h.length;e<f;e++)null!==h[e].nTr&&(m?h[e].nTr.appendChild(h[e]._anHidden[a]):h[e].nTr.insertBefore(h[e]._anHidden[a],J(d,e)[o]))}else{e=0;for(f=h.length;e<f;e++)null!==h[e].nTr&&(o=J(d,e)[a],h[e]._anHidden[a]=o,o.parentNode.removeChild(o))}g[a].bVisible=
+b;W(d,d.aoHeader);d.nTFoot&&W(d,d.aoFooter);e=0;for(f=d.aoOpenRows.length;e<f;e++)d.aoOpenRows[e].nTr.colSpan=t(d);if(c===n||c)k(d),x(d);ra(d)}};this.fnSettings=function(){return s(this[j.ext.iApiIndex])};this.fnSort=function(a){var b=s(this[j.ext.iApiIndex]);b.aaSorting=a;O(b)};this.fnSortListener=function(a,b,c){ia(s(this[j.ext.iApiIndex]),a,b,c)};this.fnUpdate=function(a,b,c,d,e){var f=s(this[j.ext.iApiIndex]),b="object"===typeof b?I(f,b):b;if(h.isArray(a)&&c===n){f.aoData[b]._aData=a.slice();
+for(c=0;c<f.aoColumns.length;c++)this.fnUpdate(v(f,b,c),b,c,!1,!1)}else if(h.isPlainObject(a)&&c===n){f.aoData[b]._aData=h.extend(!0,{},a);for(c=0;c<f.aoColumns.length;c++)this.fnUpdate(v(f,b,c),b,c,!1,!1)}else{F(f,b,c,a);var a=v(f,b,c,"display"),g=f.aoColumns[c];null!==g.fnRender&&(a=S(f,b,c),g.bUseRendered&&F(f,b,c,a));null!==f.aoData[b].nTr&&(J(f,b)[c].innerHTML=a)}c=h.inArray(b,f.aiDisplay);f.asDataSearch[c]=na(f,Y(f,b,"filter",r(f,"bSearchable")));(e===n||e)&&k(f);(d===n||d)&&aa(f);return 0};
+this.fnVersionCheck=j.ext.fnVersionCheck;this.oApi={_fnExternApiFunc:Va,_fnInitialise:ba,_fnInitComplete:$,_fnLanguageCompat:pa,_fnAddColumn:o,_fnColumnOptions:m,_fnAddData:H,_fnCreateTr:ea,_fnGatherData:ua,_fnBuildHead:va,_fnDrawHead:W,_fnDraw:x,_fnReDraw:aa,_fnAjaxUpdate:wa,_fnAjaxParameters:Ea,_fnAjaxUpdateDraw:Fa,_fnServerParams:ka,_fnAddOptionsHtml:xa,_fnFeatureHtmlTable:Ba,_fnScrollDraw:La,_fnAdjustColumnSizing:k,_fnFeatureHtmlFilter:za,_fnFilterComplete:K,_fnFilterCustom:Ia,_fnFilterColumn:Ha,
+_fnFilter:Ga,_fnBuildSearchArray:la,_fnBuildSearchRow:na,_fnFilterCreateSearch:ma,_fnDataToSearch:Ja,_fnSort:O,_fnSortAttachListener:ia,_fnSortingClasses:P,_fnFeatureHtmlPaginate:Da,_fnPageChange:qa,_fnFeatureHtmlInfo:Ca,_fnUpdateInfo:Ka,_fnFeatureHtmlLength:ya,_fnFeatureHtmlProcessing:Aa,_fnProcessingDisplay:E,_fnVisibleToColumnIndex:G,_fnColumnIndexToVisible:R,_fnNodeToDataIndex:I,_fnVisbleColumns:t,_fnCalculateEnd:y,_fnConvertToWidth:Ma,_fnCalculateColumnWidths:da,_fnScrollingWidthAdjust:Oa,_fnGetWidestNode:Na,
+_fnGetMaxLenString:Pa,_fnStringToCss:q,_fnDetectType:B,_fnSettingsFromNode:s,_fnGetDataMaster:Z,_fnGetTrNodes:T,_fnGetTdNodes:J,_fnEscapeRegex:oa,_fnDeleteIndex:ha,_fnReOrderIndex:u,_fnColumnOrdering:M,_fnLog:D,_fnClearTable:ga,_fnSaveState:ra,_fnLoadState:Sa,_fnCreateCookie:function(a,b,c,d,e){var f=new Date;f.setTime(f.getTime()+1E3*c);var c=X.location.pathname.split("/"),a=a+"_"+c.pop().replace(/[\/:]/g,"").toLowerCase(),g;null!==e?(g="function"===typeof h.parseJSON?h.parseJSON(b):eval("("+b+")"),
+b=e(a,g,f.toGMTString(),c.join("/")+"/")):b=a+"="+encodeURIComponent(b)+"; expires="+f.toGMTString()+"; path="+c.join("/")+"/";a=l.cookie.split(";");e=b.split(";")[0].length;f=[];if(4096<e+l.cookie.length+10){for(var j=0,o=a.length;j<o;j++)if(-1!=a[j].indexOf(d)){var k=a[j].split("=");try{(g=eval("("+decodeURIComponent(k[1])+")"))&&g.iCreate&&f.push({name:k[0],time:g.iCreate})}catch(m){}}for(f.sort(function(a,b){return b.time-a.time});4096<e+l.cookie.length+10;){if(0===f.length)return;d=f.pop();l.cookie=
+d.name+"=; expires=Thu, 01-Jan-1970 00:00:01 GMT; path="+c.join("/")+"/"}}l.cookie=b},_fnReadCookie:function(a){for(var b=X.location.pathname.split("/"),a=a+"_"+b[b.length-1].replace(/[\/:]/g,"").toLowerCase()+"=",b=l.cookie.split(";"),c=0;c<b.length;c++){for(var d=b[c];" "==d.charAt(0);)d=d.substring(1,d.length);if(0===d.indexOf(a))return decodeURIComponent(d.substring(a.length,d.length))}return null},_fnDetectHeader:V,_fnGetUniqueThs:N,_fnScrollBarWidth:Qa,_fnApplyToChildren:C,_fnMap:p,_fnGetRowData:Y,
+_fnGetCellData:v,_fnSetCellData:F,_fnGetObjectDataFn:Q,_fnSetObjectDataFn:L,_fnApplyColumnDefs:ta,_fnBindAction:Ra,_fnExtend:Ta,_fnCallbackReg:z,_fnCallbackFire:A,_fnJsonString:Wa,_fnRender:S,_fnNodeToColumnIndex:fa,_fnInfoMacros:ja,_fnBrowserDetect:Ua,_fnGetColumns:r};h.extend(j.ext.oApi,this.oApi);for(var sa in j.ext.oApi)sa&&(this[sa]=Va(sa));var ca=this;this.each(function(){var a=0,b,c,d;c=this.getAttribute("id");var i=!1,f=!1;if("table"!=this.nodeName.toLowerCase())D(null,0,"Attempted to initialise DataTables on a node which is not a table: "+
+this.nodeName);else{a=0;for(b=j.settings.length;a<b;a++){if(j.settings[a].nTable==this){if(e===n||e.bRetrieve)return j.settings[a].oInstance;if(e.bDestroy){j.settings[a].oInstance.fnDestroy();break}else{D(j.settings[a],0,"Cannot reinitialise DataTable.\n\nTo retrieve the DataTables object for this table, pass no arguments or see the docs for bRetrieve and bDestroy");return}}if(j.settings[a].sTableId==this.id){j.settings.splice(a,1);break}}if(null===c||""===c)this.id=c="DataTables_Table_"+j.ext._oExternConfig.iNextUnique++;
+var g=h.extend(!0,{},j.models.oSettings,{nTable:this,oApi:ca.oApi,oInit:e,sDestroyWidth:h(this).width(),sInstance:c,sTableId:c});j.settings.push(g);g.oInstance=1===ca.length?ca:h(this).dataTable();e||(e={});e.oLanguage&&pa(e.oLanguage);e=Ta(h.extend(!0,{},j.defaults),e);p(g.oFeatures,e,"bPaginate");p(g.oFeatures,e,"bLengthChange");p(g.oFeatures,e,"bFilter");p(g.oFeatures,e,"bSort");p(g.oFeatures,e,"bInfo");p(g.oFeatures,e,"bProcessing");p(g.oFeatures,e,"bAutoWidth");p(g.oFeatures,e,"bSortClasses");
+p(g.oFeatures,e,"bServerSide");p(g.oFeatures,e,"bDeferRender");p(g.oScroll,e,"sScrollX","sX");p(g.oScroll,e,"sScrollXInner","sXInner");p(g.oScroll,e,"sScrollY","sY");p(g.oScroll,e,"bScrollCollapse","bCollapse");p(g.oScroll,e,"bScrollInfinite","bInfinite");p(g.oScroll,e,"iScrollLoadGap","iLoadGap");p(g.oScroll,e,"bScrollAutoCss","bAutoCss");p(g,e,"asStripeClasses");p(g,e,"asStripClasses","asStripeClasses");p(g,e,"fnServerData");p(g,e,"fnFormatNumber");p(g,e,"sServerMethod");p(g,e,"aaSorting");p(g,
+e,"aaSortingFixed");p(g,e,"aLengthMenu");p(g,e,"sPaginationType");p(g,e,"sAjaxSource");p(g,e,"sAjaxDataProp");p(g,e,"iCookieDuration");p(g,e,"sCookiePrefix");p(g,e,"sDom");p(g,e,"bSortCellsTop");p(g,e,"iTabIndex");p(g,e,"oSearch","oPreviousSearch");p(g,e,"aoSearchCols","aoPreSearchCols");p(g,e,"iDisplayLength","_iDisplayLength");p(g,e,"bJQueryUI","bJUI");p(g,e,"fnCookieCallback");p(g,e,"fnStateLoad");p(g,e,"fnStateSave");p(g.oLanguage,e,"fnInfoCallback");z(g,"aoDrawCallback",e.fnDrawCallback,"user");
+z(g,"aoServerParams",e.fnServerParams,"user");z(g,"aoStateSaveParams",e.fnStateSaveParams,"user");z(g,"aoStateLoadParams",e.fnStateLoadParams,"user");z(g,"aoStateLoaded",e.fnStateLoaded,"user");z(g,"aoRowCallback",e.fnRowCallback,"user");z(g,"aoRowCreatedCallback",e.fnCreatedRow,"user");z(g,"aoHeaderCallback",e.fnHeaderCallback,"user");z(g,"aoFooterCallback",e.fnFooterCallback,"user");z(g,"aoInitComplete",e.fnInitComplete,"user");z(g,"aoPreDrawCallback",e.fnPreDrawCallback,"user");g.oFeatures.bServerSide&&
+g.oFeatures.bSort&&g.oFeatures.bSortClasses?z(g,"aoDrawCallback",P,"server_side_sort_classes"):g.oFeatures.bDeferRender&&z(g,"aoDrawCallback",P,"defer_sort_classes");e.bJQueryUI?(h.extend(g.oClasses,j.ext.oJUIClasses),e.sDom===j.defaults.sDom&&"lfrtip"===j.defaults.sDom&&(g.sDom='<"H"lfr>t<"F"ip>')):h.extend(g.oClasses,j.ext.oStdClasses);h(this).addClass(g.oClasses.sTable);if(""!==g.oScroll.sX||""!==g.oScroll.sY)g.oScroll.iBarWidth=Qa();g.iInitDisplayStart===n&&(g.iInitDisplayStart=e.iDisplayStart,
+g._iDisplayStart=e.iDisplayStart);e.bStateSave&&(g.oFeatures.bStateSave=!0,Sa(g,e),z(g,"aoDrawCallback",ra,"state_save"));null!==e.iDeferLoading&&(g.bDeferLoading=!0,a=h.isArray(e.iDeferLoading),g._iRecordsDisplay=a?e.iDeferLoading[0]:e.iDeferLoading,g._iRecordsTotal=a?e.iDeferLoading[1]:e.iDeferLoading);null!==e.aaData&&(f=!0);""!==e.oLanguage.sUrl?(g.oLanguage.sUrl=e.oLanguage.sUrl,h.getJSON(g.oLanguage.sUrl,null,function(a){pa(a);h.extend(true,g.oLanguage,e.oLanguage,a);ba(g)}),i=!0):h.extend(!0,
+g.oLanguage,e.oLanguage);null===e.asStripeClasses&&(g.asStripeClasses=[g.oClasses.sStripeOdd,g.oClasses.sStripeEven]);b=g.asStripeClasses.length;g.asDestroyStripes=[];if(b){c=!1;d=h(this).children("tbody").children("tr:lt("+b+")");for(a=0;a<b;a++)d.hasClass(g.asStripeClasses[a])&&(c=!0,g.asDestroyStripes.push(g.asStripeClasses[a]));c&&d.removeClass(g.asStripeClasses.join(" "))}c=[];a=this.getElementsByTagName("thead");0!==a.length&&(V(g.aoHeader,a[0]),c=N(g));if(null===e.aoColumns){d=[];a=0;for(b=
+c.length;a<b;a++)d.push(null)}else d=e.aoColumns;a=0;for(b=d.length;a<b;a++)e.saved_aoColumns!==n&&e.saved_aoColumns.length==b&&(null===d[a]&&(d[a]={}),d[a].bVisible=e.saved_aoColumns[a].bVisible),o(g,c?c[a]:null);ta(g,e.aoColumnDefs,d,function(a,b){m(g,a,b)});a=0;for(b=g.aaSorting.length;a<b;a++){g.aaSorting[a][0]>=g.aoColumns.length&&(g.aaSorting[a][0]=0);var k=g.aoColumns[g.aaSorting[a][0]];g.aaSorting[a][2]===n&&(g.aaSorting[a][2]=0);e.aaSorting===n&&g.saved_aaSorting===n&&(g.aaSorting[a][1]=
+k.asSorting[0]);c=0;for(d=k.asSorting.length;c<d;c++)if(g.aaSorting[a][1]==k.asSorting[c]){g.aaSorting[a][2]=c;break}}P(g);Ua(g);a=h(this).children("caption").each(function(){this._captionSide=h(this).css("caption-side")});b=h(this).children("thead");0===b.length&&(b=[l.createElement("thead")],this.appendChild(b[0]));g.nTHead=b[0];b=h(this).children("tbody");0===b.length&&(b=[l.createElement("tbody")],this.appendChild(b[0]));g.nTBody=b[0];g.nTBody.setAttribute("role","alert");g.nTBody.setAttribute("aria-live",
+"polite");g.nTBody.setAttribute("aria-relevant","all");b=h(this).children("tfoot");if(0===b.length&&0<a.length&&(""!==g.oScroll.sX||""!==g.oScroll.sY))b=[l.createElement("tfoot")],this.appendChild(b[0]);0<b.length&&(g.nTFoot=b[0],V(g.aoFooter,g.nTFoot));if(f)for(a=0;a<e.aaData.length;a++)H(g,e.aaData[a]);else ua(g);g.aiDisplay=g.aiDisplayMaster.slice();g.bInitialised=!0;!1===i&&ba(g)}});ca=null;return this};j.fnVersionCheck=function(e){for(var h=function(e,h){for(;e.length<h;)e+="0";return e},m=j.ext.sVersion.split("."),
+e=e.split("."),k="",n="",l=0,t=e.length;l<t;l++)k+=h(m[l],3),n+=h(e[l],3);return parseInt(k,10)>=parseInt(n,10)};j.fnIsDataTable=function(e){for(var h=j.settings,m=0;m<h.length;m++)if(h[m].nTable===e||h[m].nScrollHead===e||h[m].nScrollFoot===e)return!0;return!1};j.fnTables=function(e){var o=[];jQuery.each(j.settings,function(j,k){(!e||!0===e&&h(k.nTable).is(":visible"))&&o.push(k.nTable)});return o};j.version="1.9.4";j.settings=[];j.models={};j.models.ext={afnFiltering:[],afnSortData:[],aoFeatures:[],
+aTypes:[],fnVersionCheck:j.fnVersionCheck,iApiIndex:0,ofnSearch:{},oApi:{},oStdClasses:{},oJUIClasses:{},oPagination:{},oSort:{},sVersion:j.version,sErrMode:"alert",_oExternConfig:{iNextUnique:0}};j.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};j.models.oRow={nTr:null,_aData:[],_aSortData:[],_anHidden:[],_sRowStripe:""};j.models.oColumn={aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bUseRendered:null,bVisible:null,_bAutoType:!0,fnCreatedCell:null,fnGetData:null,
+fnRender:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};j.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bJQueryUI:!1,bLengthChange:!0,
+bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollAutoCss:!0,bScrollCollapse:!1,bScrollInfinite:!1,bServerSide:!1,bSort:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCookieCallback:null,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(e){if(1E3>e)return e;for(var h=e+"",e=h.split(""),j="",h=h.length,k=0;k<h;k++)0===k%3&&0!==k&&(j=this.oLanguage.sInfoThousands+j),j=e[h-k-1]+j;return j},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,
+fnRowCallback:null,fnServerData:function(e,j,m,k){k.jqXHR=h.ajax({url:e,data:j,success:function(e){e.sError&&k.oApi._fnLog(k,0,e.sError);h(k.oInstance).trigger("xhr",[k,e]);m(e)},dataType:"json",cache:!1,type:k.sServerMethod,error:function(e,h){"parsererror"==h&&k.oApi._fnLog(k,0,"DataTables warning: JSON data from server could not be parsed. This is caused by a JSON formatting error.")}})},fnServerParams:null,fnStateLoad:function(e){var e=this.oApi._fnReadCookie(e.sCookiePrefix+e.sInstance),j;try{j=
+"function"===typeof h.parseJSON?h.parseJSON(e):eval("("+e+")")}catch(m){j=null}return j},fnStateLoadParams:null,fnStateLoaded:null,fnStateSave:function(e,h){this.oApi._fnCreateCookie(e.sCookiePrefix+e.sInstance,this.oApi._fnJsonString(h),e.iCookieDuration,e.sCookiePrefix,e.fnCookieCallback)},fnStateSaveParams:null,iCookieDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iScrollLoadGap:100,iTabIndex:0,oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},
+oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sInfoThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},j.models.oSearch),sAjaxDataProp:"aaData",
+sAjaxSource:null,sCookiePrefix:"SpryMedia_DataTables_",sDom:"lfrtip",sPaginationType:"two_button",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET"};j.defaults.columns={aDataSort:null,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bUseRendered:!0,bVisible:!0,fnCreatedCell:null,fnRender:null,iDataSort:-1,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};j.models.oSettings={oFeatures:{bAutoWidth:null,
+bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortClasses:null,bStateSave:null},oScroll:{bAutoCss:null,bCollapse:null,bInfinite:null,iBarWidth:0,iLoadGap:null,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1},aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aoColumns:[],aoHeader:[],aoFooter:[],asDataSearch:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:null,
+asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,sPaginationType:"two_button",iCookieDuration:0,sCookiePrefix:"",fnCookieCallback:null,aoStateSave:[],aoStateLoad:[],
+oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iDisplayEnd:10,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return this.oFeatures.bServerSide?parseInt(this._iRecordsTotal,10):this.aiDisplayMaster.length},
+fnRecordsDisplay:function(){return this.oFeatures.bServerSide?parseInt(this._iRecordsDisplay,10):this.aiDisplay.length},fnDisplayEnd:function(){return this.oFeatures.bServerSide?!1===this.oFeatures.bPaginate||-1==this._iDisplayLength?this._iDisplayStart+this.aiDisplay.length:Math.min(this._iDisplayStart+this._iDisplayLength,this._iRecordsDisplay):this._iDisplayEnd},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null};j.ext=h.extend(!0,{},j.models.ext);h.extend(j.ext.oStdClasses,
+{sTable:"dataTable",sPagePrevEnabled:"paginate_enabled_previous",sPagePrevDisabled:"paginate_disabled_previous",sPageNextEnabled:"paginate_enabled_next",sPageNextDisabled:"paginate_disabled_next",sPageJUINext:"",sPageJUIPrev:"",sPageButton:"paginate_button",sPageButtonActive:"paginate_active",sPageButtonStaticDisabled:"paginate_button paginate_button_disabled",sPageFirst:"first",sPagePrevious:"previous",sPageNext:"next",sPageLast:"last",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",
+sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",
+sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sFooterTH:"",sJUIHeader:"",sJUIFooter:""});h.extend(j.ext.oJUIClasses,j.ext.oStdClasses,{sPagePrevEnabled:"fg-button ui-button ui-state-default ui-corner-left",sPagePrevDisabled:"fg-button ui-button ui-state-default ui-corner-left ui-state-disabled",sPageNextEnabled:"fg-button ui-button ui-state-default ui-corner-right",
+sPageNextDisabled:"fg-button ui-button ui-state-default ui-corner-right ui-state-disabled",sPageJUINext:"ui-icon ui-icon-circle-arrow-e",sPageJUIPrev:"ui-icon ui-icon-circle-arrow-w",sPageButton:"fg-button ui-button ui-state-default",sPageButtonActive:"fg-button ui-button ui-state-default ui-state-disabled",sPageButtonStaticDisabled:"fg-button ui-button ui-state-default ui-state-disabled",sPageFirst:"first ui-corner-tl ui-corner-bl",sPageLast:"last ui-corner-tr ui-corner-br",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",
+sSortAsc:"ui-state-default",sSortDesc:"ui-state-default",sSortable:"ui-state-default",sSortableAsc:"ui-state-default",sSortableDesc:"ui-state-default",sSortableNone:"ui-state-default",sSortJUIAsc:"css_right ui-icon ui-icon-triangle-1-n",sSortJUIDesc:"css_right ui-icon ui-icon-triangle-1-s",sSortJUI:"css_right ui-icon ui-icon-carat-2-n-s",sSortJUIAscAllowed:"css_right ui-icon ui-icon-carat-1-n",sSortJUIDescAllowed:"css_right ui-icon ui-icon-carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",
+sScrollHead:"dataTables_scrollHead ui-state-default",sScrollFoot:"dataTables_scrollFoot ui-state-default",sFooterTH:"ui-state-default",sJUIHeader:"fg-toolbar ui-toolbar ui-widget-header ui-corner-tl ui-corner-tr ui-helper-clearfix",sJUIFooter:"fg-toolbar ui-toolbar ui-widget-header ui-corner-bl ui-corner-br ui-helper-clearfix"});h.extend(j.ext.oPagination,{two_button:{fnInit:function(e,j,m){var k=e.oLanguage.oPaginate,n=function(h){e.oApi._fnPageChange(e,h.data.action)&&m(e)},k=!e.bJUI?'<a class="'+
+e.oClasses.sPagePrevDisabled+'" tabindex="'+e.iTabIndex+'" role="button">'+k.sPrevious+'</a><a class="'+e.oClasses.sPageNextDisabled+'" tabindex="'+e.iTabIndex+'" role="button">'+k.sNext+"</a>":'<a class="'+e.oClasses.sPagePrevDisabled+'" tabindex="'+e.iTabIndex+'" role="button"><span class="'+e.oClasses.sPageJUIPrev+'"></span></a><a class="'+e.oClasses.sPageNextDisabled+'" tabindex="'+e.iTabIndex+'" role="button"><span class="'+e.oClasses.sPageJUINext+'"></span></a>';h(j).append(k);var l=h("a",j),
+k=l[0],l=l[1];e.oApi._fnBindAction(k,{action:"previous"},n);e.oApi._fnBindAction(l,{action:"next"},n);e.aanFeatures.p||(j.id=e.sTableId+"_paginate",k.id=e.sTableId+"_previous",l.id=e.sTableId+"_next",k.setAttribute("aria-controls",e.sTableId),l.setAttribute("aria-controls",e.sTableId))},fnUpdate:function(e){if(e.aanFeatures.p)for(var h=e.oClasses,j=e.aanFeatures.p,k,l=0,n=j.length;l<n;l++)if(k=j[l].firstChild)k.className=0===e._iDisplayStart?h.sPagePrevDisabled:h.sPagePrevEnabled,k=k.nextSibling,
+k.className=e.fnDisplayEnd()==e.fnRecordsDisplay()?h.sPageNextDisabled:h.sPageNextEnabled}},iFullNumbersShowPages:5,full_numbers:{fnInit:function(e,j,m){var k=e.oLanguage.oPaginate,l=e.oClasses,n=function(h){e.oApi._fnPageChange(e,h.data.action)&&m(e)};h(j).append('<a  tabindex="'+e.iTabIndex+'" class="'+l.sPageButton+" "+l.sPageFirst+'">'+k.sFirst+'</a><a  tabindex="'+e.iTabIndex+'" class="'+l.sPageButton+" "+l.sPagePrevious+'">'+k.sPrevious+'</a><span></span><a tabindex="'+e.iTabIndex+'" class="'+
+l.sPageButton+" "+l.sPageNext+'">'+k.sNext+'</a><a tabindex="'+e.iTabIndex+'" class="'+l.sPageButton+" "+l.sPageLast+'">'+k.sLast+"</a>");var t=h("a",j),k=t[0],l=t[1],r=t[2],t=t[3];e.oApi._fnBindAction(k,{action:"first"},n);e.oApi._fnBindAction(l,{action:"previous"},n);e.oApi._fnBindAction(r,{action:"next"},n);e.oApi._fnBindAction(t,{action:"last"},n);e.aanFeatures.p||(j.id=e.sTableId+"_paginate",k.id=e.sTableId+"_first",l.id=e.sTableId+"_previous",r.id=e.sTableId+"_next",t.id=e.sTableId+"_last")},
+fnUpdate:function(e,o){if(e.aanFeatures.p){var m=j.ext.oPagination.iFullNumbersShowPages,k=Math.floor(m/2),l=Math.ceil(e.fnRecordsDisplay()/e._iDisplayLength),n=Math.ceil(e._iDisplayStart/e._iDisplayLength)+1,t="",r,B=e.oClasses,u,M=e.aanFeatures.p,L=function(h){e.oApi._fnBindAction(this,{page:h+r-1},function(h){e.oApi._fnPageChange(e,h.data.page);o(e);h.preventDefault()})};-1===e._iDisplayLength?n=k=r=1:l<m?(r=1,k=l):n<=k?(r=1,k=m):n>=l-k?(r=l-m+1,k=l):(r=n-Math.ceil(m/2)+1,k=r+m-1);for(m=r;m<=k;m++)t+=
+n!==m?'<a tabindex="'+e.iTabIndex+'" class="'+B.sPageButton+'">'+e.fnFormatNumber(m)+"</a>":'<a tabindex="'+e.iTabIndex+'" class="'+B.sPageButtonActive+'">'+e.fnFormatNumber(m)+"</a>";m=0;for(k=M.length;m<k;m++)u=M[m],u.hasChildNodes()&&(h("span:eq(0)",u).html(t).children("a").each(L),u=u.getElementsByTagName("a"),u=[u[0],u[1],u[u.length-2],u[u.length-1]],h(u).removeClass(B.sPageButton+" "+B.sPageButtonActive+" "+B.sPageButtonStaticDisabled),h([u[0],u[1]]).addClass(1==n?B.sPageButtonStaticDisabled:
+B.sPageButton),h([u[2],u[3]]).addClass(0===l||n===l||-1===e._iDisplayLength?B.sPageButtonStaticDisabled:B.sPageButton))}}}});h.extend(j.ext.oSort,{"string-pre":function(e){"string"!=typeof e&&(e=null!==e&&e.toString?e.toString():"");return e.toLowerCase()},"string-asc":function(e,h){return e<h?-1:e>h?1:0},"string-desc":function(e,h){return e<h?1:e>h?-1:0},"html-pre":function(e){return e.replace(/<.*?>/g,"").toLowerCase()},"html-asc":function(e,h){return e<h?-1:e>h?1:0},"html-desc":function(e,h){return e<
+h?1:e>h?-1:0},"date-pre":function(e){e=Date.parse(e);if(isNaN(e)||""===e)e=Date.parse("01/01/1970 00:00:00");return e},"date-asc":function(e,h){return e-h},"date-desc":function(e,h){return h-e},"numeric-pre":function(e){return"-"==e||""===e?0:1*e},"numeric-asc":function(e,h){return e-h},"numeric-desc":function(e,h){return h-e}});h.extend(j.ext.aTypes,[function(e){if("number"===typeof e)return"numeric";if("string"!==typeof e)return null;var h,j=!1;h=e.charAt(0);if(-1=="0123456789-".indexOf(h))return null;
+for(var k=1;k<e.length;k++){h=e.charAt(k);if(-1=="0123456789.".indexOf(h))return null;if("."==h){if(j)return null;j=!0}}return"numeric"},function(e){var h=Date.parse(e);return null!==h&&!isNaN(h)||"string"===typeof e&&0===e.length?"date":null},function(e){return"string"===typeof e&&-1!=e.indexOf("<")&&-1!=e.indexOf(">")?"html":null}]);h.fn.DataTable=j;h.fn.dataTable=j;h.fn.dataTableSettings=j.settings;h.fn.dataTableExt=j.ext};"function"===typeof define&&define.amd?define(["jquery"],L):jQuery&&!jQuery.fn.dataTable&&
+L(jQuery)})(window,document);

headphones/__init__.py

@@ -154,6 +154,12 @@ MININOVA = None
 WAFFLES = None
 WAFFLES_UID = None
 WAFFLES_PASSKEY = None
+RUTRACKER = None
+RUTRACKER_USER = None
+RUTRACKER_PASSWORD = None
+WHATCD = None
+WHATCD_USERNAME = None
+WHATCD_PASSWORD = None
 DOWNLOAD_TORRENT_DIR = None
 
 INTERFACE = None
@@ -193,6 +199,8 @@ CUSTOMSLEEP = None
 HPUSER = None
 HPPASS = None
 
+CACHE_SIZEMB = 32
+
 def CheckSection(sec):
     """ Check if INI section exists, if not create it """
     try:
@@ -248,7 +256,8 @@ def initialize():
                 LOSSLESS_DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, ADD_ARTISTS, CORRECT_METADATA, MOVE_FILES, \
                 RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, EXTRAS, AUTOWANT_UPCOMING, AUTOWANT_ALL, \
                 ADD_ALBUM_ART, EMBED_ALBUM_ART, EMBED_LYRICS, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, SEARCH_INTERVAL, \
-                TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, ISOHUNT, KAT, MININOVA, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, DOWNLOAD_TORRENT_DIR, \
+                TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, ISOHUNT, KAT, MININOVA, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, \
+                RUTRACKER, RUTRACKER_USER, RUTRACKER_PASSWORD, WHATCD, WHATCD_USERNAME, WHATCD_PASSWORD, DOWNLOAD_TORRENT_DIR, \
                 LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \
                 NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS,\
                 NZBSORG, NZBSORG_UID, NZBSORG_HASH, NEWZBIN, NEWZBIN_UID, NEWZBIN_PASSWORD, LASTFM_USERNAME, INTERFACE, FOLDER_PERMISSIONS, \
@@ -256,7 +265,7 @@ def initialize():
                 ENCODERVBRCBR, ENCODERLOSSLESS, DELETE_LOSSLESS_FILES, PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, MIRRORLIST, \
                 MIRROR, CUSTOMHOST, CUSTOMPORT, CUSTOMSLEEP, HPUSER, HPPASS, XBMC_ENABLED, XBMC_HOST, XBMC_USERNAME, XBMC_PASSWORD, XBMC_UPDATE, \
                 XBMC_NOTIFY, NMA_ENABLED, NMA_APIKEY, NMA_PRIORITY, NMA_ONSNATCH, SYNOINDEX_ENABLED, ALBUM_COMPLETION_PCT, PREFERRED_BITRATE_HIGH_BUFFER, \
-                PREFERRED_BITRATE_LOW_BUFFER
+                PREFERRED_BITRATE_LOW_BUFFER,CACHE_SIZEMB
                 
         if __INITIALIZED__:
             return False
@@ -269,6 +278,8 @@ def initialize():
         CheckSection('NZBsorg')
         CheckSection('Newzbin')
         CheckSection('Waffles')
+        CheckSection('Rutracker')
+        CheckSection('What.cd')
         CheckSection('Prowl')
         CheckSection('XBMC')
         CheckSection('NMA')
@@ -296,6 +307,7 @@ def initialize():
         API_KEY = check_setting_str(CFG, 'General', 'api_key', '')
         GIT_PATH = check_setting_str(CFG, 'General', 'git_path', '')
         LOG_DIR = check_setting_str(CFG, 'General', 'log_dir', '')
+        CACHE_DIR = check_setting_str(CFG, 'General', 'cache_dir', '')
         
         CHECK_GITHUB = bool(check_setting_int(CFG, 'General', 'check_github', 1))
         CHECK_GITHUB_ON_STARTUP = bool(check_setting_int(CFG, 'General', 'check_github_on_startup', 1))
@@ -305,7 +317,7 @@ def initialize():
         DESTINATION_DIR = check_setting_str(CFG, 'General', 'destination_dir', '')
         LOSSLESS_DESTINATION_DIR = check_setting_str(CFG, 'General', 'lossless_destination_dir', '')
         PREFERRED_QUALITY = check_setting_int(CFG, 'General', 'preferred_quality', 0)
-        PREFERRED_BITRATE = check_setting_int(CFG, 'General', 'preferred_bitrate', '')
+        PREFERRED_BITRATE = check_setting_str(CFG, 'General', 'preferred_bitrate', '')
         PREFERRED_BITRATE_HIGH_BUFFER = check_setting_int(CFG, 'General', 'preferred_bitrate_high_buffer', '')
         PREFERRED_BITRATE_LOW_BUFFER = check_setting_int(CFG, 'General', 'preferred_bitrate_low_buffer', '')
         DETECT_BITRATE = bool(check_setting_int(CFG, 'General', 'detect_bitrate', 0))
@@ -342,6 +354,14 @@ def initialize():
         WAFFLES = bool(check_setting_int(CFG, 'Waffles', 'waffles', 0))
         WAFFLES_UID = check_setting_str(CFG, 'Waffles', 'waffles_uid', '')
         WAFFLES_PASSKEY = check_setting_str(CFG, 'Waffles', 'waffles_passkey', '')
+        
+        RUTRACKER = bool(check_setting_int(CFG, 'Rutracker', 'rutracker', 0))
+        RUTRACKER_USER = check_setting_str(CFG, 'Rutracker', 'rutracker_user', '')
+        RUTRACKER_PASSWORD = check_setting_str(CFG, 'Rutracker', 'rutracker_password', '')
+
+        WHATCD = bool(check_setting_int(CFG, 'What.cd', 'whatcd', 0))
+        WHATCD_USERNAME = check_setting_str(CFG, 'What.cd', 'whatcd_username', '')
+        WHATCD_PASSWORD = check_setting_str(CFG, 'What.cd', 'whatcd_password', '')
 
         SAB_HOST = check_setting_str(CFG, 'SABnzbd', 'sab_host', '')
         SAB_USERNAME = check_setting_str(CFG, 'SABnzbd', 'sab_username', '')
@@ -412,6 +432,8 @@ def initialize():
         CUSTOMSLEEP = check_setting_int(CFG, 'General', 'customsleep', 1)
         HPUSER = check_setting_str(CFG, 'General', 'hpuser', '')
         HPPASS = check_setting_str(CFG, 'General', 'hppass', '')
+
+        CACHE_SIZEMB = check_setting_int(CFG,'Advanced','cache_sizemb',32)
         
         ALBUM_COMPLETION_PCT = check_setting_int(CFG, 'Advanced', 'album_completion_pct', 80)
         
@@ -469,8 +491,9 @@ def initialize():
         # Start the logger, silence console logging if we need to
         logger.headphones_log.initLogger(verbose=VERBOSE)
         
-        # Put the cache dir in the data dir for now
-        CACHE_DIR = os.path.join(DATA_DIR, 'cache')
+        if not CACHE_DIR:
+            # Put the cache dir in the data dir for now
+            CACHE_DIR = os.path.join(DATA_DIR, 'cache')
         if not os.path.exists(CACHE_DIR):
             try:
                 os.makedirs(CACHE_DIR)
@@ -576,6 +599,7 @@ def config_write():
     new_config['General']['api_enabled'] = int(API_ENABLED)
     new_config['General']['api_key'] = API_KEY
     new_config['General']['log_dir'] = LOG_DIR
+    new_config['General']['cache_dir'] = CACHE_DIR
     new_config['General']['git_path'] = GIT_PATH
     
     new_config['General']['check_github'] = int(CHECK_GITHUB)
@@ -620,6 +644,16 @@ def config_write():
     new_config['Waffles']['waffles'] = int(WAFFLES)
     new_config['Waffles']['waffles_uid'] = WAFFLES_UID
     new_config['Waffles']['waffles_passkey'] = WAFFLES_PASSKEY
+    
+    new_config['Rutracker'] = {}
+    new_config['Rutracker']['rutracker'] = int(RUTRACKER)
+    new_config['Rutracker']['rutracker_user'] = RUTRACKER_USER
+    new_config['Rutracker']['rutracker_password'] = RUTRACKER_PASSWORD
+
+    new_config['What.cd'] = {}
+    new_config['What.cd']['whatcd'] = int(WHATCD)
+    new_config['What.cd']['whatcd_username'] = WHATCD_USERNAME
+    new_config['What.cd']['whatcd_password'] = WHATCD_PASSWORD
 
     new_config['General']['search_interval'] = SEARCH_INTERVAL
     new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL
@@ -708,6 +742,7 @@ def config_write():
     
     new_config['Advanced'] = {}
     new_config['Advanced']['album_completion_pct'] = ALBUM_COMPLETION_PCT
+    new_config['Advanced']['cache_sizemb'] = CACHE_SIZEMB
     
     new_config.write()
 

headphones/cache.py

@@ -62,21 +62,28 @@ class Cache(object):
     def __init__(self):
         
         pass
-        
-    def _exists(self, type):
 
-        self.artwork_files = glob.glob(os.path.join(self.path_to_art_cache, self.id + '*'))
-        self.thumb_files = glob.glob(os.path.join(self.path_to_art_cache, 'T_' + self.id + '*'))
+    def _findfilesstartingwith(self,pattern,folder):
+        files = []
+        if os.path.exists(folder):
+            for fname in os.listdir(folder):
+                if fname.startswith(pattern):
+                    files.append(os.path.join(folder,fname))
+        return files
+   
+    def _exists(self, type):
+        self.artwork_files = []
+        self.thumb_files = []
 
         if type == 'artwork':
-
+            self.artwork_files = self._findfilesstartingwith(self.id,self.path_to_art_cache)
             if self.artwork_files:
                 return True
             else:
                 return False
-                
+
         elif type == 'thumb':
-            
+            self.thumb_files = self._findfilesstartingwith("T_" + self.id,self.path_to_art_cache)
             if self.thumb_files:
                 return True
             else:

headphones/db.py

@@ -40,6 +40,12 @@ class DBConnection:
     
         self.filename = filename
         self.connection = sqlite3.connect(dbFilename(filename), timeout=20)
+        #don't wait for the disk to finish writing
+        self.connection.execute("PRAGMA synchronous = OFF")
+        #journal disabled since we never do rollbacks
+        self.connection.execute("PRAGMA journal_mode = OFF")        
+        #64mb of cache memory,probably need to make it user configurable
+        self.connection.execute("PRAGMA cache_size=-%s" % (headphones.CACHE_SIZEMB*1024))
         self.connection.row_factory = sqlite3.Row
         
     def action(self, query, args=None):

headphones/helpers.py

@@ -123,7 +123,12 @@ def bytes_to_mb(bytes):
     mb = int(bytes)/1048576
     size = '%.1f MB' % mb
     return size
-    
+
+def mb_to_bytes(mb_str):
+    result = re.search('^(\d+(?:\.\d+)?)\s?(?:mb)?', mb_str, flags=re.I)
+    if result:
+        return int(float(result.group(1))*1048576)
+
 def replace_all(text, dic):
     for i, j in dic.iteritems():
         text = text.replace(i, j)

headphones/importer.py

@@ -21,7 +21,12 @@ from lib.beets.mediafile import MediaFile
 import headphones
 from headphones import logger, helpers, db, mb, albumart, lastfm
 
-various_artists_mbid = '89ad4ac3-39f7-470e-963a-56509c546377'
+blacklisted_special_artist_names = ['[anonymous]','[data]','[no artist]','[traditional]','[unknown]','Various Artists']
+blacklisted_special_artists = ['f731ccc4-e22a-43af-a747-64213329e088','33cf029c-63b0-41a0-9855-be2a3665fb3b',\
+                                '314e1c25-dde7-4e4d-b2f4-0a7b9f7c56dc','eec63d3c-3b81-4ad4-b1e4-7c147d4d2b61',\
+                                '9be7f096-97ec-4615-8957-8d40b5dcbc41','125ec42a-7229-4250-afc5-e057484327fe',\
+                                '89ad4ac3-39f7-470e-963a-56509c546377']
+
         
 def is_exists(artistid):
 
@@ -63,7 +68,7 @@ def artistlist_to_mbids(artistlist, forced=False):
         
         if not forced:
             bl_artist = myDB.action('SELECT * FROM blacklist WHERE ArtistID=?', [artistid]).fetchone()
-            if bl_artist or artistid == various_artists_mbid:
+            if bl_artist or artistid in blacklisted_special_artists:
                 logger.info("Artist ID for '%s' is either blacklisted or Various Artists. To add artist, you must do it manually (Artist ID: %s)" % (artist, artistid))
                 continue
         
@@ -99,8 +104,8 @@ def addArtisttoDB(artistid, extrasonly=False):
     from headphones import cache
     
     # Can't add various artists - throws an error from MB
-    if artistid == various_artists_mbid:
-        logger.warn('Cannot import Various Artists.')
+    if artistid in blacklisted_special_artists:
+        logger.warn('Cannot import blocked special purpose artist with id' + artistid)
         return
         
     # We'll use this to see if we should update the 'LastUpdated' time stamp
@@ -133,6 +138,14 @@ def addArtisttoDB(artistid, extrasonly=False):
         
     artist = mb.getArtist(artistid, extrasonly)
     
+    if artist and artist.get('artist_name') in blacklisted_special_artist_names:
+        logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name'))
+        myDB.action('DELETE from artists WHERE ArtistID=?', [artistid])
+        #in case it's already in the db
+        myDB.action('DELETE from albums WHERE ArtistID=?', [artistid])
+        myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid])
+        return
+
     if not artist:
         logger.warn("Error fetching artist info. ID: " + artistid)
         if dbartist is None:
@@ -158,6 +171,16 @@ def addArtisttoDB(artistid, extrasonly=False):
     
     myDB.upsert("artists", newValueDict, controlValueDict)
 
+    # See if we need to grab extras. Artist specific extras take precedence over global option
+    # Global options are set when adding a new artist
+    myDB = db.DBConnection()
+    
+    try:
+        db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone()
+        includeExtras = db_artist['IncludeExtras']
+    except IndexError:
+        includeExtras = False  
+
     for rg in artist['releasegroups']:
         
         logger.info("Now adding/updating: " + rg['title'])
@@ -167,69 +190,54 @@ def addArtisttoDB(artistid, extrasonly=False):
         # check if the album already exists
         rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
                     
-        try:    
-            releaselist = mb.getReleaseGroup(rgid)
-        except Exception, e:
-            logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title'])
+        releases = mb.get_all_releases(rgid,includeExtras)
+        if releases == []:
+            logger.info('No official releases in release group %s' % rg['title'])
             continue
-            
-        if not releaselist:
+        if not releases:
             errors = True
+            logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title'])
             continue
         
         # This will be used later to build a hybrid release     
         fullreleaselist = []
-            
-        for release in releaselist:
+
+        for release in releases:
         # What we're doing here now is first updating the allalbums & alltracks table to the most
         # current info, then moving the appropriate release into the album table and its associated
         # tracks into the tracks table
-            
-            releaseid = release['id']
-            
-            try:
-                releasedict = mb.getRelease(releaseid, include_artist_info=False)
-            except Exception, e:
-                errors = True
-                logger.info('Unable to get release information for %s: %s' % (release['id'], e))
-                continue
-           
-            if not releasedict:
-                errors = True
-                continue
+            controlValueDict = {"ReleaseID" : release['ReleaseID']}
 
-            controlValueDict = {"ReleaseID":  release['id']}
-
-            newValueDict = {"ArtistID":         artistid,
-                            "ArtistName":       artist['artist_name'],
-                            "AlbumTitle":       rg['title'],
-                            "AlbumID":          rg['id'],
-                            "AlbumASIN":        releasedict['asin'],
-                            "ReleaseDate":      releasedict['date'],
-                            "Type":             rg['type'],
-                            "ReleaseCountry":   releasedict['country'],
-                            "ReleaseFormat":    releasedict['format']
+            newValueDict = {"ArtistID":         release['ArtistID'],
+                            "ArtistName":       release['ArtistName'],
+                            "AlbumTitle":       release['AlbumTitle'],
+                            "AlbumID":          release['AlbumID'],
+                            "AlbumASIN":        release['AlbumASIN'],
+                            "ReleaseDate":      release['ReleaseDate'],
+                            "Type":             release['Type'],
+                            "ReleaseCountry":   release['ReleaseCountry'],
+                            "ReleaseFormat":    release['ReleaseFormat']
                         }
-                        
+
             myDB.upsert("allalbums", newValueDict, controlValueDict)
             
             # Build the dictionary for the fullreleaselist
-            newValueDict['ReleaseID'] = release['id']
-            newValueDict['Tracks'] = releasedict['tracks']
+            newValueDict['ReleaseID'] = release['ReleaseID']
+            newValueDict['Tracks'] = release['Tracks']
             fullreleaselist.append(newValueDict)
             
-            for track in releasedict['tracks']:
+            for track in release['Tracks']:
 
                 cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
         
                 controlValueDict = {"TrackID":      track['id'],
-                                    "ReleaseID":    release['id']}
+                                    "ReleaseID":    release['ReleaseID']}
 
-                newValueDict = {"ArtistID":         artistid,
-                                "ArtistName":       artist['artist_name'],
-                                "AlbumTitle":       rg['title'],
-                                "AlbumASIN":        releasedict['asin'],
-                                "AlbumID":          rg['id'],
+                newValueDict = {"ArtistID":         release['ArtistID'],
+                                "ArtistName":       release['ArtistName'],
+                                "AlbumTitle":       release['AlbumTitle'],
+                                "AlbumID":          release['AlbumID'],
+                                "AlbumASIN":        release['AlbumASIN'],
                                 "TrackTitle":       track['title'],
                                 "TrackDuration":    track['duration'],
                                 "TrackNumber":      track['number'],
@@ -251,7 +259,13 @@ def addArtisttoDB(artistid, extrasonly=False):
                 myDB.upsert("alltracks", newValueDict, controlValueDict)
 
         # Basically just do the same thing again for the hybrid release
-        hybridrelease = getHybridRelease(fullreleaselist)
+        # This may end up being called with an empty fullreleaselist
+        try:
+            hybridrelease = getHybridRelease(fullreleaselist)
+        except Exception, e:
+            errors = True
+            logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e))
+            continue
         
         # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
         # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
@@ -584,6 +598,8 @@ def getHybridRelease(fullreleaselist):
     """
     Returns a dictionary of best group of tracks from the list of releases & earliest release date
     """
+    if len(fullreleaselist) == 0:
+        raise Exception("getHybridRelease was called with an empty fullreleaselist")
     sortable_release_list = []
         
     for release in fullreleaselist:

headphones/librarysync.py

@@ -56,6 +56,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None):
     song_list = []
     
     for r,d,f in os.walk(dir):
+        #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one
+        #using a list comprehension will not work correctly for nested subdirectories (os.walk keeps its original list)
+        for directory in d[:]:
+            if directory.startswith("."):
+                d.remove(directory)
         for files in f:
             # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc
             if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):

headphones/mb.py

@@ -30,16 +30,12 @@ mb_lock = threading.Lock()
 
 # Quick fix to add mirror switching on the fly. Need to probably return the mbhost & mbport that's
 # being used, so we can send those values to the log
-def startmb(forcemb=False):
+def startmb():
 
     mbuser = None
     mbpass = None
     
-    # Can use headphones mirror for queries
-    if headphones.MIRROR == "headphones" or "custom":
-        forcemb=False
-    
-    if forcemb or headphones.MIRROR == "musicbrainz.org":
+    if headphones.MIRROR == "musicbrainz.org":
         mbhost = "musicbrainz.org"
         mbport = 80
         sleepytime = 1
@@ -54,16 +50,15 @@ def startmb(forcemb=False):
         mbpass = headphones.HPPASS
         sleepytime = 0
     else:
-        mbhost = "tbueter.com"
-        mbport = 5000
-        sleepytime = 0
+        return False
     
     musicbrainzngs.set_useragent("headphones","0.0","https://github.com/rembo10/headphones")
     musicbrainzngs.set_hostname(mbhost + ":" + str(mbport))
     if sleepytime == 0:
         musicbrainzngs.set_rate_limit(False)
     else:
-        musicbrainzngs.set_rate_limit(True)
+        #calling it with an it ends up blocking all requests after the first
+        musicbrainzngs.set_rate_limit(limit_or_interval=float(sleepytime))
 
     # Add headphones credentials
     if headphones.MIRROR == "headphones":
@@ -71,13 +66,10 @@ def startmb(forcemb=False):
             logger.warn("No username or password set for VIP server")
         else:
             musicbrainzngs.hpauth(mbuser,mbpass)
-
-    # Don't really need to return q anymore since ngs, but maybe we can return an 'initialized=True' instead?
-    q = musicbrainzngs
     
     logger.debug('Using the following server values:\nMBHost: %s ; MBPort: %i  ;  Sleep Interval: %i ' % (mbhost, mbport, sleepytime))
     
-    return (q, sleepytime)
+    return True
 
 def findArtist(name, limit=1):
 
@@ -89,16 +81,12 @@ def findArtist(name, limit=1):
         if any((c in chars) for c in name):
             name = '"'+name+'"'
             
-        q, sleepytime = startmb(forcemb=True)
-
         try:
-            artistResults = musicbrainzngs.search_artists(query=name,limit=limit)['artist-list']
+            artistResults = musicbrainzngs.search_artists(query='artist:'+name,limit=limit)['artist-list']
         except WebServiceError, e:
             logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e)))
             time.sleep(5)
         
-        time.sleep(sleepytime)
-        
         if not artistResults:
             return False        
         for result in artistResults:
@@ -134,27 +122,23 @@ def findArtist(name, limit=1):
 def findRelease(name, limit=1):
 
     with mb_lock:        
-        releaselistngs = []
-        releaseResultsngs = None
+        releaselist = []
+        releaseResults = None
         
         chars = set('!?')
         if any((c in chars) for c in name):
             name = '"'+name+'"'
             
-        q, sleepytime = startmb(forcemb=True)
-        
         try:
-            releaseResultsngs = musicbrainzngs.search_releases(query=name,limit=limit)['release-list']
+            releaseResults = musicbrainzngs.search_releases(query=name,limit=limit)['release-list']
         except WebServiceError, e: #need to update exceptions
             logger.warn('Attempt to query MusicBrainz for "%s" failed: %s' % (name, str(e)))
             time.sleep(5)
-        
-        time.sleep(sleepytime)
-        
-        if not releaseResultsngs:
+
+        if not releaseResults:
             return False
-        for result in releaseResultsngs:
-                        releaselistngs.append({
+        for result in releaseResults:
+                        releaselist.append({
                         'uniquename':        unicode(result['artist-credit'][0]['artist']['name']),
                         'title':             unicode(result['title']),
                         'id':                unicode(result['artist-credit'][0]['artist']['id']),
@@ -163,7 +147,7 @@ def findRelease(name, limit=1):
                         'albumurl':          unicode("http://musicbrainz.org/release/" + result['id']),#probably needs to be changed
                         'score':             int(result['ext:score'])
                         })            
-        return releaselistngs
+        return releaselist
 
 def getArtist(artistid, extrasonly=False):
 
@@ -172,8 +156,6 @@ def getArtist(artistid, extrasonly=False):
     
         artist = None
         
-        q, sleepytime = startmb()
-        
         try:
             limit = 100
             artist = musicbrainzngs.get_artist_by_id(artistid)['artist']
@@ -191,8 +173,6 @@ def getArtist(artistid, extrasonly=False):
         if not artist:
             return False
         
-        time.sleep(sleepytime)
-        
         #if 'disambiguation' in artist:
         #    uniquename = unicode(artist['sort-name'] + " (" + artist['disambiguation'] + ")")
         #else:
@@ -285,9 +265,7 @@ def getReleaseGroup(rgid):
         releaselist = []
         
         releaseGroup = None
-        
-        q, sleepytime = startmb()
-        
+               
         try:
             releaseGroup = musicbrainzngs.get_release_group_by_id(rgid,["artists","releases","media","discids",])['release-group']
         except WebServiceError, e:
@@ -308,8 +286,6 @@ def getRelease(releaseid, include_artist_info=True):
         release = {}
         results = None
         
-        q, sleepytime = startmb()
-        
         try:
             if include_artist_info:
                 results = musicbrainzngs.get_release_by_id(releaseid,["artists","release-groups","media","recordings"]).get('release')
@@ -321,8 +297,6 @@ def getRelease(releaseid, include_artist_info=True):
         
         if not results:
             return False
-        
-        time.sleep(sleepytime)
 
         release['title'] = unicode(results['title'])
         release['id'] = unicode(results['id']) 
@@ -350,24 +324,86 @@ def getRelease(releaseid, include_artist_info=True):
 
             release['artist_name'] = unicode(results['artist-credit'][0]['artist']['name'])
             release['artist_id'] = unicode(results['artist-credit'][0]['artist']['id'])
-                
-        totalTracks = 1
-        tracks = []
-        for medium in results['medium-list']:                
-            for track in medium['track-list']:
-                tracks.append({
-                        'number':        totalTracks,
-                        'title':         unicode(track['recording']['title']),
-                        'id':            unicode(track['recording']['id']),
-                        'url':           u"http://musicbrainz.org/track/" + track['recording']['id'],
-                        'duration':      int(track['length']) if 'length' in track else 0
-                        })
-                totalTracks += 1       
 
-        release['tracks'] = tracks
+        release['tracks'] = getTracksFromRelease(results)
         
         return release
 
+def get_all_releases(rgid,includeExtras=False):
+    results = []
+    try:
+        limit = 100
+        newResults = None
+        while newResults == None or len(newResults) >= limit:
+            newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results))
+            if 'release-list' not in newResults:
+                break #may want to raise an exception here instead ?
+            newResults = newResults['release-list']
+            results += newResults
+            
+    except WebServiceError, e:
+        logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e)))
+        time.sleep(5)
+        return False
+        
+    if not results or len(results) == 0:
+        return False
+
+        
+    releases = []
+    for releasedata in results:
+        #releasedata.get will return None if it doesn't have a status
+        #all official releases should have the Official status included
+        if not includeExtras and releasedata.get('status') != 'Official':
+            continue
+        
+        release = {}
+        release['AlbumTitle'] = unicode(releasedata['title'])
+        release['AlbumID'] = unicode(rgid)
+        release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None
+        release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None      
+        release['ReleaseID'] = releasedata['id']
+        if 'release-group' not in releasedata:
+            raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid)
+        release['Type'] = unicode(releasedata['release-group']['type'])
+
+
+        #making the assumption that the most important artist will be first in the list
+        if 'artist-credit' in releasedata:
+            release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id'])
+            release['ArtistName'] = unicode(releasedata['artist-credit-phrase'])
+        else:
+            logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.')
+            return False
+                
+
+        release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown'
+        #assuming that the list will contain media and that the format will be consistent
+        try:
+            release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format'])
+        except:
+            release['ReleaseFormat'] = u'Unknown'
+ 
+        release['Tracks'] = getTracksFromRelease(releasedata)
+        releases.append(release)
+
+    return releases
+
+def getTracksFromRelease(release):
+    totalTracks = 1
+    tracks = []
+    for medium in release['medium-list']:
+        for track in medium['track-list']:
+            tracks.append({
+                    'number':        totalTracks,
+                    'title':         unicode(track['recording']['title']),
+                    'id':            unicode(track['recording']['id']),
+                    'url':           u"http://musicbrainz.org/track/" + track['recording']['id'],
+                    'duration':      int(track['length']) if 'length' in track else 0
+                    })
+            totalTracks += 1      
+    return tracks
+
 # Used when there is a disambiguation
 def findArtistbyAlbum(name):
 
@@ -386,15 +422,12 @@ def findArtistbyAlbum(name):
 
     results = None
     
-    q, sleepytime = startmb(forcemb=True)
-    
     try:
         results = musicbrainzngs.search_release_groups(term).get('release-group-list')
     except WebServiceError, e:
         logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e)))
         time.sleep(5)    
     
-    time.sleep(sleepytime)
     
     if not results:
         return False
@@ -419,23 +452,19 @@ def findArtistbyAlbum(name):
     
 def findAlbumID(artist=None, album=None):
 
-    results_ngs = None
-    
-    q, sleepytime = startmb(forcemb=True)
+    results = None
     
     try:
         term = '"'+album+'" AND artist:"'+artist+'"'
-        results_ngs = musicbrainzngs.search_release_groups(term,1).get('release-group-list')
+        results = musicbrainzngs.search_release_groups(term,1).get('release-group-list')
     except WebServiceError, e:
         logger.warn('Attempt to query MusicBrainz for %s - %s failed (%s)' % (artist, album, str(e)))
         time.sleep(5)
-    
-    time.sleep(sleepytime)
-    
-    if not results_ngs:
+
+    if not results:
         return False
 
-    if len(results_ngs) < 1:
+    if len(results) < 1:
         return False    
-    rgid_ngs = unicode(results_ngs[0]['id'])
-    return rgid_ngs
+    rgid = unicode(results[0]['id'])
+    return rgid

headphones/searcher.py

@@ -15,6 +15,10 @@
 
 import urllib, urllib2, urlparse
 import lib.feedparser as feedparser
+from lib.pygazelle import api as gazelleapi
+from lib.pygazelle import encoding as gazelleencoding
+from lib.pygazelle import format as gazelleformat
+from lib.pygazelle import media as gazellemedia
 from xml.dom import minidom
 from xml.parsers.expat import ExpatError
 from StringIO import StringIO
@@ -28,6 +32,9 @@ from headphones import logger, db, helpers, classes, sab
 
 import lib.bencode as bencode
 
+import headphones.searcher_rutracker as rutrackersearch
+rutracker = rutrackersearch.Rutracker()
+
 class NewzbinDownloader(urllib.FancyURLopener):
 
     def __init__(self):
@@ -97,7 +104,8 @@ def searchforalbum(albumid=None, new=False, lossless=False):
                 else:
                     foundNZB = searchNZB(result['AlbumID'], new)
 
-            if (headphones.KAT or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES) and foundNZB == "none":
+            if (headphones.KAT or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES or headphones.RUTRACKER or headphones.WHATCD) and foundNZB == "none":
+
                 if result['Status'] == "Wanted Lossless":
                     searchTorrent(result['AlbumID'], new, losslessOnly=True)
                 else:
@@ -109,7 +117,7 @@ def searchforalbum(albumid=None, new=False, lossless=False):
         if (headphones.NZBMATRIX or headphones.NEWZNAB or headphones.NZBSORG or headphones.NEWZBIN) and (headphones.SAB_HOST or headphones.BLACKHOLE):
             foundNZB = searchNZB(albumid, new, lossless)
 
-        if (headphones.KAT or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES) and foundNZB == "none":
+        if (headphones.KAT or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES or headphones.RUTRACKER or headphones.WHATCD) and foundNZB == "none":
             searchTorrent(albumid, new, lossless)
 
 def searchNZB(albumid=None, new=False, losslessOnly=False):
@@ -632,6 +640,13 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
         results = myDB.select('SELECT ArtistName, AlbumTitle, AlbumID, ReleaseDate from albums WHERE Status="Wanted" OR Status="Wanted Lossless"')
         new = True
         
+    # rutracker login
+    
+    if headphones.RUTRACKER and results:
+        rulogin = rutracker.login(headphones.RUTRACKER_USER, headphones.RUTRACKER_PASSWORD)
+        if not rulogin:
+            logger.info(u'Could not login to rutracker, search results will exclude this provider')
+    
     for albums in results:
         
         albumid = albums[2]
@@ -642,10 +657,12 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
         except TypeError:
             year = ''
         
-        dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':''}
+        dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':' ', '*':''}
 
-        cleanalbum = helpers.latinToAscii(helpers.replace_all(albums[1], dic))
-        cleanartist = helpers.latinToAscii(helpers.replace_all(albums[0], dic))
+        semi_cleanalbum = helpers.replace_all(albums[1], dic)
+        cleanalbum = helpers.latinToAscii(semi_cleanalbum)
+        semi_cleanartist = helpers.replace_all(albums[0], dic)
+        cleanartist = helpers.latinToAscii(semi_cleanartist)
 
         # FLAC usually doesn't have a year for some reason so I'll leave it out
         # Various Artist albums might be listed as VA, so I'll leave that out too
@@ -656,7 +673,9 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
             term = cleanalbum + ' ' + year
         else:
             term = cleanartist + ' ' + cleanalbum
-            
+
+        semi_clean_artist_term = re.sub('[\.\-\/]', ' ', semi_cleanartist).encode('utf-8')
+        semi_clean_album_term = re.sub('[\.\-\/]', ' ', semi_cleanalbum).encode('utf-8')
         # Replace bad characters in the term and unicode it
         term = re.sub('[\.\-\/]', ' ', term).encode('utf-8')
         artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8')
@@ -665,6 +684,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
         logger.info("Searching torrents for %s since it was marked as wanted" % term)
         
         resultlist = []
+        pre_sorted_results = False
         minimumseeders = int(headphones.NUMBEROFSEEDERS) - 1
 
         if headphones.KAT:
@@ -805,8 +825,116 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                             logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
                         except Exception, e:
                             logger.error(u"An error occurred while trying to parse the response from Waffles.fm: %s" % e)
+        
+        # rutracker.org
+        
+        if headphones.RUTRACKER and rulogin:
+        
+            provider = "rutracker.org"
+            
+            # Ignore if release date not specified, results too unpredictable
+            
+            if not year:
+                logger.info(u'Release date not specified, ignoring for rutracker.org')
+            else:
+            
+                bitrate = False
+            
+                if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
+                    format = 'lossless'
+                    maxsize = 10000000000
+                elif headphones.PREFERRED_QUALITY == 1:
+                    format = 'lossless+mp3'
+                    maxsize = 10000000000
+                else:
+                    format = 'mp3'
+                    maxsize = 300000000
+                    if headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE:
+                        bitrate = True
+                
+                # build search url based on above
+            
+                searchURL = rutracker.searchurl(artistterm, albumterm, year, format)
+                logger.info(u'Parsing results from <a href="%s">rutracker.org</a>' % searchURL)
+            
+                # parse results and get best match
+            
+                rulist = rutracker.search(searchURL, maxsize, minimumseeders, albumid, bitrate)
+            
+                # add best match to overall results list
+            
+                if rulist:
+                    for ru in rulist:
+                        title = ru[0].decode('utf-8')
+                        size = ru[1]
+                        url = ru[2]
+                        resultlist.append((title, size, url, provider))
+                        logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
+                else:
+                    logger.info(u"No valid results found from %s" % (provider))
 
+        if headphones.WHATCD:
+            provider = "What.cd"
+            providerurl = "http://what.cd/"
 
+            bitrate = None
+            bitrate_string = bitrate
+            if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
+                format = gazelleformat.FLAC
+                maxsize = 10000000000
+            elif headphones.PREFERRED_QUALITY:
+                format=None
+                bitrate = headphones.PREFERRED_BITRATE
+                for encoding_string in gazelleencoding.ALL_ENCODINGS:
+                    if re.search(bitrate, encoding_string, flags=re.I):
+                        bitrate_string = encoding_string
+                if bitrate_string not in gazelleencoding.ALL_ENCODINGS:
+                    raise Exception("Preferred bitrate %s not recognized by %s" % (bitrate_string, provider))
+                maxsize = 10000000000
+            else:
+                format = gazelleformat.MP3
+                maxsize = 300000000
+
+            try:
+                gazelle = gazelleapi.GazelleAPI(headphones.WHATCD_USERNAME, headphones.WHATCD_PASSWORD)
+            except Exception, e:
+                gazelle = None
+                logger.warn(u"What.cd credentials incorrect or site is down. Error: %s %s" % (e.__class__.__name__, str(e)))
+
+            if gazelle:
+                logger.info(u"Searching %s..." % provider)
+                search_results = gazelle.search_torrents(artistname=semi_clean_artist_term, groupname=semi_clean_album_term,
+                                                            format=format, encoding=bitrate_string)
+
+                # filter on format, size, and num seeders
+                logger.info(u"Filtering torrents by format, maximum size, and minimum seeders...")
+                all_torrents = search_results['results']
+                match_torrents = [ torrent for torrent in all_torrents if torrent.size <= maxsize ]
+                match_torrents = [ torrent for torrent in match_torrents if torrent.seeders >= minimumseeders ]
+
+                logger.info(u"Remaining torrents: %s" % ", ".join(repr(torrent) for torrent in match_torrents))
+
+                # sort by times d/l'd
+                if not len(match_torrents):
+                    logger.info(u"No results found from %s for %s after filtering" % (provider, term))
+                elif len(match_torrents) > 1:
+                    logger.info(u"Found %d matching releases from %s for %s - %s after filtering" %
+                                (len(match_torrents), provider, artistterm, albumterm))
+                    logger.info("Sorting torrents by times snatched and preferred bitrate %s..." % bitrate_string)
+                    match_torrents.sort(key=lambda x: int(x.snatched), reverse=True)
+    #                if bitrate:
+    #                    match_torrents.sort(key=lambda x: re.match("mp3", x.getTorrentDetails(), flags=re.I), reverse=True)
+    #                    match_torrents.sort(key=lambda x: str(bitrate) in x.getTorrentFolderName(), reverse=True)
+                    logger.info(u"New order: %s" % ", ".join(repr(torrent) for torrent in match_torrents))
+
+                pre_sorted_results = True
+                for torrent in match_torrents:
+                    if not torrent.file_path:
+                        torrent.group.update_group_data() # will load the file_path for the individual torrents
+                    resultlist.append((torrent.file_path,
+                                       torrent.size,
+                                       gazelle.generate_torrent_link(torrent.id),
+                                       provider))
 
         if headphones.ISOHUNT:
             provider = "isoHunt"    
@@ -961,7 +1089,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
         
         if len(resultlist):    
                        
-            if headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE:
+            if headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE and not pre_sorted_results:
 
                 logger.debug('Target bitrate: %s kbps' % headphones.PREFERRED_BITRATE)
 
@@ -987,6 +1115,10 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                     logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1]))
                     
                     torrentlist = sorted(resultlist, key=lambda title: title[1], reverse=True)
+
+            elif pre_sorted_results:
+
+                torrentlist = resultlist
             
             else:
             
@@ -1013,7 +1145,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
 
             logger.info(u"Pre-processing result")
             
-            (data, bestqual) = preprocesstorrent(torrentlist)
+            (data, bestqual) = preprocesstorrent(torrentlist, pre_sorted_results)
             
             if data and bestqual:
                 logger.info(u'Found best result from %s: <a href="%s">%s</a> - %s' % (bestqual[3], bestqual[2], bestqual[0], helpers.bytes_to_mb(bestqual[1])))
@@ -1029,19 +1161,24 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                 
                     # Get torrent name from .torrent, this is usually used by the torrent client as the folder name
 
-
                     torrent_name = torrent_folder_name + '.torrent'
                     download_path = os.path.join(headphones.TORRENTBLACKHOLE_DIR, torrent_name)
                     try:
-			#Write the torrent file to a path derived from the TORRENTBLACKHOLE_DIR and file name.
-			torrent_file = open(download_path, 'wb')
-			torrent_file.write(data)
-			torrent_file.close()
-			#Open the fresh torrent file again so we can extract the proper torrent name
-			#Used later in post-processing.
-			torrent_file = open(download_path, 'rb')
+                        if bestqual[3] == 'rutracker.org':
+			                download_path = rutracker.get_torrent(bestqual[2], headphones.TORRENTBLACKHOLE_DIR)
+			                if not download_path:
+			                    break
+                        else:  
+			                #Write the torrent file to a path derived from the TORRENTBLACKHOLE_DIR and file name.
+			                torrent_file = open(download_path, 'wb')
+			                torrent_file.write(data)
+			                torrent_file.close()
+			                
+			            #Open the fresh torrent file again so we can extract the proper torrent name
+			            #Used later in post-processing.
+                        torrent_file = open(download_path, 'rb')
                         torrent_info = bencode.bdecode(torrent_file.read())
-			torrent_file.close()
+                        torrent_file.close()
                         torrent_folder_name = torrent_info['info'].get('name','').decode('utf-8')
                         logger.info('Torrent folder name: %s' % torrent_folder_name)
                     except Exception, e:
@@ -1051,14 +1188,28 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                 myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]])
                 myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", torrent_folder_name])
 
-def preprocesstorrent(resultlist):
+def preprocesstorrent(resultlist, pre_sorted_list=False):
     selresult = ""
     for result in resultlist:
         if selresult == "":
             selresult = result
         elif int(selresult[1]) < int(result[1]): # if size is lower than new result replace previous selected result (bigger size = better quality?)
             selresult = result
-            
+             
+    # get outta here if rutracker
+        
+    if selresult[3] == 'rutracker.org':
+        return True, selresult
+                   
+    if pre_sorted_list:
+        selresult = resultlist[0]
+    else:
+        for result in resultlist:
+            if selresult == "":
+                selresult = result
+            elif int(selresult[1]) < int(result[1]): # if size is lower than new result replace previous selected result (bigger size = better quality?)
+                selresult = result
+
     try:
         request = urllib2.Request(selresult[2])
         request.add_header('Accept-encoding', 'gzip')

headphones/searcher_rutracker.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# Headphones rutracker.org search
+# Functions called from searcher.py
+
+import urllib
+import urllib2
+import cookielib
+from urlparse import urlparse
+from bs4 import BeautifulSoup
+from headphones import logger, db
+import lib.bencode as bencode
+import os
+
+class Rutracker():
+
+    logged_in = False
+    # Stores a number of login attempts to prevent recursion.
+    #login_counter = 0
+    
+    def __init__(self):
+
+        self.cookiejar = cookielib.CookieJar()
+        self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookiejar))
+        urllib2.install_opener(self.opener)
+
+    def login(self, login, password):
+        """Implements tracker login procedure."""
+        
+        self.logged_in = False
+
+        if login is None or password is None:
+            return False
+
+        #self.login_counter += 1
+        
+        # No recursion wanted.
+        #if self.login_counter > 1:
+        #    return False
+        
+        params = urllib.urlencode({"login_username" : login,
+                                   "login_password" : password,
+                                   "login" : "Вход"})
+
+        try:
+            self.opener.open("http://login.rutracker.org/forum/login.php", params)
+        except :
+            pass
+            
+        # Check if we're logged in
+        
+        for cookie in self.cookiejar:
+            if cookie.name == 'bb_data':
+                self.logged_in = True
+        
+        return self.logged_in
+
+    def searchurl(self, artist, album, year, format):
+        """
+        Return the search url
+        """
+         
+        # Build search url
+        
+        searchterm = ''
+        if artist != 'Various Artists':
+            searchterm = artist
+            searchterm = searchterm + ' '
+        searchterm = searchterm + album
+        searchterm = searchterm + ' '
+        searchterm = searchterm + year
+        
+        providerurl = "http://rutracker.org/forum/tracker.php"
+        
+        if format == 'lossless':
+            format = '+lossless'
+        elif format == 'lossless+mp3':
+            format = '+lossless||mp3||aac'
+        else:
+            format = '+mp3||aac'
+            
+        # sort by size, descending. 
+        
+        sort = '&o=7&s=2'
+        
+        searchurl = "%s?nm=%s%s%s" % (providerurl, urllib.quote(searchterm), format, sort)
+        
+        return searchurl
+    
+    def search(self, searchurl, maxsize, minseeders, albumid, bitrate):
+        """
+        Parse the search results and return the first valid torrent
+        """
+        
+        titles = []
+        urls = []
+        seeders = []
+        sizes = []
+        torrentlist = [] 
+        rulist = []
+        
+        try:
+            
+            page = self.opener.open(searchurl, timeout=60)
+            soup = BeautifulSoup(page.read())
+            
+            # Debug
+            #logger.debug (soup.prettify()) 
+            
+            # Title
+             
+            for link in soup.find_all('a', attrs={'class' : 'med tLink bold'}): 
+                title = link.get_text()
+                titles.append(title)
+            
+            # Download URL
+
+            for link in soup.find_all('a', attrs={'class' : 'small tr-dl dl-stub'}):
+                url = link.get('href')
+                urls.append(url)
+                
+            # Seeders
+             
+            for link in soup.find_all('td', attrs={'class' : 'row4 seedmed'}): 
+                seeder = link.get_text()
+                seeders.append(seeder)
+            
+            # Size
+             
+            for link in soup.find_all('td', attrs={'class' : 'row4 small nowrap tor-size'}): 
+                size = link.u.string
+                sizes.append(size)
+                
+        except :
+            pass
+            
+        # Combine lists
+        
+        torrentlist = zip(titles, urls, seeders, sizes)
+        
+        # return if nothing found
+        
+        if not torrentlist:
+            return False
+            
+         # get headphones track count for album, return if not found
+        
+        hptrackcount = 0
+        
+        myDB = db.DBConnection()
+        tracks = myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [albumid])
+        for track in tracks:
+            hptrackcount += 1
+        
+        if not hptrackcount:
+            logger.info('headphones track info not found, cannot compare to torrent') 
+            return False
+        
+        # Return the first valid torrent, unless we want a preferred bitrate then we want all valid entries
+       
+        for torrent in torrentlist:
+            
+            returntitle = torrent[0].encode('utf-8')
+            url = torrent[1]
+            seeders = torrent[2]
+            size = torrent[3]
+            
+            # Attempt to filter out unwanted
+            
+            title = returntitle.lower()
+            
+            if 'promo' not in title and 'vinyl' not in title and 'songbook' not in title and 'tvrip' not in title and 'hdtv' not in title and 'dvd' not in title \
+              and int(size) <= maxsize and int(seeders) >= minseeders:
+                     
+                # Check torrent info
+                
+                torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])['t']
+                self.cookiejar.set_cookie(cookielib.Cookie(version=0, name='bb_dl', value=torrent_id, port=None, port_specified=False, domain='.rutracker.org', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False))
+                                          
+                # Debug
+                #for cookie in self.cookiejar:
+                #    logger.debug ('Cookie: %s' % cookie) 
+                     
+                try:
+                    page = self.opener.open(url)
+                    torrent = page.read()
+                    if torrent:
+                        decoded = bencode.bdecode(torrent)
+                        metainfo = decoded['info']
+                    page.close ()
+                except Exception, e:
+                    logger.error('Error getting torrent: %s' % e)  
+                    return False      
+                
+                # get torrent track count and check for cue
+                
+                trackcount = 0
+                cuecount = 0
+                
+                if 'files' in metainfo: # multi
+                    for pathfile in metainfo['files']:
+                        path = pathfile['path']
+                        for file in path:
+                            if '.ape' in file or '.flac' in file or '.ogg' in file or '.m4a' in file or '.aac' in file or '.mp3' in file or '.wav' in file or '.aif' in file:
+                                trackcount += 1
+                            if '.cue' in file:
+                                cuecount += 1
+                                     
+                #Torrent topic page
+        
+                topicurl = 'http://rutracker.org/forum/viewtopic.php?t=' + torrent_id
+                logger.debug ('torrent title: %s' % title)
+                logger.debug ('headphones trackcount: %s' % hptrackcount) 
+                logger.debug ('rutracker trackcount: %s' % trackcount)
+
+                # If torrent track count less than headphones track count, and there's a cue, then attempt to get track count from log(s)
+                # This is for the case where we have a single .flac/.wav which can be split by cue
+                # Not great, but shouldn't be doing this too often
+                
+                totallogcount = 0
+                if trackcount < hptrackcount and cuecount > 0 and cuecount < hptrackcount:
+                    page = self.opener.open(topicurl, timeout=60)
+                    soup = BeautifulSoup(page.read())
+                    findtoc = soup.find_all(text='TOC of the extracted CD')
+                    if not findtoc:
+                        findtoc = soup.find_all(text='TOC извлечённого CD')
+                    for toc in findtoc:
+                        logcount = 0
+                        for toccontent in toc.find_all_next(text=True):
+                            cut_string = toccontent.split('|')
+                            new_string = cut_string[0].lstrip().rstrip()
+                            if new_string == '1' or new_string == '01':
+                                logcount = 1
+                            elif logcount > 0:
+                                if new_string.isdigit():
+                                    logcount += 1
+                                else:
+                                    break
+                        totallogcount = totallogcount + logcount
+                            
+                if totallogcount > 0:
+                    trackcount = totallogcount        
+                    logger.debug ('rutracker logtrackcount: %s' % totallogcount)
+                
+                # If torrent track count = hp track count then return torrent, 
+                # if greater, check for deluxe/special/foreign editions
+                # if less, then allow if it's a single track with a cue
+                
+                valid = False
+                
+                if trackcount == hptrackcount:
+                    valid = True
+                elif trackcount > hptrackcount:
+                    if 'deluxe' in title or 'edition' in title or 'japanese' in title:
+                        valid = True
+                        
+                # return 1st valid torrent if not checking by bitrate, else add to list and return at end
+                
+                if valid:
+                    rulist.append((returntitle, size, topicurl))
+                    if not bitrate:
+                        return rulist
+                         
+        return rulist
+
+
+    def get_torrent(self, url, savelocation):
+    
+        torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])['t']
+        self.cookiejar.set_cookie(cookielib.Cookie(version=0, name='bb_dl', value=torrent_id, port=None, port_specified=False, domain='.rutracker.org', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False))
+        downloadurl = 'http://dl.rutracker.org/forum/dl.php?t=' + torrent_id                  
+        torrent_name = torrent_id + '.torrent'
+        download_path = os.path.join(savelocation, torrent_name)
+        
+        try:
+            page = self.opener.open(downloadurl)
+            torrent = page.read()
+            fp = open (download_path, 'wb')
+            fp.write (torrent)
+            fp.close ()
+        except Exception, e:
+            logger.error('Error getting torrent: %s' % e)  
+            return False      
+        
+        return download_path
+        

headphones/versioncheck.py

@@ -20,7 +20,7 @@ from headphones import logger, version
 
 import lib.simplejson as simplejson
 
-user = "rembo10"
+user = "AdeHub"
 branch = "master"
 
 def runGit(args):

headphones/webserve.py

@@ -26,10 +26,12 @@ import threading
 import headphones
 
 from headphones import logger, searcher, db, importer, mb, lastfm, librarysync
-from headphones.helpers import checked, radio
+from headphones.helpers import checked, radio,today
 
 import lib.simplejson as simplejson
 
+import sys
+
 
 
 def serve_template(templatename, **kwargs):
@@ -190,6 +192,20 @@ class WebInterface(object):
         raise cherrypy.HTTPRedirect("home")
     deleteArtist.exposed = True
     
+
+    def deleteEmptyArtists(self):
+        logger.info(u"Deleting all empty artists")
+        myDB = db.DBConnection()
+        emptyArtistIDs = [row['ArtistID'] for row in myDB.select("SELECT ArtistID FROM artists WHERE HaveTracks == 0")]
+        for ArtistID in emptyArtistIDs:
+            logger.info(u"Deleting all traces of artist: " + ArtistID)
+            myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID])
+            myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID])
+            myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID])
+            myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID])
+    deleteEmptyArtists.exposed = True     
+   
+        
     def refreshArtist(self, ArtistID):
         threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start()  
         raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID)
@@ -394,6 +410,110 @@ class WebInterface(object):
         return serve_template(templatename="logs.html", title="Log", lineList=headphones.LOG_LIST)
     logs.exposed = True
     
+
+    def getLog(self,iDisplayStart=0,iDisplayLength=100,iSortCol_0=0,sSortDir_0="desc",sSearch="",**kwargs):
+
+        iDisplayStart = int(iDisplayStart)
+        iDisplayLength = int(iDisplayLength)
+
+        filtered = []
+        if sSearch == "":
+            filtered = headphones.LOG_LIST[::]
+        else:
+            filtered = [row for row in headphones.LOG_LIST for column in row if sSearch in column]
+
+        sortcolumn = 0
+        if iSortCol_0 == '1':
+            sortcolumn = 2
+        elif iSortCol_0 == '2':
+            sortcolumn = 1
+        filtered.sort(key=lambda x:x[sortcolumn],reverse=sSortDir_0 == "desc")        
+
+        rows = filtered[iDisplayStart:(iDisplayStart+iDisplayLength)]
+        rows = [[row[0],row[2],row[1]] for row in rows]
+
+        dict = {'iTotalDisplayRecords':len(filtered),
+                'iTotalRecords':len(headphones.LOG_LIST),
+                'aaData':rows,
+                }
+        s = simplejson.dumps(dict)
+        return s
+    getLog.exposed = True
+
+    def getArtists_json(self,iDisplayStart=0,iDisplayLength=100,sSearch="",iSortCol_0='0',sSortDir_0='asc',**kwargs):
+        iDisplayStart = int(iDisplayStart)
+        iDisplayLength = int(iDisplayLength)
+        filtered = []
+        totalcount = 0        
+        myDB = db.DBConnection()
+        
+        
+        sortcolumn = 'ArtistSortName'
+        sortbyhavepercent = False
+        if iSortCol_0 == '2':
+            sortcolumn = 'Status'
+        elif iSortCol_0 == '3':
+            sortcolumn = 'ReleaseDate'
+        elif iSortCol_0 == '4':
+            sortbyhavepercent = True
+
+        if sSearch == "":
+            query = 'SELECT * from artists order by %s COLLATE NOCASE %s' % (sortcolumn,sSortDir_0)    
+            filtered = myDB.select(query)
+            totalcount = len(filtered) 
+        else:
+            query = 'SELECT * from artists WHERE ArtistSortName LIKE "%' + sSearch + '%" OR LatestAlbum LIKE "%' + sSearch +'%"' +  'ORDER BY %s COLLATE NOCASE %s' % (sortcolumn,sSortDir_0)
+            filtered = myDB.select(query)
+            totalcount = myDB.select('SELECT COUNT(*) from artists')[0][0]
+
+        if sortbyhavepercent:
+            filtered.sort(key=lambda x:(float(x['HaveTracks'])/x['TotalTracks'] if x['TotalTracks'] > 0 else 0.0,x['HaveTracks'] if x['HaveTracks'] else 0.0),reverse=sSortDir_0 == "asc")
+
+        #can't figure out how to change the datatables default sorting order when its using an ajax datasource so ill 
+        #just reverse it here and the first click on the "Latest Album" header will sort by descending release date
+        if sortcolumn == 'ReleaseDate':
+            filtered.reverse()
+            
+
+        artists = filtered[iDisplayStart:(iDisplayStart+iDisplayLength)]
+        rows = []
+        for artist in artists:
+            row = {"ArtistID":artist['ArtistID'],
+                      "ArtistSortName":artist["ArtistSortName"],
+                      "Status":artist["Status"],
+                      "TotalTracks":artist["TotalTracks"],
+                      "HaveTracks":artist["HaveTracks"],
+                      "LatestAlbum":"",                      
+                      "ReleaseDate":"",
+                      "ReleaseInFuture":"False",
+                      "AlbumID":"",
+                      }
+
+            if not row['HaveTracks']:
+                row['HaveTracks'] = 0
+            if artist['ReleaseDate'] and artist['LatestAlbum']:
+                row['ReleaseDate'] = artist['ReleaseDate']
+                row['LatestAlbum'] = artist['LatestAlbum']
+                row['AlbumID'] = artist['AlbumID']
+                if artist['ReleaseDate'] > today():
+                    row['ReleaseInFuture'] = "True"
+            elif artist['LatestAlbum']:
+                row['ReleaseDate'] = ''
+                row['LatestAlbum'] = artist['LatestAlbum']
+                row['AlbumID'] = artist['AlbumID']
+              
+            rows.append(row)
+
+
+        dict = {'iTotalDisplayRecords':len(filtered),
+                'iTotalRecords':totalcount,
+                'aaData':rows,
+                }
+        s = simplejson.dumps(dict)
+        cherrypy.response.headers['Content-type'] = 'application/json'
+        return s
+    getArtists_json.exposed=True
+
     def clearhistory(self, type=None):
         myDB = db.DBConnection()
         if type == 'all':
@@ -463,6 +583,12 @@ class WebInterface(object):
                     "use_waffles" : checked(headphones.WAFFLES),
                     "waffles_uid" : headphones.WAFFLES_UID,
                     "waffles_passkey": headphones.WAFFLES_PASSKEY,
+                    "use_rutracker" : checked(headphones.RUTRACKER),
+                    "rutracker_user" : headphones.RUTRACKER_USER,
+                    "rutracker_password": headphones.RUTRACKER_PASSWORD,
+                    "use_whatcd" : checked(headphones.WHATCD),
+                    "whatcd_username" : headphones.WHATCD_USERNAME,
+                    "whatcd_password": headphones.WHATCD_PASSWORD,
                     "pref_qual_0" : radio(headphones.PREFERRED_QUALITY, 0),
                     "pref_qual_1" : radio(headphones.PREFERRED_QUALITY, 1),
                     "pref_qual_3" : radio(headphones.PREFERRED_QUALITY, 3),
@@ -486,6 +612,7 @@ class WebInterface(object):
                     "autowant_upcoming" : checked(headphones.AUTOWANT_UPCOMING),
                     "autowant_all" : checked(headphones.AUTOWANT_ALL),
                     "log_dir" : headphones.LOG_DIR,
+                    "cache_dir" : headphones.CACHE_DIR,
                     "interface_list" : interface_list,
                     "music_encoder":        checked(headphones.MUSIC_ENCODER),
                     "encoder":      headphones.ENCODER,
@@ -519,7 +646,8 @@ class WebInterface(object):
                     "customport": headphones.CUSTOMPORT,
                     "customsleep": headphones.CUSTOMSLEEP,
                     "hpuser": headphones.HPUSER,
-                    "hppass": headphones.HPPASS
+                    "hppass": headphones.HPPASS,
+                    "cache_sizemb":headphones.CACHE_SIZEMB,
                 }
             
         # Need to convert EXTRAS to a dictionary we can pass to the config: it'll come in as a string like 2,5,6,8
@@ -538,21 +666,21 @@ class WebInterface(object):
         
         return serve_template(templatename="config.html", title="Settings", config=config)  
     config.exposed = True
-    
-    
+
+
     def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0, api_enabled=0, api_key=None, 
         download_scan_interval=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, 
         sab_category=None, download_dir=None, blackhole=0, blackhole_dir=None, usenet_retention=None, nzbmatrix=0, nzbmatrix_username=None, nzbmatrix_apikey=None, 
         newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, newzbin=0, newzbin_uid=None, 
         newzbin_password=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None, 
-        numberofseeders=10, use_isohunt=0, use_kat=0, use_mininova=0, waffles=0, waffles_uid=None, waffles_passkey=None, rename_files=0, correct_metadata=0, 
-        cleanup_files=0, add_album_art=0, embed_album_art=0, embed_lyrics=0, destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, 
-        include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0, remix=0, spokenword=0, audiobook=0, autowant_upcoming=False, autowant_all=False, 
-        interface=None, log_dir=None, music_encoder=0, encoder=None, bitrate=None, samplingfrequency=None, encoderfolder=None, advancedencoder=None, 
-        encoderoutputformat=None, encodervbrcbr=None, encoderquality=None, encoderlossless=0, delete_lossless_files=0, prowl_enabled=0, prowl_onsnatch=0, 
-        prowl_keys=None, prowl_priority=0, xbmc_enabled=0, xbmc_host=None, xbmc_username=None, xbmc_password=None, xbmc_update=0, xbmc_notify=0, nma_enabled=False, 
-        nma_apikey=None, nma_priority=0, nma_onsnatch=0, synoindex_enabled=False, mirror=None, customhost=None, customport=None, customsleep=None, hpuser=None, hppass=None, 
-        preferred_bitrate_high_buffer=None, preferred_bitrate_low_buffer=None, **kwargs):
+        numberofseeders=10, use_isohunt=0, use_kat=0, use_mininova=0, waffles=0, waffles_uid=None, waffles_passkey=None, whatcd=0, whatcd_username=None, whatcd_password=None,
+        rutracker=0, rutracker_user=None, rutracker_password=None, rename_files=0, correct_metadata=0, cleanup_files=0, add_album_art=0, embed_album_art=0, embed_lyrics=0, 
+        destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0,
+        remix=0, spokenword=0, audiobook=0, autowant_upcoming=False, autowant_all=False, interface=None, log_dir=None, cache_dir=None, music_encoder=0, encoder=None, bitrate=None, 
+        samplingfrequency=None, encoderfolder=None, advancedencoder=None, encoderoutputformat=None, encodervbrcbr=None, encoderquality=None, encoderlossless=0, 
+        delete_lossless_files=0, prowl_enabled=0, prowl_onsnatch=0, prowl_keys=None, prowl_priority=0, xbmc_enabled=0, xbmc_host=None, xbmc_username=None, xbmc_password=None, 
+        xbmc_update=0, xbmc_notify=0, nma_enabled=False, nma_apikey=None, nma_priority=0, nma_onsnatch=0, synoindex_enabled=False, mirror=None, customhost=None, customport=None, 
+        customsleep=None, hpuser=None, hppass=None, preferred_bitrate_high_buffer=None, preferred_bitrate_low_buffer=None, cache_sizemb=None, **kwargs):
 
         headphones.HTTP_HOST = http_host
         headphones.HTTP_PORT = http_port
@@ -595,6 +723,12 @@ class WebInterface(object):
         headphones.WAFFLES = waffles
         headphones.WAFFLES_UID = waffles_uid
         headphones.WAFFLES_PASSKEY = waffles_passkey
+        headphones.RUTRACKER = rutracker
+        headphones.RUTRACKER_USER = rutracker_user
+        headphones.RUTRACKER_PASSWORD = rutracker_password
+        headphones.WHATCD = whatcd
+        headphones.WHATCD_USERNAME = whatcd_username
+        headphones.WHATCD_PASSWORD = whatcd_password
         headphones.PREFERRED_QUALITY = int(preferred_quality)
         headphones.PREFERRED_BITRATE = preferred_bitrate
         headphones.PREFERRED_BITRATE_HIGH_BUFFER = preferred_bitrate_high_buffer
@@ -616,6 +750,7 @@ class WebInterface(object):
         headphones.AUTOWANT_ALL = autowant_all
         headphones.INTERFACE = interface
         headphones.LOG_DIR = log_dir
+        headphones.CACHE_DIR = cache_dir
         headphones.MUSIC_ENCODER = music_encoder
         headphones.ENCODER = encoder
         headphones.BITRATE = int(bitrate)
@@ -648,6 +783,7 @@ class WebInterface(object):
         headphones.CUSTOMSLEEP = customsleep
         headphones.HPUSER = hpuser
         headphones.HPPASS = hppass
+        headphones.CACHE_SIZEMB = cache_sizemb
 
         # Handle the variable config options. Note - keys with False values aren't getting passed
         
@@ -680,6 +816,9 @@ class WebInterface(object):
         # Write the config
         headphones.config_write()
 
+        #reconfigure musicbrainz database connection with the new values
+        mb.startmb()
+
         raise cherrypy.HTTPRedirect("config")
         
     configUpdate.exposed = True
@@ -770,3 +909,75 @@ class WebInterface(object):
         return simplejson.dumps(image_dict)
         
     getImageLinks.exposed = True
+
+class Artwork(object):
+    def index(self):
+        return "Artwork"
+    index.exposed = True
+
+    def default(self,ArtistOrAlbum="",ID=None):
+        from headphones import cache
+        ArtistID = None
+        AlbumID = None
+        if ArtistOrAlbum == "artist":
+            ArtistID = ID
+        elif ArtistOrAlbum == "album":
+            AlbumID = ID
+    
+        relpath =  cache.getArtwork(ArtistID,AlbumID)
+
+        if not relpath:
+            relpath = "data/interfaces/default/images/no-cover-art.png"
+            basedir = os.path.dirname(sys.argv[0])
+            path = os.path.join(basedir,relpath)
+            cherrypy.response.headers['Content-type'] = 'image/png'
+            cherrypy.response.headers['Cache-Control'] = 'no-cache'
+        else:
+            relpath = relpath.replace('cache/','',1)
+            path = os.path.join(headphones.CACHE_DIR,relpath)
+            fileext = os.path.splitext(relpath)[1][1::]
+            cherrypy.response.headers['Content-type'] = 'image/' + fileext
+            cherrypy.response.headers['Cache-Control'] = 'max-age=31556926'
+
+        path = os.path.normpath(path)
+        f = open(path,'rb')
+        return f.read()
+    default.exposed = True
+
+    class Thumbs(object):
+        def index(self):
+            return "Here be thumbs"
+        index.exposed = True
+        def default(self,ArtistOrAlbum="",ID=None):
+            from headphones import cache
+            ArtistID = None
+            AlbumID = None
+            if ArtistOrAlbum == "artist":
+                ArtistID = ID
+            elif ArtistOrAlbum == "album":
+                AlbumID = ID
+    
+            relpath =  cache.getThumb(ArtistID,AlbumID)
+
+            if not relpath:
+                relpath = "data/interfaces/default/images/no-cover-artist.png"
+                basedir = os.path.dirname(sys.argv[0])
+                path = os.path.join(basedir,relpath)
+                cherrypy.response.headers['Content-type'] = 'image/png'
+                cherrypy.response.headers['Cache-Control'] = 'no-cache'
+            else:
+                relpath = relpath.replace('cache/','',1)
+                path = os.path.join(headphones.CACHE_DIR,relpath)
+                fileext = os.path.splitext(relpath)[1][1::]
+                cherrypy.response.headers['Content-type'] = 'image/' + fileext
+                cherrypy.response.headers['Cache-Control'] = 'max-age=31556926'
+
+            path = os.path.normpath(path)
+            f = open(path,'rb')
+            return f.read()
+        default.exposed = True
+    
+    thumbs = Thumbs()
+    
+    
+WebInterface.artwork = Artwork()

headphones/webstart.py

@@ -31,6 +31,9 @@ def initialize(options={}):
                 'server.socket_port':   options['http_port'],
                 'server.socket_host':   options['http_host'],
                 'engine.autoreload_on': False,
+                'tools.encode.on' : True,
+                'tools.encode.encoding' : 'utf-8',
+                'tools.decode.on' : True,
         })
 
     conf = {
@@ -56,7 +59,7 @@ def initialize(options={}):
         },
         '/favicon.ico':{
             'tools.staticfile.on': True,
-            'tools.staticfile.filename': "images/favicon.ico"
+            'tools.staticfile.filename': os.path.join(os.path.abspath(os.curdir),"images" + os.sep + "favicon.ico")
         },
         '/cache':{
             'tools.staticdir.on': True,

html5lib/__init__.py

@@ -0,0 +1,17 @@
+""" 
+HTML parsing library based on the WHATWG "HTML5"
+specification. The parser is designed to be compatible with existing
+HTML found in the wild and implements well-defined error recovery that
+is largely compatible with modern desktop web browsers.
+
+Example usage:
+
+import html5lib
+f = open("my_document.html")
+tree = html5lib.parse(f) 
+"""
+__version__ = "0.95-dev"
+from html5parser import HTMLParser, parse, parseFragment
+from treebuilders import getTreeBuilder
+from treewalkers import getTreeWalker
+from serializer import serialize

html5lib/filters/_base.py

@@ -0,0 +1,10 @@
+
+class Filter(object):
+    def __init__(self, source):
+        self.source = source
+
+    def __iter__(self):
+        return iter(self.source)
+
+    def __getattr__(self, name):
+        return getattr(self.source, name)

html5lib/filters/formfiller.py

@@ -0,0 +1,127 @@
+#
+# The goal is to finally have a form filler where you pass data for
+# each form, using the algorithm for "Seeding a form with initial values"
+# See http://www.whatwg.org/specs/web-forms/current-work/#seeding
+#
+
+import _base
+
+from html5lib.constants import spaceCharacters
+spaceCharacters = u"".join(spaceCharacters)
+
+class SimpleFilter(_base.Filter):
+    def __init__(self, source, fieldStorage):
+        _base.Filter.__init__(self, source)
+        self.fieldStorage = fieldStorage
+
+    def __iter__(self):
+        field_indices = {}
+        state = None
+        field_name = None
+        for token in _base.Filter.__iter__(self):
+            type = token["type"]
+            if type in ("StartTag", "EmptyTag"):
+                name = token["name"].lower()
+                if name == "input":
+                    field_name = None
+                    field_type = None
+                    input_value_index = -1
+                    input_checked_index = -1
+                    for i,(n,v) in enumerate(token["data"]):
+                        n = n.lower()
+                        if n == u"name":
+                            field_name = v.strip(spaceCharacters)
+                        elif n == u"type":
+                            field_type = v.strip(spaceCharacters)
+                        elif n == u"checked":
+                            input_checked_index = i
+                        elif n == u"value":
+                            input_value_index = i
+
+                    value_list = self.fieldStorage.getlist(field_name)
+                    field_index = field_indices.setdefault(field_name, 0)
+                    if field_index < len(value_list):
+                        value = value_list[field_index]
+                    else:
+                        value = ""
+
+                    if field_type in (u"checkbox", u"radio"):
+                        if value_list:
+                            if token["data"][input_value_index][1] == value:
+                                if input_checked_index < 0:
+                                    token["data"].append((u"checked", u""))
+                                field_indices[field_name] = field_index + 1
+                            elif input_checked_index >= 0:
+                                del token["data"][input_checked_index]
+
+                    elif field_type not in (u"button", u"submit", u"reset"):
+                        if input_value_index >= 0:
+                            token["data"][input_value_index] = (u"value", value)
+                        else:
+                            token["data"].append((u"value", value))
+                        field_indices[field_name] = field_index + 1
+
+                    field_type = None
+                    field_name = None
+
+                elif name == "textarea":
+                    field_type = "textarea"
+                    field_name = dict((token["data"])[::-1])["name"]
+
+                elif name == "select":
+                    field_type = "select"
+                    attributes = dict(token["data"][::-1])
+                    field_name = attributes.get("name")
+                    is_select_multiple = "multiple" in attributes
+                    is_selected_option_found = False
+
+                elif field_type == "select" and field_name and name == "option":
+                    option_selected_index = -1
+                    option_value = None
+                    for i,(n,v) in enumerate(token["data"]):
+                        n = n.lower()
+                        if n == "selected":
+                            option_selected_index = i
+                        elif n == "value":
+                            option_value = v.strip(spaceCharacters)
+                    if option_value is None:
+                        raise NotImplementedError("<option>s without a value= attribute")
+                    else:
+                        value_list = self.fieldStorage.getlist(field_name)
+                        if value_list:
+                            field_index = field_indices.setdefault(field_name, 0)
+                            if field_index < len(value_list):
+                                value = value_list[field_index]
+                            else:
+                                value = ""
+                            if (is_select_multiple or not is_selected_option_found) and option_value == value:
+                                if option_selected_index < 0:
+                                    token["data"].append((u"selected", u""))
+                                field_indices[field_name] = field_index + 1
+                                is_selected_option_found = True
+                            elif option_selected_index >= 0:
+                                del token["data"][option_selected_index]
+
+            elif field_type is not None and field_name and type == "EndTag":
+                name = token["name"].lower()
+                if name == field_type:
+                    if name == "textarea":
+                        value_list = self.fieldStorage.getlist(field_name)
+                        if value_list:
+                            field_index = field_indices.setdefault(field_name, 0)
+                            if field_index < len(value_list):
+                                value = value_list[field_index]
+                            else:
+                                value = ""
+                            yield {"type": "Characters", "data": value}
+                            field_indices[field_name] = field_index + 1
+
+                    field_name = None
+
+                elif name == "option" and field_type == "select":
+                    pass # TODO: part of "option without value= attribute" processing
+
+            elif field_type == "textarea":
+                continue # ignore token
+
+            yield token

html5lib/filters/inject_meta_charset.py

@@ -0,0 +1,62 @@
+import _base
+
+class Filter(_base.Filter):
+    def __init__(self, source, encoding):
+        _base.Filter.__init__(self, source)
+        self.encoding = encoding
+
+    def __iter__(self):
+        state = "pre_head"
+        meta_found = (self.encoding is None)
+        pending = []
+
+        for token in _base.Filter.__iter__(self):
+            type = token["type"]
+            if type == "StartTag":
+                if token["name"].lower() == u"head":
+                    state = "in_head"
+
+            elif type == "EmptyTag":
+                if token["name"].lower() == u"meta":
+                   # replace charset with actual encoding
+                   has_http_equiv_content_type = False
+                   for (namespace,name),value in token["data"].iteritems():
+                       if namespace != None:
+                           continue
+                       elif name.lower() == u'charset':
+                          token["data"][(namespace,name)] = self.encoding
+                          meta_found = True
+                          break
+                       elif name == u'http-equiv' and value.lower() == u'content-type':
+                           has_http_equiv_content_type = True
+                   else:
+                       if has_http_equiv_content_type and (None, u"content") in token["data"]:
+                           token["data"][(None, u"content")] = u'text/html; charset=%s' % self.encoding
+                           meta_found = True
+
+                elif token["name"].lower() == u"head" and not meta_found:
+                    # insert meta into empty head
+                    yield {"type": "StartTag", "name": u"head",
+                           "data": token["data"]}
+                    yield {"type": "EmptyTag", "name": u"meta",
+                           "data": {(None, u"charset"): self.encoding}}
+                    yield {"type": "EndTag", "name": u"head"}
+                    meta_found = True
+                    continue
+
+            elif type == "EndTag":
+                if token["name"].lower() == u"head" and pending:
+                    # insert meta into head (if necessary) and flush pending queue
+                    yield pending.pop(0)
+                    if not meta_found:
+                        yield {"type": "EmptyTag", "name": u"meta",
+                               "data": {(None, u"charset"): self.encoding}}
+                    while pending:
+                        yield pending.pop(0)
+                    meta_found = True
+                    state = "post_head"
+
+            if state == "in_head":
+                pending.append(token)
+            else:
+                yield token

html5lib/filters/lint.py

@@ -0,0 +1,88 @@
+from gettext import gettext
+_ = gettext
+
+import _base
+from html5lib.constants import cdataElements, rcdataElements, voidElements
+
+from html5lib.constants import spaceCharacters
+spaceCharacters = u"".join(spaceCharacters)
+
+class LintError(Exception): pass
+
+class Filter(_base.Filter):
+    def __iter__(self):
+        open_elements = []
+        contentModelFlag = "PCDATA"
+        for token in _base.Filter.__iter__(self):
+            type = token["type"]
+            if type in ("StartTag", "EmptyTag"):
+                name = token["name"]
+                if contentModelFlag != "PCDATA":
+                    raise LintError(_("StartTag not in PCDATA content model flag: %s") % name)
+                if not isinstance(name, unicode):
+                    raise LintError(_(u"Tag name is not a string: %r") % name)
+                if not name:
+                    raise LintError(_(u"Empty tag name"))
+                if type == "StartTag" and name in voidElements:
+                    raise LintError(_(u"Void element reported as StartTag token: %s") % name)
+                elif type == "EmptyTag" and name not in voidElements:
+                    raise LintError(_(u"Non-void element reported as EmptyTag token: %s") % token["name"])
+                if type == "StartTag":
+                    open_elements.append(name)
+                for name, value in token["data"]:
+                    if not isinstance(name, unicode):
+                        raise LintError(_("Attribute name is not a string: %r") % name)
+                    if not name:
+                        raise LintError(_(u"Empty attribute name"))
+                    if not isinstance(value, unicode):
+                        raise LintError(_("Attribute value is not a string: %r") % value)
+                if name in cdataElements:
+                    contentModelFlag = "CDATA"
+                elif name in rcdataElements:
+                    contentModelFlag = "RCDATA"
+                elif name == "plaintext":
+                    contentModelFlag = "PLAINTEXT"
+
+            elif type == "EndTag":
+                name = token["name"]
+                if not isinstance(name, unicode):
+                    raise LintError(_(u"Tag name is not a string: %r") % name)
+                if not name:
+                    raise LintError(_(u"Empty tag name"))
+                if name in voidElements:
+                    raise LintError(_(u"Void element reported as EndTag token: %s") % name)
+                start_name = open_elements.pop()
+                if start_name != name:
+                    raise LintError(_(u"EndTag (%s) does not match StartTag (%s)") % (name, start_name))
+                contentModelFlag = "PCDATA"
+
+            elif type == "Comment":
+                if contentModelFlag != "PCDATA":
+                    raise LintError(_("Comment not in PCDATA content model flag"))
+
+            elif type in ("Characters", "SpaceCharacters"):
+                data = token["data"]
+                if not isinstance(data, unicode):
+                    raise LintError(_("Attribute name is not a string: %r") % data)
+                if not data:
+                    raise LintError(_(u"%s token with empty data") % type)
+                if type == "SpaceCharacters":
+                    data = data.strip(spaceCharacters)
+                    if data:
+                        raise LintError(_(u"Non-space character(s) found in SpaceCharacters token: ") % data)
+
+            elif type == "Doctype":
+                name = token["name"]
+                if contentModelFlag != "PCDATA":
+                    raise LintError(_("Doctype not in PCDATA content model flag: %s") % name)
+                if not isinstance(name, unicode):
+                    raise LintError(_(u"Tag name is not a string: %r") % name)
+                # XXX: what to do with token["data"] ?
+
+            elif type in ("ParseError", "SerializeError"):
+                pass
+
+            else:
+                raise LintError(_(u"Unknown token type: %s") % type)
+
+            yield token

html5lib/filters/optionaltags.py

@@ -0,0 +1,202 @@
+import _base
+
+class Filter(_base.Filter):
+    def slider(self):
+        previous1 = previous2 = None
+        for token in self.source:
+            if previous1 is not None:
+                yield previous2, previous1, token
+            previous2 = previous1
+            previous1 = token
+        yield previous2, previous1, None
+
+    def __iter__(self):
+        for previous, token, next in self.slider():
+            type = token["type"]
+            if type == "StartTag":
+                if (token["data"] or 
+                    not self.is_optional_start(token["name"], previous, next)):
+                    yield token
+            elif type == "EndTag":
+                if not self.is_optional_end(token["name"], next):
+                    yield token
+            else:
+                yield token
+
+    def is_optional_start(self, tagname, previous, next):
+        type = next and next["type"] or None
+        if tagname in 'html':
+            # An html element's start tag may be omitted if the first thing
+            # inside the html element is not a space character or a comment.
+            return type not in ("Comment", "SpaceCharacters")
+        elif tagname == 'head':
+            # A head element's start tag may be omitted if the first thing
+            # inside the head element is an element.
+            # XXX: we also omit the start tag if the head element is empty
+            if type in ("StartTag", "EmptyTag"):
+                return True
+            elif type == "EndTag":
+                return next["name"] == "head"
+        elif tagname == 'body':
+            # A body element's start tag may be omitted if the first thing
+            # inside the body element is not a space character or a comment,
+            # except if the first thing inside the body element is a script
+            # or style element and the node immediately preceding the body
+            # element is a head element whose end tag has been omitted.
+            if type in ("Comment", "SpaceCharacters"):
+                return False
+            elif type == "StartTag":
+                # XXX: we do not look at the preceding event, so we never omit
+                # the body element's start tag if it's followed by a script or
+                # a style element.
+                return next["name"] not in ('script', 'style')
+            else:
+                return True
+        elif tagname == 'colgroup':
+            # A colgroup element's start tag may be omitted if the first thing
+            # inside the colgroup element is a col element, and if the element
+            # is not immediately preceeded by another colgroup element whose
+            # end tag has been omitted.
+            if type in ("StartTag", "EmptyTag"):
+                # XXX: we do not look at the preceding event, so instead we never
+                # omit the colgroup element's end tag when it is immediately
+                # followed by another colgroup element. See is_optional_end.
+                return next["name"] == "col"
+            else:
+                return False
+        elif tagname == 'tbody':
+            # A tbody element's start tag may be omitted if the first thing
+            # inside the tbody element is a tr element, and if the element is
+            # not immediately preceeded by a tbody, thead, or tfoot element
+            # whose end tag has been omitted.
+            if type == "StartTag":
+                # omit the thead and tfoot elements' end tag when they are
+                # immediately followed by a tbody element. See is_optional_end.
+                if previous and previous['type'] == 'EndTag' and \
+                  previous['name'] in ('tbody','thead','tfoot'):
+                    return False
+                return next["name"] == 'tr'
+            else:
+                return False
+        return False
+
+    def is_optional_end(self, tagname, next):
+        type = next and next["type"] or None
+        if tagname in ('html', 'head', 'body'):
+            # An html element's end tag may be omitted if the html element
+            # is not immediately followed by a space character or a comment.
+            return type not in ("Comment", "SpaceCharacters")
+        elif tagname in ('li', 'optgroup', 'tr'):
+            # A li element's end tag may be omitted if the li element is
+            # immediately followed by another li element or if there is
+            # no more content in the parent element.
+            # An optgroup element's end tag may be omitted if the optgroup
+            # element is immediately followed by another optgroup element,
+            # or if there is no more content in the parent element.
+            # A tr element's end tag may be omitted if the tr element is
+            # immediately followed by another tr element, or if there is
+            # no more content in the parent element.
+            if type == "StartTag":
+                return next["name"] == tagname
+            else:
+                return type == "EndTag" or type is None
+        elif tagname in ('dt', 'dd'):
+            # A dt element's end tag may be omitted if the dt element is
+            # immediately followed by another dt element or a dd element.
+            # A dd element's end tag may be omitted if the dd element is
+            # immediately followed by another dd element or a dt element,
+            # or if there is no more content in the parent element.
+            if type == "StartTag":
+                return next["name"] in ('dt', 'dd')
+            elif tagname == 'dd':
+                return type == "EndTag" or type is None
+            else:
+                return False
+        elif tagname == 'p':
+            # A p element's end tag may be omitted if the p element is
+            # immediately followed by an address, article, aside,
+            # blockquote, datagrid, dialog, dir, div, dl, fieldset,
+            # footer, form, h1, h2, h3, h4, h5, h6, header, hr, menu,
+            # nav, ol, p, pre, section, table, or ul, element, or if
+            # there is no more content in the parent element.
+            if type in ("StartTag", "EmptyTag"):
+                return next["name"] in ('address', 'article', 'aside',
+                                        'blockquote', 'datagrid', 'dialog', 
+                                        'dir', 'div', 'dl', 'fieldset', 'footer',
+                                        'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+                                        'header', 'hr', 'menu', 'nav', 'ol', 
+                                        'p', 'pre', 'section', 'table', 'ul')
+            else:
+                return type == "EndTag" or type is None
+        elif tagname == 'option':
+            # An option element's end tag may be omitted if the option
+            # element is immediately followed by another option element,
+            # or if it is immediately followed by an <code>optgroup</code>
+            # element, or if there is no more content in the parent
+            # element.
+            if type == "StartTag":
+                return next["name"] in ('option', 'optgroup')
+            else:
+                return type == "EndTag" or type is None
+        elif tagname in ('rt', 'rp'):
+            # An rt element's end tag may be omitted if the rt element is
+            # immediately followed by an rt or rp element, or if there is
+            # no more content in the parent element.
+            # An rp element's end tag may be omitted if the rp element is
+            # immediately followed by an rt or rp element, or if there is
+            # no more content in the parent element.
+            if type == "StartTag":
+                return next["name"] in ('rt', 'rp')
+            else:
+                return type == "EndTag" or type is None
+        elif tagname == 'colgroup':
+            # A colgroup element's end tag may be omitted if the colgroup
+            # element is not immediately followed by a space character or
+            # a comment.
+            if type in ("Comment", "SpaceCharacters"):
+                return False
+            elif type == "StartTag":
+                # XXX: we also look for an immediately following colgroup
+                # element. See is_optional_start.
+                return next["name"] != 'colgroup'
+            else:
+                return True
+        elif tagname in ('thead', 'tbody'):
+            # A thead element's end tag may be omitted if the thead element
+            # is immediately followed by a tbody or tfoot element.
+            # A tbody element's end tag may be omitted if the tbody element
+            # is immediately followed by a tbody or tfoot element, or if
+            # there is no more content in the parent element.
+            # A tfoot element's end tag may be omitted if the tfoot element
+            # is immediately followed by a tbody element, or if there is no
+            # more content in the parent element.
+            # XXX: we never omit the end tag when the following element is
+            # a tbody. See is_optional_start.
+            if type == "StartTag":
+                return next["name"] in ['tbody', 'tfoot']
+            elif tagname == 'tbody':
+                return type == "EndTag" or type is None
+            else:
+                return False
+        elif tagname == 'tfoot':
+            # A tfoot element's end tag may be omitted if the tfoot element
+            # is immediately followed by a tbody element, or if there is no
+            # more content in the parent element.
+            # XXX: we never omit the end tag when the following element is
+            # a tbody. See is_optional_start.
+            if type == "StartTag":
+                return next["name"] == 'tbody'
+            else:
+                return type == "EndTag" or type is None
+        elif tagname in ('td', 'th'):
+            # A td element's end tag may be omitted if the td element is
+            # immediately followed by a td or th element, or if there is
+            # no more content in the parent element.
+            # A th element's end tag may be omitted if the th element is
+            # immediately followed by a td or th element, or if there is
+            # no more content in the parent element.
+            if type == "StartTag":
+                return next["name"] in ('td', 'th')
+            else:
+                return type == "EndTag" or type is None
+        return False

html5lib/filters/sanitizer.py

@@ -0,0 +1,8 @@
+import _base
+from html5lib.sanitizer import HTMLSanitizerMixin
+
+class Filter(_base.Filter, HTMLSanitizerMixin):
+    def __iter__(self):
+        for token in _base.Filter.__iter__(self):
+            token = self.sanitize_token(token)
+            if token: yield token

html5lib/filters/whitespace.py

@@ -0,0 +1,41 @@
+try:
+    frozenset
+except NameError:
+    # Import from the sets module for python 2.3
+    from sets import ImmutableSet as frozenset
+
+import re
+
+import _base
+from html5lib.constants import rcdataElements, spaceCharacters
+spaceCharacters = u"".join(spaceCharacters)
+
+SPACES_REGEX = re.compile(u"[%s]+" % spaceCharacters)
+
+class Filter(_base.Filter):
+
+    spacePreserveElements = frozenset(["pre", "textarea"] + list(rcdataElements))
+
+    def __iter__(self):
+        preserve = 0
+        for token in _base.Filter.__iter__(self):
+            type = token["type"]
+            if type == "StartTag" \
+              and (preserve or token["name"] in self.spacePreserveElements):
+                preserve += 1
+
+            elif type == "EndTag" and preserve:
+                preserve -= 1
+
+            elif not preserve and type == "SpaceCharacters" and token["data"]:
+                # Test on token["data"] above to not introduce spaces where there were not
+                token["data"] = u" "
+
+            elif not preserve and type == "Characters":
+                token["data"] = collapse_spaces(token["data"])
+
+            yield token
+
+def collapse_spaces(text):
+    return SPACES_REGEX.sub(' ', text)
+

html5lib/ihatexml.py

@@ -0,0 +1,177 @@
+import re
+
+baseChar = """[#x0041-#x005A] | [#x0061-#x007A] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x00FF] | [#x0100-#x0131] | [#x0134-#x013E] | [#x0141-#x0148] | [#x014A-#x017E] | [#x0180-#x01C3] | [#x01CD-#x01F0] | [#x01F4-#x01F5] | [#x01FA-#x0217] | [#x0250-#x02A8] | [#x02BB-#x02C1] | #x0386 | [#x0388-#x038A] | #x038C | [#x038E-#x03A1] | [#x03A3-#x03CE] | [#x03D0-#x03D6] | #x03DA | #x03DC | #x03DE | #x03E0 | [#x03E2-#x03F3] | [#x0401-#x040C] | [#x040E-#x044F] | [#x0451-#x045C] | [#x045E-#x0481] | [#x0490-#x04C4] | [#x04C7-#x04C8] | [#x04CB-#x04CC] | [#x04D0-#x04EB] | [#x04EE-#x04F5] | [#x04F8-#x04F9] | [#x0531-#x0556] | #x0559 | [#x0561-#x0586] | [#x05D0-#x05EA] | [#x05F0-#x05F2] | [#x0621-#x063A] | [#x0641-#x064A] | [#x0671-#x06B7] | [#x06BA-#x06BE] | [#x06C0-#x06CE] | [#x06D0-#x06D3] | #x06D5 | [#x06E5-#x06E6] | [#x0905-#x0939] | #x093D | [#x0958-#x0961] | [#x0985-#x098C] | [#x098F-#x0990] | [#x0993-#x09A8] | [#x09AA-#x09B0] | #x09B2 | [#x09B6-#x09B9] | [#x09DC-#x09DD] | [#x09DF-#x09E1] | [#x09F0-#x09F1] | [#x0A05-#x0A0A] | [#x0A0F-#x0A10] | [#x0A13-#x0A28] | [#x0A2A-#x0A30] | [#x0A32-#x0A33] | [#x0A35-#x0A36] | [#x0A38-#x0A39] | [#x0A59-#x0A5C] | #x0A5E | [#x0A72-#x0A74] | [#x0A85-#x0A8B] | #x0A8D | [#x0A8F-#x0A91] | [#x0A93-#x0AA8] | [#x0AAA-#x0AB0] | [#x0AB2-#x0AB3] | [#x0AB5-#x0AB9] | #x0ABD | #x0AE0 | [#x0B05-#x0B0C] | [#x0B0F-#x0B10] | [#x0B13-#x0B28] | [#x0B2A-#x0B30] | [#x0B32-#x0B33] | [#x0B36-#x0B39] | #x0B3D | [#x0B5C-#x0B5D] | [#x0B5F-#x0B61] | [#x0B85-#x0B8A] | [#x0B8E-#x0B90] | [#x0B92-#x0B95] | [#x0B99-#x0B9A] | #x0B9C | [#x0B9E-#x0B9F] | [#x0BA3-#x0BA4] | [#x0BA8-#x0BAA] | [#x0BAE-#x0BB5] | [#x0BB7-#x0BB9] | [#x0C05-#x0C0C] | [#x0C0E-#x0C10] | [#x0C12-#x0C28] | [#x0C2A-#x0C33] | [#x0C35-#x0C39] | [#x0C60-#x0C61] | [#x0C85-#x0C8C] | [#x0C8E-#x0C90] | [#x0C92-#x0CA8] | [#x0CAA-#x0CB3] | [#x0CB5-#x0CB9] | #x0CDE | [#x0CE0-#x0CE1] | [#x0D05-#x0D0C] | [#x0D0E-#x0D10] | [#x0D12-#x0D28] | [#x0D2A-#x0D39] | [#x0D60-#x0D61] | [#x0E01-#x0E2E] | #x0E30 | [#x0E32-#x0E33] | [#x0E40-#x0E45] | [#x0E81-#x0E82] | #x0E84 | [#x0E87-#x0E88] | #x0E8A | #x0E8D | [#x0E94-#x0E97] | [#x0E99-#x0E9F] | [#x0EA1-#x0EA3] | #x0EA5 | #x0EA7 | [#x0EAA-#x0EAB] | [#x0EAD-#x0EAE] | #x0EB0 | [#x0EB2-#x0EB3] | #x0EBD | [#x0EC0-#x0EC4] | [#x0F40-#x0F47] | [#x0F49-#x0F69] | [#x10A0-#x10C5] | [#x10D0-#x10F6] | #x1100 | [#x1102-#x1103] | [#x1105-#x1107] | #x1109 | [#x110B-#x110C] | [#x110E-#x1112] | #x113C | #x113E | #x1140 | #x114C | #x114E | #x1150 | [#x1154-#x1155] | #x1159 | [#x115F-#x1161] | #x1163 | #x1165 | #x1167 | #x1169 | [#x116D-#x116E] | [#x1172-#x1173] | #x1175 | #x119E | #x11A8 | #x11AB | [#x11AE-#x11AF] | [#x11B7-#x11B8] | #x11BA | [#x11BC-#x11C2] | #x11EB | #x11F0 | #x11F9 | [#x1E00-#x1E9B] | [#x1EA0-#x1EF9] | [#x1F00-#x1F15] | [#x1F18-#x1F1D] | [#x1F20-#x1F45] | [#x1F48-#x1F4D] | [#x1F50-#x1F57] | #x1F59 | #x1F5B | #x1F5D | [#x1F5F-#x1F7D] | [#x1F80-#x1FB4] | [#x1FB6-#x1FBC] | #x1FBE | [#x1FC2-#x1FC4] | [#x1FC6-#x1FCC] | [#x1FD0-#x1FD3] | [#x1FD6-#x1FDB] | [#x1FE0-#x1FEC] | [#x1FF2-#x1FF4] | [#x1FF6-#x1FFC] | #x2126 | [#x212A-#x212B] | #x212E | [#x2180-#x2182] | [#x3041-#x3094] | [#x30A1-#x30FA] | [#x3105-#x312C] | [#xAC00-#xD7A3]"""
+
+ideographic = """[#x4E00-#x9FA5] | #x3007 | [#x3021-#x3029]"""
+
+combiningCharacter = """[#x0300-#x0345] | [#x0360-#x0361] | [#x0483-#x0486] | [#x0591-#x05A1] | [#x05A3-#x05B9] | [#x05BB-#x05BD] | #x05BF | [#x05C1-#x05C2] | #x05C4 | [#x064B-#x0652] | #x0670 | [#x06D6-#x06DC] | [#x06DD-#x06DF] | [#x06E0-#x06E4] | [#x06E7-#x06E8] | [#x06EA-#x06ED] | [#x0901-#x0903] | #x093C | [#x093E-#x094C] | #x094D | [#x0951-#x0954] | [#x0962-#x0963] | [#x0981-#x0983] | #x09BC | #x09BE | #x09BF | [#x09C0-#x09C4] | [#x09C7-#x09C8] | [#x09CB-#x09CD] | #x09D7 | [#x09E2-#x09E3] | #x0A02 | #x0A3C | #x0A3E | #x0A3F | [#x0A40-#x0A42] | [#x0A47-#x0A48] | [#x0A4B-#x0A4D] | [#x0A70-#x0A71] | [#x0A81-#x0A83] | #x0ABC | [#x0ABE-#x0AC5] | [#x0AC7-#x0AC9] | [#x0ACB-#x0ACD] | [#x0B01-#x0B03] | #x0B3C | [#x0B3E-#x0B43] | [#x0B47-#x0B48] | [#x0B4B-#x0B4D] | [#x0B56-#x0B57] | [#x0B82-#x0B83] | [#x0BBE-#x0BC2] | [#x0BC6-#x0BC8] | [#x0BCA-#x0BCD] | #x0BD7 | [#x0C01-#x0C03] | [#x0C3E-#x0C44] | [#x0C46-#x0C48] | [#x0C4A-#x0C4D] | [#x0C55-#x0C56] | [#x0C82-#x0C83] | [#x0CBE-#x0CC4] | [#x0CC6-#x0CC8] | [#x0CCA-#x0CCD] | [#x0CD5-#x0CD6] | [#x0D02-#x0D03] | [#x0D3E-#x0D43] | [#x0D46-#x0D48] | [#x0D4A-#x0D4D] | #x0D57 | #x0E31 | [#x0E34-#x0E3A] | [#x0E47-#x0E4E] | #x0EB1 | [#x0EB4-#x0EB9] | [#x0EBB-#x0EBC] | [#x0EC8-#x0ECD] | [#x0F18-#x0F19] | #x0F35 | #x0F37 | #x0F39 | #x0F3E | #x0F3F | [#x0F71-#x0F84] | [#x0F86-#x0F8B] | [#x0F90-#x0F95] | #x0F97 | [#x0F99-#x0FAD] | [#x0FB1-#x0FB7] | #x0FB9 | [#x20D0-#x20DC] | #x20E1 | [#x302A-#x302F] | #x3099 | #x309A"""
+
+digit = """[#x0030-#x0039] | [#x0660-#x0669] | [#x06F0-#x06F9] | [#x0966-#x096F] | [#x09E6-#x09EF] | [#x0A66-#x0A6F] | [#x0AE6-#x0AEF] | [#x0B66-#x0B6F] | [#x0BE7-#x0BEF] | [#x0C66-#x0C6F] | [#x0CE6-#x0CEF] | [#x0D66-#x0D6F] | [#x0E50-#x0E59] | [#x0ED0-#x0ED9] | [#x0F20-#x0F29]"""
+
+extender = """#x00B7 | #x02D0 | #x02D1 | #x0387 | #x0640 | #x0E46 | #x0EC6 | #x3005 | [#x3031-#x3035] | [#x309D-#x309E] | [#x30FC-#x30FE]"""
+
+letter = " | ".join([baseChar, ideographic])
+
+#Without the 
+name = " | ".join([letter, digit, ".", "-", "_", combiningCharacter, 
+                       extender])
+nameFirst = " | ".join([letter, "_"])
+
+reChar = re.compile(r"#x([\d|A-F]{4,4})")
+reCharRange = re.compile(r"\[#x([\d|A-F]{4,4})-#x([\d|A-F]{4,4})\]")
+
+def charStringToList(chars):
+    charRanges = [item.strip() for item in chars.split(" | ")]
+    rv = []
+    for item in charRanges:
+        foundMatch = False
+        for regexp in (reChar, reCharRange):
+            match = regexp.match(item)
+            if match is not None:
+                rv.append([hexToInt(item) for item in match.groups()])
+                if len(rv[-1]) == 1:
+                    rv[-1] = rv[-1]*2
+                foundMatch = True
+                break
+        if not foundMatch:
+            assert len(item) == 1
+            
+            rv.append([ord(item)] * 2)
+    rv = normaliseCharList(rv)
+    return rv
+
+def normaliseCharList(charList):
+    charList = sorted(charList)
+    for item in charList:
+        assert item[1] >= item[0]
+    rv = []
+    i = 0
+    while i < len(charList):
+        j = 1
+        rv.append(charList[i])
+        while i + j < len(charList) and charList[i+j][0] <= rv[-1][1] + 1:
+            rv[-1][1] = charList[i+j][1]
+            j += 1
+        i += j
+    return rv
+
+#We don't really support characters above the BMP :(
+max_unicode = int("FFFF", 16)
+    
+def missingRanges(charList):
+    rv = []
+    if charList[0] != 0:
+        rv.append([0, charList[0][0] - 1])
+    for i, item in enumerate(charList[:-1]):
+        rv.append([item[1]+1, charList[i+1][0] - 1])
+    if charList[-1][1] != max_unicode:
+        rv.append([charList[-1][1] + 1, max_unicode])
+    return rv
+
+def listToRegexpStr(charList):
+    rv = []
+    for item in charList:
+        if item[0] == item[1]:
+           rv.append(escapeRegexp(unichr(item[0])))
+        else:
+            rv.append(escapeRegexp(unichr(item[0])) + "-" +
+                      escapeRegexp(unichr(item[1])))
+    return "[%s]"%"".join(rv)
+
+def hexToInt(hex_str):
+    return int(hex_str, 16)
+
+def escapeRegexp(string):
+    specialCharacters = (".", "^", "$", "*", "+", "?", "{", "}",
+                          "[", "]", "|", "(", ")", "-")
+    for char in specialCharacters:
+        string = string.replace(char, "\\" + char)
+        if char in string:
+            print string
+
+    return string
+
+#output from the above
+nonXmlNameBMPRegexp = re.compile(u'[\x00-,/:-@\\[-\\^`\\{-\xb6\xb8-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u02cf\u02d2-\u02ff\u0346-\u035f\u0362-\u0385\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482\u0487-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u0590\u05a2\u05ba\u05be\u05c0\u05c3\u05c5-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u063f\u0653-\u065f\u066a-\u066f\u06b8-\u06b9\u06bf\u06cf\u06d4\u06e9\u06ee-\u06ef\u06fa-\u0900\u0904\u093a-\u093b\u094e-\u0950\u0955-\u0957\u0964-\u0965\u0970-\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09bd\u09c5-\u09c6\u09c9-\u09ca\u09ce-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09f2-\u0a01\u0a03-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a58\u0a5d\u0a5f-\u0a65\u0a75-\u0a80\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0adf\u0ae1-\u0ae5\u0af0-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3b\u0b44-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b62-\u0b65\u0b70-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bd6\u0bd8-\u0be6\u0bf0-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3d\u0c45\u0c49\u0c4e-\u0c54\u0c57-\u0c5f\u0c62-\u0c65\u0c70-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbd\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce2-\u0ce5\u0cf0-\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d3d\u0d44-\u0d45\u0d49\u0d4e-\u0d56\u0d58-\u0d5f\u0d62-\u0d65\u0d70-\u0e00\u0e2f\u0e3b-\u0e3f\u0e4f\u0e5a-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0f17\u0f1a-\u0f1f\u0f2a-\u0f34\u0f36\u0f38\u0f3a-\u0f3d\u0f48\u0f6a-\u0f70\u0f85\u0f8c-\u0f8f\u0f96\u0f98\u0fae-\u0fb0\u0fb8\u0fba-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u20cf\u20dd-\u20e0\u20e2-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3004\u3006\u3008-\u3020\u3030\u3036-\u3040\u3095-\u3098\u309b-\u309c\u309f-\u30a0\u30fb\u30ff-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]')
+
+nonXmlNameFirstBMPRegexp = re.compile(u'[\x00-@\\[-\\^`\\{-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u0385\u0387\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u0640\u064b-\u0670\u06b8-\u06b9\u06bf\u06cf\u06d4\u06d6-\u06e4\u06e7-\u0904\u093a-\u093c\u093e-\u0957\u0962-\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09db\u09de\u09e2-\u09ef\u09f2-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a58\u0a5d\u0a5f-\u0a71\u0a75-\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abc\u0abe-\u0adf\u0ae1-\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3c\u0b3e-\u0b5b\u0b5e\u0b62-\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c5f\u0c62-\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cdd\u0cdf\u0ce2-\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d5f\u0d62-\u0e00\u0e2f\u0e31\u0e34-\u0e3f\u0e46-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eb1\u0eb4-\u0ebc\u0ebe-\u0ebf\u0ec5-\u0f3f\u0f48\u0f6a-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3006\u3008-\u3020\u302a-\u3040\u3095-\u30a0\u30fb-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]')
+
+class InfosetFilter(object):
+    replacementRegexp = re.compile(r"U[\dA-F]{5,5}")
+    def __init__(self, replaceChars = None,  
+                 dropXmlnsLocalName = False, 
+                 dropXmlnsAttrNs = False,
+                 preventDoubleDashComments = False,
+                 preventDashAtCommentEnd = False,
+                 replaceFormFeedCharacters = True):
+
+        self.dropXmlnsLocalName = dropXmlnsLocalName
+        self.dropXmlnsAttrNs = dropXmlnsAttrNs
+
+        self.preventDoubleDashComments = preventDoubleDashComments
+        self.preventDashAtCommentEnd = preventDashAtCommentEnd
+
+        self.replaceFormFeedCharacters = replaceFormFeedCharacters
+
+        self.replaceCache = {}
+
+    def coerceAttribute(self, name, namespace=None):
+        if self.dropXmlnsLocalName and name.startswith("xmlns:"):
+            #Need a datalosswarning here
+            return None
+        elif (self.dropXmlnsAttrNs and 
+              namespace == "http://www.w3.org/2000/xmlns/"):
+            return None
+        else:
+            return self.toXmlName(name)
+
+    def coerceElement(self, name, namespace=None):
+        return self.toXmlName(name)
+
+    def coerceComment(self, data):
+        if self.preventDoubleDashComments:
+            while "--" in data:
+                data = data.replace("--", "- -")
+        return data
+    
+    def coerceCharacters(self, data):
+        if self.replaceFormFeedCharacters:
+            data = data.replace("\x0C", " ")
+        #Other non-xml characters
+        return data
+
+    def toXmlName(self, name):
+        nameFirst = name[0]
+        nameRest = name[1:]
+        m = nonXmlNameFirstBMPRegexp.match(nameFirst)
+        if m:
+            nameFirstOutput = self.getReplacementCharacter(nameFirst)
+        else:
+            nameFirstOutput = nameFirst
+
+        nameRestOutput = nameRest
+        replaceChars = set(nonXmlNameBMPRegexp.findall(nameRest))
+        for char in replaceChars:
+            replacement = self.getReplacementCharacter(char)
+            nameRestOutput = nameRestOutput.replace(char, replacement)
+        return nameFirstOutput + nameRestOutput
+    
+    def getReplacementCharacter(self, char):
+        if char in self.replaceCache:
+            replacement = self.replaceCache[char]
+        else:
+            replacement = self.escapeChar(char)
+        return replacement
+
+    def fromXmlName(self, name):
+        for item in set(self.replacementRegexp.findall(name)):
+            name = name.replace(item, self.unescapeChar(item))
+        return name
+
+    def escapeChar(self, char):
+        replacement = "U" + hex(ord(char))[2:].upper().rjust(5, "0")
+        self.replaceCache[char] = replacement
+        return replacement
+
+    def unescapeChar(self, charcode):
+        return unichr(int(charcode[1:], 16))

html5lib/inputstream.py

@@ -0,0 +1,782 @@
+import codecs
+import re
+import types
+import sys
+
+from constants import EOF, spaceCharacters, asciiLetters, asciiUppercase
+from constants import encodings, ReparseException
+import utils
+
+#Non-unicode versions of constants for use in the pre-parser
+spaceCharactersBytes = frozenset([str(item) for item in spaceCharacters])
+asciiLettersBytes = frozenset([str(item) for item in asciiLetters])
+asciiUppercaseBytes = frozenset([str(item) for item in asciiUppercase])
+spacesAngleBrackets = spaceCharactersBytes | frozenset([">", "<"])
+
+invalid_unicode_re = re.compile(u"[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]")
+
+non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
+                                  0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
+                                  0x6FFFE, 0x6FFFF, 0x7FFFE, 0x7FFFF, 0x8FFFE,
+                                  0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
+                                  0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE,
+                                  0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF,
+                                  0x10FFFE, 0x10FFFF])
+
+ascii_punctuation_re = re.compile(ur"[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]")
+
+# Cache for charsUntil()
+charsUntilRegEx = {}
+        
+class BufferedStream:
+    """Buffering for streams that do not have buffering of their own
+
+    The buffer is implemented as a list of chunks on the assumption that 
+    joining many strings will be slow since it is O(n**2)
+    """
+    
+    def __init__(self, stream):
+        self.stream = stream
+        self.buffer = []
+        self.position = [-1,0] #chunk number, offset
+
+    def tell(self):
+        pos = 0
+        for chunk in self.buffer[:self.position[0]]:
+            pos += len(chunk)
+        pos += self.position[1]
+        return pos
+
+    def seek(self, pos):
+        assert pos < self._bufferedBytes()
+        offset = pos
+        i = 0
+        while len(self.buffer[i]) < offset:
+            offset -= pos
+            i += 1
+        self.position = [i, offset]
+
+    def read(self, bytes):
+        if not self.buffer:
+            return self._readStream(bytes)
+        elif (self.position[0] == len(self.buffer) and
+              self.position[1] == len(self.buffer[-1])):
+            return self._readStream(bytes)
+        else:
+            return self._readFromBuffer(bytes)
+    
+    def _bufferedBytes(self):
+        return sum([len(item) for item in self.buffer])
+
+    def _readStream(self, bytes):
+        data = self.stream.read(bytes)
+        self.buffer.append(data)
+        self.position[0] += 1
+        self.position[1] = len(data)
+        return data
+
+    def _readFromBuffer(self, bytes):
+        remainingBytes = bytes
+        rv = []
+        bufferIndex = self.position[0]
+        bufferOffset = self.position[1]
+        while bufferIndex < len(self.buffer) and remainingBytes != 0:
+            assert remainingBytes > 0
+            bufferedData = self.buffer[bufferIndex]
+            
+            if remainingBytes <= len(bufferedData) - bufferOffset:
+                bytesToRead = remainingBytes
+                self.position = [bufferIndex, bufferOffset + bytesToRead]
+            else:
+                bytesToRead = len(bufferedData) - bufferOffset
+                self.position = [bufferIndex, len(bufferedData)]
+                bufferIndex += 1
+            data = rv.append(bufferedData[bufferOffset: 
+                                          bufferOffset + bytesToRead])
+            remainingBytes -= bytesToRead
+
+            bufferOffset = 0
+
+        if remainingBytes:
+            rv.append(self._readStream(remainingBytes))
+        
+        return "".join(rv)
+        
+
+
+class HTMLInputStream:
+    """Provides a unicode stream of characters to the HTMLTokenizer.
+
+    This class takes care of character encoding and removing or replacing
+    incorrect byte-sequences and also provides column and line tracking.
+
+    """
+
+    _defaultChunkSize = 10240
+
+    def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
+        """Initialises the HTMLInputStream.
+
+        HTMLInputStream(source, [encoding]) -> Normalized stream from source
+        for use by html5lib.
+
+        source can be either a file-object, local filename or a string.
+
+        The optional encoding parameter must be a string that indicates
+        the encoding.  If specified, that encoding will be used,
+        regardless of any BOM or later declaration (such as in a meta
+        element)
+        
+        parseMeta - Look for a <meta> element containing encoding information
+
+        """
+
+        #Craziness
+        if len(u"\U0010FFFF") == 1:
+            self.reportCharacterErrors = self.characterErrorsUCS4
+            self.replaceCharactersRegexp = re.compile(u"[\uD800-\uDFFF]")
+        else:
+            self.reportCharacterErrors = self.characterErrorsUCS2
+            self.replaceCharactersRegexp = re.compile(u"([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
+
+        # List of where new lines occur
+        self.newLines = [0]
+
+        self.charEncoding = (codecName(encoding), "certain")
+
+        # Raw Stream - for unicode objects this will encode to utf-8 and set
+        #              self.charEncoding as appropriate
+        self.rawStream = self.openStream(source)
+
+        # Encoding Information
+        #Number of bytes to use when looking for a meta element with
+        #encoding information
+        self.numBytesMeta = 512
+        #Number of bytes to use when using detecting encoding using chardet
+        self.numBytesChardet = 100
+        #Encoding to use if no other information can be found
+        self.defaultEncoding = "windows-1252"
+        
+        #Detect encoding iff no explicit "transport level" encoding is supplied
+        if (self.charEncoding[0] is None):
+            self.charEncoding = self.detectEncoding(parseMeta, chardet)
+
+
+        self.reset()
+
+    def reset(self):
+        self.dataStream = codecs.getreader(self.charEncoding[0])(self.rawStream,
+                                                                 'replace')
+
+        self.chunk = u""
+        self.chunkSize = 0
+        self.chunkOffset = 0
+        self.errors = []
+
+        # number of (complete) lines in previous chunks
+        self.prevNumLines = 0
+        # number of columns in the last line of the previous chunk
+        self.prevNumCols = 0
+        
+        #Deal with CR LF and surrogates split over chunk boundaries
+        self._bufferedCharacter = None
+
+    def openStream(self, source):
+        """Produces a file object from source.
+
+        source can be either a file object, local filename or a string.
+
+        """
+        # Already a file object
+        if hasattr(source, 'read'):
+            stream = source
+        else:
+            # Otherwise treat source as a string and convert to a file object
+            if isinstance(source, unicode):
+                source = source.encode('utf-8')
+                self.charEncoding = ("utf-8", "certain")
+            try:
+                from io import BytesIO
+            except:
+                # 2to3 converts this line to: from io import StringIO  
+                from cStringIO import StringIO as BytesIO
+            stream = BytesIO(source)
+
+        if (not(hasattr(stream, "tell") and hasattr(stream, "seek")) or
+            stream is sys.stdin):
+            stream = BufferedStream(stream)
+
+        return stream
+
+    def detectEncoding(self, parseMeta=True, chardet=True):
+        #First look for a BOM
+        #This will also read past the BOM if present
+        encoding = self.detectBOM()
+        confidence = "certain"
+        #If there is no BOM need to look for meta elements with encoding 
+        #information
+        if encoding is None and parseMeta:
+            encoding = self.detectEncodingMeta()
+            confidence = "tentative"
+        #Guess with chardet, if avaliable
+        if encoding is None and chardet:
+            confidence = "tentative"
+            try:
+                from chardet.universaldetector import UniversalDetector
+                buffers = []
+                detector = UniversalDetector()
+                while not detector.done:
+                    buffer = self.rawStream.read(self.numBytesChardet)
+                    if not buffer:
+                        break
+                    buffers.append(buffer)
+                    detector.feed(buffer)
+                detector.close()
+                encoding = detector.result['encoding']
+                self.rawStream.seek(0)
+            except ImportError:
+                pass
+        # If all else fails use the default encoding
+        if encoding is None:
+            confidence="tentative"
+            encoding = self.defaultEncoding
+        
+        #Substitute for equivalent encodings:
+        encodingSub = {"iso-8859-1":"windows-1252"}
+
+        if encoding.lower() in encodingSub:
+            encoding = encodingSub[encoding.lower()]
+
+        return encoding, confidence
+
+    def changeEncoding(self, newEncoding):
+        newEncoding = codecName(newEncoding)
+        if newEncoding in ("utf-16", "utf-16-be", "utf-16-le"):
+            newEncoding = "utf-8"
+        if newEncoding is None:
+            return
+        elif newEncoding == self.charEncoding[0]:
+            self.charEncoding = (self.charEncoding[0], "certain")
+        else:
+            self.rawStream.seek(0)
+            self.reset()
+            self.charEncoding = (newEncoding, "certain")
+            raise ReparseException, "Encoding changed from %s to %s"%(self.charEncoding[0], newEncoding)
+            
+    def detectBOM(self):
+        """Attempts to detect at BOM at the start of the stream. If
+        an encoding can be determined from the BOM return the name of the
+        encoding otherwise return None"""
+        bomDict = {
+            codecs.BOM_UTF8: 'utf-8',
+            codecs.BOM_UTF16_LE: 'utf-16-le', codecs.BOM_UTF16_BE: 'utf-16-be',
+            codecs.BOM_UTF32_LE: 'utf-32-le', codecs.BOM_UTF32_BE: 'utf-32-be'
+        }
+
+        # Go to beginning of file and read in 4 bytes
+        string = self.rawStream.read(4)
+
+        # Try detecting the BOM using bytes from the string
+        encoding = bomDict.get(string[:3])         # UTF-8
+        seek = 3
+        if not encoding:
+            # Need to detect UTF-32 before UTF-16
+            encoding = bomDict.get(string)         # UTF-32
+            seek = 4
+            if not encoding:
+                encoding = bomDict.get(string[:2]) # UTF-16
+                seek = 2
+
+        # Set the read position past the BOM if one was found, otherwise
+        # set it to the start of the stream
+        self.rawStream.seek(encoding and seek or 0)
+
+        return encoding
+
+    def detectEncodingMeta(self):
+        """Report the encoding declared by the meta element
+        """
+        buffer = self.rawStream.read(self.numBytesMeta)
+        parser = EncodingParser(buffer)
+        self.rawStream.seek(0)
+        encoding = parser.getEncoding()
+        
+        if encoding in ("utf-16", "utf-16-be", "utf-16-le"):
+            encoding = "utf-8"
+
+        return encoding
+
+    def _position(self, offset):
+        chunk = self.chunk
+        nLines = chunk.count(u'\n', 0, offset)
+        positionLine = self.prevNumLines + nLines
+        lastLinePos = chunk.rfind(u'\n', 0, offset)
+        if lastLinePos == -1:
+            positionColumn = self.prevNumCols + offset
+        else:
+            positionColumn = offset - (lastLinePos + 1)
+        return (positionLine, positionColumn)
+
+    def position(self):
+        """Returns (line, col) of the current position in the stream."""
+        line, col = self._position(self.chunkOffset)
+        return (line+1, col)
+
+    def char(self):
+        """ Read one character from the stream or queue if available. Return
+            EOF when EOF is reached.
+        """
+        # Read a new chunk from the input stream if necessary
+        if self.chunkOffset >= self.chunkSize:
+            if not self.readChunk():
+                return EOF
+
+        chunkOffset = self.chunkOffset
+        char = self.chunk[chunkOffset]
+        self.chunkOffset = chunkOffset + 1
+
+        return char
+
+    def readChunk(self, chunkSize=None):
+        if chunkSize is None:
+            chunkSize = self._defaultChunkSize
+
+        self.prevNumLines, self.prevNumCols = self._position(self.chunkSize)
+
+        self.chunk = u""
+        self.chunkSize = 0
+        self.chunkOffset = 0
+
+        data = self.dataStream.read(chunkSize)
+        
+        #Deal with CR LF and surrogates broken across chunks
+        if self._bufferedCharacter:
+            data = self._bufferedCharacter + data
+            self._bufferedCharacter = None
+        elif not data:
+            # We have no more data, bye-bye stream
+            return False
+        
+        if len(data) > 1:
+            lastv = ord(data[-1])
+            if lastv == 0x0D or 0xD800 <= lastv <= 0xDBFF:
+                self._bufferedCharacter = data[-1]
+                data = data[:-1]
+        
+        self.reportCharacterErrors(data)
+        
+        # Replace invalid characters
+        # Note U+0000 is dealt with in the tokenizer
+        data = self.replaceCharactersRegexp.sub(u"\ufffd", data)
+                    
+        data = data.replace(u"\r\n", u"\n")
+        data = data.replace(u"\r", u"\n")
+
+        self.chunk = data
+        self.chunkSize = len(data)
+
+        return True
+
+    def characterErrorsUCS4(self, data):
+        for i in xrange(len(invalid_unicode_re.findall(data))):
+            self.errors.append("invalid-codepoint")
+
+    def characterErrorsUCS2(self, data):
+        #Someone picked the wrong compile option
+        #You lose
+        skip = False
+        import sys
+        for match in invalid_unicode_re.finditer(data):
+            if skip:
+                continue
+            codepoint = ord(match.group())
+            pos = match.start()
+            #Pretty sure there should be endianness issues here
+            if utils.isSurrogatePair(data[pos:pos+2]):
+                #We have a surrogate pair!
+                char_val = utils.surrogatePairToCodepoint(data[pos:pos+2])
+                if char_val in non_bmp_invalid_codepoints:
+                    self.errors.append("invalid-codepoint")
+                skip = True
+            elif (codepoint >= 0xD800 and codepoint <= 0xDFFF and
+                  pos == len(data) - 1):
+                self.errors.append("invalid-codepoint")
+            else:
+                skip = False
+                self.errors.append("invalid-codepoint")
+
+    def charsUntil(self, characters, opposite = False):
+        """ Returns a string of characters from the stream up to but not
+        including any character in 'characters' or EOF. 'characters' must be
+        a container that supports the 'in' method and iteration over its
+        characters.
+        """
+
+        # Use a cache of regexps to find the required characters
+        try:
+            chars = charsUntilRegEx[(characters, opposite)]
+        except KeyError:
+            if __debug__:
+                for c in characters: 
+                    assert(ord(c) < 128)
+            regex = u"".join([u"\\x%02x" % ord(c) for c in characters])
+            if not opposite:
+                regex = u"^%s" % regex
+            chars = charsUntilRegEx[(characters, opposite)] = re.compile(u"[%s]+" % regex)
+
+        rv = []
+
+        while True:
+            # Find the longest matching prefix
+            m = chars.match(self.chunk, self.chunkOffset)
+            if m is None:
+                # If nothing matched, and it wasn't because we ran out of chunk,
+                # then stop
+                if self.chunkOffset != self.chunkSize:
+                    break
+            else:
+                end = m.end()
+                # If not the whole chunk matched, return everything
+                # up to the part that didn't match
+                if end != self.chunkSize:
+                    rv.append(self.chunk[self.chunkOffset:end])
+                    self.chunkOffset = end
+                    break
+            # If the whole remainder of the chunk matched,
+            # use it all and read the next chunk
+            rv.append(self.chunk[self.chunkOffset:])
+            if not self.readChunk():
+                # Reached EOF
+                break
+
+        r = u"".join(rv)
+        return r
+
+    def unget(self, char):
+        # Only one character is allowed to be ungotten at once - it must
+        # be consumed again before any further call to unget
+        if char is not None:
+            if self.chunkOffset == 0:
+                # unget is called quite rarely, so it's a good idea to do
+                # more work here if it saves a bit of work in the frequently
+                # called char and charsUntil.
+                # So, just prepend the ungotten character onto the current
+                # chunk:
+                self.chunk = char + self.chunk
+                self.chunkSize += 1
+            else:
+                self.chunkOffset -= 1
+                assert self.chunk[self.chunkOffset] == char
+
+class EncodingBytes(str):
+    """String-like object with an associated position and various extra methods
+    If the position is ever greater than the string length then an exception is
+    raised"""
+    def __new__(self, value):
+        return str.__new__(self, value.lower())
+
+    def __init__(self, value):
+        self._position=-1
+    
+    def __iter__(self):
+        return self
+    
+    def next(self):
+        p = self._position = self._position + 1
+        if p >= len(self):
+            raise StopIteration
+        elif p < 0:
+            raise TypeError
+        return self[p]
+
+    def previous(self):
+        p = self._position
+        if p >= len(self):
+            raise StopIteration
+        elif p < 0:
+            raise TypeError
+        self._position = p = p - 1
+        return self[p]
+    
+    def setPosition(self, position):
+        if self._position >= len(self):
+            raise StopIteration
+        self._position = position
+    
+    def getPosition(self):
+        if self._position >= len(self):
+            raise StopIteration
+        if self._position >= 0:
+            return self._position
+        else:
+            return None
+    
+    position = property(getPosition, setPosition)
+
+    def getCurrentByte(self):
+        return self[self.position]
+    
+    currentByte = property(getCurrentByte)
+
+    def skip(self, chars=spaceCharactersBytes):
+        """Skip past a list of characters"""
+        p = self.position               # use property for the error-checking
+        while p < len(self):
+            c = self[p]
+            if c not in chars:
+                self._position = p
+                return c
+            p += 1
+        self._position = p
+        return None
+
+    def skipUntil(self, chars):
+        p = self.position
+        while p < len(self):
+            c = self[p]
+            if c in chars:
+                self._position = p
+                return c
+            p += 1
+        self._position = p
+        return None
+
+    def matchBytes(self, bytes):
+        """Look for a sequence of bytes at the start of a string. If the bytes 
+        are found return True and advance the position to the byte after the 
+        match. Otherwise return False and leave the position alone"""
+        p = self.position
+        data = self[p:p+len(bytes)]
+        rv = data.startswith(bytes)
+        if rv:
+            self.position += len(bytes)
+        return rv
+    
+    def jumpTo(self, bytes):
+        """Look for the next sequence of bytes matching a given sequence. If
+        a match is found advance the position to the last byte of the match"""
+        newPosition = self[self.position:].find(bytes)
+        if newPosition > -1:
+            # XXX: This is ugly, but I can't see a nicer way to fix this.
+            if self._position == -1:
+                self._position = 0
+            self._position += (newPosition + len(bytes)-1)
+            return True
+        else:
+            raise StopIteration
+
+class EncodingParser(object):
+    """Mini parser for detecting character encoding from meta elements"""
+
+    def __init__(self, data):
+        """string - the data to work on for encoding detection"""
+        self.data = EncodingBytes(data)
+        self.encoding = None
+
+    def getEncoding(self):
+        methodDispatch = (
+            ("<!--",self.handleComment),
+            ("<meta",self.handleMeta),
+            ("</",self.handlePossibleEndTag),
+            ("<!",self.handleOther),
+            ("<?",self.handleOther),
+            ("<",self.handlePossibleStartTag))
+        for byte in self.data:
+            keepParsing = True
+            for key, method in methodDispatch:
+                if self.data.matchBytes(key):
+                    try:
+                        keepParsing = method()    
+                        break
+                    except StopIteration:
+                        keepParsing=False
+                        break
+            if not keepParsing:
+                break
+        
+        return self.encoding
+
+    def handleComment(self):
+        """Skip over comments"""
+        return self.data.jumpTo("-->")
+
+    def handleMeta(self):
+        if self.data.currentByte not in spaceCharactersBytes:
+            #if we have <meta not followed by a space so just keep going
+            return True
+        #We have a valid meta element we want to search for attributes
+        while True:
+            #Try to find the next attribute after the current position
+            attr = self.getAttribute()
+            if attr is None:
+                return True
+            else:
+                if attr[0] == "charset":
+                    tentativeEncoding = attr[1]
+                    codec = codecName(tentativeEncoding)
+                    if codec is not None:
+                        self.encoding = codec
+                        return False
+                elif attr[0] == "content":
+                    contentParser = ContentAttrParser(EncodingBytes(attr[1]))
+                    tentativeEncoding = contentParser.parse()
+                    codec = codecName(tentativeEncoding)
+                    if codec is not None:
+                        self.encoding = codec
+                        return False
+
+    def handlePossibleStartTag(self):
+        return self.handlePossibleTag(False)
+
+    def handlePossibleEndTag(self):
+        self.data.next()
+        return self.handlePossibleTag(True)
+
+    def handlePossibleTag(self, endTag):
+        data = self.data
+        if data.currentByte not in asciiLettersBytes:
+            #If the next byte is not an ascii letter either ignore this
+            #fragment (possible start tag case) or treat it according to 
+            #handleOther
+            if endTag:
+                data.previous()
+                self.handleOther()
+            return True
+        
+        c = data.skipUntil(spacesAngleBrackets)
+        if c == "<":
+            #return to the first step in the overall "two step" algorithm
+            #reprocessing the < byte
+            data.previous()
+        else:
+            #Read all attributes
+            attr = self.getAttribute()
+            while attr is not None:
+                attr = self.getAttribute()
+        return True
+
+    def handleOther(self):
+        return self.data.jumpTo(">")
+
+    def getAttribute(self):
+        """Return a name,value pair for the next attribute in the stream, 
+        if one is found, or None"""
+        data = self.data
+        # Step 1 (skip chars)
+        c = data.skip(spaceCharactersBytes | frozenset("/"))
+        # Step 2
+        if c in (">", None):
+            return None
+        # Step 3
+        attrName = []
+        attrValue = []
+        #Step 4 attribute name
+        while True:
+            if c == "=" and attrName:   
+                break
+            elif c in spaceCharactersBytes:
+                #Step 6!
+                c = data.skip()
+                c = data.next()
+                break
+            elif c in ("/", ">"):
+                return "".join(attrName), ""
+            elif c in asciiUppercaseBytes:
+                attrName.append(c.lower())
+            elif c == None:
+                return None
+            else:
+                attrName.append(c)
+            #Step 5
+            c = data.next()
+        #Step 7
+        if c != "=":
+            data.previous()
+            return "".join(attrName), ""
+        #Step 8
+        data.next()
+        #Step 9
+        c = data.skip()
+        #Step 10
+        if c in ("'", '"'):
+            #10.1
+            quoteChar = c
+            while True:
+                #10.2
+                c = data.next()
+                #10.3
+                if c == quoteChar:
+                    data.next()
+                    return "".join(attrName), "".join(attrValue)
+                #10.4
+                elif c in asciiUppercaseBytes:
+                    attrValue.append(c.lower())
+                #10.5
+                else:
+                    attrValue.append(c)
+        elif c == ">":
+            return "".join(attrName), ""
+        elif c in asciiUppercaseBytes:
+            attrValue.append(c.lower())
+        elif c is None:
+            return None
+        else:
+            attrValue.append(c)
+        # Step 11
+        while True:
+            c = data.next()
+            if c in spacesAngleBrackets:
+                return "".join(attrName), "".join(attrValue)
+            elif c in asciiUppercaseBytes:
+                attrValue.append(c.lower())
+            elif c is None:
+                return None
+            else:
+                attrValue.append(c)
+
+
+class ContentAttrParser(object):
+    def __init__(self, data):
+        self.data = data
+    def parse(self):
+        try:
+            #Check if the attr name is charset 
+            #otherwise return
+            self.data.jumpTo("charset")
+            self.data.position += 1
+            self.data.skip()
+            if not self.data.currentByte == "=":
+                #If there is no = sign keep looking for attrs
+                return None
+            self.data.position += 1
+            self.data.skip()
+            #Look for an encoding between matching quote marks
+            if self.data.currentByte in ('"', "'"):
+                quoteMark = self.data.currentByte
+                self.data.position += 1
+                oldPosition = self.data.position
+                if self.data.jumpTo(quoteMark):
+                    return self.data[oldPosition:self.data.position]
+                else:
+                    return None
+            else:
+                #Unquoted value
+                oldPosition = self.data.position
+                try:
+                    self.data.skipUntil(spaceCharactersBytes)
+                    return self.data[oldPosition:self.data.position]
+                except StopIteration:
+                    #Return the whole remaining value
+                    return self.data[oldPosition:]
+        except StopIteration:
+            return None
+
+
+def codecName(encoding):
+    """Return the python codec name corresponding to an encoding or None if the
+    string doesn't correspond to a valid encoding."""
+    if (encoding is not None and type(encoding) in types.StringTypes):
+        canonicalName = ascii_punctuation_re.sub("", encoding).lower()
+        return encodings.get(canonicalName, None)
+    else:
+        return None

html5lib/sanitizer.py

@@ -0,0 +1,258 @@
+import re
+from xml.sax.saxutils import escape, unescape
+
+from tokenizer import HTMLTokenizer
+from constants import tokenTypes
+
+class HTMLSanitizerMixin(object):
+    """ sanitization of XHTML+MathML+SVG and of inline style attributes."""
+
+    acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area',
+        'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button',
+        'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup',
+        'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn',
+        'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset',
+        'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1',
+        'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins',
+        'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter',
+        'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option',
+        'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select',
+        'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong',
+        'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot',
+        'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video']
+      
+    mathml_elements = ['maction', 'math', 'merror', 'mfrac', 'mi',
+        'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom',
+        'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub',
+        'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder',
+        'munderover', 'none']
+      
+    svg_elements = ['a', 'animate', 'animateColor', 'animateMotion',
+        'animateTransform', 'clipPath', 'circle', 'defs', 'desc', 'ellipse',
+        'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern',
+        'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph',
+        'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect',
+        'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use']
+        
+    acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
+        'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis',
+        'background', 'balance', 'bgcolor', 'bgproperties', 'border',
+        'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding',
+        'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff',
+        'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color',
+        'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords',
+        'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default',
+        'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end',
+        'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers',
+        'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace',
+        'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing',
+        'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend',
+        'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method',
+        'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open',
+        'optimum', 'pattern', 'ping', 'point-size', 'prompt', 'pqg',
+        'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min',
+        'replace', 'required', 'rev', 'rightspacing', 'rows', 'rowspan',
+        'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src', 'start',
+        'step', 'style', 'summary', 'suppress', 'tabindex', 'target',
+        'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap',
+        'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml',
+        'width', 'wrap', 'xml:lang']
+
+    mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign',
+        'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth',
+        'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence',
+        'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace',
+        'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize',
+        'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines',
+        'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection',
+        'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show',
+        'xlink:type', 'xmlns', 'xmlns:xlink']
+  
+    svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic',
+        'arabic-form', 'ascent', 'attributeName', 'attributeType',
+        'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height',
+        'class', 'clip-path', 'color', 'color-rendering', 'content', 'cx',
+        'cy', 'd', 'dx', 'dy', 'descent', 'display', 'dur', 'end', 'fill',
+        'fill-opacity', 'fill-rule', 'font-family', 'font-size',
+        'font-stretch', 'font-style', 'font-variant', 'font-weight', 'from',
+        'fx', 'fy', 'g1', 'g2', 'glyph-name', 'gradientUnits', 'hanging',
+        'height', 'horiz-adv-x', 'horiz-origin-x', 'id', 'ideographic', 'k',
+        'keyPoints', 'keySplines', 'keyTimes', 'lang', 'marker-end',
+        'marker-mid', 'marker-start', 'markerHeight', 'markerUnits',
+        'markerWidth', 'mathematical', 'max', 'min', 'name', 'offset',
+        'opacity', 'orient', 'origin', 'overline-position',
+        'overline-thickness', 'panose-1', 'path', 'pathLength', 'points',
+        'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount',
+        'repeatDur', 'requiredExtensions', 'requiredFeatures', 'restart',
+        'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 'stop-color',
+        'stop-opacity', 'strikethrough-position', 'strikethrough-thickness',
+        'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap',
+        'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity',
+        'stroke-width', 'systemLanguage', 'target', 'text-anchor', 'to',
+        'transform', 'type', 'u1', 'u2', 'underline-position',
+        'underline-thickness', 'unicode', 'unicode-range', 'units-per-em',
+        'values', 'version', 'viewBox', 'visibility', 'width', 'widths', 'x',
+        'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole',
+        'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type',
+        'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y',
+        'y1', 'y2', 'zoomAndPan']
+
+    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc',
+        'xlink:href', 'xml:base']
+
+    svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill',
+        'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end',
+        'mask', 'stroke']
+
+    svg_allow_local_href = ['altGlyph', 'animate', 'animateColor',
+        'animateMotion', 'animateTransform', 'cursor', 'feImage', 'filter',
+        'linearGradient', 'pattern', 'radialGradient', 'textpath', 'tref',
+        'set', 'use']
+  
+    acceptable_css_properties = ['azimuth', 'background-color',
+        'border-bottom-color', 'border-collapse', 'border-color',
+        'border-left-color', 'border-right-color', 'border-top-color', 'clear',
+        'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font',
+        'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight',
+        'height', 'letter-spacing', 'line-height', 'overflow', 'pause',
+        'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness',
+        'speak', 'speak-header', 'speak-numeral', 'speak-punctuation',
+        'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent',
+        'unicode-bidi', 'vertical-align', 'voice-family', 'volume',
+        'white-space', 'width']
+  
+    acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue',
+        'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed',
+        'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left',
+        'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive',
+        'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top',
+        'transparent', 'underline', 'white', 'yellow']
+  
+    acceptable_svg_properties = [ 'fill', 'fill-opacity', 'fill-rule',
+        'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin',
+        'stroke-opacity']
+  
+    acceptable_protocols = [ 'ed2k', 'ftp', 'http', 'https', 'irc',
+        'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal',
+        'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag',
+        'ssh', 'sftp', 'rtsp', 'afs' ]
+  
+    # subclasses may define their own versions of these constants
+    allowed_elements = acceptable_elements + mathml_elements + svg_elements
+    allowed_attributes = acceptable_attributes + mathml_attributes + svg_attributes
+    allowed_css_properties = acceptable_css_properties
+    allowed_css_keywords = acceptable_css_keywords
+    allowed_svg_properties = acceptable_svg_properties
+    allowed_protocols = acceptable_protocols
+
+    # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and
+    # stripping out all # attributes not in ALLOWED_ATTRIBUTES. Style
+    # attributes are parsed, and a restricted set, # specified by
+    # ALLOWED_CSS_PROPERTIES and ALLOWED_CSS_KEYWORDS, are allowed through.
+    # attributes in ATTR_VAL_IS_URI are scanned, and only URI schemes specified
+    # in ALLOWED_PROTOCOLS are allowed.
+    #
+    #   sanitize_html('<script> do_nasty_stuff() </script>')
+    #    => &lt;script> do_nasty_stuff() &lt;/script>
+    #   sanitize_html('<a href="javascript: sucker();">Click here for $100</a>')
+    #    => <a>Click here for $100</a>
+    def sanitize_token(self, token):
+
+        # accommodate filters which use token_type differently
+        token_type = token["type"]
+        if token_type in tokenTypes.keys():
+          token_type = tokenTypes[token_type]
+
+        if token_type in (tokenTypes["StartTag"], tokenTypes["EndTag"], 
+                             tokenTypes["EmptyTag"]):
+            if token["name"] in self.allowed_elements:
+                if token.has_key("data"):
+                    attrs = dict([(name,val) for name,val in
+                                  token["data"][::-1] 
+                                  if name in self.allowed_attributes])
+                    for attr in self.attr_val_is_uri:
+                        if not attrs.has_key(attr):
+                            continue
+                        val_unescaped = re.sub("[`\000-\040\177-\240\s]+", '',
+                                               unescape(attrs[attr])).lower()
+                        #remove replacement characters from unescaped characters
+                        val_unescaped = val_unescaped.replace(u"\ufffd", "")
+                        if (re.match("^[a-z0-9][-+.a-z0-9]*:",val_unescaped) and
+                            (val_unescaped.split(':')[0] not in 
+                             self.allowed_protocols)):
+                            del attrs[attr]
+                    for attr in self.svg_attr_val_allows_ref:
+                        if attr in attrs:
+                            attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)',
+                                                 ' ',
+                                                 unescape(attrs[attr]))
+                    if (token["name"] in self.svg_allow_local_href and
+                        'xlink:href' in attrs and re.search('^\s*[^#\s].*',
+                                                            attrs['xlink:href'])):
+                        del attrs['xlink:href']
+                    if attrs.has_key('style'):
+                        attrs['style'] = self.sanitize_css(attrs['style'])
+                    token["data"] = [[name,val] for name,val in attrs.items()]
+                return token
+            else:
+                if token_type == tokenTypes["EndTag"]:
+                    token["data"] = "</%s>" % token["name"]
+                elif token["data"]:
+                    attrs = ''.join([' %s="%s"' % (k,escape(v)) for k,v in token["data"]])
+                    token["data"] = "<%s%s>" % (token["name"],attrs)
+                else:
+                    token["data"] = "<%s>" % token["name"]
+                if token.get("selfClosing"):
+                    token["data"]=token["data"][:-1] + "/>"
+
+                if token["type"] in tokenTypes.keys():
+                    token["type"] = "Characters"
+                else:
+                    token["type"] = tokenTypes["Characters"]
+
+                del token["name"]
+                return token
+        elif token_type == tokenTypes["Comment"]:
+            pass
+        else:
+            return token
+
+    def sanitize_css(self, style):
+        # disallow urls
+        style=re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ',style)
+
+        # gauntlet
+        if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): return ''
+        if not re.match("^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style): return ''
+
+        clean = []
+        for prop,value in re.findall("([-\w]+)\s*:\s*([^:;]*)",style):
+          if not value: continue
+          if prop.lower() in self.allowed_css_properties:
+              clean.append(prop + ': ' + value + ';')
+          elif prop.split('-')[0].lower() in ['background','border','margin',
+                                              'padding']:
+              for keyword in value.split():
+                  if not keyword in self.acceptable_css_keywords and \
+                      not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$",keyword):
+                      break
+              else:
+                  clean.append(prop + ': ' + value + ';')
+          elif prop.lower() in self.allowed_svg_properties:
+              clean.append(prop + ': ' + value + ';')
+
+        return ' '.join(clean)
+
+class HTMLSanitizer(HTMLTokenizer, HTMLSanitizerMixin):
+    def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True,
+                 lowercaseElementName=False, lowercaseAttrName=False, parser=None):
+        #Change case matching defaults as we only output lowercase html anyway
+        #This solution doesn't seem ideal...
+        HTMLTokenizer.__init__(self, stream, encoding, parseMeta, useChardet,
+                               lowercaseElementName, lowercaseAttrName, parser=parser)
+
+    def __iter__(self):
+        for token in HTMLTokenizer.__iter__(self):
+            token = self.sanitize_token(token)
+            if token:
+                yield token

html5lib/serializer/__init__.py

@@ -0,0 +1,17 @@
+
+from html5lib import treewalkers
+
+from htmlserializer import HTMLSerializer
+from xhtmlserializer import XHTMLSerializer
+
+def serialize(input, tree="simpletree", format="html", encoding=None,
+              **serializer_opts):
+    # XXX: Should we cache this?
+    walker = treewalkers.getTreeWalker(tree) 
+    if format == "html":
+        s = HTMLSerializer(**serializer_opts)
+    elif format == "xhtml":
+        s = XHTMLSerializer(**serializer_opts)
+    else:
+        raise ValueError, "type must be either html or xhtml"
+    return s.render(walker(input), encoding)

html5lib/serializer/htmlserializer.py

@@ -0,0 +1,312 @@
+try:
+    frozenset
+except NameError:
+    # Import from the sets module for python 2.3
+    from sets import ImmutableSet as frozenset
+
+import gettext
+_ = gettext.gettext
+
+from html5lib.constants import voidElements, booleanAttributes, spaceCharacters
+from html5lib.constants import rcdataElements, entities, xmlEntities
+from html5lib import utils
+from xml.sax.saxutils import escape
+
+spaceCharacters = u"".join(spaceCharacters)
+
+try:
+    from codecs import register_error, xmlcharrefreplace_errors
+except ImportError:
+    unicode_encode_errors = "strict"
+else:
+    unicode_encode_errors = "htmlentityreplace"
+
+    from html5lib.constants import entities
+
+    encode_entity_map = {}
+    is_ucs4 = len(u"\U0010FFFF") == 1
+    for k, v in entities.items():
+        #skip multi-character entities
+        if ((is_ucs4 and len(v) > 1) or
+            (not is_ucs4 and len(v) > 2)):
+            continue
+        if v != "&":
+            if len(v) == 2:
+                v = utils.surrogatePairToCodepoint(v)
+            else:
+                try:
+                    v = ord(v)
+                except:
+                    print v
+                    raise
+            if not v in encode_entity_map or k.islower():
+                # prefer < over < and similarly for &, >, etc.
+                encode_entity_map[v] = k
+
+    def htmlentityreplace_errors(exc):
+        if isinstance(exc, (UnicodeEncodeError, UnicodeTranslateError)):
+            res = []
+            codepoints = []
+            skip = False
+            for i, c in enumerate(exc.object[exc.start:exc.end]):
+                if skip:
+                    skip = False
+                    continue
+                index = i + exc.start
+                if utils.isSurrogatePair(exc.object[index:min([exc.end, index+2])]):
+                    codepoint = utils.surrogatePairToCodepoint(exc.object[index:index+2])
+                    skip = True
+                else:
+                    codepoint = ord(c)
+                codepoints.append(codepoint)
+            for cp in codepoints:
+                e = encode_entity_map.get(cp)
+                if e:
+                    res.append("&")
+                    res.append(e)
+                    if not e.endswith(";"):
+                        res.append(";")
+                else:
+                    res.append("&#x%s;"%(hex(cp)[2:]))
+            return (u"".join(res), exc.end)
+        else:
+            return xmlcharrefreplace_errors(exc)
+
+    register_error(unicode_encode_errors, htmlentityreplace_errors)
+
+    del register_error
+
+
+class HTMLSerializer(object):
+
+    # attribute quoting options
+    quote_attr_values = False
+    quote_char = u'"'
+    use_best_quote_char = True
+
+    # tag syntax options
+    omit_optional_tags = True
+    minimize_boolean_attributes = True
+    use_trailing_solidus = False
+    space_before_trailing_solidus = True
+
+    # escaping options
+    escape_lt_in_attrs = False
+    escape_rcdata = False
+    resolve_entities = True
+
+    # miscellaneous options
+    inject_meta_charset = True
+    strip_whitespace = False
+    sanitize = False
+
+    options = ("quote_attr_values", "quote_char", "use_best_quote_char",
+          "minimize_boolean_attributes", "use_trailing_solidus",
+          "space_before_trailing_solidus", "omit_optional_tags",
+          "strip_whitespace", "inject_meta_charset", "escape_lt_in_attrs",
+          "escape_rcdata", "resolve_entities", "sanitize")
+
+    def __init__(self, **kwargs):
+        """Initialize HTMLSerializer.
+
+        Keyword options (default given first unless specified) include:
+
+        inject_meta_charset=True|False
+          Whether it insert a meta element to define the character set of the
+          document.
+        quote_attr_values=True|False
+          Whether to quote attribute values that don't require quoting
+          per HTML5 parsing rules.
+        quote_char=u'"'|u"'"
+          Use given quote character for attribute quoting. Default is to
+          use double quote unless attribute value contains a double quote,
+          in which case single quotes are used instead.
+        escape_lt_in_attrs=False|True
+          Whether to escape < in attribute values.
+        escape_rcdata=False|True
+          Whether to escape characters that need to be escaped within normal
+          elements within rcdata elements such as style.
+        resolve_entities=True|False
+          Whether to resolve named character entities that appear in the
+          source tree. The XML predefined entities &lt; &gt; &amp; &quot; &apos;
+          are unaffected by this setting.
+        strip_whitespace=False|True
+          Whether to remove semantically meaningless whitespace. (This
+          compresses all whitespace to a single space except within pre.)
+        minimize_boolean_attributes=True|False
+          Shortens boolean attributes to give just the attribute value,
+          for example <input disabled="disabled"> becomes <input disabled>.
+        use_trailing_solidus=False|True
+          Includes a close-tag slash at the end of the start tag of void
+          elements (empty elements whose end tag is forbidden). E.g. <hr/>.
+        space_before_trailing_solidus=True|False
+          Places a space immediately before the closing slash in a tag
+          using a trailing solidus. E.g. <hr />. Requires use_trailing_solidus.
+        sanitize=False|True
+          Strip all unsafe or unknown constructs from output.
+          See `html5lib user documentation`_
+        omit_optional_tags=True|False
+          Omit start/end tags that are optional.
+
+        .. _html5lib user documentation: http://code.google.com/p/html5lib/wiki/UserDocumentation
+        """
+        if kwargs.has_key('quote_char'):
+            self.use_best_quote_char = False
+        for attr in self.options:
+            setattr(self, attr, kwargs.get(attr, getattr(self, attr)))
+        self.errors = []
+        self.strict = False
+
+    def encode(self, string):
+        assert(isinstance(string, unicode))
+        if self.encoding:
+            return string.encode(self.encoding, unicode_encode_errors)
+        else:
+            return string
+
+    def encodeStrict(self, string):
+        assert(isinstance(string, unicode))
+        if self.encoding:
+            return string.encode(self.encoding, "strict")
+        else:
+            return string
+
+    def serialize(self, treewalker, encoding=None):
+        self.encoding = encoding
+        in_cdata = False
+        self.errors = []
+        if encoding and self.inject_meta_charset:
+            from html5lib.filters.inject_meta_charset import Filter
+            treewalker = Filter(treewalker, encoding)
+        # XXX: WhitespaceFilter should be used before OptionalTagFilter
+        # for maximum efficiently of this latter filter
+        if self.strip_whitespace:
+            from html5lib.filters.whitespace import Filter
+            treewalker = Filter(treewalker)
+        if self.sanitize:
+            from html5lib.filters.sanitizer import Filter
+            treewalker = Filter(treewalker)
+        if self.omit_optional_tags:
+            from html5lib.filters.optionaltags import Filter
+            treewalker = Filter(treewalker)
+        for token in treewalker:
+            type = token["type"]
+            if type == "Doctype":
+                doctype = u"<!DOCTYPE %s" % token["name"]
+                
+                if token["publicId"]:
+                    doctype += u' PUBLIC "%s"' % token["publicId"]
+                elif token["systemId"]:
+                    doctype += u" SYSTEM"
+                if token["systemId"]:                
+                    if token["systemId"].find(u'"') >= 0:
+                        if token["systemId"].find(u"'") >= 0:
+                            self.serializeError(_("System identifer contains both single and double quote characters"))
+                        quote_char = u"'"
+                    else:
+                        quote_char = u'"'
+                    doctype += u" %s%s%s" % (quote_char, token["systemId"], quote_char)
+                
+                doctype += u">"
+                yield self.encodeStrict(doctype)
+
+            elif type in ("Characters", "SpaceCharacters"):
+                if type == "SpaceCharacters" or in_cdata:
+                    if in_cdata and token["data"].find("</") >= 0:
+                        self.serializeError(_("Unexpected </ in CDATA"))
+                    yield self.encode(token["data"])
+                else:
+                    yield self.encode(escape(token["data"]))
+
+            elif type in ("StartTag", "EmptyTag"):
+                name = token["name"]
+                yield self.encodeStrict(u"<%s" % name)
+                if name in rcdataElements and not self.escape_rcdata:
+                    in_cdata = True
+                elif in_cdata:
+                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                attributes = []
+                for (attr_namespace,attr_name),attr_value in sorted(token["data"].items()):
+                    #TODO: Add namespace support here
+                    k = attr_name
+                    v = attr_value
+                    yield self.encodeStrict(u' ')
+
+                    yield self.encodeStrict(k)
+                    if not self.minimize_boolean_attributes or \
+                      (k not in booleanAttributes.get(name, tuple()) \
+                      and k not in booleanAttributes.get("", tuple())):
+                        yield self.encodeStrict(u"=")
+                        if self.quote_attr_values or not v:
+                            quote_attr = True
+                        else:
+                            quote_attr = reduce(lambda x,y: x or (y in v),
+                                spaceCharacters + u">\"'=", False)
+                        v = v.replace(u"&", u"&amp;")
+                        if self.escape_lt_in_attrs: v = v.replace(u"<", u"&lt;")
+                        if quote_attr:
+                            quote_char = self.quote_char
+                            if self.use_best_quote_char:
+                                if u"'" in v and u'"' not in v:
+                                    quote_char = u'"'
+                                elif u'"' in v and u"'" not in v:
+                                    quote_char = u"'"
+                            if quote_char == u"'":
+                                v = v.replace(u"'", u"&#39;")
+                            else:
+                                v = v.replace(u'"', u"&quot;")
+                            yield self.encodeStrict(quote_char)
+                            yield self.encode(v)
+                            yield self.encodeStrict(quote_char)
+                        else:
+                            yield self.encode(v)
+                if name in voidElements and self.use_trailing_solidus:
+                    if self.space_before_trailing_solidus:
+                        yield self.encodeStrict(u" /")
+                    else:
+                        yield self.encodeStrict(u"/")
+                yield self.encode(u">")
+
+            elif type == "EndTag":
+                name = token["name"]
+                if name in rcdataElements:
+                    in_cdata = False
+                elif in_cdata:
+                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                yield self.encodeStrict(u"</%s>" % name)
+
+            elif type == "Comment":
+                data = token["data"]
+                if data.find("--") >= 0:
+                    self.serializeError(_("Comment contains --"))
+                yield self.encodeStrict(u"<!--%s-->" % token["data"])
+
+            elif type == "Entity":
+                name = token["name"]
+                key = name + ";"
+                if not key in entities:
+                    self.serializeError(_("Entity %s not recognized" % name))
+                if self.resolve_entities and key not in xmlEntities:
+                    data = entities[key]
+                else:
+                    data = u"&%s;" % name
+                yield self.encodeStrict(data)
+
+            else:
+                self.serializeError(token["data"])
+
+    def render(self, treewalker, encoding=None):
+        if encoding:
+            return "".join(list(self.serialize(treewalker, encoding)))
+        else:
+            return u"".join(list(self.serialize(treewalker)))
+
+    def serializeError(self, data="XXX ERROR MESSAGE NEEDED"):
+        # XXX The idea is to make data mandatory.
+        self.errors.append(data)
+        if self.strict:
+            raise SerializeError
+
+def SerializeError(Exception):
+    """Error in serialized tree"""
+    pass

html5lib/serializer/xhtmlserializer.py

@@ -0,0 +1,9 @@
+from htmlserializer import HTMLSerializer
+
+class XHTMLSerializer(HTMLSerializer):
+    quote_attr_values = True
+    minimize_boolean_attributes = False
+    use_trailing_solidus = True
+    escape_lt_in_attrs = True
+    omit_optional_tags = False
+    escape_rcdata = True

html5lib/tests/__init__.py

@@ -0,0 +1,12 @@
+import sys
+import os
+
+parent_path = os.path.abspath(os.path.join(os.path.split(__file__)[0], ".."))
+
+if not parent_path in sys.path:
+    sys.path.insert(0, parent_path)
+del parent_path
+
+from runtests import buildTestSuite
+
+import support

html5lib/tests/mockParser.py

@@ -0,0 +1,37 @@
+import sys
+import os
+
+if __name__ == '__main__':
+    #Allow us to import from the src directory
+    os.chdir(os.path.split(os.path.abspath(__file__))[0])
+    sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, "src")))
+
+from tokenizer import HTMLTokenizer
+
+class HTMLParser(object):
+    """ Fake parser to test tokenizer output """
+    def parse(self, stream, output=True):
+        tokenizer = HTMLTokenizer(stream)
+        for token in tokenizer:
+            if output:
+                print token
+
+if __name__ == "__main__":
+    x = HTMLParser()
+    if len(sys.argv) > 1:
+        if len(sys.argv) > 2:
+            import hotshot, hotshot.stats
+            prof = hotshot.Profile('stats.prof')
+            prof.runcall(x.parse, sys.argv[1], False)
+            prof.close()
+            stats = hotshot.stats.load('stats.prof')
+            stats.strip_dirs()
+            stats.sort_stats('time')
+            stats.print_stats()
+        else:
+            x.parse(sys.argv[1])
+    else:
+        print """Usage: python mockParser.py filename [stats]
+        If stats is specified the hotshots profiler will run and output the
+        stats instead.
+        """

html5lib/tests/runparsertests.py

@@ -0,0 +1,27 @@
+import sys
+import os
+import glob
+import unittest
+
+#Allow us to import the parent module
+os.chdir(os.path.split(os.path.abspath(__file__))[0])
+sys.path.insert(0, os.path.abspath(os.curdir))
+sys.path.insert(0, os.path.abspath(os.pardir))
+sys.path.insert(0, os.path.join(os.path.abspath(os.pardir), "src"))
+
+def buildTestSuite():
+    suite = unittest.TestSuite()
+    for testcase in glob.glob('test_*.py'):
+        if testcase in ("test_tokenizer.py", "test_parser.py", "test_parser2.py"):
+            module = os.path.splitext(testcase)[0]
+            suite.addTest(__import__(module).buildTestSuite())
+    return suite
+
+def main():
+    results = unittest.TextTestRunner().run(buildTestSuite())
+    return results
+
+if __name__ == "__main__":
+    results = main()
+    if not results.wasSuccessful():
+        sys.exit(1)

html5lib/tests/runtests.py

@@ -0,0 +1,20 @@
+import sys
+import os
+import glob
+import unittest
+
+def buildTestSuite():
+    suite = unittest.TestSuite()
+    for testcase in glob.glob('test_*.py'):
+        module = os.path.splitext(testcase)[0]
+        suite.addTest(__import__(module).buildTestSuite())
+    return suite
+
+def main():
+    results = unittest.TextTestRunner().run(buildTestSuite())
+    return results
+
+if __name__ == "__main__":
+    results = main()
+    if not results.wasSuccessful():
+        sys.exit(1)

html5lib/tests/support.py

@@ -0,0 +1,127 @@
+import os
+import sys
+import codecs
+import glob
+
+base_path = os.path.split(__file__)[0]
+
+if os.path.exists(os.path.join(base_path, 'testdata')):
+    #release
+    test_dir = os.path.join(base_path, 'testdata')
+else:
+    #development
+    test_dir = os.path.abspath(
+        os.path.join(base_path,
+                     os.path.pardir, os.path.pardir,
+                     os.path.pardir, 'testdata'))
+    assert os.path.exists(test_dir), "Test data not found"
+    #import the development html5lib
+    sys.path.insert(0, os.path.abspath(os.path.join(base_path, 
+                                                    os.path.pardir,
+                                                    os.path.pardir)))
+
+import html5lib
+from html5lib import html5parser, treebuilders
+del base_path
+
+#Build a dict of avaliable trees
+treeTypes = {"simpletree":treebuilders.getTreeBuilder("simpletree"),
+             "DOM":treebuilders.getTreeBuilder("dom")}
+
+#Try whatever etree implementations are avaliable from a list that are
+#"supposed" to work
+try:
+    import xml.etree.ElementTree as ElementTree
+    treeTypes['ElementTree'] = treebuilders.getTreeBuilder("etree", ElementTree, fullTree=True)
+except ImportError:
+    try:
+        import elementtree.ElementTree as ElementTree
+        treeTypes['ElementTree'] = treebuilders.getTreeBuilder("etree", ElementTree, fullTree=True)
+    except ImportError:
+        pass
+
+try:
+    import xml.etree.cElementTree as cElementTree
+    treeTypes['cElementTree'] = treebuilders.getTreeBuilder("etree", cElementTree, fullTree=True)
+except ImportError:
+    try:
+        import cElementTree
+        treeTypes['cElementTree'] = treebuilders.getTreeBuilder("etree", cElementTree, fullTree=True)
+    except ImportError:
+        pass
+    
+try:
+    import lxml.etree as lxml
+    treeTypes['lxml'] = treebuilders.getTreeBuilder("etree", lxml, fullTree=True)
+except ImportError:
+    pass
+
+try:
+    import BeautifulSoup
+    treeTypes["beautifulsoup"] = treebuilders.getTreeBuilder("beautifulsoup", fullTree=True)
+except ImportError:
+    pass
+
+def html5lib_test_files(subdirectory, files='*.dat'):
+    return glob.glob(os.path.join(test_dir,subdirectory,files))
+
+class DefaultDict(dict):
+    def __init__(self, default, *args, **kwargs):
+        self.default = default
+        dict.__init__(self, *args, **kwargs)
+    
+    def __getitem__(self, key):
+        return dict.get(self, key, self.default)
+
+class TestData(object):
+    def __init__(self, filename, newTestHeading="data"):
+        self.f = codecs.open(filename, encoding="utf8")
+        self.newTestHeading = newTestHeading
+    
+    def __iter__(self):
+        data = DefaultDict(None)
+        key=None
+        for line in self.f:
+            heading = self.isSectionHeading(line)
+            if heading:
+                if data and heading == self.newTestHeading:
+                    #Remove trailing newline
+                    data[key] = data[key][:-1]
+                    yield self.normaliseOutput(data)
+                    data = DefaultDict(None)
+                key = heading
+                data[key]=""
+            elif key is not None:
+                data[key] += line
+        if data:
+            yield self.normaliseOutput(data)
+        
+    def isSectionHeading(self, line):
+        """If the current heading is a test section heading return the heading,
+        otherwise return False"""
+        if line.startswith("#"):
+            return line[1:].strip()
+        else:
+            return False
+    
+    def normaliseOutput(self, data):
+        #Remove trailing newlines
+        for key,value in data.iteritems():
+            if value.endswith("\n"):
+                data[key] = value[:-1]
+        return data
+
+def convert(stripChars):
+    def convertData(data):
+        """convert the output of str(document) to the format used in the testcases"""
+        data = data.split("\n")
+        rv = []
+        for line in data:
+            if line.startswith("|"):
+                rv.append(line[stripChars:])
+            else:
+                rv.append(line)
+        return "\n".join(rv)
+    return convertData
+
+convertExpected = convert(2)

html5lib/tests/test_encoding.py

@@ -0,0 +1,54 @@
+import os
+import unittest
+from support import html5lib_test_files, TestData, test_dir
+
+from html5lib import HTMLParser, inputstream
+
+import re, unittest
+
+class Html5EncodingTestCase(unittest.TestCase):
+    def test_codec_name(self):
+        self.assertEquals(inputstream.codecName("utf-8"), "utf-8")
+        self.assertEquals(inputstream.codecName("utf8"), "utf-8")
+        self.assertEquals(inputstream.codecName("  utf8  "), "utf-8")
+        self.assertEquals(inputstream.codecName("ISO_8859--1"), "windows-1252")
+
+def buildTestSuite():
+    for filename in html5lib_test_files("encoding"):
+        test_name = os.path.basename(filename).replace('.dat',''). \
+            replace('-','')
+        tests = TestData(filename, "data")
+        for idx, test in enumerate(tests):
+            def encodingTest(self, data=test['data'], 
+                             encoding=test['encoding']):
+                p = HTMLParser()
+                t = p.parse(data, useChardet=False)
+                
+                errorMessage = ("Input:\n%s\nExpected:\n%s\nRecieved\n%s\n"%
+                                (data, repr(encoding.lower()), 
+                                 repr(p.tokenizer.stream.charEncoding)))
+                self.assertEquals(encoding.lower(),
+                                  p.tokenizer.stream.charEncoding[0], 
+                                  errorMessage)
+            setattr(Html5EncodingTestCase, 'test_%s_%d' % (test_name, idx+1),
+                encodingTest)
+
+    try:
+        import chardet
+        def test_chardet(self):
+            data = open(os.path.join(test_dir, "encoding" , "chardet", "test_big5.txt")).read()
+            encoding = inputstream.HTMLInputStream(data).charEncoding
+            assert encoding[0].lower() == "big5"
+        setattr(Html5EncodingTestCase, 'test_chardet', test_chardet)
+    except ImportError:
+        print "chardet not found, skipping chardet tests"
+        
+
+    return unittest.defaultTestLoader.loadTestsFromName(__name__)
+
+def main():
+    buildTestSuite()
+    unittest.main()
+
+if __name__ == "__main__":
+    main()

html5lib/tests/test_formfiller.py

@@ -0,0 +1,296 @@
+import sys
+import unittest
+
+from html5lib.filters.formfiller import SimpleFilter
+
+class FieldStorage(dict):
+    def getlist(self, name):
+        l = self[name]
+        if isinstance(l, list):
+            return l
+        elif isinstance(l, tuple) or hasattr(l, '__iter__'):
+            return list(l)
+        return [l]
+
+class TestCase(unittest.TestCase):
+    def runTest(self, input, formdata, expected):
+        try:
+            output = list(SimpleFilter(input, formdata))
+        except NotImplementedError, nie:
+            # Amnesty for those that confess...
+            print >>sys.stderr, "Not implemented:", str(nie)
+        else:
+            errorMsg = "\n".join(["\n\nInput:", str(input),
+                                  "\nForm data:", str(formdata),
+                                  "\nExpected:", str(expected),
+                                  "\nReceived:", str(output)])
+            self.assertEquals(output, expected, errorMsg)
+
+    def testSingleTextInputWithValue(self):
+        self.runTest(
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"text"), (u"name", u"foo"), (u"value", u"quux")]}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"text"), (u"name", u"foo"), (u"value", u"bar")]}])
+
+    def testSingleTextInputWithoutValue(self):
+        self.runTest(
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"text"), (u"name", u"foo")]}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"text"), (u"name", u"foo"), (u"value", u"bar")]}])
+
+    def testSingleCheckbox(self):
+        self.runTest(
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"checkbox"), (u"name", u"foo"), (u"value", u"bar")]}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"checkbox"), (u"name", u"foo"), (u"value", u"bar"), (u"checked", u"")]}])
+
+    def testSingleCheckboxShouldBeUnchecked(self):
+        self.runTest(
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"checkbox"), (u"name", u"foo"), (u"value", u"quux")]}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"checkbox"), (u"name", u"foo"), (u"value", u"quux")]}])
+
+    def testSingleCheckboxCheckedByDefault(self):
+        self.runTest(
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"checkbox"), (u"name", u"foo"), (u"value", u"bar"), (u"checked", u"")]}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"checkbox"), (u"name", u"foo"), (u"value", u"bar"), (u"checked", u"")]}])
+
+    def testSingleCheckboxCheckedByDefaultShouldBeUnchecked(self):
+        self.runTest(
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"checkbox"), (u"name", u"foo"), (u"value", u"quux"), (u"checked", u"")]}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"EmptyTag", "name": u"input",
+                "data": [(u"type", u"checkbox"), (u"name", u"foo"), (u"value", u"quux")]}])
+
+    def testSingleTextareaWithValue(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"textarea", "data": [(u"name", u"foo")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"textarea", "data": []}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"StartTag", "name": u"textarea", "data": [(u"name", u"foo")]},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"textarea", "data": []}])
+
+    def testSingleTextareaWithoutValue(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"textarea", "data": [(u"name", u"foo")]},
+             {"type": u"EndTag", "name": u"textarea", "data": []}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"StartTag", "name": u"textarea", "data": [(u"name", u"foo")]},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"textarea", "data": []}])
+
+    def testSingleSelectWithValue(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar"), (u"selected", u"")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+    def testSingleSelectWithValueShouldBeUnselected(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": "quux"}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+    def testSingleSelectWithoutValue(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"selected", u"")]},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+    def testSingleSelectWithoutValueShouldBeUnselected(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": "quux"}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+    def testSingleSelectTwoOptionsWithValue(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"quux")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar"), (u"selected", u"")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"quux")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+    def testSingleSelectTwoOptionsWithValueShouldBeUnselected(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"baz")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": "quux"}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"baz")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+    def testSingleSelectTwoOptionsWithoutValue(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": "bar"}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"selected", u"")]},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+    def testSingleSelectTwoOptionsWithoutValueShouldBeUnselected(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"baz"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": "quux"}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"bar"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": []},
+             {"type": u"Characters", "data": u"baz"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+    def testSingleSelectMultiple(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo"), (u"multiple", u"")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"quux")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": ["bar", "quux"]}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo"), (u"multiple", u"")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar"), (u"selected", u"")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"quux"), (u"selected", u"")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+    def testTwoSelect(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"quux")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []},
+             {"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"quux")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}],
+            FieldStorage({"foo": ["bar", "quux"]}),
+            [{"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar"), (u"selected", u"")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"quux")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []},
+             {"type": u"StartTag", "name": u"select", "data": [(u"name", u"foo")]},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"bar")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"StartTag", "name": u"option", "data": [(u"value", u"quux"), (u"selected", u"")]},
+             {"type": u"Characters", "data": u"quux"},
+             {"type": u"EndTag", "name": u"option", "data": []},
+             {"type": u"EndTag", "name": u"select", "data": []}])
+
+def buildTestSuite():
+    return unittest.defaultTestLoader.loadTestsFromName(__name__)
+
+def main():
+    buildTestSuite()
+    unittest.main()
+
+if __name__ == "__main__":
+    main()

html5lib/tests/test_parser.py

@@ -0,0 +1,140 @@
+import os
+import sys
+import traceback
+import StringIO
+import warnings
+import re
+
+warnings.simplefilter("error")
+
+from support import html5lib_test_files as data_files
+from support import TestData, convert, convertExpected
+import html5lib
+from html5lib import html5parser, treebuilders, constants
+
+treeTypes = {"simpletree":treebuilders.getTreeBuilder("simpletree"),
+             "DOM":treebuilders.getTreeBuilder("dom")}
+
+#Try whatever etree implementations are avaliable from a list that are
+#"supposed" to work
+try:
+    import xml.etree.ElementTree as ElementTree
+    treeTypes['ElementTree'] = treebuilders.getTreeBuilder("etree", ElementTree, fullTree=True)
+except ImportError:
+    try:
+        import elementtree.ElementTree as ElementTree
+        treeTypes['ElementTree'] = treebuilders.getTreeBuilder("etree", ElementTree, fullTree=True)
+    except ImportError:
+        pass
+
+try:
+    import xml.etree.cElementTree as cElementTree
+    treeTypes['cElementTree'] = treebuilders.getTreeBuilder("etree", cElementTree, fullTree=True)
+except ImportError:
+    try:
+        import cElementTree
+        treeTypes['cElementTree'] = treebuilders.getTreeBuilder("etree", cElementTree, fullTree=True)
+    except ImportError:
+        pass
+    
+try:
+    try:
+        import lxml.html as lxml
+    except ImportError:
+        import lxml.etree as lxml
+    treeTypes['lxml'] = treebuilders.getTreeBuilder("lxml", lxml, fullTree=True)
+except ImportError:
+    pass
+
+try:
+    import BeautifulSoup
+    treeTypes["beautifulsoup"] = treebuilders.getTreeBuilder("beautifulsoup", fullTree=True)
+except ImportError:
+    pass
+
+#Try whatever dom implementations are avaliable from a list that are
+#"supposed" to work
+try:
+    import pxdom
+    treeTypes["pxdom"] = treebuilders.getTreeBuilder("dom", pxdom)
+except ImportError:
+    pass
+
+#Run the parse error checks
+checkParseErrors = False
+
+#XXX - There should just be one function here but for some reason the testcase
+#format differs from the treedump format by a single space character
+def convertTreeDump(data):
+    return "\n".join(convert(3)(data).split("\n")[1:])
+
+namespaceExpected = re.compile(r"^(\s*)<(\S+)>", re.M).sub
+
+
+def runParserTest(innerHTML, input, expected, errors, treeClass,
+                  namespaceHTMLElements):
+    #XXX - move this out into the setup function
+    #concatenate all consecutive character tokens into a single token
+    try:
+        p = html5parser.HTMLParser(tree = treeClass,
+                                   namespaceHTMLElements=namespaceHTMLElements)
+    except constants.DataLossWarning:
+        return
+
+    try:
+        if innerHTML:
+            document = p.parseFragment(input, innerHTML)
+        else:
+            try:
+                document = p.parse(input)
+            except constants.DataLossWarning:
+                return 
+    except:
+        errorMsg = u"\n".join([u"\n\nInput:", input, u"\nExpected:", expected,
+                               u"\nTraceback:", traceback.format_exc()])
+        assert False, errorMsg.encode("utf8")
+
+    output = convertTreeDump(p.tree.testSerializer(document))
+
+    expected = convertExpected(expected)
+    if namespaceHTMLElements:
+        expected = namespaceExpected(r"\1<html \2>", expected)
+
+    errorMsg = u"\n".join([u"\n\nInput:", input, u"\nExpected:", expected,
+                           u"\nReceived:", output])
+    assert expected == output, errorMsg.encode("utf8")
+    errStr = [u"Line: %i Col: %i %s"%(line, col, 
+                                      constants.E[errorcode] % datavars if isinstance(datavars, dict) else (datavars,)) for
+              ((line,col), errorcode, datavars) in p.errors]
+
+    errorMsg2 = u"\n".join([u"\n\nInput:", input,
+                            u"\nExpected errors (" + str(len(errors)) + u"):\n" + u"\n".join(errors),
+                            u"\nActual errors (" + str(len(p.errors)) + u"):\n" + u"\n".join(errStr)])
+    if checkParseErrors:
+            assert len(p.errors) == len(errors), errorMsg2.encode("utf-8")
+
+def test_parser():
+    sys.stderr.write('Testing tree builders '+ " ".join(treeTypes.keys()) + "\n")
+    files = data_files('tree-construction')
+    
+    for filename in files:
+        testName = os.path.basename(filename).replace(".dat","")
+
+        tests = TestData(filename, "data")
+        
+        for index, test in enumerate(tests):
+            input, errors, innerHTML, expected = [test[key] for key in
+                                                      'data', 'errors',
+                                                      'document-fragment',
+                                                      'document']
+            if errors:
+                errors = errors.split("\n")
+
+            for treeName, treeCls in treeTypes.iteritems():
+                for namespaceHTMLElements in (True, False):
+                    print input
+                    yield (runParserTest, innerHTML, input, expected, errors, treeCls,
+                           namespaceHTMLElements)
+                    break
+                
+                

html5lib/tests/test_parser2.py

@@ -0,0 +1,39 @@
+import support
+from html5lib import html5parser
+from html5lib.constants import namespaces
+from html5lib.treebuilders import dom
+
+import unittest
+
+# tests that aren't autogenerated from text files
+class MoreParserTests(unittest.TestCase):
+
+  def test_assertDoctypeCloneable(self):
+    parser = html5parser.HTMLParser(tree=dom.TreeBuilder)
+    doc = parser.parse('<!DOCTYPE HTML>')
+    self.assert_(doc.cloneNode(True))
+
+  def test_line_counter(self):
+    # http://groups.google.com/group/html5lib-discuss/browse_frm/thread/f4f00e4a2f26d5c0
+    parser = html5parser.HTMLParser(tree=dom.TreeBuilder)
+    parser.parse("<pre>\nx\n&gt;\n</pre>")
+
+  def test_namespace_html_elements_0(self): 
+    parser = html5parser.HTMLParser(namespaceHTMLElements=True)
+    doc = parser.parse("<html></html>")
+    self.assert_(doc.childNodes[0].namespace == namespaces["html"])
+
+  def test_namespace_html_elements_1(self): 
+    parser = html5parser.HTMLParser(namespaceHTMLElements=False)
+    doc = parser.parse("<html></html>")
+    self.assert_(doc.childNodes[0].namespace == None)
+
+def buildTestSuite():
+  return unittest.defaultTestLoader.loadTestsFromName(__name__)
+
+def main():
+    buildTestSuite()
+    unittest.main()
+
+if __name__ == '__main__':
+    main()

html5lib/tests/test_sanitizer.py

@@ -0,0 +1,76 @@
+import os
+import sys
+import unittest
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+from html5lib import html5parser, sanitizer, constants
+
+def runSanitizerTest(name, expected, input):
+    expected = ''.join([token.toxml() for token in html5parser.HTMLParser().
+                         parseFragment(expected).childNodes])
+    expected = json.loads(json.dumps(expected))
+    assert expected == sanitize_html(input)
+
+def sanitize_html(stream):
+    return ''.join([token.toxml() for token in
+                    html5parser.HTMLParser(tokenizer=sanitizer.HTMLSanitizer).
+                     parseFragment(stream).childNodes])
+
+def test_should_handle_astral_plane_characters():
+    assert u"<p>\U0001d4b5 \U0001d538</p>" == sanitize_html("<p>&#x1d4b5; &#x1d538;</p>")
+
+def test_sanitizer():
+    for tag_name in sanitizer.HTMLSanitizer.allowed_elements:
+        if tag_name in ['caption', 'col', 'colgroup', 'optgroup', 'option', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', 'tr']:
+            continue ### TODO
+        if tag_name != tag_name.lower():
+            continue ### TODO
+        if tag_name == 'image':
+            yield (runSanitizerTest, "test_should_allow_%s_tag" % tag_name,
+              "<img title=\"1\"/>foo &lt;bad&gt;bar&lt;/bad&gt; baz",
+              "<%s title='1'>foo <bad>bar</bad> baz</%s>" % (tag_name,tag_name))
+        elif tag_name == 'br':
+            yield (runSanitizerTest, "test_should_allow_%s_tag" % tag_name,
+              "<br title=\"1\"/>foo &lt;bad&gt;bar&lt;/bad&gt; baz<br/>",
+              "<%s title='1'>foo <bad>bar</bad> baz</%s>" % (tag_name,tag_name))
+        elif tag_name in constants.voidElements:
+            yield (runSanitizerTest, "test_should_allow_%s_tag" % tag_name,
+              "<%s title=\"1\"/>foo &lt;bad&gt;bar&lt;/bad&gt; baz" % tag_name,
+              "<%s title='1'>foo <bad>bar</bad> baz</%s>" % (tag_name,tag_name))
+        else:
+            yield (runSanitizerTest, "test_should_allow_%s_tag" % tag_name,
+              "<%s title=\"1\">foo &lt;bad&gt;bar&lt;/bad&gt; baz</%s>" % (tag_name,tag_name),
+              "<%s title='1'>foo <bad>bar</bad> baz</%s>" % (tag_name,tag_name))
+
+    for tag_name in sanitizer.HTMLSanitizer.allowed_elements:
+        tag_name = tag_name.upper()
+        yield (runSanitizerTest, "test_should_forbid_%s_tag" % tag_name,
+          "&lt;%s title=\"1\"&gt;foo &lt;bad&gt;bar&lt;/bad&gt; baz&lt;/%s&gt;" % (tag_name,tag_name),
+          "<%s title='1'>foo <bad>bar</bad> baz</%s>" % (tag_name,tag_name))
+
+    for attribute_name in sanitizer.HTMLSanitizer.allowed_attributes:
+        if attribute_name != attribute_name.lower(): continue ### TODO
+        if attribute_name == 'style': continue
+        yield (runSanitizerTest, "test_should_allow_%s_attribute" % attribute_name,
+          "<p %s=\"foo\">foo &lt;bad&gt;bar&lt;/bad&gt; baz</p>" % attribute_name,
+          "<p %s='foo'>foo <bad>bar</bad> baz</p>" % attribute_name)
+
+    for attribute_name in sanitizer.HTMLSanitizer.allowed_attributes:
+        attribute_name = attribute_name.upper()
+        yield (runSanitizerTest, "test_should_forbid_%s_attribute" % attribute_name,
+          "<p>foo &lt;bad&gt;bar&lt;/bad&gt; baz</p>",
+          "<p %s='display: none;'>foo <bad>bar</bad> baz</p>" % attribute_name)
+
+    for protocol in sanitizer.HTMLSanitizer.allowed_protocols:
+        yield (runSanitizerTest, "test_should_allow_%s_uris" % protocol,
+          "<a href=\"%s\">foo</a>" % protocol,
+          """<a href="%s">foo</a>""" % protocol)
+
+    for protocol in sanitizer.HTMLSanitizer.allowed_protocols:
+        yield (runSanitizerTest, "test_should_allow_uppercase_%s_uris" % protocol,
+          "<a href=\"%s\">foo</a>" % protocol,
+        """<a href="%s">foo</a>""" % protocol)

html5lib/tests/test_serializer.py

@@ -0,0 +1,180 @@
+import os
+import unittest
+from support import html5lib_test_files
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+import html5lib
+from html5lib import html5parser, serializer, constants
+from html5lib.treewalkers._base import TreeWalker
+
+optionals_loaded = []
+
+try:
+    from lxml import etree
+    optionals_loaded.append("lxml")
+except ImportError:
+    pass
+
+default_namespace = constants.namespaces["html"]
+
+class JsonWalker(TreeWalker):
+    def __iter__(self):
+        for token in self.tree:
+            type = token[0]
+            if type == "StartTag":
+                if len(token) == 4:
+                    namespace, name, attrib = token[1:4]
+                else:
+                    namespace = default_namespace
+                    name, attrib = token[1:3]
+                yield self.startTag(namespace, name, self._convertAttrib(attrib))
+            elif type == "EndTag":
+                if len(token) == 3:
+                    namespace, name = token[1:3]
+                else:
+                    namespace = default_namespace
+                    name = token[1]
+                yield self.endTag(namespace, name)
+            elif type == "EmptyTag":
+                if len(token) == 4:
+                    namespace, name, attrib = token[1:]
+                else:
+                    namespace = default_namespace
+                    name, attrib = token[1:]
+                for token in self.emptyTag(namespace, name, self._convertAttrib(attrib)):
+                    yield token
+            elif type == "Comment":
+                yield self.comment(token[1])
+            elif type in ("Characters", "SpaceCharacters"):
+                for token in self.text(token[1]):
+                    yield token
+            elif type == "Doctype":
+                if len(token) == 4:
+                    yield self.doctype(token[1], token[2], token[3])
+                elif len(token) == 3:
+                    yield self.doctype(token[1], token[2])
+                else:
+                    yield self.doctype(token[1])
+            else:
+                raise ValueError("Unknown token type: " + type)
+    
+    def _convertAttrib(self, attribs):
+        """html5lib tree-walkers use a dict of (namespace, name): value for
+        attributes, but JSON cannot represent this. Convert from the format
+        in the serializer tests (a list of dicts with "namespace", "name",
+        and "value" as keys) to html5lib's tree-walker format."""
+        attrs = {}
+        for attrib in attribs:
+            name = (attrib["namespace"], attrib["name"])
+            assert(name not in attrs)
+            attrs[name] = attrib["value"]
+        return attrs
+
+
+def serialize_html(input, options):
+    options = dict([(str(k),v) for k,v in options.iteritems()])
+    return serializer.HTMLSerializer(**options).render(JsonWalker(input),options.get("encoding",None))
+
+def serialize_xhtml(input, options):
+    options = dict([(str(k),v) for k,v in options.iteritems()])
+    return serializer.XHTMLSerializer(**options).render(JsonWalker(input),options.get("encoding",None))
+
+def make_test(input, expected, xhtml, options):
+    result = serialize_html(input, options)
+    if len(expected) == 1:
+        assert expected[0] == result, "Expected:\n%s\nActual:\n%s\nOptions\nxhtml:False\n%s"%(expected[0], result, str(options))
+    elif result not in expected:
+        assert False, "Expected: %s, Received: %s" % (expected, result)
+
+    if not xhtml:
+        return
+
+    result = serialize_xhtml(input, options)
+    if len(xhtml) == 1:
+        assert xhtml[0] == result, "Expected:\n%s\nActual:\n%s\nOptions\nxhtml:True\n%s"%(xhtml[0], result, str(options))
+    elif result not in xhtml:
+        assert False, "Expected: %s, Received: %s" % (xhtml, result)
+
+
+class EncodingTestCase(unittest.TestCase):
+    def throwsWithLatin1(self, input):
+        self.assertRaises(UnicodeEncodeError, serialize_html, input, {"encoding": "iso-8859-1"})
+
+    def testDoctypeName(self):
+        self.throwsWithLatin1([["Doctype", u"\u0101"]])
+
+    def testDoctypePublicId(self):
+        self.throwsWithLatin1([["Doctype", u"potato", u"\u0101"]])
+
+    def testDoctypeSystemId(self):
+        self.throwsWithLatin1([["Doctype", u"potato", u"potato", u"\u0101"]])
+
+    def testCdataCharacters(self):
+        self.assertEquals("<style>&amacr;", serialize_html([["StartTag", "http://www.w3.org/1999/xhtml", "style", {}],
+                                                            ["Characters", u"\u0101"]],
+                                                           {"encoding": "iso-8859-1"}))
+
+    def testCharacters(self):
+        self.assertEquals("&amacr;", serialize_html([["Characters", u"\u0101"]],
+                                                    {"encoding": "iso-8859-1"}))
+
+    def testStartTagName(self):
+        self.throwsWithLatin1([["StartTag", u"http://www.w3.org/1999/xhtml", u"\u0101", []]])
+
+    def testEmptyTagName(self):
+        self.throwsWithLatin1([["EmptyTag", u"http://www.w3.org/1999/xhtml", u"\u0101", []]])
+
+    def testAttributeName(self):
+        self.throwsWithLatin1([["StartTag", u"http://www.w3.org/1999/xhtml", u"span", [{"namespace": None, "name": u"\u0101", "value": u"potato"}]]])
+
+    def testAttributeValue(self):
+        self.assertEquals("<span potato=&amacr;>", serialize_html([["StartTag", u"http://www.w3.org/1999/xhtml", u"span",
+                                                                    [{"namespace": None, "name": u"potato", "value": u"\u0101"}]]],
+                                                                  {"encoding": "iso-8859-1"}))
+
+    def testEndTagName(self):
+        self.throwsWithLatin1([["EndTag", u"http://www.w3.org/1999/xhtml", u"\u0101"]])
+
+    def testComment(self):
+        self.throwsWithLatin1([["Comment", u"\u0101"]])
+
+
+if "lxml" in optionals_loaded:
+    class LxmlTestCase(unittest.TestCase):
+        def setUp(self):
+            self.parser = etree.XMLParser(resolve_entities=False)
+            self.treewalker = html5lib.getTreeWalker("lxml")
+            self.serializer = serializer.HTMLSerializer()
+
+        def testEntityReplacement(self):
+            doc = """<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&beta;</html>"""
+            tree = etree.fromstring(doc, parser = self.parser).getroottree()
+            result = serializer.serialize(tree, tree="lxml", omit_optional_tags=False)
+            self.assertEquals(u"""<!DOCTYPE html SYSTEM "about:legacy-compat"><html>\u03B2</html>""", result)
+
+        def testEntityXML(self):
+            doc = """<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&gt;</html>"""
+            tree = etree.fromstring(doc, parser = self.parser).getroottree()
+            result = serializer.serialize(tree, tree="lxml", omit_optional_tags=False)
+            self.assertEquals(u"""<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&gt;</html>""", result)
+
+        def testEntityNoResolve(self):
+            doc = """<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&beta;</html>"""
+            tree = etree.fromstring(doc, parser = self.parser).getroottree()
+            result = serializer.serialize(tree, tree="lxml", omit_optional_tags=False,
+                                          resolve_entities=False)
+            self.assertEquals(u"""<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&beta;</html>""", result)
+
+def test_serializer():
+    for filename in html5lib_test_files('serializer', '*.test'):
+        tests = json.load(file(filename))
+        test_name = os.path.basename(filename).replace('.test','')
+        for index, test in enumerate(tests['tests']):
+            xhtml = test.get("xhtml", test["expected"])
+            if test_name == 'optionaltags': 
+                xhtml = None
+            yield make_test, test["input"], test["expected"], xhtml, test.get("options", {})

html5lib/tests/test_stream.py

@@ -0,0 +1,97 @@
+import support
+import unittest, codecs
+
+from html5lib.inputstream import HTMLInputStream
+
+class HTMLInputStreamShortChunk(HTMLInputStream):
+    _defaultChunkSize = 2
+
+class HTMLInputStreamTest(unittest.TestCase):
+
+    def test_char_ascii(self):
+        stream = HTMLInputStream("'", encoding='ascii')
+        self.assertEquals(stream.charEncoding[0], 'ascii')
+        self.assertEquals(stream.char(), "'")
+
+    def test_char_null(self):
+        stream = HTMLInputStream("\x00")
+        self.assertEquals(stream.char(), u'\ufffd')
+
+    def test_char_utf8(self):
+        stream = HTMLInputStream(u'\u2018'.encode('utf-8'), encoding='utf-8')
+        self.assertEquals(stream.charEncoding[0], 'utf-8')
+        self.assertEquals(stream.char(), u'\u2018')
+
+    def test_char_win1252(self):
+        stream = HTMLInputStream(u"\xa9\xf1\u2019".encode('windows-1252'))
+        self.assertEquals(stream.charEncoding[0], 'windows-1252')
+        self.assertEquals(stream.char(), u"\xa9")
+        self.assertEquals(stream.char(), u"\xf1")
+        self.assertEquals(stream.char(), u"\u2019")
+
+    def test_bom(self):
+        stream = HTMLInputStream(codecs.BOM_UTF8 + "'")
+        self.assertEquals(stream.charEncoding[0], 'utf-8')
+        self.assertEquals(stream.char(), "'")
+
+    def test_utf_16(self):
+        stream = HTMLInputStream((' '*1025).encode('utf-16'))
+        self.assert_(stream.charEncoding[0] in ['utf-16-le', 'utf-16-be'], stream.charEncoding)
+        self.assertEquals(len(stream.charsUntil(' ', True)), 1025)
+
+    def test_newlines(self):
+        stream = HTMLInputStreamShortChunk(codecs.BOM_UTF8 + "a\nbb\r\nccc\rddddxe")
+        self.assertEquals(stream.position(), (1, 0))
+        self.assertEquals(stream.charsUntil('c'), u"a\nbb\n")
+        self.assertEquals(stream.position(), (3, 0))
+        self.assertEquals(stream.charsUntil('x'), u"ccc\ndddd")
+        self.assertEquals(stream.position(), (4, 4))
+        self.assertEquals(stream.charsUntil('e'), u"x")
+        self.assertEquals(stream.position(), (4, 5))
+
+    def test_newlines2(self):
+        size = HTMLInputStream._defaultChunkSize
+        stream = HTMLInputStream("\r" * size + "\n")
+        self.assertEquals(stream.charsUntil('x'), "\n" * size)
+
+    def test_position(self):
+        stream = HTMLInputStreamShortChunk(codecs.BOM_UTF8 + "a\nbb\nccc\nddde\nf\ngh")
+        self.assertEquals(stream.position(), (1, 0))
+        self.assertEquals(stream.charsUntil('c'), u"a\nbb\n")
+        self.assertEquals(stream.position(), (3, 0))
+        stream.unget(u"\n")
+        self.assertEquals(stream.position(), (2, 2))
+        self.assertEquals(stream.charsUntil('c'), u"\n")
+        self.assertEquals(stream.position(), (3, 0))
+        stream.unget(u"\n")
+        self.assertEquals(stream.position(), (2, 2))
+        self.assertEquals(stream.char(), u"\n")
+        self.assertEquals(stream.position(), (3, 0))
+        self.assertEquals(stream.charsUntil('e'), u"ccc\nddd")
+        self.assertEquals(stream.position(), (4, 3))
+        self.assertEquals(stream.charsUntil('h'), u"e\nf\ng")
+        self.assertEquals(stream.position(), (6, 1))
+
+    def test_position2(self):
+        stream = HTMLInputStreamShortChunk("abc\nd")
+        self.assertEquals(stream.position(), (1, 0))
+        self.assertEquals(stream.char(), u"a")
+        self.assertEquals(stream.position(), (1, 1))
+        self.assertEquals(stream.char(), u"b")
+        self.assertEquals(stream.position(), (1, 2))
+        self.assertEquals(stream.char(), u"c")
+        self.assertEquals(stream.position(), (1, 3))
+        self.assertEquals(stream.char(), u"\n")
+        self.assertEquals(stream.position(), (2, 0))
+        self.assertEquals(stream.char(), u"d")
+        self.assertEquals(stream.position(), (2, 1))
+
+def buildTestSuite():
+    return unittest.defaultTestLoader.loadTestsFromName(__name__)
+
+def main():
+    buildTestSuite()
+    unittest.main()
+
+if __name__ == '__main__':
+    main()

html5lib/tests/test_tokenizer.py

@@ -0,0 +1,193 @@
+import sys
+import os
+import unittest
+import cStringIO
+import warnings
+import re
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+from support import html5lib_test_files
+from html5lib.tokenizer import HTMLTokenizer
+from html5lib import constants
+
+class TokenizerTestParser(object):
+    def __init__(self, initialState, lastStartTag=None):
+        self.tokenizer = HTMLTokenizer
+        self._state = initialState
+        self._lastStartTag = lastStartTag
+
+    def parse(self, stream, encoding=None, innerHTML=False):
+        tokenizer = self.tokenizer(stream, encoding)
+        self.outputTokens = []
+
+        tokenizer.state = getattr(tokenizer, self._state)
+        if self._lastStartTag is not None:
+            tokenizer.currentToken = {"type": "startTag", 
+                                      "name":self._lastStartTag}
+
+        types = dict((v,k) for k,v in constants.tokenTypes.iteritems())
+        for token in tokenizer:
+            getattr(self, 'process%s' % types[token["type"]])(token)
+
+        return self.outputTokens
+
+    def processDoctype(self, token):
+        self.outputTokens.append([u"DOCTYPE", token["name"], token["publicId"],
+                                  token["systemId"], token["correct"]])
+
+    def processStartTag(self, token):
+        self.outputTokens.append([u"StartTag", token["name"], 
+                                  dict(token["data"][::-1]), token["selfClosing"]])
+
+    def processEmptyTag(self, token):
+        if token["name"] not in constants.voidElements:
+            self.outputTokens.append(u"ParseError")
+        self.outputTokens.append([u"StartTag", token["name"], dict(token["data"][::-1])])
+
+    def processEndTag(self, token):
+        self.outputTokens.append([u"EndTag", token["name"], 
+                                  token["selfClosing"]])
+
+    def processComment(self, token):
+        self.outputTokens.append([u"Comment", token["data"]])
+
+    def processSpaceCharacters(self, token):
+        self.outputTokens.append([u"Character", token["data"]])
+        self.processSpaceCharacters = self.processCharacters
+
+    def processCharacters(self, token):
+        self.outputTokens.append([u"Character", token["data"]])
+
+    def processEOF(self, token):
+        pass
+
+    def processParseError(self, token):
+        self.outputTokens.append([u"ParseError", token["data"]])
+
+def concatenateCharacterTokens(tokens):
+    outputTokens = []
+    for token in tokens:
+        if not "ParseError" in token and token[0] == "Character":
+            if (outputTokens and not "ParseError" in outputTokens[-1] and
+                outputTokens[-1][0] == "Character"):
+                outputTokens[-1][1] += token[1]
+            else:
+                outputTokens.append(token)
+        else:
+            outputTokens.append(token)
+    return outputTokens
+
+def normalizeTokens(tokens):
+    # TODO: convert tests to reflect arrays
+    for i, token in enumerate(tokens):
+        if token[0] == u'ParseError':
+            tokens[i] = token[0]
+    return tokens
+
+def tokensMatch(expectedTokens, receivedTokens, ignoreErrorOrder,
+                ignoreErrors=False):
+    """Test whether the test has passed or failed
+
+    If the ignoreErrorOrder flag is set to true we don't test the relative
+    positions of parse errors and non parse errors
+    """
+    checkSelfClosing= False
+    for token in expectedTokens:
+        if (token[0] == "StartTag" and len(token) == 4
+            or token[0] == "EndTag" and len(token) == 3):
+            checkSelfClosing = True
+            break
+
+    if not checkSelfClosing:
+        for token in receivedTokens:
+            if token[0] == "StartTag" or token[0] == "EndTag":
+                token.pop()
+
+    if not ignoreErrorOrder and not ignoreErrors:
+        return expectedTokens == receivedTokens
+    else:
+        #Sort the tokens into two groups; non-parse errors and parse errors
+        tokens = {"expected":[[],[]], "received":[[],[]]}
+        for tokenType, tokenList in zip(tokens.keys(),
+                                         (expectedTokens, receivedTokens)):
+            for token in tokenList:
+                if token != "ParseError":
+                    tokens[tokenType][0].append(token)
+                else:
+                    if not ignoreErrors:
+                        tokens[tokenType][1].append(token)
+        return tokens["expected"] == tokens["received"]
+
+def unescape_test(test):
+    def decode(inp):
+        return inp.decode("unicode-escape")
+
+    test["input"] = decode(test["input"])
+    for token in test["output"]:
+        if token == "ParseError":
+            continue
+        else:
+            token[1] = decode(token[1])
+            if len(token) > 2:
+                for key, value in token[2]:
+                    del token[2][key]
+                    token[2][decode(key)] = decode(value)
+    return test
+
+
+def runTokenizerTest(test):
+    #XXX - move this out into the setup function
+    #concatenate all consecutive character tokens into a single token
+    if 'doubleEscaped' in test:
+        test = unescape_test(test)
+
+    expected = concatenateCharacterTokens(test['output'])            
+    if 'lastStartTag' not in test:
+        test['lastStartTag'] = None
+    outBuffer = cStringIO.StringIO()
+    stdout = sys.stdout
+    sys.stdout = outBuffer
+    parser = TokenizerTestParser(test['initialState'], 
+                                 test['lastStartTag'])
+    tokens = parser.parse(test['input'])
+    tokens = concatenateCharacterTokens(tokens)
+    received = normalizeTokens(tokens)
+    errorMsg = u"\n".join(["\n\nInitial state:",
+                          test['initialState'] ,
+                          "\nInput:", unicode(test['input']),
+                          "\nExpected:", unicode(expected),
+                          "\nreceived:", unicode(tokens)])
+    errorMsg = errorMsg.encode("utf-8")
+    ignoreErrorOrder = test.get('ignoreErrorOrder', False)
+    assert tokensMatch(expected, received, ignoreErrorOrder), errorMsg
+
+
+def _doCapitalize(match):
+    return match.group(1).upper()
+
+_capitalizeRe = re.compile(r"\W+(\w)").sub
+
+def capitalize(s):
+    s = s.lower()
+    s = _capitalizeRe(_doCapitalize, s)
+    return s
+
+
+def test_tokenizer():
+    for filename in html5lib_test_files('tokenizer', '*.test'):
+        tests = json.load(file(filename))
+        testName = os.path.basename(filename).replace(".test","")
+        if 'tests' in tests:
+            for index,test in enumerate(tests['tests']):
+                #Skip tests with a self closing flag
+                skip = False
+                if 'initialStates' not in test:
+                    test["initialStates"] = ["Data state"]
+                for initialState in test["initialStates"]:
+                    test["initialState"] = capitalize(initialState)
+                    yield runTokenizerTest, test
+

html5lib/tests/test_treewalkers.py

@@ -0,0 +1,311 @@
+import os
+import sys
+import StringIO
+import unittest
+import warnings
+
+warnings.simplefilter("error")
+
+from support import html5lib_test_files, TestData, convertExpected
+
+from html5lib import html5parser, treewalkers, treebuilders, constants
+from html5lib.filters.lint import Filter as LintFilter, LintError
+
+def PullDOMAdapter(node):
+    from xml.dom import Node
+    from xml.dom.pulldom import START_ELEMENT, END_ELEMENT, COMMENT, CHARACTERS
+
+    if node.nodeType in (Node.DOCUMENT_NODE, Node.DOCUMENT_FRAGMENT_NODE):
+        for childNode in node.childNodes:
+            for event in PullDOMAdapter(childNode):
+                yield event
+
+    elif node.nodeType == Node.DOCUMENT_TYPE_NODE:
+        raise NotImplementedError("DOCTYPE nodes are not supported by PullDOM")
+
+    elif node.nodeType == Node.COMMENT_NODE:
+        yield COMMENT, node
+
+    elif node.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
+        yield CHARACTERS, node
+
+    elif node.nodeType == Node.ELEMENT_NODE:
+        yield START_ELEMENT, node
+        for childNode in node.childNodes:
+            for event in PullDOMAdapter(childNode):
+                yield event
+        yield END_ELEMENT, node
+
+    else:
+        raise NotImplementedError("Node type not supported: " + str(node.nodeType))
+
+treeTypes = {
+"simpletree":  {"builder": treebuilders.getTreeBuilder("simpletree"),
+                "walker":  treewalkers.getTreeWalker("simpletree")},
+"DOM":         {"builder": treebuilders.getTreeBuilder("dom"),
+                "walker":  treewalkers.getTreeWalker("dom")},
+"PullDOM":     {"builder": treebuilders.getTreeBuilder("dom"),
+                "adapter": PullDOMAdapter,
+                "walker":  treewalkers.getTreeWalker("pulldom")},
+}
+
+#Try whatever etree implementations are available from a list that are
+#"supposed" to work
+try:
+    import xml.etree.ElementTree as ElementTree
+    treeTypes['ElementTree'] = \
+        {"builder": treebuilders.getTreeBuilder("etree", ElementTree),
+         "walker":  treewalkers.getTreeWalker("etree", ElementTree)}
+except ImportError:
+    try:
+        import elementtree.ElementTree as ElementTree
+        treeTypes['ElementTree'] = \
+            {"builder": treebuilders.getTreeBuilder("etree", ElementTree),
+             "walker":  treewalkers.getTreeWalker("etree", ElementTree)}
+    except ImportError:
+        pass
+
+try:
+    import xml.etree.cElementTree as ElementTree
+    treeTypes['cElementTree'] = \
+        {"builder": treebuilders.getTreeBuilder("etree", ElementTree),
+         "walker":  treewalkers.getTreeWalker("etree", ElementTree)}
+except ImportError:
+    try:
+        import cElementTree as ElementTree
+        treeTypes['cElementTree'] = \
+            {"builder": treebuilders.getTreeBuilder("etree", ElementTree),
+             "walker":  treewalkers.getTreeWalker("etree", ElementTree)}
+    except ImportError:
+        pass
+
+try:
+    import lxml.etree as ElementTree
+#    treeTypes['lxml_as_etree'] = \
+#        {"builder": treebuilders.getTreeBuilder("etree", ElementTree),
+#         "walker":  treewalkers.getTreeWalker("etree", ElementTree)}
+    treeTypes['lxml_native'] = \
+        {"builder": treebuilders.getTreeBuilder("lxml"),
+         "walker":  treewalkers.getTreeWalker("lxml")}
+except ImportError:
+    pass
+
+try:
+    import BeautifulSoup
+    treeTypes["beautifulsoup"] = \
+        {"builder": treebuilders.getTreeBuilder("beautifulsoup"),
+         "walker":  treewalkers.getTreeWalker("beautifulsoup")}
+except ImportError:
+    pass
+    
+#Try whatever etree implementations are available from a list that are
+#"supposed" to work
+try:
+    import pxdom
+    treeTypes['pxdom'] = \
+        {"builder": treebuilders.getTreeBuilder("dom", pxdom),
+         "walker":  treewalkers.getTreeWalker("dom")}
+except ImportError:
+    pass
+
+try:
+    from genshi.core import QName, Attrs
+    from genshi.core import START, END, TEXT, COMMENT, DOCTYPE
+
+    def GenshiAdapter(tree):
+        text = None
+        for token in treewalkers.getTreeWalker("simpletree")(tree):
+            type = token["type"]
+            if type in ("Characters", "SpaceCharacters"):
+                if text is None:
+                    text = token["data"]
+                else:
+                    text += token["data"]
+            elif text is not None:
+                yield TEXT, text, (None, -1, -1)
+                text = None
+
+            if type in ("StartTag", "EmptyTag"):
+                if token["namespace"]:
+                    name = u"{%s}%s" % (token["namespace"], token["name"])
+                else:
+                    name = token["name"]
+                yield (START,
+                       (QName(name),
+                        Attrs([(QName(attr),value) for attr,value in token["data"]])),
+                       (None, -1, -1))
+                if type == "EmptyTag":
+                    type = "EndTag"
+
+            if type == "EndTag":
+                yield END, QName(token["name"]), (None, -1, -1)
+
+            elif type == "Comment":
+                yield COMMENT, token["data"], (None, -1, -1)
+
+            elif type == "Doctype":
+                yield DOCTYPE, (token["name"], token["publicId"], 
+                                token["systemId"]), (None, -1, -1)
+
+            else:
+                pass # FIXME: What to do?
+
+        if text is not None:
+            yield TEXT, text, (None, -1, -1)
+
+    #treeTypes["genshi"] = \
+    #    {"builder": treebuilders.getTreeBuilder("simpletree"),
+    #     "adapter": GenshiAdapter,
+    #     "walker":  treewalkers.getTreeWalker("genshi")}
+except ImportError:
+    pass
+
+def concatenateCharacterTokens(tokens):
+    charactersToken = None
+    for token in tokens:
+        type = token["type"]
+        if type in ("Characters", "SpaceCharacters"):
+            if charactersToken is None:
+                charactersToken = {"type": "Characters", "data": token["data"]}
+            else:
+                charactersToken["data"] += token["data"]
+        else:
+            if charactersToken is not None:
+                yield charactersToken
+                charactersToken = None
+            yield token
+    if charactersToken is not None:
+        yield charactersToken
+
+def convertTokens(tokens):
+    output = []
+    indent = 0
+    for token in concatenateCharacterTokens(tokens):
+        type = token["type"]
+        if type in ("StartTag", "EmptyTag"):
+            if (token["namespace"] and
+                token["namespace"] != constants.namespaces["html"]):
+                if token["namespace"] in constants.prefixes:
+                    name = constants.prefixes[token["namespace"]]
+                else:
+                    name = token["namespace"]
+                name += u" " + token["name"]
+            else:
+                name = token["name"]
+            output.append(u"%s<%s>" % (" "*indent, name))
+            indent += 2
+            attrs = token["data"]
+            if attrs:
+                #TODO: Remove this if statement, attrs should always exist
+                for (namespace,name),value in sorted(attrs.items()):
+                    if namespace:
+                        if namespace in constants.prefixes:
+                            outputname = constants.prefixes[namespace]
+                        else:
+                            outputname = namespace
+                        outputname += u" " + name
+                    else:
+                        outputname = name
+                    output.append(u"%s%s=\"%s\"" % (" "*indent, outputname, value))
+            if type == "EmptyTag":
+                indent -= 2
+        elif type == "EndTag":
+            indent -= 2
+        elif type == "Comment":
+            output.append("%s<!-- %s -->" % (" "*indent, token["data"]))
+        elif type == "Doctype":
+            if token["name"]:
+                if token["publicId"]:
+                    output.append("""%s<!DOCTYPE %s "%s" "%s">"""% 
+                                  (" "*indent, token["name"], 
+                                   token["publicId"],
+                                   token["systemId"] and token["systemId"] or ""))
+                elif token["systemId"]:
+                    output.append("""%s<!DOCTYPE %s "" "%s">"""% 
+                                  (" "*indent, token["name"], 
+                                   token["systemId"]))
+                else:
+                    output.append("%s<!DOCTYPE %s>"%(" "*indent,
+                                                     token["name"]))
+            else:
+                output.append("%s<!DOCTYPE >" % (" "*indent,))
+        elif type in ("Characters", "SpaceCharacters"):
+            output.append("%s\"%s\"" % (" "*indent, token["data"]))
+        else:
+            pass # TODO: what to do with errors?
+    return u"\n".join(output)
+
+import re
+attrlist = re.compile(r"^(\s+)\w+=.*(\n\1\w+=.*)+",re.M)
+def sortattrs(x):
+  lines = x.group(0).split("\n")
+  lines.sort()
+  return "\n".join(lines)
+
+
+class TokenTestCase(unittest.TestCase):
+    def test_all_tokens(self):
+        expected = [
+            {'data': {}, 'type': 'StartTag', 'namespace': u'http://www.w3.org/1999/xhtml', 'name': u'html'},
+            {'data': {}, 'type': 'StartTag', 'namespace': u'http://www.w3.org/1999/xhtml', 'name': u'head'},
+            {'data': {}, 'type': 'EndTag', 'namespace': u'http://www.w3.org/1999/xhtml', 'name': u'head'},
+            {'data': {}, 'type': 'StartTag', 'namespace': u'http://www.w3.org/1999/xhtml', 'name': u'body'},
+            {'data': u'a', 'type': 'Characters'},
+            {'data': {}, 'type': 'StartTag', 'namespace': u'http://www.w3.org/1999/xhtml', 'name': u'div'},
+            {'data': u'b', 'type': 'Characters'},
+            {'data': {}, 'type': 'EndTag', 'namespace': u'http://www.w3.org/1999/xhtml', 'name': u'div'},
+            {'data': u'c', 'type': 'Characters'},
+            {'data': {}, 'type': 'EndTag', 'namespace': u'http://www.w3.org/1999/xhtml', 'name': u'body'},
+            {'data': {}, 'type': 'EndTag', 'namespace': u'http://www.w3.org/1999/xhtml', 'name': u'html'}
+            ]
+        for treeName, treeCls in treeTypes.iteritems():
+            p = html5parser.HTMLParser(tree = treeCls["builder"])
+            document = p.parse("<html><head></head><body>a<div>b</div>c</body></html>")
+            document = treeCls.get("adapter", lambda x: x)(document)
+            output = treeCls["walker"](document)
+            for expectedToken, outputToken in zip(expected, output):
+                self.assertEquals(expectedToken, outputToken)
+
+def run_test(innerHTML, input, expected, errors, treeClass):
+    try:
+        p = html5parser.HTMLParser(tree = treeClass["builder"])
+        if innerHTML:
+            document = p.parseFragment(StringIO.StringIO(input), innerHTML)
+        else:
+            document = p.parse(StringIO.StringIO(input))
+    except constants.DataLossWarning:
+        #Ignore testcases we know we don't pass
+        return
+
+    document = treeClass.get("adapter", lambda x: x)(document)
+    try:
+        output = convertTokens(treeClass["walker"](document))
+        output = attrlist.sub(sortattrs, output)
+        expected = attrlist.sub(sortattrs, convertExpected(expected))
+        assert expected == output, "\n".join([
+                "", "Input:", input,
+                "", "Expected:", expected,
+                "", "Received:", output
+                ])
+    except NotImplementedError:
+        pass # Amnesty for those that confess...
+            
+def test_treewalker():
+    sys.stdout.write('Testing tree walkers '+ " ".join(treeTypes.keys()) + "\n")
+
+    for treeName, treeCls in treeTypes.iteritems():
+        files = html5lib_test_files('tree-construction')
+        for filename in files:
+            testName = os.path.basename(filename).replace(".dat","")
+
+            tests = TestData(filename, "data")
+
+            for index, test in enumerate(tests):
+                (input, errors,
+                 innerHTML, expected) = [test[key] for key in ("data", "errors",
+                                                               "document-fragment",
+                                                               "document")]
+                errors = errors.split("\n")
+                yield run_test, innerHTML, input, expected, errors, treeCls
+
+

html5lib/tests/test_whitespace_filter.py

@@ -0,0 +1,123 @@
+import unittest
+
+from html5lib.filters.whitespace import Filter
+from html5lib.constants import spaceCharacters
+spaceCharacters = u"".join(spaceCharacters)
+
+class TestCase(unittest.TestCase):
+    def runTest(self, input, expected):
+        output = list(Filter(input))
+        errorMsg = "\n".join(["\n\nInput:", str(input),
+                              "\nExpected:", str(expected),
+                              "\nReceived:", str(output)])
+        self.assertEquals(output, expected, errorMsg)
+
+    def runTestUnmodifiedOutput(self, input):
+        self.runTest(input, input)
+
+    def testPhrasingElements(self):
+        self.runTestUnmodifiedOutput(
+            [{"type": u"Characters", "data": u"This is a " },
+             {"type": u"StartTag", "name": u"span", "data": [] },
+             {"type": u"Characters", "data": u"phrase" },
+             {"type": u"EndTag", "name": u"span", "data": []},
+             {"type": u"SpaceCharacters", "data": u" " },
+             {"type": u"Characters", "data": u"with" },
+             {"type": u"SpaceCharacters", "data": u" " },
+             {"type": u"StartTag", "name": u"em", "data": [] },
+             {"type": u"Characters", "data": u"emphasised text" },
+             {"type": u"EndTag", "name": u"em", "data": []},
+             {"type": u"Characters", "data": u" and an " },
+             {"type": u"StartTag", "name": u"img", "data": [[u"alt", u"image"]] },
+             {"type": u"Characters", "data": u"." }])
+
+    def testLeadingWhitespace(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"SpaceCharacters", "data": spaceCharacters},
+             {"type": u"Characters", "data": u"foo"},
+             {"type": u"EndTag", "name": u"p", "data": []}],
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"SpaceCharacters", "data": u" "},
+             {"type": u"Characters", "data": u"foo"},
+             {"type": u"EndTag", "name": u"p", "data": []}])
+
+    def testLeadingWhitespaceAsCharacters(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"Characters", "data": spaceCharacters + u"foo"},
+             {"type": u"EndTag", "name": u"p", "data": []}],
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"Characters", "data": u" foo"},
+             {"type": u"EndTag", "name": u"p", "data": []}])
+
+    def testTrailingWhitespace(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"Characters", "data": u"foo"},
+             {"type": u"SpaceCharacters", "data": spaceCharacters},
+             {"type": u"EndTag", "name": u"p", "data": []}],
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"Characters", "data": u"foo"},
+             {"type": u"SpaceCharacters", "data": u" "},
+             {"type": u"EndTag", "name": u"p", "data": []}])
+
+    def testTrailingWhitespaceAsCharacters(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"Characters", "data": u"foo" + spaceCharacters},
+             {"type": u"EndTag", "name": u"p", "data": []}],
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"Characters", "data": u"foo "},
+             {"type": u"EndTag", "name": u"p", "data": []}])
+
+    def testWhitespace(self):
+        self.runTest(
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"Characters", "data": u"foo" + spaceCharacters + "bar"},
+             {"type": u"EndTag", "name": u"p", "data": []}],
+            [{"type": u"StartTag", "name": u"p", "data": []},
+             {"type": u"Characters", "data": u"foo bar"},
+             {"type": u"EndTag", "name": u"p", "data": []}])
+
+    def testLeadingWhitespaceInPre(self):
+        self.runTestUnmodifiedOutput(
+            [{"type": u"StartTag", "name": u"pre", "data": []},
+             {"type": u"SpaceCharacters", "data": spaceCharacters},
+             {"type": u"Characters", "data": u"foo"},
+             {"type": u"EndTag", "name": u"pre", "data": []}])
+
+    def testLeadingWhitespaceAsCharactersInPre(self):
+        self.runTestUnmodifiedOutput(
+            [{"type": u"StartTag", "name": u"pre", "data": []},
+             {"type": u"Characters", "data": spaceCharacters + u"foo"},
+             {"type": u"EndTag", "name": u"pre", "data": []}])
+
+    def testTrailingWhitespaceInPre(self):
+        self.runTestUnmodifiedOutput(
+            [{"type": u"StartTag", "name": u"pre", "data": []},
+             {"type": u"Characters", "data": u"foo"},
+             {"type": u"SpaceCharacters", "data": spaceCharacters},
+             {"type": u"EndTag", "name": u"pre", "data": []}])
+
+    def testTrailingWhitespaceAsCharactersInPre(self):
+        self.runTestUnmodifiedOutput(
+            [{"type": u"StartTag", "name": u"pre", "data": []},
+             {"type": u"Characters", "data": u"foo" + spaceCharacters},
+             {"type": u"EndTag", "name": u"pre", "data": []}])
+
+    def testWhitespaceInPre(self):
+        self.runTestUnmodifiedOutput(
+            [{"type": u"StartTag", "name": u"pre", "data": []},
+             {"type": u"Characters", "data": u"foo" + spaceCharacters + "bar"},
+             {"type": u"EndTag", "name": u"pre", "data": []}])
+
+def buildTestSuite():
+    return unittest.defaultTestLoader.loadTestsFromName(__name__)
+
+def main():
+    buildTestSuite()
+    unittest.main()
+
+if __name__ == "__main__":
+    main()

html5lib/tests/testdata/encoding/test-yahoo-jp.dat

@@ -0,0 +1,10 @@
+#data
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=euc-jp">
+<!--京-->
+<title>Yahoo! JAPAN</title>
+<meta name="description" content="日本最大級のポータルサイト。検索、オークション、ニュース、メール、コミュニティ、ショッピング、など80以上のサービスを展開。あなたの生活をより豊かにする「ライフ・エンジン」を目指していきます。">
+<style type="text/css" media="all">
+#encoding
+euc_jp

html5lib/tests/testdata/encoding/tests2.dat

@@ -0,0 +1,115 @@
+#data
+<meta
+#encoding
+windows-1252
+
+#data
+<
+#encoding
+windows-1252
+
+#data
+<!
+#encoding
+windows-1252
+
+#data
+<meta charset = "
+#encoding
+windows-1252
+
+#data
+<meta charset=euc_jp
+#encoding
+windows-1252
+
+#data
+<meta <meta charset='euc_jp'>
+#encoding
+euc_jp
+
+#data
+<meta       charset    =     'euc_jp'>
+#encoding
+euc_jp
+
+#data
+<!-- -->
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+#encoding
+utf-8
+
+#data
+<!-- -->
+<meta http-equiv="Content-Type" content="text/html; charset=utf
+#encoding
+windows-1252
+
+#data
+<meta http-equiv="Content-Type<meta charset="utf-8">
+#encoding
+windows-1252
+
+#data
+<meta http-equiv="Content-Type" content="text/html; charset='utf-8'">
+#encoding
+utf-8
+
+#data
+<meta http-equiv="Content-Type" content="text/html; charset='utf-8">
+#encoding
+windows-1252
+
+#data
+<meta                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 
+#encoding
+windows-1252
+
+#data
+<meta charset                    =                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            
+#encoding
+windows-1252
+
+#data
+<meta charset=                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            utf-8
+>
+#encoding
+utf-8
+
+#data
+<meta content = "text/html;
+#encoding
+windows-1252
+
+#data
+<meta charset="UTF-16">
+#encoding
+utf-8
+
+#data
+<meta charset="UTF-16LE">
+#encoding
+utf-8
+
+#data
+<meta charset="UTF-16BE">
+#encoding
+utf-8
+
+#data
+<html a=ñ>
+<meta charset="utf-8">
+#encoding
+utf-8
+
+#data
+<html ñ>
+<meta charset="utf-8">
+#encoding
+utf-8
+
+#data
+<html>ñ
+<meta charset="utf-8">
+#encoding
+utf-8

html5lib/tests/testdata/sanitizer/tests1.dat

@@ -0,0 +1,501 @@
+[
+  {
+    "name": "IE_Comments",
+    "input": "<!--[if gte IE 4]><script>alert('XSS');</script><![endif]-->",
+    "output": ""
+  },
+
+  {
+    "name": "IE_Comments_2",
+    "input": "<![if !IE 5]><script>alert('XSS');</script><![endif]>",
+    "output": "&lt;script&gt;alert('XSS');&lt;/script&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "allow_colons_in_path_component",
+    "input": "<a href=\"./this:that\">foo</a>",
+    "output": "<a href='./this:that'>foo</a>"
+  },
+
+  {
+    "name": "background_attribute",
+    "input": "<div background=\"javascript:alert('XSS')\"></div>",
+    "output": "<div/>",
+    "xhtml": "<div></div>",
+    "rexml": "<div></div>"
+  },
+
+  {
+    "name": "bgsound",
+    "input": "<bgsound src=\"javascript:alert('XSS');\" />",
+    "output": "&lt;bgsound src=\"javascript:alert('XSS');\"/&gt;",
+    "rexml": "&lt;bgsound src=\"javascript:alert('XSS');\"&gt;&lt;/bgsound&gt;"
+  },
+
+  {
+    "name": "div_background_image_unicode_encoded",
+    "input": "<div style=\"background-image:\u00a5\u00a2\u006C\u0028'\u006a\u0061\u00a6\u0061\u00a3\u0063\u00a2\u0069\u00a0\u00a4\u003a\u0061\u006c\u0065\u00a2\u00a4\u0028.1027\u0058.1053\u0053\u0027\u0029'\u0029\">foo</div>",
+    "output": "<div style=''>foo</div>"
+  },
+
+  {
+    "name": "div_expression",
+    "input": "<div style=\"width: expression(alert('XSS'));\">foo</div>",
+    "output": "<div style=''>foo</div>"
+  },
+
+  {
+    "name": "double_open_angle_brackets",
+    "input": "<img src=http://ha.ckers.org/scriptlet.html <",
+    "output": "<img src='http://ha.ckers.org/scriptlet.html'>",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "double_open_angle_brackets_2",
+    "input": "<script src=http://ha.ckers.org/scriptlet.html <",
+    "output": "&lt;script src=\"http://ha.ckers.org/scriptlet.html\" &lt;=\"\"&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "grave_accents",
+    "input": "<img src=`javascript:alert('XSS')` />",
+    "output": "<img/>",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "img_dynsrc_lowsrc",
+    "input": "<img dynsrc=\"javascript:alert('XSS')\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "img_vbscript",
+    "input": "<img src='vbscript:msgbox(\"XSS\")' />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "input_image",
+    "input": "<input type=\"image\" src=\"javascript:alert('XSS');\" />",
+    "output": "<input type='image'/>",
+    "rexml": "<input type='image' />"
+  },
+
+  {
+    "name": "link_stylesheets",
+    "input": "<link rel=\"stylesheet\" href=\"javascript:alert('XSS');\" />",
+    "output": "&lt;link rel=\"stylesheet\" href=\"javascript:alert('XSS');\"/&gt;",
+    "rexml": "&lt;link href=\"javascript:alert('XSS');\" rel=\"stylesheet\"/&gt;"
+  },
+
+  {
+    "name": "link_stylesheets_2",
+    "input": "<link rel=\"stylesheet\" href=\"http://ha.ckers.org/xss.css\" />",
+    "output": "&lt;link rel=\"stylesheet\" href=\"http://ha.ckers.org/xss.css\"/&gt;",
+    "rexml": "&lt;link href=\"http://ha.ckers.org/xss.css\" rel=\"stylesheet\"/&gt;"
+  },
+
+  {
+    "name": "list_style_image",
+    "input": "<li style=\"list-style-image: url(javascript:alert('XSS'))\">foo</li>",
+    "output": "<li style=''>foo</li>"
+  },
+
+  {
+    "name": "no_closing_script_tags",
+    "input": "<script src=http://ha.ckers.org/xss.js?<b>",
+    "output": "&lt;script src=\"http://ha.ckers.org/xss.js?&amp;lt;b\"&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "non_alpha_non_digit",
+    "input": "<script/XSS src=\"http://ha.ckers.org/xss.js\"></script>",
+    "output": "&lt;script XSS=\"\" src=\"http://ha.ckers.org/xss.js\"&gt;&lt;/script&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "non_alpha_non_digit_2",
+    "input": "<a onclick!\\#$%&()*~+-_.,:;?@[/|\\]^`=alert(\"XSS\")>foo</a>",
+    "output": "<a>foo</a>",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "non_alpha_non_digit_3",
+    "input": "<img/src=\"http://ha.ckers.org/xss.js\"/>",
+    "output": "<img src='http://ha.ckers.org/xss.js'/>",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "non_alpha_non_digit_II",
+    "input": "<a href!\\#$%&()*~+-_.,:;?@[/|]^`=alert('XSS')>foo</a>",
+    "output": "<a>foo</a>",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "non_alpha_non_digit_III",
+    "input": "<a/href=\"javascript:alert('XSS');\">foo</a>",
+    "output": "<a>foo</a>",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "platypus",
+    "input": "<a href=\"http://www.ragingplatypus.com/\" style=\"display:block; position:absolute; left:0; top:0; width:100%; height:100%; z-index:1; background-color:black; background-image:url(http://www.ragingplatypus.com/i/cam-full.jpg); background-x:center; background-y:center; background-repeat:repeat;\">never trust your upstream platypus</a>",
+    "output": "<a href='http://www.ragingplatypus.com/' style='display: block; width: 100%; height: 100%; background-color: black; background-x: center; background-y: center;'>never trust your upstream platypus</a>"
+  },
+
+  {
+    "name": "protocol_resolution_in_script_tag",
+    "input": "<script src=//ha.ckers.org/.j></script>",
+    "output": "&lt;script src=\"//ha.ckers.org/.j\"&gt;&lt;/script&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "should_allow_anchors",
+    "input": "<a href='foo' onclick='bar'><script>baz</script></a>",
+    "output": "<a href='foo'>&lt;script&gt;baz&lt;/script&gt;</a>"
+  },
+
+  {
+    "name": "should_allow_image_alt_attribute",
+    "input": "<img alt='foo' onclick='bar' />",
+    "output": "<img alt='foo'/>",
+    "rexml": "<img alt='foo' />"
+  },
+
+  {
+    "name": "should_allow_image_height_attribute",
+    "input": "<img height='foo' onclick='bar' />",
+    "output": "<img height='foo'/>",
+    "rexml": "<img height='foo' />"
+  },
+
+  {
+    "name": "should_allow_image_src_attribute",
+    "input": "<img src='foo' onclick='bar' />",
+    "output": "<img src='foo'/>",
+    "rexml": "<img src='foo' />"
+  },
+
+  {
+    "name": "should_allow_image_width_attribute",
+    "input": "<img width='foo' onclick='bar' />",
+    "output": "<img width='foo'/>",
+    "rexml": "<img width='foo' />"
+  },
+
+  {
+    "name": "should_handle_blank_text",
+    "input": "",
+    "output": ""
+  },
+
+  {
+    "name": "should_handle_malformed_image_tags",
+    "input": "<img \"\"\"><script>alert(\"XSS\")</script>\">",
+    "output": "<img/>&lt;script&gt;alert(\"XSS\")&lt;/script&gt;\"&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "should_handle_non_html",
+    "input": "abc",
+    "output": "abc"
+  },
+
+  {
+    "name": "should_not_fall_for_ridiculous_hack",
+    "input": "<img\nsrc\n=\n\"\nj\na\nv\na\ns\nc\nr\ni\np\nt\n:\na\nl\ne\nr\nt\n(\n'\nX\nS\nS\n'\n)\n\"\n />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_0",
+    "input": "<img src=\"javascript:alert('XSS');\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_1",
+    "input": "<img src=javascript:alert('XSS') />",
+    "output": "<img/>",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_10",
+    "input": "<img src=\"jav&#x0A;ascript:alert('XSS');\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_11",
+    "input": "<img src=\"jav&#x0D;ascript:alert('XSS');\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_12",
+    "input": "<img src=\" &#14;  javascript:alert('XSS');\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_13",
+    "input": "<img src=\"&#x20;javascript:alert('XSS');\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_14",
+    "input": "<img src=\"&#xA0;javascript:alert('XSS');\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_2",
+    "input": "<img src=\"JaVaScRiPt:alert('XSS')\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_3",
+    "input": "<img src='javascript:alert(&quot;XSS&quot;)' />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_4",
+    "input": "<img src='javascript:alert(String.fromCharCode(88,83,83))' />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_5",
+    "input": "<img src='&#106;&#97;&#118;&#97;&#115;&#99;&#114;&#105;&#112;&#116;&#58;&#97;&#108;&#101;&#114;&#116;&#40;&#39;&#88;&#83;&#83;&#39;&#41;' />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_6",
+    "input": "<img src='&#0000106;&#0000097;&#0000118;&#0000097;&#0000115;&#0000099;&#0000114;&#0000105;&#0000112;&#0000116;&#0000058;&#0000097;&#0000108;&#0000101;&#0000114;&#0000116;&#0000040;&#0000039;&#0000088;&#0000083;&#0000083;&#0000039;&#0000041' />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_7",
+    "input": "<img src='&#x6A;&#x61;&#x76;&#x61;&#x73;&#x63;&#x72;&#x69;&#x70;&#x74;&#x3A;&#x61;&#x6C;&#x65;&#x72;&#x74;&#x28;&#x27;&#x58;&#x53;&#x53;&#x27;&#x29' />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_8",
+    "input": "<img src=\"jav\tascript:alert('XSS');\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_not_fall_for_xss_image_hack_9",
+    "input": "<img src=\"jav&#x09;ascript:alert('XSS');\" />",
+    "output": "<img/>",
+    "rexml": "<img />"
+  },
+
+  {
+    "name": "should_sanitize_half_open_scripts",
+    "input": "<img src=\"javascript:alert('XSS')\"",
+    "output": "<img/>",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "should_sanitize_invalid_script_tag",
+    "input": "<script/XSS SRC=\"http://ha.ckers.org/xss.js\"></script>",
+    "output": "&lt;script XSS=\"\" SRC=\"http://ha.ckers.org/xss.js\"&gt;&lt;/script&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "should_sanitize_script_tag_with_multiple_open_brackets",
+    "input": "<<script>alert(\"XSS\");//<</script>",
+    "output": "&lt;&lt;script&gt;alert(\"XSS\");//&lt;&lt;/script&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "should_sanitize_script_tag_with_multiple_open_brackets_2",
+    "input": "<iframe src=http://ha.ckers.org/scriptlet.html\n<",
+    "output": "&lt;iframe src=\"http://ha.ckers.org/scriptlet.html\" &lt;=\"\"&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "should_sanitize_tag_broken_up_by_null",
+    "input": "<scr\u0000ipt>alert(\"XSS\")</scr\u0000ipt>",
+    "output": "&lt;scr\ufffdipt&gt;alert(\"XSS\")&lt;/scr\ufffdipt&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "should_sanitize_unclosed_script",
+    "input": "<script src=http://ha.ckers.org/xss.js?<b>",
+    "output": "&lt;script src=\"http://ha.ckers.org/xss.js?&amp;lt;b\"&gt;",
+    "rexml": "Ill-formed XHTML!"
+  },
+
+  {
+    "name": "should_strip_href_attribute_in_a_with_bad_protocols",
+    "input": "<a href=\"javascript:XSS\" title=\"1\">boo</a>",
+    "output": "<a title='1'>boo</a>"
+  },
+
+  {
+    "name": "should_strip_href_attribute_in_a_with_bad_protocols_and_whitespace",
+    "input": "<a href=\" javascript:XSS\" title=\"1\">boo</a>",
+    "output": "<a title='1'>boo</a>"
+  },
+
+  {
+    "name": "should_strip_src_attribute_in_img_with_bad_protocols",
+    "input": "<img src=\"javascript:XSS\" title=\"1\">boo</img>",
+    "output": "<img title='1'/>boo",
+    "rexml": "<img title='1' />"
+  },
+
+  {
+    "name": "should_strip_src_attribute_in_img_with_bad_protocols_and_whitespace",
+    "input": "<img src=\" javascript:XSS\" title=\"1\">boo</img>",
+    "output": "<img title='1'/>boo",
+    "rexml": "<img title='1' />"
+  },
+
+  {
+    "name": "xml_base",
+    "input": "<div xml:base=\"javascript:alert('XSS');//\">foo</div>",
+    "output": "<div>foo</div>"
+  },
+
+  {
+    "name": "xul",
+    "input": "<p style=\"-moz-binding:url('http://ha.ckers.org/xssmoz.xml#xss')\">fubar</p>",
+    "output": "<p style=''>fubar</p>"
+  },
+
+  {
+    "name": "quotes_in_attributes",
+    "input": "<img src='foo' title='\"foo\" bar' />",
+    "rexml": "<img src='foo' title='\"foo\" bar' />",
+    "output": "<img title='&quot;foo&quot; bar' src='foo'/>"
+  },
+
+  {
+    "name": "uri_refs_in_svg_attributes",
+    "input": "<rect fill='url(#foo)' />",
+    "rexml": "<rect fill='url(#foo)'></rect>",
+    "xhtml": "<rect fill='url(#foo)'></rect>",
+    "output": "<rect fill='url(#foo)'/>"
+  },
+
+  {
+    "name": "absolute_uri_refs_in_svg_attributes",
+    "input": "<rect fill='url(http://bad.com/) #fff' />",
+    "rexml": "<rect fill='  #fff'></rect>",
+    "xhtml": "<rect fill='  #fff'></rect>",
+    "output": "<rect fill='  #fff'/>"
+  },
+
+  {
+    "name": "uri_ref_with_space_in svg_attribute",
+    "input": "<rect fill='url(\n#foo)' />",
+    "rexml": "<rect fill='url(\n#foo)'></rect>",
+    "xhtml": "<rect fill='url(\n#foo)'></rect>",
+    "output": "<rect fill='url(\n#foo)'/>"
+  },
+
+  {
+    "name": "absolute_uri_ref_with_space_in svg_attribute",
+    "input": "<rect fill=\"url(\nhttp://bad.com/)\" />",
+    "rexml": "<rect fill=' '></rect>",
+    "xhtml": "<rect fill=' '></rect>",
+    "output": "<rect fill=' '/>"
+  },
+
+  {
+    "name": "allow_html5_image_tag",
+    "input": "<image src='foo' />",
+    "rexml": "&lt;image src=\"foo\"&gt;&lt;/image&gt;",
+    "output": "&lt;image src=\"foo\"/&gt;"
+  },
+
+  {
+    "name": "style_attr_end_with_nothing",
+    "input": "<div style=\"color: blue\" />",
+    "output": "<div style='color: blue;'/>",
+    "xhtml": "<div style='color: blue;'></div>",
+    "rexml": "<div style='color: blue;'></div>"
+  },
+
+  {
+    "name": "style_attr_end_with_space",
+    "input": "<div style=\"color: blue \" />",
+    "output": "<div style='color: blue ;'/>",
+    "xhtml": "<div style='color: blue ;'></div>",
+    "rexml": "<div style='color: blue ;'></div>"
+  },
+
+  {
+    "name": "style_attr_end_with_semicolon",
+    "input": "<div style=\"color: blue;\" />",
+    "output": "<div style='color: blue;'/>",
+    "xhtml": "<div style='color: blue;'></div>",
+    "rexml": "<div style='color: blue;'></div>"
+  },
+
+  {
+    "name": "style_attr_end_with_semicolon_space",
+    "input": "<div style=\"color: blue; \" />",
+    "output": "<div style='color: blue;'/>",
+    "xhtml": "<div style='color: blue;'></div>",
+    "rexml": "<div style='color: blue;'></div>"
+  },
+  
+  {
+   "name": "attributes_with_embedded_quotes",
+   "input": "<img src=doesntexist.jpg\"'onerror=\"alert(1) />",
+   "output": "<img src='doesntexist.jpg&quot;&apos;onerror=&quot;alert(1)'/>",
+   "rexml": "Ill-formed XHTML!"
+  },
+  
+  {
+   "name": "attributes_with_embedded_quotes_II",
+   "input": "<img src=notthere.jpg\"\"onerror=\"alert(2) />",
+   "output": "<img src='notthere.jpg&quot;&quot;onerror=&quot;alert(2)'/>",
+   "rexml": "Ill-formed XHTML!"
+  }
+]

html5lib/tests/testdata/serializer/core.test

@@ -0,0 +1,125 @@
+{"tests": [
+
+{"description": "proper attribute value escaping",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "test \"with\" ""}]]],
+ "expected": ["<span title='test \"with\" &amp;quot;'>"]
+},
+
+{"description": "proper attribute value non-quoting",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo"}]]],
+ "expected": ["<span title=foo>"],
+ "xhtml":    ["<span title=\"foo\">"]
+},
+
+{"description": "proper attribute value non-quoting (with <)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo<bar"}]]],
+ "expected": ["<span title=foo<bar>"],
+ "xhtml":    ["<span title=\"foo&lt;bar\">"]
+},
+
+{"description": "proper attribute value quoting (with =)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo=bar"}]]],
+ "expected": ["<span title=\"foo=bar\">"]
+},
+
+{"description": "proper attribute value quoting (with >)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo>bar"}]]],
+ "expected": ["<span title=\"foo>bar\">"]
+},
+
+{"description": "proper attribute value quoting (with \")",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo\"bar"}]]],
+ "expected": ["<span title='foo\"bar'>"]
+},
+
+{"description": "proper attribute value quoting (with ')",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo'bar"}]]],
+ "expected": ["<span title=\"foo'bar\">"]
+},
+
+{"description": "proper attribute value quoting (with both \" and ')",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo'bar\"baz"}]]],
+ "expected": ["<span title=\"foo'bar&quot;baz\">"]
+},
+
+{"description": "proper attribute value quoting (with space)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo bar"}]]],
+ "expected": ["<span title=\"foo bar\">"]
+},
+
+{"description": "proper attribute value quoting (with tab)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo\tbar"}]]],
+ "expected": ["<span title=\"foo\tbar\">"]
+},
+
+{"description": "proper attribute value quoting (with LF)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo\nbar"}]]],
+ "expected": ["<span title=\"foo\nbar\">"]
+},
+
+{"description": "proper attribute value quoting (with CR)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo\rbar"}]]],
+ "expected": ["<span title=\"foo\rbar\">"]
+},
+
+{"description": "proper attribute value non-quoting (with linetab)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo\u000Bbar"}]]],
+ "expected": ["<span title=foo\u000Bbar>"],
+ "xhtml": ["<span title=\"foo\u000Bbar\">"]
+},
+
+{"description": "proper attribute value quoting (with form feed)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "foo\u000Cbar"}]]],
+ "expected": ["<span title=\"foo\u000Cbar\">"]
+},
+
+{"description": "void element (as EmptyTag token)",
+ "input": [["EmptyTag", "img", {}]],
+ "expected": ["<img>"],
+ "xhtml":    ["<img />"]
+},
+
+{"description": "void element (as StartTag token)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "img", {}]],
+ "expected": ["<img>"],
+ "xhtml":    ["<img />"]
+},
+
+{"description": "doctype in error",
+ "input": [["Doctype", "foo"]],
+ "expected": ["<!DOCTYPE foo>"]
+},
+
+{"description": "character data",
+ "options": {"encoding":"utf-8"},
+ "input": [["Characters", "a<b>c&d"]],
+ "expected": ["a&lt;b&gt;c&amp;d"]
+},
+
+{"description": "rcdata",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "script", {}], ["Characters", "a<b>c&d"]],
+ "expected": ["<script>a<b>c&d"],
+ "xhtml": ["<script>a&lt;b&gt;c&amp;d"]
+},
+
+{"description": "doctype",
+ "input": [["Doctype", "HTML"]],
+ "expected": ["<!DOCTYPE HTML>"]
+},
+
+{"description": "HTML 4.01 DOCTYPE",
+ "input": [["Doctype", "HTML",  "-//W3C//DTD HTML 4.01//EN", "http://www.w3.org/TR/html4/strict.dtd"]],
+ "expected": ["<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/html4/strict.dtd\">"]
+},
+
+{"description": "HTML 4.01 DOCTYPE without system identifer",
+ "input": [["Doctype", "HTML",  "-//W3C//DTD HTML 4.01//EN"]],
+ "expected": ["<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\">"]
+},
+
+{"description": "IBM DOCTYPE without public identifer",
+ "input": [["Doctype", "html",  "", "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd"]],
+ "expected": ["<!DOCTYPE html SYSTEM \"http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd\">"]
+}
+
+]}

html5lib/tests/testdata/serializer/injectmeta.test

@@ -0,0 +1,66 @@
+{"tests": [
+
+{"description": "no encoding",
+ "options": {"inject_meta_charset": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": [""],
+ "xhtml": ["<head></head>"]
+},
+
+{"description": "empytag head",
+ "options": {"inject_meta_charset": true, "encoding":"utf-8"},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": ["<meta charset=utf-8>"],
+ "xhtml":    ["<head><meta charset=\"utf-8\" /></head>"]
+},
+
+{"description": "head w/title",
+ "options": {"inject_meta_charset": true, "encoding":"utf-8"},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["StartTag", "http://www.w3.org/1999/xhtml","title",{}], ["Characters", "foo"],["EndTag", "http://www.w3.org/1999/xhtml", "title"], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": ["<meta charset=utf-8><title>foo</title>"],
+ "xhtml":    ["<head><meta charset=\"utf-8\" /><title>foo</title></head>"]
+},
+
+{"description": "head w/meta-charset",
+ "options": {"inject_meta_charset": true, "encoding":"utf-8"},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EmptyTag","meta",[{"namespace": null, "name": "charset", "value": "ascii"}]], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": ["<meta charset=utf-8>"],
+ "xhtml":    ["<head><meta charset=\"utf-8\" /></head>"]
+},
+
+{"description": "head w/ two meta-charset",
+ "options": {"inject_meta_charset": true, "encoding":"utf-8"},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EmptyTag","meta",[{"namespace": null, "name": "charset", "value": "ascii"}]], ["EmptyTag","meta",[{"namespace": null, "name": "charset", "value": "ascii"}]], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": ["<meta charset=utf-8><meta charset=utf-8>", "<head><meta charset=utf-8><meta charset=ascii>"],
+ "xhtml": ["<head><meta charset=\"utf-8\" /><meta charset=\"utf-8\" /></head>", "<head><meta charset=\"utf-8\" /><meta charset=\"ascii\" /></head>"]
+},
+
+{"description": "head w/robots",
+ "options": {"inject_meta_charset": true, "encoding":"utf-8"},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EmptyTag","meta",[{"namespace": null, "name": "name", "value": "robots"},{"namespace": null, "name": "content", "value": "noindex"}]], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": ["<meta charset=utf-8><meta content=noindex name=robots>"],
+ "xhtml":    ["<head><meta charset=\"utf-8\" /><meta content=\"noindex\" name=\"robots\" /></head>"]
+},
+
+{"description": "head w/robots & charset",
+ "options": {"inject_meta_charset": true, "encoding":"utf-8"},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EmptyTag","meta",[{"namespace": null, "name": "name", "value": "robots"},{"namespace": null, "name": "content", "value": "noindex"}]], ["EmptyTag","meta",[{"namespace": null, "name": "charset", "value": "ascii"}]], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": ["<meta content=noindex name=robots><meta charset=utf-8>"],
+ "xhtml":    ["<head><meta content=\"noindex\" name=\"robots\" /><meta charset=\"utf-8\" /></head>"]
+},
+
+{"description": "head w/ charset in http-equiv content-type",
+ "options": {"inject_meta_charset": true, "encoding":"utf-8"},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EmptyTag","meta",[{"namespace": null, "name": "http-equiv", "value": "content-type"}, {"namespace": null, "name": "content", "value": "text/html; charset=ascii"}]], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": ["<meta content=\"text/html; charset=utf-8\" http-equiv=content-type>"],
+ "xhtml":    ["<head><meta content=\"text/html; charset=utf-8\" http-equiv=\"content-type\" /></head>"]
+},
+
+{"description": "head w/robots & charset in http-equiv content-type",
+ "options": {"inject_meta_charset": true, "encoding":"utf-8"},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EmptyTag","meta",[{"namespace": null, "name": "name", "value": "robots"},{"namespace": null, "name": "content", "value": "noindex"}]], ["EmptyTag","meta",[{"namespace": null, "name": "http-equiv", "value": "content-type"}, {"namespace": null, "name": "content", "value": "text/html; charset=ascii"}]], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": ["<meta content=noindex name=robots><meta content=\"text/html; charset=utf-8\" http-equiv=content-type>"],
+ "xhtml": ["<head><meta content=\"noindex\" name=\"robots\" /><meta content=\"text/html; charset=utf-8\" http-equiv=\"content-type\" /></head>"]
+}
+
+]}

html5lib/tests/testdata/serializer/optionaltags.test

@@ -0,0 +1,965 @@
+{"tests": [
+
+{"description": "html start-tag followed by text, with attributes",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "html", [{"namespace": null, "name": "lang", "value": "en"}]], ["Characters", "foo"]],
+ "expected": ["<html lang=en>foo"]
+},
+
+
+
+{"description": "html start-tag followed by comment",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "html", {}], ["Comment", "foo"]],
+ "expected": ["<html><!--foo-->"]
+},
+
+{"description": "html start-tag followed by space character",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "html", {}], ["Characters", " foo"]],
+ "expected": ["<html> foo"]
+},
+
+{"description": "html start-tag followed by text",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "html", {}], ["Characters", "foo"]],
+ "expected": ["foo"]
+},
+
+{"description": "html start-tag followed by start-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "html", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["<foo>"]
+},
+
+{"description": "html start-tag followed by end-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "html", {}], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "html start-tag at EOF (shouldn't ever happen?!)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "html", {}]],
+ "expected": [""]
+},
+
+
+
+{"description": "html end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "html"], ["Comment", "foo"]],
+ "expected": ["</html><!--foo-->"]
+},
+
+{"description": "html end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "html"], ["Characters", " foo"]],
+ "expected": ["</html> foo"]
+},
+
+{"description": "html end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "html"], ["Characters", "foo"]],
+ "expected": ["foo"]
+},
+
+{"description": "html end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "html"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["<foo>"]
+},
+
+{"description": "html end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "html"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "html end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "html"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "head start-tag followed by comment",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["Comment", "foo"]],
+ "expected": ["<head><!--foo-->"]
+},
+
+{"description": "head start-tag followed by space character",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["Characters", " foo"]],
+ "expected": ["<head> foo"]
+},
+
+{"description": "head start-tag followed by text",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["Characters", "foo"]],
+ "expected": ["<head>foo"]
+},
+
+{"description": "head start-tag followed by start-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["<foo>"]
+},
+
+{"description": "head start-tag followed by end-tag (shouldn't ever happen?!)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["<head></foo>", "</foo>"]
+},
+
+{"description": "empty head element",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": [""]
+},
+
+{"description": "head start-tag followed by empty-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}], ["EmptyTag", "foo", {}]],
+ "expected": ["<foo>"]
+},
+
+{"description": "head start-tag at EOF (shouldn't ever happen?!)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "head", {}]],
+ "expected": ["<head>", ""]
+},
+
+
+
+{"description": "head end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "head"], ["Comment", "foo"]],
+ "expected": ["</head><!--foo-->"]
+},
+
+{"description": "head end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "head"], ["Characters", " foo"]],
+ "expected": ["</head> foo"]
+},
+
+{"description": "head end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "head"], ["Characters", "foo"]],
+ "expected": ["foo"]
+},
+
+{"description": "head end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "head"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["<foo>"]
+},
+
+{"description": "head end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "head"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "head end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "head"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "body start-tag followed by comment",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "body", {}], ["Comment", "foo"]],
+ "expected": ["<body><!--foo-->"]
+},
+
+{"description": "body start-tag followed by space character",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "body", {}], ["Characters", " foo"]],
+ "expected": ["<body> foo"]
+},
+
+{"description": "body start-tag followed by text",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "body", {}], ["Characters", "foo"]],
+ "expected": ["foo"]
+},
+
+{"description": "body start-tag followed by start-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "body", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["<foo>"]
+},
+
+{"description": "body start-tag followed by end-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "body", {}], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "body start-tag at EOF (shouldn't ever happen?!)",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "body", {}]],
+ "expected": [""]
+},
+
+
+
+{"description": "body end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "body"], ["Comment", "foo"]],
+ "expected": ["</body><!--foo-->"]
+},
+
+{"description": "body end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "body"], ["Characters", " foo"]],
+ "expected": ["</body> foo"]
+},
+
+{"description": "body end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "body"], ["Characters", "foo"]],
+ "expected": ["foo"]
+},
+
+{"description": "body end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "body"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["<foo>"]
+},
+
+{"description": "body end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "body"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "body end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "body"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "li end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "li"], ["Comment", "foo"]],
+ "expected": ["</li><!--foo-->"]
+},
+
+{"description": "li end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "li"], ["Characters", " foo"]],
+ "expected": ["</li> foo"]
+},
+
+{"description": "li end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "li"], ["Characters", "foo"]],
+ "expected": ["</li>foo"]
+},
+
+{"description": "li end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "li"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</li><foo>"]
+},
+
+{"description": "li end-tag followed by li start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "li"], ["StartTag", "http://www.w3.org/1999/xhtml", "li", {}]],
+ "expected": ["<li>"]
+},
+
+{"description": "li end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "li"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "li end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "li"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "dt end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dt"], ["Comment", "foo"]],
+ "expected": ["</dt><!--foo-->"]
+},
+
+{"description": "dt end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dt"], ["Characters", " foo"]],
+ "expected": ["</dt> foo"]
+},
+
+{"description": "dt end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dt"], ["Characters", "foo"]],
+ "expected": ["</dt>foo"]
+},
+
+{"description": "dt end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dt"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</dt><foo>"]
+},
+
+{"description": "dt end-tag followed by dt start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dt"], ["StartTag", "http://www.w3.org/1999/xhtml", "dt", {}]],
+ "expected": ["<dt>"]
+},
+
+{"description": "dt end-tag followed by dd start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dt"], ["StartTag", "http://www.w3.org/1999/xhtml", "dd", {}]],
+ "expected": ["<dd>"]
+},
+
+{"description": "dt end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dt"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</dt></foo>"]
+},
+
+{"description": "dt end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dt"]],
+ "expected": ["</dt>"]
+},
+
+
+
+
+{"description": "dd end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dd"], ["Comment", "foo"]],
+ "expected": ["</dd><!--foo-->"]
+},
+
+{"description": "dd end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dd"], ["Characters", " foo"]],
+ "expected": ["</dd> foo"]
+},
+
+{"description": "dd end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dd"], ["Characters", "foo"]],
+ "expected": ["</dd>foo"]
+},
+
+{"description": "dd end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dd"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</dd><foo>"]
+},
+
+{"description": "dd end-tag followed by dd start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dd"], ["StartTag", "http://www.w3.org/1999/xhtml", "dd", {}]],
+ "expected": ["<dd>"]
+},
+
+{"description": "dd end-tag followed by dt start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dd"], ["StartTag", "http://www.w3.org/1999/xhtml", "dt", {}]],
+ "expected": ["<dt>"]
+},
+
+{"description": "dd end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dd"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "dd end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "dd"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "p end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["Comment", "foo"]],
+ "expected": ["</p><!--foo-->"]
+},
+
+{"description": "p end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["Characters", " foo"]],
+ "expected": ["</p> foo"]
+},
+
+{"description": "p end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["Characters", "foo"]],
+ "expected": ["</p>foo"]
+},
+
+{"description": "p end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</p><foo>"]
+},
+
+{"description": "p end-tag followed by address start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "address", {}]],
+ "expected": ["<address>"]
+},
+
+{"description": "p end-tag followed by article start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "article", {}]],
+ "expected": ["<article>"]
+},
+
+{"description": "p end-tag followed by aside start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "aside", {}]],
+ "expected": ["<aside>"]
+},
+
+{"description": "p end-tag followed by blockquote start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "blockquote", {}]],
+ "expected": ["<blockquote>"]
+},
+
+{"description": "p end-tag followed by datagrid start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "datagrid", {}]],
+ "expected": ["<datagrid>"]
+},
+
+{"description": "p end-tag followed by dialog start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "dialog", {}]],
+ "expected": ["<dialog>"]
+},
+
+{"description": "p end-tag followed by dir start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "dir", {}]],
+ "expected": ["<dir>"]
+},
+
+{"description": "p end-tag followed by div start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "div", {}]],
+ "expected": ["<div>"]
+},
+
+{"description": "p end-tag followed by dl start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "dl", {}]],
+ "expected": ["<dl>"]
+},
+
+{"description": "p end-tag followed by fieldset start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "fieldset", {}]],
+ "expected": ["<fieldset>"]
+},
+
+{"description": "p end-tag followed by footer start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "footer", {}]],
+ "expected": ["<footer>"]
+},
+
+{"description": "p end-tag followed by form start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "form", {}]],
+ "expected": ["<form>"]
+},
+
+{"description": "p end-tag followed by h1 start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "h1", {}]],
+ "expected": ["<h1>"]
+},
+
+{"description": "p end-tag followed by h2 start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "h2", {}]],
+ "expected": ["<h2>"]
+},
+
+{"description": "p end-tag followed by h3 start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "h3", {}]],
+ "expected": ["<h3>"]
+},
+
+{"description": "p end-tag followed by h4 start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "h4", {}]],
+ "expected": ["<h4>"]
+},
+
+{"description": "p end-tag followed by h5 start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "h5", {}]],
+ "expected": ["<h5>"]
+},
+
+{"description": "p end-tag followed by h6 start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "h6", {}]],
+ "expected": ["<h6>"]
+},
+
+{"description": "p end-tag followed by header start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "header", {}]],
+ "expected": ["<header>"]
+},
+
+{"description": "p end-tag followed by hr empty-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["EmptyTag", "hr", {}]],
+ "expected": ["<hr>"]
+},
+
+{"description": "p end-tag followed by menu start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "menu", {}]],
+ "expected": ["<menu>"]
+},
+
+{"description": "p end-tag followed by nav start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "nav", {}]],
+ "expected": ["<nav>"]
+},
+
+{"description": "p end-tag followed by ol start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "ol", {}]],
+ "expected": ["<ol>"]
+},
+
+{"description": "p end-tag followed by p start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "p", {}]],
+ "expected": ["<p>"]
+},
+
+{"description": "p end-tag followed by pre start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "pre", {}]],
+ "expected": ["<pre>"]
+},
+
+{"description": "p end-tag followed by section start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "section", {}]],
+ "expected": ["<section>"]
+},
+
+{"description": "p end-tag followed by table start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "table", {}]],
+ "expected": ["<table>"]
+},
+
+{"description": "p end-tag followed by ul start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["StartTag", "http://www.w3.org/1999/xhtml", "ul", {}]],
+ "expected": ["<ul>"]
+},
+
+{"description": "p end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "p end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "p"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "optgroup end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "optgroup"], ["Comment", "foo"]],
+ "expected": ["</optgroup><!--foo-->"]
+},
+
+{"description": "optgroup end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "optgroup"], ["Characters", " foo"]],
+ "expected": ["</optgroup> foo"]
+},
+
+{"description": "optgroup end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "optgroup"], ["Characters", "foo"]],
+ "expected": ["</optgroup>foo"]
+},
+
+{"description": "optgroup end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "optgroup"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</optgroup><foo>"]
+},
+
+{"description": "optgroup end-tag followed by optgroup start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "optgroup"], ["StartTag", "http://www.w3.org/1999/xhtml", "optgroup", {}]],
+ "expected": ["<optgroup>"]
+},
+
+{"description": "optgroup end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "optgroup"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "optgroup end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "optgroup"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "option end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "option"], ["Comment", "foo"]],
+ "expected": ["</option><!--foo-->"]
+},
+
+{"description": "option end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "option"], ["Characters", " foo"]],
+ "expected": ["</option> foo"]
+},
+
+{"description": "option end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "option"], ["Characters", "foo"]],
+ "expected": ["</option>foo"]
+},
+
+{"description": "option end-tag followed by optgroup start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "option"], ["StartTag", "http://www.w3.org/1999/xhtml", "optgroup", {}]],
+ "expected": ["<optgroup>"]
+},
+
+{"description": "option end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "option"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</option><foo>"]
+},
+
+{"description": "option end-tag followed by option start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "option"], ["StartTag", "http://www.w3.org/1999/xhtml", "option", {}]],
+ "expected": ["<option>"]
+},
+
+{"description": "option end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "option"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "option end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "option"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "colgroup start-tag followed by comment",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "colgroup", {}], ["Comment", "foo"]],
+ "expected": ["<colgroup><!--foo-->"]
+},
+
+{"description": "colgroup start-tag followed by space character",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "colgroup", {}], ["Characters", " foo"]],
+ "expected": ["<colgroup> foo"]
+},
+
+{"description": "colgroup start-tag followed by text",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "colgroup", {}], ["Characters", "foo"]],
+ "expected": ["<colgroup>foo"]
+},
+
+{"description": "colgroup start-tag followed by start-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "colgroup", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["<colgroup><foo>"]
+},
+
+{"description": "first colgroup in a table with a col child",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "table", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "colgroup", {}], ["EmptyTag", "col", {}]],
+ "expected": ["<table><col>"]
+},
+
+{"description": "colgroup with a col child, following another colgroup",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "colgroup"], ["StartTag", "http://www.w3.org/1999/xhtml", "colgroup", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "col", {}]],
+ "expected": ["</colgroup><col>", "<colgroup><col>"]
+},
+
+{"description": "colgroup start-tag followed by end-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "colgroup", {}], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["<colgroup></foo>"]
+},
+
+{"description": "colgroup start-tag at EOF",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "colgroup", {}]],
+ "expected": ["<colgroup>"]
+},
+
+
+
+{"description": "colgroup end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "colgroup"], ["Comment", "foo"]],
+ "expected": ["</colgroup><!--foo-->"]
+},
+
+{"description": "colgroup end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "colgroup"], ["Characters", " foo"]],
+ "expected": ["</colgroup> foo"]
+},
+
+{"description": "colgroup end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "colgroup"], ["Characters", "foo"]],
+ "expected": ["foo"]
+},
+
+{"description": "colgroup end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "colgroup"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["<foo>"]
+},
+
+{"description": "colgroup end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "colgroup"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "colgroup end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "colgroup"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "thead end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "thead"], ["Comment", "foo"]],
+ "expected": ["</thead><!--foo-->"]
+},
+
+{"description": "thead end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "thead"], ["Characters", " foo"]],
+ "expected": ["</thead> foo"]
+},
+
+{"description": "thead end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "thead"], ["Characters", "foo"]],
+ "expected": ["</thead>foo"]
+},
+
+{"description": "thead end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "thead"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</thead><foo>"]
+},
+
+{"description": "thead end-tag followed by tbody start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "thead"], ["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}]],
+ "expected": ["<tbody>"]
+},
+
+{"description": "thead end-tag followed by tfoot start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "thead"], ["StartTag", "http://www.w3.org/1999/xhtml", "tfoot", {}]],
+ "expected": ["<tfoot>"]
+},
+
+{"description": "thead end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "thead"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</thead></foo>"]
+},
+
+{"description": "thead end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "thead"]],
+ "expected": ["</thead>"]
+},
+
+
+
+
+{"description": "tbody start-tag followed by comment",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}], ["Comment", "foo"]],
+ "expected": ["<tbody><!--foo-->"]
+},
+
+{"description": "tbody start-tag followed by space character",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}], ["Characters", " foo"]],
+ "expected": ["<tbody> foo"]
+},
+
+{"description": "tbody start-tag followed by text",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}], ["Characters", "foo"]],
+ "expected": ["<tbody>foo"]
+},
+
+{"description": "tbody start-tag followed by start-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["<tbody><foo>"]
+},
+
+{"description": "first tbody in a table with a tr child",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "table", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "tr", {}]],
+ "expected": ["<table><tr>"]
+},
+
+{"description": "tbody with a tr child, following another tbody",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tbody"], ["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "tr", {}]],
+ "expected": ["<tbody><tr>", "</tbody><tr>"]
+},
+
+{"description": "tbody with a tr child, following a thead",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "thead"], ["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "tr", {}]],
+ "expected": ["<tbody><tr>", "</thead><tr>"]
+},
+
+{"description": "tbody with a tr child, following a tfoot",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tfoot"], ["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}], ["StartTag", "http://www.w3.org/1999/xhtml", "tr", {}]],
+ "expected": ["<tbody><tr>", "</tfoot><tr>"]
+},
+
+{"description": "tbody start-tag followed by end-tag",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["<tbody></foo>"]
+},
+
+{"description": "tbody start-tag at EOF",
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}]],
+ "expected": ["<tbody>"]
+},
+
+
+
+{"description": "tbody end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tbody"], ["Comment", "foo"]],
+ "expected": ["</tbody><!--foo-->"]
+},
+
+{"description": "tbody end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tbody"], ["Characters", " foo"]],
+ "expected": ["</tbody> foo"]
+},
+
+{"description": "tbody end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tbody"], ["Characters", "foo"]],
+ "expected": ["</tbody>foo"]
+},
+
+{"description": "tbody end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tbody"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</tbody><foo>"]
+},
+
+{"description": "tbody end-tag followed by tbody start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tbody"], ["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}]],
+ "expected": ["<tbody>", "</tbody>"]
+},
+
+{"description": "tbody end-tag followed by tfoot start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tbody"], ["StartTag", "http://www.w3.org/1999/xhtml", "tfoot", {}]],
+ "expected": ["<tfoot>"]
+},
+
+{"description": "tbody end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tbody"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "tbody end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tbody"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "tfoot end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tfoot"], ["Comment", "foo"]],
+ "expected": ["</tfoot><!--foo-->"]
+},
+
+{"description": "tfoot end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tfoot"], ["Characters", " foo"]],
+ "expected": ["</tfoot> foo"]
+},
+
+{"description": "tfoot end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tfoot"], ["Characters", "foo"]],
+ "expected": ["</tfoot>foo"]
+},
+
+{"description": "tfoot end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tfoot"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</tfoot><foo>"]
+},
+
+{"description": "tfoot end-tag followed by tbody start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tfoot"], ["StartTag", "http://www.w3.org/1999/xhtml", "tbody", {}]],
+ "expected": ["<tbody>", "</tfoot>"]
+},
+
+{"description": "tfoot end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tfoot"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "tfoot end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tfoot"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "tr end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tr"], ["Comment", "foo"]],
+ "expected": ["</tr><!--foo-->"]
+},
+
+{"description": "tr end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tr"], ["Characters", " foo"]],
+ "expected": ["</tr> foo"]
+},
+
+{"description": "tr end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tr"], ["Characters", "foo"]],
+ "expected": ["</tr>foo"]
+},
+
+{"description": "tr end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tr"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</tr><foo>"]
+},
+
+{"description": "tr end-tag followed by tr start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tr"], ["StartTag", "http://www.w3.org/1999/xhtml", "tr", {}]],
+ "expected": ["<tr>", "</tr>"]
+},
+
+{"description": "tr end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tr"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "tr end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "tr"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "td end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "td"], ["Comment", "foo"]],
+ "expected": ["</td><!--foo-->"]
+},
+
+{"description": "td end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "td"], ["Characters", " foo"]],
+ "expected": ["</td> foo"]
+},
+
+{"description": "td end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "td"], ["Characters", "foo"]],
+ "expected": ["</td>foo"]
+},
+
+{"description": "td end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "td"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</td><foo>"]
+},
+
+{"description": "td end-tag followed by td start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "td"], ["StartTag", "http://www.w3.org/1999/xhtml", "td", {}]],
+ "expected": ["<td>", "</td>"]
+},
+
+{"description": "td end-tag followed by th start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "td"], ["StartTag", "http://www.w3.org/1999/xhtml", "th", {}]],
+ "expected": ["<th>", "</td>"]
+},
+
+{"description": "td end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "td"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "td end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "td"]],
+ "expected": [""]
+},
+
+
+
+
+{"description": "th end-tag followed by comment",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "th"], ["Comment", "foo"]],
+ "expected": ["</th><!--foo-->"]
+},
+
+{"description": "th end-tag followed by space character",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "th"], ["Characters", " foo"]],
+ "expected": ["</th> foo"]
+},
+
+{"description": "th end-tag followed by text",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "th"], ["Characters", "foo"]],
+ "expected": ["</th>foo"]
+},
+
+{"description": "th end-tag followed by start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "th"], ["StartTag", "http://www.w3.org/1999/xhtml", "foo", {}]],
+ "expected": ["</th><foo>"]
+},
+
+{"description": "th end-tag followed by th start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "th"], ["StartTag", "http://www.w3.org/1999/xhtml", "th", {}]],
+ "expected": ["<th>", "</th>"]
+},
+
+{"description": "th end-tag followed by td start-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "th"], ["StartTag", "http://www.w3.org/1999/xhtml", "td", {}]],
+ "expected": ["<td>", "</th>"]
+},
+
+{"description": "th end-tag followed by end-tag",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml", "th"], ["EndTag", "http://www.w3.org/1999/xhtml", "foo"]],
+ "expected": ["</foo>"]
+},
+
+{"description": "th end-tag at EOF",
+ "input": [["EndTag", "http://www.w3.org/1999/xhtml"    , "th"]],
+ "expected": [""]
+}
+
+]}

html5lib/tests/testdata/serializer/options.test

@@ -0,0 +1,60 @@
+{"tests":[
+
+{"description": "quote_char=\"'\"",
+ "options": {"quote_char": "'"},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": null, "name": "title", "value": "test 'with' quote_char"}]]],
+ "expected": ["<span title='test &#39;with&#39; quote_char'>"]
+},
+
+{"description": "quote_attr_values=true",
+ "options": {"quote_attr_values": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "button", [{"namespace": null, "name": "disabled", "value" :"disabled"}]]],
+ "expected": ["<button disabled>"],
+ "xhtml":    ["<button disabled=\"disabled\">"]
+},
+
+{"description": "quote_attr_values=true with irrelevant",
+ "options": {"quote_attr_values": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "div", [{"namespace": null, "name": "irrelevant", "value" :"irrelevant"}]]],
+ "expected": ["<div irrelevant>"],
+ "xhtml":    ["<div irrelevant=\"irrelevant\">"]
+},
+
+{"description": "use_trailing_solidus=true with void element",
+ "options": {"use_trailing_solidus": true},
+ "input": [["EmptyTag", "img", {}]],
+ "expected": ["<img />"]
+},
+
+{"description": "use_trailing_solidus=true with non-void element",
+ "options": {"use_trailing_solidus": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "div", {}]],
+ "expected": ["<div>"]
+},
+
+{"description": "minimize_boolean_attributes=false",
+ "options": {"minimize_boolean_attributes": false},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "div", [{"namespace": null, "name": "irrelevant", "value" :"irrelevant"}]]],
+ "expected": ["<div irrelevant=irrelevant>"],
+ "xhtml":    ["<div irrelevant=\"irrelevant\">"]
+},
+
+{"description": "minimize_boolean_attributes=false with empty value",
+ "options": {"minimize_boolean_attributes": false},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "div", [{"namespace": null, "name": "irrelevant", "value" :""}]]],
+ "expected": ["<div irrelevant=\"\">"]
+},
+
+{"description": "escape less than signs in attribute values",
+ "options": {"escape_lt_in_attrs": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "a", [{"namespace": null, "name": "title", "value": "a<b>c&d"}]]],
+ "expected": ["<a title=\"a&lt;b>c&amp;d\">"]
+},
+
+{"description": "rcdata",
+ "options": {"escape_rcdata": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "script", {}], ["Characters", "a<b>c&d"]],
+ "expected": ["<script>a&lt;b&gt;c&amp;d"]
+}
+
+]}

html5lib/tests/testdata/serializer/whitespace.test

@@ -0,0 +1,51 @@
+{"tests": [
+
+{"description": "bare text with leading spaces",
+ "options": {"strip_whitespace": true},
+ "input": [["Characters", "\t\r\n\u000C foo"]],
+ "expected": [" foo"]
+},
+
+{"description": "bare text with trailing spaces",
+ "options": {"strip_whitespace": true},
+ "input": [["Characters", "foo \t\r\n\u000C"]],
+ "expected": ["foo "]
+},
+
+{"description": "bare text with inner spaces",
+ "options": {"strip_whitespace": true},
+ "input": [["Characters", "foo \t\r\n\u000C bar"]],
+ "expected": ["foo bar"]
+},
+
+{"description": "text within <pre>",
+ "options": {"strip_whitespace": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "pre", {}], ["Characters", "\t\r\n\u000C foo \t\r\n\u000C bar \t\r\n\u000C"], ["EndTag", "http://www.w3.org/1999/xhtml", "pre"]],
+ "expected": ["<pre>\t\r\n\u000C foo \t\r\n\u000C bar \t\r\n\u000C</pre>"]
+},
+
+{"description": "text within <pre>, with inner markup",
+ "options": {"strip_whitespace": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "pre", {}], ["Characters", "\t\r\n\u000C fo"], ["StartTag", "http://www.w3.org/1999/xhtml", "span", {}], ["Characters", "o \t\r\n\u000C b"], ["EndTag", "http://www.w3.org/1999/xhtml", "span"], ["Characters", "ar \t\r\n\u000C"], ["EndTag", "http://www.w3.org/1999/xhtml", "pre"]],
+ "expected": ["<pre>\t\r\n\u000C fo<span>o \t\r\n\u000C b</span>ar \t\r\n\u000C</pre>"]
+},
+
+{"description": "text within <textarea>",
+ "options": {"strip_whitespace": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "textarea", {}], ["Characters", "\t\r\n\u000C foo \t\r\n\u000C bar \t\r\n\u000C"], ["EndTag", "http://www.w3.org/1999/xhtml", "textarea"]],
+ "expected": ["<textarea>\t\r\n\u000C foo \t\r\n\u000C bar \t\r\n\u000C</textarea>"]
+},
+
+{"description": "text within <script>",
+ "options": {"strip_whitespace": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "script", {}], ["Characters", "\t\r\n\u000C foo \t\r\n\u000C bar \t\r\n\u000C"], ["EndTag", "http://www.w3.org/1999/xhtml", "script"]],
+ "expected": ["<script>\t\r\n\u000C foo \t\r\n\u000C bar \t\r\n\u000C</script>"]
+},
+
+{"description": "text within <style>",
+ "options": {"strip_whitespace": true},
+ "input": [["StartTag", "http://www.w3.org/1999/xhtml", "style", {}], ["Characters", "\t\r\n\u000C foo \t\r\n\u000C bar \t\r\n\u000C"], ["EndTag", "http://www.w3.org/1999/xhtml", "style"]],
+ "expected": ["<style>\t\r\n\u000C foo \t\r\n\u000C bar \t\r\n\u000C</style>"]
+}
+
+]}

html5lib/tests/testdata/sniffer/htmlOrFeed.json

@@ -0,0 +1,43 @@
+[
+    {"type": "text/html", "input": ""},
+    {"type": "text/html", "input": "<!---->"},
+    {"type": "text/html", "input": "<!--asdfaslkjdf;laksjdf as;dkfjsd-->"},
+    {"type": "text/html", "input": "<!"},
+    {"type": "text/html", "input": "\t"},
+    {"type": "text/html", "input": "<!>"},
+    {"type": "text/html", "input": "<?"},
+    {"type": "text/html", "input": "<??>"},
+    {"type": "application/rss+xml", "input": "<rss"},
+    {"type": "application/atom+xml", "input": "<feed"},
+    {"type": "text/html", "input": "<html"},
+    {"type": "text/html", "input": "<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML 2.0//EN\">\n<html><head>\n<title>302 Found</title>\n</head><body>\n<h1>Found</h1>\n<p>The document has moved <a href=\"http://feeds.feedburner.com/gofug\">here</a>.</p>\n</body></html>\n"},
+    {"type": "text/html", "input": "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 Transitional//EN\">\r\n<HTML><HEAD>\r\n   <link rel=\"stylesheet\" type=\"text/css\" href=\"http://cache.blogads.com/289619328/feed.css\" /><link rel=\"stylesheet\" type=\"text/css\" href=\"http://cache.blogads.com/431602649/feed.css\" />\r\n<link rel=\"stylesheet\" type=\"text/css\" href=\"http://cache.blogads.com/382549546/feed.css\" />\r\n<link rel=\"stylesheet\" type=\"text/css\" href=\"http://cache.blogads.com/314618017/feed.css\" /><META http-equiv=\"expires\" content="},
+    {"type": "text/html", "input": "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">\r\n<html>\r\n<head>\r\n<title>Xiaxue - Chicken pie blogger.</title><meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\"><style type=\"text/css\">\r\n<style type=\"text/css\">\r\n<!--\r\nbody {\r\n background-color: #FFF2F2;\r\n}\r\n.style1 {font-family: Georgia, \"Times New Roman\", Times, serif}\r\n.style2 {\r\n color: #8a567c;\r\n font-size: 14px;\r\n font-family: Georgia, \"Times New Roman\", Times, serif;\r\n}\r"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\"><html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\r\n<head> \r\n<title>Google Operating System</title>\r\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\r\n<meta name=\"Description\" content=\"Unofficial news and tips about Google. A blog that watches Google's latest developments and the attempts to move your operating system online.\" />\r\n<meta name=\"generator\" c"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\"><html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\r\n<head>\r\n  <title>Assimilated Press</title>  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\r\n<meta name=\"MSSmartTagsPreventParsing\" content=\"true\" />\r\n<meta name=\"generator\" content=\"Blogger\" />\r\n<link rel=\"alternate\" type=\"application/atom+xml\" title=\"Assimilated Press - Atom\" href=\"http://assimila"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\"><html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\r\n<head>\r\n  <title>PostSecret</title>\r\n<META name=\"keywords\" Content=\"secrets, postcard, secret, postcards, postsecret, postsecrets,online confessional, post secret, post secrets, artomatic, post a secret\"><META name=\"discription\" Content=\"See a Secret...Share a Secret\">  <meta http-equiv=\"Content-Type\" content=\"te"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n<html xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'>\n  <head>\n    \n  <meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/>\n  <meta content='true' name='MSSmartTagsPreventParsing'/>\n  <meta content='blogger' name='generator'/>\n  <link rel=\"alternate\" typ"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\" dir=\"ltr\" lang=\"ja\">\n<head profile=\"http://gmpg.org/xfn/11\"> \n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />  \n<title> CMS Lever</title><link rel=\"stylesheet\" type=\"text/css\" media=\"screen\" href=\"http://s.wordpress.com/wp-content/themes/pub/twenty-eight/2813.css\"/>\n<link rel=\"alternate\" type=\"application/rss+xml\" title=\"RSS 2.0\" h"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n    \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\" dir=\"ltr\" lang=\"en\"><head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n<title> Park Avenue Peerage</title>\t<meta name=\"generator\" content=\"WordPress.com\" />\t<!-- feeds -->\n\t<link rel=\"alternate\" type=\"application/rss+xml\" title=\"RSS 2.0\" href=\"http://parkavenuepeerage.wordpress.com/feed/\" />\t<link rel=\"pingback\" href="},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n    \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\" dir=\"ltr\" lang=\"ja\"><head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n<title> \u884c\u96f2\u6d41\u6c34 -like a floating clouds and running water-</title>\t<meta name=\"generator\" content=\"WordPress.com\" />\t<!-- feeds -->\n\t<link rel=\"alternate\" type=\"application/rss+xml\" title=\"RSS 2.0\" href=\"http://shw4.wordpress.com/feed/\" />\t<li"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\"><html xmlns=\"http://www.w3.org/1999/xhtml\">\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />\n<meta name=\"generator\" content=\"http://www.typepad.com/\" />\n<title>Go Fug Yourself</title><link rel=\"stylesheet\" href=\"http://gofugyourself.typepad.com/go_fug_yourself/styles.css\" type=\"text/css\" />\n<link rel=\"alternate\" type=\"application/atom+xml\" title=\"Atom\" "},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\" dir=\"ltr\" lang=\"en\"><head profile=\"http://gmpg.org/xfn/11\">\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" /><title> Ladies&#8230;</title><meta name=\"generator\" content=\"WordPress.com\" /> <!-- leave this for stats --><link rel=\"stylesheet\" href=\"http://s.wordpress.com/wp-content/themes/default/style.css?1\" type=\"tex"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\r\n<html xmlns=\"http://www.w3.org/1999/xhtml\">\r\n<head>\r\n  <title>The Sartorialist</title>  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\r\n<meta name=\"MSSmartTagsPreventParsing\" content=\"true\" />\r\n<meta name=\"generator\" content=\"Blogger\" />\r\n<link rel=\"alternate\" type=\"application/atom+xml\" title=\"The Sartorialist - Atom\" href=\"http://thesartorialist.blogspot"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \n     \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\"><html  xmlns=\"http://www.w3.org/1999/xhtml\" lang=\"en\">\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=ISO-8859-1\" />\n<meta name=\"generator\" content=\"http://www.typepad.com/\" />\n<title>Creating Passionate Users</title><link rel=\"stylesheet\" href=\"http://headrush.typepad.com/creating_passionate_users/styles.css\" type=\"text/css\" />\n<link rel=\"alternate\" type"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\"\n\t\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\" id=\"sixapart-standard\">\n<head>\n\t<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />\n\t<meta name=\"generator\" content=\"http://www.typepad.com/\" />\n\t\n\t\n    <meta name=\"keywords\" content=\"marketing, blog, seth, ideas, respect, permission\" />\n    <meta name=\"description\" content=\"Seth Godin's riffs on marketing, respect, and the "},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\"\n\t\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\" id=\"sixapart-standard\">\n<head>\n\t<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />\n\t<meta name=\"generator\" content=\"http://www.typepad.com/\" />\n\t\n\t\n    \n    <meta name=\"description\" content=\" Western Civilization hangs in the balance. This blog is part of the solution,the cure. Get your heads out of the sand and Fight the G"},
+    {"type": "text/html", "input": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1//EN\" \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\" dir=\"ltr\" lang=\"en\">\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=pahrefhttpwwwfeedburnercomtarget_blankimgsrchttpwwwfeedburnercomfbimagespubpowered_by_fbgifaltPoweredbyFeedBurnerstyleborder0ap\" />\n<title> From Under the Rotunda</title>\n<link rel=\"stylesheet\" href=\"http://s.wordpress.com/wp-content/themes/pub/andreas04/style.css\" type=\"text/css\""},
+    {"type": "application/atom+xml", "input": "<?xml version='1.0' encoding='UTF-8'?><?xml-stylesheet href=\"http://www.blogger.com/styles/atom.css\" type=\"text/css\"?><feed xmlns='http://www.w3.org/2005/Atom' xmlns:openSearch='http://a9.com/-/spec/opensearchrss/1.0/'><id>tag:blogger.com,1999:blog-10861780</id><updated>2007-07-27T12:38:50.888-07:00</updated><title type='text'>Official Google Blog</title><link rel='alternate' type='text/html' href='http://googleblog.blogspot.com/'/><link rel='next' type='application/atom+xml' href='http://googleblog.blogs"},
+    {"type": "application/rss+xml", "input": "<?xml version='1.0' encoding='UTF-8'?><rss xmlns:atom='http://www.w3.org/2005/Atom' xmlns:openSearch='http://a9.com/-/spec/opensearchrss/1.0/' version='2.0'><channel><atom:id>tag:blogger.com,1999:blog-10861780</atom:id><lastBuildDate>Fri, 27 Jul 2007 19:38:50 +0000</lastBuildDate><title>Official Google Blog</title><description/><link>http://googleblog.blogspot.com/</link><managingEditor>Eric Case</managingEditor><generator>Blogger</generator><openSearch:totalResults>729</openSearch:totalResults><openSearc"},
+    {"type": "application/rss+xml", "input": "<?xml version=\"1.0\" encoding=\"pahrefhttpwwwfeedburnercomtarget_blankimgsrchttpwwwfeedburnercomfbimagespubpowered_by_fbgifaltPoweredbyFeedBurnerstyleborder0ap\"?>\n<!-- generator=\"wordpress/MU\" -->\n<rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txmlns:wfw=\"http://wellformedweb.org/CommentAPI/\"\n\txmlns:dc=\"http://purl.org/dc/elements/1.1/\"\n\t><channel>\n\t<title>From Under the Rotunda</title>\n\t<link>http://dannybernardi.wordpress.com</link>\n\t<description>The Monographs of Danny Ber"},
+    {"type": "application/rss+xml", "input": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!-- generator=\"wordpress/MU\" -->\n<rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txmlns:wfw=\"http://wellformedweb.org/CommentAPI/\"\n\txmlns:dc=\"http://purl.org/dc/elements/1.1/\"\n\t><channel>\n\t<title>CMS Lever</title>\n\t<link>http://kanaguri.wordpress.com</link>\n\t<description>CMS\u306e\u6c17\u306b\u306a\u3063\u305f\u3053\u3068</description>\n\t<pubDate>Wed, 18 Jul 2007 21:26:22 +0000</pubDate>\n\t<generator>http://wordpress.org/?v=MU</generator>\n\t<language>ja</languag"},
+    {"type": "application/atom+xml", "input": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<feed xmlns=\"http://www.w3.org/2005/Atom\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:thr=\"http://purl.org/syndication/thread/1.0\">\n    <title>Atlas Shrugs</title>\n    <link rel=\"self\" type=\"application/atom+xml\" href=\"http://atlasshrugs2000.typepad.com/atlas_shrugs/atom.xml\" />\n    <link rel=\"alternate\" type=\"text/html\" href=\"http://atlasshrugs2000.typepad.com/atlas_shrugs/\" />\n    <id>tag:typepad.com,2003:weblog-132946</id>\n    <updated>2007-08-15T16:07:34-04"},
+    {"type": "application/atom+xml", "input": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/atom10full.xsl\" type=\"text/xsl\" media=\"screen\"?><?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/itemcontent.css\" type=\"text/css\" media=\"screen\"?><feed xmlns=\"http://www.w3.org/2005/Atom\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:thr=\"http://purl.org/syndication/thread/1.0\" xmlns:feedburner=\"http://rssnamespace.org/feedburner/ext/1.0\">\r\n    <title>Creating Passionate Users</title>\r\n  "},
+    {"type": "application/atom+xml", "input": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/atom10full.xsl\" type=\"text/xsl\" media=\"screen\"?><?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/itemcontent.css\" type=\"text/css\" media=\"screen\"?><feed xmlns=\"http://www.w3.org/2005/Atom\" xmlns:feedburner=\"http://rssnamespace.org/feedburner/ext/1.0\">\r\n    <title>Seth's Blog</title>\r\n    <link rel=\"alternate\" type=\"text/html\" href=\"http://sethgodin.typepad.com/seths_blog/\" />\r\n    <link rel=\"s"},
+    {"type": "application/atom+xml", "input": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/atom10full.xsl\" type=\"text/xsl\" media=\"screen\"?><?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/itemcontent.css\" type=\"text/css\" media=\"screen\"?><feed xmlns=\"http://www.w3.org/2005/Atom\" xmlns:openSearch=\"http://a9.com/-/spec/opensearchrss/1.0/\" xmlns:feedburner=\"http://rssnamespace.org/feedburner/ext/1.0\"><id>tag:blogger.com,1999:blog-32454861</id><updated>2007-07-31T21:44:09.867+02:00</upd"},
+    {"type": "application/atom+xml", "input": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/atomfull.xsl\" type=\"text/xsl\" media=\"screen\"?><?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/itemcontent.css\" type=\"text/css\" media=\"screen\"?><feed xmlns=\"http://purl.org/atom/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:feedburner=\"http://rssnamespace.org/feedburner/ext/1.0\" version=\"0.3\">\r\n  <title>Go Fug Yourself</title>\r\n  <link rel=\"alternate\" type=\"text/html\" href=\"http://go"},
+    {"type": "application/rss+xml", "input": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/rss2full.xsl\" type=\"text/xsl\" media=\"screen\"?><?xml-stylesheet href=\"http://feeds.feedburner.com/~d/styles/itemcontent.css\" type=\"text/css\" media=\"screen\"?><rss xmlns:creativeCommons=\"http://backend.userland.com/creativeCommonsRssModule\" xmlns:feedburner=\"http://rssnamespace.org/feedburner/ext/1.0\" version=\"2.0\"><channel><title>Google Operating System</title><link>http://googlesystem.blogspot.com/</link>"},
+    {"type": "application/rss+xml", "input": "<?xml version=\"1.0\" encoding=\"\"?>\n<!-- generator=\"wordpress/MU\" -->\n<rss version=\"2.0\"\n\txmlns:content=\"http://purl.org/rss/1.0/modules/content/\"\n\txmlns:wfw=\"http://wellformedweb.org/CommentAPI/\"\n\txmlns:dc=\"http://purl.org/dc/elements/1.1/\"\n\t><channel>\n\t<title>Nunublog</title>\n\t<link>http://nunubh.wordpress.com</link>\n\t<description>Just Newbie Blog!</description>\n\t<pubDate>Mon, 09 Jul 2007 18:54:09 +0000</pubDate>\n\t<generator>http://wordpress.org/?v=MU</generator>\n\t<language>id</language>\n\t\t\t<item>\n\t\t<ti"},
+    {"type": "text/html", "input": "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\r\n<HEAD>\r\n<TITLE>Design*Sponge</TITLE><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\r\n<meta name=\"MSSmartTagsPreventParsing\" content=\"true\" />\r\n<meta name=\"generator\" content=\"Blogger\" />\r\n<link rel=\"alternate\" type=\"application/atom+xml\" title=\"Design*Sponge - Atom\" href=\"http://designsponge.blogspot.com/feeds/posts/default\" />\r\n<link rel=\"alternate\" type=\"application/rss+xml\" title=\"Design*Sponge - RSS\" href="},
+    {"type": "text/html", "input": "<HTML>\n<HEAD>\n<TITLE>Moved Temporarily</TITLE>\n</HEAD>\n<BODY BGCOLOR=\"#FFFFFF\" TEXT=\"#000000\">\n<H1>Moved Temporarily</H1>\nThe document has moved <A HREF=\"http://feeds.feedburner.com/thesecretdiaryofstevejobs\">here</A>.\n</BODY>\n</HTML>\n"}
+]

html5lib/tests/testdata/tokenizer/contentModelFlags.test

@@ -0,0 +1,75 @@
+{"tests": [
+
+{"description":"PLAINTEXT content model flag",
+"initialStates":["PLAINTEXT state"],
+"lastStartTag":"plaintext",
+"input":"<head>&body;",
+"output":[["Character", "<head>&body;"]]},
+
+{"description":"End tag closing RCDATA or RAWTEXT",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo</xmp>",
+"output":[["Character", "foo"], ["EndTag", "xmp"]]},
+
+{"description":"End tag closing RCDATA or RAWTEXT (case-insensitivity)",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo</xMp>",
+"output":[["Character", "foo"], ["EndTag", "xmp"]]},
+
+{"description":"End tag closing RCDATA or RAWTEXT (ending with space)",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo</xmp ",
+"output":[["Character", "foo"], "ParseError"]},
+
+{"description":"End tag closing RCDATA or RAWTEXT (ending with EOF)",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo</xmp",
+"output":[["Character", "foo</xmp"]]},
+
+{"description":"End tag closing RCDATA or RAWTEXT (ending with slash)",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo</xmp/",
+"output":[["Character", "foo"], "ParseError"]},
+
+{"description":"End tag not closing RCDATA or RAWTEXT (ending with left-angle-bracket)",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo</xmp<",
+"output":[["Character", "foo</xmp<"]]},
+
+{"description":"End tag with incorrect name in RCDATA or RAWTEXT",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"</foo>bar</xmp>",
+"output":[["Character", "</foo>bar"], ["EndTag", "xmp"]]},
+
+{"description":"End tag with incorrect name in RCDATA or RAWTEXT (starting like correct name)",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"</foo>bar</xmpaar>",
+"output":[["Character", "</foo>bar</xmpaar>"]]},
+
+{"description":"End tag closing RCDATA or RAWTEXT, switching back to PCDATA",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo</xmp></baz>",
+"output":[["Character", "foo"], ["EndTag", "xmp"], ["EndTag", "baz"]]},
+
+{"description":"RAWTEXT w/ something looking like an entity",
+"initialStates":["RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"&foo;",
+"output":[["Character", "&foo;"]]},
+
+{"description":"RCDATA w/ an entity",
+"initialStates":["RCDATA state"],
+"lastStartTag":"textarea",
+"input":"&lt;",
+"output":[["Character", "<"]]}
+
+]}

html5lib/tests/testdata/tokenizer/domjs.test

@@ -0,0 +1,90 @@
+{
+    "tests": [
+        {
+            "description":"CR in bogus comment state",
+            "input":"<?\u000d",
+            "output":["ParseError", ["Comment", "?\u000a"]]
+        },
+        {
+            "description":"CRLF in bogus comment state",
+            "input":"<?\u000d\u000a",
+            "output":["ParseError", ["Comment", "?\u000a"]]
+        },
+        {
+            "description":"NUL in RCDATA and RAWTEXT",
+            "doubleEscaped":true,
+            "initialStates":["RCDATA state", "RAWTEXT state"],
+            "input":"\\u0000",
+            "output":["ParseError", ["Character", "\\uFFFD"]]
+        },
+        {
+            "description":"skip first BOM but not later ones",
+            "input":"\uFEFFfoo\uFEFFbar",
+            "output":[["Character", "foo\uFEFFbar"]]
+        },
+        {
+            "description":"Non BMP-charref in in RCDATA",
+            "initialStates":["RCDATA state"],
+            "input":"&NotEqualTilde;",
+            "output":[["Character", "\u2242\u0338"]]
+        },
+        {
+            "description":"Bad charref in in RCDATA",
+            "initialStates":["RCDATA state"],
+            "input":"&NotEqualTild;",
+            "output":["ParseError", ["Character", "&NotEqualTild;"]]
+        },
+        {
+            "description":"lowercase endtags in RCDATA and RAWTEXT",
+            "initialStates":["RCDATA state", "RAWTEXT state"],
+            "lastStartTag":"xmp",
+            "input":"</XMP>",
+            "output":[["EndTag","xmp"]]
+        },
+        {
+            "description":"bad endtag in RCDATA and RAWTEXT",
+            "initialStates":["RCDATA state", "RAWTEXT state"],
+            "lastStartTag":"xmp",
+            "input":"</ XMP>",
+            "output":[["Character","</ XMP>"]]
+        },
+        {
+            "description":"bad endtag in RCDATA and RAWTEXT",
+            "initialStates":["RCDATA state", "RAWTEXT state"],
+            "lastStartTag":"xmp",
+            "input":"</xm>",
+            "output":[["Character","</xm>"]]
+        },
+        {
+            "description":"bad endtag in RCDATA and RAWTEXT",
+            "initialStates":["RCDATA state", "RAWTEXT state"],
+            "lastStartTag":"xmp",
+            "input":"</xm ",
+            "output":[["Character","</xm "]]
+        },
+        {
+            "description":"bad endtag in RCDATA and RAWTEXT",
+            "initialStates":["RCDATA state", "RAWTEXT state"],
+            "lastStartTag":"xmp",
+            "input":"</xm/",
+            "output":[["Character","</xm/"]]
+        },
+        {
+            "description":"Non BMP-charref in attribute",
+            "input":"<p id=\"&NotEqualTilde;\">",
+            "output":[["StartTag", "p", {"id":"\u2242\u0338"}]]
+        },
+        {
+            "description":"--!NUL in comment ",
+            "doubleEscaped":true,
+            "input":"<!----!\\u0000-->",
+            "output":["ParseError", ["Comment", "--!\\uFFFD"]]
+        },
+        {
+            "description":"space EOF after doctype ",
+            "input":"<!DOCTYPE html ",
+            "output":["ParseError", ["DOCTYPE", "html", null, null , false]]
+        }
+
+    ]
+}

html5lib/tests/testdata/tokenizer/entities.test

@@ -0,0 +1,283 @@
+{"tests": [
+
+{"description": "Undefined named entity in attribute value ending in semicolon and whose name starts with a known entity name.",
+"input":"<h a='&noti;'>",
+"output": ["ParseError", ["StartTag", "h", {"a": "&noti;"}]]},
+
+{"description": "Entity name followed by the equals sign in an attribute value.",
+"input":"<h a='&lang='>",
+"output": ["ParseError", ["StartTag", "h", {"a": "&lang="}]]},
+
+{"description": "CR as numeric entity",
+"input":"&#013;",
+"output": ["ParseError", ["Character", "\r"]]},
+
+{"description": "CR as hexadecimal numeric entity",
+"input":"&#x00D;",
+"output": ["ParseError", ["Character", "\r"]]},
+
+{"description": "Windows-1252 EURO SIGN numeric entity.",
+"input":"&#0128;",
+"output": ["ParseError", ["Character", "\u20AC"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR numeric entity.",
+"input":"&#0129;",
+"output": ["ParseError", ["Character", "\u0081"]]},
+
+{"description": "Windows-1252 SINGLE LOW-9 QUOTATION MARK numeric entity.",
+"input":"&#0130;",
+"output": ["ParseError", ["Character", "\u201A"]]},
+
+{"description": "Windows-1252 LATIN SMALL LETTER F WITH HOOK numeric entity.",
+"input":"&#0131;",
+"output": ["ParseError", ["Character", "\u0192"]]},
+
+{"description": "Windows-1252 DOUBLE LOW-9 QUOTATION MARK numeric entity.",
+"input":"&#0132;",
+"output": ["ParseError", ["Character", "\u201E"]]},
+
+{"description": "Windows-1252 HORIZONTAL ELLIPSIS numeric entity.",
+"input":"&#0133;",
+"output": ["ParseError", ["Character", "\u2026"]]},
+
+{"description": "Windows-1252 DAGGER numeric entity.",
+"input":"&#0134;",
+"output": ["ParseError", ["Character", "\u2020"]]},
+
+{"description": "Windows-1252 DOUBLE DAGGER numeric entity.",
+"input":"&#0135;",
+"output": ["ParseError", ["Character", "\u2021"]]},
+
+{"description": "Windows-1252 MODIFIER LETTER CIRCUMFLEX ACCENT numeric entity.",
+"input":"&#0136;",
+"output": ["ParseError", ["Character", "\u02C6"]]},
+
+{"description": "Windows-1252 PER MILLE SIGN numeric entity.",
+"input":"&#0137;",
+"output": ["ParseError", ["Character", "\u2030"]]},
+
+{"description": "Windows-1252 LATIN CAPITAL LETTER S WITH CARON numeric entity.",
+"input":"&#0138;",
+"output": ["ParseError", ["Character", "\u0160"]]},
+
+{"description": "Windows-1252 SINGLE LEFT-POINTING ANGLE QUOTATION MARK numeric entity.",
+"input":"&#0139;",
+"output": ["ParseError", ["Character", "\u2039"]]},
+
+{"description": "Windows-1252 LATIN CAPITAL LIGATURE OE numeric entity.",
+"input":"&#0140;",
+"output": ["ParseError", ["Character", "\u0152"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR numeric entity.",
+"input":"&#0141;",
+"output": ["ParseError", ["Character", "\u008D"]]},
+
+{"description": "Windows-1252 LATIN CAPITAL LETTER Z WITH CARON numeric entity.",
+"input":"&#0142;",
+"output": ["ParseError", ["Character", "\u017D"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR numeric entity.",
+"input":"&#0143;",
+"output": ["ParseError", ["Character", "\u008F"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR numeric entity.",
+"input":"&#0144;",
+"output": ["ParseError", ["Character", "\u0090"]]},
+
+{"description": "Windows-1252 LEFT SINGLE QUOTATION MARK numeric entity.",
+"input":"&#0145;",
+"output": ["ParseError", ["Character", "\u2018"]]},
+
+{"description": "Windows-1252 RIGHT SINGLE QUOTATION MARK numeric entity.",
+"input":"&#0146;",
+"output": ["ParseError", ["Character", "\u2019"]]},
+
+{"description": "Windows-1252 LEFT DOUBLE QUOTATION MARK numeric entity.",
+"input":"&#0147;",
+"output": ["ParseError", ["Character", "\u201C"]]},
+
+{"description": "Windows-1252 RIGHT DOUBLE QUOTATION MARK numeric entity.",
+"input":"&#0148;",
+"output": ["ParseError", ["Character", "\u201D"]]},
+
+{"description": "Windows-1252 BULLET numeric entity.",
+"input":"&#0149;",
+"output": ["ParseError", ["Character", "\u2022"]]},
+
+{"description": "Windows-1252 EN DASH numeric entity.",
+"input":"&#0150;",
+"output": ["ParseError", ["Character", "\u2013"]]},
+
+{"description": "Windows-1252 EM DASH numeric entity.",
+"input":"&#0151;",
+"output": ["ParseError", ["Character", "\u2014"]]},
+
+{"description": "Windows-1252 SMALL TILDE numeric entity.",
+"input":"&#0152;",
+"output": ["ParseError", ["Character", "\u02DC"]]},
+
+{"description": "Windows-1252 TRADE MARK SIGN numeric entity.",
+"input":"&#0153;",
+"output": ["ParseError", ["Character", "\u2122"]]},
+
+{"description": "Windows-1252 LATIN SMALL LETTER S WITH CARON numeric entity.",
+"input":"&#0154;",
+"output": ["ParseError", ["Character", "\u0161"]]},
+
+{"description": "Windows-1252 SINGLE RIGHT-POINTING ANGLE QUOTATION MARK numeric entity.",
+"input":"&#0155;",
+"output": ["ParseError", ["Character", "\u203A"]]},
+
+{"description": "Windows-1252 LATIN SMALL LIGATURE OE numeric entity.",
+"input":"&#0156;",
+"output": ["ParseError", ["Character", "\u0153"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR numeric entity.",
+"input":"&#0157;",
+"output": ["ParseError", ["Character", "\u009D"]]},
+
+{"description": "Windows-1252 EURO SIGN hexadecimal numeric entity.",
+"input":"&#x080;",
+"output": ["ParseError", ["Character", "\u20AC"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR hexadecimal numeric entity.",
+"input":"&#x081;",
+"output": ["ParseError", ["Character", "\u0081"]]},
+
+{"description": "Windows-1252 SINGLE LOW-9 QUOTATION MARK hexadecimal numeric entity.",
+"input":"&#x082;",
+"output": ["ParseError", ["Character", "\u201A"]]},
+
+{"description": "Windows-1252 LATIN SMALL LETTER F WITH HOOK hexadecimal numeric entity.",
+"input":"&#x083;",
+"output": ["ParseError", ["Character", "\u0192"]]},
+
+{"description": "Windows-1252 DOUBLE LOW-9 QUOTATION MARK hexadecimal numeric entity.",
+"input":"&#x084;",
+"output": ["ParseError", ["Character", "\u201E"]]},
+
+{"description": "Windows-1252 HORIZONTAL ELLIPSIS hexadecimal numeric entity.",
+"input":"&#x085;",
+"output": ["ParseError", ["Character", "\u2026"]]},
+
+{"description": "Windows-1252 DAGGER hexadecimal numeric entity.",
+"input":"&#x086;",
+"output": ["ParseError", ["Character", "\u2020"]]},
+
+{"description": "Windows-1252 DOUBLE DAGGER hexadecimal numeric entity.",
+"input":"&#x087;",
+"output": ["ParseError", ["Character", "\u2021"]]},
+
+{"description": "Windows-1252 MODIFIER LETTER CIRCUMFLEX ACCENT hexadecimal numeric entity.",
+"input":"&#x088;",
+"output": ["ParseError", ["Character", "\u02C6"]]},
+
+{"description": "Windows-1252 PER MILLE SIGN hexadecimal numeric entity.",
+"input":"&#x089;",
+"output": ["ParseError", ["Character", "\u2030"]]},
+
+{"description": "Windows-1252 LATIN CAPITAL LETTER S WITH CARON hexadecimal numeric entity.",
+"input":"&#x08A;",
+"output": ["ParseError", ["Character", "\u0160"]]},
+
+{"description": "Windows-1252 SINGLE LEFT-POINTING ANGLE QUOTATION MARK hexadecimal numeric entity.",
+"input":"&#x08B;",
+"output": ["ParseError", ["Character", "\u2039"]]},
+
+{"description": "Windows-1252 LATIN CAPITAL LIGATURE OE hexadecimal numeric entity.",
+"input":"&#x08C;",
+"output": ["ParseError", ["Character", "\u0152"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR hexadecimal numeric entity.",
+"input":"&#x08D;",
+"output": ["ParseError", ["Character", "\u008D"]]},
+
+{"description": "Windows-1252 LATIN CAPITAL LETTER Z WITH CARON hexadecimal numeric entity.",
+"input":"&#x08E;",
+"output": ["ParseError", ["Character", "\u017D"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR hexadecimal numeric entity.",
+"input":"&#x08F;",
+"output": ["ParseError", ["Character", "\u008F"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR hexadecimal numeric entity.",
+"input":"&#x090;",
+"output": ["ParseError", ["Character", "\u0090"]]},
+
+{"description": "Windows-1252 LEFT SINGLE QUOTATION MARK hexadecimal numeric entity.",
+"input":"&#x091;",
+"output": ["ParseError", ["Character", "\u2018"]]},
+
+{"description": "Windows-1252 RIGHT SINGLE QUOTATION MARK hexadecimal numeric entity.",
+"input":"&#x092;",
+"output": ["ParseError", ["Character", "\u2019"]]},
+
+{"description": "Windows-1252 LEFT DOUBLE QUOTATION MARK hexadecimal numeric entity.",
+"input":"&#x093;",
+"output": ["ParseError", ["Character", "\u201C"]]},
+
+{"description": "Windows-1252 RIGHT DOUBLE QUOTATION MARK hexadecimal numeric entity.",
+"input":"&#x094;",
+"output": ["ParseError", ["Character", "\u201D"]]},
+
+{"description": "Windows-1252 BULLET hexadecimal numeric entity.",
+"input":"&#x095;",
+"output": ["ParseError", ["Character", "\u2022"]]},
+
+{"description": "Windows-1252 EN DASH hexadecimal numeric entity.",
+"input":"&#x096;",
+"output": ["ParseError", ["Character", "\u2013"]]},
+
+{"description": "Windows-1252 EM DASH hexadecimal numeric entity.",
+"input":"&#x097;",
+"output": ["ParseError", ["Character", "\u2014"]]},
+
+{"description": "Windows-1252 SMALL TILDE hexadecimal numeric entity.",
+"input":"&#x098;",
+"output": ["ParseError", ["Character", "\u02DC"]]},
+
+{"description": "Windows-1252 TRADE MARK SIGN hexadecimal numeric entity.",
+"input":"&#x099;",
+"output": ["ParseError", ["Character", "\u2122"]]},
+
+{"description": "Windows-1252 LATIN SMALL LETTER S WITH CARON hexadecimal numeric entity.",
+"input":"&#x09A;",
+"output": ["ParseError", ["Character", "\u0161"]]},
+
+{"description": "Windows-1252 SINGLE RIGHT-POINTING ANGLE QUOTATION MARK hexadecimal numeric entity.",
+"input":"&#x09B;",
+"output": ["ParseError", ["Character", "\u203A"]]},
+
+{"description": "Windows-1252 LATIN SMALL LIGATURE OE hexadecimal numeric entity.",
+"input":"&#x09C;",
+"output": ["ParseError", ["Character", "\u0153"]]},
+
+{"description": "Windows-1252 REPLACEMENT CHAR hexadecimal numeric entity.",
+"input":"&#x09D;",
+"output": ["ParseError", ["Character", "\u009D"]]},
+
+{"description": "Windows-1252 LATIN SMALL LETTER Z WITH CARON hexadecimal numeric entity.",
+"input":"&#x09E;",
+"output": ["ParseError", ["Character", "\u017E"]]},
+
+{"description": "Windows-1252 LATIN CAPITAL LETTER Y WITH DIAERESIS hexadecimal numeric entity.",
+"input":"&#x09F;",
+"output": ["ParseError", ["Character", "\u0178"]]},
+
+{"description": "Decimal numeric entity followed by hex character a.",
+"input":"&#97a",
+"output": ["ParseError", ["Character", "aa"]]},
+
+{"description": "Decimal numeric entity followed by hex character A.",
+"input":"&#97A",
+"output": ["ParseError", ["Character", "aA"]]},
+
+{"description": "Decimal numeric entity followed by hex character f.",
+"input":"&#97f",
+"output": ["ParseError", ["Character", "af"]]},
+
+{"description": "Decimal numeric entity followed by hex character A.",
+"input":"&#97F",
+"output": ["ParseError", ["Character", "aF"]]}
+
+]}

html5lib/tests/testdata/tokenizer/escapeFlag.test

@@ -0,0 +1,33 @@
+{"tests": [
+
+{"description":"Commented close tag in RCDATA or RAWTEXT",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo<!--</xmp>--></xmp>",
+"output":[["Character", "foo<!--"], ["EndTag", "xmp"], ["Character", "-->"], ["EndTag", "xmp"]]},
+
+{"description":"Bogus comment in RCDATA or RAWTEXT",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo<!-->baz</xmp>",
+"output":[["Character", "foo<!-->baz"], ["EndTag", "xmp"]]},
+
+{"description":"End tag surrounded by bogus comment in RCDATA or RAWTEXT",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo<!--></xmp><!-->baz</xmp>",
+"output":[["Character", "foo<!-->"], ["EndTag", "xmp"], "ParseError", ["Comment", ""], ["Character", "baz"], ["EndTag", "xmp"]]},
+
+{"description":"Commented entities in RCDATA",
+"initialStates":["RCDATA state"],
+"lastStartTag":"xmp",
+"input":" &amp; <!-- &amp; --> &amp; </xmp>",
+"output":[["Character", " & <!-- & --> & "], ["EndTag", "xmp"]]},
+
+{"description":"Incorrect comment ending sequences in RCDATA or RAWTEXT",
+"initialStates":["RCDATA state", "RAWTEXT state"],
+"lastStartTag":"xmp",
+"input":"foo<!-- x --x>x-- >x--!>x--<></xmp>",
+"output":[["Character", "foo<!-- x --x>x-- >x--!>x--<>"], ["EndTag", "xmp"]]}
+
+]}

html5lib/tests/testdata/tokenizer/pendingSpecChanges.test

@@ -0,0 +1,7 @@
+{"tests": [
+
+{"description":"<!---- >",
+"input":"<!---- >",
+"output":["ParseError", "ParseError", ["Comment","-- >"]]}
+
+]}

html5lib/tests/testdata/tokenizer/test1.test

@@ -0,0 +1,196 @@
+{"tests": [
+
+{"description":"Correct Doctype lowercase",
+"input":"<!DOCTYPE html>",
+"output":[["DOCTYPE", "html", null, null, true]]},
+
+{"description":"Correct Doctype uppercase",
+"input":"<!DOCTYPE HTML>",
+"output":[["DOCTYPE", "html", null, null, true]]},
+
+{"description":"Correct Doctype mixed case",
+"input":"<!DOCTYPE HtMl>", 
+"output":[["DOCTYPE", "html", null, null, true]]},
+
+{"description":"Correct Doctype case with EOF",
+"input":"<!DOCTYPE HtMl", 
+"output":["ParseError", ["DOCTYPE", "html", null, null, false]]},
+
+{"description":"Truncated doctype start",
+"input":"<!DOC>", 
+"output":["ParseError", ["Comment", "DOC"]]},
+
+{"description":"Doctype in error",
+"input":"<!DOCTYPE foo>", 
+"output":[["DOCTYPE", "foo", null, null, true]]},
+
+{"description":"Single Start Tag",
+"input":"<h>",
+"output":[["StartTag", "h", {}]]},
+
+{"description":"Empty end tag",
+"input":"</>",
+"output":["ParseError"]},
+
+{"description":"Empty start tag",
+"input":"<>",
+"output":["ParseError", ["Character", "<>"]]},
+
+{"description":"Start Tag w/attribute",
+"input":"<h a='b'>",
+"output":[["StartTag", "h", {"a":"b"}]]},
+
+{"description":"Start Tag w/attribute no quotes",
+"input":"<h a=b>",
+"output":[["StartTag", "h", {"a":"b"}]]},
+
+{"description":"Start/End Tag",
+"input":"<h></h>",
+"output":[["StartTag", "h", {}], ["EndTag", "h"]]},
+
+{"description":"Two unclosed start tags",
+"input":"<p>One<p>Two",
+"output":[["StartTag", "p", {}], ["Character", "One"], ["StartTag", "p", {}], ["Character", "Two"]]},
+
+{"description":"End Tag w/attribute",
+"input":"<h></h a='b'>",
+"output":[["StartTag", "h", {}], "ParseError", ["EndTag", "h"]]},
+
+{"description":"Multiple atts",
+"input":"<h a='b' c='d'>",
+"output":[["StartTag", "h", {"a":"b", "c":"d"}]]},
+
+{"description":"Multiple atts no space",
+"input":"<h a='b'c='d'>",
+"output":["ParseError", ["StartTag", "h", {"a":"b", "c":"d"}]]},
+
+{"description":"Repeated attr",
+ "input":"<h a='b' a='d'>",
+ "output":["ParseError", ["StartTag", "h", {"a":"b"}]]},
+
+{"description":"Simple comment",
+ "input":"<!--comment-->",
+ "output":[["Comment", "comment"]]},
+
+{"description":"Comment, Central dash no space",
+ "input":"<!----->",
+ "output":["ParseError", ["Comment", "-"]]},
+
+{"description":"Comment, two central dashes",
+"input":"<!-- --comment -->",
+"output":["ParseError", ["Comment", " --comment "]]},
+
+{"description":"Unfinished comment",
+"input":"<!--comment",
+"output":["ParseError", ["Comment", "comment"]]},
+
+{"description":"Start of a comment",
+"input":"<!-",
+"output":["ParseError", ["Comment", "-"]]},
+
+{"description":"Short comment",
+ "input":"<!-->",
+ "output":["ParseError", ["Comment", ""]]},
+
+{"description":"Short comment two",
+ "input":"<!--->",
+ "output":["ParseError", ["Comment", ""]]},
+
+{"description":"Short comment three",
+ "input":"<!---->",
+ "output":[["Comment", ""]]},
+
+
+{"description":"Ampersand EOF",
+"input":"&",
+"output":[["Character", "&"]]},
+
+{"description":"Ampersand ampersand EOF",
+"input":"&&",
+"output":[["Character", "&&"]]},
+
+{"description":"Ampersand space EOF",
+"input":"& ",
+"output":[["Character", "& "]]},
+
+{"description":"Unfinished entity",
+"input":"&f",
+"output":["ParseError", ["Character", "&f"]]},
+
+{"description":"Ampersand, number sign",
+"input":"&#",
+"output":["ParseError", ["Character", "&#"]]},
+
+{"description":"Unfinished numeric entity",
+"input":"&#x",
+"output":["ParseError", ["Character", "&#x"]]},
+
+{"description":"Entity with trailing semicolon (1)",
+"input":"I'm &not;it",
+"output":[["Character","I'm \u00ACit"]]},
+
+{"description":"Entity with trailing semicolon (2)",
+"input":"I'm &notin;",
+"output":[["Character","I'm \u2209"]]},
+
+{"description":"Entity without trailing semicolon (1)",
+"input":"I'm &notit",
+"output":[["Character","I'm "], "ParseError", ["Character", "\u00ACit"]]},
+
+{"description":"Entity without trailing semicolon (2)",
+"input":"I'm &notin",
+"output":[["Character","I'm "], "ParseError", ["Character", "\u00ACin"]]},
+
+{"description":"Partial entity match at end of file",
+"input":"I'm &no",
+"output":[["Character","I'm "], "ParseError", ["Character", "&no"]]},
+
+{"description":"Non-ASCII character reference name",
+"input":"&\u00AC;",
+"output":["ParseError", ["Character", "&\u00AC;"]]},
+
+{"description":"ASCII decimal entity",
+"input":"&#0036;",
+"output":[["Character","$"]]},
+
+{"description":"ASCII hexadecimal entity",
+"input":"&#x3f;",
+"output":[["Character","?"]]},
+
+{"description":"Hexadecimal entity in attribute",
+"input":"<h a='&#x3f;'></h>",
+"output":[["StartTag", "h", {"a":"?"}], ["EndTag", "h"]]},
+
+{"description":"Entity in attribute without semicolon ending in x",
+"input":"<h a='&notx'>",
+"output":["ParseError", ["StartTag", "h", {"a":"&notx"}]]},
+
+{"description":"Entity in attribute without semicolon ending in 1",
+"input":"<h a='&not1'>",
+"output":["ParseError", ["StartTag", "h", {"a":"&not1"}]]},
+
+{"description":"Entity in attribute without semicolon ending in i",
+"input":"<h a='&noti'>",
+"output":["ParseError", ["StartTag", "h", {"a":"&noti"}]]},
+
+{"description":"Entity in attribute without semicolon",
+"input":"<h a='&COPY'>",
+"output":["ParseError", ["StartTag", "h", {"a":"\u00A9"}]]},
+
+{"description":"Unquoted attribute ending in ampersand",
+"input":"<s o=& t>",
+"output":[["StartTag","s",{"o":"&","t":""}]]},
+
+{"description":"Unquoted attribute at end of tag with final character of &, with tag followed by characters",
+"input":"<a a=a&>foo",
+"output":[["StartTag", "a", {"a":"a&"}], ["Character", "foo"]]},
+
+{"description":"plaintext element",
+ "input":"<plaintext>foobar",
+ "output":[["StartTag","plaintext",{}], ["Character","foobar"]]},
+
+{"description":"Open angled bracket in unquoted attribute value state",
+ "input":"<a a=f<>",
+ "output":["ParseError", ["StartTag", "a", {"a":"f<"}]]}
+
+]}

html5lib/tests/testdata/tokenizer/test2.test

@@ -0,0 +1,179 @@
+{"tests": [
+
+{"description":"DOCTYPE without name",
+"input":"<!DOCTYPE>",
+"output":["ParseError", "ParseError", ["DOCTYPE", "", null, null, false]]},
+
+{"description":"DOCTYPE without space before name",
+"input":"<!DOCTYPEhtml>",
+"output":["ParseError", ["DOCTYPE", "html", null, null, true]]},
+
+{"description":"Incorrect DOCTYPE without a space before name",
+"input":"<!DOCTYPEfoo>",
+"output":["ParseError", ["DOCTYPE", "foo", null, null, true]]},
+
+{"description":"DOCTYPE with publicId",
+"input":"<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML Transitional 4.01//EN\">",
+"output":[["DOCTYPE", "html", "-//W3C//DTD HTML Transitional 4.01//EN", null, true]]},
+
+{"description":"DOCTYPE with EOF after PUBLIC",
+"input":"<!DOCTYPE html PUBLIC",
+"output":["ParseError", ["DOCTYPE", "html", null, null, false]]},
+
+{"description":"DOCTYPE with EOF after PUBLIC '",
+"input":"<!DOCTYPE html PUBLIC '",
+"output":["ParseError", ["DOCTYPE", "html", "", null, false]]},
+
+{"description":"DOCTYPE with EOF after PUBLIC 'x",
+"input":"<!DOCTYPE html PUBLIC 'x",
+"output":["ParseError", ["DOCTYPE", "html", "x", null, false]]},
+
+{"description":"DOCTYPE with systemId",
+"input":"<!DOCTYPE html SYSTEM \"-//W3C//DTD HTML Transitional 4.01//EN\">",
+"output":[["DOCTYPE", "html", null, "-//W3C//DTD HTML Transitional 4.01//EN", true]]},
+
+{"description":"DOCTYPE with publicId and systemId",
+"input":"<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML Transitional 4.01//EN\" \"-//W3C//DTD HTML Transitional 4.01//EN\">",
+"output":[["DOCTYPE", "html", "-//W3C//DTD HTML Transitional 4.01//EN", "-//W3C//DTD HTML Transitional 4.01//EN", true]]},
+
+{"description":"DOCTYPE with > in double-quoted publicId",
+"input":"<!DOCTYPE html PUBLIC \">x",
+"output":["ParseError", ["DOCTYPE", "html", "", null, false], ["Character", "x"]]},
+
+{"description":"DOCTYPE with > in single-quoted publicId",
+"input":"<!DOCTYPE html PUBLIC '>x",
+"output":["ParseError", ["DOCTYPE", "html", "", null, false], ["Character", "x"]]},
+
+{"description":"DOCTYPE with > in double-quoted systemId",
+"input":"<!DOCTYPE html PUBLIC \"foo\" \">x",
+"output":["ParseError", ["DOCTYPE", "html", "foo", "", false], ["Character", "x"]]},
+
+{"description":"DOCTYPE with > in single-quoted systemId",
+"input":"<!DOCTYPE html PUBLIC 'foo' '>x",
+"output":["ParseError", ["DOCTYPE", "html", "foo", "", false], ["Character", "x"]]},
+
+{"description":"Incomplete doctype",
+"input":"<!DOCTYPE html ",
+"output":["ParseError", ["DOCTYPE", "html", null, null, false]]},
+
+{"description":"Numeric entity representing the NUL character",
+"input":"&#0000;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Hexadecimal entity representing the NUL character",
+"input":"&#x0000;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Numeric entity representing a codepoint after 1114111 (U+10FFFF)",
+"input":"&#2225222;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Hexadecimal entity representing a codepoint after 1114111 (U+10FFFF)",
+"input":"&#x1010FFFF;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Hexadecimal entity pair representing a surrogate pair",
+"input":"&#xD869;&#xDED6;",
+"output":["ParseError", ["Character", "\uFFFD"], "ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Hexadecimal entity with mixed uppercase and lowercase",
+"input":"&#xaBcD;",
+"output":[["Character", "\uABCD"]]},
+
+{"description":"Entity without a name",
+"input":"&;",
+"output":["ParseError", ["Character", "&;"]]},
+
+{"description":"Unescaped ampersand in attribute value",
+"input":"<h a='&'>",
+"output":[["StartTag", "h", { "a":"&" }]]},
+
+{"description":"StartTag containing <",
+"input":"<a<b>",
+"output":[["StartTag", "a<b", { }]]},
+
+{"description":"Non-void element containing trailing /",
+"input":"<h/>",
+"output":[["StartTag","h",{},true]]},
+
+{"description":"Void element with permitted slash",
+"input":"<br/>",
+"output":[["StartTag","br",{},true]]},
+
+{"description":"Void element with permitted slash (with attribute)",
+"input":"<br foo='bar'/>",
+"output":[["StartTag","br",{"foo":"bar"},true]]},
+
+{"description":"StartTag containing /",
+"input":"<h/a='b'>",
+"output":["ParseError", ["StartTag", "h", { "a":"b" }]]},
+
+{"description":"Double-quoted attribute value",
+"input":"<h a=\"b\">",
+"output":[["StartTag", "h", { "a":"b" }]]},
+
+{"description":"Unescaped </",
+"input":"</",
+"output":["ParseError", ["Character", "</"]]},
+
+{"description":"Illegal end tag name",
+"input":"</1>",
+"output":["ParseError", ["Comment", "1"]]},
+
+{"description":"Simili processing instruction",
+"input":"<?namespace>",
+"output":["ParseError", ["Comment", "?namespace"]]},
+
+{"description":"A bogus comment stops at >, even if preceeded by two dashes",
+"input":"<?foo-->",
+"output":["ParseError", ["Comment", "?foo--"]]},
+
+{"description":"Unescaped <",
+"input":"foo < bar",
+"output":[["Character", "foo "], "ParseError", ["Character", "< bar"]]},
+
+{"description":"Null Byte Replacement",
+"input":"\u0000",
+"output":["ParseError", ["Character", "\u0000"]]},
+
+{"description":"Comment with dash",
+"input":"<!---x",
+"output":["ParseError", ["Comment", "-x"]]},
+
+{"description":"Entity + newline",
+"input":"\nx\n&gt;\n",
+"output":[["Character","\nx\n>\n"]]},
+
+{"description":"Start tag with no attributes but space before the greater-than sign",
+"input":"<h >",
+"output":[["StartTag", "h", {}]]},
+
+{"description":"Empty attribute followed by uppercase attribute",
+"input":"<h a B=''>",
+"output":[["StartTag", "h", {"a":"", "b":""}]]},
+
+{"description":"Double-quote after attribute name",
+"input":"<h a \">",
+"output":["ParseError", ["StartTag", "h", {"a":"", "\"":""}]]},
+
+{"description":"Single-quote after attribute name",
+"input":"<h a '>",
+"output":["ParseError", ["StartTag", "h", {"a":"", "'":""}]]},
+
+{"description":"Empty end tag with following characters",
+"input":"a</>bc",
+"output":[["Character", "a"], "ParseError", ["Character", "bc"]]},
+
+{"description":"Empty end tag with following tag",
+"input":"a</><b>c",
+"output":[["Character", "a"], "ParseError", ["StartTag", "b", {}], ["Character", "c"]]},
+
+{"description":"Empty end tag with following comment",
+"input":"a</><!--b-->c",
+"output":[["Character", "a"], "ParseError", ["Comment", "b"], ["Character", "c"]]},
+
+{"description":"Empty end tag with following end tag",
+"input":"a</></b>c",
+"output":[["Character", "a"], "ParseError", ["EndTag", "b"], ["Character", "c"]]}
+
+]}