Move token normalisation to the tokenizer

Author: gsnedders

html5lib/_tokenizer.py

@@ -2,7 +2,8 @@
 
 from six import unichr as chr
 
-from collections import deque
+from collections import deque, OrderedDict
+from sys import version_info
 
 from .constants import spaceCharacters
 from .constants import entities
@@ -17,6 +18,11 @@
 
 entitiesTrie = Trie(entities)
 
+if version_info >= (3, 7):
+    attributeMap = dict
+else:
+    attributeMap = OrderedDict
+
 
 class HTMLTokenizer(object):
     """ This class takes care of tokenizing HTML.
@@ -228,6 +234,14 @@ def emitCurrentToken(self):
         # Add token to the queue to be yielded
         if (token["type"] in tagTokenTypes):
             token["name"] = token["name"].translate(asciiUpper2Lower)
+            if token["type"] == tokenTypes["StartTag"]:
+                raw = token["data"]
+                data = attributeMap(raw)
+                if len(raw) > len(data):
+                    # we had some duplicated attribute, fix so first wins
+                    data.update(raw[::-1])
+                token["data"] = data
+
             if token["type"] == tokenTypes["EndTag"]:
                 if token["data"]:
                     self.tokenQueue.append({"type": tokenTypes["ParseError"],

html5lib/html5parser.py

@@ -3,9 +3,6 @@
 
 import types
 
-from collections import OrderedDict
-from sys import version_info
-
 from . import _inputstream
 from . import _tokenizer
 
@@ -26,12 +23,6 @@
 )
 
 
-if version_info >= (3, 7):
-    attributeMap = dict
-else:
-    attributeMap = OrderedDict
-
-
 def parse(doc, treebuilder="etree", namespaceHTMLElements=True, **kwargs):
     """Parse an HTML document as a string or file-like object into a tree
 
@@ -210,7 +201,7 @@ def mainLoop(self):
         DoctypeToken = tokenTypes["Doctype"]
         ParseErrorToken = tokenTypes["ParseError"]
 
-        for token in self.normalizedTokens():
+        for token in self.tokenizer:
             prev_token = None
             new_token = token
             while new_token is not None:
@@ -268,10 +259,6 @@ def mainLoop(self):
             if reprocess:
                 assert self.phase not in phases
 
-    def normalizedTokens(self):
-        for token in self.tokenizer:
-            yield self.normalizeToken(token)
-
     def parse(self, stream, *args, **kwargs):
         """Parse a HTML document into a well-formed tree
 
@@ -333,18 +320,6 @@ def parseError(self, errorcode="XXX-undefined-error", datavars=None):
         if self.strict:
             raise ParseError(E[errorcode] % datavars)
 
-    def normalizeToken(self, token):
-        # HTML5 specific normalizations to the token stream
-        if token["type"] == tokenTypes["StartTag"]:
-            raw = token["data"]
-            data = attributeMap(raw)
-            if len(raw) > len(data):
-                # we had some duplicated attribute, fix so first wins
-                data.update(raw[::-1])
-            token["data"] = data
-
-        return token
-
     def adjustMathMLAttributes(self, token):
         adjust_attributes(token, adjustMathMLAttributes)
 
@@ -2803,8 +2778,8 @@ def processEndTag(self, token):
 def adjust_attributes(token, replacements):
     needs_adjustment = viewkeys(token['data']) & viewkeys(replacements)
     if needs_adjustment:
-        token['data'] = attributeMap((replacements.get(k, k), v)
-                                     for k, v in token['data'].items())
+        token['data'] = type(token['data'])((replacements.get(k, k), v)
+                                            for k, v in token['data'].items())
 
 
 def impliedTagToken(name, type="EndTag", attributes=None,

html5lib/tests/test_parser2.py

@@ -1,12 +1,12 @@
 from __future__ import absolute_import, division, unicode_literals
 
-from six import PY2, text_type, unichr
+from six import PY2, text_type
 
 import io
 
 from . import support  # noqa
 
-from html5lib.constants import namespaces, tokenTypes
+from html5lib.constants import namespaces
 from html5lib import parse, parseFragment, HTMLParser
 
 
@@ -53,42 +53,6 @@ def test_unicode_file():
     assert parse(io.StringIO("a")) is not None
 
 
-def test_maintain_attribute_order():
-    # This is here because we impl it in parser and not tokenizer
-    p = HTMLParser()
-    # generate loads to maximize the chance a hash-based mutation will occur
-    attrs = [(unichr(x), i) for i, x in enumerate(range(ord('a'), ord('z')))]
-    token = {'name': 'html',
-             'selfClosing': False,
-             'selfClosingAcknowledged': False,
-             'type': tokenTypes["StartTag"],
-             'data': attrs}
-    out = p.normalizeToken(token)
-    attr_order = list(out["data"].keys())
-    assert attr_order == [x for x, i in attrs]
-
-
-def test_duplicate_attribute():
-    # This is here because we impl it in parser and not tokenizer
-    doc = parse('<p class=a class=b>')
-    el = doc[1][0]
-    assert el.get("class") == "a"
-
-
-def test_maintain_duplicate_attribute_order():
-    # This is here because we impl it in parser and not tokenizer
-    p = HTMLParser()
-    attrs = [(unichr(x), i) for i, x in enumerate(range(ord('a'), ord('z')))]
-    token = {'name': 'html',
-             'selfClosing': False,
-             'selfClosingAcknowledged': False,
-             'type': tokenTypes["StartTag"],
-             'data': attrs + [('a', len(attrs))]}
-    out = p.normalizeToken(token)
-    attr_order = list(out["data"].keys())
-    assert attr_order == [x for x, i in attrs]
-
-
 def test_debug_log():
     parser = HTMLParser(debug=True)
     parser.parse("<!doctype html><title>a</title><p>b<script>c</script>d</p>e")

html5lib/tests/test_tokenizer2.py

@@ -0,0 +1,66 @@
+from __future__ import absolute_import, division, unicode_literals
+
+import io
+
+from six import unichr, text_type
+
+from html5lib._tokenizer import HTMLTokenizer
+from html5lib.constants import tokenTypes
+
+
+def ignore_parse_errors(toks):
+    for tok in toks:
+        if tok['type'] != tokenTypes['ParseError']:
+            yield tok
+
+
+def test_maintain_attribute_order():
+    # generate loads to maximize the chance a hash-based mutation will occur
+    attrs = [(unichr(x), text_type(i)) for i, x in enumerate(range(ord('a'), ord('z')))]
+    stream = io.StringIO("<span " + " ".join("%s='%s'" % (x, i) for x, i in attrs) + ">")
+
+    toks = HTMLTokenizer(stream)
+    out = list(ignore_parse_errors(toks))
+
+    assert len(out) == 1
+    assert out[0]['type'] == tokenTypes['StartTag']
+
+    attrs_tok = out[0]['data']
+    assert len(attrs_tok) == len(attrs)
+
+    for (in_name, in_value), (out_name, out_value) in zip(attrs, attrs_tok.items()):
+        assert in_name == out_name
+        assert in_value == out_value
+
+
+def test_duplicate_attribute():
+    stream = io.StringIO("<span a=1 a=2 a=3>")
+
+    toks = HTMLTokenizer(stream)
+    out = list(ignore_parse_errors(toks))
+
+    assert len(out) == 1
+    assert out[0]['type'] == tokenTypes['StartTag']
+
+    attrs_tok = out[0]['data']
+    assert len(attrs_tok) == 1
+    assert list(attrs_tok.items()) == [('a', '1')]
+
+
+def test_maintain_duplicate_attribute_order():
+    # generate loads to maximize the chance a hash-based mutation will occur
+    attrs = [(unichr(x), text_type(i)) for i, x in enumerate(range(ord('a'), ord('z')))]
+    stream = io.StringIO("<span " + " ".join("%s='%s'" % (x, i) for x, i in attrs) + " a=100>")
+
+    toks = HTMLTokenizer(stream)
+    out = list(ignore_parse_errors(toks))
+
+    assert len(out) == 1
+    assert out[0]['type'] == tokenTypes['StartTag']
+
+    attrs_tok = out[0]['data']
+    assert len(attrs_tok) == len(attrs)
+
+    for (in_name, in_value), (out_name, out_value) in zip(attrs, attrs_tok.items()):
+        assert in_name == out_name
+        assert in_value == out_value

html5lib/tests/test_treewalkers.py

@@ -1,7 +1,9 @@
 from __future__ import absolute_import, division, unicode_literals
 
 import itertools
+import sys
 
+from six import unichr, text_type
 import pytest
 
 try:
@@ -135,3 +137,65 @@ def test_lxml_xml():
     output = Lint(walker(lxmltree))
 
     assert list(output) == expected
+
+
+@pytest.mark.parametrize("treeName",
+                         [pytest.param(treeName, marks=[getattr(pytest.mark, treeName),
+                                                        pytest.mark.skipif(sys.version_info < (3, 7), reason="dict order undef")])
+                          for treeName in sorted(treeTypes.keys())])
+def test_maintain_attribute_order(treeName):
+    treeAPIs = treeTypes[treeName]
+    if treeAPIs is None:
+        pytest.skip("Treebuilder not loaded")
+
+    # generate loads to maximize the chance a hash-based mutation will occur
+    attrs = [(unichr(x), text_type(i)) for i, x in enumerate(range(ord('a'), ord('z')))]
+    data = "<span " + " ".join("%s='%s'" % (x, i) for x, i in attrs) + ">"
+
+    parser = html5parser.HTMLParser(tree=treeAPIs["builder"])
+    document = parser.parseFragment(data)
+
+    document = treeAPIs.get("adapter", lambda x: x)(document)
+    output = list(Lint(treeAPIs["walker"](document)))
+
+    assert len(output) == 2
+    assert output[0]['type'] == 'StartTag'
+    assert output[1]['type'] == "EndTag"
+
+    attrs_out = output[0]['data']
+    assert len(attrs) == len(attrs_out)
+
+    for (in_name, in_value), (out_name, out_value) in zip(attrs, attrs_out.items()):
+        assert (None, in_name) == out_name
+        assert in_value == out_value
+
+
+@pytest.mark.parametrize("treeName",
+                         [pytest.param(treeName, marks=[getattr(pytest.mark, treeName),
+                                                        pytest.mark.skipif(sys.version_info < (3, 7), reason="dict order undef")])
+                          for treeName in sorted(treeTypes.keys())])
+def test_maintain_attribute_order_adjusted(treeName):
+    treeAPIs = treeTypes[treeName]
+    if treeAPIs is None:
+        pytest.skip("Treebuilder not loaded")
+
+    # generate loads to maximize the chance a hash-based mutation will occur
+    data = "<svg a=1 refx=2 b=3 xml:lang=4 c=5>"
+
+    parser = html5parser.HTMLParser(tree=treeAPIs["builder"])
+    document = parser.parseFragment(data)
+
+    document = treeAPIs.get("adapter", lambda x: x)(document)
+    output = list(Lint(treeAPIs["walker"](document)))
+
+    assert len(output) == 2
+    assert output[0]['type'] == 'StartTag'
+    assert output[1]['type'] == "EndTag"
+
+    attrs_out = output[0]['data']
+
+    assert list(attrs_out.items()) == [((None, 'a'), '1'),
+                                       ((None, 'refX'), '2'),
+                                       ((None, 'b'), '3'),
+                                       (('http://www.w3.org/XML/1998/namespace', 'lang'), '4'),
+                                       ((None, 'c'), '5')]

html5lib/tests/tokenizer.py

@@ -40,7 +40,7 @@ def processDoctype(self, token):
 
     def processStartTag(self, token):
         self.outputTokens.append(["StartTag", token["name"],
-                                  dict(token["data"][::-1]), token["selfClosing"]])
+                                  token["data"], token["selfClosing"]])
 
     def processEmptyTag(self, token):
         if token["name"] not in constants.voidElements: