justhtml 0.12.0__py3-none-any.whl → 0.24.0__py3-none-any.whl

justhtml/tokens.py

@@ -4,7 +4,7 @@ from typing import Literal
 
 
 class Tag:
-    __slots__ = ("attrs", "kind", "name", "self_closing")
+    __slots__ = ("attrs", "kind", "name", "self_closing", "start_pos")
 
     START: Literal[0] = 0
     END: Literal[1] = 1
@@ -13,6 +13,7 @@ class Tag:
     name: str
     attrs: dict[str, str | None]
     self_closing: bool
+    start_pos: int | None
 
     def __init__(
         self,
@@ -20,11 +21,13 @@ class Tag:
         name: str,
         attrs: dict[str, str | None] | None,
         self_closing: bool = False,
+        start_pos: int | None = None,
     ) -> None:
         self.kind = kind
         self.name = name
         self.attrs = attrs if attrs is not None else {}
         self.self_closing = bool(self_closing)
+        self.start_pos = start_pos
 
 
 class CharacterTokens:
@@ -37,12 +40,14 @@ class CharacterTokens:
 
 
 class CommentToken:
-    __slots__ = ("data",)
+    __slots__ = ("data", "start_pos")
 
     data: str
+    start_pos: int | None
 
-    def __init__(self, data: str) -> None:
+    def __init__(self, data: str, start_pos: int | None = None) -> None:
         self.data = data
+        self.start_pos = start_pos
 
 
 class Doctype:

justhtml/treebuilder.py

@@ -59,6 +59,7 @@ class TreeBuilder(TreeBuilderModesMixin):
         "open_elements",
         "original_mode",
         "pending_table_text",
+        "pending_table_text_should_error",
         "quirks_mode",
         "table_text_original_mode",
         "template_modes",
@@ -86,6 +87,7 @@ class TreeBuilder(TreeBuilderModesMixin):
     open_elements: list[Any]
     original_mode: InsertionMode | None  # type: ignore[assignment]
     pending_table_text: list[str]
+    pending_table_text_should_error: bool
     quirks_mode: str
     table_text_original_mode: InsertionMode | None  # type: ignore[assignment]
     template_modes: list[InsertionMode]
@@ -118,6 +120,7 @@ class TreeBuilder(TreeBuilderModesMixin):
         self.quirks_mode = "no-quirks"
         self.ignore_lf = False
         self.active_formatting = []
+        self.pending_table_text_should_error = False
         self.insert_from_table = False
         self.pending_table_text = []
         self.template_modes = []
@@ -251,7 +254,7 @@ class TreeBuilder(TreeBuilderModesMixin):
         if self._has_element_in_button_scope("p"):
             self._generate_implied_end_tags("p")
             if self.open_elements[-1].name != "p":
-                self._parse_error("end-tag-too-early", tag_name="p")
+                self._parse_error("unexpected-end-tag", tag_name="p")
             self._pop_until_inclusive("p")
             return True
         return False
@@ -413,11 +416,7 @@ class TreeBuilder(TreeBuilderModesMixin):
                         # Tokenizer guarantees non-empty data
                         data = current_token.data
                         if "\x00" in data:
-                            self._parse_error("invalid-codepoint")
                             data = data.replace("\x00", "")
-                        if "\x0c" in data:
-                            self._parse_error("invalid-codepoint")
-                            data = data.replace("\x0c", "")
                         if data:
                             if not is_all_whitespace(data):
                                 self._reconstruct_active_formatting_elements()
@@ -497,6 +496,10 @@ class TreeBuilder(TreeBuilderModesMixin):
 
     def _append_comment_to_document(self, text: str) -> None:
         node = SimpleDomNode("#comment", data=text)
+        if self.tokenizer is not None and self.tokenizer.track_node_locations:
+            node._origin_pos = self.tokenizer.last_token_start_pos
+            if node._origin_pos is not None:
+                node._origin_line, node._origin_col = self.tokenizer.location_at_pos(node._origin_pos)
         self.document.append_child(node)
 
     def _append_comment(self, text: str, parent: Any | None = None) -> None:
@@ -506,6 +509,10 @@ class TreeBuilder(TreeBuilderModesMixin):
         if type(parent) is TemplateNode and parent.template_content:
             parent = parent.template_content
         node = SimpleDomNode("#comment", data=text)
+        if self.tokenizer is not None and self.tokenizer.track_node_locations:
+            node._origin_pos = self.tokenizer.last_token_start_pos
+            if node._origin_pos is not None:
+                node._origin_line, node._origin_col = self.tokenizer.location_at_pos(node._origin_pos)
         parent.append_child(node)
 
     def _append_text(self, text: str) -> None:
@@ -516,6 +523,9 @@ class TreeBuilder(TreeBuilderModesMixin):
                 if not text:
                     return
 
+        if "\f" in text:
+            text = text.replace("\f", " ")
+
         # Guard against empty stack
         if not self.open_elements:  # pragma: no cover
             return
@@ -532,6 +542,10 @@ class TreeBuilder(TreeBuilderModesMixin):
                     return
 
             node = TextNode(text)
+            if self.tokenizer is not None and self.tokenizer.track_node_locations:
+                node._origin_pos = self.tokenizer.last_token_start_pos
+                if node._origin_pos is not None:
+                    node._origin_line, node._origin_col = self.tokenizer.location_at_pos(node._origin_pos)
             children.append(node)
             node.parent = target
             return
@@ -552,6 +566,10 @@ class TreeBuilder(TreeBuilderModesMixin):
             return
 
         node = TextNode(text)
+        if self.tokenizer is not None and self.tokenizer.track_node_locations:
+            node._origin_pos = self.tokenizer.last_token_start_pos
+            if node._origin_pos is not None:
+                node._origin_line, node._origin_col = self.tokenizer.location_at_pos(node._origin_pos)
         reference_node = parent.children[position] if position < len(parent.children) else None
         parent.insert_before(node, reference_node)
 
@@ -582,6 +600,11 @@ class TreeBuilder(TreeBuilderModesMixin):
         else:
             node = ElementNode(tag.name, attrs=tag.attrs, namespace=namespace)
 
+        if self.tokenizer is not None and self.tokenizer.track_node_locations:
+            node._origin_pos = tag.start_pos
+            if node._origin_pos is not None:
+                node._origin_line, node._origin_col = self.tokenizer.location_at_pos(node._origin_pos)
+
         # Fast path for common case: not inserting from table
         if not self.insert_from_table:
             target = self._current_node_or_html()
@@ -799,6 +822,10 @@ class TreeBuilder(TreeBuilderModesMixin):
             entry = self.active_formatting[index]
             tag = Tag(Tag.START, entry["name"], self._clone_attributes(entry["attrs"]), False)
             new_node = self._insert_element(tag, push=True)
+            if self.tokenizer is not None and self.tokenizer.track_node_locations:
+                new_node._origin_pos = entry["node"].origin_offset
+                new_node._origin_line = entry["node"].origin_line
+                new_node._origin_col = entry["node"].origin_col
             entry["node"] = new_node
             index += 1
 
@@ -855,12 +882,19 @@ class TreeBuilder(TreeBuilderModesMixin):
     def _flush_pending_table_text(self) -> None:
         data = "".join(self.pending_table_text)
         self.pending_table_text.clear()
-        if not data:
+        if not data:  # pragma: no cover
             return
         if is_all_whitespace(data):
             self._append_text(data)
             return
-        self._parse_error("foster-parenting-character")
+
+        if self.pending_table_text_should_error:
+            # html5lib reports one foster-parenting error per non-whitespace character.
+            for ch in data:
+                if ch not in " \t\n\r\f":
+                    self._parse_error("foster-parenting-character")
+        self.pending_table_text_should_error = False
+
         previous = self.insert_from_table
         self.insert_from_table = True
         try:
@@ -1118,7 +1152,7 @@ class TreeBuilder(TreeBuilderModesMixin):
 
             # Per HTML5 spec: if first node doesn't match, it's a parse error
             if first:
-                self._parse_error("unexpected-end-tag-in-foreign-content", tag_name=token.name)
+                self._parse_error("unexpected-end-tag", tag_name=token.name)
                 first = False
 
             # If we hit an HTML element that doesn't match, process in secondary mode
@@ -1259,19 +1293,21 @@ class TreeBuilder(TreeBuilderModesMixin):
             return self.process_token(CharacterTokens(data))
 
         if self.mode == InsertionMode.IN_BODY:
-            if "\x00" in data:
-                self._parse_error("invalid-codepoint")
-                data = data.replace("\x00", "")
-
             if not data:
                 return TokenSinkResult.Continue
+            if "\x00" in data:
+                data = data.replace("\x00", "")
+                if not data:
+                    return TokenSinkResult.Continue
 
             if is_all_whitespace(data):
-                self._reconstruct_active_formatting_elements()
+                if self.active_formatting:
+                    self._reconstruct_active_formatting_elements()
                 self._append_text(data)
                 return TokenSinkResult.Continue
 
-            self._reconstruct_active_formatting_elements()
+            if self.active_formatting:
+                self._reconstruct_active_formatting_elements()
             self.frameset_ok = False
             self._append_text(data)
             return TokenSinkResult.Continue

justhtml/treebuilder_modes.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 from typing import Any
 
 from .constants import (
+    FORMAT_MARKER,
     FORMATTING_ELEMENTS,
     HEADING_ELEMENTS,
 )
@@ -54,9 +55,9 @@ class TreeBuilderModesMixin:
             return ("reprocess", InsertionMode.BEFORE_HTML, token)
         # Only Tags remain - no DOCTYPE seen, so quirks mode
         if token.kind == Tag.START:
-            self._parse_error("expected-doctype-but-got-start-tag", tag_name=token.name, token=token)
+            self._parse_error("expected-doctype-but-got-start-tag", tag_name=token.name)
         else:
-            self._parse_error("expected-doctype-but-got-end-tag", tag_name=token.name, token=token)
+            self._parse_error("expected-doctype-but-got-end-tag", tag_name=token.name)
         self._set_quirks_mode("quirks")
         return ("reprocess", InsertionMode.BEFORE_HTML, token)
 
@@ -265,11 +266,7 @@ class TreeBuilderModesMixin:
         if isinstance(token, CharacterTokens):
             data = token.data or ""
             if "\x00" in data:
-                self._parse_error("invalid-codepoint-in-body")
                 data = data.replace("\x00", "")
-            if "\x0c" in data:
-                self._parse_error("invalid-codepoint-in-body")
-                data = data.replace("\x0c", "")
             if not data or is_all_whitespace(data):
                 if data:
                     self._append_text(data)
@@ -331,6 +328,10 @@ class TreeBuilderModesMixin:
                 self.mode = InsertionMode.IN_HEAD
                 return ("reprocess", InsertionMode.IN_HEAD, token)
             if token.kind == Tag.END and token.name == "template":
+                has_template = any(node.name == "template" for node in self.open_elements)
+                if not has_template:
+                    self._parse_error("unexpected-end-tag", tag_name=token.name)
+                    return None
                 return self._mode_in_head(token)
             if token.kind == Tag.END and token.name == "body":
                 self._insert_body_if_missing()
@@ -451,6 +452,8 @@ class TreeBuilderModesMixin:
         if self.template_modes:
             self._parse_error("unexpected-start-tag", tag_name=token.name)
             return
+        # Per spec: parse error; merge attributes onto existing <html>.
+        self._parse_error("unexpected-start-tag", tag_name=token.name)
         # In IN_BODY mode, html element is always at open_elements[0]
         if self.open_elements:  # pragma: no branch
             html = self.open_elements[0]
@@ -574,6 +577,10 @@ class TreeBuilderModesMixin:
             # 3. Find formatting element
             formatting_element_index = self._find_active_formatting_index(subject)
             if formatting_element_index is None:
+                # html5lib reports a parse error when an end tag for a formatting
+                # element triggers the adoption agency algorithm but no matching
+                # active formatting entry exists.
+                self._parse_error("adoption-agency-1.3")
                 return
 
             formatting_element_entry = self.active_formatting[formatting_element_index]
@@ -651,6 +658,10 @@ class TreeBuilderModesMixin:
                 # 10.4 Replace entry with new element
                 entry = self.active_formatting[node_formatting_index]
                 new_element = self._create_element(entry["name"], entry["node"].namespace, entry["attrs"])
+                if self.tokenizer is not None and self.tokenizer.track_node_locations:
+                    new_element._origin_pos = entry["node"].origin_offset
+                    new_element._origin_line = entry["node"].origin_line
+                    new_element._origin_col = entry["node"].origin_col
                 entry["node"] = new_element
                 self.open_elements[self.open_elements.index(node)] = new_element
                 node = new_element
@@ -684,6 +695,10 @@ class TreeBuilderModesMixin:
             # 12. Create new formatting element
             entry = self.active_formatting[formatting_element_index]
             new_formatting_element = self._create_element(entry["name"], entry["node"].namespace, entry["attrs"])
+            if self.tokenizer is not None and self.tokenizer.track_node_locations:
+                new_formatting_element._origin_pos = entry["node"].origin_offset
+                new_formatting_element._origin_line = entry["node"].origin_line
+                new_formatting_element._origin_col = entry["node"].origin_col
             entry["node"] = new_formatting_element
 
             # 13. Move children of furthest block
@@ -708,6 +723,7 @@ class TreeBuilderModesMixin:
 
     def _handle_body_start_a(self, token: Any) -> Any:
         if self._has_active_formatting_entry("a"):
+            self._parse_error("unexpected-start-tag-implies-end-tag", tag_name=token.name)
             self._adoption_agency("a")
             self._remove_last_active_formatting_by_name("a")
             self._remove_last_open_element_by_name("a")
@@ -853,6 +869,7 @@ class TreeBuilderModesMixin:
     def _handle_body_end_template(self, token: Any) -> Any:
         has_template = any(node.name == "template" for node in self.open_elements)
         if not has_template:
+            self._parse_error("unexpected-end-tag", tag_name=token.name)
             return
         self._generate_implied_end_tags()
         self._pop_until_inclusive("template")
@@ -978,11 +995,33 @@ class TreeBuilderModesMixin:
         if isinstance(token, CharacterTokens):
             data = token.data or ""
             if "\x00" in data:
-                self._parse_error("unexpected-null-character")
                 data = data.replace("\x00", "")
                 if not data:
                     return None
                 token = CharacterTokens(data)
+
+            if is_all_whitespace(data):
+                self._append_text(data)
+                return None
+
+            # html5lib-tests expect that some table foster-parenting text triggered by a
+            # misnested formatting element (<a>) only produces an implied-end-tag error
+            # when the table closes, not an additional character-in-table error.
+            suppress_table_char_error = False
+            if self.active_formatting:
+                for idx in range(len(self.active_formatting) - 1, -1, -1):
+                    entry = self.active_formatting[idx]
+                    if entry is FORMAT_MARKER:
+                        break
+                    if entry["name"] == "a":
+                        if entry["node"] not in self.open_elements:
+                            suppress_table_char_error = True
+                        break
+
+            if not suppress_table_char_error:
+                self.pending_table_text_should_error = True
+            else:
+                self.pending_table_text_should_error = False
             self.pending_table_text = []
             self.table_text_original_mode = self.mode
             self.mode = InsertionMode.IN_TABLE_TEXT
@@ -1055,7 +1094,7 @@ class TreeBuilderModesMixin:
                         self.form_element = node
                         self.open_elements.pop()  # push=True always adds to stack
                     return None
-                self._parse_error("unexpected-start-tag-implies-table-voodoo", tag_name=name)
+                self._parse_error("foster-parenting-start-tag", tag_name=name)
                 previous = self.insert_from_table
                 self.insert_from_table = True
                 try:
@@ -1082,19 +1121,33 @@ class TreeBuilderModesMixin:
         if self.template_modes:
             return self._mode_in_template(token)
         if self._has_in_table_scope("table"):
-            self._parse_error("expected-closing-tag-but-got-eof", tag_name="table")
+            self._parse_error("eof-in-table")
         return None
 
     def _mode_in_table_text(self, token: Any) -> Any:
         if isinstance(token, CharacterTokens):
             # IN_TABLE mode guarantees non-empty data
             data = token.data
-            if "\x0c" in data:
-                self._parse_error("invalid-codepoint-in-table-text")
-                data = data.replace("\x0c", "")
-            if data:
-                self.pending_table_text.append(data)
+            self.pending_table_text.append(data)
             return None
+
+        if (
+            self.pending_table_text
+            and isinstance(token, Tag)
+            and token.kind == Tag.END
+            and token.name == "table"
+            and not is_all_whitespace("".join(self.pending_table_text))
+        ):
+            # If a misnested <a> exists only in the active formatting list, html5lib
+            # reports the implied close when the table ends.
+            if self.active_formatting:
+                for idx in range(len(self.active_formatting) - 1, -1, -1):
+                    entry = self.active_formatting[idx]
+                    if entry is FORMAT_MARKER:
+                        break
+                    if entry["name"] == "a" and entry["node"] not in self.open_elements:
+                        self._parse_error("unexpected-implied-end-tag-in-table-view")
+                        break
         self._flush_pending_table_text()
         original = self.table_text_original_mode or InsertionMode.IN_TABLE
         self.table_text_original_mode = None
@@ -1443,11 +1496,7 @@ class TreeBuilderModesMixin:
         if isinstance(token, CharacterTokens):
             data = token.data or ""
             if "\x00" in data:
-                self._parse_error("invalid-codepoint-in-select")
                 data = data.replace("\x00", "")
-            if "\x0c" in data:
-                self._parse_error("invalid-codepoint-in-select")
-                data = data.replace("\x0c", "")
             if data:
                 self._reconstruct_active_formatting_elements()
                 self._append_text(data)
@@ -1475,13 +1524,13 @@ class TreeBuilderModesMixin:
                     self._insert_element(token, push=True)
                     return None
                 if name == "select":
-                    self._parse_error("unexpected-start-tag-implies-end-tag", tag_name=name)
+                    self._parse_error("unexpected-select-in-select")
                     # select is always in scope in IN_SELECT mode
                     self._pop_until_any_inclusive({"select"})
                     self._reset_insertion_mode()
                     return None
                 if name in {"input", "textarea"}:
-                    self._parse_error("unexpected-start-tag-implies-end-tag", tag_name=name)
+                    self._parse_error("unexpected-start-tag-in-select", tag_name=name)
                     # select is always in scope in IN_SELECT mode
                     self._pop_until_any_inclusive({"select"})
                     self._reset_insertion_mode()
@@ -1491,7 +1540,7 @@ class TreeBuilderModesMixin:
                     self._insert_element(token, push=False)
                     return None
                 if name in {"caption", "col", "colgroup", "tbody", "td", "tfoot", "th", "thead", "tr", "table"}:
-                    self._parse_error("unexpected-start-tag-implies-end-tag", tag_name=name)
+                    self._parse_error("unexpected-start-tag-in-select", tag_name=name)
                     # select is always in scope in IN_SELECT mode
                     self._pop_until_any_inclusive({"select"})
                     self._reset_insertion_mode()
@@ -1509,6 +1558,7 @@ class TreeBuilderModesMixin:
                     self._append_active_formatting_entry(name, token.attrs, node)
                     return None
                 if name == "hr":
+                    self._parse_error("unexpected-start-tag-in-select", tag_name=name)
                     # Per spec: pop option and optgroup before inserting hr (makes hr sibling, not child)
                     if self.open_elements and self.open_elements[-1].name == "option":
                         self.open_elements.pop()
@@ -1518,22 +1568,29 @@ class TreeBuilderModesMixin:
                     self._insert_element(token, push=False)
                     return None
                 if name == "menuitem":
+                    self._parse_error("unexpected-start-tag-in-select", tag_name=name)
                     self._reconstruct_active_formatting_elements()
                     self._insert_element(token, push=True)
                     return None
                 # Allow common HTML elements in select (newer spec)
                 if name in {"p", "div", "span", "button", "datalist", "selectedcontent"}:
+                    self._parse_error("unexpected-start-tag-in-select", tag_name=name)
                     self._reconstruct_active_formatting_elements()
                     self._insert_element(token, push=not token.self_closing)
                     return None
                 if name in {"br", "img"}:
+                    self._parse_error("unexpected-start-tag-in-select", tag_name=name)
                     self._reconstruct_active_formatting_elements()
                     self._insert_element(token, push=False)
                     return None
                 if name == "plaintext":
                     # Per spec: plaintext element is inserted in select (consumes all remaining text)
+                    self._parse_error("unexpected-start-tag-in-select", tag_name=name)
                     self._reconstruct_active_formatting_elements()
                     self._insert_element(token, push=True)
+                    return None
+                # Any other start tag: parse error, ignore.
+                self._parse_error("unexpected-start-tag-in-select", tag_name=name)
                 return None
             if name == "optgroup":
                 if self.open_elements and self.open_elements[-1].name == "option":
@@ -1541,13 +1598,13 @@ class TreeBuilderModesMixin:
                 if self.open_elements and self.open_elements[-1].name == "optgroup":
                     self.open_elements.pop()
                 else:
-                    self._parse_error("unexpected-end-tag", tag_name=token.name)
+                    self._parse_error("unexpected-end-tag-in-select", tag_name=token.name)
                 return None
             if name == "option":
                 if self.open_elements and self.open_elements[-1].name == "option":
                     self.open_elements.pop()
                 else:
-                    self._parse_error("unexpected-end-tag", tag_name=token.name)
+                    self._parse_error("unexpected-end-tag-in-select", tag_name=token.name)
                 return None
             if name == "select":
                 # In IN_SELECT mode, select is always in scope - pop to it
@@ -1559,17 +1616,20 @@ class TreeBuilderModesMixin:
                 # select is always on stack in IN_SELECT mode
                 select_node = self._find_last_on_stack("select")
                 fmt_index = self._find_active_formatting_index(name)
-                if fmt_index is not None:
-                    target = self.active_formatting[fmt_index]["node"]
-                    if target in self.open_elements:  # pragma: no branch
-                        select_index = self.open_elements.index(select_node)
-                        target_index = self.open_elements.index(target)
-                        if target_index < select_index:
-                            self._parse_error("unexpected-end-tag", tag_name=name)
-                            return None
+                if fmt_index is None:
+                    self._parse_error("unexpected-end-tag-in-select", tag_name=name)
+                    return None
+                target = self.active_formatting[fmt_index]["node"]
+                if target in self.open_elements:  # pragma: no branch
+                    select_index = self.open_elements.index(select_node)
+                    target_index = self.open_elements.index(target)
+                    if target_index < select_index:
+                        self._parse_error("unexpected-end-tag-in-select", tag_name=name)
+                        return None
                 self._adoption_agency(name)
                 return None
             if name in {"p", "div", "span", "button", "datalist", "selectedcontent"}:
+                self._parse_error("unexpected-end-tag-in-select", tag_name=name)
                 # Per HTML5 spec: these end tags in select mode close the element if it's on the stack.
                 # But we must not pop across the select boundary (i.e., don't pop elements BEFORE select).
                 select_idx = None
@@ -1586,11 +1646,9 @@ class TreeBuilderModesMixin:
                         popped = self.open_elements.pop()
                         if popped.name == name:
                             break
-                else:
-                    self._parse_error("unexpected-end-tag", tag_name=name)
                 return None
             if name in {"caption", "col", "colgroup", "tbody", "td", "tfoot", "th", "thead", "tr", "table"}:
-                self._parse_error("unexpected-end-tag", tag_name=name)
+                self._parse_error("unexpected-end-tag-in-select", tag_name=name)
                 # select is always in scope in IN_SELECT mode
                 self._pop_until_any_inclusive({"select"})
                 self._reset_insertion_mode()
@@ -1773,8 +1831,11 @@ class TreeBuilderModesMixin:
     def _mode_after_frameset(self, token: Any) -> Any:
         # Per HTML5 spec §13.2.6.4.17: After frameset insertion mode
         if isinstance(token, CharacterTokens):
-            # Only whitespace characters allowed; ignore all others
-            whitespace = "".join(ch for ch in token.data if ch in "\t\n\f\r ")
+            # Only whitespace characters allowed; non-whitespace is a parse error.
+            data = token.data or ""
+            whitespace = "".join(ch for ch in data if ch in "\t\n\f\r ")
+            if any(ch not in "\t\n\f\r " for ch in data):
+                self._parse_error("unexpected-token-after-frameset")
             if whitespace:
                 self._append_text(whitespace)
             return None
@@ -1787,6 +1848,9 @@ class TreeBuilderModesMixin:
             if token.kind == Tag.END and token.name == "html":
                 self.mode = InsertionMode.AFTER_AFTER_FRAMESET
                 return None
+            if token.kind == Tag.END and token.name == "frameset":
+                self._parse_error("unexpected-token-after-frameset")
+                return None
             if token.kind == Tag.START and token.name == "noframes":
                 # Insert noframes element directly and switch to TEXT mode
                 self._insert_element(token, push=True)