justhtml 0.24.0__py3-none-any.whl → 0.38.0__py3-none-any.whl

justhtml/tokenizer.py

@@ -9,7 +9,7 @@ if TYPE_CHECKING:
 
 from .entities import decode_entities_in_text
 from .errors import generate_error_message
-from .tokens import CommentToken, Doctype, DoctypeToken, EOFToken, ParseError, Tag
+from .tokens import AnyToken, CharacterTokens, CommentToken, Doctype, DoctypeToken, EOFToken, ParseError, Tag
 
 _ATTR_VALUE_UNQUOTED_TERMINATORS = "\t\n\f >&\"'<=`\0"
 _ASCII_LOWER_TABLE = str.maketrans({chr(code): chr(code + 32) for code in range(65, 91)})
@@ -79,7 +79,14 @@ def _coerce_comment_for_xml(text: str) -> str:
 
 
 class TokenizerOpts:
-    __slots__ = ("discard_bom", "exact_errors", "initial_rawtext_tag", "initial_state", "xml_coercion")
+    __slots__ = (
+        "discard_bom",
+        "emit_bogus_markup_as_text",
+        "exact_errors",
+        "initial_rawtext_tag",
+        "initial_state",
+        "xml_coercion",
+    )
 
     discard_bom: bool
     exact_errors: bool
@@ -91,12 +98,14 @@ class TokenizerOpts:
         self,
         exact_errors: bool = False,
         discard_bom: bool = True,
+        emit_bogus_markup_as_text: bool = False,
         initial_state: int | None = None,
         initial_rawtext_tag: str | None = None,
         xml_coercion: bool = False,
     ) -> None:
         self.exact_errors = bool(exact_errors)
         self.discard_bom = bool(discard_bom)
+        self.emit_bogus_markup_as_text = bool(emit_bogus_markup_as_text)
         self.initial_state = initial_state
         self.initial_rawtext_tag = initial_rawtext_tag
         self.xml_coercion = bool(xml_coercion)
@@ -203,6 +212,7 @@ class Tokenizer:
         "text_buffer",
         "text_start_pos",
         "track_node_locations",
+        "track_tag_positions",
     )
 
     _comment_token: CommentToken
@@ -211,6 +221,7 @@ class Tokenizer:
     _tag_token: Tag
     buffer: str
     collect_errors: bool
+    track_tag_positions: bool
     track_node_locations: bool
     current_attr_name: list[str]
     current_attr_value: list[str]
@@ -252,11 +263,13 @@ class Tokenizer:
         *,
         collect_errors: bool = False,
         track_node_locations: bool = False,
+        track_tag_positions: bool = False,
     ) -> None:
         self.sink = sink
         self.opts = opts or TokenizerOpts()
         self.collect_errors = collect_errors
         self.track_node_locations = bool(track_node_locations)
+        self.track_tag_positions = bool(track_tag_positions)
         self.errors = []
 
         self.state = self.DATA
@@ -396,8 +409,9 @@ class Tokenizer:
 
     def run(self, html: str | None) -> None:
         self.initialize(html)
+        handlers = self._STATE_HANDLERS  # type: ignore[attr-defined]
         while True:
-            if self.step():
+            if handlers[self.state](self):  # type: ignore[no-any-return]
                 break
 
     # ---------------------
@@ -486,7 +500,7 @@ class Tokenizer:
                     self.state = self.TAG_NAME
                     return self._state_tag_name()
 
-                if nc == "!":
+                if nc == "!" and not self.opts.emit_bogus_markup_as_text:
                     # Optimization: Peek ahead for comments
                     if pos + 2 < length and buffer[pos + 1] == "-" and buffer[pos + 2] == "-":
                         self._flush_text()
@@ -529,12 +543,20 @@ class Tokenizer:
             self._emit_token(EOFToken())
             return True
         if c == "!":
+            if self.opts.emit_bogus_markup_as_text:
+                self._append_text("<!")
+                self.state = self.DATA
+                return False
             self.state = self.MARKUP_DECLARATION_OPEN
             return False
         if c == "/":
             self.state = self.END_TAG_OPEN
             return False
         if c == "?":
+            if self.opts.emit_bogus_markup_as_text:
+                self._append_text("<?")
+                self.state = self.DATA
+                return False
             self._emit_error("unexpected-question-mark-instead-of-tag-name")
             self.current_comment.clear()
             self._reconsume_current()
@@ -551,6 +573,11 @@ class Tokenizer:
         c = self._get_char()
         if c is None:
             self._emit_error("eof-before-tag-name")
+            if self.opts.emit_bogus_markup_as_text:
+                self._append_text("</")
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
             self._append_text("<")
             self._append_text("/")
             self._flush_text()
@@ -558,6 +585,16 @@ class Tokenizer:
             return True
         if c == ">":
             self._emit_error("empty-end-tag")
+            if self.opts.emit_bogus_markup_as_text:
+                self._append_text("</>")
+                self.state = self.DATA
+                return False
+            self.state = self.DATA
+            return False
+
+        if self.opts.emit_bogus_markup_as_text:
+            self._append_text("</")
+            self._append_text(c)
             self.state = self.DATA
             return False
 
@@ -593,6 +630,8 @@ class Tokenizer:
                     if pos < length:
                         next_char = buffer[pos]
                         if next_char in (" ", "\t", "\n", "\f"):
+                            if self.current_tag_kind == Tag.END and self.opts.emit_bogus_markup_as_text:
+                                return self._emit_raw_end_tag_as_text(pos)
                             pos += 1
                             self.pos = pos
                             self.state = self.BEFORE_ATTRIBUTE_NAME
@@ -604,6 +643,8 @@ class Tokenizer:
                                 self.state = self.DATA
                             return False
                         if next_char == "/":
+                            if self.current_tag_kind == Tag.END and self.opts.emit_bogus_markup_as_text:
+                                return self._emit_raw_end_tag_as_text(pos)
                             pos += 1
                             self.pos = pos
                             self.state = self.SELF_CLOSING_START_TAG
@@ -620,15 +661,20 @@ class Tokenizer:
             if c is None:
                 self.pos = pos
                 self._emit_error("eof-in-tag")
-                # Per HTML5 spec: EOF in tag name is a parse error, emit EOF token only
-                # The incomplete tag is discarded (not emitted as text)
+                self._emit_incomplete_tag_as_text()
                 self._emit_token(EOFToken())
                 return True
             if c in ("\t", "\n", "\f", " "):
+                if self.current_tag_kind == Tag.END and self.opts.emit_bogus_markup_as_text:
+                    self.pos = pos
+                    return self._emit_raw_end_tag_as_text(pos)
                 self.pos = pos
                 self.state = self.BEFORE_ATTRIBUTE_NAME
                 return self._state_before_attribute_name()
             if c == "/":
+                if self.current_tag_kind == Tag.END and self.opts.emit_bogus_markup_as_text:
+                    self.pos = pos
+                    return self._emit_raw_end_tag_as_text(pos)
                 self.pos = pos
                 self.state = self.SELF_CLOSING_START_TAG
                 return self._state_self_closing_start_tag()
@@ -675,6 +721,7 @@ class Tokenizer:
 
             if c is None:
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._flush_text()
                 self._emit_token(EOFToken())
                 return True
@@ -733,37 +780,43 @@ class Tokenizer:
                     pos = match.end()
 
                     if pos < length:
-                        c = buffer[pos]
-                        if c == "=":
+                        next_char = buffer[pos]
+                        if next_char == "=":
                             pos += 1
                             self.pos = pos
                             self.state = self.BEFORE_ATTRIBUTE_VALUE
                             return self._state_before_attribute_value()
-                        if c in (" ", "\t", "\n", "\f"):
+                        if next_char in (" ", "\t", "\n", "\f"):
                             pos += 1
                             self.pos = pos
                             self._finish_attribute()
                             self.state = self.AFTER_ATTRIBUTE_NAME
                             return False  # Let main loop dispatch to avoid recursion
-                        if c == ">":
+                        if next_char == ">":
                             pos += 1
                             self.pos = pos
                             self._finish_attribute()
                             if not self._emit_current_tag():
                                 self.state = self.DATA
                             return False
-                        if c == "/":
+                        if next_char == "/":
                             pos += 1
                             self.pos = pos
                             self._finish_attribute()
                             self.state = self.SELF_CLOSING_START_TAG
                             return self._state_self_closing_start_tag()
 
+            # Inline _get_char (reconsume is never True in this state)
+            if pos >= length:
+                c: str | None = None
+            else:
+                c = buffer[pos]
+                pos += 1
+            self.current_char = c
             self.pos = pos
-            c = self._get_char()  # type: ignore[assignment]
-            pos = self.pos
             if c is None:
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._flush_text()
                 self._emit_token(EOFToken())
                 return True
@@ -798,9 +851,8 @@ class Tokenizer:
             # Optimization: Skip whitespace
             if not self.reconsume:
                 if self.pos < length:
-                    match = _WHITESPACE_PATTERN.match(buffer, self.pos)
-                    if match:
-                        self.pos = match.end()
+                    if buffer[self.pos] in " \t\n\f":
+                        self.pos = _WHITESPACE_PATTERN.match(buffer, self.pos).end()  # type: ignore[union-attr]
 
             # Inline _get_char
             if self.pos >= length:
@@ -816,6 +868,7 @@ class Tokenizer:
 
             if c is None:
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._flush_text()
                 self._emit_token(EOFToken())
                 return True
@@ -846,9 +899,17 @@ class Tokenizer:
 
     def _state_before_attribute_value(self) -> bool:
         while True:
-            c = self._get_char()
+            # Inline _get_char (reconsume is never True in this state)
+            pos = self.pos
+            if pos >= self.length:
+                c: str | None = None
+            else:
+                c = self.buffer[pos]
+                self.pos = pos + 1
+            self.current_char = c
             if c is None:
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._flush_text()
                 self._emit_token(EOFToken())
                 return True
@@ -906,6 +967,7 @@ class Tokenizer:
             if self.pos >= length:
                 self.current_char = None
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._emit_token(EOFToken())
                 return True
 
@@ -961,6 +1023,7 @@ class Tokenizer:
             if self.pos >= length:
                 self.current_char = None
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._emit_token(EOFToken())
                 return True
 
@@ -1014,6 +1077,7 @@ class Tokenizer:
                 # Per HTML5 spec: EOF in attribute value is a parse error
                 # The incomplete tag is discarded (not emitted)
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._emit_token(EOFToken())
                 return True
             if c in ("\t", "\n", "\f", " "):
@@ -1049,6 +1113,7 @@ class Tokenizer:
 
         if c is None:
             self._emit_error("eof-in-tag")
+            self._emit_incomplete_tag_as_text()
             self._flush_text()
             self._emit_token(EOFToken())
             return True
@@ -1076,6 +1141,7 @@ class Tokenizer:
         c = self._get_char()
         if c is None:
             self._emit_error("eof-in-tag")
+            self._emit_incomplete_tag_as_text()
             self._flush_text()
             self._emit_token(EOFToken())
             return True
@@ -1797,6 +1863,7 @@ class Tokenizer:
                         "unexpected-null-character",
                         line=line,
                         column=column,
+                        category="tokenizer",
                         message=message,
                         source_html=self.buffer,
                     )
@@ -1878,7 +1945,12 @@ class Tokenizer:
         tag.name = name
         tag.attrs = attrs
         tag.self_closing = self.current_tag_self_closing
-        tag.start_pos = self.current_token_start_pos
+        if self.track_tag_positions:
+            tag.start_pos = self.current_token_start_pos
+            tag.end_pos = self.pos
+        else:
+            tag.start_pos = None
+            tag.end_pos = None
         self.last_token_start_pos = tag.start_pos
 
         switched_to_rawtext = False
@@ -1919,6 +1991,30 @@ class Tokenizer:
         self.current_tag_kind = Tag.START
         return switched_to_rawtext
 
+    def _emit_incomplete_tag_as_text(self) -> None:
+        if not self.opts.emit_bogus_markup_as_text:
+            return
+        start = self.current_token_start_pos
+        if start is None:  # pragma: no cover
+            return
+        raw = self.buffer[start : self.pos]
+        if raw:  # pragma: no branch
+            self._emit_token(CharacterTokens(raw))
+
+    def _emit_raw_end_tag_as_text(self, pos: int) -> bool:
+        end = self.buffer.find(">", pos)
+        if end == -1:
+            self.pos = self.length
+            self._emit_incomplete_tag_as_text()
+            self._emit_token(EOFToken())
+            return True
+        self.pos = end + 1
+        raw = self.buffer[self.current_token_start_pos : self.pos]
+        if raw:  # pragma: no branch
+            self._emit_token(CharacterTokens(raw))
+        self.state = self.DATA
+        return False
+
     def _emit_comment(self) -> None:
         data = "".join(self.current_comment)
         self.current_comment.clear()
@@ -1947,7 +2043,7 @@ class Tokenizer:
         self.current_doctype_force_quirks = False
         self._emit_token(DoctypeToken(doctype))
 
-    def _emit_token(self, token: Any) -> None:
+    def _emit_token(self, token: AnyToken) -> None:
         if self.collect_errors:
             self._record_token_position()
         self.sink.process_token(token)
@@ -1998,7 +2094,9 @@ class Tokenizer:
 
         message = generate_error_message(code)
         line = self._get_line_at_pos(self.pos)
-        self.errors.append(ParseError(code, line=line, column=column, message=message, source_html=self.buffer))
+        self.errors.append(
+            ParseError(code, line=line, column=column, category="tokenizer", message=message, source_html=self.buffer)
+        )
 
     def _emit_error_at_pos(self, code: str, pos: int) -> None:
         last_newline = self.buffer.rfind("\n", 0, pos + 1)
@@ -2009,7 +2107,9 @@ class Tokenizer:
 
         message = generate_error_message(code)
         line = self._get_line_at_pos(pos)
-        self.errors.append(ParseError(code, line=line, column=column, message=message, source_html=self.buffer))
+        self.errors.append(
+            ParseError(code, line=line, column=column, category="tokenizer", message=message, source_html=self.buffer)
+        )
 
     def _consume_if(self, literal: str) -> bool:
         end = self.pos + len(literal)

justhtml/tokens.py

@@ -4,7 +4,7 @@ from typing import Literal
 
 
 class Tag:
-    __slots__ = ("attrs", "kind", "name", "self_closing", "start_pos")
+    __slots__ = ("attrs", "end_pos", "kind", "name", "self_closing", "start_pos")
 
     START: Literal[0] = 0
     END: Literal[1] = 1
@@ -12,6 +12,7 @@ class Tag:
     kind: int
     name: str
     attrs: dict[str, str | None]
+    end_pos: int | None
     self_closing: bool
     start_pos: int | None
 
@@ -22,12 +23,14 @@ class Tag:
         attrs: dict[str, str | None] | None,
         self_closing: bool = False,
         start_pos: int | None = None,
+        end_pos: int | None = None,
     ) -> None:
         self.kind = kind
         self.name = name
         self.attrs = attrs if attrs is not None else {}
         self.self_closing = bool(self_closing)
         self.start_pos = start_pos
+        self.end_pos = end_pos
 
 
 class CharacterTokens:
@@ -84,6 +87,9 @@ class EOFToken:
     __slots__ = ()
 
 
+AnyToken = Tag | CharacterTokens | CommentToken | DoctypeToken | EOFToken
+
+
 class TokenSinkResult:
     __slots__ = ()
 
@@ -94,8 +100,9 @@ class TokenSinkResult:
 class ParseError:
     """Represents a parse error with location information."""
 
-    __slots__ = ("_end_column", "_source_html", "code", "column", "line", "message")
+    __slots__ = ("_end_column", "_source_html", "category", "code", "column", "line", "message")
 
+    category: str
     code: str
     line: int | None
     column: int | None
@@ -110,10 +117,12 @@ class ParseError:
         code: str,
         line: int | None = None,
         column: int | None = None,
+        category: str = "parse",
         message: str | None = None,
         source_html: str | None = None,
         end_column: int | None = None,
     ) -> None:
+        self.category = category
         self.code = code
         self.line = line
         self.column = column
@@ -123,7 +132,11 @@ class ParseError:
 
     def __repr__(self) -> str:
         if self.line is not None and self.column is not None:
+            if self.category != "parse":
+                return f"ParseError({self.code!r}, line={self.line}, column={self.column}, category={self.category!r})"
             return f"ParseError({self.code!r}, line={self.line}, column={self.column})"
+        if self.category != "parse":
+            return f"ParseError({self.code!r}, category={self.category!r})"
         return f"ParseError({self.code!r})"
 
     def __str__(self) -> str:
@@ -138,7 +151,12 @@ class ParseError:
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, ParseError):
             return NotImplemented
-        return self.code == other.code and self.line == other.line and self.column == other.column
+        return (
+            self.category == other.category
+            and self.code == other.code
+            and self.line == other.line
+            and self.column == other.column
+        )
 
     def as_exception(self, end_column: int | None = None) -> SyntaxError:
         """Convert to a SyntaxError-like exception with source highlighting.