justhtml 0.12.0__py3-none-any.whl → 0.38.0__py3-none-any.whl

justhtml/tokenizer.py

@@ -9,9 +9,9 @@ if TYPE_CHECKING:
 
 from .entities import decode_entities_in_text
 from .errors import generate_error_message
-from .tokens import CommentToken, Doctype, DoctypeToken, EOFToken, ParseError, Tag
+from .tokens import AnyToken, CharacterTokens, CommentToken, Doctype, DoctypeToken, EOFToken, ParseError, Tag
 
-_ATTR_VALUE_UNQUOTED_TERMINATORS = "\t\n\f >&\"'<=`\r\0"
+_ATTR_VALUE_UNQUOTED_TERMINATORS = "\t\n\f >&\"'<=`\0"
 _ASCII_LOWER_TABLE = str.maketrans({chr(code): chr(code + 32) for code in range(65, 91)})
 _RCDATA_ELEMENTS = {"title", "textarea"}
 _RAWTEXT_SWITCH_TAGS = {
@@ -29,8 +29,8 @@ _ATTR_VALUE_DOUBLE_PATTERN = re.compile(r'["&\0]')
 _ATTR_VALUE_SINGLE_PATTERN = re.compile(r"['&\0]")
 _ATTR_VALUE_UNQUOTED_PATTERN = re.compile(f"[{re.escape(_ATTR_VALUE_UNQUOTED_TERMINATORS)}]")
 
-_TAG_NAME_RUN_PATTERN = re.compile(r"[^\t\n\f />\0\r]+")
-_ATTR_NAME_RUN_PATTERN = re.compile(r"[^\t\n\f />=\0\"'<\r]+")
+_TAG_NAME_RUN_PATTERN = re.compile(r"[^\t\n\f />\0]+")
+_ATTR_NAME_RUN_PATTERN = re.compile(r"[^\t\n\f />=\0\"'<]+")
 _COMMENT_RUN_PATTERN = re.compile(r"[^-\0]+")
 _WHITESPACE_PATTERN = re.compile(r"[ \t\n\f]+")
 
@@ -44,6 +44,13 @@ for _plane in range(17):
 _XML_COERCION_PATTERN = re.compile(r"[\f\uFDD0-\uFDEF" + "".join(_xml_invalid_single_chars) + "]")
 
 
+def _is_noncharacter_codepoint(codepoint: int) -> bool:
+    if 0xFDD0 <= codepoint <= 0xFDEF:
+        return True
+    last = codepoint & 0xFFFF
+    return last == 0xFFFE or last == 0xFFFF
+
+
 def _xml_coercion_callback(match: re.Match[str]) -> str:
     if match.group(0) == "\f":
         return " "
@@ -72,7 +79,14 @@ def _coerce_comment_for_xml(text: str) -> str:
 
 
 class TokenizerOpts:
-    __slots__ = ("discard_bom", "exact_errors", "initial_rawtext_tag", "initial_state", "xml_coercion")
+    __slots__ = (
+        "discard_bom",
+        "emit_bogus_markup_as_text",
+        "exact_errors",
+        "initial_rawtext_tag",
+        "initial_state",
+        "xml_coercion",
+    )
 
     discard_bom: bool
     exact_errors: bool
@@ -84,12 +98,14 @@ class TokenizerOpts:
         self,
         exact_errors: bool = False,
         discard_bom: bool = True,
+        emit_bogus_markup_as_text: bool = False,
         initial_state: int | None = None,
         initial_rawtext_tag: str | None = None,
         xml_coercion: bool = False,
     ) -> None:
         self.exact_errors = bool(exact_errors)
         self.discard_bom = bool(discard_bom)
+        self.emit_bogus_markup_as_text = bool(emit_bogus_markup_as_text)
         self.initial_state = initial_state
         self.initial_rawtext_tag = initial_rawtext_tag
         self.xml_coercion = bool(xml_coercion)
@@ -178,11 +194,12 @@ class Tokenizer:
         "current_tag_kind",
         "current_tag_name",
         "current_tag_self_closing",
+        "current_token_start_pos",
         "errors",
-        "ignore_lf",
         "last_start_tag_name",
         "last_token_column",
         "last_token_line",
+        "last_token_start_pos",
         "length",
         "opts",
         "original_tag_name",
@@ -194,6 +211,8 @@ class Tokenizer:
         "temp_buffer",
         "text_buffer",
         "text_start_pos",
+        "track_node_locations",
+        "track_tag_positions",
     )
 
     _comment_token: CommentToken
@@ -202,6 +221,8 @@ class Tokenizer:
     _tag_token: Tag
     buffer: str
     collect_errors: bool
+    track_tag_positions: bool
+    track_node_locations: bool
     current_attr_name: list[str]
     current_attr_value: list[str]
     current_attr_value_has_amp: bool
@@ -215,11 +236,12 @@ class Tokenizer:
     current_tag_kind: int
     current_tag_name: list[str]
     current_tag_self_closing: bool
+    current_token_start_pos: int
     errors: list[ParseError]
-    ignore_lf: bool
     last_start_tag_name: str | None
     last_token_column: int
     last_token_line: int
+    last_token_start_pos: int | None
     length: int
     opts: TokenizerOpts
     original_tag_name: list[str]
@@ -234,10 +256,20 @@ class Tokenizer:
 
     # _STATE_HANDLERS is defined at the end of the file
 
-    def __init__(self, sink: Any, opts: TokenizerOpts | None = None, collect_errors: bool = False) -> None:
+    def __init__(
+        self,
+        sink: Any,
+        opts: TokenizerOpts | None = None,
+        *,
+        collect_errors: bool = False,
+        track_node_locations: bool = False,
+        track_tag_positions: bool = False,
+    ) -> None:
         self.sink = sink
         self.opts = opts or TokenizerOpts()
         self.collect_errors = collect_errors
+        self.track_node_locations = bool(track_node_locations)
+        self.track_tag_positions = bool(track_tag_positions)
         self.errors = []
 
         self.state = self.DATA
@@ -246,9 +278,10 @@ class Tokenizer:
         self.pos = 0
         self.reconsume = False
         self.current_char = ""
-        self.ignore_lf = False
         self.last_token_line = 1
         self.last_token_column = 0
+        self.current_token_start_pos = 0
+        self.last_token_start_pos = None
 
         # Reusable buffers to avoid per-token allocations.
         self.text_buffer = []
@@ -276,14 +309,20 @@ class Tokenizer:
         if html and html[0] == "\ufeff" and self.opts.discard_bom:
             html = html[1:]
 
+        # Normalize newlines per §13.2.2.5
+        if html:
+            if "\r" in html:
+                html = html.replace("\r\n", "\n").replace("\r", "\n")
+
         self.buffer = html or ""
         self.length = len(self.buffer)
         self.pos = 0
         self.reconsume = False
         self.current_char = ""
-        self.ignore_lf = False
         self.last_token_line = 1
         self.last_token_column = 0
+        self.current_token_start_pos = 0
+        self.last_token_start_pos = None
         self.errors = []
         self.text_buffer.clear()
         self.text_start_pos = 0
@@ -313,8 +352,9 @@ class Tokenizer:
         else:
             self.state = self.DATA
 
-        # Pre-compute newline positions for O(log n) line lookups
-        if self.collect_errors:
+        # Pre-compute newline positions for O(log n) line lookups.
+        # Only do this when errors are collected or when node locations are requested.
+        if self.collect_errors or self.track_node_locations:
             self._newline_positions = []
             pos = -1
             buffer = self.buffer
@@ -334,6 +374,34 @@ class Tokenizer:
             return 1
         return bisect_right(newline_positions, pos - 1) + 1
 
+    def location_at_pos(self, pos: int) -> tuple[int, int]:
+        """Return (line, column) for a 0-indexed offset in the current buffer.
+
+        Column is 1-indexed. Newline positions are computed lazily when needed.
+        """
+        newline_positions = self._newline_positions
+        if newline_positions is None:
+            newline_positions = []
+            scan = -1
+            buffer = self.buffer
+            while True:
+                scan = buffer.find("\n", scan + 1)
+                if scan == -1:
+                    break
+                newline_positions.append(scan)
+            self._newline_positions = newline_positions
+
+        line_index = bisect_right(newline_positions, pos - 1)
+        line = line_index + 1
+
+        # Compute column using newline index rather than rfind() to avoid O(n) scans.
+        if line_index == 0:
+            last_newline = -1
+        else:
+            last_newline = newline_positions[line_index - 1]
+        column = pos - last_newline
+        return line, column
+
     def step(self) -> bool:
         """Run one step of the tokenizer state machine. Returns True if EOF reached."""
         handler = self._STATE_HANDLERS[self.state]  # type: ignore[attr-defined]
@@ -341,8 +409,9 @@ class Tokenizer:
 
     def run(self, html: str | None) -> None:
         self.initialize(html)
+        handlers = self._STATE_HANDLERS  # type: ignore[attr-defined]
         while True:
-            if self.step():
+            if handlers[self.state](self):  # type: ignore[no-any-return]
                 break
 
     # ---------------------
@@ -356,9 +425,8 @@ class Tokenizer:
             return self.buffer[peek_pos]
         return None
 
-    def _append_text_chunk(self, chunk: str, *, ends_with_cr: bool = False) -> None:
+    def _append_text_chunk(self, chunk: str) -> None:
         self._append_text(chunk)
-        self.ignore_lf = ends_with_cr
 
     # ---------------------
     # State handlers
@@ -392,12 +460,12 @@ class Tokenizer:
 
             if end > pos:
                 chunk = buffer[pos:end]
-
-                if "\r" in chunk:
-                    chunk = chunk.replace("\r\n", "\n").replace("\r", "\n")
-
+                if self.collect_errors and not chunk.isascii():
+                    base_pos = pos
+                    for offset, ch in enumerate(chunk):
+                        if _is_noncharacter_codepoint(ord(ch)):
+                            self._emit_error_at_pos("noncharacter-in-input-stream", base_pos + offset)
                 self._append_text(chunk)
-                self.ignore_lf = chunk.endswith("\r")
 
                 pos = end
                 self.pos = pos
@@ -410,8 +478,8 @@ class Tokenizer:
             pos += 1
             self.pos = pos
             self.current_char = c
-            self.ignore_lf = False
             # c is always '<' here due to find() optimization above
+            self.current_token_start_pos = pos - 1
             # Optimization: Peek ahead for common tag starts
             if pos < length:
                 nc = buffer[pos]
@@ -432,7 +500,7 @@ class Tokenizer:
                     self.state = self.TAG_NAME
                     return self._state_tag_name()
 
-                if nc == "!":
+                if nc == "!" and not self.opts.emit_bogus_markup_as_text:
                     # Optimization: Peek ahead for comments
                     if pos + 2 < length and buffer[pos + 1] == "-" and buffer[pos + 2] == "-":
                         self._flush_text()
@@ -475,12 +543,20 @@ class Tokenizer:
             self._emit_token(EOFToken())
             return True
         if c == "!":
+            if self.opts.emit_bogus_markup_as_text:
+                self._append_text("<!")
+                self.state = self.DATA
+                return False
             self.state = self.MARKUP_DECLARATION_OPEN
             return False
         if c == "/":
             self.state = self.END_TAG_OPEN
             return False
         if c == "?":
+            if self.opts.emit_bogus_markup_as_text:
+                self._append_text("<?")
+                self.state = self.DATA
+                return False
             self._emit_error("unexpected-question-mark-instead-of-tag-name")
             self.current_comment.clear()
             self._reconsume_current()
@@ -497,6 +573,11 @@ class Tokenizer:
         c = self._get_char()
         if c is None:
             self._emit_error("eof-before-tag-name")
+            if self.opts.emit_bogus_markup_as_text:
+                self._append_text("</")
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
             self._append_text("<")
             self._append_text("/")
             self._flush_text()
@@ -504,6 +585,16 @@ class Tokenizer:
             return True
         if c == ">":
             self._emit_error("empty-end-tag")
+            if self.opts.emit_bogus_markup_as_text:
+                self._append_text("</>")
+                self.state = self.DATA
+                return False
+            self.state = self.DATA
+            return False
+
+        if self.opts.emit_bogus_markup_as_text:
+            self._append_text("</")
+            self._append_text(c)
             self.state = self.DATA
             return False
 
@@ -518,15 +609,15 @@ class Tokenizer:
         append_tag_char = self.current_tag_name.append
         buffer = self.buffer
         length = self.length
+        pos = self.pos
 
         while True:
             # Inline _consume_tag_name_run
-            # Note: reconsume and ignore_lf are never True when entering TAG_NAME
-            pos = self.pos
+            # Note: reconsume is never True when entering TAG_NAME
             if pos < length:
                 # Optimization: Check for common terminators before regex
                 match = None
-                if buffer[pos] not in "\t\n\f />\0\r":
+                if buffer[pos] not in "\t\n\f />\0":
                     match = _TAG_NAME_RUN_PATTERN.match(buffer, pos)
 
                 if match:
@@ -534,46 +625,68 @@ class Tokenizer:
                     if not chunk.islower():
                         chunk = chunk.translate(_ASCII_LOWER_TABLE)
                     append_tag_char(chunk)
-                    self.pos = match.end()
-
-                    if self.pos < length:
-                        c = buffer[self.pos]
-                        if c in (" ", "\t", "\n", "\f", "\r"):
-                            self.pos += 1
-                            if c == "\r":
-                                self.ignore_lf = True
+                    pos = match.end()
+
+                    if pos < length:
+                        next_char = buffer[pos]
+                        if next_char in (" ", "\t", "\n", "\f"):
+                            if self.current_tag_kind == Tag.END and self.opts.emit_bogus_markup_as_text:
+                                return self._emit_raw_end_tag_as_text(pos)
+                            pos += 1
+                            self.pos = pos
                             self.state = self.BEFORE_ATTRIBUTE_NAME
                             return self._state_before_attribute_name()
-                        if c == ">":
-                            self.pos += 1
+                        if next_char == ">":
+                            pos += 1
+                            self.pos = pos
                             if not self._emit_current_tag():
                                 self.state = self.DATA
                             return False
-                        if c == "/":
-                            self.pos += 1
+                        if next_char == "/":
+                            if self.current_tag_kind == Tag.END and self.opts.emit_bogus_markup_as_text:
+                                return self._emit_raw_end_tag_as_text(pos)
+                            pos += 1
+                            self.pos = pos
                             self.state = self.SELF_CLOSING_START_TAG
                             return self._state_self_closing_start_tag()
 
-            c = self._get_char()  # type: ignore[assignment]
+            # Inline _get_char
+            # Note: reconsume is never True in this state.
+            if pos >= length:
+                c: str | None = None
+            else:
+                c = buffer[pos]
+                pos += 1
+            self.current_char = c
             if c is None:
+                self.pos = pos
                 self._emit_error("eof-in-tag")
-                # Per HTML5 spec: EOF in tag name is a parse error, emit EOF token only
-                # The incomplete tag is discarded (not emitted as text)
+                self._emit_incomplete_tag_as_text()
                 self._emit_token(EOFToken())
                 return True
             if c in ("\t", "\n", "\f", " "):
+                if self.current_tag_kind == Tag.END and self.opts.emit_bogus_markup_as_text:
+                    self.pos = pos
+                    return self._emit_raw_end_tag_as_text(pos)
+                self.pos = pos
                 self.state = self.BEFORE_ATTRIBUTE_NAME
                 return self._state_before_attribute_name()
             if c == "/":
+                if self.current_tag_kind == Tag.END and self.opts.emit_bogus_markup_as_text:
+                    self.pos = pos
+                    return self._emit_raw_end_tag_as_text(pos)
+                self.pos = pos
                 self.state = self.SELF_CLOSING_START_TAG
                 return self._state_self_closing_start_tag()
             if c == ">":
                 # In slow path, tag name is only first char (from DATA),
                 # so no rawtext elements possible - always set DATA state
+                self.pos = pos
                 self._emit_current_tag()
                 self.state = self.DATA
                 return False
             # c == "\0" - the only remaining possibility after fast-path
+            self.pos = pos
             self._emit_error("unexpected-null-character")
             append_tag_char(replacement)
 
@@ -583,7 +696,7 @@ class Tokenizer:
 
         while True:
             # Optimization: Skip whitespace
-            if not self.reconsume and not self.ignore_lf:
+            if not self.reconsume:
                 if self.pos < length:
                     # Check if current char is whitespace before running regex
                     if buffer[self.pos] in " \t\n\f":
@@ -603,25 +716,12 @@ class Tokenizer:
 
             self.current_char = c
 
-            if c == " ":
-                self.ignore_lf = False
-                continue
-            if c == "\n":
-                if self.ignore_lf:
-                    self.ignore_lf = False
-                # Line tracking now computed on-demand via _get_line_at_pos()
-                continue
-            if c == "\t" or c == "\f":
-                self.ignore_lf = False
-                continue
-            if c == "\r":
-                self.ignore_lf = False
-                if self.pos < length and buffer[self.pos] == "\n":
-                    self.pos += 1
+            if c in (" ", "\n", "\t", "\f"):
                 continue
 
             if c is None:
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._flush_text()
                 self._emit_token(EOFToken())
                 return True
@@ -661,52 +761,62 @@ class Tokenizer:
         append_attr_char = self.current_attr_name.append
         buffer = self.buffer
         length = self.length
+        pos = self.pos
 
         while True:
             # Inline _consume_attribute_name_run
-            if not self.reconsume and not self.ignore_lf:
-                pos = self.pos
-                if pos < length:
-                    # Optimization: Check for common terminators before regex
-                    match = None
-                    if buffer[pos] not in "\t\n\f />=\0\"'<\r":
-                        match = _ATTR_NAME_RUN_PATTERN.match(buffer, pos)
-
-                    if match:
-                        chunk = match.group(0)
-                        if not chunk.islower():
-                            chunk = chunk.translate(_ASCII_LOWER_TABLE)
-                        append_attr_char(chunk)
-                        self.pos = match.end()
-
-                        if self.pos < length:
-                            c = buffer[self.pos]
-                            if c == "=":
-                                self.pos += 1
-                                self.state = self.BEFORE_ATTRIBUTE_VALUE
-                                return self._state_before_attribute_value()
-                            if c in (" ", "\t", "\n", "\f", "\r"):
-                                self.pos += 1
-                                if c == "\r":
-                                    self.ignore_lf = True
-                                self._finish_attribute()
-                                self.state = self.AFTER_ATTRIBUTE_NAME
-                                return False  # Let main loop dispatch to avoid recursion
-                            if c == ">":
-                                self.pos += 1
-                                self._finish_attribute()
-                                if not self._emit_current_tag():
-                                    self.state = self.DATA
-                                return False
-                            if c == "/":
-                                self.pos += 1
-                                self._finish_attribute()
-                                self.state = self.SELF_CLOSING_START_TAG
-                                return self._state_self_closing_start_tag()
-
-            c = self._get_char()  # type: ignore[assignment]
+            # Note: reconsume is never True in this state.
+            if pos < length:
+                # Optimization: Check for common terminators before regex
+                match = None
+                if buffer[pos] not in "\t\n\f />=\0\"'<":
+                    match = _ATTR_NAME_RUN_PATTERN.match(buffer, pos)
+
+                if match:
+                    chunk = match.group(0)
+                    if not chunk.islower():
+                        chunk = chunk.translate(_ASCII_LOWER_TABLE)
+                    append_attr_char(chunk)
+                    pos = match.end()
+
+                    if pos < length:
+                        next_char = buffer[pos]
+                        if next_char == "=":
+                            pos += 1
+                            self.pos = pos
+                            self.state = self.BEFORE_ATTRIBUTE_VALUE
+                            return self._state_before_attribute_value()
+                        if next_char in (" ", "\t", "\n", "\f"):
+                            pos += 1
+                            self.pos = pos
+                            self._finish_attribute()
+                            self.state = self.AFTER_ATTRIBUTE_NAME
+                            return False  # Let main loop dispatch to avoid recursion
+                        if next_char == ">":
+                            pos += 1
+                            self.pos = pos
+                            self._finish_attribute()
+                            if not self._emit_current_tag():
+                                self.state = self.DATA
+                            return False
+                        if next_char == "/":
+                            pos += 1
+                            self.pos = pos
+                            self._finish_attribute()
+                            self.state = self.SELF_CLOSING_START_TAG
+                            return self._state_self_closing_start_tag()
+
+            # Inline _get_char (reconsume is never True in this state)
+            if pos >= length:
+                c: str | None = None
+            else:
+                c = buffer[pos]
+                pos += 1
+            self.current_char = c
+            self.pos = pos
             if c is None:
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._flush_text()
                 self._emit_token(EOFToken())
                 return True
@@ -730,8 +840,7 @@ class Tokenizer:
                 self._emit_error("unexpected-null-character")
                 append_attr_char(replacement)
                 continue
-            if c in ('"', "'", "<"):
-                self._emit_error("unexpected-character-in-attribute-name")
+            self._emit_error("unexpected-character-in-attribute-name")
             append_attr_char(c)
 
     def _state_after_attribute_name(self) -> bool:
@@ -740,11 +849,10 @@ class Tokenizer:
 
         while True:
             # Optimization: Skip whitespace
-            if not self.reconsume and not self.ignore_lf:
+            if not self.reconsume:
                 if self.pos < length:
-                    match = _WHITESPACE_PATTERN.match(buffer, self.pos)
-                    if match:
-                        self.pos = match.end()
+                    if buffer[self.pos] in " \t\n\f":
+                        self.pos = _WHITESPACE_PATTERN.match(buffer, self.pos).end()  # type: ignore[union-attr]
 
             # Inline _get_char
             if self.pos >= length:
@@ -755,25 +863,12 @@ class Tokenizer:
 
             self.current_char = c
 
-            if c == " ":
-                self.ignore_lf = False
-                continue
-            if c == "\n":
-                # Note: Only reachable when ignore_lf=True (CR-LF handling)
-                # Standalone \n is caught by whitespace optimization
-                self.ignore_lf = False
+            if c in (" ", "\n", "\t", "\f"):
                 continue
-            if c == "\r":
-                self.ignore_lf = True
-                continue
-            if c == "\t" or c == "\f":
-                self.ignore_lf = False
-                continue
-
-            self.ignore_lf = False
 
             if c is None:
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._flush_text()
                 self._emit_token(EOFToken())
                 return True
@@ -804,9 +899,17 @@ class Tokenizer:
 
     def _state_before_attribute_value(self) -> bool:
         while True:
-            c = self._get_char()
+            # Inline _get_char (reconsume is never True in this state)
+            pos = self.pos
+            if pos >= self.length:
+                c: str | None = None
+            else:
+                c = self.buffer[pos]
+                self.pos = pos + 1
+            self.current_char = c
             if c is None:
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._flush_text()
                 self._emit_token(EOFToken())
                 return True
@@ -857,10 +960,6 @@ class Tokenizer:
                     if end != next_quote:
                         chunk = buffer[pos:end]
 
-                    # Normalize chunk for value if needed
-                    if "\r" in chunk:
-                        chunk = chunk.replace("\r\n", "\n").replace("\r", "\n")
-
                     self.current_attr_value.append(chunk)
                     self.pos = end
 
@@ -868,6 +967,7 @@ class Tokenizer:
             if self.pos >= length:
                 self.current_char = None
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._emit_token(EOFToken())
                 return True
 
@@ -916,10 +1016,6 @@ class Tokenizer:
                     if end != next_quote:
                         chunk = buffer[pos:end]
 
-                    # Normalize chunk for value if needed
-                    if "\r" in chunk:
-                        chunk = chunk.replace("\r\n", "\n").replace("\r", "\n")
-
                     self.current_attr_value.append(chunk)
                     self.pos = end
 
@@ -927,6 +1023,7 @@ class Tokenizer:
             if self.pos >= length:
                 self.current_char = None
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._emit_token(EOFToken())
                 return True
 
@@ -965,11 +1062,22 @@ class Tokenizer:
                         self.current_attr_value.append(buffer[pos:end])
                         self.pos = end
 
-            c = self._get_char()
+            # Inline _get_char
+            if self.reconsume:
+                self.reconsume = False
+                c = self.current_char
+            elif self.pos >= length:
+                c = None
+            else:
+                c = buffer[self.pos]
+                self.pos += 1
+            self.current_char = c
+
             if c is None:
                 # Per HTML5 spec: EOF in attribute value is a parse error
                 # The incomplete tag is discarded (not emitted)
                 self._emit_error("eof-in-tag")
+                self._emit_incomplete_tag_as_text()
                 self._emit_token(EOFToken())
                 return True
             if c in ("\t", "\n", "\f", " "):
@@ -995,9 +1103,17 @@ class Tokenizer:
 
     def _state_after_attribute_value_quoted(self) -> bool:
         """After attribute value (quoted) state per HTML5 spec §13.2.5.42"""
-        c = self._get_char()
+        # Inline _get_char
+        if self.pos >= self.length:
+            c: str | None = None
+        else:
+            c = self.buffer[self.pos]
+            self.pos += 1
+        self.current_char = c
+
         if c is None:
             self._emit_error("eof-in-tag")
+            self._emit_incomplete_tag_as_text()
             self._flush_text()
             self._emit_token(EOFToken())
             return True
@@ -1025,6 +1141,7 @@ class Tokenizer:
         c = self._get_char()
         if c is None:
             self._emit_error("eof-in-tag")
+            self._emit_incomplete_tag_as_text()
             self._flush_text()
             self._emit_token(EOFToken())
             return True
@@ -1125,7 +1242,14 @@ class Tokenizer:
         while True:
             if self._consume_comment_run():
                 continue
-            c = self._get_char()
+            # Inline _get_char
+            if self.pos >= self.length:
+                c: str | None = None
+            else:
+                c = self.buffer[self.pos]
+                self.pos += 1
+            self.current_char = c
+
             if c is None:
                 self._emit_error("eof-in-comment")
                 self._emit_comment()
@@ -1264,7 +1388,7 @@ class Tokenizer:
         while True:
             c = self._get_char()
             if c is None:
-                self._emit_error("eof-in-doctype-name")
+                self._emit_error("eof-in-doctype")
                 self.current_doctype_force_quirks = True
                 self._emit_doctype()
                 self._emit_token(EOFToken())
@@ -1291,7 +1415,7 @@ class Tokenizer:
         while True:
             c = self._get_char()
             if c is None:
-                self._emit_error("eof-in-doctype-name")
+                self._emit_error("eof-in-doctype")
                 self.current_doctype_force_quirks = True
                 self._emit_doctype()
                 self._emit_token(EOFToken())
@@ -1675,36 +1799,19 @@ class Tokenizer:
             self.reconsume = False
             return self.current_char
 
-        buffer = self.buffer
         pos = self.pos
-        length = self.length
-        while True:
-            if pos >= length:
-                self.pos = pos
-                self.current_char = None
-                return None
-
-            c = buffer[pos]
-            pos += 1
-
-            if c == "\r":
-                self.ignore_lf = True
-                self.current_char = "\n"
-                self.pos = pos
-                return "\n"
-
-            if c == "\n":
-                if self.ignore_lf:
-                    self.ignore_lf = False
-                    continue
-                # Line tracking now computed on-demand via _get_line_at_pos()
+        if pos >= self.length:
+            self.current_char = None
+            return None
 
-            else:
-                self.ignore_lf = False
-
-            self.current_char = c
-            self.pos = pos
-            return c
+        c = self.buffer[pos]
+        self.pos = pos + 1
+        self.current_char = c
+        if c == "<":
+            self.current_token_start_pos = pos
+        if self.collect_errors and not c.isascii() and _is_noncharacter_codepoint(ord(c)):
+            self._emit_error_at_pos("noncharacter-in-input-stream", pos)
+        return c
 
     def _reconsume_current(self) -> None:
         self.reconsume = True
@@ -1731,10 +1838,38 @@ class Tokenizer:
         raw_len = len(data)
 
         self.text_buffer.clear()
-        if self.state == self.DATA and "\0" in data:
-            count = data.count("\0")
-            for _ in range(count):
-                self._emit_error("unexpected-null-character")
+        # U+0000 NULL is a parse error in text.
+        # Emit one error per NULL at the *actual* character position.
+        if "\0" in data:
+            base_pos = self.text_start_pos
+            search_from = 0
+            while True:
+                idx = data.find("\0", search_from)
+                if idx == -1:
+                    break
+                error_pos = base_pos + idx
+
+                # Compute column at error_pos (1-indexed).
+                last_newline = self.buffer.rfind("\n", 0, error_pos + 1)
+                if last_newline == -1:
+                    column = error_pos + 1
+                else:
+                    column = error_pos - last_newline
+                line = self._get_line_at_pos(error_pos)
+
+                message = generate_error_message("unexpected-null-character")
+                self.errors.append(
+                    ParseError(
+                        "unexpected-null-character",
+                        line=line,
+                        column=column,
+                        category="tokenizer",
+                        message=message,
+                        source_html=self.buffer,
+                    )
+                )
+
+                search_from = idx + 1
 
         # Per HTML5 spec:
         # - RCDATA state (title, textarea): decode character references
@@ -1747,13 +1882,16 @@ class Tokenizer:
             pass
         else:
             if "&" in data:
-                data = decode_entities_in_text(data)
+                report_error = self._emit_error if self.collect_errors else None
+                data = decode_entities_in_text(data, report_error=report_error)
         # Apply XML coercion if enabled
         if self.opts.xml_coercion:
             data = _coerce_text_for_xml(data)
 
         # Record position at END of raw text (1-indexed column = raw_len)
-        self._record_text_end_position(raw_len)
+        if self.collect_errors:
+            self._record_text_end_position(raw_len)
+        self.last_token_start_pos = self.text_start_pos
         self.sink.process_characters(data)
         # Note: process_characters never returns Plaintext or RawData
         # State switches happen via _emit_current_tag instead
@@ -1785,7 +1923,8 @@ class Tokenizer:
         else:
             value = "".join(attr_value_buffer)
         if self.current_attr_value_has_amp:
-            value = decode_entities_in_text(value, in_attribute=True)
+            report_error = self._emit_error if self.collect_errors else None
+            value = decode_entities_in_text(value, in_attribute=True, report_error=report_error)
         attrs[name] = value
         attr_value_buffer.clear()
         self.current_attr_value_has_amp = False
@@ -1806,6 +1945,13 @@ class Tokenizer:
         tag.name = name
         tag.attrs = attrs
         tag.self_closing = self.current_tag_self_closing
+        if self.track_tag_positions:
+            tag.start_pos = self.current_token_start_pos
+            tag.end_pos = self.pos
+        else:
+            tag.start_pos = None
+            tag.end_pos = None
+        self.last_token_start_pos = tag.start_pos
 
         switched_to_rawtext = False
         if self.current_tag_kind == Tag.START:
@@ -1831,7 +1977,8 @@ class Tokenizer:
         # Remember current state before emitting
 
         # Emit token to sink
-        self._record_token_position()
+        if self.collect_errors:
+            self._record_token_position()
         result = self.sink.process_token(tag)
         if result == 1:  # TokenSinkResult.Plaintext
             self.state = self.PLAINTEXT
@@ -1844,6 +1991,30 @@ class Tokenizer:
         self.current_tag_kind = Tag.START
         return switched_to_rawtext
 
+    def _emit_incomplete_tag_as_text(self) -> None:
+        if not self.opts.emit_bogus_markup_as_text:
+            return
+        start = self.current_token_start_pos
+        if start is None:  # pragma: no cover
+            return
+        raw = self.buffer[start : self.pos]
+        if raw:  # pragma: no branch
+            self._emit_token(CharacterTokens(raw))
+
+    def _emit_raw_end_tag_as_text(self, pos: int) -> bool:
+        end = self.buffer.find(">", pos)
+        if end == -1:
+            self.pos = self.length
+            self._emit_incomplete_tag_as_text()
+            self._emit_token(EOFToken())
+            return True
+        self.pos = end + 1
+        raw = self.buffer[self.current_token_start_pos : self.pos]
+        if raw:  # pragma: no branch
+            self._emit_token(CharacterTokens(raw))
+        self.state = self.DATA
+        return False
+
     def _emit_comment(self) -> None:
         data = "".join(self.current_comment)
         self.current_comment.clear()
@@ -1851,6 +2022,8 @@ class Tokenizer:
         if self.opts.xml_coercion:
             data = _coerce_comment_for_xml(data)
         self._comment_token.data = data
+        self._comment_token.start_pos = self.current_token_start_pos
+        self.last_token_start_pos = self._comment_token.start_pos
         self._emit_token(self._comment_token)
 
     def _emit_doctype(self) -> None:
@@ -1870,8 +2043,9 @@ class Tokenizer:
         self.current_doctype_force_quirks = False
         self._emit_token(DoctypeToken(doctype))
 
-    def _emit_token(self, token: Any) -> None:
-        self._record_token_position()
+    def _emit_token(self, token: AnyToken) -> None:
+        if self.collect_errors:
+            self._record_token_position()
         self.sink.process_token(token)
         # Note: process_token never returns Plaintext or RawData for state switches
         # State switches happen via _emit_current_tag checking sink response
@@ -1881,8 +2055,6 @@ class Tokenizer:
 
         Per the spec, the position should be at the end of the token (after the last char).
         """
-        if not self.collect_errors:
-            return
         # pos points after the last consumed character, which is exactly what we want
         pos = self.pos
         last_newline = self.buffer.rfind("\n", 0, pos)
@@ -1899,8 +2071,6 @@ class Tokenizer:
         Uses text_start_pos + raw_len to compute where text ends, matching html5lib's
         behavior of reporting the column of the last character (1-indexed).
         """
-        if not self.collect_errors:
-            return
         # Position of last character of text (0-indexed)
         end_pos = self.text_start_pos + raw_len
         last_newline = self.buffer.rfind("\n", 0, end_pos)
@@ -1924,7 +2094,22 @@ class Tokenizer:
 
         message = generate_error_message(code)
         line = self._get_line_at_pos(self.pos)
-        self.errors.append(ParseError(code, line=line, column=column, message=message, source_html=self.buffer))
+        self.errors.append(
+            ParseError(code, line=line, column=column, category="tokenizer", message=message, source_html=self.buffer)
+        )
+
+    def _emit_error_at_pos(self, code: str, pos: int) -> None:
+        last_newline = self.buffer.rfind("\n", 0, pos + 1)
+        if last_newline == -1:
+            column = pos + 1
+        else:
+            column = pos - last_newline
+
+        message = generate_error_message(code)
+        line = self._get_line_at_pos(pos)
+        self.errors.append(
+            ParseError(code, line=line, column=column, category="tokenizer", message=message, source_html=self.buffer)
+        )
 
     def _consume_if(self, literal: str) -> bool:
         end = self.pos + len(literal)
@@ -1953,21 +2138,9 @@ class Tokenizer:
         if pos >= length:
             return False
 
-        # Handle ignore_lf for CRLF sequences
-        if self.ignore_lf and pos < length and self.buffer[pos] == "\n":
-            self.ignore_lf = False
-            pos += 1
-            self.pos = pos
-            if pos >= length:
-                return False
-
         match = _COMMENT_RUN_PATTERN.match(self.buffer, pos)
         if match:
             chunk = match.group(0)
-            # Handle CRLF normalization for comments
-            if "\r" in chunk:
-                chunk = chunk.replace("\r\n", "\n").replace("\r", "\n")
-                self.ignore_lf = chunk.endswith("\r")
             self.current_comment.append(chunk)
             self.pos = match.end()
             return True
@@ -2061,7 +2234,7 @@ class Tokenizer:
             # Consume everything up to the special character
             if next_special > pos:
                 chunk = buffer[pos:next_special]
-                self._append_text_chunk(chunk, ends_with_cr=chunk.endswith("\r"))
+                self._append_text_chunk(chunk)
                 pos = next_special
                 self.pos = pos
 
@@ -2073,7 +2246,6 @@ class Tokenizer:
 
             # Handle special characters - we're at one of them after find()
             if null_index == pos:
-                self.ignore_lf = False
                 self._emit_error("unexpected-null-character")
                 self._append_text("\ufffd")
                 pos += 1
@@ -2188,9 +2360,7 @@ class Tokenizer:
             if null_index != -1 and null_index < next_special:
                 if null_index > pos:
                     chunk = buffer[pos:null_index]
-                    self._append_text_chunk(chunk, ends_with_cr=chunk.endswith("\r"))
-                else:
-                    self.ignore_lf = False
+                    self._append_text_chunk(chunk)
                 self._emit_error("unexpected-null-character")
                 self._append_text("\ufffd")
                 pos = null_index + 1
@@ -2199,14 +2369,14 @@ class Tokenizer:
             if lt_index == -1:
                 if pos < length:
                     chunk = buffer[pos:length]
-                    self._append_text_chunk(chunk, ends_with_cr=chunk.endswith("\r"))
+                    self._append_text_chunk(chunk)
                 self.pos = length
                 self._flush_text()
                 self._emit_token(EOFToken())
                 return True
             if lt_index > pos:
                 chunk = buffer[pos:lt_index]
-                self._append_text_chunk(chunk, ends_with_cr=chunk.endswith("\r"))
+                self._append_text_chunk(chunk)
             pos = lt_index + 1
             self.pos = pos
             # Handle script escaped transition before treating '<' as markup boundary