justhtml 0.6.0__py3-none-any.whl

justhtml/tokenizer.py

@@ -0,0 +1,2590 @@
+import re
+from bisect import bisect_right
+
+from .entities import decode_entities_in_text
+from .errors import generate_error_message
+from .tokens import CommentToken, Doctype, DoctypeToken, EOFToken, ParseError, Tag
+
+_ATTR_VALUE_UNQUOTED_TERMINATORS = "\t\n\f >&\"'<=`\r\0"
+_ASCII_LOWER_TABLE = str.maketrans({chr(code): chr(code + 32) for code in range(65, 91)})
+_RCDATA_ELEMENTS = {"title", "textarea"}
+_RAWTEXT_SWITCH_TAGS = {
+    "script",
+    "style",
+    "xmp",
+    "iframe",
+    "noembed",
+    "noframes",
+    "textarea",
+    "title",
+}
+
+_ATTR_VALUE_DOUBLE_PATTERN = re.compile(r'["&\0]')
+_ATTR_VALUE_SINGLE_PATTERN = re.compile(r"['&\0]")
+_ATTR_VALUE_UNQUOTED_PATTERN = re.compile(f"[{re.escape(_ATTR_VALUE_UNQUOTED_TERMINATORS)}]")
+
+_TAG_NAME_RUN_PATTERN = re.compile(r"[^\t\n\f />\0\r]+")
+_ATTR_NAME_RUN_PATTERN = re.compile(r"[^\t\n\f />=\0\"'<\r]+")
+_COMMENT_RUN_PATTERN = re.compile(r"[^-\0]+")
+_WHITESPACE_PATTERN = re.compile(r"[ \t\n\f]+")
+
+# XML Coercion Regex
+_xml_invalid_single_chars = []
+for _plane in range(17):
+    _base = _plane * 0x10000
+    _xml_invalid_single_chars.append(chr(_base + 0xFFFE))
+    _xml_invalid_single_chars.append(chr(_base + 0xFFFF))
+
+_XML_COERCION_PATTERN = re.compile(r"[\f\uFDD0-\uFDEF" + "".join(_xml_invalid_single_chars) + "]")
+
+
+def _xml_coercion_callback(match):
+    if match.group(0) == "\f":
+        return " "
+    return "\ufffd"
+
+
+def _coerce_text_for_xml(text):
+    """Apply XML coercion to text content."""
+    # Fast path for ASCII
+    if text.isascii():
+        if "\f" in text:
+            return text.replace("\f", " ")
+        return text
+
+    if not _XML_COERCION_PATTERN.search(text):
+        return text
+    return _XML_COERCION_PATTERN.sub(_xml_coercion_callback, text)
+
+
+def _coerce_comment_for_xml(text):
+    """Apply XML coercion to comment content - handle double hyphens."""
+    # Replace -- with - - (with space)
+    if "--" in text:
+        return text.replace("--", "- -")
+    return text
+
+
+class TokenizerOpts:
+    __slots__ = ("discard_bom", "exact_errors", "initial_rawtext_tag", "initial_state", "xml_coercion")
+
+    def __init__(
+        self,
+        exact_errors=False,
+        discard_bom=True,
+        initial_state=None,
+        initial_rawtext_tag=None,
+        xml_coercion=False,
+    ):
+        self.exact_errors = bool(exact_errors)
+        self.discard_bom = bool(discard_bom)
+        self.initial_state = initial_state
+        self.initial_rawtext_tag = initial_rawtext_tag
+        self.xml_coercion = bool(xml_coercion)
+
+
+class Tokenizer:
+    DATA = 0
+    TAG_OPEN = 1
+    END_TAG_OPEN = 2
+    TAG_NAME = 3
+    BEFORE_ATTRIBUTE_NAME = 4
+    ATTRIBUTE_NAME = 5
+    AFTER_ATTRIBUTE_NAME = 6
+    BEFORE_ATTRIBUTE_VALUE = 7
+    ATTRIBUTE_VALUE_DOUBLE = 8
+    ATTRIBUTE_VALUE_SINGLE = 9
+    ATTRIBUTE_VALUE_UNQUOTED = 10
+    AFTER_ATTRIBUTE_VALUE_QUOTED = 11
+    SELF_CLOSING_START_TAG = 12
+    MARKUP_DECLARATION_OPEN = 13
+    COMMENT_START = 14
+    COMMENT_START_DASH = 15
+    COMMENT = 16
+    COMMENT_END_DASH = 17
+    COMMENT_END = 18
+    COMMENT_END_BANG = 19
+    BOGUS_COMMENT = 20
+    DOCTYPE = 21
+    BEFORE_DOCTYPE_NAME = 22
+    DOCTYPE_NAME = 23
+    AFTER_DOCTYPE_NAME = 24
+    BOGUS_DOCTYPE = 25
+    AFTER_DOCTYPE_PUBLIC_KEYWORD = 26
+    AFTER_DOCTYPE_SYSTEM_KEYWORD = 27
+    BEFORE_DOCTYPE_PUBLIC_IDENTIFIER = 28
+    DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED = 29
+    DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED = 30
+    AFTER_DOCTYPE_PUBLIC_IDENTIFIER = 31
+    BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS = 32
+    BEFORE_DOCTYPE_SYSTEM_IDENTIFIER = 33
+    DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED = 34
+    DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED = 35
+    AFTER_DOCTYPE_SYSTEM_IDENTIFIER = 36
+    CDATA_SECTION = 37
+    CDATA_SECTION_BRACKET = 38
+    CDATA_SECTION_END = 39
+    RCDATA = 40
+    RCDATA_LESS_THAN_SIGN = 41
+    RCDATA_END_TAG_OPEN = 42
+    RCDATA_END_TAG_NAME = 43
+    RAWTEXT = 44
+    RAWTEXT_LESS_THAN_SIGN = 45
+    RAWTEXT_END_TAG_OPEN = 46
+    RAWTEXT_END_TAG_NAME = 47
+    PLAINTEXT = 48
+    SCRIPT_DATA_ESCAPED = 49
+    SCRIPT_DATA_ESCAPED_DASH = 50
+    SCRIPT_DATA_ESCAPED_DASH_DASH = 51
+    SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN = 52
+    SCRIPT_DATA_ESCAPED_END_TAG_OPEN = 53
+    SCRIPT_DATA_ESCAPED_END_TAG_NAME = 54
+    SCRIPT_DATA_DOUBLE_ESCAPE_START = 55
+    SCRIPT_DATA_DOUBLE_ESCAPED = 56
+    SCRIPT_DATA_DOUBLE_ESCAPED_DASH = 57
+    SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH = 58
+    SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN = 59
+    SCRIPT_DATA_DOUBLE_ESCAPE_END = 60
+
+    __slots__ = (
+        "_comment_token",
+        "_newline_positions",
+        "_state_handlers",
+        "_tag_token",
+        "buffer",
+        "collect_errors",
+        "current_attr_name",
+        "current_attr_value",
+        "current_attr_value_has_amp",
+        "current_char",
+        "current_comment",
+        "current_doctype_force_quirks",
+        "current_doctype_name",
+        "current_doctype_public",
+        "current_doctype_system",
+        "current_tag_attrs",
+        "current_tag_kind",
+        "current_tag_name",
+        "current_tag_self_closing",
+        "errors",
+        "ignore_lf",
+        "last_start_tag_name",
+        "last_token_column",
+        "last_token_line",
+        "length",
+        "opts",
+        "original_tag_name",
+        "pos",
+        "rawtext_tag_name",
+        "reconsume",
+        "sink",
+        "state",
+        "temp_buffer",
+        "text_buffer",
+        "text_start_pos",
+    )
+
+    # _STATE_HANDLERS is defined at the end of the file
+
+    def __init__(self, sink, opts=None, collect_errors=False):
+        self.sink = sink
+        self.opts = opts or TokenizerOpts()
+        self.collect_errors = collect_errors
+        self.errors = []
+
+        self.state = self.DATA
+        self.buffer = ""
+        self.length = 0
+        self.pos = 0
+        self.reconsume = False
+        self.current_char = ""
+        self.ignore_lf = False
+        self.last_token_line = 1
+        self.last_token_column = 0
+
+        # Reusable buffers to avoid per-token allocations.
+        self.text_buffer = []
+        self.text_start_pos = 0
+        self.current_tag_name = []
+        self.current_tag_attrs = {}
+        self.current_attr_name = []
+        self.current_attr_value = []
+        self.current_attr_value_has_amp = False
+        self.current_tag_self_closing = False
+        self.current_tag_kind = Tag.START
+        self.current_comment = []
+        self.current_doctype_name = []
+        self.current_doctype_public = None  # None = not set, [] = empty string
+        self.current_doctype_system = None  # None = not set, [] = empty string
+        self.current_doctype_force_quirks = False
+        self.last_start_tag_name = None
+        self.rawtext_tag_name = None
+        self.original_tag_name = []
+        self.temp_buffer = []
+        self._tag_token = Tag(Tag.START, "", {}, False)
+        self._comment_token = CommentToken("")
+
+    def initialize(self, html):
+        if html and html[0] == "\ufeff" and self.opts.discard_bom:
+            html = html[1:]
+
+        self.buffer = html or ""
+        self.length = len(self.buffer)
+        self.pos = 0
+        self.reconsume = False
+        self.current_char = ""
+        self.ignore_lf = False
+        self.last_token_line = 1
+        self.last_token_column = 0
+        self.errors = []
+        self.text_buffer.clear()
+        self.text_start_pos = 0
+        self.current_tag_name.clear()
+        self.current_tag_attrs = {}
+        self.current_attr_name.clear()
+        self.current_attr_value.clear()
+        self.current_attr_value_has_amp = False
+        self.current_comment.clear()
+        self.current_doctype_name.clear()
+        self.current_doctype_public = None
+        self.current_doctype_system = None
+        self.current_doctype_force_quirks = False
+        self.current_tag_self_closing = False
+        self.current_tag_kind = Tag.START
+        self.rawtext_tag_name = self.opts.initial_rawtext_tag
+        self.temp_buffer.clear()
+        self.last_start_tag_name = None
+        self._tag_token.kind = Tag.START
+        self._tag_token.name = ""
+        self._tag_token.attrs = {}
+        self._tag_token.self_closing = False
+
+        initial_state = self.opts.initial_state
+        if isinstance(initial_state, int):
+            self.state = initial_state
+        else:
+            self.state = self.DATA
+
+        # Pre-compute newline positions for O(log n) line lookups
+        if self.collect_errors:
+            self._newline_positions = []
+            pos = -1
+            buffer = self.buffer
+            while True:
+                pos = buffer.find("\n", pos + 1)
+                if pos == -1:
+                    break
+                self._newline_positions.append(pos)
+        else:
+            self._newline_positions = None
+
+    def _get_line_at_pos(self, pos):
+        """Get line number (1-indexed) for a position using binary search."""
+        # Line number = count of newlines before pos + 1
+        return bisect_right(self._newline_positions, pos - 1) + 1
+
+    def step(self):
+        """Run one step of the tokenizer state machine. Returns True if EOF reached."""
+        handler = self._STATE_HANDLERS[self.state]
+        return handler(self)
+
+    def run(self, html):
+        self.initialize(html)
+        while True:
+            if self.step():
+                break
+
+    # ---------------------
+    # Helper methods
+    # ---------------------
+
+    def _peek_char(self, offset):
+        """Peek ahead at character at current position + offset without consuming"""
+        peek_pos = self.pos + offset
+        if peek_pos < self.length:
+            return self.buffer[peek_pos]
+        return None
+
+    def _append_text_chunk(self, chunk, *, ends_with_cr=False):
+        self._append_text(chunk)
+        self.ignore_lf = ends_with_cr
+
+    # ---------------------
+    # State handlers
+    # ---------------------
+
+    def _state_data(self):
+        buffer = self.buffer
+        length = self.length
+        pos = self.pos
+        while True:
+            if self.reconsume:
+                # Note: reconsume is never True at EOF in DATA state
+                self.reconsume = False
+                self.pos -= 1
+                pos = self.pos
+
+            if pos >= length:
+                self.pos = length
+                self.current_char = None
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+
+            # Optimized loop using find
+            next_lt = buffer.find("<", pos)
+
+            if next_lt == -1:
+                next_lt = length
+
+            end = next_lt
+
+            if end > pos:
+                chunk = buffer[pos:end]
+
+                if "\r" in chunk:
+                    chunk = chunk.replace("\r\n", "\n").replace("\r", "\n")
+
+                self._append_text(chunk)
+                self.ignore_lf = chunk.endswith("\r")
+
+                pos = end
+                self.pos = pos
+                if pos >= length:
+                    continue
+
+            # After find("<"), we're always at '<' unless reconsume is True
+            # But reconsume only happens after TAG_OPEN which reconsumed '<'
+            c = buffer[pos]
+            pos += 1
+            self.pos = pos
+            self.current_char = c
+            self.ignore_lf = False
+            # c is always '<' here due to find() optimization above
+            # Optimization: Peek ahead for common tag starts
+            if pos < length:
+                nc = buffer[pos]
+                if ("a" <= nc <= "z") or ("A" <= nc <= "Z"):
+                    self._flush_text()
+                    # Inline _start_tag(Tag.START)
+                    self.current_tag_kind = Tag.START
+                    self.current_tag_name.clear()
+                    self.current_attr_name.clear()
+                    self.current_attr_value.clear()
+                    self.current_attr_value_has_amp = False
+                    self.current_tag_self_closing = False
+
+                    if "A" <= nc <= "Z":
+                        nc = chr(ord(nc) + 32)
+                    self.current_tag_name.append(nc)
+                    self.pos += 1
+                    self.state = self.TAG_NAME
+                    return self._state_tag_name()
+
+                if nc == "!":
+                    # Optimization: Peek ahead for comments
+                    if pos + 2 < length and buffer[pos + 1] == "-" and buffer[pos + 2] == "-":
+                        self._flush_text()
+                        self.pos += 3  # Consume !--
+                        self.current_comment.clear()
+                        self.state = self.COMMENT_START
+                        return self._state_comment_start()
+
+                if nc == "/":
+                    # Check next char for end tag
+                    if pos + 1 < length:
+                        nnc = buffer[pos + 1]
+                        if ("a" <= nnc <= "z") or ("A" <= nnc <= "Z"):
+                            self._flush_text()
+                            # Inline _start_tag(Tag.END)
+                            self.current_tag_kind = Tag.END
+                            self.current_tag_name.clear()
+                            self.current_attr_name.clear()
+                            self.current_attr_value.clear()
+                            self.current_attr_value_has_amp = False
+                            self.current_tag_self_closing = False
+
+                            if "A" <= nnc <= "Z":
+                                nnc = chr(ord(nnc) + 32)
+                            self.current_tag_name.append(nnc)
+                            self.pos += 2  # Consume / and nnc
+                            self.state = self.TAG_NAME
+                            return self._state_tag_name()
+
+            self._flush_text()
+            self.state = self.TAG_OPEN
+            return self._state_tag_open()
+
+    def _state_tag_open(self):
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-before-tag-name")
+            self._append_text("<")
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == "!":
+            self.state = self.MARKUP_DECLARATION_OPEN
+            return False
+        if c == "/":
+            self.state = self.END_TAG_OPEN
+            return False
+        if c == "?":
+            self._emit_error("unexpected-question-mark-instead-of-tag-name")
+            self.current_comment.clear()
+            self._reconsume_current()
+            self.state = self.BOGUS_COMMENT
+            return False
+
+        self._emit_error("invalid-first-character-of-tag-name")
+        self._append_text("<")
+        self._reconsume_current()
+        self.state = self.DATA
+        return False
+
+    def _state_end_tag_open(self):
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-before-tag-name")
+            self._append_text("<")
+            self._append_text("/")
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == ">":
+            self._emit_error("empty-end-tag")
+            self.state = self.DATA
+            return False
+
+        self._emit_error("invalid-first-character-of-tag-name")
+        self.current_comment.clear()
+        self._reconsume_current()
+        self.state = self.BOGUS_COMMENT
+        return False
+
+    def _state_tag_name(self):
+        replacement = "\ufffd"
+        append_tag_char = self.current_tag_name.append
+        buffer = self.buffer
+        length = self.length
+
+        while True:
+            # Inline _consume_tag_name_run
+            # Note: reconsume and ignore_lf are never True when entering TAG_NAME
+            pos = self.pos
+            if pos < length:
+                # Optimization: Check for common terminators before regex
+                match = None
+                if buffer[pos] not in "\t\n\f />\0\r":
+                    match = _TAG_NAME_RUN_PATTERN.match(buffer, pos)
+
+                if match:
+                    chunk = match.group(0)
+                    if not chunk.islower():
+                        chunk = chunk.translate(_ASCII_LOWER_TABLE)
+                    append_tag_char(chunk)
+                    self.pos = match.end()
+
+                    if self.pos < length:
+                        c = buffer[self.pos]
+                        if c in (" ", "\t", "\n", "\f", "\r"):
+                            self.pos += 1
+                            if c == "\r":
+                                self.ignore_lf = True
+                            self.state = self.BEFORE_ATTRIBUTE_NAME
+                            return self._state_before_attribute_name()
+                        if c == ">":
+                            self.pos += 1
+                            if not self._emit_current_tag():
+                                self.state = self.DATA
+                            return False
+                        if c == "/":
+                            self.pos += 1
+                            self.state = self.SELF_CLOSING_START_TAG
+                            return self._state_self_closing_start_tag()
+
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-tag")
+                # Per HTML5 spec: EOF in tag name is a parse error, emit EOF token only
+                # The incomplete tag is discarded (not emitted as text)
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                self.state = self.BEFORE_ATTRIBUTE_NAME
+                return self._state_before_attribute_name()
+            if c == "/":
+                self.state = self.SELF_CLOSING_START_TAG
+                return self._state_self_closing_start_tag()
+            if c == ">":
+                # In slow path, tag name is only first char (from DATA),
+                # so no rawtext elements possible - always set DATA state
+                self._emit_current_tag()
+                self.state = self.DATA
+                return False
+            # c == "\0" - the only remaining possibility after fast-path
+            self._emit_error("unexpected-null-character")
+            append_tag_char(replacement)
+
+    def _state_before_attribute_name(self):
+        buffer = self.buffer
+        length = self.length
+
+        while True:
+            # Optimization: Skip whitespace
+            if not self.reconsume and not self.ignore_lf:
+                if self.pos < length:
+                    # Check if current char is whitespace before running regex
+                    if buffer[self.pos] in " \t\n\f":
+                        match = _WHITESPACE_PATTERN.match(buffer, self.pos)
+                        if match:
+                            self.pos = match.end()
+
+            # Inline _get_char
+            if self.reconsume:  # pragma: no cover
+                self.reconsume = False
+                c = self.current_char
+            elif self.pos >= length:
+                c = None
+            else:
+                c = buffer[self.pos]
+                self.pos += 1
+
+            self.current_char = c
+
+            if c == " ":
+                self.ignore_lf = False
+                continue
+            if c == "\n":
+                if self.ignore_lf:
+                    self.ignore_lf = False
+                # Line tracking now computed on-demand via _get_line_at_pos()
+                continue
+            if c == "\t" or c == "\f":
+                self.ignore_lf = False
+                continue
+            if c == "\r":
+                self.ignore_lf = False
+                if self.pos < length and buffer[self.pos] == "\n":
+                    self.pos += 1
+                continue
+
+            if c is None:
+                self._emit_error("eof-in-tag")
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+
+            if c == "/":
+                self.state = self.SELF_CLOSING_START_TAG
+                return False
+            if c == ">":
+                self._finish_attribute()
+                if not self._emit_current_tag():
+                    self.state = self.DATA
+                return False
+            if c == "=":
+                self._emit_error("unexpected-equals-sign-before-attribute-name")
+                self.current_attr_name.clear()
+                self.current_attr_value.clear()
+                self.current_attr_value_has_amp = False
+                self.current_attr_name.append("=")
+                self.state = self.ATTRIBUTE_NAME
+                return False  # Let main loop dispatch to avoid recursion
+
+            self.current_attr_name.clear()
+            self.current_attr_value.clear()
+            self.current_attr_value_has_amp = False
+            if c == "\0":
+                self._emit_error("unexpected-null-character")
+                c = "\ufffd"
+            elif "A" <= c <= "Z":
+                c = chr(ord(c) + 32)
+
+            self.current_attr_name.append(c)
+            self.state = self.ATTRIBUTE_NAME
+            return False  # Let main loop dispatch to avoid recursion
+
+    def _state_attribute_name(self):
+        replacement = "\ufffd"
+        append_attr_char = self.current_attr_name.append
+        buffer = self.buffer
+        length = self.length
+
+        while True:
+            # Inline _consume_attribute_name_run
+            if not self.reconsume and not self.ignore_lf:
+                pos = self.pos
+                if pos < length:
+                    # Optimization: Check for common terminators before regex
+                    match = None
+                    if buffer[pos] not in "\t\n\f />=\0\"'<\r":
+                        match = _ATTR_NAME_RUN_PATTERN.match(buffer, pos)
+
+                    if match:
+                        chunk = match.group(0)
+                        if not chunk.islower():
+                            chunk = chunk.translate(_ASCII_LOWER_TABLE)
+                        append_attr_char(chunk)
+                        self.pos = match.end()
+
+                        if self.pos < length:
+                            c = buffer[self.pos]
+                            if c == "=":
+                                self.pos += 1
+                                self.state = self.BEFORE_ATTRIBUTE_VALUE
+                                return self._state_before_attribute_value()
+                            if c in (" ", "\t", "\n", "\f", "\r"):
+                                self.pos += 1
+                                if c == "\r":
+                                    self.ignore_lf = True
+                                self._finish_attribute()
+                                self.state = self.AFTER_ATTRIBUTE_NAME
+                                return False  # Let main loop dispatch to avoid recursion
+                            if c == ">":
+                                self.pos += 1
+                                self._finish_attribute()
+                                if not self._emit_current_tag():
+                                    self.state = self.DATA
+                                return False
+                            if c == "/":
+                                self.pos += 1
+                                self._finish_attribute()
+                                self.state = self.SELF_CLOSING_START_TAG
+                                return self._state_self_closing_start_tag()
+
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-tag")
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                self._finish_attribute()
+                self.state = self.AFTER_ATTRIBUTE_NAME
+                return False  # Let main loop dispatch to avoid recursion
+            if c == "/":
+                self._finish_attribute()
+                self.state = self.SELF_CLOSING_START_TAG
+                return self._state_self_closing_start_tag()
+            if c == "=":
+                self.state = self.BEFORE_ATTRIBUTE_VALUE
+                return self._state_before_attribute_value()
+            if c == ">":
+                self._finish_attribute()
+                if not self._emit_current_tag():
+                    self.state = self.DATA
+                return False
+            if c == "\0":
+                self._emit_error("unexpected-null-character")
+                append_attr_char(replacement)
+                continue
+            if c in ('"', "'", "<"):
+                self._emit_error("unexpected-character-in-attribute-name")
+            append_attr_char(c)
+
+    def _state_after_attribute_name(self):
+        buffer = self.buffer
+        length = self.length
+
+        while True:
+            # Optimization: Skip whitespace
+            if not self.reconsume and not self.ignore_lf:
+                if self.pos < length:
+                    match = _WHITESPACE_PATTERN.match(buffer, self.pos)
+                    if match:
+                        self.pos = match.end()
+
+            # Inline _get_char
+            if self.pos >= length:
+                c = None
+            else:
+                c = buffer[self.pos]
+                self.pos += 1
+
+            self.current_char = c
+
+            if c == " ":
+                self.ignore_lf = False
+                continue
+            if c == "\n":
+                # Note: Only reachable when ignore_lf=True (CR-LF handling)
+                # Standalone \n is caught by whitespace optimization
+                self.ignore_lf = False
+                continue
+            if c == "\r":
+                self.ignore_lf = True
+                continue
+            if c == "\t" or c == "\f":
+                self.ignore_lf = False
+                continue
+
+            self.ignore_lf = False
+
+            if c is None:
+                self._emit_error("eof-in-tag")
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+            if c == "/":
+                self._finish_attribute()
+                self.state = self.SELF_CLOSING_START_TAG
+                return False
+            if c == "=":
+                self.state = self.BEFORE_ATTRIBUTE_VALUE
+                return False
+            if c == ">":
+                self._finish_attribute()
+                if not self._emit_current_tag():
+                    self.state = self.DATA
+                return False
+            self._finish_attribute()
+            self.current_attr_name.clear()
+            self.current_attr_value.clear()
+            self.current_attr_value_has_amp = False
+            if c == "\0":
+                self._emit_error("unexpected-null-character")
+                c = "\ufffd"
+            elif "A" <= c <= "Z":
+                c = chr(ord(c) + 32)
+            self.current_attr_name.append(c)
+            self.state = self.ATTRIBUTE_NAME
+            return False  # Let main loop dispatch to avoid recursion
+
+    def _state_before_attribute_value(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-tag")
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                continue
+            if c == '"':
+                self.state = self.ATTRIBUTE_VALUE_DOUBLE
+                return self._state_attribute_value_double()
+            if c == "'":
+                self.state = self.ATTRIBUTE_VALUE_SINGLE
+                return self._state_attribute_value_single()
+            if c == ">":
+                self._emit_error("missing-attribute-value")
+                self._finish_attribute()
+                if not self._emit_current_tag():
+                    self.state = self.DATA
+                return False
+            self._reconsume_current()
+            self.state = self.ATTRIBUTE_VALUE_UNQUOTED
+            return self._state_attribute_value_unquoted()
+
+    def _state_attribute_value_double(self):
+        replacement = "\ufffd"
+        stop_pattern = _ATTR_VALUE_DOUBLE_PATTERN
+        buffer = self.buffer
+        length = self.length
+
+        while True:
+            # Inline _consume_attribute_value_run
+            pos = self.pos
+            if pos < length:
+                # Optimization: Optimistically look for quote
+                next_quote = buffer.find('"', pos)
+                if next_quote == -1:
+                    next_quote = length
+
+                # Check if we skipped other terminators
+                chunk = buffer[pos:next_quote]
+                if "&" in chunk or "\0" in chunk:
+                    # Fallback to regex if complex chars present
+                    match = stop_pattern.search(buffer, pos)
+                    # Note: match is always found because we checked for & or \0 above
+                    end = match.start()
+                else:
+                    end = next_quote
+
+                if end > pos:
+                    # chunk is already valid if we took the fast path
+                    if end != next_quote:
+                        chunk = buffer[pos:end]
+
+                    # Normalize chunk for value if needed
+                    if "\r" in chunk:
+                        chunk = chunk.replace("\r\n", "\n").replace("\r", "\n")
+
+                    self.current_attr_value.append(chunk)
+                    self.pos = end
+
+            # Inlined _get_char logic
+            if self.pos >= length:
+                self.current_char = None
+                self._emit_error("eof-in-tag")
+                self._emit_token(EOFToken())
+                return True
+
+            c = buffer[self.pos]
+            self.pos += 1
+
+            self.current_char = c
+
+            if c == '"':
+                self.state = self.AFTER_ATTRIBUTE_VALUE_QUOTED
+                return self._state_after_attribute_value_quoted()
+            if c == "&":
+                self._append_attr_value_char("&")
+                self.current_attr_value_has_amp = True
+            else:
+                # c == "\0" - the only remaining possibility after fast-path
+                self._emit_error("unexpected-null-character")
+                self._append_attr_value_char(replacement)
+
+    def _state_attribute_value_single(self):
+        replacement = "\ufffd"
+        stop_pattern = _ATTR_VALUE_SINGLE_PATTERN
+        buffer = self.buffer
+        length = self.length
+
+        while True:
+            # Inline _consume_attribute_value_run
+            pos = self.pos
+            if pos < length:
+                # Optimization: Optimistically look for quote
+                next_quote = buffer.find("'", pos)
+                if next_quote == -1:
+                    next_quote = length
+
+                # Check if we skipped other terminators
+                chunk = buffer[pos:next_quote]
+                if "&" in chunk or "\0" in chunk:
+                    # Fallback to regex if complex chars present
+                    match = stop_pattern.search(buffer, pos)
+                    # Note: match is always found because we checked for & or \0 above
+                    end = match.start()
+                else:
+                    end = next_quote
+
+                if end > pos:
+                    # chunk is already valid if we took the fast path
+                    if end != next_quote:
+                        chunk = buffer[pos:end]
+
+                    # Normalize chunk for value if needed
+                    if "\r" in chunk:
+                        chunk = chunk.replace("\r\n", "\n").replace("\r", "\n")
+
+                    self.current_attr_value.append(chunk)
+                    self.pos = end
+
+            # Inlined _get_char logic
+            if self.pos >= length:
+                self.current_char = None
+                self._emit_error("eof-in-tag")
+                self._emit_token(EOFToken())
+                return True
+
+            c = buffer[self.pos]
+            self.pos += 1
+
+            self.current_char = c
+
+            if c == "'":
+                self.state = self.AFTER_ATTRIBUTE_VALUE_QUOTED
+                return self._state_after_attribute_value_quoted()
+            if c == "&":
+                self._append_attr_value_char("&")
+                self.current_attr_value_has_amp = True
+            else:
+                # c == "\0" - the only remaining possibility after fast-path
+                self._emit_error("unexpected-null-character")
+                self._append_attr_value_char(replacement)
+
+    def _state_attribute_value_unquoted(self):
+        replacement = "\ufffd"
+        stop_pattern = _ATTR_VALUE_UNQUOTED_PATTERN
+        buffer = self.buffer
+        length = self.length
+
+        while True:
+            # Inline _consume_attribute_value_run
+            if not self.reconsume:
+                pos = self.pos
+                if pos < length:
+                    match = stop_pattern.search(buffer, pos)
+                    # Note: match is always found - pattern matches terminators or EOF
+                    end = match.start() if match else length
+
+                    if end > pos:
+                        self.current_attr_value.append(buffer[pos:end])
+                        self.pos = end
+
+            c = self._get_char()
+            if c is None:
+                # Per HTML5 spec: EOF in attribute value is a parse error
+                # The incomplete tag is discarded (not emitted)
+                self._emit_error("eof-in-tag")
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                self._finish_attribute()
+                self.state = self.BEFORE_ATTRIBUTE_NAME
+                return False
+            if c == ">":
+                self._finish_attribute()
+                if not self._emit_current_tag():
+                    self.state = self.DATA
+                return False
+            if c == "&":
+                self._append_attr_value_char("&")
+                self.current_attr_value_has_amp = True
+                continue
+            if c in ('"', "'", "<", "=", "`"):
+                self._emit_error("unexpected-character-in-unquoted-attribute-value")
+            if c == "\0":
+                self._emit_error("unexpected-null-character")
+                self._append_attr_value_char(replacement)
+                continue
+            self._append_attr_value_char(c)
+
+    def _state_after_attribute_value_quoted(self):
+        """After attribute value (quoted) state per HTML5 spec §13.2.5.42"""
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-in-tag")
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c in ("\t", "\n", "\f", " "):
+            self._finish_attribute()
+            self.state = self.BEFORE_ATTRIBUTE_NAME
+            return False
+        if c == "/":
+            self._finish_attribute()
+            self.state = self.SELF_CLOSING_START_TAG
+            return False
+        if c == ">":
+            self._finish_attribute()
+            if not self._emit_current_tag():
+                self.state = self.DATA
+            return False
+        # Anything else: parse error, reconsume in before attribute name state
+        self._emit_error("missing-whitespace-between-attributes")
+        self._finish_attribute()
+        self._reconsume_current()
+        self.state = self.BEFORE_ATTRIBUTE_NAME
+        return False
+
+    def _state_self_closing_start_tag(self):
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-in-tag")
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == ">":
+            self.current_tag_self_closing = True
+            self._emit_current_tag()
+            self.state = self.DATA
+            return False
+        self._emit_error("unexpected-character-after-solidus-in-tag")
+        self._reconsume_current()
+        self.state = self.BEFORE_ATTRIBUTE_NAME
+        return False
+
+    def _state_markup_declaration_open(self):
+        # Note: Comment handling (<!--) is optimized in DATA state fast-path
+        # This code only handles DOCTYPE and CDATA, or malformed markup
+        if self._consume_case_insensitive("DOCTYPE"):
+            self.current_doctype_name.clear()
+            self.current_doctype_public = None
+            self.current_doctype_system = None
+            self.current_doctype_force_quirks = False
+            self.state = self.DOCTYPE
+            return False
+        if self._consume_if("[CDATA["):
+            # CDATA sections are only valid in foreign content (SVG/MathML)
+            # Check if the adjusted current node is in a foreign namespace
+            stack = self.sink.open_elements
+            if stack:
+                current = stack[-1]
+                if current and current.namespace not in {None, "html"}:
+                    # Proper CDATA section in foreign content
+                    self.state = self.CDATA_SECTION
+                    return False
+            # Treat as bogus comment in HTML context, preserving "[CDATA[" prefix
+            self._emit_error("cdata-in-html-content")
+            self.current_comment.clear()
+            # Add the consumed "[CDATA[" text to the comment
+            for ch in "[CDATA[":
+                self.current_comment.append(ch)
+            self.state = self.BOGUS_COMMENT
+            return False
+        self._emit_error("incorrectly-opened-comment")
+        self.current_comment.clear()
+        # Don't reconsume - bogus comment starts from current position
+        self.state = self.BOGUS_COMMENT
+        return False
+
+    def _state_comment_start(self):
+        replacement = "\ufffd"
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-in-comment")
+            self._emit_comment()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self.state = self.COMMENT_START_DASH
+            return False
+        if c == ">":
+            self._emit_error("abrupt-closing-of-empty-comment")
+            self._emit_comment()
+            self.state = self.DATA
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self.current_comment.append(replacement)
+        else:
+            self.current_comment.append(c)
+        self.state = self.COMMENT
+        return False
+
+    def _state_comment_start_dash(self):
+        replacement = "\ufffd"
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-in-comment")
+            self._emit_comment()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self.state = self.COMMENT_END
+            return False
+        if c == ">":
+            self._emit_error("abrupt-closing-of-empty-comment")
+            self._emit_comment()
+            self.state = self.DATA
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self.current_comment.extend(("-", replacement))
+        else:
+            self.current_comment.extend(("-", c))
+        self.state = self.COMMENT
+        return False
+
+    def _state_comment(self):
+        replacement = "\ufffd"
+        while True:
+            if self._consume_comment_run():
+                continue
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-comment")
+                self._emit_comment()
+                self._emit_token(EOFToken())
+                return True
+            if c == "-":
+                self.state = self.COMMENT_END_DASH
+                return False
+            # c == "\0" - the only remaining possibility after _consume_comment_run
+            self._emit_error("unexpected-null-character")
+            self.current_comment.append(replacement)
+
+    def _state_comment_end_dash(self):
+        replacement = "\ufffd"
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-in-comment")
+            self._emit_comment()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self.state = self.COMMENT_END
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self.current_comment.extend(("-", replacement))
+            self.state = self.COMMENT
+            return False
+        # Per spec: append "-" and current char, switch to COMMENT state
+        self.current_comment.extend(("-", c))
+        self.state = self.COMMENT
+        return False
+
+    def _state_comment_end(self):
+        replacement = "\ufffd"
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-in-comment")
+            self._emit_comment()
+            self._emit_token(EOFToken())
+            return True
+        if c == ">":
+            self._emit_comment()
+            self.state = self.DATA
+            return False
+        if c == "!":
+            self.state = self.COMMENT_END_BANG
+            return False
+        if c == "-":
+            self.current_comment.append("-")
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self.current_comment.extend(("--", replacement))
+            self.state = self.COMMENT
+            return False
+        self._emit_error("incorrectly-closed-comment")
+        self.current_comment.extend(("--", c))
+        self.state = self.COMMENT
+        return False
+
+    def _state_comment_end_bang(self):
+        replacement = "\ufffd"
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-in-comment")
+            self._emit_comment()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self.current_comment.append("-")
+            self.current_comment.append("-")
+            self.current_comment.append("!")
+            self.state = self.COMMENT_END_DASH
+            return False
+        if c == ">":
+            self._emit_error("incorrectly-closed-comment")
+            self._emit_comment()
+            self.state = self.DATA
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self.current_comment.append("-")
+            self.current_comment.append("-")
+            self.current_comment.append("!")
+            self.current_comment.append(replacement)
+            self.state = self.COMMENT
+            return False
+        self.current_comment.append("-")
+        self.current_comment.append("-")
+        self.current_comment.append("!")
+        self.current_comment.append(c)
+        self.state = self.COMMENT
+        return False
+
+    def _state_bogus_comment(self):
+        replacement = "\ufffd"
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_comment()
+                self._emit_token(EOFToken())
+                return True
+            if c == ">":
+                self._emit_comment()
+                self.state = self.DATA
+                return False
+            if c == "\0":
+                self.current_comment.append(replacement)
+            else:
+                self.current_comment.append(c)
+
+    def _state_doctype(self):
+        c = self._get_char()
+        if c is None:
+            self._emit_error("eof-in-doctype")
+            self.current_doctype_force_quirks = True
+            self._emit_doctype()
+            self._emit_token(EOFToken())
+            return True
+        if c in ("\t", "\n", "\f", " "):
+            self.state = self.BEFORE_DOCTYPE_NAME
+            return False
+        if c == ">":
+            self._emit_error("expected-doctype-name-but-got-right-bracket")
+            self.current_doctype_force_quirks = True
+            self._emit_doctype()
+            self.state = self.DATA
+            return False
+        self._emit_error("missing-whitespace-before-doctype-name")
+        self._reconsume_current()
+        self.state = self.BEFORE_DOCTYPE_NAME
+        return False
+
+    def _state_before_doctype_name(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-doctype-name")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                return False
+            if c == ">":
+                self._emit_error("expected-doctype-name-but-got-right-bracket")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            if "A" <= c <= "Z":
+                self.current_doctype_name.append(chr(ord(c) + 32))
+            elif c == "\0":
+                self._emit_error("unexpected-null-character")
+                self.current_doctype_name.append("\ufffd")
+            else:
+                self.current_doctype_name.append(c)
+            self.state = self.DOCTYPE_NAME
+            return False
+
+    def _state_doctype_name(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-doctype-name")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                self.state = self.AFTER_DOCTYPE_NAME
+                return False
+            if c == ">":
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            if "A" <= c <= "Z":
+                self.current_doctype_name.append(chr(ord(c) + 32))
+                continue
+            if c == "\0":
+                self._emit_error("unexpected-null-character")
+                self.current_doctype_name.append("\ufffd")
+                continue
+            self.current_doctype_name.append(c)
+
+    def _state_after_doctype_name(self):
+        if self._consume_case_insensitive("PUBLIC"):
+            self.state = self.AFTER_DOCTYPE_PUBLIC_KEYWORD
+            return False
+        if self._consume_case_insensitive("SYSTEM"):
+            self.state = self.AFTER_DOCTYPE_SYSTEM_KEYWORD
+            return False
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-doctype")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                continue
+            if c == ">":
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self._emit_error("missing-whitespace-after-doctype-name")
+            self.current_doctype_force_quirks = True
+            self._reconsume_current()
+            self.state = self.BOGUS_DOCTYPE
+            return False
+
+    def _state_after_doctype_public_keyword(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("missing-quote-before-doctype-public-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                self.state = self.BEFORE_DOCTYPE_PUBLIC_IDENTIFIER
+                return False
+            if c == '"':
+                self._emit_error("missing-whitespace-before-doctype-public-identifier")
+                self.current_doctype_public = []
+                self.state = self.DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED
+                return False
+            if c == "'":
+                self._emit_error("missing-whitespace-before-doctype-public-identifier")
+                self.current_doctype_public = []
+                self.state = self.DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED
+                return False
+            if c == ">":
+                self._emit_error("missing-doctype-public-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self._emit_error("unexpected-character-after-doctype-public-keyword")
+            self.current_doctype_force_quirks = True
+            self._reconsume_current()
+            self.state = self.BOGUS_DOCTYPE
+            return False
+
+    def _state_after_doctype_system_keyword(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("missing-quote-before-doctype-system-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                self.state = self.BEFORE_DOCTYPE_SYSTEM_IDENTIFIER
+                return False
+            if c == '"':
+                self._emit_error("missing-whitespace-after-doctype-public-identifier")
+                self.current_doctype_system = []
+                self.state = self.DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED
+                return False
+            if c == "'":
+                self._emit_error("missing-whitespace-after-doctype-public-identifier")
+                self.current_doctype_system = []
+                self.state = self.DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED
+                return False
+            if c == ">":
+                self._emit_error("missing-doctype-system-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self._emit_error("unexpected-character-after-doctype-system-keyword")
+            self.current_doctype_force_quirks = True
+            self._reconsume_current()
+            self.state = self.BOGUS_DOCTYPE
+            return False
+
+    def _state_before_doctype_public_identifier(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("missing-doctype-public-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                continue
+            if c == '"':
+                self.current_doctype_public = []
+                self.state = self.DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED
+                return False
+            if c == "'":
+                self.current_doctype_public = []
+                self.state = self.DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED
+                return False
+            if c == ">":
+                self._emit_error("missing-doctype-public-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self._emit_error("missing-quote-before-doctype-public-identifier")
+            self.current_doctype_force_quirks = True
+            self._reconsume_current()
+            self.state = self.BOGUS_DOCTYPE
+            return False
+
+    def _state_doctype_public_identifier_double_quoted(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-doctype-public-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c == '"':
+                self.state = self.AFTER_DOCTYPE_PUBLIC_IDENTIFIER
+                return False
+            if c == "\0":
+                self._emit_error("unexpected-null-character")
+                self.current_doctype_public.append("\ufffd")
+                continue
+            if c == ">":
+                self._emit_error("abrupt-doctype-public-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self.current_doctype_public.append(c)
+
+    def _state_doctype_public_identifier_single_quoted(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-doctype-public-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c == "'":
+                self.state = self.AFTER_DOCTYPE_PUBLIC_IDENTIFIER
+                return False
+            if c == "\0":
+                self._emit_error("unexpected-null-character")
+                self.current_doctype_public.append("\ufffd")
+                continue
+            if c == ">":
+                self._emit_error("abrupt-doctype-public-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self.current_doctype_public.append(c)
+
+    def _state_after_doctype_public_identifier(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("missing-whitespace-between-doctype-public-and-system-identifiers")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                self.state = self.BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS
+                return False
+            if c == ">":
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            if c == '"':
+                self._emit_error("missing-whitespace-between-doctype-public-and-system-identifiers")
+                self.current_doctype_system = []
+                self.state = self.DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED
+                return False
+            if c == "'":
+                self._emit_error("missing-whitespace-between-doctype-public-and-system-identifiers")
+                self.current_doctype_system = []
+                self.state = self.DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED
+                return False
+            self._emit_error("unexpected-character-after-doctype-public-identifier")
+            self.current_doctype_force_quirks = True
+            self._reconsume_current()
+            self.state = self.BOGUS_DOCTYPE
+            return False
+
+    def _state_between_doctype_public_and_system_identifiers(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("missing-quote-before-doctype-system-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                continue
+            if c == ">":
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            if c == '"':
+                self.current_doctype_system = []
+                self.state = self.DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED
+                return False
+            if c == "'":
+                self.current_doctype_system = []
+                self.state = self.DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED
+                return False
+            self._emit_error("missing-quote-before-doctype-system-identifier")
+            self.current_doctype_force_quirks = True
+            self._reconsume_current()
+            self.state = self.BOGUS_DOCTYPE
+            return False
+
+    def _state_before_doctype_system_identifier(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("missing-doctype-system-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                continue
+            if c == '"':
+                self.current_doctype_system = []
+                self.state = self.DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED
+                return False
+            if c == "'":
+                self.current_doctype_system = []
+                self.state = self.DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED
+                return False
+            if c == ">":
+                self._emit_error("missing-doctype-system-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self._emit_error("missing-quote-before-doctype-system-identifier")
+            self.current_doctype_force_quirks = True
+            self._reconsume_current()
+            self.state = self.BOGUS_DOCTYPE
+            return False
+
+    def _state_doctype_system_identifier_double_quoted(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-doctype-system-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c == '"':
+                self.state = self.AFTER_DOCTYPE_SYSTEM_IDENTIFIER
+                return False
+            if c == "\0":
+                self._emit_error("unexpected-null-character")
+                self.current_doctype_system.append("\ufffd")
+                continue
+            if c == ">":
+                self._emit_error("abrupt-doctype-system-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self.current_doctype_system.append(c)
+
+    def _state_doctype_system_identifier_single_quoted(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-doctype-system-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c == "'":
+                self.state = self.AFTER_DOCTYPE_SYSTEM_IDENTIFIER
+                return False
+            if c == "\0":
+                self._emit_error("unexpected-null-character")
+                self.current_doctype_system.append("\ufffd")
+                continue
+            if c == ">":
+                self._emit_error("abrupt-doctype-system-identifier")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self.current_doctype_system.append(c)
+
+    def _state_after_doctype_system_identifier(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-doctype")
+                self.current_doctype_force_quirks = True
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c in ("\t", "\n", "\f", " "):
+                continue
+            if c == ">":
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+            self._emit_error("unexpected-character-after-doctype-system-identifier")
+            self._reconsume_current()
+            self.state = self.BOGUS_DOCTYPE
+            return False
+
+    def _state_bogus_doctype(self):
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_doctype()
+                self._emit_token(EOFToken())
+                return True
+            if c == ">":
+                self._emit_doctype()
+                self.state = self.DATA
+                return False
+
+    # ---------------------
+    # Low-level helpers
+    # ---------------------
+
+    def _get_char(self):
+        if self.reconsume:
+            self.reconsume = False
+            return self.current_char
+
+        buffer = self.buffer
+        pos = self.pos
+        length = self.length
+        while True:
+            if pos >= length:
+                self.pos = pos
+                self.current_char = None
+                return None
+
+            c = buffer[pos]
+            pos += 1
+
+            if c == "\r":
+                self.ignore_lf = True
+                self.current_char = "\n"
+                self.pos = pos
+                return "\n"
+
+            if c == "\n":
+                if self.ignore_lf:
+                    self.ignore_lf = False
+                    continue
+                # Line tracking now computed on-demand via _get_line_at_pos()
+
+            else:
+                self.ignore_lf = False
+
+            self.current_char = c
+            self.pos = pos
+            return c
+
+    def _reconsume_current(self):
+        self.reconsume = True
+
+    def _append_text(self, text):
+        """Append text to buffer, recording start position if this is the first chunk."""
+        if not self.text_buffer:
+            # Record where text started (current position before this chunk)
+            self.text_start_pos = self.pos
+        self.text_buffer.append(text)
+
+    def _flush_text(self):
+        if not self.text_buffer:
+            return
+
+        # Optimization: Avoid join for single chunk
+        # text_buffer is never populated with empty strings
+        if len(self.text_buffer) == 1:
+            data = self.text_buffer[0]
+        else:
+            data = "".join(self.text_buffer)
+
+        # Calculate raw text length before any processing for position tracking
+        raw_len = len(data)
+
+        self.text_buffer.clear()
+        if self.state == self.DATA and "\0" in data:
+            count = data.count("\0")
+            for _ in range(count):
+                self._emit_error("unexpected-null-character")
+
+        # Per HTML5 spec:
+        # - RCDATA state (title, textarea): decode character references
+        # - RAWTEXT state (style, script, etc): do NOT decode
+        # - PLAINTEXT state: do NOT decode
+        # - CDATA sections: do NOT decode
+        if self.state >= self.PLAINTEXT or self.CDATA_SECTION <= self.state <= self.CDATA_SECTION_END:
+            pass
+        elif self.state >= self.RAWTEXT:
+            pass
+        else:
+            if "&" in data:
+                data = decode_entities_in_text(data)
+        # Apply XML coercion if enabled
+        if self.opts.xml_coercion:
+            data = _coerce_text_for_xml(data)
+
+        # Record position at END of raw text (1-indexed column = raw_len)
+        self._record_text_end_position(raw_len)
+        self.sink.process_characters(data)
+        # Note: process_characters never returns Plaintext or RawData
+        # State switches happen via _emit_current_tag instead
+
+    def _append_attr_value_char(self, c):
+        self.current_attr_value.append(c)
+
+    def _finish_attribute(self):
+        attr_name_buffer = self.current_attr_name
+        if not attr_name_buffer:
+            return
+        if len(attr_name_buffer) == 1:
+            name = attr_name_buffer[0]
+        else:
+            name = "".join(attr_name_buffer)
+        attrs = self.current_tag_attrs
+        is_duplicate = name in attrs
+        attr_name_buffer.clear()
+        attr_value_buffer = self.current_attr_value
+        if is_duplicate:
+            self._emit_error("duplicate-attribute")
+            attr_value_buffer.clear()
+            self.current_attr_value_has_amp = False
+            return
+        if not attr_value_buffer:
+            value = ""
+        elif len(attr_value_buffer) == 1:
+            value = attr_value_buffer[0]
+        else:
+            value = "".join(attr_value_buffer)
+        if self.current_attr_value_has_amp:
+            value = decode_entities_in_text(value, in_attribute=True)
+        attrs[name] = value
+        attr_value_buffer.clear()
+        self.current_attr_value_has_amp = False
+
+    def _emit_current_tag(self):
+        name_parts = self.current_tag_name
+        part_count = len(name_parts)
+        # Note: part_count is always >= 1 because fast-path appends before entering TAG_NAME
+        if part_count == 1:
+            name = name_parts[0]
+        else:
+            name = "".join(name_parts)
+        attrs = self.current_tag_attrs
+        self.current_tag_attrs = {}
+
+        tag = self._tag_token
+        tag.kind = self.current_tag_kind
+        tag.name = name
+        tag.attrs = attrs
+        tag.self_closing = self.current_tag_self_closing
+
+        switched_to_rawtext = False
+        if self.current_tag_kind == Tag.START:
+            self.last_start_tag_name = name
+            needs_rawtext_check = name in _RAWTEXT_SWITCH_TAGS or name == "plaintext"
+            if needs_rawtext_check:
+                stack = self.sink.open_elements
+                current_node = stack[-1] if stack else None
+                namespace = current_node.namespace if current_node else None
+                if namespace is None or namespace == "html":
+                    if name in _RCDATA_ELEMENTS:
+                        self.state = self.RCDATA
+                        self.rawtext_tag_name = name
+                        switched_to_rawtext = True
+                    elif name in _RAWTEXT_SWITCH_TAGS:
+                        self.state = self.RAWTEXT
+                        self.rawtext_tag_name = name
+                        switched_to_rawtext = True
+                    else:
+                        # Must be "plaintext" - the only other way needs_rawtext_check can be True
+                        self.state = self.PLAINTEXT
+                        switched_to_rawtext = True
+        # Remember current state before emitting
+
+        # Emit token to sink
+        self._record_token_position()
+        result = self.sink.process_token(tag)
+        if result == 1:  # TokenSinkResult.Plaintext
+            self.state = self.PLAINTEXT
+            switched_to_rawtext = True
+
+        self.current_tag_name.clear()
+        self.current_attr_name.clear()
+        self.current_attr_value.clear()
+        self.current_tag_self_closing = False
+        self.current_tag_kind = Tag.START
+        return switched_to_rawtext
+
+    def _emit_comment(self):
+        data = "".join(self.current_comment)
+        self.current_comment.clear()
+        # Apply XML coercion if enabled
+        if self.opts.xml_coercion:
+            data = _coerce_comment_for_xml(data)
+        self._comment_token.data = data
+        self._emit_token(self._comment_token)
+
+    def _emit_doctype(self):
+        name = "".join(self.current_doctype_name) if self.current_doctype_name else None
+        # If public_id/system_id is a list (even empty), join it; if None, keep None
+        public_id = "".join(self.current_doctype_public) if self.current_doctype_public is not None else None
+        system_id = "".join(self.current_doctype_system) if self.current_doctype_system is not None else None
+        doctype = Doctype(
+            name=name,
+            public_id=public_id,
+            system_id=system_id,
+            force_quirks=self.current_doctype_force_quirks,
+        )
+        self.current_doctype_name.clear()
+        self.current_doctype_public = None
+        self.current_doctype_system = None
+        self.current_doctype_force_quirks = False
+        self._emit_token(DoctypeToken(doctype))
+
+    def _emit_token(self, token):
+        self._record_token_position()
+        self.sink.process_token(token)
+        # Note: process_token never returns Plaintext or RawData for state switches
+        # State switches happen via _emit_current_tag checking sink response
+
+    def _record_token_position(self):
+        """Record current position as 0-indexed column for the last emitted token.
+
+        Per the spec, the position should be at the end of the token (after the last char).
+        """
+        if not self.collect_errors:
+            return
+        # pos points after the last consumed character, which is exactly what we want
+        pos = self.pos
+        last_newline = self.buffer.rfind("\n", 0, pos)
+        if last_newline == -1:
+            column = pos  # 0-indexed from start
+        else:
+            column = pos - last_newline - 1  # 0-indexed from after newline
+        self.last_token_line = self._get_line_at_pos(pos)
+        self.last_token_column = column
+
+    def _record_text_end_position(self, raw_len):
+        """Record position at end of text token (after last character).
+
+        Uses text_start_pos + raw_len to compute where text ends, matching html5lib's
+        behavior of reporting the column of the last character (1-indexed).
+        """
+        if not self.collect_errors:
+            return
+        # Position of last character of text (0-indexed)
+        end_pos = self.text_start_pos + raw_len
+        last_newline = self.buffer.rfind("\n", 0, end_pos)
+        if last_newline == -1:
+            column = end_pos  # 1-indexed column = end_pos (position after last char)
+        else:
+            column = end_pos - last_newline - 1
+        self.last_token_line = self._get_line_at_pos(end_pos)
+        self.last_token_column = column
+
+    def _emit_error(self, code):
+        if not self.collect_errors:
+            return
+        # Compute column on-demand: scan backwards to find last newline
+        pos = max(0, self.pos - 1)  # Current position being processed
+        last_newline = self.buffer.rfind("\n", 0, pos + 1)
+        if last_newline == -1:
+            column = pos + 1  # 1-indexed from start of input
+        else:
+            column = pos - last_newline  # 1-indexed from after newline
+
+        message = generate_error_message(code)
+        line = self._get_line_at_pos(self.pos)
+        self.errors.append(ParseError(code, line=line, column=column, message=message, source_html=self.buffer))
+
+    def _consume_if(self, literal):
+        end = self.pos + len(literal)
+        if end > self.length:
+            return False
+        segment = self.buffer[self.pos : end]
+        if segment != literal:
+            return False
+        self.pos = end
+        return True
+
+    def _consume_case_insensitive(self, literal):
+        end = self.pos + len(literal)
+        if end > self.length:
+            return False
+        segment = self.buffer[self.pos : end]
+        if segment.lower() != literal.lower():
+            return False
+        self.pos = end
+        return True
+
+    def _consume_comment_run(self):
+        # Note: Comments are never reconsumed
+        pos = self.pos
+        length = self.length
+        if pos >= length:
+            return False
+
+        # Handle ignore_lf for CRLF sequences
+        if self.ignore_lf and pos < length and self.buffer[pos] == "\n":
+            self.ignore_lf = False
+            pos += 1
+            self.pos = pos
+            if pos >= length:
+                return False
+
+        match = _COMMENT_RUN_PATTERN.match(self.buffer, pos)
+        if match:
+            chunk = match.group(0)
+            # Handle CRLF normalization for comments
+            if "\r" in chunk:
+                chunk = chunk.replace("\r\n", "\n").replace("\r", "\n")
+                self.ignore_lf = chunk.endswith("\r")
+            self.current_comment.append(chunk)
+            self.pos = match.end()
+            return True
+        return False
+
+    def _state_cdata_section(self):
+        # CDATA section state - consume characters until we see ']'
+        while True:
+            c = self._get_char()
+            if c is None:
+                self._emit_error("eof-in-cdata")
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+            if c == "]":
+                self.state = self.CDATA_SECTION_BRACKET
+                return False
+            self._append_text(c)
+
+    def _state_cdata_section_bracket(self):
+        # Seen one ']', check for second ']'
+        c = self._get_char()
+        if c == "]":
+            self.state = self.CDATA_SECTION_END
+            return False
+        # False alarm, emit the ']' we saw and continue
+        self._append_text("]")
+        if c is None:
+            self._emit_error("eof-in-cdata")
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        self._reconsume_current()
+        self.state = self.CDATA_SECTION
+        return False
+
+    def _state_cdata_section_end(self):
+        # Seen ']]', check for '>'
+        c = self._get_char()
+        if c == ">":
+            # End of CDATA section
+            self._flush_text()
+            self.state = self.DATA
+            return False
+        # Not the end - we saw ']]' but not '>'. Emit one ']' and check if the next char is another ']'
+        self._append_text("]")
+        if c is None:
+            # EOF after ']]' - emit the second ']' too
+            self._append_text("]")
+            self._emit_error("eof-in-cdata")
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == "]":
+            # Still might be ']]>' sequence, stay in CDATA_SECTION_END
+            return False
+        # Not a bracket, so emit the second ']', reconsume current char and go back to CDATA_SECTION
+        self._append_text("]")
+        self._reconsume_current()
+        self.state = self.CDATA_SECTION
+        return False
+
+    def _state_rcdata(self):
+        buffer = self.buffer
+        length = self.length
+        pos = self.pos
+        while True:
+            if self.reconsume:
+                self.reconsume = False
+                if self.current_char is None:
+                    self._flush_text()
+                    self._emit_token(EOFToken())
+                    return True
+                self.pos -= 1
+                pos = self.pos
+
+            # Optimized loop using find
+            lt_index = buffer.find("<", pos)
+            amp_index = buffer.find("&", pos)
+            null_index = buffer.find("\0", pos)
+
+            # Find the nearest special character
+            next_special = length
+            if lt_index != -1:
+                next_special = lt_index
+            if amp_index != -1 and amp_index < next_special:
+                next_special = amp_index
+            if null_index != -1 and null_index < next_special:
+                next_special = null_index
+
+            # Consume everything up to the special character
+            if next_special > pos:
+                chunk = buffer[pos:next_special]
+                self._append_text_chunk(chunk, ends_with_cr=chunk.endswith("\r"))
+                pos = next_special
+                self.pos = pos
+
+            # Handle EOF
+            if pos >= length:
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+
+            # Handle special characters - we're at one of them after find()
+            if null_index == pos:
+                self.ignore_lf = False
+                self._emit_error("unexpected-null-character")
+                self._append_text("\ufffd")
+                pos += 1
+                self.pos = pos
+            elif amp_index == pos:
+                # Ampersand in RCDATA - will be decoded by _flush_text
+                self._append_text("&")
+                pos += 1
+                self.pos = pos
+            else:
+                # lt_index == pos - the only remaining possibility
+                # Less-than sign - might be start of end tag
+                pos += 1
+                self.pos = pos
+                self.state = self.RCDATA_LESS_THAN_SIGN
+                return False
+
+    def _state_rcdata_less_than_sign(self):
+        c = self._get_char()
+        if c == "/":
+            self.current_tag_name.clear()
+            self.state = self.RCDATA_END_TAG_OPEN
+            return False
+        self._append_text("<")
+        self._reconsume_current()
+        self.state = self.RCDATA
+        return False
+
+    def _state_rcdata_end_tag_open(self):
+        c = self._get_char()
+        if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+            self.current_tag_name.append(c.lower())
+            self.original_tag_name.append(c)
+            self.state = self.RCDATA_END_TAG_NAME
+            return False
+        self.text_buffer.extend(("<", "/"))
+        self._reconsume_current()
+        self.state = self.RCDATA
+        return False
+
+    def _state_rcdata_end_tag_name(self):
+        # Check if this matches the opening tag name
+        while True:
+            c = self._get_char()
+            if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+                self.current_tag_name.append(c.lower())
+                self.original_tag_name.append(c)
+                continue
+            # End of tag name - check if it matches
+            tag_name = "".join(self.current_tag_name)
+            if tag_name == self.rawtext_tag_name:
+                if c == ">":
+                    attrs = []
+                    tag = Tag(Tag.END, tag_name, attrs, False)
+                    self._flush_text()
+                    self._emit_token(tag)
+                    self.state = self.DATA
+                    self.rawtext_tag_name = None
+                    self.original_tag_name.clear()
+                    return False
+                if c in (" ", "\t", "\n", "\r", "\f"):
+                    # Whitespace after tag name - switch to BEFORE_ATTRIBUTE_NAME
+                    self.current_tag_kind = Tag.END
+                    self.current_tag_attrs = {}
+                    self.state = self.BEFORE_ATTRIBUTE_NAME
+                    return False
+                if c == "/":
+                    self._flush_text()
+                    self.current_tag_kind = Tag.END
+                    self.current_tag_attrs = {}
+                    self.state = self.SELF_CLOSING_START_TAG
+                    return False
+            # If we hit EOF or tag doesn't match, emit as text
+            if c is None:
+                # EOF - emit incomplete tag as text (preserve original case) then EOF
+                self.text_buffer.extend(("<", "/"))
+                for ch in self.original_tag_name:
+                    self._append_text(ch)
+                self.current_tag_name.clear()
+                self.original_tag_name.clear()
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+            # Not a matching end tag - emit as text (preserve original case)
+            self.text_buffer.extend(("<", "/"))
+            for ch in self.original_tag_name:
+                self._append_text(ch)
+            self.current_tag_name.clear()
+            self.original_tag_name.clear()
+            self._reconsume_current()
+            self.state = self.RCDATA
+            return False
+
+    def _state_rawtext(self):
+        buffer = self.buffer
+        length = self.length
+        pos = self.pos
+        while True:
+            if self.reconsume:
+                self.reconsume = False
+                if self.current_char is None:
+                    self._flush_text()
+                    self._emit_token(EOFToken())
+                    return True
+                self.pos -= 1
+                pos = self.pos
+
+            # Optimized loop using find
+            lt_index = buffer.find("<", pos)
+            null_index = buffer.find("\0", pos)
+            next_special = lt_index if lt_index != -1 else length
+            if null_index != -1 and null_index < next_special:
+                if null_index > pos:
+                    chunk = buffer[pos:null_index]
+                    self._append_text_chunk(chunk, ends_with_cr=chunk.endswith("\r"))
+                else:
+                    self.ignore_lf = False
+                self._emit_error("unexpected-null-character")
+                self._append_text("\ufffd")
+                pos = null_index + 1
+                self.pos = pos
+                continue
+            if lt_index == -1:
+                if pos < length:
+                    chunk = buffer[pos:length]
+                    self._append_text_chunk(chunk, ends_with_cr=chunk.endswith("\r"))
+                self.pos = length
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+            if lt_index > pos:
+                chunk = buffer[pos:lt_index]
+                self._append_text_chunk(chunk, ends_with_cr=chunk.endswith("\r"))
+            pos = lt_index + 1
+            self.pos = pos
+            # Handle script escaped transition before treating '<' as markup boundary
+            if self.rawtext_tag_name == "script":
+                next1 = self._peek_char(0)
+                next2 = self._peek_char(1)
+                next3 = self._peek_char(2)
+                if next1 == "!" and next2 == "-" and next3 == "-":
+                    self.text_buffer.extend(["<", "!", "-", "-"])
+                    self._get_char()
+                    self._get_char()
+                    self._get_char()
+                    self.state = self.SCRIPT_DATA_ESCAPED
+                    return False
+            self.state = self.RAWTEXT_LESS_THAN_SIGN
+            return False
+
+    def _state_rawtext_less_than_sign(self):
+        c = self._get_char()
+        if c == "/":
+            self.current_tag_name.clear()
+            self.state = self.RAWTEXT_END_TAG_OPEN
+            return False
+        self._append_text("<")
+        self._reconsume_current()
+        self.state = self.RAWTEXT
+        return False
+
+    def _state_rawtext_end_tag_open(self):
+        c = self._get_char()
+        if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+            self.current_tag_name.append(c.lower())
+            self.original_tag_name.append(c)
+            self.state = self.RAWTEXT_END_TAG_NAME
+            return False
+        self.text_buffer.extend(("<", "/"))
+        self._reconsume_current()
+        self.state = self.RAWTEXT
+        return False
+
+    def _state_rawtext_end_tag_name(self):
+        # Check if this matches the opening tag name
+        while True:
+            c = self._get_char()
+            if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+                self.current_tag_name.append(c.lower())
+                self.original_tag_name.append(c)
+                continue
+            # End of tag name - check if it matches
+            tag_name = "".join(self.current_tag_name)
+            if tag_name == self.rawtext_tag_name:
+                if c == ">":
+                    attrs = []
+                    tag = Tag(Tag.END, tag_name, attrs, False)
+                    self._flush_text()
+                    self._emit_token(tag)
+                    self.state = self.DATA
+                    self.rawtext_tag_name = None
+                    self.original_tag_name.clear()
+                    return False
+                if c in (" ", "\t", "\n", "\r", "\f"):
+                    # Whitespace after tag name - switch to BEFORE_ATTRIBUTE_NAME
+                    self.current_tag_kind = Tag.END
+                    self.current_tag_attrs = {}
+                    self.state = self.BEFORE_ATTRIBUTE_NAME
+                    return False
+                if c == "/":
+                    self._flush_text()
+                    self.current_tag_kind = Tag.END
+                    self.current_tag_attrs = {}
+                    self.state = self.SELF_CLOSING_START_TAG
+                    return False
+            # If we hit EOF or tag doesn't match, emit as text
+            if c is None:
+                # EOF - emit incomplete tag as text (preserve original case) then EOF
+                self.text_buffer.extend(("<", "/"))
+                for ch in self.original_tag_name:
+                    self._append_text(ch)
+                self.current_tag_name.clear()
+                self.original_tag_name.clear()
+                self._flush_text()
+                self._emit_token(EOFToken())
+                return True
+            # Not a matching end tag - emit as text (preserve original case)
+            self.text_buffer.extend(("<", "/"))
+            for ch in self.original_tag_name:
+                self._append_text(ch)
+            self.current_tag_name.clear()
+            self.original_tag_name.clear()
+            self._reconsume_current()
+            self.state = self.RAWTEXT
+            return False
+
+    def _state_plaintext(self):
+        # PLAINTEXT state - consume everything as text, no end tag
+        if self.pos < self.length:
+            remaining = self.buffer[self.pos :]
+            # Replace null bytes with replacement character
+            if "\0" in remaining:
+                remaining = remaining.replace("\0", "\ufffd")
+                self._emit_error("unexpected-null-character")
+            self._append_text(remaining)
+            self.pos = self.length
+        self._flush_text()
+        self._emit_token(EOFToken())
+        return True
+
+    def _state_script_data_escaped(self):
+        c = self._get_char()
+        if c is None:
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self._append_text("-")
+            self.state = self.SCRIPT_DATA_ESCAPED_DASH
+            return False
+        if c == "<":
+            self.state = self.SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self._append_text("\ufffd")
+            return False
+        self._append_text(c)
+        return False
+
+    def _state_script_data_escaped_dash(self):
+        c = self._get_char()
+        if c is None:
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self._append_text("-")
+            self.state = self.SCRIPT_DATA_ESCAPED_DASH_DASH
+            return False
+        if c == "<":
+            self.state = self.SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self._append_text("\ufffd")
+            self.state = self.SCRIPT_DATA_ESCAPED
+            return False
+        self._append_text(c)
+        self.state = self.SCRIPT_DATA_ESCAPED
+        return False
+
+    def _state_script_data_escaped_dash_dash(self):
+        c = self._get_char()
+        if c is None:
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self._append_text("-")
+            return False
+        if c == "<":
+            self._append_text("<")
+            self.state = self.SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN
+            return False
+        if c == ">":
+            self._append_text(">")
+            self.state = self.RAWTEXT
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self._append_text("\ufffd")
+            self.state = self.SCRIPT_DATA_ESCAPED
+            return False
+        self._append_text(c)
+        self.state = self.SCRIPT_DATA_ESCAPED
+        return False
+
+    def _state_script_data_escaped_less_than_sign(self):
+        c = self._get_char()
+        if c == "/":
+            self.temp_buffer.clear()
+            self.state = self.SCRIPT_DATA_ESCAPED_END_TAG_OPEN
+            return False
+        if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+            self.temp_buffer.clear()
+            self._append_text("<")
+            self._reconsume_current()
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPE_START
+            return False
+        self._append_text("<")
+        self._reconsume_current()
+        self.state = self.SCRIPT_DATA_ESCAPED
+
+        return False
+
+    def _state_script_data_escaped_end_tag_open(self):
+        c = self._get_char()
+        if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+            self.current_tag_name.clear()
+            self.original_tag_name.clear()
+            self._reconsume_current()
+            self.state = self.SCRIPT_DATA_ESCAPED_END_TAG_NAME
+            return False
+        self.text_buffer.extend(("<", "/"))
+        self._reconsume_current()
+        self.state = self.SCRIPT_DATA_ESCAPED
+        return False
+
+    def _state_script_data_escaped_end_tag_name(self):
+        c = self._get_char()
+        if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+            self.current_tag_name.append(c.lower())
+            self.original_tag_name.append(c)
+            self.temp_buffer.append(c)
+            return False
+        # Check if this is an appropriate end tag
+        tag_name = "".join(self.current_tag_name)
+        is_appropriate = tag_name == self.rawtext_tag_name
+
+        if is_appropriate:
+            if c in (" ", "\t", "\n", "\r", "\f"):
+                self.current_tag_kind = Tag.END
+                self.current_tag_attrs = {}
+                self.state = self.BEFORE_ATTRIBUTE_NAME
+                return False
+            if c == "/":
+                self._flush_text()
+                self.current_tag_kind = Tag.END
+                self.current_tag_attrs = {}
+                self.state = self.SELF_CLOSING_START_TAG
+                return False
+            if c == ">":
+                self._flush_text()
+                attrs = []
+                tag = Tag(Tag.END, tag_name, attrs, False)
+                self._emit_token(tag)
+                self.state = self.DATA
+                self.rawtext_tag_name = None
+                self.current_tag_name.clear()
+                self.original_tag_name.clear()
+                return False
+        # Not an appropriate end tag
+        self.text_buffer.extend(("<", "/"))
+        for ch in self.temp_buffer:
+            self._append_text(ch)
+        self._reconsume_current()
+        self.state = self.SCRIPT_DATA_ESCAPED
+        return False
+
+    def _state_script_data_double_escape_start(self):
+        c = self._get_char()
+        if c in (" ", "\t", "\n", "\r", "\f", "/", ">"):
+            # Check if temp_buffer contains "script"
+            temp = "".join(self.temp_buffer).lower()
+            if temp == "script":
+                self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED
+            else:
+                self.state = self.SCRIPT_DATA_ESCAPED
+            self._append_text(c)
+            return False
+        if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+            self.temp_buffer.append(c)
+            self._append_text(c)
+            return False
+        self._reconsume_current()
+        self.state = self.SCRIPT_DATA_ESCAPED
+        return False
+
+    def _state_script_data_double_escaped(self):
+        c = self._get_char()
+        if c is None:
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self._append_text("-")
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED_DASH
+            return False
+        if c == "<":
+            self._append_text("<")
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self._append_text("\ufffd")
+            return False
+        self._append_text(c)
+        return False
+
+    def _state_script_data_double_escaped_dash(self):
+        c = self._get_char()
+        if c is None:
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self._append_text("-")
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH
+            return False
+        if c == "<":
+            self._append_text("<")
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self._append_text("\ufffd")
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED
+            return False
+        self._append_text(c)
+        self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED
+        return False
+
+    def _state_script_data_double_escaped_dash_dash(self):
+        c = self._get_char()
+        if c is None:
+            self._flush_text()
+            self._emit_token(EOFToken())
+            return True
+        if c == "-":
+            self._append_text("-")
+            return False
+        if c == "<":
+            self._append_text("<")
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN
+
+            return False
+        if c == ">":
+            self._append_text(">")
+            self.state = self.RAWTEXT
+
+            return False
+        if c == "\0":
+            self._emit_error("unexpected-null-character")
+            self._append_text("\ufffd")
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED
+            return False
+        self._append_text(c)
+        self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED
+        return False
+
+    def _state_script_data_double_escaped_less_than_sign(self):
+        c = self._get_char()
+        if c == "/":
+            self.temp_buffer.clear()
+            self._append_text("/")
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPE_END
+            return False
+        if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+            self.temp_buffer.clear()
+            self._reconsume_current()
+            self.state = self.SCRIPT_DATA_DOUBLE_ESCAPE_START
+            return False
+        self._reconsume_current()
+        self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED
+        return False
+
+    def _state_script_data_double_escape_end(self):
+        c = self._get_char()
+        if c in (" ", "\t", "\n", "\r", "\f", "/", ">"):
+            # Check if temp_buffer contains "script"
+            temp = "".join(self.temp_buffer).lower()
+
+            if temp == "script":
+                self.state = self.SCRIPT_DATA_ESCAPED
+            else:
+                self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED
+            self._append_text(c)
+            return False
+        if c is not None and ("A" <= c <= "Z" or "a" <= c <= "z"):
+            self.temp_buffer.append(c)
+            self._append_text(c)
+            return False
+        self._reconsume_current()
+        self.state = self.SCRIPT_DATA_DOUBLE_ESCAPED
+        return False
+
+
+Tokenizer._STATE_HANDLERS = [
+    Tokenizer._state_data,
+    Tokenizer._state_tag_open,
+    Tokenizer._state_end_tag_open,
+    Tokenizer._state_tag_name,
+    Tokenizer._state_before_attribute_name,
+    Tokenizer._state_attribute_name,
+    Tokenizer._state_after_attribute_name,
+    Tokenizer._state_before_attribute_value,
+    Tokenizer._state_attribute_value_double,
+    Tokenizer._state_attribute_value_single,
+    Tokenizer._state_attribute_value_unquoted,
+    Tokenizer._state_after_attribute_value_quoted,
+    Tokenizer._state_self_closing_start_tag,
+    Tokenizer._state_markup_declaration_open,
+    Tokenizer._state_comment_start,
+    Tokenizer._state_comment_start_dash,
+    Tokenizer._state_comment,
+    Tokenizer._state_comment_end_dash,
+    Tokenizer._state_comment_end,
+    Tokenizer._state_comment_end_bang,
+    Tokenizer._state_bogus_comment,
+    Tokenizer._state_doctype,
+    Tokenizer._state_before_doctype_name,
+    Tokenizer._state_doctype_name,
+    Tokenizer._state_after_doctype_name,
+    Tokenizer._state_bogus_doctype,
+    Tokenizer._state_after_doctype_public_keyword,
+    Tokenizer._state_after_doctype_system_keyword,
+    Tokenizer._state_before_doctype_public_identifier,
+    Tokenizer._state_doctype_public_identifier_double_quoted,
+    Tokenizer._state_doctype_public_identifier_single_quoted,
+    Tokenizer._state_after_doctype_public_identifier,
+    Tokenizer._state_between_doctype_public_and_system_identifiers,
+    Tokenizer._state_before_doctype_system_identifier,
+    Tokenizer._state_doctype_system_identifier_double_quoted,
+    Tokenizer._state_doctype_system_identifier_single_quoted,
+    Tokenizer._state_after_doctype_system_identifier,
+    Tokenizer._state_cdata_section,
+    Tokenizer._state_cdata_section_bracket,
+    Tokenizer._state_cdata_section_end,
+    Tokenizer._state_rcdata,
+    Tokenizer._state_rcdata_less_than_sign,
+    Tokenizer._state_rcdata_end_tag_open,
+    Tokenizer._state_rcdata_end_tag_name,
+    Tokenizer._state_rawtext,
+    Tokenizer._state_rawtext_less_than_sign,
+    Tokenizer._state_rawtext_end_tag_open,
+    Tokenizer._state_rawtext_end_tag_name,
+    Tokenizer._state_plaintext,
+    Tokenizer._state_script_data_escaped,
+    Tokenizer._state_script_data_escaped_dash,
+    Tokenizer._state_script_data_escaped_dash_dash,
+    Tokenizer._state_script_data_escaped_less_than_sign,
+    Tokenizer._state_script_data_escaped_end_tag_open,
+    Tokenizer._state_script_data_escaped_end_tag_name,
+    Tokenizer._state_script_data_double_escape_start,
+    Tokenizer._state_script_data_double_escaped,
+    Tokenizer._state_script_data_double_escaped_dash,
+    Tokenizer._state_script_data_double_escaped_dash_dash,
+    Tokenizer._state_script_data_double_escaped_less_than_sign,
+    Tokenizer._state_script_data_double_escape_end,
+]