justhtml 0.6.0__py3-none-any.whl → 0.33.0__py3-none-any.whl

justhtml/serialize.py

@@ -2,35 +2,288 @@
 
 # ruff: noqa: PERF401
 
-from justhtml.constants import FOREIGN_ATTRIBUTE_ADJUSTMENTS, VOID_ELEMENTS
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from .constants import FOREIGN_ATTRIBUTE_ADJUSTMENTS, SPECIAL_ELEMENTS, VOID_ELEMENTS, WHITESPACE_PRESERVING_ELEMENTS
+from .sanitize import DEFAULT_DOCUMENT_POLICY, DEFAULT_POLICY, SanitizationPolicy, _sanitize
+
+# Matches characters that prevent an attribute value from being unquoted.
+# Note: This matches the logic of the previous loop-based implementation.
+# It checks for space characters, quotes, equals sign, and greater-than.
+_UNQUOTED_ATTR_VALUE_INVALID = re.compile(r'[ \t\n\f\r"\'=>]')
+
+
+def _escape_text(text: str | None) -> str:
+    if not text:
+        return ""
+    # Minimal, but matches html5lib serializer expectations in core cases.
+    return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+
+
+def _choose_attr_quote(value: str | None, forced_quote_char: str | None = None) -> str:
+    if forced_quote_char in {'"', "'"}:
+        return forced_quote_char
+    if value is None:
+        return '"'
+    # value is assumed to be a string
+    if '"' in value and "'" not in value:
+        return "'"
+    return '"'
+
+
+def _escape_attr_value(value: str | None, quote_char: str, *, escape_lt_in_attrs: bool = False) -> str:
+    if value is None:
+        return ""
+    # value is assumed to be a string
+    value = value.replace("&", "&amp;")
+    if escape_lt_in_attrs:
+        value = value.replace("<", "&lt;")
+    # Note: html5lib's default serializer does not escape '>' in attrs.
+    if quote_char == '"':
+        return value.replace('"', "&quot;")
+    return value.replace("'", "&#39;")
+
+
+def _can_unquote_attr_value(value: str | None) -> bool:
+    if value is None:
+        return False
+    # Optimization: use regex instead of loop
+    return not _UNQUOTED_ATTR_VALUE_INVALID.search(value)
+
+
+def _serializer_minimize_attr_value(name: str, value: str | None, minimize_boolean_attributes: bool) -> bool:
+    if not minimize_boolean_attributes:
+        return False
+    if value is None or value == "":
+        return True
+    if value == name:
+        return True
+    return value.lower() == name
+
+
+def serialize_start_tag(
+    name: str,
+    attrs: dict[str, str | None] | None,
+    *,
+    quote_attr_values: bool = True,
+    minimize_boolean_attributes: bool = True,
+    quote_char: str | None = None,
+    escape_lt_in_attrs: bool = False,
+    use_trailing_solidus: bool = False,
+    is_void: bool = False,
+) -> str:
+    attrs = attrs or {}
+    parts: list[str] = ["<", name]
+    if attrs:
+        for key, value in attrs.items():
+            if _serializer_minimize_attr_value(key, value, minimize_boolean_attributes):
+                parts.extend([" ", key])
+                continue
+
+            if value is None:
+                parts.extend([" ", key, '=""'])
+                continue
+
+            # value is guaranteed to be a string here because attrs is dict[str, str | None]
+            value_str = value
+            if value_str == "":
+                parts.extend([" ", key, '=""'])
+                continue
+
+            if not quote_attr_values and _can_unquote_attr_value(value_str):
+                escaped = value_str.replace("&", "&amp;")
+                if escape_lt_in_attrs:
+                    escaped = escaped.replace("<", "&lt;")
+                parts.extend([" ", key, "=", escaped])
+            else:
+                quote = _choose_attr_quote(value_str, quote_char)
+                escaped = _escape_attr_value(value_str, quote, escape_lt_in_attrs=escape_lt_in_attrs)
+                parts.extend([" ", key, "=", quote, escaped, quote])
+
+    if use_trailing_solidus and is_void:
+        parts.append(" />")
+    else:
+        parts.append(">")
+    return "".join(parts)
 
 
-def to_html(node, indent=0, indent_size=2, pretty=True):
+def serialize_end_tag(name: str) -> str:
+    return f"</{name}>"
+
+
+def to_html(
+    node: Any,
+    indent: int = 0,
+    indent_size: int = 2,
+    *,
+    pretty: bool = True,
+    safe: bool = True,
+    policy: SanitizationPolicy | None = None,
+) -> str:
     """Convert node to HTML string."""
+    if safe:
+        if policy is None and node.name == "#document":
+            node = _sanitize(node, policy=DEFAULT_DOCUMENT_POLICY)
+        else:
+            node = _sanitize(node, policy=policy or DEFAULT_POLICY)
     if node.name == "#document":
         # Document root - just render children
-        parts = []
+        parts: list[str] = []
         for child in node.children or []:
-            parts.append(_node_to_html(child, indent, indent_size, pretty))
+            parts.append(_node_to_html(child, indent, indent_size, pretty, in_pre=False))
         return "\n".join(parts) if pretty else "".join(parts)
-    return _node_to_html(node, indent, indent_size, pretty)
+    return _node_to_html(node, indent, indent_size, pretty, in_pre=False)
+
+
+def _collapse_html_whitespace(text: str) -> str:
+    """Collapse HTML whitespace runs to a single space and trim edges.
+
+    This matches how HTML rendering treats most whitespace in text nodes, and is
+    used only for pretty-printing in non-preformatted contexts.
+    """
+    if not text:
+        return ""
+
+    # Optimization: split() handles whitespace collapsing efficiently.
+    # Note: split() treats \v as whitespace, which is not HTML whitespace.
+    # But \v is extremely rare in HTML.
+    if "\v" in text:
+        parts: list[str] = []
+        in_whitespace = False
+        for ch in text:
+            if ch in {" ", "\t", "\n", "\f", "\r"}:
+                if not in_whitespace:
+                    parts.append(" ")
+                    in_whitespace = True
+                continue
 
+            parts.append(ch)
+            in_whitespace = False
 
-def _node_to_html(node, indent=0, indent_size=2, pretty=True):
+        collapsed = "".join(parts)
+        return collapsed.strip(" ")
+
+    return " ".join(text.split())
+
+
+def _normalize_formatting_whitespace(text: str) -> str:
+    """Normalize formatting whitespace within a text node.
+
+    Converts newlines/tabs/CR/FF to regular spaces and collapses runs that
+    include such formatting whitespace to a single space.
+
+    Pure space runs are preserved as-is (so existing double-spaces remain).
+    """
+    if not text:
+        return ""
+
+    if "\n" not in text and "\r" not in text and "\t" not in text and "\f" not in text:
+        return text
+
+    starts_with_formatting = text[0] in {"\n", "\r", "\t", "\f"}
+    ends_with_formatting = text[-1] in {"\n", "\r", "\t", "\f"}
+
+    out: list[str] = []
+    in_ws = False
+    saw_formatting_ws = False
+
+    for ch in text:
+        if ch == " ":
+            if in_ws:
+                # Only collapse if this whitespace run included formatting whitespace.
+                if saw_formatting_ws:
+                    continue
+                out.append(" ")
+                continue
+            in_ws = True
+            saw_formatting_ws = False
+            out.append(" ")
+            continue
+
+        if ch in {"\n", "\r", "\t", "\f"}:
+            if in_ws:
+                saw_formatting_ws = True
+                continue
+            in_ws = True
+            saw_formatting_ws = True
+            out.append(" ")
+            continue
+
+        in_ws = False
+        saw_formatting_ws = False
+        out.append(ch)
+
+    normalized = "".join(out)
+    if starts_with_formatting and normalized.startswith(" "):
+        normalized = normalized[1:]
+    if ends_with_formatting and normalized.endswith(" "):
+        normalized = normalized[:-1]
+    return normalized
+
+
+def _is_whitespace_text_node(node: Any) -> bool:
+    return node.name == "#text" and (node.data or "").strip() == ""
+
+
+def _should_pretty_indent_children(children: list[Any]) -> bool:
+    for child in children:
+        if child is None:
+            continue
+        name = child.name
+        if name == "#comment":
+            return False
+        if name == "#text" and (child.data or "").strip():
+            return False
+
+    element_children: list[Any] = [
+        child for child in children if child is not None and child.name not in {"#text", "#comment"}
+    ]
+    if not element_children:
+        return True
+    if len(element_children) == 1:
+        only_child = element_children[0]
+        if only_child.name in SPECIAL_ELEMENTS:
+            return True
+        if only_child.name == "a":
+            # If an anchor wraps block-ish content (valid HTML5), treat it as block-ish
+            # for pretty-printing so the parent can indent it on its own line.
+            for grandchild in only_child.children or []:
+                if grandchild is None:
+                    continue
+                if grandchild.name in SPECIAL_ELEMENTS:
+                    return True
+        return False
+
+    # Safe indentation rule: only insert inter-element whitespace when we won't
+    # be placing it between two adjacent inline/phrasing elements.
+    prev_is_special = element_children[0].name in SPECIAL_ELEMENTS
+    for child in element_children[1:]:
+        current_is_special = child.name in SPECIAL_ELEMENTS
+        if not prev_is_special and not current_is_special:
+            return False
+        prev_is_special = current_is_special
+    return True
+
+
+def _node_to_html(node: Any, indent: int = 0, indent_size: int = 2, pretty: bool = True, *, in_pre: bool) -> str:
     """Helper to convert a node to HTML."""
-    prefix = " " * (indent * indent_size) if pretty else ""
-    newline = "\n" if pretty else ""
-    name = node.name
+    prefix = " " * (indent * indent_size) if pretty and not in_pre else ""
+    name: str = node.name
+    content_pre = in_pre or name in WHITESPACE_PRESERVING_ELEMENTS
+    newline = "\n" if pretty and not content_pre else ""
 
     # Text node
     if name == "#text":
-        text = node.data
-        if pretty:
+        text: str | None = node.data
+        if pretty and not in_pre:
             text = text.strip() if text else ""
             if text:
-                return f"{prefix}{text}"
+                return f"{prefix}{_escape_text(text)}"
             return ""
-        return text or ""
+        return _escape_text(text) if text else ""
 
     # Comment node
     if name == "#comment":
@@ -42,58 +295,166 @@ def _node_to_html(node, indent=0, indent_size=2, pretty=True):
 
     # Document fragment
     if name == "#document-fragment":
-        parts = []
+        parts: list[str] = []
         for child in node.children or []:
-            child_html = _node_to_html(child, indent, indent_size, pretty)
+            child_html = _node_to_html(child, indent, indent_size, pretty, in_pre=in_pre)
             if child_html:
                 parts.append(child_html)
         return newline.join(parts) if pretty else "".join(parts)
 
     # Element node
-    attrs = node.attrs or {}
+    attrs: dict[str, str | None] = node.attrs or {}
 
     # Build opening tag
-    attr_str = ""
-    if attrs:
-        attr_parts = []
-        for key, value in attrs.items():
-            if value is None:
-                attr_parts.append(key)
-            elif value == "":
-                attr_parts.append(key)
-            else:
-                # Escape quotes in attribute values
-                escaped = str(value).replace("&", "&amp;").replace('"', "&quot;")
-                attr_parts.append(f'{key}="{escaped}"')
-        if attr_parts:  # pragma: no branch
-            attr_str = " " + " ".join(attr_parts)
+    open_tag = serialize_start_tag(name, attrs)
 
     # Void elements
     if name in VOID_ELEMENTS:
-        return f"{prefix}<{name}{attr_str}>"
+        return f"{prefix}{open_tag}"
 
     # Elements with children
-    children = node.children or []
+    # Template special handling: HTML templates store contents in `template_content`.
+    if name == "template" and node.namespace in {None, "html"} and node.template_content is not None:
+        children: list[Any] = node.template_content.children or []
+    else:
+        children = node.children or []
     if not children:
-        return f"{prefix}<{name}{attr_str}></{name}>"
+        return f"{prefix}{open_tag}{serialize_end_tag(name)}"
 
     # Check if all children are text-only (inline rendering)
-    all_text = all(hasattr(c, "name") and c.name == "#text" for c in children)
+    all_text = all(c.name == "#text" for c in children)
+
+    if all_text and pretty and not content_pre:
+        # Serializer controls sanitization at the to_html() entry point; avoid
+        # implicit re-sanitization during rendering.
+        text_content = node.to_text(separator="", strip=False, safe=False)
+        text_content = _collapse_html_whitespace(text_content)
+        return f"{prefix}{open_tag}{_escape_text(text_content)}{serialize_end_tag(name)}"
+
+    if pretty and content_pre:
+        inner = "".join(
+            _node_to_html(child, indent + 1, indent_size, pretty, in_pre=True)
+            for child in children
+            if child is not None
+        )
+        return f"{prefix}{open_tag}{inner}{serialize_end_tag(name)}"
+
+    if pretty and not content_pre and not _should_pretty_indent_children(children):
+        # For block-ish elements that contain only element children and whitespace-only
+        # text nodes, we can still format each child on its own line (only when there
+        # is already whitespace separating element siblings).
+        if name in SPECIAL_ELEMENTS:
+            has_comment = False
+            has_element = False
+            has_whitespace_between_elements = False
+
+            first_element_index: int | None = None
+            last_element_index: int | None = None
+
+            previous_was_element = False
+            saw_whitespace_since_last_element = False
+            for i, child in enumerate(children):
+                if child is None:
+                    continue
+                if child.name == "#comment":
+                    has_comment = True
+                    break
+                if child.name == "#text":
+                    # Track whether there is already whitespace between element siblings.
+                    if previous_was_element and not (child.data or "").strip():
+                        saw_whitespace_since_last_element = True
+                    continue
+
+                has_element = True
+                if first_element_index is None:
+                    first_element_index = i
+                last_element_index = i
+                if previous_was_element and saw_whitespace_since_last_element:
+                    has_whitespace_between_elements = True
+                previous_was_element = True
+                saw_whitespace_since_last_element = False
+
+            can_indent_non_whitespace_text = True
+            if has_element and first_element_index is not None and last_element_index is not None:
+                for i, child in enumerate(children):
+                    if child is None or child.name != "#text":
+                        continue
+                    if not (child.data or "").strip():
+                        continue
+                    # Only allow non-whitespace text *after* the last element.
+                    # Leading text or text between elements could gain new spaces
+                    # due to indentation/newlines.
+                    if i < first_element_index or first_element_index < i < last_element_index:
+                        can_indent_non_whitespace_text = False
+                        break
+
+            if has_element and has_whitespace_between_elements and not has_comment and can_indent_non_whitespace_text:
+                inner_lines: list[str] = []
+                for child in children:
+                    if child is None:
+                        continue
+                    if child.name == "#text":
+                        text = _collapse_html_whitespace(child.data or "")
+                        if text:
+                            inner_lines.append(f"{' ' * ((indent + 1) * indent_size)}{_escape_text(text)}")
+                        continue
+                    child_html = _node_to_html(child, indent + 1, indent_size, pretty=True, in_pre=content_pre)
+                    if child_html:
+                        inner_lines.append(child_html)
+                if inner_lines:
+                    inner = "\n".join(inner_lines)
+                    return f"{prefix}{open_tag}\n{inner}\n{prefix}{serialize_end_tag(name)}"
+
+        inner_parts: list[str] = []
+
+        first_non_none_index: int | None = None
+        last_non_none_index: int | None = None
+        for i, child in enumerate(children):
+            if child is None:
+                continue
+            if first_non_none_index is None:
+                first_non_none_index = i
+            last_non_none_index = i
+
+        for i, child in enumerate(children):
+            if child is None:
+                continue
+
+            if child.name == "#text":
+                data = child.data or ""
+                if not data.strip():
+                    # Drop leading/trailing formatting whitespace in compact mode.
+                    if i == first_non_none_index or i == last_non_none_index:
+                        continue
+                    # Preserve intentional small spacing, but collapse large formatting gaps.
+                    if "\n" in data or "\r" in data or "\t" in data or len(data) > 2:
+                        inner_parts.append(" ")
+                        continue
+
+                data = _normalize_formatting_whitespace(data)
+                child_html = _escape_text(data) if data else ""
+            else:
+                # Even when we can't safely insert whitespace *between* siblings, we can
+                # still pretty-print each element subtree to improve readability.
+                child_html = _node_to_html(child, 0, indent_size, pretty=True, in_pre=content_pre)
+            if child_html:
+                inner_parts.append(child_html)
 
-    if all_text and pretty:
-        return f"{prefix}<{name}{attr_str}>{node.text}</{name}>"
+        return f"{prefix}{open_tag}{''.join(inner_parts)}{serialize_end_tag(name)}"
 
     # Render with child indentation
-    parts = [f"{prefix}<{name}{attr_str}>"]
+    parts = [f"{prefix}{open_tag}"]
     for child in children:
-        child_html = _node_to_html(child, indent + 1, indent_size, pretty)
+        if pretty and not content_pre and _is_whitespace_text_node(child):
+            continue
+        child_html = _node_to_html(child, indent + 1, indent_size, pretty, in_pre=content_pre)
         if child_html:
             parts.append(child_html)
-    parts.append(f"{prefix}</{name}>")
+    parts.append(f"{prefix}{serialize_end_tag(name)}")
     return newline.join(parts) if pretty else "".join(parts)
 
 
-def to_test_format(node, indent=0):
+def to_test_format(node: Any, indent: int = 0) -> str:
     """Convert node to html5lib test format string.
 
     This format is used by html5lib-tests for validating parser output.
@@ -105,26 +466,26 @@ def to_test_format(node, indent=0):
     return _node_to_test_format(node, indent)
 
 
-def _node_to_test_format(node, indent):
+def _node_to_test_format(node: Any, indent: int) -> str:
     """Helper to convert a node to test format."""
     if node.name == "#comment":
-        comment = node.data or ""
+        comment: str = node.data or ""
         return f"| {' ' * indent}<!-- {comment} -->"
 
     if node.name == "!doctype":
         return _doctype_to_test_format(node)
 
     if node.name == "#text":
-        text = node.data or ""
+        text: str = node.data or ""
         return f'| {" " * indent}"{text}"'
 
     # Regular element
     line = f"| {' ' * indent}<{_qualified_name(node)}>"
     attribute_lines = _attrs_to_test_format(node, indent)
 
-    # Template special handling
-    if node.name == "template" and hasattr(node, "template_content") and node.template_content:
-        sections = [line]
+    # Template special handling (only HTML namespace templates have template_content)
+    if node.name == "template" and node.namespace in {None, "html"} and node.template_content is not None:
+        sections: list[str] = [line]
         if attribute_lines:
             sections.extend(attribute_lines)
         content_line = f"| {' ' * (indent + 2)}content"
@@ -142,24 +503,24 @@ def _node_to_test_format(node, indent):
     return "\n".join(sections)
 
 
-def _qualified_name(node):
+def _qualified_name(node: Any) -> str:
     """Get the qualified name of a node (with namespace prefix if needed)."""
     if node.namespace and node.namespace not in {"html", None}:
         return f"{node.namespace} {node.name}"
-    return node.name
+    return str(node.name)
 
 
-def _attrs_to_test_format(node, indent):
+def _attrs_to_test_format(node: Any, indent: int) -> list[str]:
     """Format element attributes for test output."""
     if not node.attrs:
         return []
 
-    formatted = []
+    formatted: list[str] = []
     padding = " " * (indent + 2)
 
     # Prepare display names for sorting
-    display_attrs = []
-    namespace = node.namespace
+    display_attrs: list[tuple[str, str]] = []
+    namespace: str | None = node.namespace
     for attr_name, attr_value in node.attrs.items():
         value = attr_value or ""
         display_name = attr_name
@@ -177,15 +538,15 @@ def _attrs_to_test_format(node, indent):
     return formatted
 
 
-def _doctype_to_test_format(node):
+def _doctype_to_test_format(node: Any) -> str:
     """Format DOCTYPE node for test output."""
     doctype = node.data
 
-    name = doctype.name or ""
-    public_id = doctype.public_id
-    system_id = doctype.system_id
+    name: str = doctype.name or ""
+    public_id: str | None = doctype.public_id
+    system_id: str | None = doctype.system_id
 
-    parts = ["| <!DOCTYPE"]
+    parts: list[str] = ["| <!DOCTYPE"]
     if name:
         parts.append(f" {name}")
     else:

justhtml/stream.py

@@ -1,15 +1,33 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+from .encoding import decode_html
 from .tokenizer import Tokenizer
 from .tokens import CommentToken, DoctypeToken, Tag
 
+# Type alias for stream events
+StreamEvent = tuple[str, Any]
+
+
+class _DummyNode:
+    namespace: str = "html"
+
 
 class StreamSink:
     """A sink that buffers tokens for the stream API."""
 
-    def __init__(self):
+    tokens: list[StreamEvent]
+    open_elements: list[_DummyNode]
+
+    def __init__(self) -> None:
         self.tokens = []
         self.open_elements = []  # Required by tokenizer for rawtext checks
 
-    def process_token(self, token):
+    def process_token(self, token: Tag | CommentToken | DoctypeToken | Any) -> int:
         # Tokenizer reuses token objects, so we must copy data
         if isinstance(token, Tag):
             # Copy tag data
@@ -24,10 +42,7 @@ class StreamSink:
                 # We need a dummy object with namespace for tokenizer checks
                 # Tokenizer checks: stack[-1].namespace
                 # We can just use a simple object
-                class DummyNode:
-                    namespace = "html"
-
-                self.open_elements.append(DummyNode())
+                self.open_elements.append(_DummyNode())
             else:  # Tag.END
                 if self.open_elements:
                     self.open_elements.pop()
@@ -43,19 +58,28 @@ class StreamSink:
 
         return 0  # TokenSinkResult.Continue
 
-    def process_characters(self, data):
+    def process_characters(self, data: str) -> None:
         """Handle character data from tokenizer."""
         self.tokens.append(("text", data))
 
 
-def stream(html):
+def stream(
+    html: str | bytes | bytearray | memoryview,
+    *,
+    encoding: str | None = None,
+) -> Generator[StreamEvent, None, None]:
     """
     Stream HTML events from the given HTML string.
     Yields tuples of (event_type, data).
     """
+    html_str: str
+    if isinstance(html, (bytes, bytearray, memoryview)):
+        html_str, _ = decode_html(bytes(html), transport_encoding=encoding)
+    else:
+        html_str = html
     sink = StreamSink()
     tokenizer = Tokenizer(sink)
-    tokenizer.initialize(html)
+    tokenizer.initialize(html_str)
 
     while True:
         # Run one step of the tokenizer
@@ -64,7 +88,7 @@ def stream(html):
         # Yield any tokens produced by this step
         if sink.tokens:
             # Coalesce text tokens
-            text_buffer = []
+            text_buffer: list[str] = []
             for event, data in sink.tokens:
                 if event == "text":
                     text_buffer.append(data)