spot-sdk-python 2.0.0b13__tar.gz → 2.1.0b1__tar.gz

{spot_sdk_python-2.0.0b13 → spot_sdk_python-2.1.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spot-sdk-python
-Version: 2.0.0b13
+Version: 2.1.0b1
 Summary: SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers
 Author: SPOT Project
 Author-email: SPOT Project <spot@sonn.lu>
@@ -50,7 +50,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b13
+pip install spot-sdk-python==2.0.0
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for

{spot_sdk_python-2.0.0b13 → spot_sdk_python-2.1.0b1}/README.md

@@ -26,7 +26,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b13
+pip install spot-sdk-python==2.0.0
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for

{spot_sdk_python-2.0.0b13 → spot_sdk_python-2.1.0b1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "spot-sdk-python"
-version = "2.0.0b13"
+version = "2.1.0b1"
 description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
 readme = "README.md"
 license = "Apache-2.0"

{spot_sdk_python-2.0.0b13 → spot_sdk_python-2.1.0b1}/pyproject.toml.orig

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "spot-sdk-python"
-version = "2.0.0b13"
+version = "2.1.0b1"
 description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
 readme = "README.md"
 license = "Apache-2.0"

{spot_sdk_python-2.0.0b13 → spot_sdk_python-2.1.0b1}/spot_sdk/email.py

@@ -26,9 +26,14 @@ class Address(BaseModel):
         default="",
         description="As browsers read a host (WHATWG URL Standard): UTS 46-mapped"
         " (lower-cased...), A-labels decoded, no trailing dots; one that can be"
-        " no DNS name only lower-cased: over 1 012 characters or a label over 252,"
-        " invisible ones aside (DNS allows 253 and 63, and NFC composes at most"
-        " 4 code points into one)",
+        " no DNS name only lower-cased: over 1 012 characters, a label over 252"
+        " or over 63 combining marks in a row as UTS 46 maps them, invisible ones"
+        " aside (DNS allows"
+        " 253 and 63, NFC composes at most 4 code points into one, and an"
+        " A-label spells each mark with a character at least); one whose"
+        " A-labels not yet decoded do not fit in the 32 768 characters of"
+        " distinct A-labels a message decodes keeps all its A-labels encoded, as"
+        " UTS 46 maps them (host_limit)",
     )
 
 
@@ -83,9 +88,14 @@ class Url(BaseModel):
         " reads it: percent-decoded, UTS 46-mapped (lower-cased...), A-labels"
         " decoded when UTS 46 leaves them as they are, no trailing dots; one"
         " browsers refuse is kept as read, one that can be no DNS name only"
-        " lower-cased: over 1 012 characters or a label over 252, invisible ones"
-        " aside (DNS allows 253 and 63, and NFC composes at most 4 code points"
-        ' into one); "" when none'
+        " lower-cased: over 1 012 characters, a label over 252 or over 63"
+        " combining marks in a row as UTS 46 maps them, invisible ones aside (DNS"
+        " allows 253 and 63,"
+        " NFC composes at most 4 code points into one, and an A-label spells"
+        " each mark with a character at least); one whose A-labels not yet"
+        " decoded do not fit in the 32 768 characters of distinct A-labels a"
+        " message decodes keeps all its A-labels encoded, as UTS 46 maps them"
+        ' (host_limit); "" when none'
     )
     found_in: Literal["text", "html_href", "html_text", "header"] = Field(
         description="html_text: written in the HTML's visible text, not in an attribute"

{spot_sdk_python-2.0.0b13 → spot_sdk_python-2.1.0b1}/spot_sdk/mime.py

@@ -18,10 +18,12 @@ Anyone can send a message, so the parser never fails on its content:
 - the work is bounded. A part keeps at most ``MAX_HEADERS`` headers, at most
   ``MAX_PARTS`` parts are read, nesting deeper than ``MAX_DEPTH`` is not, a
   boundary longer than ``MAX_BOUNDARY_LENGTH`` is malformed, text, HTML and
-  header values are cut at ``max_text_bytes``, and at most ``MAX_URLS`` URLs
-  are kept. Each bound hit is an anomaly; the ones that leave content out set
-  ``EmailView.truncated``. At most ``MAX_ANOMALIES`` anomalies are recorded,
-  ``MAX_ANOMALIES_PER_CODE`` of each code, but always the first of each;
+  header values are cut at ``max_text_bytes``, at most ``MAX_URLS`` URLs are
+  kept, and the hosts decode at most ``MAX_A_LABEL_CHARACTERS`` characters of
+  distinct A-labels. Each bound hit is an anomaly; the ones that leave content
+  out set ``EmailView.truncated``. At most ``MAX_ANOMALIES`` anomalies are
+  recorded, ``MAX_ANOMALIES_PER_CODE`` of each code, but always the first of
+  each;
 - the view is bounded: its JSON takes at most ``MAX_VIEW_BYTES``, the sum of
   the budgets of its lists and long strings (see :func:`parse`).
 
@@ -87,6 +89,12 @@ MAX_BOUNDARY_LENGTH = 1_000
 MAX_ANOMALIES = 1_000
 MAX_ANOMALIES_PER_CODE = 100
 MAX_URLS = 10_000
+# The characters of distinct A-labels a message's hosts decode: Python decodes
+# Punycode a character at a time, and a message whose links each held other
+# A-labels took half the CPU bound (sdk#28). Each A-label is decoded once a
+# message; 32 768 characters are 520 A-labels of the longest, 63 characters,
+# and decoding them takes about 60 ms.
+MAX_A_LABEL_CHARACTERS = 1 << 15
 # The view's budgets, in bytes of its JSON as model_dump_json(by_alias=True)
 # writes it: UTF-8, escapes included. A list keeps its first entries within
 # its budget (the headers each one that fits), a string its start; each cut is
@@ -181,7 +189,12 @@ HTML_HIDDEN_TEXT = "html_hidden_text"
 """Text a reader may see was left out of visible_text: hidden by an inline style,
 the hidden attribute or the browser's own style (a closed <dialog>, a
 <datalist>), in SVG outside its text elements, or inside <head>, <iframe>,
-<noembed>, <noframes> or <template>."""
+<noembed>, <noframes> or <template>. The detail is how many characters it
+hid, counted as visible_text counts its own (the characters that render as
+nothing removed, whitespace collapsed; not the separators between its nodes
+and lines), over all the part's hidden text; the text itself is never in the
+view. There is no detail when some hidden text could not be counted, a
+template whose content is not read: the amount is unknown."""
 TRANSFER_ENCODING_JUNK = "transfer_encoding_junk"
 """A base64 body goes on after padding, which decoders read differently. The
 data ends at padding that ends a quantum, as RFC 2045 allows, and the rest is
@@ -209,6 +222,13 @@ BIDI_CONTROL = "bidi_control"
 to U+202E, U+2066 to U+2069), which reorder what a reader sees: the
 right-to-left-override file name. Noted where INVISIBLE_CHARACTERS is, with how
 many as the detail; such a string gets both notes."""
+HOST_LIMIT = "host_limit"
+"""The message's hosts hold more than MAX_A_LABEL_CHARACTERS characters of
+distinct A-labels, read in the view's order (the from, reply_to, to and cc
+domains, then the URLs, the first entry a list leaves out included): a host
+whose A-labels not yet decoded do not fit in what is left keeps all its
+A-labels encoded, as UTS 46 maps them. The other hosts are read as url_host
+reads them, the From domain always."""
 ANOMALY_LIMIT = "anomaly_limit"
 """More than MAX_ANOMALIES anomalies were met: the later ones are not recorded."""
 ANOMALY_CODES = (
@@ -231,6 +251,7 @@ ANOMALY_CODES = (
     URL_LIMIT,
     INVISIBLE_CHARACTERS,
     BIDI_CONTROL,
+    HOST_LIMIT,
     ANOMALY_LIMIT,
 )
 """Every anomaly code, in the order above."""
@@ -306,6 +327,9 @@ _HIDDEN_TAGS = frozenset(
 # Leaving out a script, a style or the title is ordinary; leaving out other
 # text hides what a reader may see.
 _ORDINARY_HIDDEN_TAGS = frozenset({"script", "style", "title"})
+# Elements whose text the pattern reader leaves out, though it hides text a
+# reader may see: that text cannot be counted by pattern.
+_UNREAD_BY_PATTERN = re.compile(r"<(?:iframe|noembed|noframes)\b", re.IGNORECASE)
 # Template content is parsed again from its serialisation, only in documents
 # with at most this many template tags: Lexbor serialises nested templates
 # recursively, about 100 bytes of stack each, and 1 500 crashed a thread with
@@ -635,11 +659,13 @@ def parse(
     nodes, left_out = _walk(root, budget=len(raw))
     _note_ambiguous_boundaries(nodes)
     fields, cut_headers = _fields(root, max_text_bytes)
+    punycode = _Punycode()  # the A-labels of the addresses' and URLs' hosts
     read, cut_from = _header_fields(
         root.firsts,
         fields,
         {authserv_id.lower() for authserv_id in trusted_authserv_ids},
         max_text_bytes,
+        punycode,
     )
     given = {"source": _Cutter(), "envelope": _Cutter()}
     source = given["source"](source)
@@ -671,24 +697,26 @@ def parse(
     text = bodies.get("text/plain")
     html = bodies.get("text/html")
     cut_urls = _Cutter()
-    found = [_header_urls(fields, cut_urls)]  # the URLs, built as they are kept
+    # The URLs, built as they are kept.
+    found = [_header_urls(fields, cut_urls, punycode)]
     shown: _Body | None = None  # the body the visible text is read from
     visible_text = ""
     if text:
-        found.append(_urls_in(text.content, "text", text.part_id, cut_urls))
+        found.append(_urls_in(text.content, "text", text.part_id, cut_urls, punycode))
         seen = _visible_body(text, text.content)
         if seen.strip():
             shown, visible_text = text, seen
     if html:
         reader = _read_html(html.content)
-        found.append(_html_urls(reader.links, html.part_id, cut_urls))
+        found.append(_html_urls(reader.links, html.part_id, cut_urls, punycode))
         html_text = reader.text
-        found.append(_urls_in(html_text, "html_text", html.part_id, cut_urls))
+        found.append(_urls_in(html_text, "html_text", html.part_id, cut_urls, punycode))
         if shown is None:
             # Removing them may leave runs of spaces and empty lines.
             shown, visible_text = html, _lines(_visible_body(html, html_text))
         if reader.hidden_text:
-            html.part.note(HTML_HIDDEN_TEXT)
+            amount = [] if reader.uncounted else [str(reader.hidden_characters)]
+            html.part.note(HTML_HIDDEN_TEXT, None, *amount)
         if reader.limit:
             html.part.note(HTML_LIMIT, None, reader.limit)
         if reader.unresolved:
@@ -718,6 +746,8 @@ def parse(
     )
     if left_out_url is not None or cut_urls.cut:
         notes.append((_Note(URL_LIMIT), "1"))
+    if punycode.spent:
+        notes.append((_Note(HOST_LIMIT), "1"))
     notes += [
         (_Note(code, f"url:{index}", (detail,)), None)
         for index, url in enumerate(urls)
@@ -1557,11 +1587,42 @@ def _unquote(value: str) -> str:
     return _QUOTED_PAIR.sub(r"\1", quoted.group(1)) if quoted else value
 
 
-def _domain(host: str) -> str:
+class _Punycode:
+    """The A-labels a message's hosts decode (``_decoded``): each once, and
+    ``MAX_A_LABEL_CHARACTERS`` characters of them in all. A host whose
+    A-labels not yet decoded do not fit in what is left keeps all its labels
+    as UTS 46 maps them, and ``spent`` says one did."""
+
+    def __init__(self) -> None:
+        self.left = MAX_A_LABEL_CHARACTERS
+        self.spent = False
+        self._read: dict[str, str] = {}  # each A-label decoded, as read
+
+    def host(self, mapped: str) -> str:
+        """``mapped``, a host as UTS 46 maps it (``_mapped_host``), with its
+        A-labels decoded; as it is when the A-labels not yet decoded do not fit
+        in what is left. Its labels are split at full stops, as UTS 46 splits a
+        mapped domain."""
+        if "xn--" not in mapped:  # no A-label: a repeated host costs no loop
+            return mapped
+        labels = mapped.split(".")
+        new = {
+            label for label in labels if _is_a_label(label) and label not in self._read
+        }
+        size = sum(map(len, new))
+        if size > self.left:
+            self.spent = True
+            return mapped
+        self.left -= size
+        self._read.update((label, _decoded(label)) for label in new)
+        return ".".join(self._read.get(label, label) for label in labels)
+
+
+def _domain(host: str, punycode: _Punycode) -> str:
     """``host`` as browsers read it, as the WHATWG URL Standard has them do,
     without its trailing dots: mapped as UTS 46 maps it (case, width, the
-    characters it ignores...), each A-label decoded (``_decoded``). A label
-    UTS 46 does not map stays as written, lower-cased.
+    characters it ignores...), each A-label decoded (``_decoded``) within
+    ``punycode``. A label UTS 46 does not map stays as written, lower-cased.
 
     A host that can be no DNS name (``_can_be_dns_name``) reaches no
     resolver: it is only lower-cased, neither mapped nor decoded. That bounds
@@ -1573,32 +1634,38 @@ def _domain(host: str) -> str:
     a label against the Bidi rule, or with a joiner out of context, is read
     all the same.
 
-    A host of at most ``_MAX_DOMAIN`` characters is read once while it stays
-    among the last 256 the process read: a hostile message repeats one host
+    A host of at most ``_MAX_DOMAIN`` characters is mapped once while it stays
+    among the last 256 the process mapped: a hostile message repeats one host
     up to MAX_URLS times (sdk#27). UTS 46 maps a character to at most 18, so
-    the cache holds 19 MiB at most. A longer host is read each time: it is a
+    the cache holds 19 MiB at most. A longer host is mapped each time: it is a
     DNS name only once its invisible characters or its trailing dots are
-    removed."""
-    if len(host) > _MAX_DOMAIN:
-        return _read_domain(host)
-    return _cached_domain(host)
+    removed. Its A-labels are decoded by ``punycode``, once a message: the
+    cache cannot bound a message whose hosts each hold other A-labels
+    (sdk#28)."""
+    mapped = (
+        _mapped_host(host) if len(host) > _MAX_DOMAIN else _cached_mapped_host(host)
+    )
+    if mapped is None:
+        return host.rstrip(_DOT_CHARACTERS).lower()
+    return punycode.host(mapped)
 
 
-def _read_domain(host: str) -> str:
-    """``host`` read as ``_domain`` reads it, each time."""
+def _mapped_host(host: str) -> str | None:
+    """``host`` as ``_domain`` maps it, each time, its A-labels not yet decoded;
+    None when it can be no DNS name."""
     host = host.rstrip(_DOT_CHARACTERS)
     if not _can_be_dns_name(host):
-        return host.lower()
+        return None
     try:
         labels = _remapped(host).split(".")
     except ValueError:  # a code point UTS 46 does not map: each label alone
         labels = [_mapped(label) for label in _DOTS.split(host)]
     while labels and not labels[-1]:
         labels.pop()
-    return ".".join(map(_decoded, labels))
+    return ".".join(labels)
 
 
-_cached_domain = functools.lru_cache(maxsize=256)(_read_domain)
+_cached_mapped_host = functools.lru_cache(maxsize=256)(_mapped_host)
 
 
 def _can_be_dns_name(host: str) -> bool:
@@ -1652,27 +1719,29 @@ def _remapped(text: str) -> str:
     return unicodedata.normalize("NFC", "".join(pieces))
 
 
+def _is_a_label(label: str) -> bool:
+    """Whether a mapped label is an A-label a resolver may reach: ``xn--``, and
+    ``_MAX_A_LABEL`` characters at most. A longer one is a DNS label no
+    resolver reaches: it stays as it is. That bounds Python's Punycode
+    decoding, quadratic in the label."""
+    return label.startswith("xn--") and len(label) <= _MAX_A_LABEL
+
+
 def _decoded(label: str) -> str:
-    """A mapped label as browsers show it: an A-label decoded from Punycode
-    when what it decodes to is a label UTS 46 leaves as it is (its own
-    criterion: valid code points, in NFC), and not ASCII alone; any other
-    label as it is.
+    """A mapped A-label (``_is_a_label``) as browsers show it: decoded from
+    Punycode when what it decodes to is a label UTS 46 leaves as it is (its
+    own criterion: valid code points, in NFC), and not ASCII alone; else as
+    it is.
 
     So an A-label that decodes to what UTS 46 would change stays encoded: a
     fullwidth or a capital letter, a soft hyphen, a full stop, a code point
     it does not map. Mapping it would read a dead A-label as another host,
     a brand's own (``xn--mi7ca7anb7c`` is a fullwidth ``paypal``).
 
-    An A-label over ``_MAX_A_LABEL`` characters is a DNS label no resolver
-    reaches: it stays as it is. That bounds Python's Punycode decoding,
-    quadratic in the label.
-
     There is no other check: IDNA 2008's CONTEXTO rules rescanned a label for
     each code point they concern, and UTS 46's joiner checks took 70 us a
     joiner on idna 3.20, so hostile labels cost seconds a message.
     """
-    if not label.startswith("xn--") or len(label) > _MAX_A_LABEL:
-        return label
     try:
         decoded = label[4:].encode("ascii").decode("punycode")
         if (
@@ -1685,9 +1754,10 @@ def _decoded(label: str) -> str:
     return label
 
 
-def _addresses(value: str, cut: _Cutter) -> Iterator[Address]:
-    """The mailboxes of an address header, each string cut to its budget; a
-    value that does not parse is kept whole."""
+def _addresses(value: str, cut: _Cutter, punycode: _Punycode) -> Iterator[Address]:
+    """The mailboxes of an address header, each string cut to its budget, each
+    domain's A-labels decoded within ``punycode``; a value that does not parse
+    is kept whole."""
     mailboxes = getaddresses([value])
     if not mailboxes or not all("@" in address for _, address in mailboxes):
         value = value.strip()
@@ -1698,7 +1768,7 @@ def _addresses(value: str, cut: _Cutter) -> Iterator[Address]:
         yield Address(
             display_name=cut(_decode_words(name)),
             address=cut(address),
-            domain=cut(_domain(address.rpartition("@")[2])),
+            domain=cut(_domain(address.rpartition("@")[2], punycode)),
         )
 
 
@@ -1729,12 +1799,14 @@ def _header_fields(
     fields: list[tuple[str, str]],
     trusted: set[str],
     limit: int,
+    punycode: _Punycode,
 ) -> tuple[_HeaderFields, list[str]]:
     """The view's fields read from the first header of each name (``firsts``,
     the whole header block's, each value cut at ``limit`` bytes), and the
     authentication results from the trusted headers on top: within their
     budgets. Each string is cut to ``_STRING_BYTES``, the subject to
-    ``_SUBJECT_BYTES``, and each list to ``_LIST_BYTES``.
+    ``_SUBJECT_BYTES``, and each list to ``_LIST_BYTES``; the addresses'
+    A-labels are decoded within ``punycode``.
 
     Also the headers a field was cut from, as ``header:`` subjects.
     """
@@ -1750,7 +1822,7 @@ def _header_fields(
         return first.get(name, ("", ""))[1]
 
     def addresses(name: str) -> list[Address]:
-        return cuts[name].listed(_addresses(header(name), cuts[name]))
+        return cuts[name].listed(_addresses(header(name), cuts[name], punycode))
 
     def message_id(name: str) -> str | None:
         return cuts[name](header(name).strip()) or None
@@ -1758,7 +1830,7 @@ def _header_fields(
     authentication, authentication_cut = _authentication(fields, trusted)
     read = _HeaderFields(
         subject=cuts["subject"](_decode_words(header("subject")), _SUBJECT_BYTES),
-        from_=next(_addresses(header("from"), cuts["from"]), None),
+        from_=next(_addresses(header("from"), cuts["from"], punycode), None),
         reply_to=addresses("reply-to"),
         to=addresses("to"),
         cc=addresses("cc"),
@@ -1933,6 +2005,12 @@ def _lines(text: str) -> str:
     return "\n".join(line for line in lines if line)
 
 
+def _shown_length(text: str) -> int:
+    """The characters ``text`` adds to ``visible_text``: those that render as
+    nothing removed, its whitespace collapsed."""
+    return len(" ".join(visible(text).split()))
+
+
 def url_host(text: str) -> str:
     """The host a browser would reach for ``text``; "" when there is none: the
     host ``parse`` records in ``Url.host``. A backslash reads as a slash, and
@@ -1958,11 +2036,17 @@ def url_host(text: str) -> str:
     character NFKC turns into a delimiter (U+2100 is ``a/c``), in time
     quadratic in a run of combining marks.
     """
+    # A host alone holds far fewer A-label characters than a message decodes.
+    return _url_host(text, _Punycode())
+
+
+def _url_host(text: str, punycode: _Punycode) -> str:
+    """``url_host(text)``, its A-labels decoded within ``punycode``."""
     if text[:4].lower() == "www.":
         text = "http://" + text
     # Browsers read a backslash as a slash in web URLs.
     host = _hostname(text.replace("\\", "/"))
-    return _domain(_clean(unquote_url(host))) if host else ""
+    return _domain(_clean(unquote_url(host)), punycode) if host else ""
 
 
 def _hostname(url: str) -> str | None:
@@ -2022,13 +2106,15 @@ def _url(
     found_in: Literal["text", "html_href", "html_text", "header"],
     part_id: str | None,
     cut: _Cutter,
+    punycode: _Punycode,
     anchor_text: str | None = None,
 ) -> Url:
     """A URL as the view records it: cut to ``_URL_BYTES``, its host (of the
-    whole URL) and anchor text to ``_STRING_BYTES``."""
+    whole URL, its A-labels decoded within ``punycode``) and anchor text to
+    ``_STRING_BYTES``."""
     return Url(
         url=cut(url, _URL_BYTES),
-        host=cut(url_host(url)),
+        host=cut(_url_host(url, punycode)),
         found_in=found_in,
         part_id=part_id,
         anchor_text=cut.optional(anchor_text),
@@ -2036,14 +2122,21 @@ def _url(
 
 
 def _urls_in(
-    text: str, found_in: Literal["text", "html_text"], part_id: str, cut: _Cutter
+    text: str,
+    found_in: Literal["text", "html_text"],
+    part_id: str,
+    cut: _Cutter,
+    punycode: _Punycode,
 ) -> Iterator[Url]:
     """The ``http(s)://`` and ``www.`` URLs written in a text."""
     for match in _URL.finditer(text):
-        yield _url(match.group().rstrip(_URL_TRAILER), found_in, part_id, cut)
+        url = match.group().rstrip(_URL_TRAILER)
+        yield _url(url, found_in, part_id, cut, punycode)
 
 
-def _header_urls(fields: list[tuple[str, str]], cut: _Cutter) -> Iterator[Url]:
+def _header_urls(
+    fields: list[tuple[str, str]], cut: _Cutter, punycode: _Punycode
+) -> Iterator[Url]:
     """The URLs of the List-Unsubscribe headers (RFC 2369)."""
     for name, value in fields:
         if name.lower() != "list-unsubscribe":
@@ -2051,13 +2144,15 @@ def _header_urls(fields: list[tuple[str, str]], cut: _Cutter) -> Iterator[Url]:
         for item in _ANGLE_BRACKETS.findall(value):
             url = "".join(item.split())
             if url:
-                yield _url(url, "header", None, cut)
+                yield _url(url, "header", None, cut, punycode)
 
 
-def _html_urls(links: list[_Link], part_id: str, cut: _Cutter) -> Iterator[Url]:
+def _html_urls(
+    links: list[_Link], part_id: str, cut: _Cutter, punycode: _Punycode
+) -> Iterator[Url]:
     """The URLs of an HTML part's attributes."""
     for link in links:
-        yield _url(link.url, "html_href", part_id, cut, link.anchor_text)
+        yield _url(link.url, "html_href", part_id, cut, punycode, link.anchor_text)
 
 
 @dataclass(slots=True)
@@ -2117,19 +2212,27 @@ class _HtmlReader:
     the text of ``<textarea>``, ``<xmp>`` and ``<plaintext>`` as raw markup, so
     it is visible. Mail clients run no script, so ``<noscript>`` content is
     visible too. Stylesheets are not evaluated. Leaving out text other than a
-    script, a style or the title sets ``hidden_text``; URLs in hidden elements
-    are still collected. A hiding property whose value is known only when
-    rendering hides nothing, and is added to ``unresolved``.
+    script, a style or the title sets ``hidden_text``, and adds what it holds
+    to ``hidden_characters``, counted as ``visible_text`` counts. URLs in
+    hidden elements are still collected. A hiding property whose value is
+    known only when rendering hides nothing, and is added to ``unresolved``.
 
     Lexbor keeps a template's content out of the tree, so it is parsed again
     from the template's serialisation: when the document has at most
     ``_MAX_TEMPLATES`` template tags, and for the document's length in all.
-    Otherwise, a template's content is not read, and is taken to hide text.
+    Otherwise, a template's content is not read, and is taken to hide text,
+    which is not counted: ``uncounted``. The content of a template over the
+    HTML budget once serialised is counted from its text read by pattern,
+    which errs high when a ``<textarea>`` makes the rest read as raw text;
+    with an ``<iframe>``, ``<noembed>`` or ``<noframes>``, whose text that
+    reader leaves out, it is ``uncounted`` too.
     """
 
     def __init__(self, html: str) -> None:
         self.links: list[_Link] = []
         self.hidden_text = False
+        self.hidden_characters = 0
+        self.uncounted = False  # hidden text that could not be counted
         self.unresolved: set[str] = set()  # properties known only when rendering
         self._chunks: list[str] = []
         self._anchors: list[list[str]] = []  # the text of each open link
@@ -2212,7 +2315,9 @@ class _HtmlReader:
 
     def _text(self, text: str, style: _Style) -> None:
         if style.hides:
-            self.hidden_text |= not style.ordinary and bool(text.strip())
+            if not style.ordinary and text.strip():
+                self.hidden_text = True
+                self.hidden_characters += _shown_length(text)
         else:
             # Only block elements break lines; a newline in the source is a space.
             self._emit(text.replace("\n", " "))
@@ -2224,7 +2329,7 @@ class _HtmlReader:
 
     def _template_content(self, template: LexborNode) -> list[LexborNode]:
         if self._budget <= 0:
-            self.hidden_text = True
+            self.hidden_text = self.uncounted = True
             return []
         html = template.html or ""
         # Lexbor escapes ">" in attribute values: the first one ends the start tag.
@@ -2232,6 +2337,12 @@ class _HtmlReader:
         self._budget -= len(content)
         if _html_budget(content):
             self.hidden_text = True
+            if _UNREAD_BY_PATTERN.search(content):
+                self.uncounted = True
+            else:
+                text = _read_by_pattern(content)[0]
+                # A line is a block: its characters, as a text node's.
+                self.hidden_characters += sum(map(_shown_length, text.split("\n")))
             return []
         return _top_nodes(content)