spot-sdk-python 2.0.0b12__tar.gz → 2.1.0b1__tar.gz

{spot_sdk_python-2.0.0b12 → spot_sdk_python-2.1.0b1}/PKG-INFO

@@ -1,25 +1,24 @@
 Metadata-Version: 2.4
 Name: spot-sdk-python
-Version: 2.0.0b12
+Version: 2.1.0b1
 Summary: SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers
-License: Apache-2.0
 Author: SPOT Project
-Author-email: spot@sonn.lu
-Requires-Python: >=3.11,<3.16
-Classifier: License :: OSI Approved :: Apache Software License
+Author-email: SPOT Project <spot@sonn.lu>
+License-Expression: Apache-2.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3.15
-Requires-Dist: fastapi (>=0.133.0,<1)
-Requires-Dist: httpx2 (>=2.13.1,<3)
-Requires-Dist: idna (>=3.11,<4)
-Requires-Dist: pydantic (>=2.10,<3.0)
-Requires-Dist: pydantic-settings (>=2.0.0,<3.0.0)
-Requires-Dist: selectolax (>=1.0,<1.1)
-Requires-Dist: starlette (>=1.7.0,<2)
+Requires-Dist: pydantic>=2.10,<3
+Requires-Dist: pydantic-settings>=2.0.0,<3
+Requires-Dist: httpx2>=2.13.1,<3
+Requires-Dist: selectolax>=1.0,<1.1
+Requires-Dist: idna>=3.11,<4
+Requires-Dist: fastapi>=0.133.0,<1
+Requires-Dist: starlette>=1.7.0,<2
+Requires-Python: >=3.11, <3.16
 Project-URL: Homepage, https://sonn.lu
 Description-Content-Type: text/markdown
 
@@ -51,7 +50,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b12
+pip install spot-sdk-python==2.0.0
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for
@@ -116,4 +115,3 @@ token=...)`.
 ## License
 
 Apache License 2.0.
-

{spot_sdk_python-2.0.0b12 → spot_sdk_python-2.1.0b1}/README.md

@@ -26,7 +26,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b12
+pip install spot-sdk-python==2.0.0
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for

spot_sdk_python-2.1.0b1/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["uv_build>=0.12.22,<0.13"]
+build-backend = "uv_build"
+
+[project]
+name = "spot-sdk-python"
+version = "2.1.0b1"
+description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.11,<3.16"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3.15",
+]
+dependencies = [
+    "pydantic>=2.10,<3",
+    "pydantic-settings>=2.0.0,<3",
+    "httpx2>=2.13.1,<3",
+    "selectolax>=1.0,<1.1",
+    "idna>=3.11,<4",
+    "fastapi>=0.133.0,<1",
+    "starlette>=1.7.0,<2",
+]
+
+[[project.authors]]
+name = "SPOT Project"
+email = "spot@sonn.lu"
+
+[project.urls]
+Homepage = "https://sonn.lu"
+
+[tool.uv.build-backend]
+module-name = "spot_sdk"
+module-root = ""
+
+[tool.ruff]
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "I",
+    "N",
+    "W",
+]
+ignore = ["E501"]

spot_sdk_python-2.1.0b1/pyproject.toml.orig

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["uv_build>=0.12.22,<0.13"]
+build-backend = "uv_build"
+
+[project]
+name = "spot-sdk-python"
+version = "2.1.0b1"
+description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
+readme = "README.md"
+license = "Apache-2.0"
+authors = [{ name = "SPOT Project", email = "spot@sonn.lu" }]
+requires-python = ">=3.11,<3.16"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3.15",
+]
+dependencies = [
+    "pydantic>=2.10,<3",
+    "pydantic-settings>=2.0.0,<3",
+    "httpx2>=2.13.1,<3",
+    "selectolax>=1.0,<1.1",
+    "idna>=3.11,<4",
+    "fastapi>=0.133.0,<1",
+    "starlette>=1.7.0,<2",
+]
+
+[project.urls]
+Homepage = "https://sonn.lu"
+
+[tool.uv.build-backend]
+module-name = "spot_sdk"
+module-root = ""
+
+[tool.ruff]
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "W"]
+ignore = ["E501"]  # Line too long

{spot_sdk_python-2.0.0b12 → spot_sdk_python-2.1.0b1}/spot_sdk/email.py

@@ -26,9 +26,14 @@ class Address(BaseModel):
         default="",
         description="As browsers read a host (WHATWG URL Standard): UTS 46-mapped"
         " (lower-cased...), A-labels decoded, no trailing dots; one that can be"
-        " no DNS name only lower-cased: over 1 012 characters or a label over 252,"
-        " invisible ones aside (DNS allows 253 and 63, and NFC composes at most"
-        " 4 code points into one)",
+        " no DNS name only lower-cased: over 1 012 characters, a label over 252"
+        " or over 63 combining marks in a row as UTS 46 maps them, invisible ones"
+        " aside (DNS allows"
+        " 253 and 63, NFC composes at most 4 code points into one, and an"
+        " A-label spells each mark with a character at least); one whose"
+        " A-labels not yet decoded do not fit in the 32 768 characters of"
+        " distinct A-labels a message decodes keeps all its A-labels encoded, as"
+        " UTS 46 maps them (host_limit)",
     )
 
 
@@ -83,9 +88,14 @@ class Url(BaseModel):
         " reads it: percent-decoded, UTS 46-mapped (lower-cased...), A-labels"
         " decoded when UTS 46 leaves them as they are, no trailing dots; one"
         " browsers refuse is kept as read, one that can be no DNS name only"
-        " lower-cased: over 1 012 characters or a label over 252, invisible ones"
-        " aside (DNS allows 253 and 63, and NFC composes at most 4 code points"
-        ' into one); "" when none'
+        " lower-cased: over 1 012 characters, a label over 252 or over 63"
+        " combining marks in a row as UTS 46 maps them, invisible ones aside (DNS"
+        " allows 253 and 63,"
+        " NFC composes at most 4 code points into one, and an A-label spells"
+        " each mark with a character at least); one whose A-labels not yet"
+        " decoded do not fit in the 32 768 characters of distinct A-labels a"
+        " message decodes keeps all its A-labels encoded, as UTS 46 maps them"
+        ' (host_limit); "" when none'
     )
     found_in: Literal["text", "html_href", "html_text", "header"] = Field(
         description="html_text: written in the HTML's visible text, not in an attribute"

{spot_sdk_python-2.0.0b12 → spot_sdk_python-2.1.0b1}/spot_sdk/knowledge.py

@@ -64,8 +64,9 @@ class KnowledgeDocument(BaseModel):
     id: str = Field(
         ...,
         description=(
-            "Stable id, unique within its source (e.g. 'employee:alice@co.com'). "
-            "Upsert is idempotent on (source, id)."
+            "Stable id, unique within its source, e.g. a key of the source's own or a "
+            "keyed hash. It appears in URL paths and logs, so it never holds personal "
+            "data such as an email address. Upsert is idempotent on (source, id)."
         ),
     )
     content: str = Field(

{spot_sdk_python-2.0.0b12 → spot_sdk_python-2.1.0b1}/spot_sdk/mime.py

@@ -18,10 +18,12 @@ Anyone can send a message, so the parser never fails on its content:
 - the work is bounded. A part keeps at most ``MAX_HEADERS`` headers, at most
   ``MAX_PARTS`` parts are read, nesting deeper than ``MAX_DEPTH`` is not, a
   boundary longer than ``MAX_BOUNDARY_LENGTH`` is malformed, text, HTML and
-  header values are cut at ``max_text_bytes``, and at most ``MAX_URLS`` URLs
-  are kept. Each bound hit is an anomaly; the ones that leave content out set
-  ``EmailView.truncated``. At most ``MAX_ANOMALIES`` anomalies are recorded,
-  ``MAX_ANOMALIES_PER_CODE`` of each code, but always the first of each;
+  header values are cut at ``max_text_bytes``, at most ``MAX_URLS`` URLs are
+  kept, and the hosts decode at most ``MAX_A_LABEL_CHARACTERS`` characters of
+  distinct A-labels. Each bound hit is an anomaly; the ones that leave content
+  out set ``EmailView.truncated``. At most ``MAX_ANOMALIES`` anomalies are
+  recorded, ``MAX_ANOMALIES_PER_CODE`` of each code, but always the first of
+  each;
 - the view is bounded: its JSON takes at most ``MAX_VIEW_BYTES``, the sum of
   the budgets of its lists and long strings (see :func:`parse`).
 
@@ -38,7 +40,9 @@ import base64
 import binascii
 import codecs
 import email
+import functools
 import hashlib
+import ipaddress
 import json
 import math
 import re
@@ -54,8 +58,8 @@ from email.utils import decode_params, getaddresses, parsedate_to_datetime, unqu
 from html import unescape as unescape_html
 from itertools import chain
 from typing import Any, Literal, NamedTuple, TypeAlias, TypeVar, cast
+from urllib.parse import SplitResult
 from urllib.parse import unquote as unquote_url
-from urllib.parse import urlsplit
 from uuid import UUID
 
 import idna
@@ -85,6 +89,12 @@ MAX_BOUNDARY_LENGTH = 1_000
 MAX_ANOMALIES = 1_000
 MAX_ANOMALIES_PER_CODE = 100
 MAX_URLS = 10_000
+# The characters of distinct A-labels a message's hosts decode: Python decodes
+# Punycode a character at a time, and a message whose links each held other
+# A-labels took half the CPU bound (sdk#28). Each A-label is decoded once a
+# message; 32 768 characters are 520 A-labels of the longest, 63 characters,
+# and decoding them takes about 60 ms.
+MAX_A_LABEL_CHARACTERS = 1 << 15
 # The view's budgets, in bytes of its JSON as model_dump_json(by_alias=True)
 # writes it: UTF-8, escapes included. A list keeps its first entries within
 # its budget (the headers each one that fits), a string its start; each cut is
@@ -179,7 +189,12 @@ HTML_HIDDEN_TEXT = "html_hidden_text"
 """Text a reader may see was left out of visible_text: hidden by an inline style,
 the hidden attribute or the browser's own style (a closed <dialog>, a
 <datalist>), in SVG outside its text elements, or inside <head>, <iframe>,
-<noembed>, <noframes> or <template>."""
+<noembed>, <noframes> or <template>. The detail is how many characters it
+hid, counted as visible_text counts its own (the characters that render as
+nothing removed, whitespace collapsed; not the separators between its nodes
+and lines), over all the part's hidden text; the text itself is never in the
+view. There is no detail when some hidden text could not be counted, a
+template whose content is not read: the amount is unknown."""
 TRANSFER_ENCODING_JUNK = "transfer_encoding_junk"
 """A base64 body goes on after padding, which decoders read differently. The
 data ends at padding that ends a quantum, as RFC 2045 allows, and the rest is
@@ -207,6 +222,13 @@ BIDI_CONTROL = "bidi_control"
 to U+202E, U+2066 to U+2069), which reorder what a reader sees: the
 right-to-left-override file name. Noted where INVISIBLE_CHARACTERS is, with how
 many as the detail; such a string gets both notes."""
+HOST_LIMIT = "host_limit"
+"""The message's hosts hold more than MAX_A_LABEL_CHARACTERS characters of
+distinct A-labels, read in the view's order (the from, reply_to, to and cc
+domains, then the URLs, the first entry a list leaves out included): a host
+whose A-labels not yet decoded do not fit in what is left keeps all its
+A-labels encoded, as UTS 46 maps them. The other hosts are read as url_host
+reads them, the From domain always."""
 ANOMALY_LIMIT = "anomaly_limit"
 """More than MAX_ANOMALIES anomalies were met: the later ones are not recorded."""
 ANOMALY_CODES = (
@@ -229,6 +251,7 @@ ANOMALY_CODES = (
     URL_LIMIT,
     INVISIBLE_CHARACTERS,
     BIDI_CONTROL,
+    HOST_LIMIT,
     ANOMALY_LIMIT,
 )
 """Every anomaly code, in the order above."""
@@ -304,6 +327,9 @@ _HIDDEN_TAGS = frozenset(
 # Leaving out a script, a style or the title is ordinary; leaving out other
 # text hides what a reader may see.
 _ORDINARY_HIDDEN_TAGS = frozenset({"script", "style", "title"})
+# Elements whose text the pattern reader leaves out, though it hides text a
+# reader may see: that text cannot be counted by pattern.
+_UNREAD_BY_PATTERN = re.compile(r"<(?:iframe|noembed|noframes)\b", re.IGNORECASE)
 # Template content is parsed again from its serialisation, only in documents
 # with at most this many template tags: Lexbor serialises nested templates
 # recursively, about 100 bytes of stack each, and 1 500 crashed a thread with
@@ -519,6 +545,38 @@ _MAX_LABEL = 4 * 63
 _MAX_A_LABEL = 63
 # The most characters idna maps a call (from 3.17).
 _IDNA_INPUT = 1024
+# NFC reorders a run of combining marks (of a canonical combining class
+# other than 0) in time quadratic in it. A label with more marks in a
+# row than an A-label holds characters maps to no DNS label: NFC composes at
+# most 3 of them into the character before (U+1F8F), and the 61 or more left
+# are code points an A-label spells with a character each at least, after its
+# "xn--": 65 characters.
+_MAX_MARKS = _MAX_A_LABEL
+_MARK_RUN = re.compile(rb"(?<![^\x00])[^\x00]{%d}" % (_MAX_MARKS + 1))
+# The characters UTS 46 maps to other marks than their own class counts (idna
+# 3.20, Unicode 14.0 and 15.1): to two, to marks though they are of class 0,
+# or to an iota, no mark.
+_MARK_READINGS = str.maketrans(
+    {
+        "\u0344": "\u0308\u0301",
+        "\u0345": "\u03b9",
+        "\u0f73": "\u0f71\u0f72",
+        "\u0f75": "\u0f71\u0f74",
+        "\u0f81": "\u0f71\u0f80",
+        "\uff9e": "\u3099",
+        "\uff9f": "\u309a",
+    }
+)
+# Python's urlsplit reads a URL without its leading C0 controls and spaces,
+# and without its tabs and line breaks. Its netloc follows "//", after the
+# scheme if any, up to the first of "/?#".
+_URL_LEADING = "".join(map(chr, range(0x21)))
+_URL_REMOVED = "\t\r\n"
+_NETLOC = re.compile(r"(?:[A-Za-z][A-Za-z0-9+.-]*:)?//([^/?#]*)")
+# What urlsplit refuses NFKC to make of a netloc's characters.
+_URL_DELIMITERS = frozenset("/?#@:")
+# An IPvFuture address, in brackets (RFC 3986).
+_IPV_FUTURE = re.compile(r"v[a-fA-F0-9]+\..+")
 
 
 # A parameter as email.utils decodes it: a value, or an RFC 2231
@@ -601,11 +659,13 @@ def parse(
     nodes, left_out = _walk(root, budget=len(raw))
     _note_ambiguous_boundaries(nodes)
     fields, cut_headers = _fields(root, max_text_bytes)
+    punycode = _Punycode()  # the A-labels of the addresses' and URLs' hosts
     read, cut_from = _header_fields(
         root.firsts,
         fields,
         {authserv_id.lower() for authserv_id in trusted_authserv_ids},
         max_text_bytes,
+        punycode,
     )
     given = {"source": _Cutter(), "envelope": _Cutter()}
     source = given["source"](source)
@@ -637,24 +697,26 @@ def parse(
     text = bodies.get("text/plain")
     html = bodies.get("text/html")
     cut_urls = _Cutter()
-    found = [_header_urls(fields, cut_urls)]  # the URLs, built as they are kept
+    # The URLs, built as they are kept.
+    found = [_header_urls(fields, cut_urls, punycode)]
     shown: _Body | None = None  # the body the visible text is read from
     visible_text = ""
     if text:
-        found.append(_urls_in(text.content, "text", text.part_id, cut_urls))
+        found.append(_urls_in(text.content, "text", text.part_id, cut_urls, punycode))
         seen = _visible_body(text, text.content)
         if seen.strip():
             shown, visible_text = text, seen
     if html:
         reader = _read_html(html.content)
-        found.append(_html_urls(reader.links, html.part_id, cut_urls))
+        found.append(_html_urls(reader.links, html.part_id, cut_urls, punycode))
         html_text = reader.text
-        found.append(_urls_in(html_text, "html_text", html.part_id, cut_urls))
+        found.append(_urls_in(html_text, "html_text", html.part_id, cut_urls, punycode))
         if shown is None:
             # Removing them may leave runs of spaces and empty lines.
             shown, visible_text = html, _lines(_visible_body(html, html_text))
         if reader.hidden_text:
-            html.part.note(HTML_HIDDEN_TEXT)
+            amount = [] if reader.uncounted else [str(reader.hidden_characters)]
+            html.part.note(HTML_HIDDEN_TEXT, None, *amount)
         if reader.limit:
             html.part.note(HTML_LIMIT, None, reader.limit)
         if reader.unresolved:
@@ -684,6 +746,8 @@ def parse(
     )
     if left_out_url is not None or cut_urls.cut:
         notes.append((_Note(URL_LIMIT), "1"))
+    if punycode.spent:
+        notes.append((_Note(HOST_LIMIT), "1"))
     notes += [
         (_Note(code, f"url:{index}", (detail,)), None)
         for index, url in enumerate(urls)
@@ -1523,45 +1587,115 @@ def _unquote(value: str) -> str:
     return _QUOTED_PAIR.sub(r"\1", quoted.group(1)) if quoted else value
 
 
-def _domain(host: str) -> str:
+class _Punycode:
+    """The A-labels a message's hosts decode (``_decoded``): each once, and
+    ``MAX_A_LABEL_CHARACTERS`` characters of them in all. A host whose
+    A-labels not yet decoded do not fit in what is left keeps all its labels
+    as UTS 46 maps them, and ``spent`` says one did."""
+
+    def __init__(self) -> None:
+        self.left = MAX_A_LABEL_CHARACTERS
+        self.spent = False
+        self._read: dict[str, str] = {}  # each A-label decoded, as read
+
+    def host(self, mapped: str) -> str:
+        """``mapped``, a host as UTS 46 maps it (``_mapped_host``), with its
+        A-labels decoded; as it is when the A-labels not yet decoded do not fit
+        in what is left. Its labels are split at full stops, as UTS 46 splits a
+        mapped domain."""
+        if "xn--" not in mapped:  # no A-label: a repeated host costs no loop
+            return mapped
+        labels = mapped.split(".")
+        new = {
+            label for label in labels if _is_a_label(label) and label not in self._read
+        }
+        size = sum(map(len, new))
+        if size > self.left:
+            self.spent = True
+            return mapped
+        self.left -= size
+        self._read.update((label, _decoded(label)) for label in new)
+        return ".".join(self._read.get(label, label) for label in labels)
+
+
+def _domain(host: str, punycode: _Punycode) -> str:
     """``host`` as browsers read it, as the WHATWG URL Standard has them do,
     without its trailing dots: mapped as UTS 46 maps it (case, width, the
-    characters it ignores...), each A-label decoded (``_decoded``). A label
-    UTS 46 does not map stays as written, lower-cased.
+    characters it ignores...), each A-label decoded (``_decoded``) within
+    ``punycode``. A label UTS 46 does not map stays as written, lower-cased.
 
     A host that can be no DNS name (``_can_be_dns_name``) reaches no
     resolver: it is only lower-cased, neither mapped nor decoded. That bounds
     the work a host costs: CPython reorders combining marks for NFC in time
-    quadratic in their run, and a run ends at a dot.
+    quadratic in their run, and a longer run than ``_MAX_MARKS`` makes no DNS
+    name.
 
     It reads the host, and does not decide whether a browser would accept it:
     a label against the Bidi rule, or with a joiner out of context, is read
-    all the same."""
+    all the same.
+
+    A host of at most ``_MAX_DOMAIN`` characters is mapped once while it stays
+    among the last 256 the process mapped: a hostile message repeats one host
+    up to MAX_URLS times (sdk#27). UTS 46 maps a character to at most 18, so
+    the cache holds 19 MiB at most. A longer host is mapped each time: it is a
+    DNS name only once its invisible characters or its trailing dots are
+    removed. Its A-labels are decoded by ``punycode``, once a message: the
+    cache cannot bound a message whose hosts each hold other A-labels
+    (sdk#28)."""
+    mapped = (
+        _mapped_host(host) if len(host) > _MAX_DOMAIN else _cached_mapped_host(host)
+    )
+    if mapped is None:
+        return host.rstrip(_DOT_CHARACTERS).lower()
+    return punycode.host(mapped)
+
+
+def _mapped_host(host: str) -> str | None:
+    """``host`` as ``_domain`` maps it, each time, its A-labels not yet decoded;
+    None when it can be no DNS name."""
     host = host.rstrip(_DOT_CHARACTERS)
     if not _can_be_dns_name(host):
-        return host.lower()
+        return None
     try:
         labels = _remapped(host).split(".")
     except ValueError:  # a code point UTS 46 does not map: each label alone
         labels = [_mapped(label) for label in _DOTS.split(host)]
     while labels and not labels[-1]:
         labels.pop()
-    return ".".join(map(_decoded, labels))
+    return ".".join(labels)
+
+
+_cached_mapped_host = functools.lru_cache(maxsize=256)(_mapped_host)
 
 
 def _can_be_dns_name(host: str) -> bool:
     """Whether ``host`` may map to a DNS name, counted on what a resolver could
     see: without its invisible characters (``visible``, which holds all those
     UTS 46 removes) and its trailing dots, it keeps at most ``_MAX_DOMAIN``
-    characters, and each label at most ``_MAX_LABEL``. UTS 46 maps any other
+    characters, each label at most ``_MAX_LABEL``, and at most ``_MAX_MARKS``
+    combining marks in a row as UTS 46 maps them. UTS 46 maps any other
     character to one or more, so the count never exceeds the mapped host's
-    before NFC. It only counts: the host is read as UTS 46 maps it."""
+    before NFC. The joiners, the only invisible characters UTS 46 keeps, may
+    split a run counted whole: no ASCII either, they leave the label as long
+    in its A-label. It only counts: the host is read as UTS 46 maps it."""
     counted = visible(host).rstrip(_DOT_CHARACTERS)
-    return len(counted) <= _MAX_DOMAIN and all(
-        len(label) <= _MAX_LABEL for label in _DOTS.split(counted)
+    return (
+        len(counted) <= _MAX_DOMAIN
+        and all(len(label) <= _MAX_LABEL for label in _DOTS.split(counted))
+        and not _too_many_marks(counted)
     )
 
 
+def _too_many_marks(text: str) -> bool:
+    """Whether ``text`` holds more than ``_MAX_MARKS`` combining marks in a row,
+    as UTS 46 maps them (``_MARK_READINGS``). In linear time: the classes, all
+    under 256, are matched as bytes, from the start of each run."""
+    if text.isascii():
+        return False
+    classes = bytes(map(unicodedata.combining, text.translate(_MARK_READINGS)))
+    return _MARK_RUN.search(classes) is not None
+
+
 def _mapped(label: str) -> str:
     """A label mapped as UTS 46 maps it; as written, lower-cased, when it does
     not map."""
@@ -1585,27 +1719,29 @@ def _remapped(text: str) -> str:
     return unicodedata.normalize("NFC", "".join(pieces))
 
 
+def _is_a_label(label: str) -> bool:
+    """Whether a mapped label is an A-label a resolver may reach: ``xn--``, and
+    ``_MAX_A_LABEL`` characters at most. A longer one is a DNS label no
+    resolver reaches: it stays as it is. That bounds Python's Punycode
+    decoding, quadratic in the label."""
+    return label.startswith("xn--") and len(label) <= _MAX_A_LABEL
+
+
 def _decoded(label: str) -> str:
-    """A mapped label as browsers show it: an A-label decoded from Punycode
-    when what it decodes to is a label UTS 46 leaves as it is (its own
-    criterion: valid code points, in NFC), and not ASCII alone; any other
-    label as it is.
+    """A mapped A-label (``_is_a_label``) as browsers show it: decoded from
+    Punycode when what it decodes to is a label UTS 46 leaves as it is (its
+    own criterion: valid code points, in NFC), and not ASCII alone; else as
+    it is.
 
     So an A-label that decodes to what UTS 46 would change stays encoded: a
     fullwidth or a capital letter, a soft hyphen, a full stop, a code point
     it does not map. Mapping it would read a dead A-label as another host,
     a brand's own (``xn--mi7ca7anb7c`` is a fullwidth ``paypal``).
 
-    An A-label over ``_MAX_A_LABEL`` characters is a DNS label no resolver
-    reaches: it stays as it is. That bounds Python's Punycode decoding,
-    quadratic in the label.
-
     There is no other check: IDNA 2008's CONTEXTO rules rescanned a label for
     each code point they concern, and UTS 46's joiner checks took 70 us a
     joiner on idna 3.20, so hostile labels cost seconds a message.
     """
-    if not label.startswith("xn--") or len(label) > _MAX_A_LABEL:
-        return label
     try:
         decoded = label[4:].encode("ascii").decode("punycode")
         if (
@@ -1618,9 +1754,10 @@ def _decoded(label: str) -> str:
     return label
 
 
-def _addresses(value: str, cut: _Cutter) -> Iterator[Address]:
-    """The mailboxes of an address header, each string cut to its budget; a
-    value that does not parse is kept whole."""
+def _addresses(value: str, cut: _Cutter, punycode: _Punycode) -> Iterator[Address]:
+    """The mailboxes of an address header, each string cut to its budget, each
+    domain's A-labels decoded within ``punycode``; a value that does not parse
+    is kept whole."""
     mailboxes = getaddresses([value])
     if not mailboxes or not all("@" in address for _, address in mailboxes):
         value = value.strip()
@@ -1631,7 +1768,7 @@ def _addresses(value: str, cut: _Cutter) -> Iterator[Address]:
         yield Address(
             display_name=cut(_decode_words(name)),
             address=cut(address),
-            domain=cut(_domain(address.rpartition("@")[2])),
+            domain=cut(_domain(address.rpartition("@")[2], punycode)),
         )
 
 
@@ -1662,12 +1799,14 @@ def _header_fields(
     fields: list[tuple[str, str]],
     trusted: set[str],
     limit: int,
+    punycode: _Punycode,
 ) -> tuple[_HeaderFields, list[str]]:
     """The view's fields read from the first header of each name (``firsts``,
     the whole header block's, each value cut at ``limit`` bytes), and the
     authentication results from the trusted headers on top: within their
     budgets. Each string is cut to ``_STRING_BYTES``, the subject to
-    ``_SUBJECT_BYTES``, and each list to ``_LIST_BYTES``.
+    ``_SUBJECT_BYTES``, and each list to ``_LIST_BYTES``; the addresses'
+    A-labels are decoded within ``punycode``.
 
     Also the headers a field was cut from, as ``header:`` subjects.
     """
@@ -1683,7 +1822,7 @@ def _header_fields(
         return first.get(name, ("", ""))[1]
 
     def addresses(name: str) -> list[Address]:
-        return cuts[name].listed(_addresses(header(name), cuts[name]))
+        return cuts[name].listed(_addresses(header(name), cuts[name], punycode))
 
     def message_id(name: str) -> str | None:
         return cuts[name](header(name).strip()) or None
@@ -1691,7 +1830,7 @@ def _header_fields(
     authentication, authentication_cut = _authentication(fields, trusted)
     read = _HeaderFields(
         subject=cuts["subject"](_decode_words(header("subject")), _SUBJECT_BYTES),
-        from_=next(_addresses(header("from"), cuts["from"]), None),
+        from_=next(_addresses(header("from"), cuts["from"], punycode), None),
         reply_to=addresses("reply-to"),
         to=addresses("to"),
         cc=addresses("cc"),
@@ -1866,6 +2005,12 @@ def _lines(text: str) -> str:
     return "\n".join(line for line in lines if line)
 
 
+def _shown_length(text: str) -> int:
+    """The characters ``text`` adds to ``visible_text``: those that render as
+    nothing removed, its whitespace collapsed."""
+    return len(" ".join(visible(text).split()))
+
+
 def url_host(text: str) -> str:
     """The host a browser would reach for ``text``; "" when there is none: the
     host ``parse`` records in ``Url.host``. A backslash reads as a slash, and
@@ -1878,19 +2023,82 @@ def url_host(text: str) -> str:
     dropped. A label UTS 46 does not map stays as written, lower-cased, a
     host browsers refuse (a space, a slash, a label against the Bidi rule) is
     kept as read, and a host that can be no DNS name is only lower-cased:
-    never an exception. That is a host that keeps over 1 012 characters, or a
-    label over 252, once its invisible characters are removed: a DNS name
-    holds 253, a label 63, and NFC composes at most 4 code points into one.
-    An A-label over 63 characters stays encoded.
+    never an exception. That is a host that keeps over 1 012 characters, a
+    label over 252, or over 63 combining marks in a row, once its invisible
+    characters are removed: a DNS name holds 253, a label 63, NFC composes at
+    most 4 code points into one, and an A-label spells each code point that is
+    not ASCII with a character at least. An A-label over 63 characters stays
+    encoded.
+
+    The host is found as CPython 3.11's urlsplit reads it (3.13 also accepts
+    an IPvFuture ``V``), its ``.hostname``, and none where urlsplit refuses the
+    URL, but in linear time: urlsplit puts the netloc in NFKC to refuse a
+    character NFKC turns into a delimiter (U+2100 is ``a/c``), in time
+    quadratic in a run of combining marks.
     """
+    # A host alone holds far fewer A-label characters than a message decodes.
+    return _url_host(text, _Punycode())
+
+
+def _url_host(text: str, punycode: _Punycode) -> str:
+    """``url_host(text)``, its A-labels decoded within ``punycode``."""
     if text[:4].lower() == "www.":
         text = "http://" + text
+    # Browsers read a backslash as a slash in web URLs.
+    host = _hostname(text.replace("\\", "/"))
+    return _domain(_clean(unquote_url(host)), punycode) if host else ""
+
+
+def _hostname(url: str) -> str | None:
+    """``urlsplit(url).hostname``, as CPython 3.11's urlsplit reads it (3.13
+    also accepts an IPvFuture ``V``); None where urlsplit refuses ``url``: a
+    bracket without its pair, brackets misplaced or around no IPv6 or IPvFuture
+    address (``_accepted_brackets``), or a character NFKC turns into a
+    delimiter. That last is checked a character at a time: NFKC composes no
+    delimiter, and the netloc's own ``@`` and ``:`` do not count."""
+    netloc = _netloc(url)
+    if ("[" in netloc) != ("]" in netloc):
+        return None
+    if "[" in netloc and not _accepted_brackets(netloc.rpartition("@")[2]):
+        return None
+    if any(
+        not _URL_DELIMITERS.isdisjoint(unicodedata.normalize("NFKD", character))
+        for character in set(netloc)
+        if not character.isascii()
+    ):
+        return None
+    return SplitResult("", netloc, "", "", "").hostname
+
+
+def _netloc(url: str) -> str:
+    """The netloc urlsplit reads in ``url``, unchecked; "" when it has none."""
+    url = url.lstrip(_URL_LEADING)
+    for removed in _URL_REMOVED:
+        url = url.replace(removed, "")
+    found = _NETLOC.match(url)
+    return found.group(1) if found else ""
+
+
+def _accepted_brackets(hostinfo: str) -> bool:
+    """Whether urlsplit accepts the brackets of a netloc whose ``hostinfo``
+    follows its last ``@``: nothing before them, only a port after them, and an
+    IPv6 or IPvFuture address in them. With the brackets in the userinfo, the
+    host itself must be such an address."""
+    before, bracket, bracketed = hostinfo.partition("[")
+    if not bracket:
+        host = before.partition(":")[0]
+    elif before:
+        return False
+    else:
+        host, _, port = bracketed.partition("]")
+        if port and not port.startswith(":"):
+            return False
+    if host.startswith("v"):
+        return _IPV_FUTURE.fullmatch(host) is not None
     try:
-        # Browsers read a backslash as a slash in web URLs.
-        host = urlsplit(text.replace("\\", "/")).hostname
+        return ipaddress.ip_address(host).version == 6
     except ValueError:
-        return ""
-    return _domain(_clean(unquote_url(host))) if host else ""
+        return False
 
 
 def _url(
@@ -1898,13 +2106,15 @@ def _url(
     found_in: Literal["text", "html_href", "html_text", "header"],
     part_id: str | None,
     cut: _Cutter,
+    punycode: _Punycode,
     anchor_text: str | None = None,
 ) -> Url:
     """A URL as the view records it: cut to ``_URL_BYTES``, its host (of the
-    whole URL) and anchor text to ``_STRING_BYTES``."""
+    whole URL, its A-labels decoded within ``punycode``) and anchor text to
+    ``_STRING_BYTES``."""
     return Url(
         url=cut(url, _URL_BYTES),
-        host=cut(url_host(url)),
+        host=cut(_url_host(url, punycode)),
         found_in=found_in,
         part_id=part_id,
         anchor_text=cut.optional(anchor_text),
@@ -1912,14 +2122,21 @@ def _url(
 
 
 def _urls_in(
-    text: str, found_in: Literal["text", "html_text"], part_id: str, cut: _Cutter
+    text: str,
+    found_in: Literal["text", "html_text"],
+    part_id: str,
+    cut: _Cutter,
+    punycode: _Punycode,
 ) -> Iterator[Url]:
     """The ``http(s)://`` and ``www.`` URLs written in a text."""
     for match in _URL.finditer(text):
-        yield _url(match.group().rstrip(_URL_TRAILER), found_in, part_id, cut)
+        url = match.group().rstrip(_URL_TRAILER)
+        yield _url(url, found_in, part_id, cut, punycode)
 
 
-def _header_urls(fields: list[tuple[str, str]], cut: _Cutter) -> Iterator[Url]:
+def _header_urls(
+    fields: list[tuple[str, str]], cut: _Cutter, punycode: _Punycode
+) -> Iterator[Url]:
     """The URLs of the List-Unsubscribe headers (RFC 2369)."""
     for name, value in fields:
         if name.lower() != "list-unsubscribe":
@@ -1927,13 +2144,15 @@ def _header_urls(fields: list[tuple[str, str]], cut: _Cutter) -> Iterator[Url]:
         for item in _ANGLE_BRACKETS.findall(value):
             url = "".join(item.split())
             if url:
-                yield _url(url, "header", None, cut)
+                yield _url(url, "header", None, cut, punycode)
 
 
-def _html_urls(links: list[_Link], part_id: str, cut: _Cutter) -> Iterator[Url]:
+def _html_urls(
+    links: list[_Link], part_id: str, cut: _Cutter, punycode: _Punycode
+) -> Iterator[Url]:
     """The URLs of an HTML part's attributes."""
     for link in links:
-        yield _url(link.url, "html_href", part_id, cut, link.anchor_text)
+        yield _url(link.url, "html_href", part_id, cut, punycode, link.anchor_text)
 
 
 @dataclass(slots=True)
@@ -1993,19 +2212,27 @@ class _HtmlReader:
     the text of ``<textarea>``, ``<xmp>`` and ``<plaintext>`` as raw markup, so
     it is visible. Mail clients run no script, so ``<noscript>`` content is
     visible too. Stylesheets are not evaluated. Leaving out text other than a
-    script, a style or the title sets ``hidden_text``; URLs in hidden elements
-    are still collected. A hiding property whose value is known only when
-    rendering hides nothing, and is added to ``unresolved``.
+    script, a style or the title sets ``hidden_text``, and adds what it holds
+    to ``hidden_characters``, counted as ``visible_text`` counts. URLs in
+    hidden elements are still collected. A hiding property whose value is
+    known only when rendering hides nothing, and is added to ``unresolved``.
 
     Lexbor keeps a template's content out of the tree, so it is parsed again
     from the template's serialisation: when the document has at most
     ``_MAX_TEMPLATES`` template tags, and for the document's length in all.
-    Otherwise, a template's content is not read, and is taken to hide text.
+    Otherwise, a template's content is not read, and is taken to hide text,
+    which is not counted: ``uncounted``. The content of a template over the
+    HTML budget once serialised is counted from its text read by pattern,
+    which errs high when a ``<textarea>`` makes the rest read as raw text;
+    with an ``<iframe>``, ``<noembed>`` or ``<noframes>``, whose text that
+    reader leaves out, it is ``uncounted`` too.
     """
 
     def __init__(self, html: str) -> None:
         self.links: list[_Link] = []
         self.hidden_text = False
+        self.hidden_characters = 0
+        self.uncounted = False  # hidden text that could not be counted
         self.unresolved: set[str] = set()  # properties known only when rendering
         self._chunks: list[str] = []
         self._anchors: list[list[str]] = []  # the text of each open link
@@ -2088,7 +2315,9 @@ class _HtmlReader:
 
     def _text(self, text: str, style: _Style) -> None:
         if style.hides:
-            self.hidden_text |= not style.ordinary and bool(text.strip())
+            if not style.ordinary and text.strip():
+                self.hidden_text = True
+                self.hidden_characters += _shown_length(text)
         else:
             # Only block elements break lines; a newline in the source is a space.
             self._emit(text.replace("\n", " "))
@@ -2100,7 +2329,7 @@ class _HtmlReader:
 
     def _template_content(self, template: LexborNode) -> list[LexborNode]:
         if self._budget <= 0:
-            self.hidden_text = True
+            self.hidden_text = self.uncounted = True
             return []
         html = template.html or ""
         # Lexbor escapes ">" in attribute values: the first one ends the start tag.
@@ -2108,6 +2337,12 @@ class _HtmlReader:
         self._budget -= len(content)
         if _html_budget(content):
             self.hidden_text = True
+            if _UNREAD_BY_PATTERN.search(content):
+                self.uncounted = True
+            else:
+                text = _read_by_pattern(content)[0]
+                # A line is a block: its characters, as a text node's.
+                self.hidden_characters += sum(map(_shown_length, text.split("\n")))
             return []
         return _top_nodes(content)
 

{spot_sdk_python-2.0.0b12 → spot_sdk_python-2.1.0b1}/spot_sdk/workflow.py

@@ -15,6 +15,10 @@ ResponseMode = Literal["observe", "enforce"]
 DECISIVE_SIGNAL_PATTERN = r"^[a-z0-9_]+(\.[a-z0-9_]+)*\.([a-z0-9_]+|\*)$"
 """A signal name, or a prefix ending in ``.*``."""
 
+# A workflow id goes into URLs, log lines and the commit messages of the config
+# history: letters, digits, dots, underscores and hyphens only (sdk#26)
+WORKFLOW_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
+
 
 class FailureStrategy(StrEnum):
     """Strategy for handling analyzer failures."""
@@ -163,7 +167,11 @@ class VerdictPolicy(BaseModel):
 class Workflow(BaseModel):
     """Complete workflow configuration for email analysis."""
 
-    id: str = Field(..., description="Workflow ID")
+    id: str = Field(
+        ...,
+        pattern=WORKFLOW_ID_PATTERN,
+        description="Workflow ID: a letter or digit, then up to 127 letters, digits, dots, underscores or hyphens",
+    )
     name: str = Field(..., description="Workflow name")
     version: int = Field(default=1, ge=1, description="Workflow version")
     description: str = Field(default="", description="Workflow description")

spot_sdk_python-2.0.0b12/pyproject.toml

@@ -1,38 +0,0 @@
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
-
-[tool.poetry]
-name = "spot-sdk-python"
-version = "2.0.0b12"
-description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
-authors = ["SPOT Project <spot@sonn.lu>"]
-license = "Apache-2.0"
-homepage = "https://sonn.lu"
-readme = "README.md"
-packages = [
-    {include = "spot_sdk"}
-]
-
-[tool.poetry.dependencies]
-python = ">=3.11,<3.16"
-pydantic = "^2.10"
-pydantic-settings = "^2.0.0"
-httpx2 = ">=2.13.1,<3"
-selectolax = ">=1.0,<1.1"
-idna = ">=3.11,<4"
-fastapi = ">=0.133.0,<1"
-starlette = ">=1.7.0,<2"
-
-[tool.poetry.group.dev.dependencies]
-pytest = "^9.0.3"
-pytest-asyncio = "^1.3.0"
-pytest-cov = "^4.1.0"
-mypy = "^1.5.0"
-
-[tool.ruff]
-target-version = "py311"
-
-[tool.ruff.lint]
-select = ["E", "F", "I", "N", "W"]
-ignore = ["E501"]  # Line too long