spot-sdk-python 2.0.0b3__tar.gz → 2.0.0b4__tar.gz

{spot_sdk_python-2.0.0b3 → spot_sdk_python-2.0.0b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spot-sdk-python
-Version: 2.0.0b3
+Version: 2.0.0b4
 Summary: SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers
 License: Apache-2.0
 Author: SPOT Project
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3.15
 Requires-Dist: fastapi (>=0.115.3,<1)
 Requires-Dist: httpx (>=0.25,<1)
+Requires-Dist: idna (>=3.11,<4)
 Requires-Dist: pydantic (>=2.10,<3.0)
 Requires-Dist: pydantic-settings (>=2.0.0,<3.0.0)
 Requires-Dist: selectolax (>=1.0,<1.1)
@@ -49,7 +50,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b3
+pip install spot-sdk-python==2.0.0b4
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for

{spot_sdk_python-2.0.0b3 → spot_sdk_python-2.0.0b4}/README.md

@@ -26,7 +26,7 @@ plugins share:
 ## Install
 
 ```bash
-pip install spot-sdk-python==2.0.0b3
+pip install spot-sdk-python==2.0.0b4
 ```
 
 Python 3.11 to 3.15. Version 2 implements plugin contract 2, for

{spot_sdk_python-2.0.0b3 → spot_sdk_python-2.0.0b4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "spot-sdk-python"
-version = "2.0.0b3"
+version = "2.0.0b4"
 description = "SPOT plugin SDK: the plugin contract, the plugin runtime, its clients and test helpers"
 authors = ["SPOT Project <spot@sonn.lu>"]
 license = "Apache-2.0"
@@ -20,6 +20,7 @@ pydantic = "^2.10"
 pydantic-settings = "^2.0.0"
 httpx = ">=0.25,<1"
 selectolax = ">=1.0,<1.1"
+idna = ">=3.11,<4"
 fastapi = ">=0.115.3,<1"
 
 [tool.poetry.group.dev.dependencies]

{spot_sdk_python-2.0.0b3 → spot_sdk_python-2.0.0b4}/spot_sdk/__init__.py

@@ -42,7 +42,7 @@ from .workflow import (
 )
 
 if TYPE_CHECKING:
-    from .app import SyncState, create_plugin_app
+    from .app import PluginHealth, SyncState, create_plugin_app
     from .clients import SpotAPIError, SpotClient
     from .knowledge import KnowledgeClient, KnowledgeDocument, chunk_text, content_hash
     from .llm import LLMClient, LLMOutputError, LLMToolCall, LLMToolCalls, LLMToolRound
@@ -50,7 +50,7 @@ if TYPE_CHECKING:
 # The modules that import FastAPI, Starlette or httpx load on first use, so importing
 # `spot_sdk.mime` or the models stays light (PEP 562).
 _LAZY = {
-    **dict.fromkeys(("SyncState", "create_plugin_app"), ".app"),
+    **dict.fromkeys(("PluginHealth", "SyncState", "create_plugin_app"), ".app"),
     **dict.fromkeys(("SpotAPIError", "SpotClient"), ".clients"),
     **dict.fromkeys(
         ("KnowledgeClient", "KnowledgeDocument", "chunk_text", "content_hash"),
@@ -122,6 +122,7 @@ __all__ = [
     "SettingField",
     "SettingType",
     # Plugin runtime
+    "PluginHealth",
     "PluginSettings",
     "SyncState",
     "create_plugin_app",

{spot_sdk_python-2.0.0b3 → spot_sdk_python-2.0.0b4}/spot_sdk/app.py

@@ -3,7 +3,8 @@
 ``create_plugin_app`` builds it from the plugin's manifest, its settings
 class and its handlers:
 
-- ``GET /health`` and ``GET /manifest`` are open;
+- ``GET /health`` and ``GET /manifest`` are open. ``/health`` answers 200
+  with the plugin's state, from its ``health`` check when it has one;
 - ``POST /internal/analyze``, ``POST /internal/respond``, and
   ``POST /internal/sync`` with ``GET /internal/state``, exist for the
   handlers given. Each checks ``Authorization: Bearer <the plugin's own
@@ -17,6 +18,10 @@ block. Either way the clients send the call's ``X-Correlation-ID`` on to SPOT.
 Neither the logs nor the error answers hold the request's content: a
 handler's exception is logged by its type and place only, and an invalid
 request is answered with the fields refused, never the values sent.
+
+The app's OpenAPI document (``app.openapi()``; the app serves none) shows the
+contract as it runs: each route's request body, its status codes, the error
+answers the app gives itself, and the Bearer token on ``/internal/*``.
 """
 
 from __future__ import annotations
@@ -28,11 +33,13 @@ import traceback
 from collections.abc import Awaitable, Callable, Iterator
 from contextlib import contextmanager
 from datetime import UTC, datetime
-from typing import Any, TypeAlias, TypeVar
+from typing import Any, Literal, TypeAlias, TypeVar
 
 from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
 from fastapi.responses import JSONResponse
-from pydantic import BaseModel, Field, SecretStr, ValidationError
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
+from pydantic.json_schema import models_json_schema
 from pydantic_core import PydanticSerializationError
 from starlette.concurrency import run_in_threadpool
 from starlette.types import Lifespan
@@ -56,7 +63,15 @@ AnalyzeHandler: TypeAlias = Callable[
 RespondHandler: TypeAlias = Callable[
     [RespondRequest, SettingsT], None | Awaitable[None]
 ]
-"""``respond(request, settings) -> None``; it acts only when ``request.mode`` is enforce."""
+"""``respond(request, settings) -> None``; it acts only when ``request.mode`` is enforce.
+
+Core calls it at least once per job, not exactly once: it retries a call that
+timed out, which may still complete, since the handler runs inside the
+request; and when it restarts between its calls and its acknowledgement of the
+job's result, it calls every responder of the job again. So a responder acts
+once per ``request.job_id``: it keeps the job ids it has acted on, or makes its
+action idempotent (a ticket keyed by the job id).
+"""
 SyncHandler: TypeAlias = Callable[[SettingsT], int | Awaitable[int]]
 """``sync(settings) -> int``: the number of documents written."""
 
@@ -80,6 +95,65 @@ class SyncState(BaseModel):
     )
 
 
+class PluginHealth(BaseModel):
+    """The plugin's state, as its ``health`` check returns it and ``GET /health``
+    reports it.
+
+    ``/health`` is open to anyone who reaches the plugin: the reason says why
+    in a few words of the plugin's own ("loading the model"), and never holds
+    an email's content, a setting or an exception's message. Core's logs and
+    the dashboard show it, so it holds no control character. An instance is
+    validated again wherever it is read, even one built without validation.
+    """
+
+    model_config = ConfigDict(frozen=True, revalidate_instances="always")
+
+    status: Literal["healthy", "starting", "unhealthy"]
+    reason: str | None = Field(
+        default=None,
+        max_length=200,
+        pattern=r"^[^\x00-\x1f\x7f]*$",
+        description="Why, in a few words; no content, no control character",
+    )
+
+
+class ErrorAnswer(BaseModel):
+    """An error the app answers itself: 401 without the plugin's own token, 500
+    when the handler fails. It never holds the request's content."""
+
+    detail: str = Field(
+        description='"unauthorized", "analysis failed", "response failed"'
+    )
+
+
+class RefusedField(BaseModel):
+    """A field of a request body that does not validate: never the value sent."""
+
+    type: str
+    loc: list[str | int]
+    msg: str
+
+
+class InvalidRequestAnswer(BaseModel):
+    """The 422 for a request body that does not validate: the fields refused."""
+
+    detail: list[RefusedField]
+
+
+HealthCheck: TypeAlias = Callable[[], PluginHealth | Awaitable[PluginHealth]]
+"""``health() -> PluginHealth``, called on each ``GET /health``."""
+_HEALTHY = PluginHealth(status="healthy")
+# The errors a route that reads its body answers itself, by status.
+_BODY_ERRORS: dict[int | str, dict[str, Any]] = {
+    422: {"model": InvalidRequestAnswer, "description": "The body does not validate"},
+    500: {"model": ErrorAnswer, "description": "The handler failed"},
+}
+_SCHEMAS = "#/components/schemas/{model}"
+_HEALTH_CHECK_FAILED = PluginHealth(
+    status="unhealthy", reason="the health check failed"
+)
+
+
 def create_plugin_app(
     manifest: Manifest,
     settings_cls: type[SettingsT],
@@ -88,6 +162,7 @@ def create_plugin_app(
     respond: RespondHandler[SettingsT] | None = None,
     sync: SyncHandler[SettingsT] | None = None,
     lifespan: Lifespan[FastAPI] | None = None,
+    health: HealthCheck | None = None,
 ) -> FastAPI:
     """Build the plugin's app; the settings are read once, here.
 
@@ -98,9 +173,21 @@ def create_plugin_app(
     ``lifespan`` is passed to FastAPI as is: an async context manager run
     around the app's life, to load a model at startup, say.
 
+    ``health``, called on each ``GET /health``, returns the plugin's state:
+    ``starting`` while the model loads, say, or ``unhealthy`` when it did not.
+    Without it, the plugin is ``healthy``. ``/health`` always answers 200, so
+    that liveness checks pass, with ``{"status", "version"}`` and the
+    ``reason`` when there is one. A check that raises, or returns no valid
+    ``PluginHealth``, makes the plugin ``unhealthy`` for a fixed reason, and
+    is logged as the handlers are: never with the exception's message.
+
     ``POST /internal/sync`` answers 202 at once and runs ``sync`` in the
     background. While a sync runs, another call starts nothing and answers
     202 with the running sync's state.
+
+    The app's ``openapi()`` describes the contract. The request bodies are
+    components of that document alone, so read it from this app: another
+    FastAPI app that includes its routes leaves their ``$ref`` dangling.
     """
     _check_kinds(manifest, {"analyze": analyze, "respond": respond, "sync": sync})
     settings = settings_cls()
@@ -110,7 +197,7 @@ def create_plugin_app(
             "mounts it as /run/secrets/spot_plugin_token; outside a container, set "
             "SPOT_PLUGIN_TOKEN."
         )
-    app = FastAPI(
+    app = _PluginApp(
         title=manifest.id,
         version=manifest.version,
         openapi_url=None,
@@ -118,8 +205,12 @@ def create_plugin_app(
     )
 
     @app.get("/health")
-    async def health() -> dict[str, str]:
-        return {"status": "healthy", "version": manifest.version}
+    async def health_route() -> dict[str, str]:
+        state = await _health(health)
+        answer = {"status": state.status, "version": manifest.version}
+        if state.reason is not None:
+            answer["reason"] = state.reason
+        return answer
 
     @app.get("/manifest")
     async def get_manifest() -> Manifest:
@@ -128,11 +219,21 @@ def create_plugin_app(
     internal = APIRouter(
         prefix="/internal",
         dependencies=[Depends(_token_check(settings.spot_plugin_token))],
+        responses={
+            401: {"model": ErrorAnswer, "description": "Without the plugin's own token"}
+        },
     )
 
     if analyze is not None:
 
-        @internal.post("/analyze")
+        @internal.post(
+            "/analyze",
+            openapi_extra=app.reads(AnalyzeRequest),
+            responses={
+                200: {"model": AnalyzerVerdict, "description": "The verdict"},
+                **_BODY_ERRORS,
+            },
+        )
         async def analyze_route(request: Request) -> Response:
             call = await _parse(request, AnalyzeRequest)
             with _correlated(request):
@@ -155,7 +256,13 @@ def create_plugin_app(
 
     if respond is not None:
 
-        @internal.post("/respond")
+        @internal.post(
+            "/respond",
+            status_code=204,
+            response_class=Response,
+            openapi_extra=app.reads(RespondRequest),
+            responses=_BODY_ERRORS,
+        )
         async def respond_route(request: Request) -> Response:
             call = await _parse(request, RespondRequest)
             with _correlated(request):
@@ -182,6 +289,40 @@ def create_plugin_app(
     return app
 
 
+class _PluginApp(FastAPI):
+    """The plugin's app. Its OpenAPI document also holds the schemas of the
+    request bodies its routes read themselves, which FastAPI leaves out."""
+
+    def __init__(self, **options: Any) -> None:
+        super().__init__(**options)
+        self._read_bodies: list[type[BaseModel]] = []
+
+    def reads(self, model: type[BaseModel]) -> dict[str, Any]:
+        """The ``openapi_extra`` of a route that reads ``model`` from its body
+        itself, validated (``_parse``)."""
+        self._read_bodies.append(model)
+        schema = {"$ref": _SCHEMAS.format(model=model.__name__)}
+        return {
+            "requestBody": {
+                "required": True,
+                "content": {"application/json": {"schema": schema}},
+            }
+        }
+
+    def openapi(self) -> dict[str, Any]:
+        if self.openapi_schema is not None:
+            return self.openapi_schema
+        document = super().openapi()  # kept as self.openapi_schema
+        _, definitions = models_json_schema(
+            [(model, "validation") for model in self._read_bodies],
+            ref_template=_SCHEMAS,
+        )
+        schemas = document.setdefault("components", {}).setdefault("schemas", {})
+        for name, schema in definitions.get("$defs", {}).items():
+            schemas.setdefault(name, schema)
+        return document
+
+
 _HANDLER_KINDS = {
     "analyze": PluginKind.ANALYZER,
     "respond": PluginKind.RESPONDER,
@@ -239,11 +380,37 @@ class _SyncRunner:
             self.state.in_progress = False
 
 
-def _token_check(token: SecretStr) -> Callable[[Request], Awaitable[None]]:
+async def _health(check: HealthCheck | None) -> PluginHealth:
+    """The plugin's state: its check's answer, healthy without one, unhealthy
+    when the check fails."""
+    if check is None:
+        return _HEALTHY
+    try:
+        answer = await _run(check)
+    except Exception as error:
+        _log_failure("health check", error)
+        return _HEALTH_CHECK_FAILED
+    try:
+        return PluginHealth.model_validate(answer)
+    except ValidationError as error:
+        _log_invalid_answer("health check", error)
+        return _HEALTH_CHECK_FAILED
+
+
+# The plugin's own token, as Authorization: Bearer <token> carries it; the
+# check below answers 401 itself.
+_BEARER = HTTPBearer(auto_error=False, description="The plugin's own token")
+
+
+def _token_check(
+    token: SecretStr,
+) -> Callable[[HTTPAuthorizationCredentials | None], Awaitable[None]]:
     expected = token.get_secret_value().encode()
 
-    async def check_token(request: Request) -> None:
-        if not _is_bearer(request.headers.get("authorization"), expected):
+    async def check_token(
+        credentials: HTTPAuthorizationCredentials | None = Depends(_BEARER),
+    ) -> None:
+        if not _is_token(credentials, expected):
             raise HTTPException(
                 status_code=401,
                 detail="unauthorized",
@@ -253,12 +420,11 @@ def _token_check(token: SecretStr) -> Callable[[Request], Awaitable[None]]:
     return check_token
 
 
-def _is_bearer(header: str | None, token: bytes) -> bool:
-    """Whether ``header`` is ``Bearer <token>``; an empty token matches nothing."""
-    if not token or header is None:
+def _is_token(credentials: HTTPAuthorizationCredentials | None, token: bytes) -> bool:
+    """Whether the Bearer credentials are ``token``; an empty token matches nothing."""
+    if not token or credentials is None:
         return False
-    scheme, _, presented = header.partition(" ")
-    return scheme.lower() == "bearer" and hmac.compare_digest(presented.encode(), token)
+    return hmac.compare_digest(credentials.credentials.encode(), token)
 
 
 async def _parse(request: Request, model: type[ModelT]) -> ModelT:

{spot_sdk_python-2.0.0b3 → spot_sdk_python-2.0.0b4}/spot_sdk/email.py

@@ -22,7 +22,14 @@ class Address(BaseModel):
 
     display_name: str = ""
     address: str = Field(description="As written; may be invalid, never rejected")
-    domain: str = Field(default="", description="Lower-cased, IDNA-decoded")
+    domain: str = Field(
+        default="",
+        description="As browsers read a host (WHATWG URL Standard): UTS 46-mapped"
+        " (lower-cased...), A-labels decoded, no trailing dots; one that can be"
+        " no DNS name only lower-cased: over 1 012 characters or a label over 252,"
+        " invisible ones aside (DNS allows 253 and 63, and NFC composes at most"
+        " 4 code points into one)",
+    )
 
 
 class Header(BaseModel):
@@ -71,7 +78,15 @@ class Url(BaseModel):
     """A URL found in the message."""
 
     url: str = Field(description="As found")
-    host: str = Field(description='Lower-cased, IDNA-decoded ("" when unparsable)')
+    host: str = Field(
+        description="The host a browser would reach, as the WHATWG URL Standard"
+        " reads it: percent-decoded, UTS 46-mapped (lower-cased...), A-labels"
+        " decoded when UTS 46 leaves them as they are, no trailing dots; one"
+        " browsers refuse is kept as read, one that can be no DNS name only"
+        " lower-cased: over 1 012 characters or a label over 252, invisible ones"
+        " aside (DNS allows 253 and 63, and NFC composes at most 4 code points"
+        ' into one); "" when none'
+    )
     found_in: Literal["text", "html_href", "html_text", "header"] = Field(
         description="html_text: written in the HTML's visible text, not in an attribute"
     )
@@ -142,7 +157,9 @@ class EmailView(BaseModel):
         default=None, description="First text/html body part, decoded"
     )
     visible_text: str = Field(
-        default="", description="text, else the HTML's visible text"
+        default="",
+        description="text, else the HTML's visible text; without the characters"
+        " that render as nothing (spot_sdk.mime.visible)",
     )
     urls: list[Url] = Field(default_factory=list)
     truncated: bool = Field(

{spot_sdk_python-2.0.0b3 → spot_sdk_python-2.0.0b4}/spot_sdk/manifest.py

@@ -10,9 +10,9 @@ from __future__ import annotations
 
 import json
 from enum import StrEnum
-from typing import Any
+from typing import Any, Self
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 
 CONTRACT_VERSION = 2
 
@@ -58,8 +58,27 @@ class SettingType(StrEnum):
     SECRET = "secret"
 
 
+# The values each setting type takes; a bool, which Python counts as an int,
+# is a boolean's only.
+_SETTING_VALUES: dict[SettingType, tuple[type, ...]] = {
+    SettingType.STRING: (str,),
+    SettingType.INTEGER: (int,),
+    SettingType.NUMBER: (int, float),
+    SettingType.BOOLEAN: (bool,),
+    SettingType.URL: (str,),
+    SettingType.SECRET: (str,),
+}
+
+
 class SettingField(BaseModel):
-    """One setting of a plugin, as ``spot.plugin.settings`` declares it."""
+    """One setting of a plugin, as ``spot.plugin.settings`` declares it.
+
+    An ``enum`` that no value can satisfy is refused, so that the plugin
+    fails when its manifest is built rather than when an admin configures it:
+    an empty one, or one whose values do not fit ``type`` (``integer``: int,
+    never bool; ``number``: int or float, never bool; ``boolean``: bool;
+    ``string``, ``url`` and ``secret``: str).
+    """
 
     type: SettingType
     required: bool = False
@@ -67,9 +86,27 @@ class SettingField(BaseModel):
     description: str = ""
     secret: bool = False
     enum: list[Any] | None = Field(
-        default=None, description="Allowed values, on any type"
+        default=None, description="Allowed values, each of the setting's type"
     )
 
+    @model_validator(mode="after")
+    def _check_enum(self) -> Self:
+        if self.enum is None:
+            return self
+        if not self.enum:
+            raise ValueError("the enum is empty: no value can satisfy it")
+        for index, value in enumerate(self.enum):
+            if not _fits(value, self.type):
+                raise ValueError(f"enum[{index}] does not fit the type {self.type}")
+        return self
+
+
+def _fits(value: Any, setting_type: SettingType) -> bool:
+    """Whether ``value`` is a value of ``setting_type``."""
+    if isinstance(value, bool):
+        return setting_type is SettingType.BOOLEAN
+    return isinstance(value, _SETTING_VALUES[setting_type])
+
 
 class Resources(BaseModel):
     """Memory and CPU a plugin container may use."""

{spot_sdk_python-2.0.0b3 → spot_sdk_python-2.0.0b4}/spot_sdk/mime.py

@@ -42,6 +42,7 @@ import hashlib
 import json
 import math
 import re
+import unicodedata
 from array import array
 from collections import Counter, defaultdict
 from collections.abc import Collection, Iterable, Iterator
@@ -53,9 +54,11 @@ from email.utils import decode_params, getaddresses, parsedate_to_datetime, unqu
 from html import unescape as unescape_html
 from itertools import chain
 from typing import Any, Literal, NamedTuple, TypeAlias, TypeVar, cast
+from urllib.parse import unquote as unquote_url
 from urllib.parse import urlsplit
 from uuid import UUID
 
+import idna
 from pydantic import BaseModel
 from pydantic_core import to_json
 from selectolax.lexbor import LexborHTMLParser, LexborNode
@@ -193,6 +196,10 @@ URL_LIMIT = "url_limit"
 """The message has more than MAX_URLS URLs, or they take more than
 MAX_URLS_BYTES: the later ones are not recorded. Or a URL was cut at 8 KiB of
 JSON, or its host or anchor text at 1 KiB."""
+INVISIBLE_CHARACTERS = "invisible_characters"
+"""Characters that render as nothing (see :func:`visible`) were removed from
+visible_text, or from a text part they left blank: noted at that part, with how
+many as the detail."""
 ANOMALY_LIMIT = "anomaly_limit"
 """More than MAX_ANOMALIES anomalies were met: the later ones are not recorded."""
 ANOMALY_CODES = (
@@ -213,6 +220,7 @@ ANOMALY_CODES = (
     HTML_LIMIT,
     HTML_STYLE_UNRESOLVED,
     URL_LIMIT,
+    INVISIBLE_CHARACTERS,
     ANOMALY_LIMIT,
 )
 """Every anomaly code, in the order above."""
@@ -458,6 +466,49 @@ _BLOCK_TAGS = frozenset(
 )
 # A srcset candidate's URL: up to whitespace.
 _SRCSET_URL = re.compile(r"[\t\n\f\r ,]*+([^\t\n\f\r ]++)")
+# The characters that render as nothing: Default_Ignorable_Code_Point, as
+# Unicode 16.0.0 lists it in DerivedCoreProperties.txt
+# (https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt), 4 174
+# code points, adjacent ranges merged. Zero-width space and joiners, the soft
+# hyphen, bidi controls, fillers, variation selectors, tags, and code points
+# reserved for more.
+_DEFAULT_IGNORABLE = (
+    (0x00AD, 0x00AD),
+    (0x034F, 0x034F),
+    (0x061C, 0x061C),
+    (0x115F, 0x1160),
+    (0x17B4, 0x17B5),
+    (0x180B, 0x180F),
+    (0x200B, 0x200F),
+    (0x202A, 0x202E),
+    (0x2060, 0x206F),
+    (0x3164, 0x3164),
+    (0xFE00, 0xFE0F),
+    (0xFEFF, 0xFEFF),
+    (0xFFA0, 0xFFA0),
+    (0xFFF0, 0xFFF8),
+    (0x1BCA0, 0x1BCA3),
+    (0x1D173, 0x1D17A),
+    (0xE0000, 0xE0FFF),
+)
+_INVISIBLE = re.compile(
+    "["
+    + "".join(f"\\U{first:08x}-\\U{last:08x}" for first, last in _DEFAULT_IGNORABLE)
+    + "]+"
+)
+# The label separators: the full stop, and those UTS 46 maps to it.
+_DOT_CHARACTERS = ".\u3002\uff0e\uff61"
+_DOTS = re.compile(f"[{_DOT_CHARACTERS}]")
+# A DNS name holds 253 characters at most, without its trailing dot, and a
+# label 63 (RFC 1035). A host maps to no DNS name when it keeps more than
+# four times as many characters once the invisible ones are removed: UTS 46
+# removes no other, and NFC composes at most 4 code points into one (U+1F8F).
+_MAX_DOMAIN = 4 * 253
+_MAX_LABEL = 4 * 63
+# An A-label is the DNS label itself.
+_MAX_A_LABEL = 63
+# The most characters idna maps a call (from 3.17).
+_IDNA_INPUT = 1024
 
 
 # A parameter as email.utils decodes it: a value, or an RFC 2231
@@ -502,6 +553,12 @@ def parse(
     most that many bytes of their part, and their URLs and the visible text
     from all of what was read, not from the cut the view stores.
 
+    The visible text is the text, or the HTML's visible text when the text is
+    blank, without the characters that render as nothing (:func:`visible`): a
+    text of them alone is blank. Removing any is the anomaly
+    ``invisible_characters``, at the part they were removed from, the visible
+    text's or a blank text's; the other strings keep them, as written.
+
     The view's JSON (``model_dump_json(by_alias=True)``, in UTF-8) takes at
     most ``MAX_VIEW_BYTES``, about 6.84 MiB, whatever the message, with
     ``max_text_bytes`` at most its default. Each list and each long string has
@@ -568,17 +625,21 @@ def parse(
     html = bodies.get("text/html")
     cut_urls = _Cutter()
     found = [_header_urls(fields, cut_urls)]  # the URLs, built as they are kept
+    shown: _Body | None = None  # the body the visible text is read from
+    visible_text = ""
     if text:
         found.append(_urls_in(text.content, "text", text.part_id, cut_urls))
-    shown = text if text and text.content.strip() else None  # the visible text's
-    visible_text = shown.content if shown else ""
+        seen = _visible_body(text, text.content)
+        if seen.strip():
+            shown, visible_text = text, seen
     if html:
         reader = _read_html(html.content)
         found.append(_html_urls(reader.links, html.part_id, cut_urls))
         html_text = reader.text
         found.append(_urls_in(html_text, "html_text", html.part_id, cut_urls))
         if shown is None:
-            shown, visible_text = html, html_text
+            # Removing them may leave runs of spaces and empty lines.
+            shown, visible_text = html, _lines(_visible_body(html, html_text))
         if reader.hidden_text:
             html.part.note(HTML_HIDDEN_TEXT)
         if reader.limit:
@@ -877,6 +938,15 @@ class _Body:
     content: str
 
 
+def _visible_body(body: _Body, text: str) -> str:
+    """``text`` read from a body, without the characters that render as
+    nothing; removing any is noted on the body's part."""
+    seen = visible(text)
+    if len(seen) < len(text):
+        body.part.note(INVISIBLE_CHARACTERS, None, str(len(text) - len(seen)))
+    return seen
+
+
 def _cut_body(body: _Body, content: str, max_bytes: int) -> str:
     """``content`` read from a body, cut to ``max_bytes`` of JSON; a cut is
     noted on the body's part."""
@@ -1392,12 +1462,98 @@ def _unquote(value: str) -> str:
 
 
 def _domain(host: str) -> str:
-    """``host`` lower-cased and IDNA-decoded, or lower-cased when it is not IDNA."""
-    host = host.lower()
+    """``host`` as browsers read it, as the WHATWG URL Standard has them do,
+    without its trailing dots: mapped as UTS 46 maps it (case, width, the
+    characters it ignores...), each A-label decoded (``_decoded``). A label
+    UTS 46 does not map stays as written, lower-cased.
+
+    A host that can be no DNS name (``_can_be_dns_name``) reaches no
+    resolver: it is only lower-cased, neither mapped nor decoded. That bounds
+    the work a host costs: CPython reorders combining marks for NFC in time
+    quadratic in their run, and a run ends at a dot.
+
+    It reads the host, and does not decide whether a browser would accept it:
+    a label against the Bidi rule, or with a joiner out of context, is read
+    all the same."""
+    host = host.rstrip(_DOT_CHARACTERS)
+    if not _can_be_dns_name(host):
+        return host.lower()
     try:
-        return host.encode("ascii").decode("idna")
-    except UnicodeError:
-        return host
+        labels = _remapped(host).split(".")
+    except ValueError:  # a code point UTS 46 does not map: each label alone
+        labels = [_mapped(label) for label in _DOTS.split(host)]
+    while labels and not labels[-1]:
+        labels.pop()
+    return ".".join(map(_decoded, labels))
+
+
+def _can_be_dns_name(host: str) -> bool:
+    """Whether ``host`` may map to a DNS name, counted on what a resolver could
+    see: without its invisible characters (``visible``, which holds all those
+    UTS 46 removes) and its trailing dots, it keeps at most ``_MAX_DOMAIN``
+    characters, and each label at most ``_MAX_LABEL``. UTS 46 maps any other
+    character to one or more, so the count never exceeds the mapped host's
+    before NFC. It only counts: the host is read as UTS 46 maps it."""
+    counted = visible(host).rstrip(_DOT_CHARACTERS)
+    return len(counted) <= _MAX_DOMAIN and all(
+        len(label) <= _MAX_LABEL for label in _DOTS.split(counted)
+    )
+
+
+def _mapped(label: str) -> str:
+    """A label mapped as UTS 46 maps it; as written, lower-cased, when it does
+    not map."""
+    try:
+        return _remapped(label)
+    except ValueError:  # idna.IDNAError, a UnicodeError
+        return label.lower()
+
+
+def _remapped(text: str) -> str:
+    """``text`` mapped as UTS 46 maps it, however long it is written. idna maps
+    at most ``_IDNA_INPUT`` characters a call, and UTS 46 maps each character
+    alone: a longer text is mapped in pieces, then put in NFC whole. A code
+    point UTS 46 does not map raises idna.IDNAError, a ValueError."""
+    if len(text) <= _IDNA_INPUT:
+        return idna.uts46_remap(text, std3_rules=False)
+    pieces = (
+        idna.uts46_remap(text[start : start + _IDNA_INPUT], std3_rules=False)
+        for start in range(0, len(text), _IDNA_INPUT)
+    )
+    return unicodedata.normalize("NFC", "".join(pieces))
+
+
+def _decoded(label: str) -> str:
+    """A mapped label as browsers show it: an A-label decoded from Punycode
+    when what it decodes to is a label UTS 46 leaves as it is (its own
+    criterion: valid code points, in NFC), and not ASCII alone; any other
+    label as it is.
+
+    So an A-label that decodes to what UTS 46 would change stays encoded: a
+    fullwidth or a capital letter, a soft hyphen, a full stop, a code point
+    it does not map. Mapping it would read a dead A-label as another host,
+    a brand's own (``xn--mi7ca7anb7c`` is a fullwidth ``paypal``).
+
+    An A-label over ``_MAX_A_LABEL`` characters is a DNS label no resolver
+    reaches: it stays as it is. That bounds Python's Punycode decoding,
+    quadratic in the label.
+
+    There is no other check: IDNA 2008's CONTEXTO rules rescanned a label for
+    each code point they concern, and UTS 46's joiner checks took 70 us a
+    joiner on idna 3.20, so hostile labels cost seconds a message.
+    """
+    if not label.startswith("xn--") or len(label) > _MAX_A_LABEL:
+        return label
+    try:
+        decoded = label[4:].encode("ascii").decode("punycode")
+        if (
+            not decoded.isascii()
+            and idna.uts46_remap(decoded, std3_rules=False) == decoded
+        ):
+            return decoded
+    except ValueError:  # a UnicodeError: not ASCII, not Punycode, not mapped
+        pass
+    return label
 
 
 def _addresses(value: str, cut: _Cutter) -> Iterator[Address]:
@@ -1631,10 +1787,39 @@ def _unclosed_parentheses(value: str) -> bytearray:
 # ----------------------------------------------------------------------- #
 
 
+def visible(text: str) -> str:
+    """``text`` without the characters that render as nothing: Unicode 16.0's
+    Default_Ignorable_Code_Point (zero-width space and joiners, the soft
+    hyphen, bidi controls, variation selectors, tags...), which split words a
+    reader sees whole. ``parse`` builds ``visible_text`` with it; the other
+    strings of the view keep them, as written (the subject, the headers,
+    display names, anchor text, the text and the HTML)."""
+    return _INVISIBLE.sub("", text)
+
+
+def _lines(text: str) -> str:
+    """``text`` with its whitespace collapsed, one line per line that is not blank."""
+    lines = (" ".join(line.split()) for line in text.split("\n"))
+    return "\n".join(line for line in lines if line)
+
+
 def url_host(text: str) -> str:
-    """The host a browser would reach for ``text``, lower-cased and IDNA-decoded;
-    "" when there is none. A backslash reads as a slash, and ``www.`` is a host
-    without a scheme: the host ``parse`` records in ``Url.host``."""
+    """The host a browser would reach for ``text``; "" when there is none: the
+    host ``parse`` records in ``Url.host``. A backslash reads as a slash, and
+    ``www.`` is a host without a scheme.
+
+    As the WHATWG URL Standard has browsers do, the host is percent-decoded,
+    as UTF-8 (a NUL becomes U+FFFD, as in the rest of the view), then mapped
+    as UTS 46 maps it (lower-cased, among others), and each A-label decoded
+    when it decodes to a label UTS 46 leaves as it is; its trailing dots are
+    dropped. A label UTS 46 does not map stays as written, lower-cased, a
+    host browsers refuse (a space, a slash, a label against the Bidi rule) is
+    kept as read, and a host that can be no DNS name is only lower-cased:
+    never an exception. That is a host that keeps over 1 012 characters, or a
+    label over 252, once its invisible characters are removed: a DNS name
+    holds 253, a label 63, and NFC composes at most 4 code points into one.
+    An A-label over 63 characters stays encoded.
+    """
     if text[:4].lower() == "www.":
         text = "http://" + text
     try:
@@ -1642,7 +1827,7 @@ def url_host(text: str) -> str:
         host = urlsplit(text.replace("\\", "/")).hostname
     except ValueError:
         return ""
-    return _domain(host) if host else ""
+    return _domain(_clean(unquote_url(host))) if host else ""
 
 
 def _url(
@@ -1775,8 +1960,7 @@ class _HtmlReader:
     @property
     def text(self) -> str:
         """The visible text: one line per block, whitespace collapsed."""
-        lines = (" ".join(line.split()) for line in "".join(self._chunks).split("\n"))
-        return "\n".join(line for line in lines if line)
+        return _lines("".join(self._chunks))
 
     def _read(self, nodes: list[LexborNode]) -> None:
         """Walk the tree depth-first, without recursion: the tree may be deep."""

{spot_sdk_python-2.0.0b3 → spot_sdk_python-2.0.0b4}/spot_sdk/testing/views.py

@@ -10,7 +10,7 @@ from typing import Any
 from uuid import uuid4
 
 from ..email import EmailView
-from ..mime import parse
+from ..mime import parse, visible
 
 
 def view_from_eml(eml: bytes | str | os.PathLike[str], **meta: Any) -> EmailView:
@@ -32,7 +32,7 @@ def make_view(**fields: Any) -> EmailView:
 
     The defaults are an empty message: a new id, received now from
     ``"test"``, no headers. As the parser does, ``visible_text`` is the
-    ``text`` when only ``text`` is given.
+    ``text`` without its invisible characters when only ``text`` is given.
     """
     defaults: dict[str, Any] = {
         "id": uuid4(),
@@ -43,5 +43,5 @@ def make_view(**fields: Any) -> EmailView:
         "headers": [],
     }
     if fields.get("text"):
-        defaults["visible_text"] = fields["text"]
+        defaults["visible_text"] = visible(fields["text"])
     return EmailView.model_validate({**defaults, **fields})