cloakapi 0.5.1__py3-none-any.whl

cloakapi/_request_strings.py

@@ -0,0 +1,1267 @@
+"""
+The request-body CHANNEL LIST for the Python SDK: which strings of a relay request
+carry customer text, walked by ONE module that the sync client, the async client, the
+messages resource, the chat resource, ``create`` and ``stream`` all share.
+
+WHY THIS FILE EXISTS
+--------------------
+Until this module, ``_prepare_messages`` tokenised ``messages[].content`` when it was a
+``str`` and nothing else, and ``MessagesResource.create`` bolted a second, separate
+tokenise call onto ``system=<str>``. Everything else a caller can put text in went out
+as typed, in ``create`` AND ``stream``: the top-level ``metadata``, a message's own
+``metadata``, ``tool_calls[].function.arguments``, ``tool_use.input``, the ``text`` of a
+list-shaped ``content`` (Anthropic text blocks, OpenAI text parts), ``tool_result``
+content (string or nested text blocks) and a ``system`` given as text blocks. Measured on
+the wire (loopback capture): 44 of 56 probes carried the raw e-mail
+and IBAN. Commit ``cdefce5b7`` told customers every SDK tokenised those channels.
+
+Two copies of one rule drift, so the walk lives here once and takes the per-string
+transform as an argument (``tok``). The client binds ``tok`` to its conversation ledger
+(:func:`bind_tokeniser`), so every string in a channel below shares one token map and one
+set of stand-ins: the same e-mail in ``metadata`` and in a message becomes the same token.
+
+CONTRACT OF EVERY FUNCTION BELOW
+--------------------------------
+* Only string TEXT is handed to ``tok``. Identifiers and structure (``role``, ``type``,
+  ``id``, ``name``, ``tool_use_id``, ``tool_call_id``, image/document ``source``,
+  ``cache_control``, ``signature``) are never touched, and dict KEYS are never touched.
+* Copy on change: the caller's objects are never mutated, and a value in which no string
+  changed is returned as the SAME object, so PII-free input crosses the wire exactly as
+  supplied.
+* Unknown or odd shapes (``None``, numbers, non-dict blocks) pass through unchanged
+  instead of raising.
+* Nothing here raises because of what a string CONTAINS. The only refusals are for a
+  body that does not have the shape of a request (a message or block that is not an
+  object, ``attachments`` that is not a list).
+
+WHERE A DETECTED VALUE SITS (one rule for every CloakAPI SDK)
+-------------------------------------------------------------
+* Text the model reads (message text, system, tool results, tool-call argument VALUES,
+  descriptions, metadata values, attachment text and names): replaced when the
+  engine or the structured rules can replace it. A ``thinking`` block and a
+  ``redacted_thinking`` block stay byte for byte as written, with or without a
+  ``signature``: the text, the signature and ``data`` are not replaced, so a
+  provider signature still covers the bytes it signed. The rules still run over
+  that text. Personal data they see is sent as written and carries
+  ``unchecked_content`` / ``value_not_replaced``. Thai and Khmer text still
+  carries ``unchecked_content`` / ``unverified_script``. The send is not stopped.
+* An object KEY in free-form data that is not request or message ``metadata``
+  is replaced when the structured rules match it (:func:`settle_structure`).
+  A stand-in that would equal a sibling key is not applied.
+  Metadata keys are never tokenised (owner decision F-068, 2026-11-14). The
+  value under that key is still tokenised. The send is not stopped.
+* Structure that must stay as it is: a tool name and a ``tool_choice`` name (a
+  provider rejects a renamed tool), a key inside tool-call arguments, a tool schema's
+  ``enum``/``const``/``default`` literal or property name, an identifier (``id``,
+  ``tool_use_id``, ``tool_call_id``, ``type``, ``role``, ``model``), a JSON number.
+  Left as written. A value a substantive rule matches carries
+  ``unchecked_content`` / ``value_not_replaced``. Bare-word and bare-date matches
+  on a tool name, a ``tool_choice`` name or a schema ``enum``/``const``/``default``
+  do not: they are not personal data in those places and do not degrade the request.
+* Inline media and media references (image, audio or file bytes, ``data:`` URLs, remote
+  URLs, file ids): this path cannot read them. Sent as provided; the request carries
+  ``unchecked_content`` / ``media_not_checked``.
+
+The notices are recorded on the request's own state (``tok.note_unreplaced``), reach the
+review step before anything is sent, and come back on the reply.
+
+NUMERIC IDENTIFIER REVIEW
+-------------------------
+Numbers are not text. Long numeric leaves and exponent-form numbers in metadata,
+tool input, and JSON tool arguments keep their JSON type and are sent with an
+``unchecked_content`` notice. Leaving a number as written does not lower coverage
+to ``rules_only``: coverage stays what the detector reported.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+import math
+import re
+import threading
+from collections.abc import Callable
+from typing import Any
+
+from .coverage import Coverage
+from .errors import RequestBodyRejectedError
+from .pii.tokenizer import detokenize, tokenize_text_ex
+
+Tokenise = Callable[[str], str]
+#: For a caller that binds a tokeniser to something that is not a client.
+_SHARED_LOCK = threading.RLock()
+_STRUCTURAL_STRING_KEYS = {"type", "role", "model", "id", "enum"}
+#: Strings that must round-trip byte for byte. Never rewritten; a detected value
+#: in one of them is reported, not replaced.
+_IDENTIFIER_KEYS = {"id", "tool_use_id", "tool_call_id", "type", "role", "model", "signature"}
+#: Leaves of a media object that hold bytes or a reference, not text.
+_OPAQUE_MEDIA_KEYS = {"data", "url", "file_data", "content_b64", "file_id"}
+_SCHEMA_LITERAL_KEYS = ("enum", "const", "default")
+#: Bare codes and bare dates. They match ordinary words (``VALIDATE``, ``PLATFORM``)
+#: and ISO dates used as structure. They do not apply to tool names, ``tool_choice``
+#: names, schema literals or metadata keys.
+_BARE_STRUCTURAL_TYPES = frozenset({"FIN_BIC_SWIFT", "DATE_OF_BIRTH"})
+_TOOL_NAME_BLOCK_TYPES = frozenset({"tool_use", "server_tool_use", "mcp_tool_use", "function_call"})
+#: A provider's own encrypted payload inside a block it returned earlier. Not text.
+_PROVIDER_OPAQUE_KEYS = {"encrypted_content", "encrypted_index"}
+
+# Kinds of place where a value is left as written (see the module docstring).
+PLACE_OBJECT_KEY = "object_key"
+PLACE_ARGUMENT_KEY = "tool_argument_key"
+PLACE_TOOL_SCHEMA = "tool_schema"
+PLACE_IDENTIFIER = "identifier"
+PLACE_NUMBER = "number"
+PLACE_MEDIA = "media"
+PLACE_UNKNOWN_BLOCK = "unknown_block"
+PLACE_AS_WRITTEN = "as_written"
+#: Content block types this SDK names. Another type is walked like other blocks
+#: (every string leaf outside protocol fields is tokenised) and reported.
+_KNOWN_BLOCK_TYPES = frozenset({
+    "text", "image", "document", "tool_use", "tool_result", "server_tool_use", "mcp_tool_use",
+    "thinking", "redacted_thinking", "input_text", "input_image", "input_audio", "input_file",
+    "output_text", "refusal", "image_url", "file", "audio", "output_audio", "tool_call", "function_call",
+})
+_JSON_NUMBER = re.compile(r"(?<![\w.])-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?(?![\w.])")
+
+
+def _has_sensitive_number(value: Any) -> bool:
+    """Flag numeric leaves that cannot be tokenised without changing JSON type."""
+    if isinstance(value, bool) or value is None:
+        return False
+    if isinstance(value, int):
+        return len(str(abs(value))) > 5
+    return isinstance(value, float) and not _gateway_accepts_float(value)
+
+
+def _gateway_accepts_float(value: float) -> bool:
+    """Mirror PrivacyTokenisationMiddleware's `%.15G` numeric primitive check."""
+    if not math.isfinite(value) or abs(value) >= 100_000.0:
+        return False
+    rendered = format(abs(value), ".15G")
+    if "E" in rendered:
+        return False
+    digits = rendered.replace(".", "").lstrip("0")
+    return len(digits) <= 5
+
+
+def _has_sensitive_numbers(value: Any) -> bool:
+    """Check whether number leaves need a reduced-coverage review."""
+    if isinstance(value, dict):
+        return any(_has_sensitive_numbers(item) for item in value.values())
+    if isinstance(value, (list, tuple)):
+        return any(_has_sensitive_numbers(item) for item in value)
+    return _has_sensitive_number(value)
+
+
+def _has_structured_pii(text: str) -> bool:
+    """Check opaque JSON labels with the local structured detector, without rewriting them."""
+    from .pii.patterns import detect_pii
+
+    return bool(detect_pii(text))
+
+
+def _substantive_pii(text: str) -> bool:
+    """A match other than a bare word (BIC/SWIFT) or a bare date.
+
+    Those two rule types fire on tool names, schema enums and metadata keys that
+    are not personal data (``VALIDATE``, ``PLATFORM``, ``2024-01-01``). They do
+    not count in those places. Every other rule type still does.
+    """
+    from .pii.patterns import detect_pii
+
+    return any(match.pii_type not in _BARE_STRUCTURAL_TYPES for match in detect_pii(text))
+
+
+def _note_tool_name(value: str, tok: Tokenise) -> None:
+    """A tool name is never rewritten. A substantive match is a notice."""
+    if _substantive_pii(value):
+        _note(tok, PLACE_IDENTIFIER)
+
+
+def _has_sensitive_tool_argument_number(value: Any) -> bool:
+    """Flag OpenAI argument numbers that cannot be rewritten without changing type."""
+    if isinstance(value, bool) or value is None:
+        return False
+    if isinstance(value, int):
+        digits = len(str(abs(value)))
+        return 6 <= digits <= 16
+    elif isinstance(value, float):
+        return not _gateway_accepts_float(value)
+    elif isinstance(value, dict):
+        return any(_has_sensitive_tool_argument_number(item) for item in value.values())
+    elif isinstance(value, (list, tuple)):
+        return any(_has_sensitive_tool_argument_number(item) for item in value)
+    return False
+
+
+def _has_sensitive_json_number(text: str) -> bool:
+    """Flag risky JSON number tokens before parsing loses their original notation."""
+    # Ignore quoted strings and keys: digit-heavy user text remains tokenisable text.
+    strings_masked = re.sub(r'"(?:[^"\\]|\\.)*"', lambda match: " " * len(match.group(0)), text)
+    for match in _JSON_NUMBER.finditer(strings_masked):
+        literal = match.group(0)
+        if "e" in literal.lower():
+            return True
+        if "." not in literal and 6 <= len(literal.lstrip("-")) <= 16:
+            return True
+        if "." in literal:
+            number = json.loads(literal)
+            if isinstance(number, float) and not _gateway_accepts_float(number):
+                return True
+    return False
+
+
+def _mark_reduced_coverage(tok: Tokenise) -> None:
+    mark = getattr(tok, "mark_reduced_coverage", None)
+    if callable(mark):
+        mark()
+
+
+def _note(tok: Tokenise, place: str) -> None:
+    """Record on the request that a value in this kind of place is sent as written."""
+    note = getattr(tok, "note_unreplaced", None)
+    if callable(note):
+        note(place)
+
+
+def _is_inline_payload(text: str) -> bool:
+    """A ``data:`` URL: bytes, not text. Never handed to the detector."""
+    return text.startswith("data:") and ";base64," in text[:256]
+
+
+def tokenise_unclassified(value: Any, tok: Tokenise) -> Any:
+    """Tokenise every string in an unclassified JSON field, retaining protocol labels."""
+    if isinstance(value, str):
+        done = tok(value)
+        return value if done == value else done
+    if isinstance(value, dict):
+        changed = False
+        result: dict[Any, Any] = {}
+        for key, item in value.items():
+            # A structural string is never rewritten. A detected value in one is
+            # reported on the request by settle_structure(); it does not stop the send.
+            done = (
+                item
+                if key in _STRUCTURAL_STRING_KEYS or key in _PROVIDER_OPAQUE_KEYS
+                else tokenise_unclassified(item, tok)
+            )
+            result[key] = done
+            changed = changed or done is not item
+        return result if changed else value
+    if isinstance(value, list):
+        items = [tokenise_unclassified(item, tok) for item in value]
+        return items if any(a is not b for a, b in zip(items, value, strict=True)) else value
+    if isinstance(value, tuple):
+        items = [tokenise_unclassified(item, tok) for item in value]
+        return items if any(a is not b for a, b in zip(items, value, strict=True)) else value
+    return value
+
+
+def bind_tokeniser(client: Any, state: dict[str, Any] | None = None) -> Tokenise:
+    """A ``str -> str`` transform bound to ``client``'s conversation ledger.
+
+    Engine-first (shared FFI engine: shared rules + checksums + NER), pure-Python rules fallback
+    when the library is unavailable. The ledger is passed IN and the advanced ledger stored
+    back on every call, so a later string (or turn) reuses an earlier stand-in.
+    """
+
+    call_state = state if state is not None else {
+        "token_map": client._token_map,
+        "bindings": client._bindings,
+        "source_texts": [],
+        "processed_texts": [],
+    }
+
+    def tok(text: str) -> str:
+        if not text:
+            # The engine refuses an empty input ("file is empty"), and there is nothing to
+            # tokenise: an empty string is a value, not a document. Walking `metadata`, tool
+            # arguments and content blocks reaches empty strings far more often than message
+            # `content` ever did, and one of them must not fail the whole request. (The Ruby
+            # and PHP clients already return an empty string unchanged.)
+            return text
+        if _is_inline_payload(text):
+            # Bytes written as text (an image as a data: URL). Not text the
+            # detector can read, and rewriting inside it would corrupt it.
+            note_unreplaced(PLACE_MEDIA)
+            return text
+        # One string at a time per client: read the conversation ledger, run
+        # detection, write the advanced ledger back. Two threads sharing a
+        # client used to interleave here; the one that finished last wrote a
+        # ledger that had never seen the other's values, so a value could lose
+        # its stand-in and get another one later. Detection is untouched; the
+        # calls are put in order.
+        with getattr(client, "_tokenise_lock", None) or _SHARED_LOCK:
+            return _tokenise_one(text)
+
+    def _tokenise_one(text: str) -> str:
+        source_texts = call_state.setdefault("source_texts", [])
+        processed_texts = call_state.setdefault("processed_texts", [])
+        # `_prepare_body_strings` walks messages after `_prepare_messages` has
+        # already tokenised them. Those exact output objects are not caller text;
+        # avoid treating their generated markers as literal collisions.
+        if not any(text is processed for processed in processed_texts):
+            source_texts.append(text)
+        res = tokenize_text_ex(
+            text,
+            call_state["token_map"],
+            seed_hex=client.seed_hex,
+            mode=client.surrogate_mode,
+            bindings=call_state["bindings"],
+        )
+        call_state["token_map"].clear()
+        call_state["token_map"].update(res.token_map)
+        # A caller may have supplied the same marker literally in this field or
+        # an earlier field. Keep it literal by excluding every colliding key
+        # from this request's local restore map; never reject the request.
+        for marker in tuple(call_state["token_map"]):
+            if any(marker and marker in source for source in source_texts):
+                del call_state["token_map"][marker]
+        processed_texts.append(res.text)
+        call_state["bindings"].clear()
+        call_state["bindings"].update(res.bindings)
+        client._bindings_revision = getattr(client, "_bindings_revision", 0) + 1
+        client._last_surrogate_mode = res.surrogate_mode
+        prior = call_state.get("coverage")
+        result_coverage = getattr(res, "coverage", Coverage.FULL)
+        call_state["coverage"] = Coverage.least(prior, result_coverage) if prior else result_coverage
+        client._last_coverage = call_state["coverage"]
+        # What the detector itself reported, apart from values left as written.
+        seen = call_state.get("detector_coverage")
+        call_state["detector_coverage"] = (
+            Coverage.least(seen, result_coverage) if seen else result_coverage
+        )
+        # Kept with THIS request (the client attributes above are "the latest
+        # request" and are overwritten by an overlapping call): the style that
+        # was written, and whether the structured rules prepared any string.
+        call_state["surrogate_mode"] = res.surrogate_mode
+        if getattr(res, "engine", None) == "regex":
+            call_state["rules_prepared"] = True
+        return res.text
+
+    def mark_reduced_coverage() -> None:
+        # A number left as written is reported. It does not lower coverage:
+        # ``rules_only`` means the name layer did not run, which a number does not decide.
+        client._last_value_not_replaced = True
+        call_state["value_not_replaced"] = True
+        note_unreplaced(PLACE_NUMBER)
+
+    def note_unreplaced(place: str) -> None:
+        places = call_state.setdefault("unreplaced_places", [])
+        if place not in places:
+            places.append(place)
+
+    def would_change(texts: list[str]) -> bool:
+        """Whether the detector would replace anything in any of ``texts``. Changes
+        nothing: it runs on a copy of the request's state (one copy for the whole
+        list), so a value that is going to stay as written gets no stand-in."""
+        texts = [text for text in texts if text and not _is_inline_payload(text)]
+        if not texts:
+            return False
+        with getattr(client, "_tokenise_lock", None) or _SHARED_LOCK:
+            token_map = dict(call_state["token_map"])
+            bindings = copy.deepcopy(call_state["bindings"])
+            for text in texts:
+                res = tokenize_text_ex(
+                    text,
+                    token_map,
+                    seed_hex=client.seed_hex,
+                    mode=client.surrogate_mode,
+                    bindings=bindings,
+                )
+                if res.text != text:
+                    return True
+        return False
+
+    tok.mark_reduced_coverage = mark_reduced_coverage  # type: ignore[attr-defined]
+    tok.note_unreplaced = note_unreplaced  # type: ignore[attr-defined]
+    tok.would_change = would_change  # type: ignore[attr-defined]
+    tok.state = call_state  # type: ignore[attr-defined]
+    return tok
+
+
+# ── generic JSON-shaped values ───────────────────────────────────────────────
+
+
+def tokenise_leaves(value: Any, tok: Tokenise) -> Any:
+    """Tokenise every string leaf of an arbitrary JSON-shaped value (dict values, list items).
+
+    Non-strings (numbers, booleans, ``None``) and dict keys are left as supplied.
+    """
+    if isinstance(value, str):
+        out = tok(value)
+        return value if out == value else out
+    if isinstance(value, dict):
+        changed = False
+        new: dict[Any, Any] = {}
+        for key, item in value.items():
+            done = tokenise_leaves(item, tok)
+            changed = changed or done is not item
+            new[key] = done
+        return new if changed else value
+    if isinstance(value, (list, tuple)):
+        items = [tokenise_leaves(item, tok) for item in value]
+        if any(done is not item for done, item in zip(items, value, strict=True)):
+            return items
+        return value
+    return value
+
+
+# A JSON string literal. In VALID JSON every `"` outside a literal opens one, so scanning left to
+# right finds the literals unambiguously; a literal followed by `:` is an object KEY.
+_JSON_STRING = re.compile(r'"(?:[^"\\]|\\.)*"', re.DOTALL)
+_KEY_COLON = re.compile(r"\s*:")
+
+
+def _splice_json_strings(text: str, tok: Tokenise) -> str:
+    """Tokenise the string VALUES of a valid JSON text IN PLACE, leaving every other byte as is.
+
+    Each value literal is decoded, tokenised, and (only when it changed) written back as a fresh
+    literal. Whitespace, key order, numbers (``12345678901234567890``, ``1.0``, ``1e2``), ``{}``
+    against ``[]`` and object KEYS are the caller's own bytes. Decoding a literal first is what
+    lets the detector see ``Bj\\u00f8rn`` as ``Bjørn``.
+    """
+
+    def repl(match: re.Match[str]) -> str:
+        if _KEY_COLON.match(text, match.end()):
+            return match.group(0)
+        literal = match.group(0)
+        value = json.loads(literal)
+        out = tok(value)
+        return literal if out == value else json.dumps(out, ensure_ascii=False)
+
+    return _JSON_STRING.sub(repl, text)
+
+
+def tokenise_arguments(arguments: Any, tok: Tokenise) -> Any:
+    """OpenAI ``tool_calls[].function.arguments``: a JSON STRING (or, for some callers, an object).
+
+    A JSON string has its string VALUES tokenised in place (:func:`_splice_json_strings`); when
+    none changed, the caller's own string comes back untouched. Malformed JSON is tokenised as
+    plain text; valid JSON beyond the decoder's recursion limit uses the key-preserving literal
+    walker. An object/list is walked leaf by leaf.
+    """
+    if isinstance(arguments, str):
+        try:
+            json.loads(arguments)
+        except RecursionError:
+            # A valid but deeply nested JSON value can exceed the decoder's
+            # recursion limit. The literal scanner is iterative and still
+            # distinguishes keys from string values, so do not text-tokenise
+            # the whole document and rewrite object keys.
+            return _splice_json_strings(arguments, tok)
+        except ValueError:
+            return tok(arguments)
+        return _splice_json_strings(arguments, tok)
+    if isinstance(arguments, int) and not isinstance(arguments, bool):
+        # A root number stays a number (owner decision 2026-11-13). The caller
+        # already recorded value_not_replaced when the digits look like personal
+        # data. Replacing it would change the JSON type.
+        return arguments
+    return tokenise_leaves(arguments, tok)
+
+
+# ── message content and blocks ───────────────────────────────────────────────
+
+
+def _tokenise_block(block: Any, tok: Tokenise) -> Any:
+    """One content block (Anthropic block, OpenAI content part). Text-bearing keys only."""
+    if not isinstance(block, dict):
+        raise RequestBodyRejectedError()
+    block_type = block.get("type")
+    if block_type == "redacted_thinking":
+        # Provider blob. Not text this pass can rewrite. A value the rules still
+        # match is sent as written and named; the request is not stopped.
+        data = block.get("data")
+        if isinstance(data, str) and _substantive_pii(data):
+            _note(tok, PLACE_AS_WRITTEN)
+        return block
+    if block_type == "thinking":
+        # THINKING-AS-WRITTEN. The provider signs these bytes. Replacing the
+        # text and leaving ``signature`` makes the two disagree, so the block
+        # is returned as the caller supplied it, signature or not. The rules
+        # still run over the thinking text. A substantive match is sent as
+        # written and named. A reply does not restore thinking.
+        thinking = block.get("thinking")
+        if isinstance(thinking, str) and _substantive_pii(thinking):
+            _note(tok, PLACE_AS_WRITTEN)
+        return block
+    if isinstance(block_type, str) and block_type not in _KNOWN_BLOCK_TYPES:
+        _note(tok, PLACE_UNKNOWN_BLOCK)
+    # A block type this SDK does not know (a provider adds them: a file part, a
+    # server tool call and its result, a search result, a refusal) is a valid
+    # request, not a malformed one. It is walked like every other block: its
+    # text-bearing strings are tokenised, bytes and references are left as
+    # provided and reported as media. Refusing it stopped a caller from
+    # continuing a conversation that holds such a block.
+    new: dict[str, Any] | None = None
+
+    def put(key: str, value: Any) -> None:
+        nonlocal new
+        if new is None:
+            new = dict(block)
+        new[key] = value
+
+    text = block.get("text")
+    if isinstance(text, str):
+        done = tok(text)
+        if done != text:
+            put("text", done)
+    name = block.get("name")
+    if isinstance(name, str):
+        if block.get("type") in _TOOL_NAME_BLOCK_TYPES:
+            _note_tool_name(name, tok)
+        else:
+            done = tok(name)
+            if done != name:
+                put("name", done)
+    for key in ("function_call", "audio", "input_audio", "image_url", "file"):
+        if key in block:
+            value = block[key]
+            done = _tokenise_text_leaves_of_media(value, tok)
+            if done is not value:
+                put(key, done)
+    # A document whose source is text (not bytes) is text for the model.
+    source = block.get("source")
+    if isinstance(source, dict):
+        if source.get("type") == "text" and isinstance(source.get("data"), str):
+            done = tok(source["data"])
+            if done != source["data"]:
+                put("source", {**source, "data": done})
+        elif source.get("type") == "content" and "content" in source:
+            done = tokenise_content(source["content"], tok)
+            if done is not source["content"]:
+                put("source", {**source, "content": done})
+    elif isinstance(source, str) and source:
+        # A string source is the citation source of a search result: text, not media.
+        done = tok(source)
+        if done != source:
+            put("source", done)
+    # Anthropic tool_use / server_tool_use / mcp_tool_use: the arguments are an OBJECT in
+    # `input`. Keyed on the presence of a container `input`, not on `type`, so block types
+    # Anthropic adds next are covered too.
+    tool_input = block.get("input")
+    if isinstance(tool_input, (dict, list, tuple)):
+        if _has_sensitive_numbers(tool_input):
+            _mark_reduced_coverage(tok)
+        done = tokenise_leaves(tool_input, tok)
+        if done is not tool_input:
+            put("input", done)
+    # Anthropic tool_result nests its own `content` (a string or a list of blocks).
+    if "content" in block:
+        nested = block["content"]
+        done = tokenise_content(nested, tok)
+        if done is not nested:
+            put("content", done)
+    result = block if new is None else new
+    classified = {"type", "text", "name", "function_call", "audio", "input_audio", "image_url", "file", "thinking", "input", "content", "source", "cache_control", "signature", "id", "tool_use_id", "tool_call_id"}
+    for key, value in block.items():
+        if key not in classified and key not in _PROVIDER_OPAQUE_KEYS:
+            done = tokenise_unclassified(value, tok)
+            if done is not value:
+                if result is block:
+                    result = dict(block)
+                result[key] = done
+    return result
+
+
+def tokenise_content(content: Any, tok: Tokenise) -> Any:
+    """Message ``content``: text, or list/tuple content blocks (recursive).
+
+    ``json.dumps`` serialises tuples as arrays too, so walk them just like lists.
+    """
+    if isinstance(content, str):
+        out = tok(content)
+        return content if out == content else out
+    if isinstance(content, dict):
+        return _tokenise_block(content, tok)
+    if isinstance(content, (list, tuple)):
+        blocks = [_tokenise_block(block, tok) for block in content]
+        if any(done is not block for done, block in zip(blocks, content, strict=True)):
+            return blocks
+    return content
+
+
+def tokenise_message(message: Any, tok: Tokenise) -> Any:
+    """Tokenise every free-text slot on an OpenAI/Anthropic message."""
+    if not isinstance(message, dict):
+        raise RequestBodyRejectedError()
+    new: dict[str, Any] | None = None
+
+    def put(key: str, value: Any) -> None:
+        nonlocal new
+        if new is None:
+            new = dict(message)
+        new[key] = value
+
+    if "content" in message:
+        content = message["content"]
+        done = tokenise_content(content, tok)
+        if done is not content:
+            put("content", done)
+    if "metadata" in message:
+        meta = message["metadata"]
+        if _has_sensitive_numbers(meta):
+            _mark_reduced_coverage(tok)
+        done = tokenise_leaves(meta, tok)
+        if done is not meta:
+            put("metadata", done)
+    # These fields are in the TypeScript request-body slot list.  They can carry
+    # user supplied prose even though they are usually treated as identifiers.
+    for key in ("name", "function_call", "audio", "input_audio"):
+        if key not in message:
+            continue
+        value = message[key]
+        if key == "function_call":
+            done = _tokenise_legacy_function_call(value, tok)
+        elif key in ("audio", "input_audio") or key == "attachments":
+            done = _tokenise_text_leaves_of_media(value, tok)
+        else:
+            done = tok(value) if isinstance(value, str) else value
+        if done is not value:
+            put(key, done)
+    if "attachments" in message:
+        done = tokenise_attachments(message["attachments"], tok)
+        if done is not message["attachments"]:
+            put("attachments", done)
+    calls = message.get("tool_calls")
+    if isinstance(calls, (list, tuple)):
+        new_calls = [_tokenise_tool_call(call, tok) for call in calls]
+        if any(done is not call for done, call in zip(new_calls, calls, strict=True)):
+            put("tool_calls", new_calls)
+    classified = {"role", "content", "metadata", "name", "function_call", "audio", "input_audio", "attachments", "tool_calls", "tool_call_id", "tool_use_id", "id", "type"}
+    for key, value in message.items():
+        if key not in classified:
+            done = tokenise_unclassified(value, tok)
+            if done is not value:
+                put(key, done)
+    return message if new is None else new
+
+
+def _tokenise_tool_call(call: Any, tok: Tokenise) -> Any:
+    """OpenAI tool call: keep the function name as written (a match is a notice), tokenise the arguments."""
+    if not isinstance(call, dict):
+        raise RequestBodyRejectedError()
+    fn = call.get("function")
+    if not isinstance(fn, dict):
+        raise RequestBodyRejectedError()
+    new_fn = dict(fn)
+    changed = False
+    for key in ("name", "description", "arguments"):
+        if key not in fn:
+            continue
+        value = fn[key]
+        if key == "arguments":
+            if isinstance(value, str):
+                try:
+                    json.loads(value)
+                except (ValueError, RecursionError):
+                    pass
+                else:
+                    if _has_sensitive_json_number(value):
+                        _mark_reduced_coverage(tok)
+            else:
+                if _has_sensitive_tool_argument_number(value):
+                    _mark_reduced_coverage(tok)
+            done = tokenise_arguments(value, tok)
+        elif key == "description":
+            done = tokenise_unclassified(value, tok)
+        elif isinstance(value, str):
+            _note_tool_name(value, tok)
+            done = value
+        else:
+            done = value
+        if done is not value:
+            new_fn[key] = done
+            changed = True
+    for key, value in fn.items():
+        if key not in {"name", "description", "arguments"}:
+            done = tokenise_unclassified(value, tok)
+            if done is not value:
+                new_fn[key] = done
+                changed = True
+    new_call = call
+    for key, value in call.items():
+        if key not in {"function", "id", "type"}:
+            done = tokenise_unclassified(value, tok)
+            if done is not value:
+                if new_call is call:
+                    new_call = dict(call)
+                new_call[key] = done
+                changed = True
+    if not changed:
+        return call
+    return {**new_call, "function": new_fn}
+
+
+# ── top-level channels of the relay body ─────────────────────────────────────
+
+
+def tokenise_system(system: Any, tok: Tokenise) -> Any:
+    """Anthropic ``system``: a string or list/tuple of text blocks."""
+    if isinstance(system, str):
+        out = tok(system)
+        return system if out == system else out
+    if isinstance(system, (list, tuple)):
+        parts = [_tokenise_system_part(part, tok) for part in system]
+        if any(done is not part for done, part in zip(parts, system, strict=True)):
+            return parts
+    if isinstance(system, dict):
+        return _tokenise_system_part(system, tok)
+    return system
+
+
+def _tokenise_system_part(part: Any, tok: Tokenise) -> Any:
+    if not isinstance(part, dict):
+        raise RequestBodyRejectedError()
+    result = dict(part)
+    changed = False
+    text = part.get("text")
+    if isinstance(text, str):
+        done = tok(text)
+        if done != text:
+            result["text"] = done
+            changed = True
+    if "content" in part:
+        done = tokenise_content(part["content"], tok)
+        if done is not part["content"]:
+            result["content"] = done
+            changed = True
+    for key, value in part.items():
+        if key not in {"type", "text", "content", "cache_control", "signature", "id"}:
+            done = tokenise_unclassified(value, tok)
+            if done is not value:
+                result[key] = done
+                changed = True
+    return result if changed else part
+
+
+def tokenise_body_strings(body: dict[str, Any], tok: Tokenise) -> dict[str, Any]:
+    """Tokenise body text slots not already handled by message traversal.
+
+    Structural fields stay untouched; a schema contract literal is left as written and
+    reported on the request. Returns a new dict only when a string changed.
+    """
+    new: dict[str, Any] | None = None
+    for key, walk in (("system", tokenise_system), ("metadata", tokenise_leaves),
+                      ("user", tokenise_leaves),
+                      ("clipboard", tokenise_leaves), ("preprompt", tokenise_leaves),
+                      ("prompt", tokenise_leaves), ("tool_choice", tokenise_tool_choice)):
+        if key not in body:
+            continue
+        if key == "metadata" and _has_sensitive_numbers(body[key]):
+            _mark_reduced_coverage(tok)
+        done = walk(body[key], tok)
+        if done is not body[key]:
+            if new is None:
+                new = dict(body)
+            new[key] = done
+    for key in ("attachments",):
+        if key in body:
+            done = tokenise_attachments(body[key], tok)
+            if done is not body[key]:
+                if new is None:
+                    new = dict(body)
+                new[key] = done
+    if "stop_sequences" in body:
+        done = tokenise_unclassified(body["stop_sequences"], tok)
+        if done is not body["stop_sequences"]:
+            if new is None:
+                new = dict(body)
+            new["stop_sequences"] = done
+    tools = body.get("tools")
+    if isinstance(tools, (list, tuple)):
+        # Tool definitions have prose and identifiers. Walk text fields and
+        # schema descriptions/examples, but reject a schema contract literal if
+        # tokenising it would silently change the tool's meaning.
+        processed = []
+        changed = False
+        for tool in tools:
+            if not isinstance(tool, dict):
+                raise RequestBodyRejectedError()
+            done = _tokenise_tool_definition(tool, tok)
+            processed.append(done)
+            changed = changed or done is not tool
+        if changed:
+            if new is None:
+                new = dict(body)
+            new["tools"] = processed
+    classified = {"model", "messages", "system", "metadata", "user", "clipboard", "preprompt", "prompt", "tool_choice", "attachments", "tools", "stop_sequences"}
+    for key, value in body.items():
+        if key not in classified:
+            done = tokenise_unclassified(value, tok)
+            if done is not value:
+                if new is None:
+                    new = dict(body)
+                new[key] = done
+    return body if new is None else new
+
+
+def assert_no_thai_khmer(value: Any) -> bool:
+    """Return whether this request contains Thai or Khmer text."""
+    if isinstance(value, str):
+        return any(
+            "\u0e00" <= char <= "\u0e7f"
+            or "\u1780" <= char <= "\u17ff"
+            or "\u19e0" <= char <= "\u19ff"
+            for char in value
+        )
+    found = False
+    if isinstance(value, dict):
+        for key, item in value.items():
+            found = assert_no_thai_khmer(key) or found
+            found = assert_no_thai_khmer(item) or found
+    elif isinstance(value, (list, tuple)):
+        for item in value:
+            found = assert_no_thai_khmer(item) or found
+    return found
+
+
+def tokenise_attachments(value: Any, tok: Tokenise) -> Any:
+    """Walk the text/name fields of attachments. Bytes and URLs are left as provided."""
+    if not isinstance(value, list):
+        raise RequestBodyRejectedError()
+    output = []
+    changed = False
+    for attachment in value:
+        if not isinstance(attachment, dict):
+            raise RequestBodyRejectedError()
+        next_attachment = dict(attachment)
+        item_changed = False
+        for key in ("text", "name", "filename"):
+            text = attachment.get(key)
+            if isinstance(text, str):
+                done = tok(text)
+                if done != text:
+                    next_attachment[key] = done
+                    item_changed = True
+        for key, value in attachment.items():
+            if key not in {"text", "name", "filename", "id", "type"} | _OPAQUE_MEDIA_KEYS:
+                done = tokenise_unclassified(value, tok)
+                if done is not value:
+                    next_attachment[key] = done
+                    item_changed = True
+        output.append(next_attachment if item_changed else attachment)
+        changed = changed or item_changed
+    return output if changed else value
+
+
+def _tokenise_legacy_function_call(value: Any, tok: Tokenise) -> Any:
+    """OpenAI ``function_call``: the name stays; argument values are tokenised."""
+    if not isinstance(value, dict):
+        return _tokenise_text_leaves_of_media(value, tok)
+    result = value
+    name = value.get("name")
+    if isinstance(name, str):
+        _note_tool_name(name, tok)
+    if "arguments" in value:
+        done = tokenise_arguments(value["arguments"], tok)
+        if done is not value["arguments"]:
+            result = {**value, "arguments": done}
+    for key, item in value.items():
+        if key in {"name", "arguments"}:
+            continue
+        changed = tokenise_unclassified(item, tok)
+        if changed is not item:
+            if result is value:
+                result = dict(value)
+            result[key] = changed
+    return result
+
+
+def tokenise_tool_choice(value: Any, tok: Tokenise) -> Any:
+    """A ``tool_choice`` name is never rewritten. A substantive match is a notice."""
+    if isinstance(value, str):
+        _note_tool_name(value, tok)
+        return value
+    if not isinstance(value, dict):
+        raise RequestBodyRejectedError()
+    result = dict(value)
+    changed = False
+    if isinstance(result.get("name"), str):
+        _note_tool_name(result["name"], tok)
+    fn = result.get("function")
+    if isinstance(fn, dict):
+        next_fn = dict(fn)
+        if isinstance(next_fn.get("name"), str):
+            _note_tool_name(next_fn["name"], tok)
+        for key, item in fn.items():
+            if key == "name":
+                continue
+            done = tokenise_unclassified(item, tok)
+            if done is not item:
+                next_fn[key] = done
+                changed = True
+        if changed:
+            result["function"] = next_fn
+    for key, item in value.items():
+        if key in {"name", "function"}:
+            continue
+        done = tokenise_unclassified(item, tok)
+        if done is not item:
+            result[key] = done
+            changed = True
+    return result if changed else value
+
+
+def _tokenise_text_leaves_of_media(value: Any, tok: Tokenise) -> Any:
+    """String leaves of a media or call object, except the leaves that hold bytes or a reference."""
+    if isinstance(value, str):
+        out = tok(value)
+        return value if out == value else out
+    if isinstance(value, dict):
+        changed = False
+        new: dict[Any, Any] = {}
+        for key, item in value.items():
+            done = item if key in _OPAQUE_MEDIA_KEYS else _tokenise_text_leaves_of_media(item, tok)
+            changed = changed or done is not item
+            new[key] = done
+        return new if changed else value
+    if isinstance(value, (list, tuple)):
+        items = [_tokenise_text_leaves_of_media(item, tok) for item in value]
+        if any(done is not item for done, item in zip(items, value, strict=True)):
+            return items
+    return value
+
+
+def _holds_media(value: dict[Any, Any], parent: str) -> bool:
+    """Whether this object carries bytes or a reference to media this path did not read."""
+    source = value.get("source")
+    if (
+        isinstance(source, dict)
+        and source.get("type") not in ("text", "content")
+        and any(source.get(key) for key in ("data", "url", "file_id"))
+    ):
+        return True
+    image_url = value.get("image_url")
+    if (isinstance(image_url, dict) and image_url.get("url")) or (
+        isinstance(image_url, str) and image_url
+    ):
+        return True
+    file = value.get("file")
+    if isinstance(file, dict) and any(file.get(key) for key in ("file_data", "file_id")):
+        return True
+    for key in ("input_audio", "audio"):
+        audio = value.get(key)
+        if isinstance(audio, dict) and any(audio.get(k) for k in ("data", "id", "url")):
+            return True
+    return parent == "attachments" and any(value.get(key) for key in _OPAQUE_MEDIA_KEYS)
+
+
+def _json_text_keys(text: str) -> list[str]:
+    """The object keys of a valid JSON text (empty for anything else)."""
+    try:
+        json.loads(text)
+    except (ValueError, RecursionError):
+        return []
+    keys = []
+    for match in _JSON_STRING.finditer(text):
+        if _KEY_COLON.match(text, match.end()):
+            try:
+                key = json.loads(match.group(0))
+            except ValueError:
+                continue
+            if isinstance(key, str):
+                keys.append(key)
+    return keys
+
+
+_TOOL_USE_BLOCK_TYPES = ("tool_use", "server_tool_use", "mcp_tool_use")
+
+
+def _is_tool_argument_place(trail: tuple[str, ...], key: str, holder: dict[Any, Any], item: Any) -> bool:
+    """Whether ``holder[key]`` is tool-call argument data.
+
+    Only a message carries it: the ``arguments`` of ``messages[].tool_calls[].function`` or
+    of legacy ``messages[].function_call``, and the ``input`` of a tool_use block in message
+    content (also inside tool_result content). ``trail`` is the path from the body root,
+    with ``[]`` for a list index. A property that merely has one of those names elsewhere
+    (``metadata``, ``user``, ...) is free-form data, so its keys are replaced like any
+    other key (F-001).
+    """
+    if len(trail) < 2 or trail[0] != "messages" or trail[1] != "[]":
+        return False
+    rest = trail[2:]
+    if key == "arguments":
+        return rest == ("tool_calls", "[]", "function") or rest == ("function_call",)
+    if key != "input" or holder.get("type") not in _TOOL_USE_BLOCK_TYPES:
+        return False
+    return (
+        isinstance(item, (dict, list, tuple))
+        and len(rest) >= 2
+        and len(rest) % 2 == 0
+        and all(segment == ("content" if index % 2 == 0 else "[]") for index, segment in enumerate(rest))
+    )
+
+
+def _child_context(key: str, item: Any, holder: dict[Any, Any], parent: str, trail: tuple[str, ...]) -> str:
+    """Which rule applies below ``key``: tool-call arguments, a tool schema, or free-form data."""
+    if _is_tool_argument_place(trail, key, holder, item):
+        return "arguments"
+    if (
+        key == "input_schema"
+        or (key == "parameters" and parent == "function")
+        or (key == "schema" and parent == "json_schema")
+    ):
+        return "schema"
+    return "free"
+
+
+def settle_structure(body: Any, tok: Tokenise) -> Any:
+    """Last pass over a body whose text has been tokenised: keys, identifiers, literals, media.
+
+    * A key in free-form data that the structured rules match is REPLACED where it stands,
+      except a metadata key, which is never renamed (owner decision F-068, 2026-11-14).
+      Keys inside tool-call arguments and tool schemas follow the tool's contract and
+      are not renamed.
+    * Everything else that still holds a detected value in a place that must not be
+      rewritten is noted on the request (``tok.note_unreplaced``), as is media.
+
+    Never raises for content. Returns the same object when no key was renamed.
+    """
+    state = getattr(tok, "state", None)
+    token_map = state.get("token_map", {}) if isinstance(state, dict) else {}
+
+    def detected(text: str) -> bool:
+        # Short structural strings only; a stand-in this request wrote is not a detection.
+        if not text or len(text) > 512:
+            return False
+        for marker in token_map:
+            if marker and marker in text:
+                text = text.replace(marker, " ")
+        return _has_structured_pii(text)
+
+    def _substantive_literal(value: Any) -> bool:
+        """Schema literal that a rule other than a bare word or a bare date matches."""
+        if isinstance(value, str):
+            return _substantive_pii(value)
+        if isinstance(value, dict):
+            return any(_substantive_literal(item) for item in value.values())
+        if isinstance(value, (list, tuple)):
+            return any(_substantive_literal(item) for item in value)
+        return False
+
+    def visit(value: Any, ctx: str, parent: str, trail: tuple[str, ...], *, metadata_keys: bool = False) -> Any:
+        if isinstance(value, (list, tuple)):
+            items = [visit(item, ctx, parent, trail + ("[]",), metadata_keys=metadata_keys) for item in value]
+            if any(done is not item for done, item in zip(items, value, strict=True)):
+                return items
+            return value
+        if not isinstance(value, dict):
+            return value
+        if _holds_media(value, parent):
+            _note(tok, PLACE_MEDIA)
+        out: dict[Any, Any] = {}
+        changed = False
+        for key, item in value.items():
+            new_key = key
+            child_ctx = ctx
+            child_metadata = metadata_keys or key == "metadata"
+            if isinstance(key, str):
+                if ctx == "free":
+                    child_ctx = _child_context(key, item, value, parent, trail)
+                if detected(key) and not metadata_keys:
+                    if ctx == "free":
+                        renamed = tok(key)
+                        if renamed != key and renamed not in value and renamed not in out:
+                            new_key = renamed
+                            changed = True
+                        else:
+                            _note(tok, PLACE_OBJECT_KEY)
+                    elif ctx == "arguments":
+                        _note(tok, PLACE_ARGUMENT_KEY)
+                    else:
+                        _note(tok, PLACE_TOOL_SCHEMA)
+                if key in _IDENTIFIER_KEYS and isinstance(item, str) and detected(item):
+                    _note(tok, PLACE_IDENTIFIER)
+                if (
+                    key in _SCHEMA_LITERAL_KEYS
+                    and (ctx == "schema" or key == "enum")
+                    and _substantive_literal(item)
+                ):
+                    _note(tok, PLACE_TOOL_SCHEMA)
+                if (
+                    child_ctx == "arguments"
+                    and isinstance(item, str)
+                    and any(detected(name) for name in _json_text_keys(item))
+                ):
+                    _note(tok, PLACE_ARGUMENT_KEY)
+            done = visit(
+                item,
+                child_ctx,
+                key if isinstance(key, str) else "",
+                trail + (key if isinstance(key, str) else "?",),
+                metadata_keys=child_metadata,
+            )
+            changed = changed or done is not item
+            out[new_key] = done
+        return out if changed else value
+
+    return visit(body, "free", "", ())
+
+
+def assert_no_inline_media(value: Any) -> None:
+    """Kept for 0.4.x imports. Inline media no longer stops a send; see settle_structure()."""
+    return None
+
+
+# ── restore in a reply ───────────────────────────────────────────────────────
+
+
+def restore_arguments(arguments: Any, token_map: dict[str, str]) -> Any:
+    """Put original values back into the string VALUES of reply tool-call arguments.
+
+    A JSON string keeps every other byte; an object is walked leaf by leaf. Keys are
+    not touched. Text that is not JSON is restored as plain text.
+    """
+    if not token_map:
+        return arguments
+
+    def back(text: str) -> str:
+        return detokenize(text, token_map)
+
+    if isinstance(arguments, str):
+        try:
+            json.loads(arguments)
+        except (ValueError, RecursionError):
+            return back(arguments)
+        return _splice_json_strings(arguments, back)
+    return tokenise_leaves(arguments, back)
+
+
+def restore_reply_tool_calls(message: dict[str, Any], token_map: dict[str, str]) -> dict[str, Any]:
+    """A chat-completion reply message with the argument values of its tool calls restored."""
+    result = message
+    calls = message.get("tool_calls")
+    if isinstance(calls, list):
+        new_calls = []
+        for call in calls:
+            fn = call.get("function") if isinstance(call, dict) else None
+            if isinstance(fn, dict) and "arguments" in fn:
+                restored = restore_arguments(fn["arguments"], token_map)
+                if restored is not fn["arguments"] and restored != fn["arguments"]:
+                    call = {**call, "function": {**fn, "arguments": restored}}
+            new_calls.append(call)
+        if any(new is not old for new, old in zip(new_calls, calls, strict=True)):
+            result = {**result, "tool_calls": new_calls}
+    legacy = message.get("function_call")
+    if isinstance(legacy, dict) and "arguments" in legacy:
+        restored = restore_arguments(legacy["arguments"], token_map)
+        if restored is not legacy["arguments"] and restored != legacy["arguments"]:
+            result = {**result, "function_call": {**legacy, "arguments": restored}}
+    return result
+
+
+def _tokenise_tool_definition(tool: Any, tok: Tokenise) -> Any:
+    if not isinstance(tool, dict):
+        raise RequestBodyRejectedError()
+    result = dict(tool)
+    changed = False
+    for key in ("name", "description"):
+        value = tool.get(key)
+        if isinstance(value, str):
+            if key == "name":
+                _note_tool_name(value, tok)
+                continue
+            done = tok(value)
+            if done != value:
+                result[key] = done
+                changed = True
+    for key in ("input_schema",):
+        if key in tool:
+            done = _tokenise_schema_prose(tool[key], tok)
+            if done is not tool[key]:
+                result[key] = done
+                changed = True
+    fn = tool.get("function")
+    if isinstance(fn, dict):
+        next_fn = dict(fn)
+        fn_changed = False
+        for key in ("name", "description"):
+            value = fn.get(key)
+            if isinstance(value, str):
+                if key == "name":
+                    _note_tool_name(value, tok)
+                    continue
+                done = tok(value)
+                if done != value:
+                    next_fn[key] = done
+                    fn_changed = True
+        if "parameters" in fn:
+            done = _tokenise_schema_prose(fn["parameters"], tok)
+            if done is not fn["parameters"]:
+                next_fn["parameters"] = done
+                fn_changed = True
+        for key, value in fn.items():
+            if key not in {"name", "description", "parameters", "type"}:
+                done = tokenise_unclassified(value, tok)
+                if done is not value:
+                    next_fn[key] = done
+                    fn_changed = True
+        if fn_changed:
+            result["function"] = next_fn
+            changed = True
+    for key, value in tool.items():
+        if key not in {"name", "description", "input_schema", "function", "type"}:
+            done = tokenise_unclassified(value, tok)
+            if done is not value:
+                result[key] = done
+                changed = True
+    return result if changed else tool
+
+
+def _literal_would_change(value: Any, tok: Tokenise) -> bool:
+    """Whether the detector would replace anything in a schema literal. Changes nothing."""
+    strings: list[str] = []
+
+    def collect(item: Any) -> None:
+        if isinstance(item, str):
+            strings.append(item)
+        elif isinstance(item, dict):
+            for child in item.values():
+                collect(child)
+        elif isinstance(item, (list, tuple)):
+            for child in item:
+                collect(child)
+
+    collect(value)
+    # Bare-word and bare-date matches are not a schema-literal notice.
+    return any(_substantive_pii(text) for text in strings)
+
+
+def _tokenise_schema_prose(value: Any, tok: Tokenise) -> Any:
+    if isinstance(value, list):
+        items = [_tokenise_schema_prose(item, tok) for item in value]
+        return items if any(a is not b for a, b in zip(items, value, strict=True)) else value
+    if not isinstance(value, dict):
+        return value
+    result = dict(value)
+    changed = False
+    for key in ("description", "title"):
+        text = value.get(key)
+        if isinstance(text, str):
+            done = tok(text)
+            if done != text:
+                result[key] = done
+                changed = True
+    examples = value.get("examples")
+    if isinstance(examples, list):
+        done = tokenise_leaves(examples, tok)
+        if done is not examples:
+            result["examples"] = done
+            changed = True
+    for key, item in value.items():
+        if key in ("description", "title", "examples"):
+            continue
+        if key in _SCHEMA_LITERAL_KEYS:
+            # The model must echo such a literal exactly and the caller's code
+            # compares it, so it is not rewritten. A literal the detector would
+            # have replaced is reported on the request; the send goes on.
+            if _literal_would_change(item, tok):
+                _note(tok, PLACE_TOOL_SCHEMA)
+            continue
+        done = _tokenise_schema_prose(item, tok)
+        if done is not item:
+            result[key] = done
+            changed = True
+    return result if changed else value