mostlyright-data 0.31.12__tar.gz → 0.31.13__tar.gz

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.31.12
+Version: 0.31.13
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/
@@ -46,6 +46,13 @@ Once a day, after a command has printed its answer, `mr-data` asks the Python pa
 a newer release is published and says so in one line on stderr. `MOSTLYRIGHT_RELEASE_CHECK=0`
 switches that off.
 
+Every call to Mostly Right names the coding agent running `mr-data` in `X-Mostlyright-Agent`
+(`claude-code`, `codex`, `gemini-cli`, `cursor`, `ci` or `none`, read from the variables those
+hosts set), and a program that imports this package sends it too. An `mr-data` command also sends
+one random id in `X-Mostlyright-Invocation`, so the requests of one command can be grouped; a
+library call sends none. Neither is sent to storage hosts. `MOSTLYRIGHT_NO_AGENT_HEADERS=1`
+switches both off.
+
 ## Sign in
 
 ```bash

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/README.md

@@ -31,6 +31,13 @@ Once a day, after a command has printed its answer, `mr-data` asks the Python pa
 a newer release is published and says so in one line on stderr. `MOSTLYRIGHT_RELEASE_CHECK=0`
 switches that off.
 
+Every call to Mostly Right names the coding agent running `mr-data` in `X-Mostlyright-Agent`
+(`claude-code`, `codex`, `gemini-cli`, `cursor`, `ci` or `none`, read from the variables those
+hosts set), and a program that imports this package sends it too. An `mr-data` command also sends
+one random id in `X-Mostlyright-Invocation`, so the requests of one command can be grouped; a
+library call sends none. Neither is sent to storage hosts. `MOSTLYRIGHT_NO_AGENT_HEADERS=1`
+switches both off.
+
 ## Sign in
 
 ```bash

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/thin/router.py

@@ -52,6 +52,7 @@ from collections.abc import Callable, Mapping, Sequence
 from typing import Any
 
 from mostlyright.data_harness import skill_assets
+from mostlyright.data_harness.thin import user_agent
 from mostlyright.data_harness.thin.transport import ThinLaneError
 from mostlyright.data_harness.thin.vocabulary import (
     CLASSIFICATION,
@@ -781,6 +782,7 @@ def main(argv: Sequence[str] | None = None) -> int:
     # not write, it never raises, and `MOSTLYRIGHT_SKILL_AUTOINSTALL=0` turns it off. A skill that
     # arrived only when somebody knew to ask for it helped exactly the people who did not need it.
     skill_assets.ensure_installed()
+    user_agent.begin_invocation()
 
     arguments = list(sys.argv[1:] if argv is None else argv)
     hosted_requested = _hosted_flag_present(arguments)

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/thin/runs.py

@@ -46,6 +46,7 @@ from mostlyright.data_harness.thin.transport import (
     UrlLibTransport,
     signed_transfer_url,
 )
+from mostlyright.data_harness.thin.user_agent import agent_headers
 
 #: The contract version every payload below is written against. ``studio_boundary`` pins the
 #: generated client to the same 3.0.0 and to one OpenAPI source digest; this constant is the thin
@@ -279,6 +280,7 @@ class StudioApiClient:
         headers = {
             "Accept": "application/json",
             CLIENT_HEADER: CLIENT_NAME,
+            **agent_headers(),
         }
         if extra_headers:
             headers.update(extra_headers)

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/thin/session.py

@@ -42,6 +42,7 @@ from mostlyright.data_harness.thin.transport import (
     UrlLibTransport,
     service_origin,
 )
+from mostlyright.data_harness.thin.user_agent import agent_headers
 from mostlyright.data_harness.ux.credentials import ResolvedCloudCredentials
 
 #: Cloud's token-exchange route. Held identical to ``hosted_deploy.CLOUD_TOKEN_PATH`` by a test.
@@ -396,7 +397,7 @@ def open_studio_session(
             status, raw, headers = selected.request(
                 "POST",
                 f"{cloud_url}{CLOUD_TOKEN_PATH}",
-                {"Accept": "application/json", "x-api-key": credentials.raw_key},
+                {"Accept": "application/json", "x-api-key": credentials.raw_key, **agent_headers()},
                 b"",
                 MAX_TOKEN_RESPONSE_BYTES,
             )
@@ -953,6 +954,7 @@ class CloudClient:
         headers = {
             "Accept": "application/json",
             "Authorization": f"Bearer {self._credentials.raw_key}",
+            **agent_headers(),
         }
         encoded: bytes | None = None
         if body is not None:

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/thin/stream.py

@@ -49,6 +49,7 @@ from mostlyright.data_harness.thin.session import (
     stated_sentence,
 )
 from mostlyright.data_harness.thin.transport import ThinLaneError, Transport, UrlLibTransport
+from mostlyright.data_harness.thin.user_agent import agent_headers
 
 #: Upstream's frame names. The first two are the tick/proof partition; ``end`` is its
 #: end-of-stream notice.
@@ -593,6 +594,7 @@ class RunEventStream:
         headers = {
             "Accept": "text/event-stream",
             CLIENT_HEADER: CLIENT_NAME,
+            **agent_headers(),
             **session.authorization(),
         }
         # Resume in Studio's own cursor space when we have one. `from_seq` is only ever the

mostlyright_data-0.31.13/src/mostlyright/data_harness/thin/user_agent.py

@@ -0,0 +1,234 @@
+"""The identity every first-party request to Mostly Right states.
+
+Without it the standard library sends ``Python-urllib/3.x``, which is what every unattended
+script on the internet sends: the CLI's own traffic could not be told apart from anything else in
+a server log, so nothing downstream could measure it, rate-limit it separately, or exempt it.
+
+The name follows the convention the two crawler lanes already use -- a product token, a version,
+and the contact segment that says where to reach the people running it -- so an origin and our own
+logs see one family of identities from this deployment rather than one per code path.
+
+Two more headers ride beside it on first-party calls only (:func:`agent_headers`): which coding
+agent, if any, is driving this process, and, under the ``mr-data`` command, one id shared by every
+request that command makes. They go where ``X-Mostlyright-Client`` goes and on Cloud's own
+routes, never through the transport, for the reason ``thin.runs.CLIENT_HEADER`` gives: the
+transport also carries signed transfers to storage nobody here owns.
+"""
+
+from __future__ import annotations
+
+import os
+import secrets
+from collections.abc import Mapping
+from functools import cache
+from importlib.metadata import PackageNotFoundError, version
+from typing import NamedTuple
+
+#: The contact segment ``hosted_worker.CRAWLER_CONTACT_SEGMENT`` states for the keyed-source
+#: agents and ``hosted_crawler_fetch.PUBLIC_HTTPS_USER_AGENT`` carries for public sources, with
+#: the measurement that makes it load-bearing recorded at the first of those.
+#:
+#: It is spelled again here rather than imported because both of those modules import ``pyarrow``
+#: at module scope and this package installs with no compiled extension at all -- and they are
+#: Studio's now, so importing them is not a choice this repository has. The spellings are held
+#: against each other by ``tests/test_thin_user_agent.py`` under
+#: ``make verify-cross-repo STUDIO_ROOT=...``, which reads the other one out of Studio's source.
+CRAWLER_CONTACT_SEGMENT = "(+https://mostlyright.md/crawler; crawler@mostlyright.md)"
+
+#: The distribution this client is installed as, and the product token it therefore reports.
+DISTRIBUTION_NAME = "mostlyright-data"
+
+#: The version reported from a source tree, where no distribution metadata exists to read. A
+#: number rather than a word, so every value this header can carry parses the same way.
+UNINSTALLED_VERSION = "0.0.0"
+
+_USER_AGENT_HEADER = "User-Agent"
+
+#: The header naming the agent driving this process, one of :data:`AGENTS`.
+AGENT_HEADER = "X-Mostlyright-Agent"
+
+#: The header carrying :func:`invocation_id`, sent only after :func:`begin_invocation`.
+INVOCATION_HEADER = "X-Mostlyright-Invocation"
+
+#: Set to anything but ``0`` or empty, and neither header above is sent.
+NO_AGENT_HEADERS_ENV = "MOSTLYRIGHT_NO_AGENT_HEADERS"
+
+AGENT_CLAUDE_CODE = "claude-code"
+AGENT_CODEX = "codex"
+AGENT_GEMINI_CLI = "gemini-cli"
+AGENT_CURSOR = "cursor"
+AGENT_CI = "ci"
+AGENT_NONE = "none"
+
+
+class AgentMarker(NamedTuple):
+    """One variable an agent host exports into the commands it runs.
+
+    ``conversation`` says whether the attendance check (``ux.attendance``) reads the variable as
+    a conversation existing. Only a marker every headless run of that host lacks may say so:
+    ``CODEX_SANDBOX`` is set by ``codex exec`` on a cron job too, so it names Codex here and is
+    not read as a person being there.
+    """
+
+    agent: str
+    variable: str
+    conversation: bool
+
+
+#: Every marker, grouped by agent in precedence order.
+#:
+#: Presence of a non-empty value is the signal and the value is never read: these are the hosts'
+#: variables, and their values are theirs to change. A coding agent outranks ``ci``, so
+#: ``claude -p`` inside a GitHub Action reports ``claude-code``. Nothing that every terminal of an
+#: editor exports is listed, because a person typing there would read as the agent.
+AGENT_MARKERS: tuple[AgentMarker, ...] = (
+    AgentMarker(AGENT_CLAUDE_CODE, "CLAUDECODE", True),
+    AgentMarker(AGENT_CLAUDE_CODE, "CLAUDE_CODE_SESSION", True),
+    AgentMarker(AGENT_CODEX, "CODEX_SESSION", True),
+    AgentMarker(AGENT_CODEX, "CODEX_SANDBOX", False),
+    AgentMarker(AGENT_CODEX, "CODEX_SANDBOX_NETWORK_DISABLED", False),
+    AgentMarker(AGENT_GEMINI_CLI, "GEMINI_CLI", False),
+    AgentMarker(AGENT_CURSOR, "CURSOR_AGENT", False),
+    AgentMarker(AGENT_CI, "CI", False),
+    AgentMarker(AGENT_CI, "GITHUB_ACTIONS", False),
+)
+
+#: Every value :data:`AGENT_HEADER` can carry.
+AGENTS: tuple[str, ...] = (*dict.fromkeys(marker.agent for marker in AGENT_MARKERS), AGENT_NONE)
+
+#: What :func:`invocation_id` returns: 16 random bytes as 32 lowercase hex digits.
+INVOCATION_ID_PATTERN = r"^[0-9a-f]{32}$"
+
+_invocation: str | None = None
+
+
+@cache
+def client_user_agent() -> str:
+    """Return the ``User-Agent`` this client presents to Mostly Right.
+
+    Cached because the answer is a property of the installed environment, and reading distribution
+    metadata walks ``sys.path``; a command that makes twenty calls would otherwise pay for it
+    twenty times.
+    """
+
+    try:
+        installed = version(DISTRIBUTION_NAME)
+    except PackageNotFoundError:
+        installed = UNINSTALLED_VERSION
+    return f"{DISTRIBUTION_NAME}/{installed} {CRAWLER_CONTACT_SEGMENT}"
+
+
+def with_user_agent(headers: Mapping[str, str]) -> dict[str, str]:
+    """Return ``headers`` as a new dict carrying this client's identity.
+
+    A caller that stated its own ``User-Agent`` keeps it, under whatever spelling it used: HTTP
+    header names are case-insensitive, so adding a second one under a different capitalization
+    would send two identities rather than override one.
+    """
+
+    resolved = dict(headers)
+    if any(name.lower() == "user-agent" for name in resolved):
+        return resolved
+    resolved[_USER_AGENT_HEADER] = client_user_agent()
+    return resolved
+
+
+def agent_env(agent: str) -> tuple[str, ...]:
+    """Return the variables that name ``agent``; empty for ``none`` and for a name not listed."""
+
+    return tuple(marker.variable for marker in AGENT_MARKERS if marker.agent == agent)
+
+
+def conversation_env() -> tuple[str, ...]:
+    """Return the markers the attendance check reads as a conversation existing."""
+
+    return tuple(marker.variable for marker in AGENT_MARKERS if marker.conversation)
+
+
+def first_set(names: tuple[str, ...], environ: Mapping[str, str] | None = None) -> str | None:
+    """Return the first of ``names`` set to a non-empty value, or ``None``."""
+
+    table = os.environ if environ is None else environ
+    return next((name for name in names if (table.get(name) or "").strip()), None)
+
+
+def detect_agent(environ: Mapping[str, str] | None = None) -> str:
+    """Return the agent driving this process, one of :data:`AGENTS`."""
+
+    table = os.environ if environ is None else environ
+    for marker in AGENT_MARKERS:
+        if (table.get(marker.variable) or "").strip():
+            return marker.agent
+    return AGENT_NONE
+
+
+def begin_invocation() -> str:
+    """Draw a fresh invocation id for the command starting now, and return it.
+
+    Called by the ``mr-data`` entry point once per command. A program importing this package
+    never calls it, so its requests carry no invocation id rather than one that lasts as long as
+    the process does.
+    """
+
+    global _invocation
+    _invocation = secrets.token_hex(16)
+    return _invocation
+
+
+def end_invocation() -> None:
+    """Forget the current invocation id, so later requests carry none."""
+
+    global _invocation
+    _invocation = None
+
+
+def invocation_id() -> str | None:
+    """Return the current command's id (see :data:`INVOCATION_ID_PATTERN`), or ``None``."""
+
+    return _invocation
+
+
+def agent_headers(environ: Mapping[str, str] | None = None) -> dict[str, str]:
+    """Return the agent and invocation headers, or none when :data:`NO_AGENT_HEADERS_ENV` is set.
+
+    The invocation header is present only inside a command :func:`begin_invocation` started. A
+    first-party call adds these itself; the transport never does.
+    """
+
+    table = os.environ if environ is None else environ
+    if (table.get(NO_AGENT_HEADERS_ENV) or "").strip() not in ("", "0"):
+        return {}
+    headers = {AGENT_HEADER: detect_agent(table)}
+    if _invocation is not None:
+        headers[INVOCATION_HEADER] = _invocation
+    return headers
+
+
+__all__ = [
+    "AGENTS",
+    "AGENT_CI",
+    "AGENT_CLAUDE_CODE",
+    "AGENT_CODEX",
+    "AGENT_CURSOR",
+    "AGENT_GEMINI_CLI",
+    "AGENT_HEADER",
+    "AGENT_MARKERS",
+    "AGENT_NONE",
+    "CRAWLER_CONTACT_SEGMENT",
+    "DISTRIBUTION_NAME",
+    "INVOCATION_HEADER",
+    "INVOCATION_ID_PATTERN",
+    "NO_AGENT_HEADERS_ENV",
+    "UNINSTALLED_VERSION",
+    "AgentMarker",
+    "agent_env",
+    "agent_headers",
+    "begin_invocation",
+    "client_user_agent",
+    "conversation_env",
+    "detect_agent",
+    "end_invocation",
+    "first_set",
+    "invocation_id",
+    "with_user_agent",
+]

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/thin/v4_public.py

@@ -52,7 +52,7 @@ from mostlyright.data_harness.thin.transport import (
     UrlLibTransport,
     service_origin,
 )
-from mostlyright.data_harness.thin.user_agent import with_user_agent
+from mostlyright.data_harness.thin.user_agent import agent_headers, with_user_agent
 from mostlyright.data_harness.ux.credentials import (
     CLOUD_URL_ENV,
     DEFAULT_CLOUD_URL,
@@ -245,6 +245,7 @@ class PublicCatalogClient:
             {
                 "Accept": accept,
                 CLIENT_HEADER: CLIENT_NAME,
+                **agent_headers(),
             }
         )
 
@@ -960,6 +961,7 @@ class PublicQueryClient(CloudClient):
         headers = {
             "Accept": "application/json",
             "Authorization": f"Bearer {self._credentials.raw_key}",
+            **agent_headers(),
         }
         status, raw, response_headers = self._transport.request(
             "GET",

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/thin/v4_query.py

@@ -75,6 +75,7 @@ from mostlyright.data_harness.thin.transport import (
     Transport,
     UrlLibTransport,
 )
+from mostlyright.data_harness.thin.user_agent import agent_headers
 from mostlyright.data_harness.thin.v4 import (
     GET_RUN_PATH,
     GET_RUN_QUERY_PATH,
@@ -329,6 +330,7 @@ class CloudRunQueryClient(CloudClient):
         headers = {
             "Accept": "application/json",
             "Authorization": f"Bearer {self._credentials.raw_key}",
+            **agent_headers(),
         }
         encoded: bytes | None = None
         if body is not None:

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/ux/attendance.py

@@ -107,6 +107,13 @@ from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from typing import Any
 
+from mostlyright.data_harness.thin.user_agent import (
+    AGENT_CI,
+    agent_env,
+    conversation_env,
+    first_set,
+)
+
 #: The variable the process that spawns ``mr-data`` sets to say whether the session it is driving
 #: has a person in it. Exactly ``"1"`` is attended and exactly ``"0"`` is unattended -- and the
 #: ``"0"`` is a veto no flag overrides, for the reason the module docstring gives. Every other value
@@ -156,7 +163,7 @@ _UNATTENDED_VALUE = "0"
 #: ⚠ AND ONLY A NON-EMPTY VALUE COUNTS. A shell that exports an empty variable has said nothing,
 #: and reading that as "a person is here" would make ``export CLAUDECODE=`` a way to turn the
 #: inference on by accident.
-AGENT_HOST_ENV: tuple[str, ...] = ("CLAUDECODE", "CLAUDE_CODE_SESSION", "CODEX_SESSION")
+AGENT_HOST_ENV: tuple[str, ...] = conversation_env()
 
 #: The variables a headless runner sets, which take an agent-host marker back off the table.
 #:
@@ -170,7 +177,7 @@ AGENT_HOST_ENV: tuple[str, ...] = ("CLAUDECODE", "CLAUDE_CODE_SESSION", "CODEX_S
 #: Buildkite all export it; ``GITHUB_ACTIONS`` is named beside it because the scenario this closes
 #: is literally ``claude -p`` in a GitHub Action. Presence of a non-empty value is the whole test,
 #: for the reason the agent-host markers give: the values are somebody else's to change.
-HEADLESS_HOST_ENV: tuple[str, ...] = ("CI", "GITHUB_ACTIONS")
+HEADLESS_HOST_ENV: tuple[str, ...] = agent_env(AGENT_CI)
 
 
 @dataclass(frozen=True)
@@ -264,11 +271,9 @@ def resolve_attendance(
         )
     # Step 5: the host named itself, which is a conversation existing -- unless a headless runner
     # named itself too, in which case the conversation is a print-mode one with nobody in it.
-    named = next((name for name in AGENT_HOST_ENV if (table.get(name) or "").strip()), None)
+    named = first_set(AGENT_HOST_ENV, table)
     if named is not None:
-        headless = next(
-            (name for name in HEADLESS_HOST_ENV if (table.get(name) or "").strip()), None
-        )
+        headless = first_set(HEADLESS_HOST_ENV, table)
         if headless is not None:
             return Attendance(
                 attended=False,

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/ux/cloud_auth.py

@@ -16,7 +16,7 @@ from collections.abc import Callable, Mapping
 from dataclasses import dataclass
 from typing import Any, Final
 
-from mostlyright.data_harness.thin.user_agent import with_user_agent
+from mostlyright.data_harness.thin.user_agent import agent_headers, with_user_agent
 
 REQUEST_TIMEOUT_SECONDS: Final = 10.0
 MAX_RESPONSE_BYTES: Final = 64 * 1024
@@ -241,7 +241,7 @@ class CloudAuthClient:
     def _request(self, method: str, path: str) -> CloudAuthResponse:
         request = urllib.request.Request(
             urllib.parse.urljoin(self._base_url + "/", path.lstrip("/")),
-            headers=with_user_agent({"x-api-key": self._raw_key}),
+            headers=with_user_agent({"x-api-key": self._raw_key, **agent_headers()}),
             method=method,
         )
         try:

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/ux/commands/clarify.py

@@ -34,7 +34,12 @@ from collections.abc import Mapping
 from typing import Any
 
 from mostlyright.data_harness.thin.transport import ThinLaneError
-from mostlyright.data_harness.ux.attendance import ATTENDANCE_ENV, resolve_attendance
+from mostlyright.data_harness.ux.attendance import (
+    AGENT_HOST_ENV,
+    ATTENDANCE_ENV,
+    HEADLESS_HOST_ENV,
+    resolve_attendance,
+)
 from mostlyright.data_harness.ux.clarification import (
     CONTRACT_ALREADY_AGREED,
     checked_question,
@@ -118,10 +123,10 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
         help=(
             "state that a person is here to answer, which is the one reliable way to say so; an "
             "explicit refusal, by --unattended or MOSTLYRIGHT_ATTENDED=0, still wins. Without it, "
-            "an agent host that names itself in the environment (CLAUDECODE, "
-            "CLAUDE_CODE_SESSION, CODEX_SESSION) is read as a conversation with somebody in it -- "
-            "but NOT when CI or GITHUB_ACTIONS is set beside it, because an agent host on a runner "
-            "is headless. A cron job carries none of these and stays unattended"
+            "an agent host that names itself in the environment "
+            f"({', '.join(AGENT_HOST_ENV)}) is read as a conversation with somebody in it -- "
+            f"but NOT when {' or '.join(HEADLESS_HOST_ENV)} is set beside it, because an agent "
+            "host on a runner is headless. A cron job carries none of these and stays unattended"
         ),
     )
     presence.add_argument(

{mostlyright_data-0.31.12 → mostlyright_data-0.31.13}/src/mostlyright/data_harness/ux/login.py

@@ -49,6 +49,7 @@ from urllib.request import (
 
 from mostlyright.data_harness.canonical import CanonicalJSONError, parse_foreign_json
 from mostlyright.data_harness.key_seam import LEGACY_API_KEY_ENV
+from mostlyright.data_harness.thin.user_agent import agent_headers
 from mostlyright.data_harness.ux.cloud_auth import CloudAuthClient, CloudAuthError
 from mostlyright.data_harness.ux.credential_store import (
     CredentialEnvelope,
@@ -291,7 +292,11 @@ def _post_json(
     """
 
     body = json.dumps(dict(payload), sort_keys=True).encode("utf-8")
-    request_headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    request_headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        **agent_headers(),
+    }
     request_headers.update(headers or {})
     try:
         status, raw = transport.request("POST", url, request_headers, body, MAX_RESPONSE_BYTES)
@@ -324,7 +329,7 @@ def _get_json(
     caller reads a 404 as "this cloud does not offer the endpoint yet" rather than as a failure.
     """
 
-    request_headers = {"Accept": "application/json"}
+    request_headers = {"Accept": "application/json", **agent_headers()}
     request_headers.update(headers or {})
     try:
         status, raw = transport.request("GET", url, request_headers, None, MAX_RESPONSE_BYTES)

mostlyright_data-0.31.12/src/mostlyright/data_harness/thin/user_agent.py

@@ -1,76 +0,0 @@
-"""The identity every first-party request to Mostly Right states.
-
-Without it the standard library sends ``Python-urllib/3.x``, which is what every unattended
-script on the internet sends: the CLI's own traffic could not be told apart from anything else in
-a server log, so nothing downstream could measure it, rate-limit it separately, or exempt it.
-
-The name follows the convention the two crawler lanes already use -- a product token, a version,
-and the contact segment that says where to reach the people running it -- so an origin and our own
-logs see one family of identities from this deployment rather than one per code path.
-"""
-
-from __future__ import annotations
-
-from collections.abc import Mapping
-from functools import cache
-from importlib.metadata import PackageNotFoundError, version
-
-#: The contact segment ``hosted_worker.CRAWLER_CONTACT_SEGMENT`` states for the keyed-source
-#: agents and ``hosted_crawler_fetch.PUBLIC_HTTPS_USER_AGENT`` carries for public sources, with
-#: the measurement that makes it load-bearing recorded at the first of those.
-#:
-#: It is spelled again here rather than imported because both of those modules import ``pyarrow``
-#: at module scope and this package installs with no compiled extension at all -- and they are
-#: Studio's now, so importing them is not a choice this repository has. The spellings are held
-#: against each other by ``tests/test_thin_user_agent.py`` under
-#: ``make verify-cross-repo STUDIO_ROOT=...``, which reads the other one out of Studio's source.
-CRAWLER_CONTACT_SEGMENT = "(+https://mostlyright.md/crawler; crawler@mostlyright.md)"
-
-#: The distribution this client is installed as, and the product token it therefore reports.
-DISTRIBUTION_NAME = "mostlyright-data"
-
-#: The version reported from a source tree, where no distribution metadata exists to read. A
-#: number rather than a word, so every value this header can carry parses the same way.
-UNINSTALLED_VERSION = "0.0.0"
-
-_USER_AGENT_HEADER = "User-Agent"
-
-
-@cache
-def client_user_agent() -> str:
-    """Return the ``User-Agent`` this client presents to Mostly Right.
-
-    Cached because the answer is a property of the installed environment, and reading distribution
-    metadata walks ``sys.path``; a command that makes twenty calls would otherwise pay for it
-    twenty times.
-    """
-
-    try:
-        installed = version(DISTRIBUTION_NAME)
-    except PackageNotFoundError:
-        installed = UNINSTALLED_VERSION
-    return f"{DISTRIBUTION_NAME}/{installed} {CRAWLER_CONTACT_SEGMENT}"
-
-
-def with_user_agent(headers: Mapping[str, str]) -> dict[str, str]:
-    """Return ``headers`` as a new dict carrying this client's identity.
-
-    A caller that stated its own ``User-Agent`` keeps it, under whatever spelling it used: HTTP
-    header names are case-insensitive, so adding a second one under a different capitalization
-    would send two identities rather than override one.
-    """
-
-    resolved = dict(headers)
-    if any(name.lower() == "user-agent" for name in resolved):
-        return resolved
-    resolved[_USER_AGENT_HEADER] = client_user_agent()
-    return resolved
-
-
-__all__ = [
-    "CRAWLER_CONTACT_SEGMENT",
-    "DISTRIBUTION_NAME",
-    "UNINSTALLED_VERSION",
-    "client_user_agent",
-    "with_user_agent",
-]