mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/thin/vocabulary.py

@@ -0,0 +1,368 @@
+"""Every ``mr-data`` name, the lane it runs in, and the written reason it runs there.
+
+WHY A REASON AND NOT JUST A LANE. PR #308 landed one hosted lane and one word for everything
+else: forty-five commands were ``local``, which is true and says nothing. It cannot distinguish
+``mr-data show`` -- which a person types about a hosted run every day and which had no hosted
+answer -- from ``mr-data init``, which makes a folder on this computer and is meaningless without
+one, from ``mr-data export-hosted-candidate``, which exists only to hand a locally built table to
+the backend and is not part of anybody's hosted day. Those three want three different sentences
+when they refuse, and a table with one column cannot give them.
+
+So each name below carries a lane AND the sentence that justifies it. The sentence is not
+decoration: :mod:`mostlyright.data_harness.thin.router` prints it in the refusal, the thin help
+groups by lane, and ``docs/HOSTED-PARITY.md`` is generated from nothing else -- there is one
+statement of why a command is where it is, and every surface reads it from here.
+
+THE FIVE LANES.
+
+``HOSTED``
+    Implemented against Studio V3 and reachable on a bare interpreter. The command means the same
+    thing it means locally, takes the same flags, and prints the same receipt.
+
+``PORTABLE``
+    Already pure Python before ADR 0021. The thin profile runs the SAME implementation the local
+    profile runs, not a copy of it.
+
+``HOSTED_PENDING``
+    User-facing, and it should be hosted. It is not, and the reason names the exact missing
+    capability rather than saying "later". Each of these is blocked on something outside this
+    repository, and the sentence says what would unblock it. A gap that is written down is a gap
+    somebody can close; one recorded as "local" is a gap nobody can see.
+
+``WORKER``
+    An operator or worker surface. It runs inside the backend's own execution environment, never
+    on a person's laptop, so it has no thin lane by construction and never will. What it needs is
+    a sentence instead of a traceback.
+
+``LOCAL``
+    Local tooling, behind the ``[local]`` extra by design. Either it is about files on this
+    computer -- and there are none in a hosted build -- or the hosted lane already answers the
+    same question under a different name, which the reason says.
+
+⚠ THE SET OF NAMES IS THE COMMAND LINE'S, NOT THIS FILE'S. ``tests/test_thin_client.py`` asserts
+:data:`VOCABULARY` equals the live parser's subcommands, so a command cannot be added to one and
+forgotten in the other.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+#: Implemented by the thin lane against Studio.
+HOSTED = "hosted"
+
+#: Pure Python already, and identical in both profiles.
+PORTABLE = "portable"
+
+#: User-facing, not hosted yet, and blocked on a named capability outside this repository.
+HOSTED_PENDING = "hosted-pending"
+
+#: A backend operator surface, which is never a thin-client concern.
+WORKER = "worker"
+
+#: Needs the ``[local]`` extra, by design.
+LOCAL = "local"
+
+#: Every lane, in the order a reader should meet them.
+LANES: tuple[str, ...] = (HOSTED, PORTABLE, HOSTED_PENDING, WORKER, LOCAL)
+
+
+@dataclass(frozen=True)
+class Lane:
+    """One command's lane and the written reason it is in that lane.
+
+    ``reason`` is one sentence, in the second person where it addresses somebody at a terminal,
+    and it must survive being printed on its own: it is what a refusal says after the command
+    name, with no surrounding paragraph to lean on.
+    """
+
+    lane: str
+    reason: str
+
+
+# --------------------------------------------------------------------------------------------
+# The table
+# --------------------------------------------------------------------------------------------
+#
+# Sorted by name. Read it as the answer to one question asked fifty-eight times: if somebody types
+# this on a machine that has no local execution lane, what should happen?
+
+CLASSIFICATION: dict[str, Lane] = {
+    "acquire": Lane(
+        HOSTED,
+        "it fetches one public address in Studio's clean room and prints the Receipt Studio "
+        "bound, naming which party resolved the pinned Reader",
+    ),
+    "acquire-cancel": Lane(
+        HOSTED,
+        "it cancels one active public acquisition by the workspace and session coordinate its "
+        "Receipt printed, reusing a durable idempotency journal if the response was lost",
+    ),
+    "acquire-slices": Lane(
+        HOSTED_PENDING,
+        "the hosted request contract admits one URL and a slice order needs two origin-bound "
+        "locators plus an ordered plan, and its safety rests on those N+2 requests sharing one "
+        "DNS epoch -- which fanning them out as one hosted session each would silently lose",
+    ),
+    "adapters": Lane(
+        LOCAL,
+        "it lists what the source adapters an acquisition document registers can do here, which "
+        "is a fact about this installation and not about the backend",
+    ),
+    "agent-request-validate": Lane(
+        LOCAL,
+        "it checks a request document against the contracts compiled into this installation, and "
+        "reaches no network at all",
+    ),
+    "approve": Lane(
+        HOSTED,
+        "it records the decision you are asking for, prints where the person who must grant it "
+        "does so, and reads back what they decided; it never decides",
+    ),
+    "author": Lane(
+        LOCAL,
+        "it turns a YAML file on this computer into a Recipe file on this computer; the hosted "
+        "lane answers recipe authoring with `propose`, which drives Studio's own authoring routes",
+    ),
+    "auth": Lane(
+        PORTABLE,
+        "it validates the effective device key with Cloud and explains the token boundary without "
+        "requiring the local execution lane",
+    ),
+    "build": Lane(HOSTED, "it queues one hosted build and says where to watch it"),
+    "catalog-author": Lane(LOCAL, "it writes one public catalog generation to a folder here"),
+    "catalog-fill": Lane(LOCAL, "it stages and publishes one public catalog successor here"),
+    "catalog-harvest": Lane(LOCAL, "it sweeps a provider into a folder here, under a key file"),
+    "catalog-publish": Lane(LOCAL, "it publishes one authored generation from a folder here"),
+    "catalog-search": Lane(LOCAL, "it searches a catalog installed on this computer"),
+    "catalog-update": Lane(LOCAL, "it installs a catalog release onto this computer"),
+    "deploy": Lane(
+        LOCAL,
+        "it stages a Build made on this computer and activates it; the hosted lane answers this "
+        "with `propose`, which opens the recipe proposal a hosted build activates, so there is "
+        "nothing here to stage",
+    ),
+    "deploy-dataset": Lane(HOSTED, "it downloads one hosted run's dataset and verifies its digest"),
+    "deploy-plan": Lane(
+        LOCAL,
+        "it writes the request document for deploying a Build made on this computer, and contacts "
+        "nothing; the hosted lane answers this with `propose`, which drives the deployment routes "
+        "directly from an authored bundle",
+    ),
+    "deploy-status": Lane(
+        HOSTED, "it reports what Studio says about the run one deployment queued"
+    ),
+    "diff": Lane(HOSTED, "it compares two hosted runs and says what changed"),
+    "export-hosted-candidate": Lane(
+        WORKER,
+        "it packages a Build made on this computer for the backend to ingest, which is the local "
+        "build lane's own handoff step and not part of any hosted day",
+    ),
+    "fleet": Lane(HOSTED, "it says which hosted runs are waiting for a person"),
+    "index": Lane(LOCAL, "it builds a private search index over Builds on this computer"),
+    "init": Lane(LOCAL, "it makes a workbench folder on this computer"),
+    "inspect": Lane(HOSTED, "it shows the sealed evidence Studio holds for one hosted run"),
+    "inventory": Lane(
+        LOCAL,
+        "it lists the operations, types, joins and engines this installation can run, and a "
+        "hosted profile runs none of them",
+    ),
+    "list": Lane(HOSTED, "it lists the hosted runs in this workspace"),
+    "login": Lane(PORTABLE, "sign in and store a device credential"),
+    "note": Lane(
+        HOSTED,
+        "it writes one cell of the operator-facing decision record into the hosted run's own "
+        "log, where the notebook and the live view both read it; it is prose, not evidence",
+    ),
+    "notebook": Lane(HOSTED, "it brings back the notebook Studio rendered for one hosted run"),
+    "peek": Lane(HOSTED, "it shows the columns, types and first rows of one hosted table version"),
+    "plan": Lane(
+        LOCAL,
+        "it checks and locks the plan in a workbench folder on this computer; the hosted lane "
+        "answers this with `propose`, whose table plan is created against Studio directly",
+    ),
+    "plan-check": Lane(
+        LOCAL,
+        "it builds a plan into a throwaway folder on this computer to see whether it works",
+    ),
+    "preflight": Lane(
+        LOCAL,
+        "it reports what would stop a build on this computer, which is the one question a hosted "
+        "profile never has to ask",
+    ),
+    "propose": Lane(
+        HOSTED,
+        "it authors one hosted recipe proposal from a bundle of documents -- the Dataset, "
+        "question, sources, table plan and the recipe -- and opens the approval a person settles, "
+        "so a hosted build has a proposal to name; it asks and never grants",
+    ),
+    "recipe-approve": Lane(
+        HOSTED,
+        "it is the same ask about one Table recipe approval, which a signed-in editor confirms in "
+        "the dashboard rather than under a step-up; the hosted lane asks and never grants",
+    ),
+    "recipe-export": Lane(LOCAL, "it writes a checked Recipe to a file on this computer"),
+    "recipe-run": Lane(
+        LOCAL,
+        "it executes a Recipe on this computer; `mr-data run --hosted` is the hosted lane's answer "
+        "to the same question and it is already here",
+    ),
+    "recipe-validate": Lane(
+        LOCAL, "it checks and fingerprints a Recipe file on this computer, reaching nothing"
+    ),
+    "recipe-verify": Lane(
+        LOCAL, "it re-checks a Recipe version against inputs pinned on this computer"
+    ),
+    "record": Lane(LOCAL, "it seals messages already recorded on this computer into batches"),
+    "research-close": Lane(
+        HOSTED,
+        "it closes one hosted research session, cancelling the probes still queued, so its "
+        "worker exits and the service behind it scales back to zero",
+    ),
+    "research-open": Lane(
+        HOSTED,
+        "it opens one hosted research session against a Dataset and a question, and shows the "
+        "worker warming up rather than hiding the cold start",
+    ),
+    "research-probe": Lane(
+        HOSTED,
+        "it asks a research session's warm worker one question about a source, follows the "
+        "session's own event stream until the probe settles, and prints the answer; the answer is "
+        "exploration, never evidence",
+    ),
+    "research-status": Lane(
+        HOSTED,
+        "it reports Studio's own reading of one research session, and of one probe with its "
+        "answer when asked",
+    ),
+    "resume": Lane(LOCAL, "it picks up an interrupted build in a folder on this computer"),
+    "review": Lane(
+        LOCAL,
+        "it seals a review decision from reports on this computer; hosted review decisions are "
+        "Studio's approvals surface and carry its step-up requirement",
+    ),
+    "review-setup": Lane(LOCAL, "it creates protected review material on this computer"),
+    "run": Lane(HOSTED, "it queues one hosted build and says where to watch it"),
+    "search": Lane(LOCAL, "it searches stores published or indexed on this computer"),
+    "serve": Lane(LOCAL, "it serves a dataset from this computer on a local address"),
+    "show": Lane(HOSTED, "it shows what one hosted run holds"),
+    "sign-review": Lane(LOCAL, "it signs a reviewer report with a key held on this computer"),
+    "source-audit": Lane(
+        LOCAL, "it checks proposed sources against the contracts compiled into this installation"
+    ),
+    "status": Lane(HOSTED, "it reports Studio's own reading of one hosted run, with its receipts"),
+    "verify": Lane(HOSTED, "it re-checks one hosted run against the evidence Studio sealed"),
+    "view": Lane(HOSTED, "it opens the notebook Studio rendered for one hosted run"),
+    "watch": Lane(HOSTED, "it streams one hosted run's live progress, resuming across stream cuts"),
+    "whoami": Lane(PORTABLE, "it is the compatibility alias for remotely truthful `auth status`"),
+    "workflow": Lane(
+        LOCAL,
+        "it prints the local build workflow and who decides at each step; the hosted workflow is "
+        "the dashboard's",
+    ),
+}
+
+#: Every command name and its lane. The half of :data:`CLASSIFICATION` the router branches on,
+#: derived rather than restated so the two can never disagree.
+VOCABULARY: dict[str, str] = {name: item.lane for name, item in CLASSIFICATION.items()}
+
+
+def names_in(lane: str) -> tuple[str, ...]:
+    """Every command in one lane, sorted."""
+
+    return tuple(sorted(name for name, item in CLASSIFICATION.items() if item.lane == lane))
+
+
+def reason_for(command: str) -> str:
+    """The written reason one command runs where it runs."""
+
+    return CLASSIFICATION[command].reason
+
+
+# --------------------------------------------------------------------------------------------
+# The other six executables
+# --------------------------------------------------------------------------------------------
+#
+# `mr-data` is not the only console script this distribution installs. The six below are the
+# backend's own executables: they are started by the worker image with a job descriptor and a
+# capability descriptor, they speak the fenced internal protocol, and none of them has any meaning
+# on a laptop. On a thin install every one of them used to end in a traceback out of `pyarrow` or
+# `cryptography` before its first line ran, because the module holding the entry point imports
+# both at module scope.
+#
+# ⚠ THE GUARD CANNOT LIVE IN THOSE MODULES. A check inside `hosted_worker.producer_main` is
+# unreachable on a thin install: the import that fails is the one that loads the module the
+# function is defined in. So the console script points at a shim that imports nothing until it has
+# decided, and `hosted_worker.py` is not touched at all -- which is also what keeps this out of the
+# way of the lane editing that file this run.
+
+#: Console script name -> (module, attribute) it delegates to once the local lane is present, and
+#: the sentence that says what the executable is for.
+WORKER_ENTRY_POINTS: dict[str, tuple[str, str, str]] = {
+    "mr-data-bootstrap": (
+        "mostlyright.data_harness.hosted_bootstrap",
+        "main",
+        "prepares a hosted worker's execution environment before a job runs",
+    ),
+    "mr-data-catalog-job": (
+        "mostlyright.data_harness.catalog_job",
+        "catalog_job_main",
+        "runs one stage of the sealed source catalogue's harvest, author and publish pipeline",
+    ),
+    "mr-data-coordinator": (
+        "mostlyright.data_harness.coordinator",
+        "coordinator_main",
+        "runs the hosted work loop that claims and executes attempts",
+    ),
+    "mr-data-crawler": (
+        "mostlyright.data_harness.hosted_crawler",
+        "crawler_main",
+        "runs the isolated hosted fetcher that holds no credential",
+    ),
+    "mr-data-crawler-job": (
+        "mostlyright.data_harness.hosted_crawler_job",
+        "crawler_job_main",
+        "runs one isolated hosted fetch job to completion",
+    ),
+    "mr-data-producer": (
+        "mostlyright.data_harness.hosted_worker",
+        "producer_main",
+        "runs one hosted build attempt inside the backend's worker image",
+    ),
+    "mr-data-verifier": (
+        "mostlyright.data_harness.hosted_worker",
+        "verifier_main",
+        "independently re-checks one hosted build attempt inside the backend's worker image",
+    ),
+    "mr-data-session-worker": (
+        "mostlyright.data_harness.hosted_session_worker",
+        "session_worker_main",
+        "serves one warm research session inside the backend's Cloud Run session-worker service",
+    ),
+    "mr-data-session-worker-job": (
+        "mostlyright.data_harness.hosted_session_worker",
+        "session_worker_job_main",
+        "runs one research session to terminal state inside a backend Cloud Run Job task",
+    ),
+    "mr-data-ingestion-job": (
+        "mostlyright.data_harness.hosted_ingestion_job",
+        "ingestion_job_main",
+        "runs one fenced resumable-ingestion delivery inside a backend Cloud Run Job task",
+    ),
+}
+
+
+__all__ = [
+    "CLASSIFICATION",
+    "HOSTED",
+    "HOSTED_PENDING",
+    "LANES",
+    "LOCAL",
+    "PORTABLE",
+    "VOCABULARY",
+    "WORKER",
+    "WORKER_ENTRY_POINTS",
+    "Lane",
+    "names_in",
+    "reason_for",
+]

mostlyright/data_harness/thin/workers.py

@@ -0,0 +1,164 @@
+"""The backend executables' front doors, so a thin install answers instead of crashing.
+
+``mr-data-producer``, ``mr-data-verifier``, ``mr-data-coordinator``, ``mr-data-bootstrap``,
+``mr-data-crawler``, ``mr-data-crawler-job``, ``mr-data-catalog-job``,
+``mr-data-session-worker``, ``mr-data-session-worker-job`` and ``mr-data-ingestion-job`` are the
+backend's own executables.
+Most are started by the worker image with a job descriptor and a capability descriptor on numbered
+file descriptors. The session-worker Job instead receives only an opaque dispatch identifier and
+Studio exchange coordinates, then obtains its fenced lease through the authenticated exchange;
+it receives no capability descriptor. Nothing about either form makes sense on a laptop, and ADR
+0021 does not give them a hosted lane because they ARE the hosted lane.
+
+What they should not do is end in a traceback. Every one of those entry points is defined in a
+module that imports ``pyarrow`` or ``cryptography`` at module scope, so on a thin install the
+failure happened during the import that loaded the entry point itself -- before any code of ours
+could say anything. PR #308 recorded that as a known gap and left it, because closing it looked
+like it meant editing ``hosted_worker.py``.
+
+It does not. The console script points HERE instead, and this module imports nothing at all until
+it has decided which of the two answers it owes:
+
+* with the ``[local]`` extra present, it resolves the real entry point and hands the arguments
+  over unchanged -- byte-identical behaviour, one extra function call in the path;
+* without it, it prints one typed sentence naming the executable and what it is for, on stderr,
+  through the same error map every other ``mr-data`` refusal goes through, and exits 2.
+
+``hosted_worker.py`` is not touched by any of this, which is what a reviewer should check first.
+"""
+
+from __future__ import annotations
+
+import importlib
+import json
+import sys
+from collections.abc import Sequence
+
+from mostlyright.data_harness.thin.vocabulary import WORKER_ENTRY_POINTS
+
+#: The code every one of these refusals carries. One code, not six: the answer is the same answer
+#: -- this executable belongs to the backend -- and six codes would be six ways to say it.
+WORKER_ENTRY_POINT_CODE = "THIN_WORKER_ENTRY_POINT_IS_LOCAL"
+
+
+def _refuse(executable: str, purpose: str, argv: Sequence[str]) -> int:
+    """Say what this executable is, where it runs, and what to install to run it here."""
+
+    from mostlyright.data_harness.thin.router import LOCAL_INSTALL_HINT
+    from mostlyright.data_harness.ux.remediation import remediation_for
+    from mostlyright.data_harness.ux.render import render_error
+
+    detail = (
+        f"{executable} is a backend executable -- it {purpose} -- and this installation is the "
+        f"hosted profile, which carries no execution lane; {LOCAL_INSTALL_HINT}"
+    )
+    remediation = remediation_for(WORKER_ENTRY_POINT_CODE)
+    if "--json" in argv:
+        print(
+            json.dumps(
+                {
+                    "schema_version": "mr-data-error.v1",
+                    "code": WORKER_ENTRY_POINT_CODE,
+                    "error": detail,
+                    "headline": detail,
+                    "remediation": list(remediation),
+                },
+                ensure_ascii=False,
+                sort_keys=True,
+            ),
+            file=sys.stderr,
+            flush=True,
+        )
+    else:
+        print(
+            render_error(detail, code=WORKER_ENTRY_POINT_CODE, remediation=remediation),
+            file=sys.stderr,
+            flush=True,
+        )
+    return 2
+
+
+def _dispatch(executable: str, argv: Sequence[str] | None) -> int:
+    """Hand one backend executable's arguments to its real entry point, or refuse by name."""
+
+    from mostlyright.data_harness.thin.router import local_lane_available
+
+    module_name, attribute, purpose = WORKER_ENTRY_POINTS[executable]
+    arguments = list(sys.argv[1:] if argv is None else argv)
+    if not local_lane_available():
+        return _refuse(executable, purpose, arguments)
+    entry_point = getattr(importlib.import_module(module_name), attribute)
+    return int(entry_point(arguments))
+
+
+def bootstrap_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-bootstrap", argv)
+
+
+def catalog_job_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-catalog-job", argv)
+
+
+def coordinator_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-coordinator", argv)
+
+
+def crawler_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-crawler", argv)
+
+
+def crawler_job_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-crawler-job", argv)
+
+
+def producer_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-producer", argv)
+
+
+def verifier_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-verifier", argv)
+
+
+def session_worker_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-session-worker", argv)
+
+
+def session_worker_job_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-session-worker-job", argv)
+
+
+def ingestion_job_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-ingestion-job", argv)
+
+
+#: Console script name -> the function in this module that backs it. Held equal to
+#: ``pyproject.toml``'s ``[project.scripts]`` by ``tests/test_thin_client.py``, so a backend
+#: executable cannot be added on one side alone.
+SHIMS = {
+    "mr-data-bootstrap": bootstrap_main,
+    "mr-data-catalog-job": catalog_job_main,
+    "mr-data-coordinator": coordinator_main,
+    "mr-data-crawler": crawler_main,
+    "mr-data-crawler-job": crawler_job_main,
+    "mr-data-producer": producer_main,
+    "mr-data-verifier": verifier_main,
+    "mr-data-session-worker": session_worker_main,
+    "mr-data-session-worker-job": session_worker_job_main,
+    "mr-data-ingestion-job": ingestion_job_main,
+}
+
+
+__all__ = [
+    "SHIMS",
+    "WORKER_ENTRY_POINT_CODE",
+    "bootstrap_main",
+    "catalog_job_main",
+    "coordinator_main",
+    "crawler_job_main",
+    "crawler_main",
+    "ingestion_job_main",
+    "producer_main",
+    "session_worker_job_main",
+    "session_worker_main",
+    "verifier_main",
+]

mostlyright/data_harness/ucum/TABLE-PIN.json

@@ -0,0 +1,40 @@
+{
+  "schema": "ucum-pin.v1",
+  "covers": "src/mostlyright/data_harness/ucum",
+  "standard": "UCUM",
+  "standard_version": "2.2",
+  "standard_release": "2024-06-28",
+  "specification": "https://ucum.org/ucum",
+  "subset": "docs/UNITS.md",
+  "canonical_tree_digest": {
+    "algorithm": "sha256",
+    "rule": [
+      "Collect every regular file under the covered path except this one.",
+      "Order them by their path relative to that root, compared as a tuple of path components.",
+      "Fold each file into one running sha256: the relative POSIX path as UTF-8, one NUL byte, then the sha256 digest of the file's bytes.",
+      "The hexadecimal digest of the running sha256 is the tree digest."
+    ],
+    "note": "The same rule vendor/VENDOR.lock.json states, and TABLE-PIN.json is outside it because a digest cannot cover the file recording it."
+  },
+  "artifacts": [
+    {
+      "path": "ucum-subset.v1.json",
+      "sha256": "24caf113858cd4fb58dfb800e1b0a04236d5718e68073607d3fba30227e35d23",
+      "bytes": 14237
+    }
+  ],
+  "tree_sha256": "4e3301c3f46ee162bfd006e92d2bdaa89e74cafe6228064d0b0871d702365a12",
+  "file_count": 1,
+  "consumers": [
+    {
+      "name": "mostlyright.data_harness.units",
+      "note": "Reads the table once per process and resolves every declared unit against it. Nothing else opens these files at run time."
+    }
+  ],
+  "update_procedure": [
+    "State the change in docs/UNITS.md first: the subset is a documented contract, not whatever the table happens to hold.",
+    "Add or amend the atom with both its recorded definition and its recorded factor, so the two can be checked against each other.",
+    "Recompute the digests here, and the file count with them. Do not update one alone.",
+    "Run `uv run --extra dev python -m pytest -q tests/test_units_table.py tests/test_units.py`."
+  ]
+}