mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/commands/catalog_harvest.py

@@ -0,0 +1,545 @@
+"""``mr-data catalog-harvest``: stage one keyed provider sweep, or just prove the key boundary.
+
+This is the first of the three explicit provider-scale mutations.  It reaches exactly one
+endpoint, authorises with a key it reads out of a file the caller names and never keeps, and
+writes a private staging root plus one secret-free receipt.  It publishes no catalog: authoring
+and publication are separate commands with separate receipts, so a sweep that stopped part-way
+cannot be mistaken for a generation.
+
+``--preflight-only`` is the whole of the key boundary and nothing else.  It opens the named file
+through a retained no-follow parent chain, checks ownership, link count, mode, size and
+before/after identity, and answers whether the file was accepted.  It never prints the key, never
+prints the path, and never makes a request.
+
+The receipt is the resume journal.  An invocation stopped by ``--max-pages-this-invocation``
+installs a typed-incomplete receipt carrying the observation timestamp, the retained checkpoint
+digest and this invocation's page count; the exact same argv with ``--resume`` inserted reads that
+receipt back, continues from the retained cursor, and replaces it.  The invocation cap itself is
+recorded as invocation evidence and is deliberately not part of the durable semantic configuration
+digest, so a resume may drop it while every API parameter and cumulative bound stays exact.
+
+``--pace-in-process`` changes who waits out the provider's minimum request interval, never how
+long that wait is.  Without it, an invocation that has committed a page hands the remaining wait
+back and stops, and something outside has to resume it; a paced sweep of any size is therefore a
+sweep of one-page invocations, and each one re-authenticates the whole evidence tree it is
+resuming into, so the cost of the sweep grows with the square of the records already staged.
+With it, the same interval is waited out here and the walk continues, so that authentication is
+paid once for the invocation.  The page commit and the resume journal are untouched either way:
+a paced sweep killed mid-wait resumes from exactly the checkpoint an unpaced one would leave.
+"""
+
+from __future__ import annotations
+
+import argparse
+import time
+from collections.abc import Mapping
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+from mostlyright.data_harness.acquisition.http import (
+    DATAGOV_V4_AUTHENTICATION_KIND,
+    DatagovV4Authorization,
+    PinnedHttpsRetriever,
+    RetrievalLimits,
+    StdlibPinnedTransport,
+)
+from mostlyright.data_harness.acquisition.url_policy import EgressPolicy, SystemResolver
+from mostlyright.data_harness.sources.catalog.bounded_io import (
+    BoundedReadFailure,
+    measure_bounded_tree,
+)
+from mostlyright.data_harness.sources.catalog.fill import (
+    DATAGOV_V4_SORT_ORDERS,
+    CatalogFillConfig,
+    CatalogFillLimits,
+    run_catalog_fill,
+)
+from mostlyright.data_harness.sources.catalog.generation_receipt import (
+    HARVEST_PREFLIGHT_SCHEMA,
+    HARVEST_RECEIPT_SCHEMA,
+    MAX_STAGE_RECEIPT_BYTES,
+    ReceiptInstallation,
+    install_receipt,
+    load_execution_coordinate,
+    publication_output_lock,
+    receipt_installation_lock,
+    require_output_root_identity,
+    retained_incomplete_receipt,
+    root_coordinate,
+)
+from mostlyright.data_harness.sources.catalog.harvest.datagov_v4 import DATAGOV_V4_ENDPOINT
+from mostlyright.data_harness.sources.contracts import SourceContractError
+from mostlyright.data_harness.ux.commands import CommandSpec
+
+#: The one provider this command harvests, and the sort orders its cursor contract admits.
+PROVIDERS = ("datagov_v4",)
+SORT_ORDERS = DATAGOV_V4_SORT_ORDERS
+
+HARVEST_INCOMPLETE_STATUS = "harvest_incomplete"
+HARVEST_COMPLETE_STATUS = "harvest_complete"
+
+#: Bounds this command owns rather than passes through, so a caller cannot state one twice.
+MAX_DISK_SCAN_MEMBERS = 5_000_000
+
+#: The flags the preflight form of this command forbids: it produces nothing and spends nothing.
+_PREFLIGHT_FORBIDDEN = (
+    "output_root",
+    "receipt_file",
+    "execution_manifest",
+    "execution_commit",
+    "per_page",
+    "sort",
+    "max_requests",
+    "max_responses",
+    "max_pages",
+    "max_records",
+    "max_response_bytes",
+    "max_network_bytes",
+    "max_disk_bytes",
+    "max_wall_seconds",
+    "min_request_interval_ms",
+    "max_convergence_passes",
+    "max_conflicting_identifiers",
+    "max_pages_this_invocation",
+    "pace_in_process",
+    "resume",
+)
+
+#: Everything a real harvest must state before it makes one request.
+_REQUIRED = (
+    "output_root",
+    "receipt_file",
+    "execution_manifest",
+    "execution_commit",
+    "per_page",
+    "sort",
+    "max_requests",
+    "max_responses",
+    "max_pages",
+    "max_records",
+    "max_response_bytes",
+    "max_network_bytes",
+    "max_disk_bytes",
+    "max_wall_seconds",
+    "min_request_interval_ms",
+    "max_convergence_passes",
+)
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--provider",
+        choices=PROVIDERS,
+        required=True,
+        help="the closed keyed public catalog provider",
+    )
+    parser.add_argument(
+        "--api-key-file",
+        type=Path,
+        required=True,
+        help="the path of a file you own holding one key line; it is read, never kept or printed",
+    )
+    parser.add_argument(
+        "--preflight-only",
+        action="store_true",
+        help="check the key file boundary and answer; make no request and write nothing",
+    )
+    parser.add_argument(
+        "--output-root", type=Path, help="the private staging root this sweep is written into"
+    )
+    parser.add_argument("--receipt-file", type=Path, help="where this stage's receipt is installed")
+    parser.add_argument(
+        "--execution-manifest", type=Path, help="the committed execution coordinate document"
+    )
+    parser.add_argument("--execution-commit", help="the exact commit that coordinate was cut at")
+    parser.add_argument("--per-page", type=int, help="records requested per provider page")
+    parser.add_argument(
+        "--sort", choices=SORT_ORDERS, help="the closed provider sort the cursor contract requires"
+    )
+    parser.add_argument(
+        "--max-conflicting-identifiers",
+        type=int,
+        help=(
+            "how many identifiers the provider may publish twice with different content before "
+            "the sweep refuses to converge (default 0). Each one is still resolved "
+            "deterministically and the count is recorded in the receipt, so a catalogue can "
+            "never hide them"
+        ),
+    )
+    parser.add_argument("--max-requests", type=int, help="maximum aggregate network requests")
+    parser.add_argument("--max-responses", type=int, help="maximum aggregate network responses")
+    parser.add_argument("--max-pages", type=int, help="maximum aggregate provider pages")
+    parser.add_argument("--max-records", type=int, help="maximum aggregate provider records")
+    parser.add_argument("--max-response-bytes", type=int, help="maximum bytes in one response")
+    parser.add_argument("--max-network-bytes", type=int, help="maximum bytes across every response")
+    parser.add_argument(
+        "--max-disk-bytes", type=int, help="maximum bytes this sweep may leave in its staging root"
+    )
+    parser.add_argument("--max-wall-seconds", type=int, help="maximum cumulative retrieval seconds")
+    parser.add_argument(
+        "--min-request-interval-ms",
+        type=int,
+        help="smallest gap between two requests, in milliseconds",
+    )
+    parser.add_argument(
+        "--max-convergence-passes",
+        type=int,
+        help="maximum complete passes allowed to reach convergence",
+    )
+    parser.add_argument(
+        "--max-pages-this-invocation",
+        type=int,
+        help="stop this process after this many newly committed pages and leave a resumable stop",
+    )
+    parser.add_argument(
+        "--pace-in-process",
+        action="store_true",
+        help="wait the request interval out here and keep going, instead of stopping at each page",
+    )
+    parser.add_argument(
+        "--resume",
+        action="store_true",
+        help="continue the sweep this receipt file was left incomplete at",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    if args.preflight_only:
+        return _preflight(args)
+    _require_full_form(args)
+    output_root = Path(args.output_root)
+    with publication_output_lock(
+        output_root,
+        label="output_root",
+        code_prefix="HARVEST",
+        refusal=SourceContractError,
+    ) as output_descriptor:
+        with receipt_installation_lock(
+            args.receipt_file,
+            label="receipt_file",
+            code_prefix="HARVEST",
+            refusal=SourceContractError,
+        ) as receipt_installation:
+            return _run_locked(
+                args,
+                output_root=output_root,
+                output_descriptor=output_descriptor,
+                receipt_installation=receipt_installation,
+            )
+
+
+def _run_locked(
+    args: argparse.Namespace,
+    *,
+    output_root: Path,
+    output_descriptor: int,
+    receipt_installation: ReceiptInstallation,
+) -> dict[str, Any]:
+    execution = load_execution_coordinate(args.execution_manifest, args.execution_commit)
+    retained = None
+    if args.resume:
+        retained = retained_incomplete_receipt(
+            args.receipt_file,
+            maximum=MAX_STAGE_RECEIPT_BYTES,
+            label="receipt_file",
+            schema=HARVEST_RECEIPT_SCHEMA,
+            incomplete_status=HARVEST_INCOMPLETE_STATUS,
+            output_root=output_root,
+            code_prefix="HARVEST",
+            refusal=SourceContractError,
+        )
+        if retained is None:
+            raise SourceContractError(
+                "HARVEST_RESUME_RECEIPT",
+                "receipt_file",
+                "a resume continues the typed-incomplete receipt its interruption left behind",
+            )
+        if retained["execution"]["root_sha256"] != execution["root_sha256"]:
+            raise SourceContractError(
+                "HARVEST_RESUME_RECEIPT",
+                "execution_manifest",
+                "the retained sweep was harvested from another execution coordinate",
+            )
+    observed_at = _observed_at(retained)
+    config = _config(args, observed_at)
+    authorization = DatagovV4Authorization.from_key_file(args.api_key_file)
+    result = run_catalog_fill(
+        config=config,
+        staging_root=output_root,
+        retriever=_retriever(args),
+        expected_checkpoint_digest=None if retained is None else retained["checkpoint_sha256"],
+        max_pages_this_invocation=args.max_pages_this_invocation,
+        authorization=authorization,
+        staging_descriptor=output_descriptor,
+        pacing_sleep=time.sleep if args.pace_in_process else None,
+    )
+    del authorization
+    disk_bytes = _disk_bytes(output_descriptor, maximum_bytes=args.max_disk_bytes)
+    receipt = result.receipt.to_dict()
+    complete = result.status == "complete"
+    previous_pages = 0 if retained is None else retained["invocation"]["cumulative_pages"]
+    payload: dict[str, Any] = {
+        "schema_version": HARVEST_RECEIPT_SCHEMA,
+        "status": HARVEST_COMPLETE_STATUS if complete else HARVEST_INCOMPLETE_STATUS,
+        "reason_code": result.reason_code,
+        "provider": args.provider,
+        "endpoint": config.endpoint,
+        "harvester_coordinate": config.coordinate,
+        "authentication_kind": DATAGOV_V4_AUTHENTICATION_KIND,
+        "observed_at": observed_at,
+        "checkpoint_sha256": result.checkpoint_sha256,
+        "config_sha256": receipt["config_sha256"],
+        "staging_sha256": result.eligible_staging_sha256,
+        "output_root_sha256": root_coordinate(output_root),
+        "output_bytes": disk_bytes,
+        "fill_receipt": receipt,
+        "fill_receipt_sha256": result.receipt.digest,
+        "measures": {
+            "requests": receipt["requests"],
+            "responses": receipt["responses"],
+            "pages": receipt["pages"],
+            "provider_records": receipt["unique_records"],
+        },
+        "sort": config.sort,
+        "conflicting_identifiers": receipt.get("conflicting_identifiers"),
+        "invocation": {
+            "max_pages_this_invocation": args.max_pages_this_invocation,
+            "paced_in_process": bool(args.pace_in_process),
+            "pages_this_invocation": receipt["pages"] - previous_pages,
+            "cumulative_pages": receipt["pages"],
+            "resumed": bool(args.resume),
+        },
+        "execution": execution,
+        "limits": _limits_evidence(args),
+    }
+
+    def commit_guard() -> None:
+        require_output_root_identity(
+            output_root,
+            output_descriptor,
+            label="output_root",
+            code_prefix="HARVEST",
+            refusal=SourceContractError,
+        )
+        if _disk_bytes(output_descriptor, maximum_bytes=args.max_disk_bytes) != disk_bytes:
+            raise SourceContractError(
+                "HARVEST_DISK_LIMIT",
+                "output_root",
+                "the retained staging tree changed while its receipt was installed",
+            )
+
+    install_receipt(
+        args.receipt_file,
+        payload,
+        maximum=MAX_STAGE_RECEIPT_BYTES,
+        label="receipt_file",
+        code_prefix="HARVEST",
+        refusal=SourceContractError,
+        replaces=retained,
+        installation=receipt_installation,
+        commit_guard=commit_guard,
+    )
+    return payload
+
+
+# ------------------------------------------------------------------------------------------------
+# The key-file boundary
+# ------------------------------------------------------------------------------------------------
+
+
+def _require_preflight_form(args: argparse.Namespace) -> None:
+    """Refuse a preflight that states an argument the preflight form does not take.
+
+    Separated from :func:`_preflight` so the refusal is reachable without reading the key file
+    or reaching a provider: a spec that cannot run must be refusable by a deployment check.
+    """
+
+    stated = [name for name in _PREFLIGHT_FORBIDDEN if _stated(args, name)]
+    if stated:
+        raise SourceContractError(
+            "HARVEST_ARGUMENTS",
+            stated[0],
+            "the preflight form checks the key boundary and takes no output, execution or budget "
+            f"argument, and this one states {', '.join(sorted(stated))}",
+        )
+
+
+def validate(args: argparse.Namespace) -> None:
+    """Refuse an argument set this command would refuse, without running it.
+
+    ``run`` calls the same two refusals on the same two branches, so a namespace this accepts is
+    one ``run`` reaches its first side effect with, and a namespace this refuses is one ``run``
+    refuses identically. That equivalence is the whole point: it lets ``mr-data-catalog-job
+    --dry-run`` answer "would this spec run?" rather than only "does argparse accept it?".
+    """
+
+    if args.preflight_only:
+        _require_preflight_form(args)
+        return
+    _require_full_form(args)
+
+
+def _preflight(args: argparse.Namespace) -> dict[str, Any]:
+    _require_preflight_form(args)
+    authorization = DatagovV4Authorization.from_key_file(args.api_key_file)
+    del authorization
+    return {
+        "schema_version": HARVEST_PREFLIGHT_SCHEMA,
+        "status": "harvest_preflight_accepted",
+        "provider": args.provider,
+        "endpoint": DATAGOV_V4_ENDPOINT,
+        "authentication_kind": DATAGOV_V4_AUTHENTICATION_KIND,
+        "key_file_boundary": "accepted",
+    }
+
+
+def _stated(args: argparse.Namespace, name: str) -> bool:
+    # Identity, not equality. `0 == False` in Python, so `value not in (None, False)` reported
+    # `--per-page 0` and `--min-request-interval-ms 0` as unstated and let them through the
+    # preflight refusal they are forbidden by.
+    value = getattr(args, name, None)
+    return value is not None and value is not False
+
+
+def _require_full_form(args: argparse.Namespace) -> None:
+    missing = [name for name in _REQUIRED if getattr(args, name, None) is None]
+    if missing:
+        raise SourceContractError(
+            "HARVEST_ARGUMENTS",
+            missing[0],
+            "a harvest states its output, its execution coordinate and every budget before it "
+            "makes one request",
+        )
+    for name in (
+        "per_page",
+        "max_requests",
+        "max_responses",
+        "max_pages",
+        "max_records",
+        "max_response_bytes",
+        "max_network_bytes",
+        "max_disk_bytes",
+        "max_wall_seconds",
+        "max_convergence_passes",
+    ):
+        value = getattr(args, name)
+        if type(value) is not int or value < 1:
+            raise SourceContractError(
+                "HARVEST_ARGUMENTS", name, "must be a positive canonical decimal integer"
+            )
+    if type(args.min_request_interval_ms) is not int or args.min_request_interval_ms < 0:
+        raise SourceContractError(
+            "HARVEST_ARGUMENTS", "min_request_interval_ms", "must be a non-negative integer"
+        )
+    if args.max_pages_this_invocation is not None and (
+        type(args.max_pages_this_invocation) is not int or args.max_pages_this_invocation < 1
+    ):
+        raise SourceContractError(
+            "HARVEST_ARGUMENTS", "max_pages_this_invocation", "must be a positive integer"
+        )
+
+
+# ------------------------------------------------------------------------------------------------
+# Configuration, transport, and the two things this command measures for itself
+# ------------------------------------------------------------------------------------------------
+
+
+def _config(args: argparse.Namespace, observed_at: str) -> CatalogFillConfig:
+    return CatalogFillConfig(
+        protocol=args.provider,
+        endpoint=DATAGOV_V4_ENDPOINT,
+        observed_at=observed_at,
+        page_size=args.per_page,
+        max_convergence_passes=args.max_convergence_passes,
+        sort=args.sort,
+        max_conflicting_identifiers=args.max_conflicting_identifiers or 0,
+        authentication_kind=DATAGOV_V4_AUTHENTICATION_KIND,
+        minimum_request_interval_ns=args.min_request_interval_ms * 1_000_000,
+        limits=CatalogFillLimits(
+            max_requests=args.max_requests,
+            max_pages=args.max_pages,
+            max_records=args.max_records,
+            max_response_bytes=args.max_response_bytes,
+            max_aggregate_bytes=args.max_network_bytes,
+            max_responses=args.max_responses,
+            max_wall_time_ns=args.max_wall_seconds * 1_000_000_000,
+        ),
+    )
+
+
+def _retriever(args: argparse.Namespace) -> Any:
+    """The one pinned transport this command reaches the provider through."""
+
+    hostname = urlsplit(DATAGOV_V4_ENDPOINT).hostname
+    assert hostname is not None
+    return PinnedHttpsRetriever(
+        resolver=SystemResolver(),
+        egress_policy=EgressPolicy(allowed_hostnames=(hostname,)),
+        transport=StdlibPinnedTransport(),
+        limits=RetrievalLimits(
+            max_response_bytes=args.max_response_bytes,
+            max_aggregate_response_bytes=min(args.max_network_bytes, 1 << 40),
+            max_redirects=0,
+            max_requests=16,
+        ),
+    )
+
+
+def _observed_at(retained: Mapping[str, Any] | None) -> str:
+    """The sweep's observation timestamp: minted once, then carried by the receipt it left."""
+
+    if retained is not None:
+        return str(retained["observed_at"])
+    return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _disk_bytes(root_descriptor: int, *, maximum_bytes: int) -> int:
+    """Bytes held by one retained tree, measured incrementally and fail-closed."""
+
+    try:
+        total, _members = measure_bounded_tree(
+            root_descriptor,
+            maximum_members=MAX_DISK_SCAN_MEMBERS,
+            maximum_bytes=maximum_bytes,
+        )
+        return total
+    except BoundedReadFailure as error:
+        raise SourceContractError(
+            "HARVEST_DISK_LIMIT",
+            "output_root",
+            "the retained staging tree is unsafe, changed, or exceeds its stated bounds",
+        ) from error
+
+
+def _limits_evidence(args: argparse.Namespace) -> dict[str, int]:
+    return {
+        "per_page": args.per_page,
+        "max_requests": args.max_requests,
+        "max_responses": args.max_responses,
+        "max_pages": args.max_pages,
+        "max_records": args.max_records,
+        "max_response_bytes": args.max_response_bytes,
+        "max_network_bytes": args.max_network_bytes,
+        "max_disk_bytes": args.max_disk_bytes,
+        "max_wall_seconds": args.max_wall_seconds,
+        "min_request_interval_ms": args.min_request_interval_ms,
+        "max_convergence_passes": args.max_convergence_passes,
+    }
+
+
+def _exit_code(payload: Mapping[str, Any]) -> int:
+    return (
+        0 if payload.get("status") in {HARVEST_COMPLETE_STATUS, "harvest_preflight_accepted"} else 1
+    )
+
+
+SPEC = CommandSpec(
+    name="catalog-harvest",
+    help="stage one keyed public catalog sweep, or just check the key file boundary",
+    add_arguments=add_arguments,
+    run=run,
+    exit_code=_exit_code,
+    validate=validate,
+)