mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/commands/catalog_fill.py

@@ -0,0 +1,523 @@
+"""``mr-data catalog-fill``: bounded staging followed by one complete publication.
+
+This is the small-catalog command: one invocation stages a whole public catalog and publishes one
+successor from it, with the authored entries handed in as one document.
+
+A keyed provider-scale catalog is not filled this way and this command refuses to try. That work
+is three separate commands with three separate receipts -- ``catalog-harvest``, then
+``catalog-author``, then ``catalog-publish`` -- because a sweep of several hundred thousand
+records is interrupted and resumed rather than run once, and because the key that reaches it
+belongs to exactly one of those three.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import stat
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+from mostlyright.data_harness.acquisition.http import (
+    PinnedHttpsRetriever,
+    RetrievalLimits,
+    StdlibPinnedTransport,
+)
+from mostlyright.data_harness.acquisition.url_policy import EgressPolicy, SystemResolver
+from mostlyright.data_harness.canonical import parse_canonical_json, sha256_bytes
+from mostlyright.data_harness.sources.catalog.contracts import (
+    CatalogEntry,
+    catalog_entry_from_dict,
+)
+from mostlyright.data_harness.sources.catalog.delta import (
+    checkpoint_catalog_delta_classification,
+    classify_catalog_delta,
+    publish_retrieval_catalog_successor,
+)
+from mostlyright.data_harness.sources.catalog.embedding import LexicalHashingBackend
+from mostlyright.data_harness.sources.catalog.fill import (
+    CatalogFillConfig,
+    CatalogFillLimits,
+    run_catalog_fill,
+)
+from mostlyright.data_harness.sources.catalog.retrieval_manifest import load_retrieval_catalog
+from mostlyright.data_harness.sources.catalog.rights_decisions import load_rights_decisions
+from mostlyright.data_harness.sources.contracts import SourceContractError
+from mostlyright.data_harness.ux.commands import CommandSpec
+
+_AUTHORING_SCHEMA = "mr-data-catalog-fill-authoring.v1"
+_MAX_AUTHORING_BYTES = 64 * 1024 * 1024
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+
+
+@dataclass(frozen=True)
+class _AuthoringBundle:
+    candidates_by_record_id: dict[str, CatalogEntry]
+    predecessor_record_ids: dict[str, str]
+    predecessor_chains: dict[str, tuple[CatalogEntry, ...]]
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--protocol",
+        choices=("ckan", "stac", "sdmx"),
+        required=True,
+        help=(
+            "the closed public catalog protocol; a keyed provider is harvested, authored and "
+            "published as three separate commands instead"
+        ),
+    )
+    parser.add_argument("--endpoint", required=True, help="credential-free public HTTPS endpoint")
+    parser.add_argument(
+        "--output",
+        dest="output_root",
+        type=Path,
+        required=True,
+        help="the explicit public catalog output root",
+    )
+    parser.add_argument(
+        "--predecessor",
+        dest="predecessor_root",
+        type=Path,
+        help="the exact predecessor catalog root, when updating",
+    )
+    parser.add_argument(
+        "--predecessor-sha256", help="the exact predecessor retrieval-manifest fingerprint"
+    )
+    parser.add_argument(
+        "--staging",
+        dest="staging_root",
+        type=Path,
+        required=True,
+        help="the private resumable staging root",
+    )
+    parser.add_argument(
+        "--checkpoint",
+        "--checkpoint-sha256",
+        dest="checkpoint_sha256",
+        help="the exact retained checkpoint fingerprint required for resume",
+    )
+    parser.add_argument(
+        "--observed-at", required=True, help="the canonical UTC observation timestamp"
+    )
+    parser.add_argument(
+        "--page-size", type=int, required=True, help="records requested per provider page"
+    )
+    parser.add_argument(
+        "--provider-count-reconciliation",
+        choices=("required-when-reported",),
+        required=True,
+        help="require equality whenever the provider reports a total",
+    )
+    parser.add_argument(
+        "--max-convergence-passes",
+        type=int,
+        required=True,
+        help="maximum complete passes allowed to reach convergence",
+    )
+    parser.add_argument(
+        "--max-requests", type=int, required=True, help="maximum aggregate network requests"
+    )
+    parser.add_argument(
+        "--max-responses", type=int, required=True, help="maximum aggregate network responses"
+    )
+    parser.add_argument(
+        "--max-pages", type=int, required=True, help="maximum aggregate provider pages"
+    )
+    parser.add_argument(
+        "--max-records", type=int, required=True, help="maximum aggregate provider records"
+    )
+    parser.add_argument(
+        "--max-response-bytes",
+        type=int,
+        required=True,
+        help="maximum bytes in one response",
+    )
+    parser.add_argument(
+        "--max-aggregate-bytes",
+        type=int,
+        required=True,
+        help="maximum bytes across all responses",
+    )
+    parser.add_argument(
+        "--max-wall-time-ns",
+        type=int,
+        required=True,
+        help="maximum cumulative retrieval time in nanoseconds",
+    )
+    parser.add_argument(
+        "--max-pages-this-invocation",
+        type=int,
+        help="stop after this many newly fetched pages and return an incomplete checkpoint",
+    )
+    parser.add_argument(
+        "--authoring",
+        type=Path,
+        required=True,
+        help="bounded canonical current-entry and predecessor mapping document",
+    )
+    parser.add_argument(
+        "--authoring-sha256", required=True, help="the exact authoring-document fingerprint"
+    )
+    parser.add_argument(
+        "--rights-decisions",
+        type=Path,
+        help="bounded canonical human rights-decision document for queued records",
+    )
+    parser.add_argument(
+        "--rights-decisions-sha256",
+        help="the exact rights-decision-document fingerprint",
+    )
+    parser.add_argument(
+        "--neural-mode",
+        choices=("lexical-only", "sidecar"),
+        required=True,
+        help="the explicit representation mode to publish",
+    )
+    parser.add_argument(
+        "--neural-model-root",
+        type=Path,
+        help="the explicit governed sidecar root for neural mode",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    _validate_coordinate_pairs(args)
+    fill_result = _run_fill(args)
+    base = _fill_payload(args, fill_result)
+    if fill_result.status != "complete":
+        return {
+            **base,
+            "status": "catalog_fill_incomplete",
+            "manifest_sha256": None,
+            "publication_sha256": None,
+        }
+
+    # No caller-authored facts are read before the deterministic fill proves a complete,
+    # reconciled generation. In particular, a pilot or budget stop cannot reach publication.
+    bundle = _load_authoring_bundle(args.authoring, args.authoring_sha256)
+    rights_decisions = None
+    if args.rights_decisions is not None:
+        rights_decisions = load_rights_decisions(
+            args.rights_decisions, args.rights_decisions_sha256
+        )
+    lexical_backend, neural_backend, fallback_reason = _publication_backends(args)
+    try:
+        predecessor, predecessor_directory = _load_predecessor(args, neural_backend)
+        classification = classify_catalog_delta(
+            predecessor=predecessor,
+            predecessor_record_ids=bundle.predecessor_record_ids,
+            candidates_by_record_id=bundle.candidates_by_record_id,
+            fill_result=fill_result,
+            staging_root=args.staging_root,
+            predecessor_chains=bundle.predecessor_chains,
+            rights_decisions=rights_decisions,
+            rights_decisions_sha256=(
+                args.rights_decisions_sha256 if rights_decisions is not None else None
+            ),
+        )
+        classification_checkpoint = checkpoint_catalog_delta_classification(
+            classification,
+            staging_root=args.staging_root,
+            lexical_backend=lexical_backend,
+            neural_backend=neural_backend,
+        )
+        # This is the only public-writer call in the command. It happens after complete fill,
+        # strict authoring capture, predecessor verification, classification, and checkpointing.
+        publication = publish_retrieval_catalog_successor(
+            classification,
+            classification_checkpoint_sha256=classification_checkpoint,
+            staging_root=args.staging_root,
+            lexical_backend=lexical_backend,
+            neural_backend=neural_backend,
+            directory=args.output_root,
+            predecessor_directory=predecessor_directory,
+            egress_policy=_publication_citation_policy(args.endpoint, rights_decisions),
+            fallback_reason=fallback_reason,
+        )
+    finally:
+        close = getattr(neural_backend, "close", None)
+        if callable(close):
+            close()
+
+    return {
+        **base,
+        "status": "catalog_published",
+        "rights_decisions_sha256": classification.rights_decisions_sha256,
+        "classification_sha256": classification.digest,
+        "classification_checkpoint_sha256": classification_checkpoint,
+        "manifest_sha256": publication.manifest_sha256,
+        "publication_sha256": publication.digest,
+        "counts": {
+            "unchanged": publication.unchanged,
+            "changed": publication.changed,
+            "new": publication.new,
+            "absent": publication.absent,
+            "flagged": publication.flagged,
+        },
+        "representations": {
+            "lexical": {
+                "reused_vector_sha256s": list(publication.lexical_reused_vector_sha256s),
+                "created_vector_sha256s": list(publication.lexical_created_vector_sha256s),
+                "encoder_invocations": publication.lexical_encoder_invocations,
+            },
+            "neural": {
+                "reused_vector_sha256s": list(publication.neural_reused_vector_sha256s),
+                "created_vector_sha256s": list(publication.neural_created_vector_sha256s),
+                "encoder_invocations": publication.neural_encoder_invocations,
+            },
+        },
+        "fallback_reason": publication.fallback_reason,
+    }
+
+
+def _run_fill(args: argparse.Namespace):
+    limits = CatalogFillLimits(
+        max_requests=args.max_requests,
+        max_pages=args.max_pages,
+        max_records=args.max_records,
+        max_response_bytes=args.max_response_bytes,
+        max_aggregate_bytes=args.max_aggregate_bytes,
+        max_responses=args.max_responses,
+        max_wall_time_ns=args.max_wall_time_ns,
+    )
+    config = CatalogFillConfig(
+        protocol=args.protocol,
+        endpoint=args.endpoint,
+        observed_at=args.observed_at,
+        page_size=args.page_size,
+        max_convergence_passes=args.max_convergence_passes,
+        limits=limits,
+        predecessor_sha256=args.predecessor_sha256,
+    )
+    retriever = PinnedHttpsRetriever(
+        resolver=SystemResolver(),
+        egress_policy=_endpoint_policy(args.endpoint),
+        transport=StdlibPinnedTransport(),
+        limits=RetrievalLimits(
+            max_response_bytes=min(args.max_response_bytes, 1 << 34),
+            max_aggregate_response_bytes=min(args.max_aggregate_bytes, 1 << 40),
+            max_redirects=5,
+            max_requests=16,
+        ),
+    )
+    return run_catalog_fill(
+        config=config,
+        staging_root=args.staging_root,
+        retriever=retriever,
+        expected_checkpoint_digest=args.checkpoint_sha256,
+        max_pages_this_invocation=args.max_pages_this_invocation,
+    )
+
+
+def _fill_payload(args: argparse.Namespace, result) -> dict[str, Any]:
+    receipt = result.receipt
+    return {
+        "schema_version": "mr-data-catalog-fill.v1",
+        "reason_code": result.reason_code,
+        "fill_receipt_sha256": receipt.digest,
+        "fill_receipt": receipt.to_dict(),
+        "flagged_queue_sha256": receipt.flagged_queue_sha256,
+        "staging_sha256": result.eligible_staging_sha256,
+        "checkpoint_sha256": result.checkpoint_sha256,
+        "provider_count_reconciliation": args.provider_count_reconciliation,
+        "convergence_passes": receipt.convergence_passes,
+    }
+
+
+def _validate_coordinate_pairs(args: argparse.Namespace) -> None:
+    predecessor_pair = (args.predecessor_root is None, args.predecessor_sha256 is None)
+    if predecessor_pair not in {(True, True), (False, False)}:
+        raise SourceContractError(
+            "CATALOG_PREDECESSOR_COORDINATE",
+            "predecessor",
+            "predecessor root and exact digest must be supplied together",
+        )
+    decisions_pair = (args.rights_decisions is None, args.rights_decisions_sha256 is None)
+    if decisions_pair not in {(True, True), (False, False)}:
+        raise SourceContractError(
+            "RIGHTS_DECISIONS_COORDINATE",
+            "rights_decisions",
+            "rights decisions and their exact digest must be supplied together",
+        )
+    if args.neural_mode == "sidecar" and args.neural_model_root is None:
+        raise SourceContractError(
+            "NEURAL_MODEL_ROOT_REQUIRED",
+            "neural_model_root",
+            "sidecar mode requires one explicit local model root",
+        )
+    if args.neural_mode == "lexical-only" and args.neural_model_root is not None:
+        raise SourceContractError(
+            "NEURAL_MODEL_ROOT_UNEXPECTED",
+            "neural_model_root",
+            "lexical-only mode accepts no model root",
+        )
+
+
+def _endpoint_policy(endpoint: str) -> EgressPolicy:
+    hostname = urlsplit(endpoint).hostname
+    if hostname is None:
+        raise SourceContractError("FILL_ENDPOINT", "endpoint", "must name one public host")
+    return EgressPolicy(allowed_hostnames=(hostname.lower(),))
+
+
+def _publication_citation_policy(endpoint: str, rights_decisions) -> EgressPolicy:
+    """The hosts a sealed entry may cite: the provider, plus what a person's decision cites.
+
+    Machine-derived citations all name the provider endpoint, so the fetch policy and the
+    citation policy used to be the same object. A human rights decision breaks that identity
+    by construction: the person looked at a licence page, and licence pages live on the
+    publisher's own site, not on its data API. The widening is bounded and receipted -- only
+    hosts named as evidence inside the digest-verified decision document join the allowlist,
+    and the document's digest rides the published classification. Network egress is untouched:
+    this policy governs which citations a sealed entry may carry, and the fetch retriever
+    keeps the endpoint-only policy.
+    """
+
+    endpoint_policy = _endpoint_policy(endpoint)
+    if not rights_decisions:
+        return endpoint_policy
+    hosts = set(endpoint_policy.allowed_hostnames)
+    for decision in rights_decisions.values():
+        for reference in decision.rights.evidence:
+            hostname = urlsplit(reference.uri).hostname
+            if hostname is not None:
+                hosts.add(hostname.lower())
+    return EgressPolicy(allowed_hostnames=tuple(sorted(hosts)))
+
+
+def _publication_backends(args: argparse.Namespace):
+    lexical = LexicalHashingBackend()
+    if args.neural_mode == "lexical-only":
+        return lexical, None, "NEURAL_NOT_REQUESTED"
+    from mostlyright.data_harness.sources.catalog.neural import resolve_local_neural_backend
+
+    return lexical, resolve_local_neural_backend(args.neural_model_root), None
+
+
+def _load_predecessor(args: argparse.Namespace, neural_backend):
+    if args.predecessor_root is None:
+        return (), None
+    loaded = load_retrieval_catalog(
+        args.predecessor_root,
+        expected_manifest_sha256=args.predecessor_sha256,
+        neural_backend=neural_backend,
+    )
+    return loaded.lexical.entries, args.predecessor_root
+
+
+def _load_authoring_bundle(path: Path, expected_sha256: str) -> _AuthoringBundle:
+    if not isinstance(expected_sha256, str) or _DIGEST.fullmatch(expected_sha256) is None:
+        raise SourceContractError(
+            "CATALOG_AUTHORING_DIGEST", "authoring_sha256", "must be a lowercase SHA-256"
+        )
+    raw = _read_bounded_regular(path)
+    if sha256_bytes(raw) != expected_sha256:
+        raise SourceContractError(
+            "CATALOG_AUTHORING_DIGEST",
+            "authoring",
+            "authoring bytes differ from the required digest",
+        )
+    value = parse_canonical_json(raw)
+    expected = {
+        "schema_version",
+        "candidates_by_record_id",
+        "predecessor_record_ids",
+        "predecessor_chains",
+    }
+    if (
+        not isinstance(value, dict)
+        or set(value) != expected
+        or value["schema_version"] != _AUTHORING_SCHEMA
+    ):
+        raise SourceContractError(
+            "CATALOG_AUTHORING_SCHEMA", "authoring", "authoring fields or version differ"
+        )
+    candidates_raw = _mapping(value["candidates_by_record_id"], "candidates_by_record_id")
+    predecessor_ids_raw = _mapping(value["predecessor_record_ids"], "predecessor_record_ids")
+    chains_raw = _mapping(value["predecessor_chains"], "predecessor_chains")
+    all_keys = (*candidates_raw, *predecessor_ids_raw, *chains_raw)
+    if any(not isinstance(key, str) for key in all_keys):
+        raise SourceContractError(
+            "CATALOG_AUTHORING_SCHEMA", "authoring", "mapping keys must be strings"
+        )
+    candidates = {
+        key: catalog_entry_from_dict(candidate) for key, candidate in candidates_raw.items()
+    }
+    predecessor_ids: dict[str, str] = {}
+    for key, entry_id in predecessor_ids_raw.items():
+        if not isinstance(entry_id, str):
+            raise SourceContractError(
+                "CATALOG_AUTHORING_SCHEMA",
+                "predecessor_record_ids",
+                "entry identities must be strings",
+            )
+        predecessor_ids[key] = entry_id
+    chains: dict[str, tuple[CatalogEntry, ...]] = {}
+    for entry_id, raw_chain in chains_raw.items():
+        if not isinstance(raw_chain, list):
+            raise SourceContractError(
+                "CATALOG_AUTHORING_SCHEMA", "predecessor_chains", "chains must be arrays"
+            )
+        chains[entry_id] = tuple(catalog_entry_from_dict(item) for item in raw_chain)
+    return _AuthoringBundle(candidates, predecessor_ids, chains)
+
+
+def _read_bounded_regular(path: Path) -> bytes:
+    flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        descriptor = os.open(path, flags)
+    except OSError as error:
+        raise SourceContractError(
+            "CATALOG_AUTHORING_UNREADABLE", "authoring", "cannot open the authoring document"
+        ) from error
+    try:
+        opened = os.fstat(descriptor)
+        if not stat.S_ISREG(opened.st_mode) or opened.st_nlink != 1:
+            raise SourceContractError(
+                "CATALOG_AUTHORING_FILE",
+                "authoring",
+                "must be a single-link regular file",
+            )
+        if opened.st_size > _MAX_AUTHORING_BYTES:
+            raise SourceContractError(
+                "CATALOG_AUTHORING_LIMIT", "authoring", "exceeds the authoring byte limit"
+            )
+        chunks: list[bytes] = []
+        remaining = _MAX_AUTHORING_BYTES + 1
+        while remaining:
+            chunk = os.read(descriptor, min(remaining, 1 << 16))
+            if not chunk:
+                break
+            chunks.append(chunk)
+            remaining -= len(chunk)
+    finally:
+        os.close(descriptor)
+    raw = b"".join(chunks)
+    if len(raw) > _MAX_AUTHORING_BYTES:
+        raise SourceContractError(
+            "CATALOG_AUTHORING_LIMIT", "authoring", "exceeds the authoring byte limit"
+        )
+    return raw
+
+
+def _mapping(value: Any, path: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise SourceContractError("CATALOG_AUTHORING_SCHEMA", path, "must be an object")
+    return value
+
+
+def _exit_code(payload: dict[str, Any]) -> int:
+    return 0 if payload.get("status") == "catalog_published" else 1
+
+
+SPEC = CommandSpec(
+    name="catalog-fill",
+    help="stage, reconcile, and publish one bounded public catalog successor",
+    add_arguments=add_arguments,
+    run=run,
+    exit_code=_exit_code,
+)