mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/commands/author.py

@@ -0,0 +1,79 @@
+"""``mr-data author`` -- write the Recipe in YAML, get the Recipe you can approve.
+
+Three gates stand between the file a person writes and the fingerprint `mr-data approve` binds:
+the reader refuses rather than guesses, `parse_fresh_recipe` checks the result against the
+contract field by field and resolves every Reader pin, and the written file is created without
+overwriting anything.
+
+This module is the command and nothing else. The reading lives in
+:mod:`mostlyright.data_harness.ux.author_yaml`, so the same step is available to a browser page
+without going through argparse. Nothing here prints, and nothing here catches a refusal: the
+command line renders it with the sentences the remediation map supplies.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import canonical_json_bytes
+from mostlyright.data_harness.recipe import RecipeError, parse_fresh_recipe
+from mostlyright.data_harness.ux.author_yaml import parse_author_yaml, read_author_source
+from mostlyright.data_harness.ux.commands import CommandSpec
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    # The flag names match recipe-export deliberately: a person who knows one knows the other.
+    parser.add_argument(
+        "--recipe",
+        type=Path,
+        required=True,
+        help="the YAML file you wrote the Recipe in",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        required=True,
+        help="where to write the Recipe; nothing is ever overwritten",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    # Imported here, not at the top: this package is loaded while the command line is still being
+    # set up. Reusing the command line's own writer is what keeps this file's publication receipt
+    # identical to the one every other written output carries.
+    from mostlyright.data_harness import cli
+
+    recipe = parse_fresh_recipe(parse_author_yaml(read_author_source(args.recipe)))
+    raw = canonical_json_bytes(recipe.to_dict())
+    try:
+        publication = cli._write_output_exclusive(
+            args.output,
+            raw,
+            purpose="recipe",
+            authority="stdout_recipe_receipt",
+        )
+    except FileExistsError as exc:
+        raise RecipeError(
+            "RECIPE_OUTPUT_EXISTS",
+            "output",
+            "recipe output already exists",
+        ) from exc
+    return {
+        "status": "recipe_authored",
+        "output": str(args.output),
+        "recipe_id": recipe.recipe_id,
+        "recipe_version": recipe.recipe_version,
+        "recipe_digest": recipe.digest,
+        "expected_size_bytes": len(raw),
+        "publication": publication,
+    }
+
+
+SPEC = CommandSpec(
+    name="author",
+    help="turn a YAML recipe description into a recipe you can approve",
+    add_arguments=add_arguments,
+    run=run,
+)

mostlyright/data_harness/ux/commands/catalog_author.py

@@ -0,0 +1,403 @@
+"""``catalog-author``: stream one converged v4 generation into bounded authoring shards.
+
+This module is the explicit second stage of the provider-scale harvest -> author -> publish
+workflow.  It reads a private staging root and an exact harvest receipt, and writes immutable
+authoring shards behind one atomic manifest plus one secret-free receipt.  It reaches no network,
+accepts no key material, opens no predecessor catalog, and never loads a whole generation into
+memory; every one of those properties is a structural consequence of what it imports.
+
+A ``--resume`` may replace only the exact typed-incomplete receipt its interrupted run left at the
+same path, after checking that the retained receipt belongs to this stage and describes this
+output root. A fresh receipt path remains exclusive, and a complete receipt is never overwritten.
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    parse_canonical_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.catalog.authoring_shards import (
+    AUTHORING_RECEIPT_SCHEMA,
+    MAX_AUTHORING_SHARD_BYTES,
+    MAX_AUTHORING_SHARD_ENTRIES,
+    AuthoringLimits,
+    AuthoringSelection,
+    CatalogAuthoringRefused,
+    run_catalog_authoring,
+)
+from mostlyright.data_harness.sources.catalog.bounded_io import (
+    BoundedReadFailure,
+    read_bounded_path,
+)
+from mostlyright.data_harness.sources.catalog.coverage import (
+    COVERAGE_EXHAUSTIVE_RULE,
+    SELECTION_RULES,
+)
+from mostlyright.data_harness.sources.catalog.generation_receipt import (
+    ReceiptInstallation,
+    install_receipt,
+    load_execution_coordinate,
+    publication_output_lock,
+    receipt_installation_lock,
+    require_output_root_identity,
+    retained_incomplete_receipt,
+    root_coordinate,
+)
+from mostlyright.data_harness.ux.commands import CommandSpec
+
+# The exact stage receipts this command will accept as the source of a fill checkpoint digest.
+# A receipt is a coordinate carrier, never an authority: the live staging checkpoint decides
+# whether the generation converged.
+HARVEST_RECEIPT_SCHEMAS = frozenset({"mr-data-catalog-fill.v1", "mr-data-catalog-harvest.v1"})
+MAX_HARVEST_RECEIPT_BYTES = 8_388_608
+MAX_AUTHOR_RECEIPT_BYTES = 8_388_608
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--staging-root",
+        type=Path,
+        required=True,
+        help="the private staging root holding one converged harvest generation",
+    )
+    parser.add_argument(
+        "--harvest-receipt",
+        type=Path,
+        required=True,
+        help="the exact harvest receipt naming the staging checkpoint to author",
+    )
+    parser.add_argument(
+        "--harvest-receipt-sha256", required=True, help="the exact harvest receipt fingerprint"
+    )
+    parser.add_argument(
+        "--policy", required=True, help="the exact reviewed authoring policy coordinate"
+    )
+    parser.add_argument(
+        "--output-root", type=Path, required=True, help="the authoring shard output root"
+    )
+    parser.add_argument(
+        "--receipt-file", type=Path, required=True, help="where this stage's receipt is installed"
+    )
+    parser.add_argument(
+        "--execution-manifest",
+        type=Path,
+        help="the committed source and installed-wheel coordinate document",
+    )
+    parser.add_argument("--execution-commit", help="the exact commit that coordinate was cut at")
+    parser.add_argument(
+        "--max-records", type=int, required=True, help="maximum authored provider records"
+    )
+    parser.add_argument(
+        "--max-input-bytes", type=int, required=True, help="maximum normalized input bytes read"
+    )
+    parser.add_argument(
+        "--max-shard-entries",
+        type=int,
+        required=True,
+        help=f"entries per authoring shard, at most {MAX_AUTHORING_SHARD_ENTRIES}",
+    )
+    parser.add_argument(
+        "--max-shard-bytes",
+        type=int,
+        required=True,
+        help=f"bytes per authoring shard, at most {MAX_AUTHORING_SHARD_BYTES}",
+    )
+    parser.add_argument(
+        "--max-disk-bytes", type=int, required=True, help="maximum local working disk bytes"
+    )
+    parser.add_argument(
+        "--max-wall-seconds", type=int, required=True, help="maximum cumulative authoring seconds"
+    )
+    parser.add_argument(
+        "--coverage-rule",
+        choices=sorted(SELECTION_RULES),
+        default=None,
+        help=(
+            "the registered rule that selects a bounded part of the converged sweep; "
+            "omit both coverage flags to author every record the sweep found"
+        ),
+    )
+    parser.add_argument(
+        "--coverage-bound",
+        type=int,
+        default=None,
+        help=(
+            "how many records that rule selects. This is not --max-records: --max-records is a "
+            "resource budget whose exhaustion still yields an incomplete generation, so set it "
+            "at least as large as this bound or the run stops at AUTHOR_RECORD_LIMIT"
+        ),
+    )
+    parser.add_argument(
+        "--resume",
+        action="store_true",
+        help="continue the retained partial generation from its exact journal digest",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    output_root = Path(args.output_root)
+    with publication_output_lock(
+        output_root,
+        label="output_root",
+        code_prefix="AUTHOR",
+        refusal=CatalogAuthoringRefused,
+    ) as output_descriptor:
+        with receipt_installation_lock(
+            args.receipt_file,
+            label="receipt_file",
+            code_prefix="AUTHOR",
+            refusal=CatalogAuthoringRefused,
+        ) as receipt_installation:
+            return _run_locked(
+                args,
+                output_root=output_root,
+                output_descriptor=output_descriptor,
+                receipt_installation=receipt_installation,
+            )
+
+
+def _run_locked(
+    args: argparse.Namespace,
+    *,
+    output_root: Path,
+    output_descriptor: int,
+    receipt_installation: ReceiptInstallation,
+) -> dict[str, Any]:
+    manifest_path = getattr(args, "execution_manifest", None)
+    execution_commit = getattr(args, "execution_commit", None)
+    if (manifest_path is None) != (execution_commit is None):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_EXECUTION_COORDINATE",
+            "execution_manifest",
+            "execution manifest and commit are supplied together",
+        )
+    execution = (
+        None
+        if manifest_path is None
+        else load_execution_coordinate(Path(manifest_path), execution_commit)
+    )
+    checkpoint_sha256, harvest_execution = _harvest_checkpoint(
+        args.harvest_receipt, args.harvest_receipt_sha256
+    )
+    if harvest_execution is not None and execution != harvest_execution:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_EXECUTION_COORDINATE",
+            "execution_manifest",
+            "provider-scale authoring must execute the exact harvest coordinate",
+        )
+    selection = _selection(args)
+    replaces = _replaceable_receipt(
+        Path(args.receipt_file), output_root, resume=args.resume, execution=execution
+    )
+    expected_journal_sha256 = None
+    if args.resume:
+        expected_journal_sha256 = _retained_journal_sha256(output_root)
+    result = run_catalog_authoring(
+        staging_root=Path(args.staging_root),
+        expected_fill_checkpoint_sha256=checkpoint_sha256,
+        policy_id=args.policy,
+        output_root=output_root,
+        limits=AuthoringLimits(
+            max_records=args.max_records,
+            max_input_bytes=args.max_input_bytes,
+            max_shard_entries=args.max_shard_entries,
+            max_shard_bytes=args.max_shard_bytes,
+            max_disk_bytes=args.max_disk_bytes,
+            max_wall_seconds=args.max_wall_seconds,
+        ),
+        selection=selection,
+        expected_journal_sha256=expected_journal_sha256,
+        output_descriptor=output_descriptor,
+    )
+    payload = {
+        **result.receipt,
+        "harvest_receipt_sha256": args.harvest_receipt_sha256,
+        "output_root_sha256": root_coordinate(output_root),
+    }
+    if execution is not None:
+        payload["execution"] = execution
+
+    def commit_guard() -> None:
+        require_output_root_identity(
+            output_root,
+            output_descriptor,
+            label="output_root",
+            code_prefix="AUTHOR",
+            refusal=CatalogAuthoringRefused,
+        )
+
+    install_receipt(
+        Path(args.receipt_file),
+        payload,
+        maximum=MAX_AUTHOR_RECEIPT_BYTES,
+        label="receipt_file",
+        code_prefix="AUTHOR",
+        refusal=CatalogAuthoringRefused,
+        replaces=replaces,
+        installation=receipt_installation,
+        commit_guard=commit_guard,
+    )
+    return payload
+
+
+def _selection(args: argparse.Namespace) -> AuthoringSelection:
+    """The coverage this invocation was asked for, stated by two flags or by neither.
+
+    Both flags or neither: a rule without a bound does not say how much of the sweep to take,
+    and a bound without a rule does not say which records, so each alone is an instruction this
+    command cannot carry out. Refusing here rather than picking a default for the missing half
+    keeps an operator from discovering at publication that they published a catalogue nobody
+    asked for.
+
+    The default is exhaustive, which is what this command did before a bound could be stated.
+    That direction matters: forgetting a flag can only publish a whole catalogue, never a
+    silently partial one.
+    """
+
+    rule = getattr(args, "coverage_rule", None)
+    bound = getattr(args, "coverage_bound", None)
+    if rule is None and bound is None:
+        return AuthoringSelection()
+    if rule is None or bound is None:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_COVERAGE",
+            "coverage_rule",
+            "a bounded coverage states its rule and its bound together",
+        )
+    if rule == COVERAGE_EXHAUSTIVE_RULE:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_COVERAGE",
+            "coverage_rule",
+            "an exhaustive coverage is what omitting both flags asks for",
+        )
+    return AuthoringSelection(rule=rule, bound=bound)
+
+
+def _replaceable_receipt(
+    path: Path,
+    output_root: Path,
+    *,
+    resume: bool,
+    execution: Mapping[str, Any] | None = None,
+) -> dict[str, Any] | None:
+    """The typed-incomplete receipt this resume is allowed to replace, or nothing at all.
+
+    Installation is otherwise ``O_EXCL``: evidence is never overwritten. The one exception is the
+    receipt an interrupted invocation of *this* stage left at *this* destination for *this* output
+    root, which the resume that continues it replaces rather than leaving two documents describing
+    one generation.
+    """
+
+    if not resume:
+        return None
+    retained = retained_incomplete_receipt(
+        path,
+        maximum=MAX_AUTHOR_RECEIPT_BYTES,
+        label="receipt_file",
+        schema=AUTHORING_RECEIPT_SCHEMA,
+        incomplete_status="incomplete",
+        output_root=output_root,
+        code_prefix="AUTHOR",
+        refusal=CatalogAuthoringRefused,
+    )
+    if retained is not None and retained.get("execution") != execution:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_RESUME_RECEIPT",
+            "execution_manifest",
+            "the retained authoring stage used another execution coordinate",
+        )
+    return retained
+
+
+def _harvest_checkpoint(path: Path, expected_sha256: Any) -> tuple[str, Mapping[str, Any] | None]:
+    if not _is_digest(expected_sha256):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_HARVEST_RECEIPT_DIGEST",
+            "harvest_receipt_sha256",
+            "must be a lowercase SHA-256",
+        )
+    raw = _read_bounded_regular(path, maximum=MAX_HARVEST_RECEIPT_BYTES)
+    if sha256_bytes(raw) != expected_sha256:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_HARVEST_RECEIPT_DIGEST",
+            "harvest_receipt",
+            "harvest receipt bytes differ from the required digest",
+        )
+    try:
+        payload = parse_canonical_json(raw)
+    except CanonicalJSONError as error:
+        raise CatalogAuthoringRefused(
+            "AUTHOR_HARVEST_RECEIPT_SCHEMA", "harvest_receipt", "receipt is not canonical JSON"
+        ) from error
+    if (
+        not isinstance(payload, Mapping)
+        or payload.get("schema_version") not in HARVEST_RECEIPT_SCHEMAS
+        or not _is_digest(payload.get("checkpoint_sha256"))
+    ):
+        raise CatalogAuthoringRefused(
+            "AUTHOR_HARVEST_RECEIPT_SCHEMA",
+            "harvest_receipt",
+            "receipt must name one accepted stage schema and an exact checkpoint digest",
+        )
+    execution = payload.get("execution")
+    if payload["schema_version"] == "mr-data-catalog-harvest.v1":
+        if not isinstance(execution, Mapping):
+            raise CatalogAuthoringRefused(
+                "AUTHOR_HARVEST_RECEIPT_SCHEMA",
+                "harvest_receipt",
+                "provider-scale harvest receipt must carry its execution coordinate",
+            )
+        return payload["checkpoint_sha256"], execution
+    return payload["checkpoint_sha256"], None
+
+
+def _retained_journal_sha256(output_root: Path) -> str:
+    raw = _read_bounded_regular(output_root / "journal.json", maximum=MAX_AUTHOR_RECEIPT_BYTES)
+    return sha256_bytes(raw)
+
+
+def _read_bounded_regular(path: Path, *, maximum: int) -> bytes:
+    try:
+        raw = read_bounded_path(path, maximum=maximum)
+    except (BoundedReadFailure, OSError) as error:
+        if isinstance(error, OSError):
+            raise CatalogAuthoringRefused(
+                "AUTHOR_INPUT_UNREADABLE", str(path.name), "cannot open the required document"
+            ) from error
+        if error.reason not in {"limit", "type", "changed"}:
+            raise CatalogAuthoringRefused(
+                "AUTHOR_INPUT_UNREADABLE", str(path.name), "cannot open the required document"
+            ) from error
+        raise CatalogAuthoringRefused(
+            "AUTHOR_INPUT_LIMIT",
+            str(path.name),
+            "must be a stable bounded single-link regular file",
+        ) from error
+    assert raw is not None
+    return raw
+
+
+def _is_digest(value: Any) -> bool:
+    return (
+        isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
+    )
+
+
+def _exit_code(payload: Mapping[str, Any]) -> int:
+    return 0 if payload.get("status") == "complete" else 1
+
+
+SPEC = CommandSpec(
+    name="catalog-author",
+    help="author one converged public catalog generation into bounded authoring shards",
+    add_arguments=add_arguments,
+    run=run,
+    exit_code=_exit_code,
+)