mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/commands/search.py

@@ -0,0 +1,440 @@
+"""``mr-data search`` over already-published exact local stores.
+
+A v1/v2 public catalog uses its sealed lookup contract. A range-local packed generation is larger
+than any one answer should be allowed to read, so it is searched only under a work budget the
+caller states in full -- eight caps, none of which has a default here or anywhere below here --
+together with the composite generation receipt that describes the generation being asked. Those
+ten values are handed to the service unchanged; this module chooses none of them.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.local_search.contracts import (
+    _RIGHTS_STATUSES,
+    BuildField,
+    BuildTimeCoverage,
+    LocalSearchContractError,
+)
+from mostlyright.data_harness.local_search.retrieval import (
+    MAX_QUERY_CHARS,
+    MAX_RETRIEVAL_LIMIT,
+    BuildFilters,
+    ExactDigestFilter,
+    RecipeIdentityFilter,
+    SourceIdentityFilter,
+)
+from mostlyright.data_harness.local_search.service import capture_local_search
+from mostlyright.data_harness.sources.catalog.gating import CatalogRequirement
+from mostlyright.data_harness.sources.contracts import (
+    DATA_FORMATS,
+    OBLIGATION_LIFECYCLES,
+    SourceContractError,
+)
+from mostlyright.data_harness.ux.commands import CommandSpec
+
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+_DIGEST_KINDS = ("candidate", "dataset", "facts", "lineage", "requirements", "validation_policy")
+_LOGICAL_TYPES = ("date", "float64", "int64", "string")
+_FORBIDDEN_PAYLOAD_KEYS = frozenset({"locator", "relative_path", "score", "scores"})
+
+#: Every cap one packed search may spend, in the order the work contract states them. There is no
+#: default for any of them here: a default would be an answer to "how much may this question cost"
+#: given by whoever forgot to ask rather than by the caller who has to live with it.
+_WORK_BUDGET = (
+    "max_ranges",
+    "max_packs",
+    "max_member_bytes",
+    "max_posting_terms",
+    "max_candidates",
+    "max_open_files",
+    "max_query_encodes",
+    "max_elapsed_seconds",
+)
+
+#: The composite receipt that describes the packed generation being asked, and its exact bytes.
+_PACKED_COORDINATES = ("public_generation_receipt", "public_generation_receipt_sha256")
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--public-index-root", type=Path, required=True, help="the sealed public catalog root"
+    )
+    parser.add_argument(
+        "--public-manifest-sha256",
+        type=_sha256,
+        required=True,
+        help="the exact public retrieval-manifest fingerprint",
+    )
+    parser.add_argument(
+        "--private-index-root", type=Path, required=True, help="the sealed private index root"
+    )
+    parser.add_argument(
+        "--private-manifest-sha256",
+        type=_sha256,
+        required=True,
+        help="the exact private index-manifest fingerprint",
+    )
+    parser.add_argument(
+        "--build-root", type=Path, required=True, help="the root for selected Build replay"
+    )
+    parser.add_argument(
+        "--question", type=_question, required=True, help="the bounded search question"
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        choices=range(1, MAX_RETRIEVAL_LIMIT + 1),
+        required=True,
+        help="the maximum matches considered from each store",
+    )
+    parser.add_argument(
+        "--public-backend",
+        choices=("lexical", "neural"),
+        required=True,
+        help="the closed public retrieval backend",
+    )
+    parser.add_argument(
+        "--private-backend",
+        choices=("lexical", "neural"),
+        required=True,
+        help="the closed private retrieval backend",
+    )
+    parser.add_argument(
+        "--neural-model-root",
+        type=Path,
+        help="the explicit governed sidecar root when neural is selected",
+    )
+    parser.add_argument(
+        "--allow-lexical-fallback",
+        action="store_true",
+        help=(
+            "allow an explicitly requested unavailable public neural representation to use lexical"
+        ),
+    )
+
+    parser.add_argument(
+        "--source-format",
+        dest="source_formats",
+        action="append",
+        choices=tuple(sorted(DATA_FORMATS)),
+        required=True,
+        help="an acceptable public source format; repeat to allow more",
+    )
+    parser.add_argument(
+        "--source-max-update-seconds",
+        type=int,
+        required=True,
+        help="the slowest acceptable public source update interval",
+    )
+    parser.add_argument(
+        "--source-intended-use", required=True, help="the declared use for source rights checks"
+    )
+    parser.add_argument(
+        "--source-lifecycle",
+        choices=tuple(sorted(OBLIGATION_LIFECYCLES)),
+        required=True,
+        help="the lifecycle used for source obligation checks",
+    )
+
+    parser.add_argument(
+        "--grain", action="append", help="one exact required grain field; repeat for more"
+    )
+    parser.add_argument(
+        "--time-start-inclusive", help="first included date for the required overlap"
+    )
+    parser.add_argument("--time-end-exclusive", help="first excluded date for the required overlap")
+    parser.add_argument(
+        "--required-field",
+        dest="required_fields",
+        action="append",
+        type=_field,
+        help="required NAME:LOGICAL_TYPE; repeat for more fields",
+    )
+    parser.add_argument(
+        "--source-identity",
+        dest="source_identities",
+        action="append",
+        type=_source_identity,
+        help="required SOURCE_ID:CONTENT_SHA256; repeat for more sources",
+    )
+    parser.add_argument(
+        "--rights-status",
+        dest="rights_statuses",
+        action="append",
+        choices=tuple(sorted(_RIGHTS_STATUSES)),
+        help="one acceptable sealed rights status; repeat for more",
+    )
+    parser.add_argument("--recipe-id", help="the exact required recipe identifier")
+    parser.add_argument("--recipe-version", type=int, help="the exact required recipe version")
+    parser.add_argument(
+        "--recipe-sha256", type=_sha256, help="the exact required recipe fingerprint"
+    )
+    parser.add_argument(
+        "--digest-kind", choices=_DIGEST_KINDS, help="the kind of exact Build digest required"
+    )
+    parser.add_argument("--digest-sha256", type=_sha256, help="the exact required Build digest")
+
+    parser.add_argument(
+        "--public-generation-receipt",
+        type=Path,
+        help="the composite receipt of the packed public catalog being asked",
+    )
+    parser.add_argument(
+        "--public-generation-receipt-sha256",
+        type=_sha256,
+        help="the exact composite receipt fingerprint",
+    )
+    parser.add_argument(
+        "--max-ranges", type=int, help="most key ranges this packed search may consider"
+    )
+    parser.add_argument("--max-packs", type=int, help="most packs this packed search may open")
+    parser.add_argument(
+        "--max-member-bytes", type=int, help="most bytes this packed search may read"
+    )
+    parser.add_argument(
+        "--max-posting-terms", type=int, help="most posting terms this packed search may read"
+    )
+    parser.add_argument(
+        "--max-candidates", type=int, help="most entries this packed search may examine"
+    )
+    parser.add_argument(
+        "--max-open-files", type=int, help="most files this packed search may hold open at once"
+    )
+    parser.add_argument(
+        "--max-query-encodes", type=int, help="most times the question itself may be encoded"
+    )
+    parser.add_argument(
+        "--max-elapsed-seconds", type=int, help="longest this packed search may take"
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    time_overlap = _time_filter(args.time_start_inclusive, args.time_end_exclusive)
+    recipe = _recipe_filter(args.recipe_id, args.recipe_version, args.recipe_sha256)
+    exact_digest = _digest_filter(args.digest_kind, args.digest_sha256)
+    try:
+        filters = BuildFilters(
+            grain=None if args.grain is None else tuple(args.grain),
+            time_overlap=time_overlap,
+            required_fields=tuple(args.required_fields or ()),
+            sources=tuple(sorted(args.source_identities or (), key=lambda item: item.source_id)),
+            rights_statuses=tuple(sorted(args.rights_statuses or ())),
+            recipe=recipe,
+            exact_digest=exact_digest,
+        )
+    except LocalSearchContractError as error:
+        raise SourceContractError(error.code, error.path, error.detail) from error
+    requirement = CatalogRequirement(
+        intended_use=args.source_intended_use,
+        lifecycle=args.source_lifecycle,
+        required_data_formats=tuple(sorted(set(args.source_formats))),
+        max_update_frequency_seconds=args.source_max_update_seconds,
+        require_open_access=True,
+        require_live=True,
+    )
+    packed_receipt, work_limits = _packed_contract(args)
+    try:
+        captured = capture_local_search(
+            public_index_root=args.public_index_root,
+            expected_public_manifest_sha256=args.public_manifest_sha256,
+            private_index_root=args.private_index_root,
+            expected_private_manifest_sha256=args.private_manifest_sha256,
+            build_root=args.build_root,
+            question=args.question,
+            source_requirement=requirement,
+            build_filters=filters,
+            public_backend=args.public_backend,
+            allow_lexical_fallback=args.allow_lexical_fallback,
+            private_backend=args.private_backend,
+            neural_model_root=args.neural_model_root,
+            public_packed_receipt=packed_receipt,
+            catalog_work_limits=work_limits,
+            limit=args.limit,
+        )
+    except LocalSearchContractError as error:
+        raise SourceContractError(error.code, error.path, error.detail) from error
+    _assert_closed_payload(captured.payload)
+    return captured.payload
+
+
+def _packed_contract(args: argparse.Namespace) -> tuple[Any | None, Any | None]:
+    """The packed generation's receipt and its work budget, stated in full or not at all.
+
+    Ten values decide whether this is a packed search: the composite receipt, its fingerprint, and
+    the eight caps. Stating some of them is not a smaller question -- it is a question nobody has
+    said the cost of -- so a partial set is refused here rather than completed with a guess.
+    """
+
+    stated = [name for name in (*_PACKED_COORDINATES, *_WORK_BUDGET) if _stated(args, name)]
+    if not stated:
+        return None, None
+    missing = [name for name in (*_PACKED_COORDINATES, *_WORK_BUDGET) if not _stated(args, name)]
+    if missing:
+        raise SourceContractError(
+            "LOCAL_SEARCH_WORK_BUDGET",
+            missing[0],
+            "a packed catalog is searched under its composite receipt and all eight work caps, "
+            "or it is not searched at all",
+        )
+    from mostlyright.data_harness.sources.catalog.generation_receipt import (
+        MAX_COMPOSITE_ARRAY_ITEMS,
+        MAX_COMPOSITE_RECEIPT_BYTES,
+        MAX_COMPOSITE_RECEIPT_DEPTH,
+        load_bounded_json,
+        verify_generation_receipt,
+    )
+    from mostlyright.data_harness.sources.catalog.packed_retrieval import CatalogSearchWorkLimits
+
+    _raw, receipt = load_bounded_json(
+        args.public_generation_receipt,
+        label="public_generation_receipt",
+        maximum=MAX_COMPOSITE_RECEIPT_BYTES,
+        max_depth=MAX_COMPOSITE_RECEIPT_DEPTH,
+        max_members=MAX_COMPOSITE_ARRAY_ITEMS,
+        max_items=MAX_COMPOSITE_ARRAY_ITEMS,
+        expected_sha256=args.public_generation_receipt_sha256,
+    )
+    verify_generation_receipt(receipt)
+    publication = receipt["publication"]
+    if publication.get("expected_head_sha256") != args.public_manifest_sha256:
+        raise SourceContractError(
+            "LOCAL_SEARCH_WORK_BUDGET",
+            "public_manifest_sha256",
+            "this composite receipt describes another packed generation",
+        )
+    return publication["packed_receipt"], CatalogSearchWorkLimits(
+        **{name: getattr(args, name) for name in _WORK_BUDGET}
+    )
+
+
+def _stated(args: argparse.Namespace, name: str) -> bool:
+    return getattr(args, name, None) is not None
+
+
+def _time_filter(start: str | None, end: str | None) -> BuildTimeCoverage | None:
+    if (start is None) != (end is None):
+        raise SourceContractError(
+            "LOCAL_SEARCH_FILTER_PAIR",
+            "time_overlap",
+            "time start and end must be supplied together",
+        )
+    if start is None:
+        return None
+    try:
+        return BuildTimeCoverage(start, end)  # type: ignore[arg-type]
+    except LocalSearchContractError as error:
+        raise SourceContractError(error.code, error.path, error.detail) from error
+
+
+def _recipe_filter(
+    recipe_id: str | None, recipe_version: int | None, recipe_sha256: str | None
+) -> RecipeIdentityFilter | None:
+    supplied = (recipe_id, recipe_version, recipe_sha256)
+    if any(value is not None for value in supplied) and not all(
+        value is not None for value in supplied
+    ):
+        raise SourceContractError(
+            "LOCAL_SEARCH_FILTER_PAIR",
+            "recipe",
+            "recipe id, version, and digest must be supplied together",
+        )
+    if recipe_id is None:
+        return None
+    try:
+        return RecipeIdentityFilter(
+            recipe_id,
+            recipe_version,
+            recipe_sha256,  # type: ignore[arg-type]
+        )
+    except LocalSearchContractError as error:
+        raise SourceContractError(error.code, error.path, error.detail) from error
+
+
+def _digest_filter(kind: str | None, digest: str | None) -> ExactDigestFilter | None:
+    if (kind is None) != (digest is None):
+        raise SourceContractError(
+            "LOCAL_SEARCH_FILTER_PAIR",
+            "exact_digest",
+            "digest kind and digest must be supplied together",
+        )
+    if kind is None:
+        return None
+    try:
+        return ExactDigestFilter(kind, digest)  # type: ignore[arg-type]
+    except LocalSearchContractError as error:
+        raise SourceContractError(error.code, error.path, error.detail) from error
+
+
+def _field(value: str) -> BuildField:
+    name, separator, logical_type = value.rpartition(":")
+    if not separator or not name or logical_type not in _LOGICAL_TYPES:
+        raise argparse.ArgumentTypeError(
+            f"must be NAME:LOGICAL_TYPE where type is one of {list(_LOGICAL_TYPES)}"
+        )
+    try:
+        return BuildField(name, logical_type)
+    except LocalSearchContractError as error:
+        raise argparse.ArgumentTypeError(error.detail) from error
+
+
+def _source_identity(value: str) -> SourceIdentityFilter:
+    source_id, separator, digest = value.rpartition(":")
+    if not separator or not source_id or _DIGEST.fullmatch(digest) is None:
+        raise argparse.ArgumentTypeError("must be SOURCE_ID:CONTENT_SHA256")
+    try:
+        return SourceIdentityFilter(source_id, digest)
+    except LocalSearchContractError as error:
+        raise argparse.ArgumentTypeError(error.detail) from error
+
+
+def _sha256(value: str) -> str:
+    if _DIGEST.fullmatch(value) is None:
+        raise argparse.ArgumentTypeError("must be a lowercase SHA-256 digest")
+    return value
+
+
+def _question(value: str) -> str:
+    if not 1 <= len(value) <= MAX_QUERY_CHARS:
+        raise argparse.ArgumentTypeError(f"must contain 1 to {MAX_QUERY_CHARS} characters")
+    return value
+
+
+def _assert_closed_payload(value: Any) -> None:
+    if isinstance(value, Mapping):
+        for key, member in value.items():
+            normalized = str(key).lower()
+            if normalized in _FORBIDDEN_PAYLOAD_KEYS or normalized.endswith("_score"):
+                raise SourceContractError(
+                    "LOCAL_SEARCH_OUTPUT_BOUNDARY",
+                    "search.payload",
+                    "service payload exposed a private score or mutable locator",
+                )
+            _assert_closed_payload(member)
+    elif isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
+        for member in value:
+            _assert_closed_payload(member)
+
+
+def _exit_code(payload: Mapping[str, Any]) -> int:
+    """A search that stopped before it finished is not an answer, so it does not exit zero.
+
+    Nothing was ruled out: the budget closed. A script that treated that as "no fitting source"
+    would draw the one conclusion the result cannot support, so the exit code says so as well.
+    """
+
+    return 0 if payload.get("status") == "local_search_complete" else 1
+
+
+SPEC = CommandSpec(
+    name="search",
+    help="search exact published public and private local stores without changing them",
+    add_arguments=add_arguments,
+    run=run,
+    exit_code=_exit_code,
+)

mostlyright/data_harness/ux/commands/show.py

@@ -0,0 +1,61 @@
+"""``mr-data show`` -- read one build back: what it answers, what a row is, and how it checked out.
+
+This module is the command and nothing else: its arguments, and the one call that turns them into
+the payload both renderings are built from. The reading itself lives in
+:mod:`mostlyright.data_harness.ux.readers`, so the same facts are available to a browser page
+without going through argparse.
+
+Nothing here prints, and nothing here catches: a refusal is a typed error, and the command line
+renders it with the sentences the remediation map supplies.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.ux.commands import CommandSpec
+from mostlyright.data_harness.ux.readers import build_facts, plain_card
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "run_dir",
+        type=Path,
+        help="a build folder, or the Workbench folder that holds one",
+    )
+    parser.add_argument(
+        "--card",
+        action="store_true",
+        help="print the plain-language summary of the data and nothing else",
+    )
+    parser.add_argument(
+        "--members",
+        action="store_true",
+        help="also list every file the Receipt covers",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    # Imported here, not at the top: this package is loaded while the command line is still being
+    # set up. Calling the command line's own helper rather than restating its rule is what stops
+    # show and verify ever disagreeing about which folder a Workbench folder's run lives in.
+    from mostlyright.data_harness.cli import _candidate_run_target
+
+    target = _candidate_run_target(args.run_dir)
+    facts = build_facts(target)
+    card = plain_card(facts)
+    if args.card:
+        return {"status": "build_shown", "run_dir": str(facts.run_dir), "card": card}
+    payload = facts.to_dict(members=args.members)
+    payload["card"] = card
+    return payload
+
+
+SPEC = CommandSpec(
+    name="show",
+    help="show one build: what it answers, what a row is, its columns, checks, and fingerprints",
+    add_arguments=add_arguments,
+    run=run,
+)

mostlyright/data_harness/ux/commands/whoami.py

@@ -0,0 +1,37 @@
+"""``mr-data whoami`` -- compatibility spelling for remotely truthful ``auth status``."""
+
+from __future__ import annotations
+
+import argparse
+from typing import Any
+
+from mostlyright.data_harness.ux.commands import CommandSpec, auth
+
+DESCRIPTION = (
+    "Compatibility alias for mr-data auth status. Validates the effective device credential with "
+    "Cloud and reports stored metadata separately; it never prints a credential."
+)
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.description = DESCRIPTION
+    parser.add_argument(
+        "--credential-store",
+        choices=("secure-file",),
+        help="explicitly use the secure-file fallback for first credential migration",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    payload = dict(auth.status_payload(credential_store=args.credential_store))
+    payload["compatibility_alias"] = "whoami"
+    return payload
+
+
+SPEC = CommandSpec(
+    name="whoami",
+    help="compatibility alias for auth status",
+    add_arguments=add_arguments,
+    run=run,
+    exit_code=auth.exit_code,
+)