mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/commands/login.py

@@ -0,0 +1,63 @@
+"""``mr-data login`` -- sign in to the Mostly Right cloud with a device code.
+
+This module is the command and nothing else: its arguments and the one call that turns them into
+the payload both renderings are built from. The device flow itself lives in
+:mod:`mostlyright.data_harness.ux.login`, so the same flow is available to a caller that is not
+going through argparse.
+
+The only thing this command prints outside the one payload ``--json`` renders or ``_emit`` turns
+into plain lines is the device code and approval address, on stderr, and only in plain mode --
+matching how ``build``, ``run`` and ``recipe-run`` already print a stderr warning beside a result
+that still goes through ``_emit``. ``--json`` gets none of it, so a caller asking for one
+machine-readable object gets exactly one.
+"""
+
+from __future__ import annotations
+
+import argparse
+from typing import Any
+
+from mostlyright.data_harness.ux.commands import CommandSpec
+from mostlyright.data_harness.ux.login import CLOUD_URL_ENV, announce_device_code, run_login
+
+DESCRIPTION = (
+    "Sign in to the Mostly Right cloud from this machine. Prints a short code and opens a browser "
+    "to approve it there, then stores a durable device credential at ~/.mostlyright/credentials, "
+    f"mode 0600. The raw credential is never printed; only its prefix and last four characters "
+    f"are. Set {CLOUD_URL_ENV} to sign in against a different cloud."
+)
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.description = DESCRIPTION
+    parser.add_argument(
+        "--device-name",
+        help="what to call this device in the cloud; defaults to this machine's hostname",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="compatibility alias for safe auth rotate; never truncates credentials in place",
+    )
+    parser.add_argument(
+        "--credential-store",
+        choices=("secure-file",),
+        help="explicitly use the secure-file credential store",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    return run_login(
+        device_name=args.device_name,
+        force=args.force,
+        credential_store=args.credential_store,
+        announce=None if args.json else announce_device_code,
+    )
+
+
+SPEC = CommandSpec(
+    name="login",
+    help="sign in to the Mostly Right cloud and store a device credential",
+    add_arguments=add_arguments,
+    run=run,
+)

mostlyright/data_harness/ux/commands/peek.py

@@ -0,0 +1,236 @@
+"""``mr-data peek`` -- open any file or public https address and show what is in it.
+
+This module is the command: its arguments, and the one call that turns them into the payload both
+renderings are built from. Everything the command actually does lives in
+:mod:`mostlyright.data_harness.ux.peek`, so browser rendering can use the same facts
+without going through argparse.
+
+Nothing here prints, and nothing here catches: a refusal is a typed error, and the command line
+renders it with the sentences the remediation map supplies.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import parse_json
+from mostlyright.data_harness.readers.contracts import ReaderError, identifier, semver
+from mostlyright.data_harness.ux.commands import CommandSpec
+from mostlyright.data_harness.ux.peek import (
+    DEFAULT_SAMPLE_ROWS,
+    MAX_SAMPLE_ROWS,
+    PEEK_FORMATS,
+    peek_path,
+    peek_url,
+)
+
+# An address is a scheme followed by `://`. Requiring the slashes is what keeps a Windows path
+# such as `C:\data\rows.csv` a path rather than a scheme called `c`.
+_ADDRESS = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*://")
+
+
+def _rows(value: str) -> int:
+    """Bound the row count at the command line, before anything is opened."""
+
+    try:
+        rows = int(value)
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"{value!r} is not a whole number of rows") from None
+    if not 0 <= rows <= MAX_SAMPLE_ROWS:
+        raise argparse.ArgumentTypeError(
+            f"the number of rows to show must be from 0 to {MAX_SAMPLE_ROWS}"
+        )
+    return rows
+
+
+def _json_column(value: str) -> tuple[str, str]:
+    name, separator, pointer = value.partition("=")
+    if not separator or not name:
+        raise argparse.ArgumentTypeError("must be NAME=/json/pointer (the pointer may be empty)")
+    return name, pointer
+
+
+def _reader_coordinate(value: str) -> tuple[str, str]:
+    family, separator, version = value.rpartition("@")
+    if not separator:
+        raise argparse.ArgumentTypeError("must be an exact FAMILY@VERSION Reader coordinate")
+    try:
+        return identifier(family, "peek.reader.family"), semver(version, "peek.reader.version")
+    except ReaderError as error:
+        raise argparse.ArgumentTypeError(error.detail) from None
+
+
+def _reader_options(value: str) -> Any:
+    """Parse strict JSON; the selected certified family owns the closed option shape."""
+
+    try:
+        return parse_json(value)
+    except ValueError as error:
+        raise argparse.ArgumentTypeError(str(error)) from None
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "target",
+        help="a file on this machine, or an https address",
+    )
+    parser.add_argument(
+        "--format",
+        dest="data_format",
+        choices=PEEK_FORMATS,
+        default=None,
+        help="say what the file is when its name does not; the bytes are still checked against it",
+    )
+    parser.add_argument(
+        "--rows",
+        type=_rows,
+        default=DEFAULT_SAMPLE_ROWS,
+        help=f"how many rows to show, from 0 to {MAX_SAMPLE_ROWS} (default {DEFAULT_SAMPLE_ROWS})",
+    )
+    parser.add_argument(
+        "--allow-redirect-host",
+        dest="allow_redirect_hosts",
+        action="append",
+        default=None,
+        metavar="HOST",
+        help="also allow this host when an address redirects; repeat for more than one",
+    )
+    parser.add_argument(
+        "--reader",
+        type=_reader_coordinate,
+        default=None,
+        metavar="FAMILY@VERSION",
+        help="open the source with this exact certified Reader coordinate",
+    )
+    parser.add_argument(
+        "--reader-options",
+        type=_reader_options,
+        default=argparse.SUPPRESS,
+        metavar="JSON",
+        help=(
+            "strict JSON settings for --reader; the selected Reader rejects unknown or "
+            "invalid settings"
+        ),
+    )
+    parser.add_argument(
+        "--json-records-pointer",
+        default=None,
+        metavar="POINTER",
+        help="project nested JSON records selected by this RFC 6901 pointer",
+    )
+    parser.add_argument(
+        "--json-expand",
+        action="append",
+        default=None,
+        metavar="POINTER",
+        help="expand this array pointer in order; repeat for nested arrays",
+    )
+    parser.add_argument(
+        "--json-column",
+        action="append",
+        type=_json_column,
+        default=None,
+        metavar="NAME=POINTER",
+        help="required projected column; repeat for more columns",
+    )
+    parser.add_argument(
+        "--json-optional-column",
+        action="append",
+        type=_json_column,
+        default=None,
+        metavar="NAME=POINTER",
+        help="optional projected column; repeat for more columns",
+    )
+    parser.add_argument(
+        "--json-document-format",
+        choices=("json", "ndjson"),
+        default=None,
+        help="nested document encoding (default: json)",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    target = str(args.target)
+    json_requested = any(
+        item is not None
+        for item in (
+            args.json_records_pointer,
+            args.json_expand,
+            args.json_column,
+            args.json_optional_column,
+            args.json_document_format,
+        )
+    )
+    reader_coordinate = args.reader
+    reader_options_supplied = hasattr(args, "reader_options")
+    reader_options = getattr(args, "reader_options", None)
+    if reader_options_supplied and reader_coordinate is None:
+        raise ReaderError(
+            "READER_OPTIONS",
+            "peek.reader_options",
+            "--reader-options requires an exact --reader FAMILY@VERSION",
+        )
+    if reader_options_supplied and not isinstance(reader_options, dict):
+        raise ReaderError(
+            "READER_OPTIONS",
+            "peek.reader_options",
+            "--reader-options must be a JSON object",
+        )
+    if reader_coordinate is not None and args.data_format is not None:
+        raise ReaderError(
+            "READER_OPTIONS",
+            "peek.format",
+            "--format is for direct tabular preview and cannot be combined with --reader",
+        )
+    if reader_coordinate is not None and json_requested:
+        raise ReaderError(
+            "READER_OPTIONS",
+            "peek.reader",
+            "--reader cannot be combined with the JSON projection shorthand flags",
+        )
+    if reader_coordinate is not None and not reader_options_supplied:
+        reader_options = {}
+    if json_requested:
+        reader_coordinate = ("json.tabular", "1.0.0")
+        reader_options = {
+            "document_format": args.json_document_format or "json",
+            "records_pointer": args.json_records_pointer or "",
+            "expand": list(args.json_expand or ()),
+            "columns": [
+                {"name": name, "pointer": pointer, "required": required}
+                for required, items in (
+                    (True, args.json_column or ()),
+                    (False, args.json_optional_column or ()),
+                )
+                for name, pointer in items
+            ],
+        }
+    if _ADDRESS.match(target):
+        result = peek_url(
+            target,
+            allow_redirect_hosts=tuple(args.allow_redirect_hosts or ()),
+            data_format=args.data_format,
+            sample_rows=args.rows,
+            reader_coordinate=reader_coordinate,
+            reader_options=reader_options,
+        )
+    else:
+        result = peek_path(
+            Path(target),
+            data_format=args.data_format,
+            sample_rows=args.rows,
+            reader_coordinate=reader_coordinate,
+            reader_options=reader_options,
+        )
+    return result.to_dict()
+
+
+SPEC = CommandSpec(
+    name="peek",
+    help="open any file or public https address and show its columns, types, and first rows",
+    add_arguments=add_arguments,
+    run=run,
+)

mostlyright/data_harness/ux/commands/plan_check.py

@@ -0,0 +1,90 @@
+"""``mr-data plan-check`` -- try a plan by building it, into a folder that is thrown away after.
+
+This module is the command and nothing else: its arguments, and the one call that turns them into
+the payload both renderings are built from. The build itself lives in
+:mod:`mostlyright.data_harness.ux.plan_check`, so the same answer is available to a browser page
+without going through argparse.
+
+The engines offered here are the same closed set ``build`` offers, imported from
+:mod:`mostlyright.data_harness.pipeline` rather than restated, so the two cannot drift into
+disagreeing about what this machine can build with.
+
+Nothing here prints, and nothing here catches: a plan that does not build is refused in exactly the
+words the real build refuses it in, and the command line renders that refusal with the sentences
+the error-remediation map supplies.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.pipeline import (
+    _VERIFY_ALLOWLISTED_ENGINES,
+    REFERENCE_BACKEND_NAME,
+)
+from mostlyright.data_harness.ux.commands import CommandSpec
+from mostlyright.data_harness.ux.plan_check import check_plan
+
+# Said in the command's own help, not only in a design document. Both halves matter: it really
+# builds, so what it says is true; and it really deletes, so it can be run again and again.
+DESCRIPTION = (
+    "Build this plan into a folder that is deleted afterwards, to find out whether it works "
+    "before you commit to it. This is the same build the build command runs, with nothing "
+    "skipped and no limit relaxed, so a plan that fails here would have failed there. The data "
+    "fingerprint it reports is the one your real build will produce; the build's own fingerprint "
+    "is not reported, because a dry run is not run by anybody."
+)
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.description = DESCRIPTION
+    parser.add_argument(
+        "plan",
+        type=Path,
+        help="the plan to try",
+    )
+    parser.add_argument(
+        "--input-root",
+        type=Path,
+        required=True,
+        help="the folder your sources are read from",
+    )
+    parser.add_argument(
+        "--engine",
+        choices=sorted(_VERIFY_ALLOWLISTED_ENGINES),
+        default=REFERENCE_BACKEND_NAME,
+        help="which engine to build with; the same choices the build command takes",
+    )
+    parser.add_argument(
+        "--keep",
+        action="store_true",
+        help=(
+            "leave the folder it built into in place and name it, for looking at a failure; "
+            "run chmod -R u+w on it before removing it"
+        ),
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    # Imported here, not at the top: this package is loaded while the command line is still being
+    # set up. Reading the plan through the command line's own reader is what keeps a malformed
+    # plan reading the same here as it does everywhere else.
+    from mostlyright.data_harness import cli
+
+    result = check_plan(
+        cli._read_json(args.plan),
+        input_root=args.input_root,
+        engine=args.engine,
+        keep=args.keep,
+    )
+    return result.to_dict()
+
+
+SPEC = CommandSpec(
+    name="plan-check",
+    help="build this plan into a throwaway folder to see whether it works, then throw it away",
+    add_arguments=add_arguments,
+    run=run,
+)

mostlyright/data_harness/ux/commands/preflight.py

@@ -0,0 +1,97 @@
+"""``mr-data preflight`` -- what would stop a build here, before you start one.
+
+This module is the command and nothing else: its arguments, the one call that turns them into the
+payload both renderings are built from, and the exit code a script can act on. The checks live in
+:mod:`mostlyright.data_harness.ux.preflight`, so the same answer is available to a browser page
+without going through argparse.
+
+Every argument is optional. With none of them the three platform checks still run, because those
+are the ones that cost a whole build to discover the hard way.
+
+``--workload`` is the argument that asks about the *job* rather than about the machine: whether
+this plan's own sources fit the ceilings the engine holds whatever it is run on. It costs one
+``lstat`` per source and opens none of them, and it is the answer that otherwise arrives as a
+container's ``MemoryError`` after the upload and the approval.
+
+The exit code is 1 when anything is blocking, so ``mr-data preflight && mr-data build ...`` does
+what it looks like it does. Nothing is refused and nothing is raised: a machine that cannot build
+is an answer this command was asked for, and it is printed in full either way.
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.pipeline import _VERIFY_ALLOWLISTED_ENGINES
+from mostlyright.data_harness.ux.commands import CommandSpec
+from mostlyright.data_harness.ux.preflight import BLOCKED_KEY, run_preflight
+
+DESCRIPTION = (
+    "Check what would stop a build before you start one. With no arguments it reports what this "
+    "machine itself can and cannot do; name a workbench folder, an input folder, a path to write "
+    "to, or an engine and each of those is checked too. Name a workload -- a workbench folder, a "
+    "plan, or a recipe -- and the job itself is measured against the ceilings the engine will "
+    "judge it by, so a job that cannot run is refused here rather than inside a container after "
+    "its sources have been uploaded. Nothing is built, nothing is written, no source is opened "
+    "and nothing is sent anywhere. The exit code is 1 when something would stop you."
+)
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.description = DESCRIPTION
+    parser.add_argument(
+        "--workspace",
+        type=Path,
+        help="a workbench folder to report the stage of",
+    )
+    parser.add_argument(
+        "--input-root",
+        type=Path,
+        help="the folder your sources would be read from",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        help="the path a build would be written to",
+    )
+    parser.add_argument(
+        "--engine",
+        choices=sorted(_VERIFY_ALLOWLISTED_ENGINES),
+        help="which engine you mean to build with; the same choices the build command takes",
+    )
+    parser.add_argument(
+        "--workload",
+        type=Path,
+        help=(
+            "the job to measure: a workbench folder, a plan, or a recipe. A workbench folder "
+            "already holds its sources; a plan or a recipe names them relative to --input-root"
+        ),
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    return run_preflight(
+        workspace=args.workspace,
+        input_root=args.input_root,
+        output=args.output,
+        engine=args.engine,
+        workload=args.workload,
+    ).to_dict()
+
+
+def exit_code(payload: Mapping[str, Any]) -> int:
+    """1 when something would stop a build, so this reads as a gate in a script."""
+
+    return 1 if payload.get(BLOCKED_KEY) else 0
+
+
+SPEC = CommandSpec(
+    name="preflight",
+    help="check what would stop a build before you start it",
+    add_arguments=add_arguments,
+    run=run,
+    exit_code=exit_code,
+)

mostlyright/data_harness/ux/commands/record.py

@@ -0,0 +1,107 @@
+"""``mr-data record`` -- turn recorded live messages into sealed batches a build can read.
+
+A live connection can never be replayed, so a build never holds one.  Capture is a separate
+job: it takes the messages a venue sent, groups them into bounded batches, seals each batch
+under the digest of its own bytes, and writes down what arrived on which connection.  A build
+then reads those sealed batches and reaches nothing at all.
+
+This command is the second half of that job.  It does not connect to anything.  It reads
+messages somebody already recorded, applies the venue's own rules for grouping and naming them,
+and writes the sealed result.  Opening the connection belongs to a trusted role that does not
+exist here yet, and this build reaches no network anywhere.
+
+Nothing here needs a clock.  Every time in the sealed result comes out of the recording, so the
+same recording always seals to the same bytes and the same names, on any machine, on any day.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.sources.stream_connector import (
+    STREAM_CONNECTOR_IDS,
+    stream_connector,
+)
+from mostlyright.data_harness.sources.stream_recorder import (
+    capture_root,
+    seal_recorded_capture,
+)
+from mostlyright.data_harness.ux.commands import CommandSpec
+
+DESCRIPTION = (
+    "Turn messages already recorded from a live source into sealed batches a build can read. "
+    "This command connects to nothing: you point it at a file of recorded messages and at a "
+    "folder to write into, and it groups, names and seals them under the chosen source's own "
+    "rules. Every dropped connection is written down as one, because a build that spans a hole "
+    "in what was captured has to be able to see the hole."
+)
+
+CAPTURE_RECORD_VERSION = "mr-data-stream-capture.v1"
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.description = DESCRIPTION
+    parser.add_argument(
+        "--connector",
+        choices=STREAM_CONNECTOR_IDS,
+        required=True,
+        help="which live source these messages came from",
+    )
+    parser.add_argument(
+        "--replay",
+        type=Path,
+        required=True,
+        help="the file of recorded messages to seal, one JSON object per line",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        required=True,
+        help="the folder to write the sealed batches into",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    # Imported here, not at the top: this package is loaded while the command line is still
+    # being set up, and the boundary module is what sets it up.
+    from mostlyright.data_harness import cli
+
+    config = stream_connector(args.connector)
+    recording = cli._read_bytes(args.replay)
+    captures_root = args.output.absolute()
+    manifest = seal_recorded_capture(
+        config=config,
+        recording=recording,
+        captures_root=captures_root,
+    )
+    return {
+        "schema_version": CAPTURE_RECORD_VERSION,
+        "status": "stream_recorded",
+        "connector_id": manifest.connector_id,
+        "connector_config_digest": manifest.connector_config_digest,
+        "stream_id": manifest.stream_id,
+        "capture_digest": manifest.digest,
+        "capture_folder": str(capture_root(captures_root, manifest.stream_id)),
+        "data_format": manifest.data_format,
+        "batch_filename": manifest.batch_filename,
+        "connection_count": len(manifest.epochs),
+        "recorded_gap_count": manifest.gap_count,
+        "kept_message_count": manifest.admitted_event_count,
+        "skipped_message_count": manifest.ignored_message_count,
+        "batch_count": len(manifest.batches),
+        "sealed_batch_names": [batch.storage_name for batch in manifest.batches],
+        "bookmarks": [batch.next_bookmark for batch in manifest.batches],
+        "recorded_through": manifest.recorded_through,
+        "network_fetch_performed": False,
+        "live_connection_opened": False,
+    }
+
+
+SPEC = CommandSpec(
+    name="record",
+    help="seal already-recorded live messages into batches a build can read",
+    add_arguments=add_arguments,
+    run=run,
+)

mostlyright/data_harness/ux/commands/review_setup.py

@@ -0,0 +1,47 @@
+"""``mr-data review-setup`` -- create protected material for independent local review."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.ux.commands import CommandSpec
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.description = (
+        "Create protected operator material for one sealed Build. This setup does not approve "
+        "the Build and does not mint a release attestation."
+    )
+    parser.add_argument("workspace", type=Path, help="the work folder containing the sealed Build")
+    parser.add_argument(
+        "--operator-root",
+        type=Path,
+        required=True,
+        help="a new protected folder for enrollment, private keys, and role tokens",
+    )
+    parser.add_argument(
+        "--anchor-output",
+        type=Path,
+        required=True,
+        help="a new file to install separately at the coordinator's fixed trust-anchor path",
+    )
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    from mostlyright.data_harness.operator_setup import provision_review_setup
+
+    return provision_review_setup(
+        args.workspace,
+        operator_root=args.operator_root,
+        anchor_path=args.anchor_output,
+    ).receipt()
+
+
+SPEC = CommandSpec(
+    name="review-setup",
+    help="create protected review material; this does not approve a Build",
+    add_arguments=add_arguments,
+    run=run,
+)