mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/approve.py

@@ -0,0 +1,485 @@
+"""What ``mr-data approve`` derives, and the plain summary of what freezing a Recipe binds.
+
+Approving a Recipe used to mean typing four identifiers correctly -- an approval name, an approver,
+a timestamp, and a policy fingerprint -- and the README's own example passed sixty-four zeros for
+the last one, which is how that goes in practice. Everything here exists so none of those has to be
+typed, and so the person is shown what they are freezing before it is frozen.
+
+Two rules hold this module together.
+
+**Every derived value satisfies the contract by construction.** The validators in
+:mod:`mostlyright.data_harness.recipe` are the truth about what a ``RecipeApproval`` accepts, and
+the derivations below are written against them, not against a copy of their grammar.
+
+**The approver is a person who ran a command.** ``derive_approved_by`` reads the flag the operator
+passed or the owner of this process, and nothing else -- not the environment, not the Recipe, not a
+file. When neither yields a usable name it refuses, because an approval naming the wrong person is
+worse than an approval that did not happen.
+
+Nothing here prints, reads a file, or writes one. :func:`approval_summary` returns structured
+facts for terminal and browser rendering without going through argparse.
+"""
+
+from __future__ import annotations
+
+import os
+import pwd
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from types import MappingProxyType
+from typing import Any
+
+from mostlyright.data_harness.canonical import canonical_json_bytes, sha256_bytes
+from mostlyright.data_harness.local_contracts import GraphTablePlan
+from mostlyright.data_harness.recipe import (
+    FrozenRecipe,
+    RecipeError,
+    _identifier,
+    _timestamp,
+)
+from mostlyright.data_harness.ux.render import render_lines
+
+# The identifier grammar `recipe._IDENT` accepts: a leading letter or digit, then up to 127 more
+# characters drawn from letters, digits, and `_ . -`. The derivations below fit inside it rather
+# than hand the contract a value it would refuse.
+MAX_IDENTIFIER_CHARACTERS = 128
+IDENTIFIER_CHARACTERS = "_.-"
+APPROVAL_ID_PREFIX = "approval-"
+FINGERPRINT_CHARACTERS = 12
+
+# The canonical UTC grammar `recipe._timestamp` accepts, to the second.
+TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+# The approval policy when the operator supplies none: a policy with no rules written down. Its
+# fingerprint is a real sha256 of real canonical bytes, which is the whole point -- a fingerprint
+# of sixty-four zeros means "I did not think about this", and that is worse than no fingerprint.
+#
+# Changing this value changes every fingerprint derived from it. It is a published constant, not
+# an implementation detail: an approval recorded last year can only be re-checked against the same
+# bytes it was hashed from.
+EMPTY_APPROVAL_POLICY: Mapping[str, Any] = MappingProxyType({})
+
+
+# ------------------------------------------------------------------------------------------------
+# The derivations
+# ------------------------------------------------------------------------------------------------
+
+
+def derive_approval_id(recipe: FrozenRecipe) -> str:
+    """Name this approval after the exact Recipe it approves.
+
+    The name carries the Recipe's own name, its version, and the first
+    :data:`FINGERPRINT_CHARACTERS` characters of its fingerprint, so reading the name tells you
+    what was approved. It is derived from the Recipe alone and from nothing else, so approving the
+    same Recipe twice derives the same name: the second run is a visible refusal to overwrite
+    rather than a second approval of the same thing under a different name.
+    """
+
+    tail = f"-v{recipe.recipe_version}-{recipe.digest[:FINGERPRINT_CHARACTERS]}"
+    budget = MAX_IDENTIFIER_CHARACTERS - len(APPROVAL_ID_PREFIX) - len(tail)
+    stem = recipe.recipe_id[:budget].rstrip(IDENTIFIER_CHARACTERS) if budget > 0 else ""
+    derived = f"{APPROVAL_ID_PREFIX}{stem}{tail}" if stem else f"approval{tail}"
+    return _identifier(derived, "approval.approval_id")
+
+
+def derive_approved_by(explicit: str | None = None) -> str:
+    """Name the person approving: the flag they passed, or the owner of this process.
+
+    There is deliberately no third source. The environment is not one of them either: the standard
+    ``getpass.getuser`` reads ``LOGNAME``/``USER``/``LNAME``/``USERNAME`` before it reads the
+    password database, so an approval built on it would name whoever last exported a variable
+    (T-29-26). This reads the process owner from the password database directly.
+
+    A name that cannot be established is a refusal naming ``--approved-by``. Nothing is invented:
+    this field is what the review trail rests on.
+    """
+
+    if explicit is not None:
+        if _as_identifier(explicit) != explicit.strip():
+            raise RecipeError(
+                "RECIPE_APPROVER",
+                "approval.approved_by",
+                "--approved-by must be letters, digits, and - . _ only, starting with a letter "
+                "or digit",
+            )
+        return _identifier(explicit.strip(), "approval.approved_by")
+    try:
+        owner = _process_owner()
+    except OSError as error:
+        raise RecipeError(
+            "RECIPE_APPROVER",
+            "approval.approved_by",
+            "this account has no name on this machine; pass --approved-by to say who is approving",
+        ) from error
+    derived = _as_identifier(owner)
+    if not derived:
+        raise RecipeError(
+            "RECIPE_APPROVER",
+            "approval.approved_by",
+            "the owner of this process has no usable name; pass --approved-by to say who is "
+            "approving",
+        )
+    return _identifier(derived, "approval.approved_by")
+
+
+def derive_approved_at(explicit: str | None = None) -> str:
+    """The moment of approval, in the exact UTC grammar the contract accepts.
+
+    An explicit value is checked rather than trusted, so a test can pin the timestamp and a typo
+    is named at the point it was typed.
+    """
+
+    if explicit is not None:
+        return _timestamp(explicit, "approval.approved_at")
+    return _timestamp(datetime.now(UTC).strftime(TIMESTAMP_FORMAT), "approval.approved_at")
+
+
+def derive_policy_digest(policy_value: Mapping[str, Any] | None = None) -> str:
+    """Fingerprint the approval policy the approver acted under.
+
+    With no policy supplied this is the fingerprint of :data:`EMPTY_APPROVAL_POLICY` -- a real hash
+    of the real canonical bytes of a policy with no rules in it. It is never sixty-four zeros.
+    """
+
+    value = EMPTY_APPROVAL_POLICY if policy_value is None else policy_value
+    if not isinstance(value, Mapping):
+        raise RecipeError(
+            "RECIPE_APPROVAL_POLICY",
+            "approval.approval_policy",
+            "an approval policy must be a JSON object",
+        )
+    try:
+        raw = canonical_json_bytes(dict(value))
+    except (TypeError, ValueError) as error:
+        raise RecipeError(
+            "RECIPE_APPROVAL_POLICY",
+            "approval.approval_policy",
+            "an approval policy must hold JSON values only",
+        ) from error
+    return sha256_bytes(raw)
+
+
+def _process_owner() -> str:
+    """The name of the account this process runs as, read from the password database."""
+
+    try:
+        return pwd.getpwuid(os.getuid()).pw_name
+    except KeyError as error:
+        raise OSError("this user id has no password database entry") from error
+
+
+def _as_identifier(value: str) -> str:
+    """``value`` reduced to the identifier grammar, or the empty string when nothing survives."""
+
+    kept = [
+        character
+        for character in value.strip()
+        if character.isascii() and (character.isalnum() or character in IDENTIFIER_CHARACTERS)
+    ]
+    while kept and not (kept[0].isascii() and kept[0].isalnum()):
+        kept.pop(0)
+    return "".join(kept)[:MAX_IDENTIFIER_CHARACTERS]
+
+
+# ------------------------------------------------------------------------------------------------
+# What you are freezing
+# ------------------------------------------------------------------------------------------------
+
+# Plain sentences for the checks the pinned engine always runs. An unknown check id falls through
+# to its own name, so a new engine check appears in the summary the day it ships.
+_CHECK_SENTENCES: Mapping[str, str] = MappingProxyType(
+    {
+        "EXACT_CANDIDATE_REPLAY": "the sealed result replays byte for byte",
+        "RECIPE_DRIFT": "the result stays inside the refresh bounds below",
+        "SOURCE_AUTHORITY": "every input file matches the fingerprint the run pins",
+        "OUTPUT_SEMANTICS": "every column keeps the type and meaning declared above",
+        "TEMPORAL_INTEGRITY": "dates and times stay inside the approved window",
+    }
+)
+
+_RELATIONSHIPS: Mapping[str, str] = MappingProxyType(
+    {
+        "one_to_one": "one row on each side matches at most one row on the other",
+        "many_to_one": "many rows on the left match one row on the right",
+    }
+)
+
+_CLEANING_SENTENCES: Mapping[str, str] = MappingProxyType(
+    {
+        "trim": "trim the spaces around {columns} in {source}",
+        "empty_to_null": "treat empty text as missing in {columns} of {source}",
+        "rename": "rename {columns} in {source}",
+        "cast": "read {columns} in {source}",
+    }
+)
+
+_CLOSING = (
+    "Freezing is permanent: this fingerprint is what runs from now on, it can be replayed for as "
+    "long as the inputs are kept, and any change at all means a new Recipe with a new fingerprint. "
+    "Nothing is approved until this command finishes, and nothing can approve it for you."
+)
+
+
+@dataclass(frozen=True)
+class ApprovalSummary:
+    """Everything approving one Recipe binds, in the words a person reads before deciding.
+
+    Every field is derived from one parsed :class:`FrozenRecipe` and from no second source, so the
+    summary cannot describe something other than what the fingerprint binds (T-29-27).
+    """
+
+    recipe: str
+    recipe_digest: str
+    question: str
+    one_row_is: str
+    population: str
+    columns: tuple[str, ...]
+    sources: tuple[str, ...]
+    cleaning: tuple[str, ...]
+    join: str
+    checks: tuple[str, ...]
+    refresh_bounds: str
+    backfill_window: str
+    engine: str
+    what_it_is_for: str
+    plan_shape: str = "joined"
+    closing: str = _CLOSING
+
+    def to_dict(self) -> dict[str, Any]:
+        """The structured facts, in the order a person reads them.
+
+        Facts taken from the Recipe -- column names, source names, check names -- go in values,
+        never in keys, so nothing the data happens to be called is shown as a label.
+        """
+
+        steps_label = (
+            "the ordered graph steps" if self.plan_shape == "graph" else "the cleaning steps"
+        )
+        structure_label = "the graph output" if self.plan_shape == "graph" else "the join"
+        return {
+            "what this answers": self.question,
+            "one row is": self.one_row_is,
+            "it covers": self.population,
+            "what the result may be used for": self.what_it_is_for,
+            "the columns": _numbered("column", self.columns),
+            "the sources": _numbered("source", self.sources),
+            steps_label: _numbered("step", self.cleaning),
+            structure_label: self.join,
+            "the checks it must pass": _numbered("check", self.checks),
+            "the refresh bounds": self.refresh_bounds,
+            "the backfill window": self.backfill_window,
+            "the pinned versions": self.engine,
+            "the recipe": self.recipe,
+            "the fingerprint being frozen": self.recipe_digest,
+            "what freezing means": self.closing,
+        }
+
+    def lines(self) -> tuple[str, ...]:
+        """The terminal rendering of :meth:`to_dict`, and not a second truth.
+
+        The same renderer the command line uses for every other payload is used here, so what a
+        person reads on the terminal and what a caller reads under ``--json`` are the same facts
+        in the same order.
+        """
+
+        return tuple(render_lines(self.to_dict()))
+
+
+def approval_summary(recipe: FrozenRecipe) -> ApprovalSummary:
+    """Say, in plain words, exactly what approving ``recipe`` freezes."""
+
+    plan = recipe.transform_plan
+    requirements = recipe.requirements
+    proposals = {item.source_id: item for item in recipe.source_proposals}
+    plan_sources = {item.source_id: item for item in plan.sources}
+    semantics = {item.name: item for item in recipe.output_semantics}
+    predecessor = (
+        "no earlier version"
+        if recipe.predecessor_recipe_digest is None
+        else f"replacing the version fingerprinted {recipe.predecessor_recipe_digest}"
+    )
+    covers = requirements.time_range
+    graph_plan = isinstance(plan, GraphTablePlan)
+    return ApprovalSummary(
+        recipe=f"{recipe.recipe_id}, version {recipe.recipe_version}, {predecessor}",
+        recipe_digest=recipe.digest,
+        question=recipe.question.text,
+        one_row_is=requirements.output_grain.description,
+        population=(
+            f"{requirements.population} From {covers.start_inclusive} up to {covers.end_exclusive}."
+        ),
+        columns=tuple(_column_sentence(semantics[name]) for name in plan.select),
+        sources=tuple(
+            _source_sentence(
+                source,
+                plan_sources.get(source.source_id),
+                proposals.get(source.source_id),
+            )
+            for source in sorted(recipe.sources, key=lambda item: item.source_id)
+        ),
+        cleaning=(
+            tuple(_graph_node_sentence(node) for node in plan.nodes)
+            if graph_plan
+            else tuple(_cleaning_sentence(step) for step in plan.cleaning)
+        ),
+        join=_graph_output_sentence(plan) if graph_plan else _join_sentence(plan.join),
+        checks=_check_sentences(recipe, plan.quality),
+        refresh_bounds=_refresh_sentence(recipe.refresh_policy),
+        backfill_window=_backfill_sentence(recipe.backfill_policy),
+        engine=(
+            f"engine {recipe.engine_version}, operations {recipe.operation_contract_version}, "
+            f"readers {recipe.reader_contract_version}, "
+            f"PyArrow {recipe.pyarrow_version}, checks {recipe.validation_policy_version} "
+            f"({recipe.validation_policy_digest})"
+        ),
+        what_it_is_for=_intent_sentence(plan.output_intent),
+        plan_shape="graph" if graph_plan else "joined",
+    )
+
+
+# What the frozen intent means, said as a permission rather than as the identifier it is stored
+# under. These are the three `local_contracts._OUTPUT_INTENTS`, and the guard below fails when that
+# set grows a fourth, because an intent with no sentence here would be frozen unstated.
+_INTENT_SENTENCES: Mapping[str, str] = MappingProxyType(
+    {
+        "local_use": "use on this machine only -- not published, not hosted",
+        "redistribute": "passing the result on to others",
+        "host": "serving the result from a hosted service",
+    }
+)
+
+
+def _intent_sentence(intent: str) -> str:
+    """What this Recipe declares the result is for, and what that costs every source.
+
+    This is not decoration beside the rights already listed per source. It is the frozen claim the
+    build enforces: `pipeline` refuses to build when a source's recorded permissions do not include
+    this intent, so freezing `redistribute` and freezing `local_use` are two different approvals of
+    the same data, and a summary that shows neither asked somebody to agree to whichever one it
+    happened to be.
+    """
+
+    meaning = _INTENT_SENTENCES.get(intent)
+    plain = f"{intent}: {meaning}" if meaning is not None else intent
+    return f"{plain}. Every source above has to permit it, and a build stops when one does not."
+
+
+def _column_sentence(field: Any) -> str:
+    parts = [f"{field.name}: {field.logical_type}, {field.semantic_type}"]
+    if field.unit not in {"none", "count"}:
+        parts.append(f"in {field.unit}")
+    elif field.unit == "count":
+        parts.append("a count")
+    if field.temporal_role != "none":
+        parts.append(f"used as the {field.temporal_role.replace('_', ' ')}")
+    if field.timezone is not None:
+        parts.append(f"in {field.timezone}")
+    return ", ".join(parts)
+
+
+def _source_sentence(source: Any, plan_source: Any, proposal: Any) -> str:
+    parts = [f"{source.source_id}: {source.plan_source_path} ({source.data_format})"]
+    if proposal is not None:
+        parts.append(f"chosen as {proposal.display_name}, rights {proposal.rights_status}")
+    if plan_source is not None:
+        rights = plan_source.rights
+        parts.append(f"from {plan_source.origin}, obtained by {plan_source.acquisition_method}")
+        parts.append(f"rights {rights.status}, may {', '.join(rights.permissions)}")
+        parts.append(f"rights evidence {rights.evidence}")
+    parts.append(f"opened by {source.adapter_id} {source.adapter_version}")
+    parts.append(f"treated as {source.declared_classification}, under {source.lawful_basis}")
+    parts.append(
+        "keeps no Bookmark"
+        if source.watermark_kind == "none"
+        else f"Bookmark from {source.watermark_field} ({source.watermark_kind})"
+    )
+    return "; ".join(parts)
+
+
+def _cleaning_sentence(step: Any) -> str:
+    columns = step.columns
+    if columns and isinstance(columns[0], tuple):
+        written = ", ".join(f"{name} as {target}" for name, target in columns)
+    else:
+        written = ", ".join(str(name) for name in columns)
+    template = _CLEANING_SENTENCES.get(step.operation)
+    if template is None:
+        return f"{step.operation} {written} in {step.source_id}"
+    return template.format(columns=written, source=step.source_id)
+
+
+def _join_sentence(join: Any) -> str:
+    relationship = _RELATIONSHIPS.get(join.cardinality, join.cardinality)
+    return (
+        f"{join.left} joined to {join.right} on {', '.join(join.on)} "
+        f"({join.kind} join): {relationship} ({join.cardinality})"
+    )
+
+
+def _graph_node_sentence(node: Any) -> str:
+    inputs = ", ".join(node.inputs) if node.inputs else "none"
+    parameters = canonical_json_bytes(node.to_dict()["parameters"]).decode("utf-8")
+    return (
+        f"{node.node_id}: {node.operation} {node.operation_version}; "
+        f"inputs: {inputs}; parameters {parameters}"
+    )
+
+
+def _graph_output_sentence(plan: GraphTablePlan) -> str:
+    join_nodes = tuple(node.node_id for node in plan.nodes if node.operation == "join")
+    joins = f"join steps {', '.join(join_nodes)}" if join_nodes else "no join step"
+    return (
+        f"the closed graph ends at {plan.terminal} after {len(plan.nodes)} ordered steps; {joins}"
+    )
+
+
+def _check_sentences(recipe: FrozenRecipe, quality: Any) -> tuple[str, ...]:
+    written = [f"at least {quality.min_rows} rows"]
+    if quality.not_null:
+        written.append(f"never missing: {', '.join(quality.not_null)}")
+    written.append(f"each {', '.join(recipe.transform_plan.grain)} appears exactly once")
+    written.extend(
+        f"{check_id}: {_CHECK_SENTENCES.get(check_id, 'the pinned engine check of that name')}"
+        for check_id in recipe.required_check_ids
+    )
+    return tuple(written)
+
+
+def _refresh_sentence(policy: Any) -> str:
+    additions = (
+        "new columns are allowed" if policy.allow_column_additions else "new columns are refused"
+    )
+    return (
+        f"a later run must return between {policy.min_row_count_percent}% and "
+        f"{policy.max_row_count_percent}% of the rows this version has, and {additions}"
+    )
+
+
+def _backfill_sentence(policy: Any) -> str:
+    envelope = policy.coverage_envelope
+    scope = policy.partition_scope
+    fields = ", ".join(scope.fields) if scope.fields else "the whole result"
+    pieces = scope.max_partitions_per_run
+    return (
+        f"backfills may cover {envelope.start_inclusive} up to {envelope.end_exclusive}, "
+        f"at most {_duration(policy.max_window_seconds)} in one run, "
+        f"over {fields} in at most {pieces} {'piece' if pieces == 1 else 'pieces'} per run, "
+        f"overlapping released coverage: {policy.overlap_policy}"
+    )
+
+
+def _duration(seconds: int) -> str:
+    """Seconds, with the plain reading of them alongside when it is an exact number of days."""
+
+    if seconds % 86_400 == 0:
+        days = seconds // 86_400
+        return f"{seconds} seconds ({days} {'day' if days == 1 else 'days'})"
+    return f"{seconds} seconds"
+
+
+def _numbered(noun: str, items: Sequence[str]) -> dict[str, str]:
+    """One numbered key per item, so the order a person reads is the order the Recipe runs."""
+
+    width = len(str(max(len(items), 1)))
+    return {f"{noun} {index:0{width}d}": item for index, item in enumerate(items, start=1)}