mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ux/inventory.py

@@ -0,0 +1,427 @@
+"""What this build can actually do, derived from the sets the code itself is held to.
+
+Until now the only way to find out which cleaning steps exist, which types a cast may target, or
+which engines this build has was to read the source. An agent driving this harness had no way at
+all, so it guessed, and a guess about capability becomes a plan that cannot be built.
+
+**Every list here is imported, never re-typed.** The frozensets below are the same objects the
+contract checker measures a Recipe against. ``tests/test_ux_inventory.py`` asserts each rendered
+list equals the sorted frozenset, so an inventory that has drifted from what the code does fails a
+test rather than misleading somebody.
+
+Three sentences in this file are the reason it is worth shipping at all, and each of them is a lie
+the obvious version tells:
+
+**Six operations.** Four cleaning steps a Recipe may name, plus joining and choosing which columns
+to keep. The last two are operations in every sense that matters -- they are what an engine
+restriction would key on -- and an inventory that prints only four undercounts what this build does.
+
+**The restriction table is empty, and that is said out loud.** No operation is restricted on any
+engine. Omitting the section would read as "unknown"; the answer is "none", and those are different
+answers to the question an agent is asking.
+
+**Five source formats, one sealed table format.** ``_DATA_FORMATS`` is the vocabulary a source
+option may declare. The build path consumes canonical CSV directly or as normalized output from a
+certified Reader. Printing five formats without that sentence beside them is the single most
+likely way this command causes harm, so the two facts are in one section and a test holds them
+there.
+
+**Six column types, all executable.** Recipe declarations and local plan casts share one
+authoritative vocabulary. ``timestamp_utc`` includes the timezone in the name because the Recipe
+requires UTC; nothing here calls a naive timestamp UTC.
+
+The checks are the ones the shipping build path records (``MIN_ROWS``, ``NOT_NULL_<column>``,
+``UNIQUE_GRAIN``), not ``repair.contracts.ALL_QUALITY_CHECK_IDS``. Those nine gates belong to a part
+of the tree that is imported only by ``repair/`` and reachable from no command; a test asserts the
+two sets do not overlap, and that inequality is the correct outcome rather than a bug.
+
+Nothing here reads the filesystem, and nothing here prints. :meth:`Inventory.to_dict` returns the
+one payload both renderings are built from.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from mostlyright.data_harness.backends.restrictions import _ENGINE_RESTRICTED_OPS
+from mostlyright.data_harness.local_contracts import (
+    _ACQUISITION_METHODS,
+    _CAST_TYPES,
+    _CLEANING_OPERATIONS,
+    _DATA_FORMATS,
+    _FEASIBILITY_DECISIONS,
+    _FITNESS_DECISIONS,
+    _JOIN_CARDINALITIES,
+    _LIVE_ENDPOINTS,
+    _LOCATOR_KINDS,
+    _OUTPUT_INTENTS,
+    _PERMISSIONS,
+    _PROPOSAL_RIGHTS,
+    _RIGHTS_STATUSES,
+    _TARGET_POLICIES,
+)
+from mostlyright.data_harness.operation_registry import (
+    OPERATION_REGISTRY,
+    RETIRED_OPERATIONS,
+)
+from mostlyright.data_harness.pipeline import _VERIFY_ALLOWLISTED_ENGINES
+from mostlyright.data_harness.readers.registry import TOOLBOX
+from mostlyright.data_harness.recipe import LOGICAL_TYPES
+from mostlyright.data_harness.ux.render import render_human
+
+# Every Recipe logical type is a target of the local plan's deterministic cast step. This text is
+# derived from the compatibility alias so inventory cannot drift from plan validation.
+_CASTABLE = ", ".join(sorted(_CAST_TYPES))
+
+# The one status this command reports.
+INVENTORY_STATUS = "inventory_listed"
+
+# The sections, by the name a person reads and types after ``--section``. They are keys in the
+# payload as well as values, because a section name is a name this command chose rather than one
+# taken from the data, so relabelling it would not invent a name the data does not have.
+CLEANING_STEPS = "cleaning steps"
+COLUMN_TYPES = "column types"
+JOINS = "joins"
+ENGINES = "engines"
+ENGINE_RESTRICTIONS = "engine restrictions"
+CHECKS = "checks"
+SOURCE_FORMATS = "source formats"
+READERS = "Reader coordinates"
+SOURCE_LOCATIONS = "source locations"
+ARRIVAL = "how a source arrives"
+RIGHTS_STATUSES = "rights statuses"
+RIGHTS_CLAIMS = "rights claims"
+RIGHTS_ALLOW = "what the rights allow"
+BUILD_INTENT = "what a build is for"
+SOURCE_FIT = "source fit"
+ADDRESS_HEALTH = "address health"
+TARGETS = "requirement targets"
+FEASIBILITY = "feasibility"
+DATASET_GRAPH = "dataset graph operations"
+
+# The sentence the empty restriction table is reported with. A constant because a test pins it: the
+# difference between this sentence and an absent section is the difference between "none" and
+# "unknown".
+NO_RESTRICTIONS = "no operation is restricted on any engine"
+
+# The two operation identities that are not cleaning steps. Named here rather than counted from a
+# plan, because this is the surface `backends/restrictions.py` keys on, and it says so in its own
+# docstring: each cleaning step's operation name, plus joining and choosing columns.
+_OPERATION_IDENTITIES = frozenset({"join", "select"})
+
+# The checks the shipping build path records, in the order `pipeline._validate_output` records
+# them. The middle one is written with its column placeholder because there is one per column that
+# may not be empty, and a list of the columns of somebody else's Recipe would be no answer at all.
+_SHIPPING_CHECKS: tuple[str, ...] = ("MIN_ROWS", "NOT_NULL_<column>", "UNIQUE_GRAIN")
+
+
+@dataclass(frozen=True)
+class InventorySection:
+    """One closed set, and the sentence a reader needs in order not to misread it.
+
+    ``note`` is never optional. A list of words with no sentence beside it is where an agent
+    invents capability: five source formats read as five readable formats unless something on the
+    same screen says otherwise.
+    """
+
+    name: str
+    names: tuple[str, ...]
+    note: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"count": len(self.names), "names": list(self.names), "note": self.note}
+
+
+@dataclass(frozen=True)
+class Inventory:
+    """Every closed set this build is held to, in the order a person reads them."""
+
+    sections: tuple[InventorySection, ...]
+
+    def section(self, name: str) -> InventorySection:
+        for section in self.sections:
+            if section.name == name:
+                return section
+        raise KeyError(name)
+
+    def to_dict(self, *, section: str | None = None) -> dict[str, Any]:
+        """The one payload both renderings are built from, filtered to one section on request."""
+
+        chosen = self.sections if section is None else (self.section(section),)
+        return {
+            "status": INVENTORY_STATUS,
+            "sections": {item.name: item.to_dict() for item in chosen},
+        }
+
+    def lines(self, *, section: str | None = None) -> list[str]:
+        """Exactly the plain lines the command line prints, from exactly the same payload."""
+
+        return render_human(self.to_dict(section=section)).splitlines()
+
+
+def _retired_sentence() -> str:
+    """Say which listed operations are retired, in the same words for however many there are.
+
+    Derived from the live table: a name that stops being retired stops being named here, and one
+    that starts being retired starts being named, without this sentence being edited.
+    """
+
+    retired = tuple(name for name in OPERATION_REGISTRY if name in RETIRED_OPERATIONS)
+    if not retired:
+        return "Every one of them may be named by a new plan."
+    if len(retired) == 1:
+        subject, pronoun = f"{retired[0]} is retired", "it"
+    else:
+        subject = f"{', '.join(retired[:-1])} and {retired[-1]} are retired"
+        pronoun = "one of them"
+    return (
+        f"{subject}: still executed when replaying a Build that was sealed with {pronoun}, and "
+        "refused in any newly authored plan. Every other one may be named by a new plan."
+    )
+
+
+def build_inventory() -> Inventory:
+    """Read every closed set out of the live code and describe it.
+
+    Called fresh each time rather than computed once at import, so a test may put an entry in the
+    restriction table and see the sentence about it change.
+    """
+
+    return Inventory(
+        sections=(
+            InventorySection(
+                name=DATASET_GRAPH,
+                names=tuple(OPERATION_REGISTRY),
+                note=(
+                    "The closed local-graph-table-plan.v1 graph operations executed by the shared "
+                    "deterministic kernel; order is contract-significant and every node emits "
+                    "sealed operation evidence. " + _retired_sentence()
+                    # Read off the live table rather than written out, so a name cannot be
+                    # retired in the registry and stay advertised as available here.  This list
+                    # is what an agent reads before proposing operations, so a retired name
+                    # listed with no sentence beside it is exactly where one gets written into a
+                    # new plan.
+                ),
+            ),
+            InventorySection(
+                name=CLEANING_STEPS,
+                names=tuple(sorted(set(_CLEANING_OPERATIONS) | set(_OPERATION_IDENTITIES))),
+                note=(
+                    "There are six, and these are all of them: four steps a Recipe may name over "
+                    "a column, plus joining two sources and choosing which columns to keep."
+                ),
+            ),
+            InventorySection(
+                name=COLUMN_TYPES,
+                names=tuple(sorted(LOGICAL_TYPES)),
+                note=(
+                    "The types a column may be declared as. A value that will not hold exactly in "
+                    "the type you asked for stops the build; it is never rounded or blanked. "
+                    f"A cast step inside a plan may target every type: {_CASTABLE}. "
+                    "timestamp_utc means "
+                    "microseconds at UTC and nothing else -- a column at another offset, or with "
+                    "no zone at all, is not that type."
+                ),
+            ),
+            InventorySection(
+                name=JOINS,
+                names=tuple(sorted(_JOIN_CARDINALITIES)),
+                note=(
+                    "The relationship a Recipe declares between two sources. Every row on the "
+                    "left must find its match, and one row in must give one row out."
+                ),
+            ),
+            InventorySection(
+                name=ENGINES,
+                names=tuple(sorted(_VERIFY_ALLOWLISTED_ENGINES)),
+                note=(
+                    "Which engine does the work is chosen when you build, not written into a "
+                    "Recipe. All three produce the same derived data members; execution evidence "
+                    "records the chosen engine, so the complete Build digest intentionally "
+                    "differs. The sealed known-good samples check that narrower data-byte parity."
+                ),
+            ),
+            _restriction_section(),
+            InventorySection(
+                name=CHECKS,
+                names=_SHIPPING_CHECKS,
+                note=(
+                    "Every build must pass these before it is sealed: the fewest rows you will "
+                    "accept, one check per column that may not be empty, and that no two rows "
+                    "describe the same thing."
+                ),
+            ),
+            InventorySection(
+                name=SOURCE_FORMATS,
+                names=tuple(sorted(_DATA_FORMATS)),
+                note=(
+                    "A source option may declare any of these five. The build consumes canonical "
+                    "CSV directly or as normalized output from a certified Reader; the Reader "
+                    "coordinates section is the executable source-format authority. Executable "
+                    "support is limited to the certified Reader coordinates listed below."
+                ),
+            ),
+            InventorySection(
+                name=READERS,
+                names=tuple(f"{family_id}@{version}" for family_id, version in TOOLBOX.families()),
+                note=(
+                    "These are certified Toolbox coordinates. On validated Ubuntu both the "
+                    "stock user.file path and the generic public-HTTPS path route every "
+                    "coordinate; coordinate-specific optional dependencies, settings, and "
+                    "budgets still apply. Every other host reports typed Reader "
+                    "unavailability, macOS included since ADR 0021, because it has no "
+                    "enforceable Reader memory boundary."
+                ),
+            ),
+            InventorySection(
+                name=SOURCE_LOCATIONS,
+                names=tuple(sorted(_LOCATOR_KINDS)),
+                note=(
+                    "Where a source option says its data is. This build reads a file at a path "
+                    "under the input folder you name; the rest are what a source option may "
+                    "declare."
+                ),
+            ),
+            InventorySection(
+                name=ARRIVAL,
+                names=tuple(sorted(_ACQUISITION_METHODS)),
+                note=(
+                    "How the bytes reached your input folder. It is recorded with the Build and "
+                    "sealed into its Receipt; adapter and Reader acquisition evidence is reported "
+                    "separately when the public local-file Reader path is used."
+                ),
+            ),
+            InventorySection(
+                name=RIGHTS_STATUSES,
+                names=tuple(sorted(_RIGHTS_STATUSES)),
+                note=(
+                    "What a Recipe records about its right to read one file. A build refuses "
+                    "outright on unknown and on prohibited."
+                ),
+            ),
+            InventorySection(
+                name=RIGHTS_CLAIMS,
+                names=tuple(sorted(_PROPOSAL_RIGHTS)),
+                note=(
+                    "What a source option claims about its own rights, before anything is built. "
+                    "This is a claim being weighed, not the basis a build reads under."
+                ),
+            ),
+            InventorySection(
+                name=RIGHTS_ALLOW,
+                names=tuple(sorted(_PERMISSIONS)),
+                note=(
+                    "What the recorded rights permit doing with the data: using it on your own "
+                    "machine, passing it on, or running it as a live dataset."
+                ),
+            ),
+            InventorySection(
+                name=BUILD_INTENT,
+                names=tuple(sorted(_OUTPUT_INTENTS)),
+                note=(
+                    "What a Recipe says its Build is for. It is checked against what every "
+                    "source's rights allow, and the narrower of the two wins."
+                ),
+            ),
+            InventorySection(
+                name=SOURCE_FIT,
+                names=tuple(sorted(_FITNESS_DECISIONS)),
+                note=(
+                    "Whether a source option was taken, kept as a fallback, or turned down. Only "
+                    "a chosen one reaches a Recipe."
+                ),
+            ),
+            InventorySection(
+                name=ADDRESS_HEALTH,
+                names=tuple(sorted(_LIVE_ENDPOINTS)),
+                note=(
+                    "What a source option records about the address it names. It is a recorded "
+                    "observation, not something this build goes and re-checks for you."
+                ),
+            ),
+            InventorySection(
+                name=TARGETS,
+                names=tuple(sorted(_TARGET_POLICIES)),
+                note=(
+                    "Whether a requirement's target has to be met once it turns out to be "
+                    "reachable at all. A newly authored Recipe can only use none: the one "
+                    "operation that ever built a target is retired, and asking for a target "
+                    "that has to be met leaves a Recipe nothing to meet it with."
+                ),
+            ),
+            InventorySection(
+                name=FEASIBILITY,
+                names=tuple(sorted(_FEASIBILITY_DECISIONS)),
+                note=(
+                    "The verdict recorded against the question itself, before any source is "
+                    "chosen. A question recorded as unsupported is not built."
+                ),
+            ),
+        )
+    )
+
+
+def _restriction_section() -> InventorySection:
+    """The per-operation, per-engine restriction table, whether it holds anything or not.
+
+    Today it holds nothing: every operation reaches the same bytes on all three engines, and the
+    sealed known-good samples are what establishes that. The sentence is derived from the table
+    rather than written under it, so the day an entry is added the section stops saying none.
+    """
+
+    if not _ENGINE_RESTRICTED_OPS:
+        return InventorySection(
+            name=ENGINE_RESTRICTIONS,
+            names=(),
+            note=(
+                f"There is a table of operations some engine may not run, and it is empty: "
+                f"{NO_RESTRICTIONS}. This section is printed empty on purpose, because a section "
+                f"that was left out would read as unknown and the answer is none."
+            ),
+        )
+    return InventorySection(
+        name=ENGINE_RESTRICTIONS,
+        names=tuple(
+            f"{operation}: not on {', '.join(sorted(engines))}"
+            for operation, engines in sorted(_ENGINE_RESTRICTED_OPS.items())
+        ),
+        note=(
+            "These operations cannot reach the same bytes on the engines named beside them, so "
+            "asking for that pairing is refused rather than built."
+        ),
+    )
+
+
+# The sections that exist, derived from the inventory itself so the command line's list of what you
+# may ask for cannot fall behind what there is to ask for.
+SECTION_NAMES: tuple[str, ...] = tuple(section.name for section in build_inventory().sections)
+
+
+__all__ = [
+    "ADDRESS_HEALTH",
+    "ARRIVAL",
+    "BUILD_INTENT",
+    "CHECKS",
+    "CLEANING_STEPS",
+    "COLUMN_TYPES",
+    "DATASET_GRAPH",
+    "ENGINES",
+    "ENGINE_RESTRICTIONS",
+    "FEASIBILITY",
+    "INVENTORY_STATUS",
+    "JOINS",
+    "NO_RESTRICTIONS",
+    "READERS",
+    "RIGHTS_ALLOW",
+    "RIGHTS_CLAIMS",
+    "RIGHTS_STATUSES",
+    "SECTION_NAMES",
+    "SOURCE_FIT",
+    "SOURCE_FORMATS",
+    "SOURCE_LOCATIONS",
+    "TARGETS",
+    "Inventory",
+    "InventorySection",
+    "build_inventory",
+]