mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/readers/contracts.py

@@ -0,0 +1,542 @@
+"""The contract every Reader family implements, and the typed failures it may raise.
+
+A Reader family is the code that turns fetched bytes into rows.  This module states the
+shape of that hand-off and nothing else: it decodes nothing, registers nothing, and reaches
+no filesystem.  Four things live here.
+
+``ReaderError``
+    The one failure type, carrying a stable machine-readable code, the subject that failed,
+    and a human detail.  The codes are a closed table, ``READER_ERROR_CODES``, validated at
+    construction so a typo becomes a failure instead of an unmapped error string. The table is
+    the interface consumed by error remediation, so codes are added here and nowhere else.
+
+``ReaderPin``
+    The three fields a recipe seals to say which code decoded its bytes and with which
+    settings.  The pin is all-present or all-absent, never half-stated, and its
+    ``options_digest`` is computed exactly the way the recipe computes its own
+    ``decode_options_digest``: sha256 over ``canonical_json_bytes`` of the options mapping.
+    It is never computed a second way.
+
+``ReaderBudgets``
+    The resource ceilings a family decodes under.  ``narrowed_by`` takes a per-field minimum
+    against a caps mapping, so a recipe may narrow a budget and may never widen one.  The
+    caps mapping is accepted as a plain mapping rather than the recipe's own typed object,
+    because importing ``recipe`` here would close an import cycle.
+
+``ReaderResult`` and ``ReaderFamily``
+    What a decode returns and what a family must offer.  A decode receives exact bytes and
+    returns exact bytes and touches no filesystem: no temporary directory, no extraction, no
+    spill.
+
+Decode options reach this module already screened for secrets and already bounded by the
+recipe layer.  This module states that dependency rather than duplicating the screen; a
+second, differently-worded screen would be a second declaration of the same rule.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass, fields
+from types import MappingProxyType
+from typing import Any, Protocol
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    sha256_bytes,
+)
+from mostlyright.data_harness.formats import (
+    DATA_FORMATS,
+    FORMAT_MEDIA_TYPES,
+    READER_CONTRACT_VERSION,
+)
+
+__all__ = [
+    "DECODE_FLAGS",
+    "READER_CONTRACT_VERSION",
+    "READER_ERROR_CODES",
+    "ReaderBudgets",
+    "ReaderError",
+    "ReaderFamily",
+    "ReaderPin",
+    "ReaderResult",
+    "identifier",
+    "semver",
+]
+
+# Error remediation consumes this closed table. Add new codes here, not at individual raise sites.
+READER_ERROR_CODES: frozenset[str] = frozenset(
+    {
+        # Resolution and contract shape.
+        "READER_FAMILY_UNKNOWN",
+        "READER_DEPENDENCY_UNAVAILABLE",
+        "READER_CONTRACT",
+        "READER_OPTIONS",
+        "READER_ADMISSION",
+        "READER_BUDGET",
+        # Container members.
+        "READER_MEMBER_MISSING",
+        "READER_MEMBER_AMBIGUOUS",
+        "READER_MEMBER_TYPE",
+        "READER_NESTED_CONTAINER",
+        # Spreadsheet coordinates.
+        "READER_SHEET_MISSING",
+        "READER_RANGE",
+        # Weather-model admission, refused before decode.
+        "READER_GRID_UNKNOWN",
+        "READER_PACKING_UNKNOWN",
+        # Bytes a family admitted and then could not decode: truncated, corrupted in transit, or
+        # damaged at the publisher.  It is separate from the admission codes on purpose, because
+        # the fix is different in kind: an admission refusal is repaired by editing the recipe or
+        # by widening a reviewed allowlist, and this one is repaired by fetching the file again.
+        # A family whose decoder reports its own failures translates them into this code rather
+        # than letting the decoder's exception type escape, since an untyped failure carries no
+        # code for the remediation map to bind to.
+        "READER_DECODE",
+        # Output and the pre-refresh warm-up check.
+        "READER_OUTPUT",
+        "READER_WARMUP",
+    }
+)
+
+# The closed vocabulary of facts a decode may report about itself.  A flag is a fact the
+# receipt carries forward, not a warning: "this spreadsheet held formulas, and the values
+# read were the ones the authoring application had cached" is a provenance statement a
+# reader of the dataset is owed.  Flags are sorted and de-duplicated on the result, so two
+# runs of the same decode produce the same tuple and the same receipt bytes.
+#
+# ``contains_masked_points`` is the weather family's: a message may carry a bitmap marking points
+# that hold no value, and a point the recipe named which the message marked absent becomes an
+# empty cell.  Without the flag that empty cell is ambiguous -- it reads the same as a Reader that
+# had a problem -- and with it the receipt states that the publisher marked the point absent.
+DECODE_FLAGS: frozenset[str] = frozenset(
+    {"contains_formula_cells", "contains_cached_values", "contains_masked_points"}
+)
+
+# Identity and version rules, spelled the way the recipe spells them.  They are duplicated
+# rather than imported because importing ``recipe`` here would close an import cycle; a test
+# asserts the two patterns letter for letter so the duplication cannot drift.
+_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
+_SEMVER = re.compile(r"^[1-9][0-9]*\.[0-9]+\.[0-9]+$")
+
+# The same ceiling the recipe applies to a sealed decode-options object.
+_MAX_DECODE_OPTIONS_BYTES = 256 * 1024
+
+# A sealed snapshot filename is a bare name in a coordinator-owned directory: no separator,
+# no parent reference, no leading dot.
+_FILENAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$")
+
+# Nesting depth is fixed rather than tunable.  A budget that could ask for depth 2 would be
+# the archive-in-archive bug class re-entering by configuration, so the only admissible
+# value is 1 and any other value is refused where the budget is built.
+_FIXED_NESTING_DEPTH = 1
+
+
+# The attributes the interpreter owns rather than this class.  Sealing them seals nothing: an
+# exception whose traceback cannot be set is not immutable, it is broken.  ``contextlib`` and
+# the exception-chaining machinery assign ``__traceback__`` at Python level, so a Reader
+# refusal crossing a context manager would surface as ``AttributeError: cannot set
+# '__traceback__'`` instead of as its own code -- the loudest failure replaced by a confusing
+# one.  What must not be rewritable is the code, the subject, and the detail, and those stay
+# sealed exactly as they were.
+_INTERPRETER_OWNED_ATTRIBUTES = frozenset(
+    {"__traceback__", "__cause__", "__context__", "__suppress_context__", "__notes__"}
+)
+
+
+class ReaderError(ValueError):
+    """One stable, fail-closed Reader failure: a code, the subject, and a detail.
+
+    The three-part shape mirrors the source-contract error the acquisition layer already
+    raises, so a caller handling one handles the other the same way.  Instances are
+    immutable once constructed: an error that a handler could rewrite on its way up is an
+    error whose code no longer means anything.
+    """
+
+    def __init__(self, code: str, subject: str, detail: str) -> None:
+        if not isinstance(code, str) or code not in READER_ERROR_CODES:
+            raise ValueError(
+                f"unknown Reader error code {code!r}; add it to READER_ERROR_CODES so "
+                "error remediation can handle it"
+            )
+        super().__init__(f"{subject}: {detail} [{code}]")
+        self.__dict__["code"] = code
+        self.__dict__["subject"] = subject
+        self.__dict__["detail"] = detail
+        self.__dict__["_sealed"] = True
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        if self.__dict__.get("_sealed") and name not in _INTERPRETER_OWNED_ATTRIBUTES:
+            raise AttributeError(f"ReaderError is immutable; cannot set {name!r}")
+        super().__setattr__(name, value)
+
+    def __delattr__(self, name: str) -> None:
+        raise AttributeError(f"ReaderError is immutable; cannot delete {name!r}")
+
+
+def identifier(value: Any, subject: str) -> str:
+    """Admit a canonical identifier, on the recipe's own rule.
+
+    Public because the registry validates a family's declared identity against exactly
+    this rule.  One rule, one implementation, so a family that could be registered can
+    always be pinned and a family that can be pinned can always be registered.
+    """
+
+    if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None:
+        raise ReaderError("READER_CONTRACT", subject, "must be a canonical identifier")
+    return value
+
+
+def semver(value: Any, subject: str) -> str:
+    """Admit a canonical positive semver, on the recipe's own rule."""
+
+    if not isinstance(value, str) or _SEMVER.fullmatch(value) is None:
+        raise ReaderError("READER_CONTRACT", subject, "must be a canonical positive semver")
+    return value
+
+
+def _strict_json_object(value: Any, subject: str) -> dict[str, Any]:
+    """Admit a bounded, canonicalisable JSON object on the recipe's own terms."""
+
+    if not isinstance(value, Mapping):
+        raise ReaderError("READER_OPTIONS", subject, "must be an object")
+    try:
+        raw = canonical_json_bytes(dict(value))
+    except CanonicalJSONError as error:
+        raise ReaderError(
+            "READER_OPTIONS",
+            subject,
+            "contains values that cannot be sealed canonically",
+        ) from error
+    if len(raw) > _MAX_DECODE_OPTIONS_BYTES:
+        raise ReaderError("READER_OPTIONS", subject, "exceeds 256 KiB")
+    return dict(value)
+
+
+@dataclass(frozen=True)
+class ReaderPin:
+    """The exact decoder a recipe sealed, and the exact settings it ran with.
+
+    ``family_id`` and ``family_version`` resolve one entry of the Toolbox on an exact key.
+    ``decode_options`` are the family's own settings, sealed verbatim, so replaying the
+    recipe replays the decode.  Every knob is inside ``options_digest``: changing one
+    changes the recipe digest and invalidates an approval granted against the old settings.
+    """
+
+    family_id: str
+    family_version: str
+    decode_options: Mapping[str, Any]
+
+    def __post_init__(self) -> None:
+        identifier(self.family_id, "reader.pin.family_id")
+        semver(self.family_version, "reader.pin.family_version")
+        object.__setattr__(
+            self,
+            "decode_options",
+            MappingProxyType(_strict_json_object(self.decode_options, "reader.pin.decode_options")),
+        )
+
+    @classmethod
+    def from_recipe_fields(
+        cls,
+        family_id: str | None,
+        family_version: str | None,
+        decode_options: Mapping[str, Any] | None,
+    ) -> ReaderPin | None:
+        """Build the pin from a recipe source's three fields, or None when none are stated.
+
+        All three present or all three absent is the only admissible shape.  A source that
+        names a decoder without pinning its settings, or pins settings without naming a
+        decoder, could not be replayed from its own record.  Absent means today's direct
+        fetch, where no Reader family decoded anything.  This is the one construction path,
+        so no caller has to re-derive the rule.
+        """
+
+        present = tuple(item is not None for item in (family_id, family_version, decode_options))
+        if not any(present):
+            return None
+        if not all(present):
+            raise ReaderError(
+                "READER_CONTRACT",
+                "reader.pin",
+                "family_id, family_version, and decode_options are all present or all absent",
+            )
+        assert family_id is not None and family_version is not None
+        assert decode_options is not None
+        return cls(family_id, family_version, decode_options)
+
+    @property
+    def options_digest(self) -> str:
+        """Digest over the exact decode settings, computed the recipe's one way."""
+
+        return sha256_bytes(canonical_json_bytes(dict(self.decode_options)))
+
+    @property
+    def coordinate(self) -> str:
+        """The ``id@version`` coordinate, for a refusal message a person can act on."""
+
+        return f"{self.family_id}@{self.family_version}"
+
+
+@dataclass(frozen=True)
+class ReaderBudgets:
+    """The ceilings one decode runs under.  A recipe may narrow them; never widen them.
+
+    The defaults are the family-level defaults that apply when a recipe states no caps.
+    ``narrowed_by`` is per-field minimum only, which is what makes a stated cap unable to
+    buy more room than the family already allows.
+    """
+
+    max_input_bytes: int = 16 * 1024 * 1024
+    max_output_bytes: int = 16 * 1024 * 1024
+    max_uncompressed_bytes: int = 128 * 1024 * 1024
+    max_expansion_ratio: int = 100
+    max_declared_cells: int = 10_000_000
+    max_container_members: int = 1_024
+    max_nesting_depth: int = _FIXED_NESTING_DEPTH
+    max_rows: int = 1_000_000
+    max_columns: int = 1_024
+
+    def __post_init__(self) -> None:
+        for item in fields(self):
+            value = getattr(self, item.name)
+            if type(value) is not int or value < 1:
+                raise ReaderError(
+                    "READER_BUDGET",
+                    f"reader.budgets.{item.name}",
+                    "must be a positive integer",
+                )
+        if self.max_nesting_depth != _FIXED_NESTING_DEPTH:
+            raise ReaderError(
+                "READER_BUDGET",
+                "reader.budgets.max_nesting_depth",
+                "must be exactly 1: nesting depth is a security invariant, not a budget",
+            )
+
+    def narrowed_by(self, caps: Mapping[str, Any] | None) -> ReaderBudgets:
+        """Return budgets no wider than these on any field.
+
+        A cap naming a field this contract does not have is refused by name rather than
+        ignored, because a silently ignored cap is a budget a recipe believes it set.
+        """
+
+        if caps is None:
+            return self
+        if not isinstance(caps, Mapping):
+            raise ReaderError("READER_BUDGET", "reader.budgets.caps", "must be a mapping or absent")
+        known = {item.name for item in fields(self)}
+        unknown = sorted(str(key) for key in caps if key not in known)
+        if unknown:
+            raise ReaderError(
+                "READER_BUDGET",
+                "reader.budgets.caps",
+                f"names no such budget: {', '.join(unknown)}",
+            )
+        narrowed: dict[str, int] = {}
+        for name in known:
+            current = getattr(self, name)
+            if name not in caps:
+                narrowed[name] = current
+                continue
+            requested = caps[name]
+            if type(requested) is not int or requested < 1:
+                raise ReaderError(
+                    "READER_BUDGET",
+                    f"reader.budgets.caps.{name}",
+                    "must be a positive integer",
+                )
+            narrowed[name] = min(current, requested)
+        return ReaderBudgets(**narrowed)
+
+
+def bulk_default_budgets() -> ReaderBudgets:
+    """Family defaults for the bulk-capable text and container families.
+
+    Sized for EPA-scale public sources under the raised hosted Courier caps: a 256 MiB
+    fetched archive whose decode output may reach the 4 GiB normalized budget. The
+    uncompressed ceiling rises with the output budget because a container's declared
+    member size is checked against it before extraction. Spreadsheet and GRIB families
+    keep their own, tighter certified defaults.
+    """
+
+    return ReaderBudgets(
+        max_input_bytes=268_435_456,
+        max_output_bytes=4_294_967_296,
+        max_uncompressed_bytes=4_294_967_296,
+        max_declared_cells=400_000_000,
+        max_rows=10_000_000,
+    )
+
+
+@dataclass(frozen=True)
+class ReaderResult:
+    """What one decode produced: exact output bytes and the facts about producing them.
+
+    ``content`` is already the canonical output encoding.  The Reader owns the byte
+    stability of the sealed artifact, so the bytes here are the bytes that get sealed.
+    """
+
+    content: bytes
+    data_format: str
+    media_type: str
+    filename: str
+    row_count: int
+    column_names: tuple[str, ...]
+    declared_cell_count: int
+    flags: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.content, (bytes, bytearray)) or len(self.content) == 0:
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.content",
+                "must be nonempty exact bytes",
+            )
+        object.__setattr__(self, "content", bytes(self.content))
+        if self.data_format not in DATA_FORMATS:
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.data_format",
+                "is not a format the table admits",
+            )
+        if self.media_type not in FORMAT_MEDIA_TYPES[self.data_format]:
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.media_type",
+                "is not a media type the table maps to this format",
+            )
+        if not isinstance(self.filename, str) or _FILENAME.fullmatch(self.filename) is None:
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.filename",
+                "must be a confined bare filename",
+            )
+        if type(self.row_count) is not int or self.row_count < 0:
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.row_count",
+                "must be a non-negative integer",
+            )
+        if not isinstance(self.column_names, tuple) or not self.column_names:
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.column_names",
+                "must be a nonempty tuple",
+            )
+        for index, name in enumerate(self.column_names):
+            if not isinstance(name, str) or not name or "\x00" in name:
+                raise ReaderError(
+                    "READER_OUTPUT",
+                    f"reader.result.column_names[{index}]",
+                    "must be nonempty text without a NUL byte",
+                )
+        if len(set(self.column_names)) != len(self.column_names):
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.column_names",
+                "must not repeat a column name",
+            )
+        if type(
+            self.declared_cell_count
+        ) is not int or self.declared_cell_count < self.row_count * len(self.column_names):
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.declared_cell_count",
+                "must be an integer no smaller than the emitted table geometry",
+            )
+        if not isinstance(self.flags, tuple):
+            raise ReaderError("READER_OUTPUT", "reader.result.flags", "must be a tuple")
+        unknown = sorted(str(flag) for flag in self.flags if flag not in DECODE_FLAGS)
+        if unknown:
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.flags",
+                f"names no such decode flag: {', '.join(unknown)}",
+            )
+        object.__setattr__(self, "flags", tuple(sorted(set(self.flags))))
+
+    def validate_for(self, output_format: str, budgets: ReaderBudgets) -> None:
+        """Bind the result to the family that produced it and the budgets it ran under.
+
+        Kept separate from construction because a result does not know which family built
+        it, and a family's declared output format is the thing being checked.
+        """
+
+        if self.data_format != output_format:
+            raise ReaderError(
+                "READER_OUTPUT",
+                "reader.result.data_format",
+                "differs from the format the family declares it emits",
+            )
+        if len(self.content) > budgets.max_output_bytes:
+            raise ReaderError(
+                "READER_BUDGET",
+                "reader.result.content",
+                "exceeds the output byte budget",
+            )
+        if self.row_count > budgets.max_rows:
+            raise ReaderError("READER_BUDGET", "reader.result.row_count", "exceeds the row budget")
+        if len(self.column_names) > budgets.max_columns:
+            raise ReaderError(
+                "READER_BUDGET",
+                "reader.result.column_names",
+                "exceeds the column budget",
+            )
+        if self.declared_cell_count > budgets.max_declared_cells:
+            raise ReaderError(
+                "READER_BUDGET",
+                "reader.result.declared_cell_count",
+                "exceeds the declared-cell budget",
+            )
+
+
+class ReaderFamily(Protocol):
+    """One certified decoder, resolved by exact identity from the closed Toolbox.
+
+    A family is deliberately not runtime-checkable.  Membership of the Toolbox is decided
+    by the registry at registration time against this contract's requirements, not by an
+    isinstance test at a call site, so a partially-shaped object cannot slip through by
+    happening to answer the attributes an isinstance check samples.
+    """
+
+    @property
+    def family_id(self) -> str:
+        """The identifier half of the exact resolution key."""
+
+    @property
+    def family_version(self) -> str:
+        """The version half of the exact resolution key.  Never resolved approximately."""
+
+    @property
+    def contract_version(self) -> str:
+        """The Reader-contract label this family was built against."""
+
+    @property
+    def output_format(self) -> str:
+        """The one format this family emits, which the result must match."""
+
+    @property
+    def accepted_media_types(self) -> tuple[str, ...]:
+        """The media types this family admits as input, sorted and de-duplicated."""
+
+    @property
+    def default_budgets(self) -> ReaderBudgets:
+        """The ceilings that apply when a recipe states no caps."""
+
+    def validate_options(self, options: Mapping[str, Any]) -> Mapping[str, Any]:
+        """Admit this family's own decode settings, refusing an unknown key by name."""
+
+    def decode(
+        self,
+        content: bytes,
+        pin: ReaderPin,
+        budgets: ReaderBudgets,
+    ) -> ReaderResult:
+        """Turn exact input bytes into exact output bytes, touching no filesystem.
+
+        The decode receives the bytes it is given and returns the bytes it produced.  It
+        creates no temporary directory, extracts no archive member to disk, and spills
+        nothing: a container is streamed in memory or it is refused.  That is the whole
+        security argument for admitting containers at all, and it admits no exception.
+        """