mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/backends/restrictions.py

@@ -0,0 +1,126 @@
+"""Typed engine-restriction contract: which operations an engine may not run.
+
+An operation that provably cannot reach cross-engine byte parity on some engine must be
+listed here so that engine gives a typed refusal instead of a silently divergent digest.
+
+The table is currently empty. Every operation in the closed set reaches byte parity across
+reference / pandas / polars, proven by the sealed goldens and the conformance matrix, so
+this module is the mechanism with no data: a closed per-operation by per-engine table plus
+a typed refusal.
+
+``_ENGINE_RESTRICTED_OPS`` maps an operation identity to the set of excluded engine names.
+An entry is a claim that the operation provably cannot reach byte parity on those engines,
+so adding one is a contract change: it needs a golden that establishes the divergence
+before the entry is trusted.
+
+Two states, not one
+-------------------
+"Cannot run this operation" has two distinct causes and they oblige different answers:
+
+``RESTRICTED`` -- an implementation exists but provably cannot reach cross-engine byte
+parity, so it must refuse rather than diverge. Admitting a restriction needs evidence: a
+golden that establishes the divergence comes first, then the table entry.
+
+``UNIMPLEMENTED`` -- no implementation exists for that engine at all. The obligation is an
+implementation or an honest refusal, not a restriction entry. Recording an unimplemented
+engine as restricted would claim evidence of divergence that was never gathered.
+
+``engine_operation_support`` returns an ``EngineOperationSupport`` member rather than a
+bool so a caller cannot collapse the two into "unsupported". An enum is the right shape
+here because the states are a closed set with no payload; a dataclass would invite fields
+that drift from the table.
+
+``assert_engine_supports_plan`` is called once from ``pipeline.build_candidate`` after the
+selected engine is resolved. With an empty table it is a no-op on every build; it raises the
+instant a restricted operation meets an excluded engine.
+
+The operation-identity vocabulary is the plan's operation surface: each cleaning step's
+``operation`` name (``trim`` / ``empty_to_null`` / ``rename`` / ``cast``), plus ``"join"``
+and ``"select"``.
+
+This module is deliberately dependency-light and dynamic-import-free: it lives under
+``backends/`` and is scanned by that package's static dynamic-import ban.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mostlyright.data_harness.local_contracts import TablePlan
+
+# Closed per-operation by per-engine restriction table; see the module docstring for the
+# rule an entry has to meet. Deliberately empty: no divergence golden exists, so no claim
+# of divergence is admissible.
+_ENGINE_RESTRICTED_OPS: dict[str, frozenset[str]] = {}
+
+
+class EngineOperationSupport(Enum):
+    """Whether an engine runs an operation, refuses it, or does not implement it."""
+
+    SUPPORTED = "supported"
+    RESTRICTED = "restricted"
+    UNIMPLEMENTED = "unimplemented"
+
+
+def engine_operation_support(engine: str, operation: str) -> EngineOperationSupport:
+    """Report how ``engine`` stands with respect to ``operation``.
+
+    The engine set is read from the registry rather than re-listed here; a second list of
+    engine names would be exactly the duplication the format table just removed. The
+    registry is resolved lazily inside this function, matching the lazy ``BuildError``
+    import below, so this module stays dependency-light and the package's static
+    dynamic-import ban keeps passing.
+    """
+
+    from mostlyright.data_harness.backends.registry import registered_backends
+
+    if engine not in registered_backends():
+        return EngineOperationSupport.UNIMPLEMENTED
+    excluded = _ENGINE_RESTRICTED_OPS.get(operation)
+    if excluded is not None and engine in excluded:
+        return EngineOperationSupport.RESTRICTED
+    return EngineOperationSupport.SUPPORTED
+
+
+def _plan_operations(plan: TablePlan) -> frozenset[str]:
+    """The set of op-identities a plan uses: each cleaning step's ``operation`` name, plus
+    ``"join"`` and ``"select"``. This is the surface the restriction table keys on."""
+
+    operations: set[str] = set()
+    for step in getattr(plan, "cleaning", ()):
+        operations.add(step.operation)
+    if getattr(plan, "join", None) is not None:
+        operations.add("join")
+    if getattr(plan, "select", None):
+        operations.add("select")
+    return frozenset(operations)
+
+
+def assert_engine_supports_plan(engine: str, plan: TablePlan) -> None:
+    """Fail closed if ``plan`` uses an operation that is restricted on ``engine``.
+
+    For any operation the plan uses that appears in ``_ENGINE_RESTRICTED_OPS`` with
+    ``engine`` in its excluded set, raise a typed ``BuildError("ENGINE_RESTRICTED", ...)``
+    rather than let the build produce a divergent digest. With the table empty this is a
+    no-op on every plan.
+    """
+
+    if not _ENGINE_RESTRICTED_OPS:
+        # Fast path while the table is empty: the mechanism never fires.
+        return
+
+    for operation in _plan_operations(plan):
+        excluded = _ENGINE_RESTRICTED_OPS.get(operation)
+        if excluded is not None and engine in excluded:
+            # Lazy import keeps ``import backends`` free of a module-load dependency on
+            # ``pipeline`` (the same cycle-avoidance the registry uses). Only this failure
+            # path needs ``BuildError``.
+            from mostlyright.data_harness.pipeline import BuildError
+
+            raise BuildError(
+                "ENGINE_RESTRICTED",
+                f"operation {operation!r} is engine-restricted on {engine!r} "
+                "(it cannot reach cross-engine byte parity on that engine)",
+            )

mostlyright/data_harness/canonical.py

@@ -0,0 +1,333 @@
+"""Strict JSON parsing, canonical bytes, and content digests.
+
+This module is intentionally small and dependency-free.  It is the Harness-private
+canonicalization authority; Studio wire canonicalization remains owned by Studio's generated
+client and contract set.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from typing import Any, NoReturn
+
+MAX_CANONICAL_DEPTH = 64
+MAX_CANONICAL_MEMBERS = 100_000
+MAX_CANONICAL_STRING_LENGTH = 1_000_000
+MAX_CANONICAL_INTEGER = (1 << 63) - 1
+
+
+class CanonicalJSONError(ValueError):
+    """A typed strict-JSON or canonicalization failure."""
+
+    def __init__(self, code: str, path: str, detail: str) -> None:
+        self.code = code
+        self.path = path
+        self.detail = detail
+        super().__init__(f"{path}: {detail} [{code}]")
+
+
+def parse_json(value: str | bytes | bytearray | memoryview) -> Any:
+    """Parse strict UTF-8 JSON while rejecting duplicate keys and non-integer numbers."""
+
+    return _parse(value, fractions="reject")
+
+
+def parse_foreign_json(value: str | bytes | bytearray | memoryview) -> Any:
+    """Parse a third-party document with the same hardening, but accept fractional numbers.
+
+    The Harness data model is integer-only, and :func:`parse_json` enforces that because every
+    value it returns may end up inside a digested artifact. A document somebody else published is a
+    different problem. A STAC Collection is required by its own specification to carry
+    ``extent.spatial.bbox``, and real bboxes are fractional (``-178.1333``); so are ``gsd``,
+    ``center_wavelength`` and most ``summaries`` entries. Refusing them would mean refusing every
+    real ``/collections`` response, which is not a hardening decision but an outage.
+
+    Everything else stays exactly as strict: duplicate object keys, byte-order marks, non-strict
+    UTF-8, ``NaN``/``Infinity``, out-of-range integers, unpaired surrogates, depth, member count and
+    string length are all refused identically. Only finite fractional numbers become admissible, and
+    only as *read* values.
+
+    Floats still cannot reach an artifact. :func:`canonical_json_bytes` -- the sole input to every
+    digest in the repository -- refuses a float unchanged, so a float read here can be inspected by
+    a parser but can never be sealed, digested or written into a canonical document. That is the
+    boundary this pair of functions draws: read what the world publishes, write only what the model
+    admits.
+    """
+
+    return _parse(value, fractions="accept")
+
+
+def _parse(value: str | bytes | bytearray | memoryview, *, fractions: str) -> Any:
+    if isinstance(value, str):
+        text = value
+    elif isinstance(value, (bytes, bytearray, memoryview)):
+        raw = bytes(value)
+        if raw.startswith(b"\xef\xbb\xbf"):
+            raise CanonicalJSONError("JSON_BOM", "$", "UTF-8 byte-order marks are not allowed")
+        try:
+            text = raw.decode("utf-8", errors="strict")
+        except UnicodeDecodeError as exc:
+            raise CanonicalJSONError(
+                "JSON_UTF8",
+                "$",
+                f"input is not strict UTF-8 at byte {exc.start}",
+            ) from None
+    else:
+        raise CanonicalJSONError(
+            "JSON_INPUT_TYPE",
+            "$",
+            "JSON input must be text or a bytes-like value",
+        )
+
+    if text.startswith("\ufeff"):
+        raise CanonicalJSONError("JSON_BOM", "$", "Unicode byte-order marks are not allowed")
+    accept_fractions = fractions == "accept"
+    try:
+        parsed = json.loads(
+            text,
+            object_pairs_hook=_reject_duplicate_pairs,
+            parse_int=_parse_integer,
+            parse_float=_parse_fraction if accept_fractions else _reject_fraction,
+            parse_constant=_reject_constant,
+        )
+    except CanonicalJSONError:
+        raise
+    except json.JSONDecodeError as exc:
+        raise CanonicalJSONError(
+            "JSON_SYNTAX",
+            "$",
+            f"invalid JSON at line {exc.lineno}, column {exc.colno}",
+        ) from None
+    return _normalize_json(
+        parsed,
+        "$",
+        depth=0,
+        members=[0],
+        accept_fractions=accept_fractions,
+    )
+
+
+def canonical_json_bytes(value: Any) -> bytes:
+    """Return deterministic UTF-8 JSON bytes for the strict Harness data model."""
+
+    normalized = _normalize_json(value, "$", depth=0, members=[0])
+    return json.dumps(
+        normalized,
+        ensure_ascii=False,
+        allow_nan=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    ).encode("utf-8")
+
+
+def parse_canonical_json(value: str | bytes | bytearray | memoryview) -> Any:
+    """Parse JSON and require that its source encoding is already canonical."""
+
+    if isinstance(value, str):
+        try:
+            raw = value.encode("utf-8", errors="strict")
+        except UnicodeEncodeError as exc:
+            raise CanonicalJSONError(
+                "JSON_UNICODE",
+                "$",
+                f"text contains an invalid Unicode code point at character {exc.start}",
+            ) from None
+    elif isinstance(value, (bytes, bytearray, memoryview)):
+        raw = bytes(value)
+    else:
+        raise CanonicalJSONError(
+            "JSON_INPUT_TYPE",
+            "$",
+            "JSON input must be text or a bytes-like value",
+        )
+    parsed = parse_json(raw)
+    if raw != canonical_json_bytes(parsed):
+        raise CanonicalJSONError(
+            "JSON_NONCANONICAL",
+            "$",
+            "JSON bytes are not the canonical UTF-8 representation",
+        )
+    return parsed
+
+
+def canonical_sha256(value: Any) -> str:
+    """Return lowercase SHA-256 over :func:`canonical_json_bytes`."""
+
+    return hashlib.sha256(canonical_json_bytes(value)).hexdigest()
+
+
+def sha256_bytes(value: bytes | bytearray | memoryview) -> str:
+    """Return lowercase SHA-256 over exact bytes."""
+
+    return hashlib.sha256(bytes(value)).hexdigest()
+
+
+def _normalize_json(
+    value: Any,
+    path: str,
+    *,
+    depth: int,
+    members: list[int],
+    accept_fractions: bool = False,
+) -> Any:
+    if depth > MAX_CANONICAL_DEPTH:
+        raise CanonicalJSONError(
+            "JSON_DEPTH",
+            path,
+            f"nesting exceeds {MAX_CANONICAL_DEPTH} levels",
+        )
+    members[0] += 1
+    if members[0] > MAX_CANONICAL_MEMBERS:
+        raise CanonicalJSONError(
+            "JSON_MEMBERS",
+            path,
+            f"value exceeds {MAX_CANONICAL_MEMBERS} members",
+        )
+
+    if value is None or type(value) is bool:
+        return value
+    if type(value) is int:
+        if not -MAX_CANONICAL_INTEGER - 1 <= value <= MAX_CANONICAL_INTEGER:
+            raise CanonicalJSONError(
+                "JSON_INTEGER_RANGE",
+                path,
+                "integer is outside the signed 64-bit range",
+            )
+        return value
+    if type(value) is float:
+        if not math.isfinite(value):
+            raise CanonicalJSONError(
+                "JSON_NONFINITE",
+                path,
+                "non-finite JSON numbers are unsupported",
+            )
+        if accept_fractions:
+            return value
+        raise CanonicalJSONError(
+            "JSON_NONINTEGER_NUMBER",
+            path,
+            "non-integer JSON numbers are unsupported",
+        )
+    if isinstance(value, str):
+        if len(value) > MAX_CANONICAL_STRING_LENGTH:
+            raise CanonicalJSONError(
+                "JSON_STRING_LENGTH",
+                path,
+                f"string exceeds {MAX_CANONICAL_STRING_LENGTH} code points",
+            )
+        _validate_unicode(value, path)
+        return value
+    if isinstance(value, dict):
+        if any(not isinstance(key, str) for key in value):
+            raise CanonicalJSONError(
+                "JSON_OBJECT_KEY",
+                path,
+                "object keys must all be strings",
+            )
+        result: dict[str, Any] = {}
+        for key, item in value.items():
+            _validate_unicode(key, f"{path}.<key>")
+            child = f"{path}.{key}" if key else f"{path}['']"
+            result[key] = _normalize_json(
+                item,
+                child,
+                depth=depth + 1,
+                members=members,
+                accept_fractions=accept_fractions,
+            )
+        return result
+    if isinstance(value, (list, tuple)):
+        return [
+            _normalize_json(
+                item,
+                f"{path}[{index}]",
+                depth=depth + 1,
+                members=members,
+                accept_fractions=accept_fractions,
+            )
+            for index, item in enumerate(value)
+        ]
+    raise CanonicalJSONError(
+        "JSON_VALUE_TYPE",
+        path,
+        f"unsupported value type {type(value).__name__}",
+    )
+
+
+def _validate_unicode(value: str, path: str) -> None:
+    """Refuse a string carrying a lone surrogate, in one step rather than one step per character.
+
+    UTF-8 can encode every code point except a surrogate, so ``str.encode`` *is* the surrogate
+    check: it raises :class:`UnicodeEncodeError` on exactly the code points ``0xD800..0xDFFF`` and
+    on nothing else, and ``start`` is the character index the walk used to report. The two forms
+    refuse the same strings with the same code, path and character; this one does it in C.
+
+    The ASCII fast path costs one C-level scan and no allocation. ASCII is a strict subset of the
+    non-surrogate code points, so a string that passes it cannot contain one.
+    """
+
+    if value.isascii():
+        return
+    try:
+        value.encode("utf-8", errors="strict")
+    except UnicodeEncodeError as error:
+        raise CanonicalJSONError(
+            "JSON_UNICODE",
+            path,
+            f"unpaired surrogate at character {error.start}",
+        ) from None
+
+
+def _reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in result:
+            raise CanonicalJSONError(
+                "JSON_DUPLICATE_KEY",
+                f"$.{key}",
+                f"duplicate object key {key!r}",
+            )
+        result[key] = value
+    return result
+
+
+def _parse_integer(value: str) -> int:
+    parsed = int(value)
+    if not -MAX_CANONICAL_INTEGER - 1 <= parsed <= MAX_CANONICAL_INTEGER:
+        raise CanonicalJSONError(
+            "JSON_INTEGER_RANGE",
+            "$",
+            "integer is outside the signed 64-bit range",
+        )
+    return parsed
+
+
+def _parse_fraction(value: str) -> float:
+    """Read a fractional literal from a foreign document, refusing a non-finite one."""
+
+    parsed = float(value)
+    if not math.isfinite(parsed):
+        raise CanonicalJSONError(
+            "JSON_NONFINITE",
+            "$",
+            f"non-finite JSON number {value!r} is unsupported",
+        )
+    return parsed
+
+
+def _reject_fraction(value: str) -> NoReturn:
+    raise CanonicalJSONError(
+        "JSON_NONINTEGER_NUMBER",
+        "$",
+        f"non-integer JSON number {value!r} is unsupported",
+    )
+
+
+def _reject_constant(value: str) -> NoReturn:
+    raise CanonicalJSONError(
+        "JSON_NONFINITE",
+        "$",
+        f"non-finite JSON number {value!r} is unsupported",
+    )