mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/formats.py

@@ -0,0 +1,236 @@
+"""The single closed table of data formats the harness accepts.
+
+Every layer that admits, refuses, or describes a data format reads its membership from
+this module.  No other module re-declares the list.  A test walks the package and fails
+if one does, because a format accepted by one layer and refused by another is a bug in
+its own right rather than a cosmetic inconsistency.
+
+The table is closed.  There is no plugin discovery, no import-by-name, and no
+environment override: membership is exactly the values written below. Add a Reader family here
+before implementing its decoder.
+
+Two vocabularies, not one
+-------------------------
+The harness has two format-shaped sets and they are deliberately different.
+
+``DATA_FORMATS`` is the wire-format set: the payload encodings a source adapter can hand
+back as bytes.  ``PLAN_DATA_FORMATS`` is the plan-contract set, which additionally
+carries ``api`` and ``stream``.  Those two are acquisition modes -- how bytes are
+reached -- not encodings of bytes.  They are modelled as two named tables so the
+distinction stays visible and typed, and they must never be collapsed into one set.
+
+This module imports nothing from the rest of the package.  It is a leaf, so every layer
+including ``acquisition`` and ``backends`` can read it without creating an import cycle.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from types import MappingProxyType
+
+__all__ = [
+    "ARCHIVE_OR_EXECUTABLE_MAGICS",
+    "DATA_FORMATS",
+    "DIRECT_FETCH_MEDIA_TYPES",
+    "FORMAT_MEDIA_TYPES",
+    "FORMAT_SUFFIXES",
+    "PARSER_FORMATS_BY_OPERATION",
+    "PARSER_FORMAT_ORDER",
+    "PLAN_DATA_FORMATS",
+    "READER_CONTRACT_VERSION",
+    "READER_OUTPUT_FORMATS",
+    "READER_WORKER_MEMORY_BYTES",
+    "SLICE_SNAPSHOT_SUFFIX",
+    "SNAPSHOT_SUFFIXES",
+    "SNAPSHOT_SUFFIX_PATTERN",
+    "WEAK_TEXT_MEDIA_TYPES",
+]
+
+# The label of the Reader contract a recipe pins itself to.  A Reader family is the code
+# that turns fetched bytes into rows; this label versions the shape of that pin -- the
+# family identifier, its version, and its decode options -- not any individual family.
+# Bumping this label invalidates recipes that name the previous contract.
+READER_CONTRACT_VERSION = "harness-reader-contract.v1"
+
+# The one process-memory ceiling shared by Reader admission and the operating-system sandbox.
+# It lives in this dependency-free leaf because acquisition consumes Readers and Readers must not
+# import acquisition back. A family-level worst-case proof is required to stay below this value.
+READER_WORKER_MEMORY_BYTES = 512 * 1024 * 1024
+
+
+# Encodings a Reader family may emit. Reader output is canonical CSV regardless of source format.
+#
+# This is a different question from ``DATA_FORMATS``.  That table answers "what encoding may
+# arrive on the wire"; this one answers "what encoding may a decoder produce".  A wire format
+# with no Reader that emits it is normal, and so is the reverse, so neither is derived from
+# the other.  Zip, gzip, tar, xlsx, tsv, and grib2 appear in neither: they are Reader
+# families, not encodings a parser is ever handed.
+READER_OUTPUT_FORMATS: tuple[str, ...] = ("csv",)
+
+# The ordered parser table.  Order is digest-load-bearing: this exact sequence is
+# embedded in the sandbox policy document built by
+# ``acquisition.sandbox._policy_digest``, and the attestation a coordinator supplies is
+# compared against the digest of that document.  Reordering these names rotates the
+# attestation for every operation that parses bytes, so treat the order as part of the
+# contract and not as an implementation detail.
+PARSER_FORMAT_ORDER: tuple[str, ...] = ("csv", "json", "ndjson", "parquet")
+
+# The wire-format set: payload encodings an adapter emits and a parser admits.
+DATA_FORMATS: frozenset[str] = frozenset(PARSER_FORMAT_ORDER)
+
+# The plan-contract set.  ``api`` and ``stream`` are acquisition modes, not wire
+# formats: they describe how bytes are reached rather than how bytes are encoded.  A
+# plan may name them; a parser may never be handed them.  See the vocabulary map in
+# ``sources/contracts.py``.
+PLAN_DATA_FORMATS: frozenset[str] = frozenset({"csv", "parquet", "json", "api", "stream"})
+
+# Media types accepted for each wire format, used to admit or refuse a response by its
+# declared content type.
+FORMAT_MEDIA_TYPES: Mapping[str, frozenset[str]] = MappingProxyType(
+    {
+        "csv": frozenset({"text/csv"}),
+        "json": frozenset({"application/json"}),
+        "ndjson": frozenset({"application/x-ndjson"}),
+        "parquet": frozenset({"application/vnd.apache.parquet"}),
+    }
+)
+
+# The weak *text* label: what a server sends when it declines to describe a text file at all.
+# It is NOT a member of the table above and must never become one -- ``FORMAT_MEDIA_TYPES``
+# answers "what encoding may be sealed", and its members are what a recipe may name and what
+# the sandbox policy pins.  ``text/plain`` names no encoding; it is the absence of a claim.
+#
+# Named for text on purpose.  ``application/octet-stream`` is the other weak label this
+# codebase knows -- it is what ``SIDECAR_MEDIA_TYPES`` admits and what the widened archive and
+# spreadsheet coordinates admit -- and it does not belong here.  Adding it would admit an
+# opaque binary download as csv, json or ndjson at both the fetch and the parse, for every
+# direct source, with no certified coordinate having approved it; the archive-magic guard in
+# ``parsing`` catches a container but not a binary whose first bytes match no known prefix.
+# Widening to that label is a separate change with its own evidence.
+WEAK_TEXT_MEDIA_TYPES: frozenset[str] = frozenset({"text/plain"})
+
+# The formats whose bytes are text, and so the only ones a weak text label can plausibly carry.
+# ``parquet`` is excluded because a Parquet file is never plain text: admitting the label there
+# would widen a refusal without ever matching something a publisher actually sends.
+_TEXT_SHAPED_FORMATS: frozenset[str] = frozenset({"csv", "json", "ndjson"})
+
+# What a *direct* fetch -- a source with no Reader pin -- admits, and what its parse admits.
+#
+# Kept separate from ``FORMAT_MEDIA_TYPES`` rather than folded into it, because the two answer
+# different questions and only one of them is the sealing vocabulary.  A Reader-pinned fetch
+# consults neither: it takes its admitted set from the pinned family, where
+# ``delimited_text@1.1.0`` and ``json.tabular@1.1.0`` carry the weak label and their ``1.0.0``
+# siblings deliberately do not.  A direct fetch has no recipe to protect and no coordinate to
+# pin, so its widening lives here or it does not exist.
+#
+# This is the one clause of the slice-vocabulary invariant that this map changes, and it is
+# changed on purpose: a parser now admits ``text/plain`` for a text-shaped format.  The other
+# clauses hold unchanged -- no recipe may name it (``PLAN_DATA_FORMATS`` is format names), and
+# no deletion sweep tracks it (the sweep is suffix-driven).  The residual risk is the one
+# already recorded for the widened Reader coordinates: a plain-text error body can parse as a
+# one-column table with no rows, which is visible at review before anything is sealed.
+DIRECT_FETCH_MEDIA_TYPES: Mapping[str, frozenset[str]] = MappingProxyType(
+    {
+        data_format: (
+            accepted | WEAK_TEXT_MEDIA_TYPES if data_format in _TEXT_SHAPED_FORMATS else accepted
+        )
+        for data_format, accepted in FORMAT_MEDIA_TYPES.items()
+    }
+)
+
+# Filename suffixes accepted for each wire format.  A format may carry more than one.
+FORMAT_SUFFIXES: Mapping[str, frozenset[str]] = MappingProxyType(
+    {
+        "csv": frozenset({".csv"}),
+        "json": frozenset({".json"}),
+        "ndjson": frozenset({".jsonl", ".ndjson"}),
+        "parquet": frozenset({".parquet"}),
+    }
+)
+
+# What an archive, an executable, or a rich document looks like from its first bytes.  Not a
+# format table: these are the things that are never a table, listed by the prefix that gives
+# them away.
+#
+# It lives in the leaf because two layers refuse on it and they must agree.  The tabular
+# parser refuses these bytes outright, so an archive is never parsed as data.  The container
+# Readers refuse a member whose own first bytes match, so an archive one layer inside an
+# archive is never opened.  Those are two layers of one invariant; a second copy of the list
+# would let them drift into disagreeing about what an archive is, which is a security-
+# relevant disagreement and not a cosmetic one.  ``readers/`` cannot import ``acquisition/``
+# -- that back-edge closes an import cycle -- so the one statement lives here, where both
+# sides can read it.
+#
+# The table is public for the same reason: a cross-package import of another module's private
+# name is not a contract.  Adding or removing a prefix changes what the harness refuses and is
+# a reviewed edit, never a tidy-up.
+ARCHIVE_OR_EXECUTABLE_MAGICS: tuple[bytes, ...] = (
+    b"PK\x03\x04",
+    b"PK\x05\x06",
+    b"PK\x07\x08",
+    b"\x1f\x8b",
+    b"BZh",
+    b"\xfd7zXZ\x00",
+    b"7z\xbc\xaf'\x1c",
+    b"Rar!\x1a\x07",
+    b"\x7fELF",
+    b"MZ",
+    b"\xca\xfe\xba\xbe",
+    b"%PDF-",
+)
+
+# Which formats each sandbox operation may hand to a parser.  This is enforced, not merely
+# declared: ``acquisition.sandbox._execute_worker_request`` refuses a request whose
+# ``data_format`` is outside its operation's entry, and refuses outright when the operation
+# has no entry.  The same mapping is embedded in the operation's policy document, so the
+# attestation a coordinator binds and the rule the worker applies are the one table.
+#
+# Presence in this mapping is also what makes an operation rotatable by a format addition; an
+# operation that is absent from it never rotates on one, because its policy document carries
+# no format list at all.
+#
+# ``probe`` is deliberately absent: it returns from the sandbox worker before the request's
+# data format is ever read, so it hands nothing to a parser.
+#
+# Values are the shared ``PARSER_FORMAT_ORDER`` by reference rather than re-spelled
+# literals, because the order is digest-load-bearing and because a re-spelled list here
+# would trip the anti-duplication gate -- which would be that gate working correctly.
+# Add Reader families here so attestation changes remain scoped to decode operations.
+# The decode operation is scoped to what a Reader may emit rather than to the wire table, and
+# that separation is the consequence worth stating: adding a **wire** format rotates ``parse``
+# and ``retrieve_and_parse`` and leaves the decode operation exactly where it was, while
+# adding a **Reader output** encoding rotates the decode operation alone. Neither event
+# touches the other's attestation, which is what the per-operation scoping was built for.
+PARSER_FORMATS_BY_OPERATION: Mapping[str, tuple[str, ...]] = MappingProxyType(
+    {
+        "parse": PARSER_FORMAT_ORDER,
+        "retrieve_and_parse": PARSER_FORMAT_ORDER,
+        "decode_and_parse": READER_OUTPUT_FORMATS,
+        "retrieve_decode_and_parse": READER_OUTPUT_FORMATS,
+    }
+)
+
+# The one sealed-snapshot suffix that is deliberately not a data format.  A fetched slice
+# is a byte range of an object, not a parseable document: no parser in this build reads it,
+# and putting it in ``DATA_FORMATS`` would claim a parser that does not exist and would
+# rotate every format-table digest to say so.
+#
+# It lives here rather than beside the sealing function because sealing and deletion have to
+# agree about it.  A suffix that could be written but not matched for removal would strand
+# snapshots no retention sweep can find -- the exact hazard the derivation below exists to
+# prevent -- and that is what happened when this suffix was declared at the sealing site and
+# unioned in there alone.
+SLICE_SNAPSHOT_SUFFIX: str = "slice"
+
+# Every suffix a sealed content-addressed snapshot may carry: the wire formats, plus the one
+# non-format above.  This is the set both halves read.  ``_is_allowlisted_snapshot_suffix``
+# admits exactly this set when writing, and ``SNAPSHOT_SUFFIX_PATTERN`` below is the
+# alternation the deletion regex matches with -- so what can be sealed and what can be
+# removed are one derived set rather than two lists that drift.
+SNAPSHOT_SUFFIXES: frozenset[str] = DATA_FORMATS | frozenset({SLICE_SNAPSHOT_SUFFIX})
+
+# The alternation fragment the sealed-snapshot filename regex is built from, derived
+# from the set above rather than spelled out.  Deriving it is what stops a suffix from
+# becoming admissible for sealing while staying unmatchable for deletion, which would
+# strand snapshots that no retention sweep can find.
+SNAPSHOT_SUFFIX_PATTERN: str = "|".join(sorted(SNAPSHOT_SUFFIXES))