mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/readers/delimited.py

@@ -0,0 +1,257 @@
+"""``delimited_text``: a tab-separated source, at the cost of one file and no table edit.
+
+This family exists to make a point as much as to read a file, so the point is written here
+where the next person to propose the alternative will find it.
+
+A delimiter is a setting, not a wire format
+-------------------------------------------
+The sealed artifact is always canonical CSV -- ``formats.READER_OUTPUT_FORMATS`` has one
+member and the reason is recorded there.  So a tab-separated source is a Reader family whose
+decode options name a delimiter, and the format tables do not move at all.  Under the other
+design, where the sealed artifact keeps the source's own encoding, ``tsv`` would have to
+enter ``PARSER_FORMAT_ORDER`` -- which rotates the ``parse`` and ``retrieve_and_parse``
+policy digests on both platform clauses, re-pins four literals, and widens the media-type
+table, the suffix table, the snapshot-suffix pattern, the deletion regex, the plan-format
+enum, and the parser dispatch.  ``raw/{source_id}.csv`` would then be a lie.  None of that
+happens here, and ``tests/h3/test_readers_delimited.py`` asserts that none of it happened.
+
+Thin on purpose
+---------------
+Admission is "are these bytes valid UTF-8 text within budget", the decode is
+``tabular.decode_delimited``, and the emission is ``tabular.encode_canonical_csv``.  The same
+two functions decode a zip member, a gzip stream, and a tar member, which is what lets four
+paths produce one answer.  If this module ever grows a parsing rule of its own, that rule has
+been duplicated out of ``tabular.py`` and the four paths have started to disagree.
+
+Four decisions, each with its reason
+------------------------------------
+Every one of these is a refusal, and a refusal is only defensible if the reason is written
+next to it.
+
+* **The delimiter set is closed** to the single characters in ``ADMITTED_DELIMITERS``: tab
+  and comma today.  Every additional delimiter is a new parser surface and a new class of
+  ambiguity -- a semicolon file in a locale that uses the comma as a decimal separator reads
+  as a different table under each -- and no supported source needs one. Widening the
+  set is a source edit that goes through review, exactly like the format table.
+* **The encoding is strict UTF-8**, spelled one way.  A byte sequence that is not valid UTF-8
+  is refused rather than replaced, because a replacement character in sealed data is silent
+  corruption: the artifact verifies clean forever and the value it carries is not the value
+  the publisher wrote.  ``UTF-8`` and ``utf-8-sig`` are refused as spellings too, since two
+  spellings of one setting seal to two different options digests for one decode.
+* **A leading byte-order mark is consumed**, by ``decode_delimited``, before the header is
+  read.  A mark that survives into the first column name produces a dataset whose first
+  column silently fails every join, and "whatever the standard library happens to do" is not
+  a rule anyone can check.  A mark anywhere else in the text is a character and stays.
+* **A headerless source is refused.**  ``header: false`` would need the recipe to supply the
+  column names, and this family's closed option set has no way to state them; naming columns
+  by position instead would put names into a dataset that nobody wrote down and no recipe
+  describes.  So the option is admitted only as ``true``, and reading a headerless source is
+  a reviewed widening -- a ``columns`` option -- rather than something a decoder invents.
+  The shared helper still names columns positionally for the container families, which pass
+  ``header`` through from their own settings; that is their decision to make, not this one's.
+
+Import discipline: this module reads ``formats``, ``readers.contracts``, and
+``readers.tabular``.  It reaches no filesystem and imports nothing from ``acquisition``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from mostlyright.data_harness.formats import (
+    FORMAT_MEDIA_TYPES,
+    FORMAT_SUFFIXES,
+    READER_CONTRACT_VERSION,
+    READER_OUTPUT_FORMATS,
+)
+from mostlyright.data_harness.readers.contracts import (
+    ReaderBudgets,
+    ReaderError,
+    ReaderPin,
+    ReaderResult,
+    bulk_default_budgets,
+)
+from mostlyright.data_harness.readers.tabular import (
+    FALLBACK_STEM,
+    decode_delimited_stream,
+    encode_canonical_csv,
+    sealed_filename,
+)
+
+__all__ = ["ADMITTED_DELIMITERS", "ADMITTED_ENCODINGS", "DelimitedTextReader"]
+
+# The one encoding a Reader may seal, unpacked rather than restated.  A second member in the
+# leaf's table makes this line raise at import, which is the loud failure that forces a family
+# to choose its output encoding deliberately.
+(_OUTPUT_FORMAT,) = READER_OUTPUT_FORMATS
+_OUTPUT_MEDIA_TYPE = sorted(FORMAT_MEDIA_TYPES[_OUTPUT_FORMAT])[0]
+_OUTPUT_SUFFIX = sorted(FORMAT_SUFFIXES[_OUTPUT_FORMAT])[0]
+
+# The closed delimiter table.  Two single characters, and the module docstring says why the
+# set is closed rather than open.  Adding one is a reviewed source edit.
+ADMITTED_DELIMITERS: tuple[str, ...] = ("\t", ",")
+
+# The closed encoding table, one spelling of one encoding.  A tuple rather than a bare
+# constant because the shape is the same shape the delimiter set has, and the next entry --
+# if there is ever an honest one -- goes through the same review.
+ADMITTED_ENCODINGS: tuple[str, ...] = ("utf-8",)
+
+# The whole option vocabulary.  ``delimiter`` is required because it is the whole of what
+# this family decides; the other two have stated defaults, so an ordinary recipe names one
+# setting and a strict recipe names three.
+_REQUIRED_OPTION_KEYS = ("delimiter",)
+_OPTIONAL_OPTION_KEYS = ("header", "encoding")
+_OPTION_KEYS = (*_REQUIRED_OPTION_KEYS, *_OPTIONAL_OPTION_KEYS)
+
+
+def _validate_options(options: Any, *, subject: str) -> dict[str, Any]:
+    """Admit this family's closed option set, refusing an unknown key by name.
+
+    Strict on purpose.  The options digest is sealed into the recipe forever, so a key that
+    was accepted and ignored would be an unreviewable difference between two recipes that
+    look identical to everyone who reads them.
+    """
+
+    if not isinstance(options, Mapping):
+        raise ReaderError("READER_OPTIONS", subject, "must be an object")
+    unknown = sorted(str(key) for key in options if key not in _OPTION_KEYS)
+    if unknown:
+        raise ReaderError(
+            "READER_OPTIONS",
+            subject,
+            f"names no such setting: {', '.join(unknown)}; this family takes "
+            f"{', '.join(sorted(_OPTION_KEYS))}",
+        )
+
+    delimiter = options.get("delimiter")
+    if not isinstance(delimiter, str) or delimiter not in ADMITTED_DELIMITERS:
+        raise ReaderError(
+            "READER_OPTIONS",
+            f"{subject}.delimiter",
+            "must name one of the delimiters this family admits: a tab or a comma. The set "
+            "is closed because every additional delimiter is a new parser surface and a new "
+            "class of ambiguity; widening it is a reviewed source edit",
+        )
+
+    encoding = options.get("encoding", ADMITTED_ENCODINGS[0])
+    if not isinstance(encoding, str) or encoding not in ADMITTED_ENCODINGS:
+        raise ReaderError(
+            "READER_OPTIONS",
+            f"{subject}.encoding",
+            f"must be exactly {ADMITTED_ENCODINGS[0]!r}. Text that is not valid UTF-8 is "
+            "refused rather than decoded with replacement characters, because a replacement "
+            "character in sealed data is corruption that verifies clean forever",
+        )
+
+    header = options.get("header", True)
+    if not isinstance(header, bool):
+        raise ReaderError("READER_OPTIONS", f"{subject}.header", "must be true or false")
+    if not header:
+        raise ReaderError(
+            "READER_OPTIONS",
+            f"{subject}.header",
+            "must be true: a headerless source needs the recipe to supply its column names, "
+            "this family's settings have no way to state them, and naming columns by position "
+            "would put names into the dataset that nobody wrote down. Add a header row to the "
+            "source, or widen this family with a reviewed columns option",
+        )
+
+    return {"delimiter": delimiter, "header": header, "encoding": encoding}
+
+
+@dataclass(frozen=True)
+class DelimitedTextReader:
+    """``delimited_text``: delimited text read from bytes, sealed as canonical CSV.
+
+    The order of the work is options, decode, emit -- three steps, because the admission this
+    family performs *is* the decode: "is this valid UTF-8 text, within budget, whose rows are
+    the width its header states" is exactly what ``decode_delimited`` answers, and asking it
+    twice in two places is how two answers come to disagree.
+
+    ``accepted_media_types`` admits the two media types a delimited source arrives under.
+    That is an input-side declaration and it is not a format table: what may be *sealed* is
+    still ``csv`` alone, which is why ``text/tab-separated-values`` appears here and in no
+    table in ``formats``.
+    """
+
+    family_id: str = "delimited_text"
+    family_version: str = "1.0.0"
+    contract_version: str = READER_CONTRACT_VERSION
+    output_format: str = _OUTPUT_FORMAT
+    accepted_media_types: tuple[str, ...] = ("text/csv", "text/tab-separated-values")
+    default_budgets: ReaderBudgets = field(default_factory=bulk_default_budgets)
+
+    def validate_options(self, options: Mapping[str, Any]) -> Mapping[str, Any]:
+        return _validate_options(options, subject="reader.delimited_text.decode_options")
+
+    def decode(self, content: bytes, pin: ReaderPin, budgets: ReaderBudgets) -> ReaderResult:
+        options = self.validate_options(pin.decode_options)
+        # Parsed rows feed the canonical encoder one at a time rather than being drained
+        # into a table first.  A year-scale source is millions of rows, and the table form
+        # would hold every cell as a live object at once -- the decode's memory ceiling is
+        # the row, not the file.
+        columns, rows = decode_delimited_stream(
+            content,
+            delimiter=str(options["delimiter"]),
+            header=bool(options["header"]),
+            budgets=budgets,
+        )
+        row_count = 0
+
+        def counted() -> Iterator[tuple[str, ...]]:
+            nonlocal row_count
+            for row in rows:
+                row_count += 1
+                yield row
+
+        payload = encode_canonical_csv(columns, counted(), budgets=budgets)
+        return ReaderResult(
+            content=payload,
+            data_format=_OUTPUT_FORMAT,
+            media_type=_OUTPUT_MEDIA_TYPE,
+            # Loose delimited text carries no name of its own: the bytes are the whole input,
+            # and the source's filename is not part of what was decoded.  So the shared
+            # fallback stem is used deliberately, exactly as the gzip family uses it, rather
+            # than a name invented here or taken from somewhere the decode never looked.
+            filename=sealed_filename(FALLBACK_STEM, suffix=_OUTPUT_SUFFIX),
+            row_count=row_count,
+            column_names=columns,
+            declared_cell_count=row_count * len(columns),
+        )
+
+
+@dataclass(frozen=True)
+class DelimitedTextReaderV1_1(DelimitedTextReader):
+    """``delimited_text@1.1.0`` admits ``text/plain`` as a label, and decodes nothing new.
+
+    A publisher that serves comma-separated bytes under ``text/plain`` is not describing a
+    different file; it is declining to describe the file at all.  ``acquisition/http.py``
+    already writes that rule down for the byte-stream spelling -- a weak label "says only
+    that a server declined to say anything", and the control is the structural check behind
+    the sandbox boundary, never the header.  Version ``1.0.0`` admitted the two labels that
+    name the format outright, which refused whole archives that were never unfit: Iowa
+    Environmental Mesonet serves its ASOS/METAR CSV under ``text/plain``, and so does a large
+    share of public agency endpoints.
+
+    This is an admission change and not a decode change.  ``decode`` is inherited untouched,
+    so the bytes a recipe seals under this coordinate are the bytes ``1.0.0`` would have
+    sealed; what moves is only which responses reach the decoder.  It is a new coordinate for
+    the reason ``archive.zip@1.1.0`` is one: ``1.0.0`` stays closed to the weak label, so an
+    already-approved recipe cannot silently begin admitting responses its review never saw.
+
+    The label is the only thing widened.  ``text/html`` remains outside every allowlist, which
+    is what keeps the bot-wall refusal intact, and the shared decoder's own refusals -- strict
+    UTF-8, a NUL byte, a repeated column name, a row whose width differs from its header --
+    are unchanged and remain the whole admission.  Archive and executable bytes arriving under
+    this label are refused as invalid UTF-8 rather than by prefix: the magic table is enforced
+    in ``acquisition/parsing`` and in the container families, and this family imports neither.
+    """
+
+    family_version: str = "1.1.0"
+    accepted_media_types: tuple[str, ...] = (
+        "text/csv",
+        "text/plain",
+        "text/tab-separated-values",
+    )

mostlyright/data_harness/readers/grib2/__init__.py

@@ -0,0 +1,33 @@
+"""The weather-model Reader separates admission from decoding.
+
+A weather model publishes its output as GRIB2 messages, and a GRIB2 message states its
+geometry and its packing method in its headers, before any value appears.  That is the whole
+reason this package is split the way it is.  ``admission`` reads those headers, decides
+whether this product is willing to open the message at all, and returns the offsets of the
+data section without reading it. The decoder requires that admission result.
+
+``MESSAGES_ROOT`` is the vendored evidence: three real messages, committed with their
+provenance, from which every number in the allowlist was read.  They are also the family's
+known-good samples under ``readers/samples/weather.grib2/1.0.0``. Admission and sample checks read
+the same files.
+
+The path is spelled out rather than imported from ``readers.samples``, which would close an
+import cycle: the sample loader reads the Toolbox, the Toolbox holds the weather family, and the
+weather family is in this package.
+
+Import discipline: this package reads ``readers.contracts`` and the standard library.  Like
+every module under ``readers/``, it must not import ``acquisition``, ``sources``, ``recipe``, or
+``pipeline``; those layers import this one and a back-edge would close an import cycle.
+"""
+
+from __future__ import annotations
+
+import pathlib
+
+__all__ = ["MESSAGES_ROOT"]
+
+# The vendored messages ship inside the package because that is where they are needed: the
+# warm-up check runs in production and its file must be installed with the code that opens it.
+MESSAGES_ROOT = (
+    pathlib.Path(__file__).resolve().parent.parent / "samples" / "weather.grib2" / "1.0.0"
+)