mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/readers/tabular.py

@@ -0,0 +1,510 @@
+"""The one canonical output encoder, the pinned float rule, and the shared delimited reader.
+
+Every Reader family emits through this module.  The sealed artifact is a Reader's output,
+so byte-stability is a Reader-contract obligation rather than a per-family accident: two
+runs of the same decode on two hosts produce the same bytes, therefore the same digest,
+therefore the same receipt.
+
+The float rule, pinned
+----------------------
+This is the contract other families are held to, so it is written as prose a reviewer can
+check rather than left to whatever a formatter happens to do.
+
+* Values are IEEE-754 binary64 end to end.  Nothing is widened, narrowed, or rounded on the
+  way to text, and no fused multiply-add or extended-precision accumulation is permitted in
+  a path that produces a value rendered here.
+* The text form is the shortest decimal string that parses back to the identical bit
+  pattern.  Shortest round-trip, not fixed precision: a formatter that emits more digits is
+  as wrong as one that emits fewer, because the extra digits are not information and they
+  move the digest.
+* No locale, no thousands separator, and no exponent normalisation beyond the shortest form
+  itself.  The decimal point is a full stop and nothing else.
+* Negative zero renders as ``-0.0`` and is never folded into ``0.0``.  They are two values,
+  they compare equal, and a fold would make two different inputs seal to one artifact.
+* A non-finite value is never rendered.  It is a missing cell instead.  This is what keeps
+  the output inside the boundary the sandbox IPC enforces, which refuses non-finite numbers
+  outright: a missing value has already become an empty cell before anything crosses it.
+
+The missing-value rule
+----------------------
+An absent value and a non-finite value are both the empty cell.  The empty cell is the only
+representation of "no value here": there is no sentinel, no ``NULL`` text, and no
+distinction between an absent value and an empty string once the bytes are sealed.
+
+One decoder, one delimiter parameter
+------------------------------------
+``decode_delimited`` takes its delimiter as an argument, which is the whole of
+tab-separated support and the whole of decoding a container's member.  A separate
+tab-separated reader would be a second implementation of one thing, so there is not one.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import itertools
+import math
+import re
+from collections.abc import Iterable, Iterator, Mapping, Sequence
+from typing import Any
+
+from mostlyright.data_harness.fast_delimited import certified_rows, certifies
+from mostlyright.data_harness.readers.contracts import ReaderBudgets, ReaderError
+
+__all__ = [
+    "FALLBACK_STEM",
+    "check_declared_size",
+    "decode_delimited",
+    "decode_delimited_stream",
+    "encode_canonical_csv",
+    "format_float",
+    "sealed_filename",
+]
+
+# A column name may not carry a control character.  The rule is the parser's own, so a table
+# a Reader produces is a table the parser would have admitted.
+_SAFE_COLUMN = re.compile(r"^[^\x00-\x1f\x7f]{1,256}$")
+
+# The quoting settings.  Fixed, not configurable: a configurable quote character would be a
+# knob that moves the sealed bytes without moving the recipe digest.
+_QUOTE_CHARACTER = '"'
+
+# The characters a derived sealed filename may carry, from the Reader contract's own rule.
+_FILENAME_SAFE = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.-")
+_FILENAME_STEM_BYTES = 200
+
+# The stem used when a family has no caller-supplied name to derive one from, and when a
+# supplied name leaves nothing usable behind.  Public because a family with no name of its own
+# -- a gzip stream carries none -- passes it in deliberately rather than inventing a second one.
+FALLBACK_STEM = "decoded"
+
+
+def sealed_filename(stem_source: str, *, suffix: str) -> str:
+    """Derive the sealed artifact's bare filename from whatever the source called the thing.
+
+    Derived rather than taken.  An archive member name, a worksheet title -- both are text a
+    stranger chose, and the sealed filename has to satisfy the Reader contract's confined
+    bare-name rule.  Anything outside that rule becomes an underscore, and a name left with
+    nothing usable becomes a fixed stem instead of an empty one.
+
+    One implementation, shared by every family.  A second copy would let two families disagree
+    about what a confined name is, and the disagreement would surface as a ``READER_OUTPUT``
+    refusal from the result contract rather than as anything a reader of the code could see.
+    """
+
+    base = stem_source.replace("\\", "/").rsplit("/", 1)[-1]
+    stem = base[: base.rindex(".")] if "." in base[1:] else base
+    kept = "".join(character if character in _FILENAME_SAFE else "_" for character in stem)
+    kept = kept.lstrip("_.-")[:_FILENAME_STEM_BYTES]
+    return (kept or FALLBACK_STEM) + suffix
+
+
+def format_float(value: float) -> str:
+    """Render one binary64 value as the shortest decimal text that round-trips.
+
+    See the float rule in the module docstring; this function is that rule's one
+    implementation.  A non-finite value is refused rather than rendered, because the caller
+    that has one is meant to emit a missing cell instead.
+    """
+
+    if not isinstance(value, float) or isinstance(value, bool):
+        raise ReaderError(
+            "READER_OUTPUT",
+            "reader.output.value",
+            "the float rule applies to binary64 values only",
+        )
+    if not math.isfinite(value):
+        raise ReaderError(
+            "READER_OUTPUT",
+            "reader.output.value",
+            "a non-finite value is never rendered; emit a missing cell instead",
+        )
+    # ``repr`` is Python's shortest-round-trip binary64 formatter.  It is named here rather
+    # than reimplemented because a hand-rolled shortest-form search is exactly the code that
+    # would drift from the rule the docstring pins.
+    return repr(value)
+
+
+def _render_cell(value: Any, subject: str) -> str:
+    """Render one value as canonical cell text, or as the empty cell when it is missing."""
+
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        if "\x00" in value:
+            raise ReaderError("READER_OUTPUT", subject, "a cell may not carry a NUL byte")
+        return value
+    if isinstance(value, bool):
+        # Pinned lower-case, so a spreadsheet's boolean seals the same way on every host.
+        return "true" if value else "false"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        return "" if not math.isfinite(value) else format_float(value)
+    raise ReaderError(
+        "READER_OUTPUT",
+        subject,
+        f"no pinned rendering exists for a {type(value).__name__} cell",
+    )
+
+
+def _validate_columns(
+    columns: Sequence[str], budgets: ReaderBudgets, subject: str
+) -> tuple[str, ...]:
+    if isinstance(columns, str) or not isinstance(columns, Sequence) or not columns:
+        raise ReaderError("READER_OUTPUT", subject, "must be a nonempty sequence of column names")
+    if len(columns) > budgets.max_columns:
+        raise ReaderError("READER_BUDGET", subject, "exceeds the column budget")
+    if any(not isinstance(name, str) or _SAFE_COLUMN.fullmatch(name) is None for name in columns):
+        raise ReaderError(
+            "READER_OUTPUT",
+            subject,
+            "column names must be bounded text without control characters",
+        )
+    if len(set(columns)) != len(columns):
+        raise ReaderError("READER_OUTPUT", subject, "column names must be unique")
+    return tuple(columns)
+
+
+def check_declared_size(
+    *,
+    row_count: int,
+    column_count: int,
+    budgets: ReaderBudgets,
+    subject: str,
+) -> None:
+    """Refuse a declared geometry that is over budget, before anything is decoded.
+
+    A spreadsheet states its used range and a weather-model message states its grid, so the
+    size of the result is knowable before the first cell is read.  A family calls this with
+    the declared numbers in hand, which is what turns a declared-size bomb into a refusal
+    instead of an allocation.
+    """
+
+    for name, value in (("row_count", row_count), ("column_count", column_count)):
+        if type(value) is not int or value < 0:
+            raise ReaderError(
+                "READER_BUDGET",
+                f"{subject}.{name}",
+                "a declared size must be a non-negative integer",
+            )
+    if column_count > budgets.max_columns:
+        raise ReaderError("READER_BUDGET", subject, "the declared column count is over budget")
+    if row_count > budgets.max_rows:
+        raise ReaderError("READER_BUDGET", subject, "the declared row count is over budget")
+    if row_count * column_count > budgets.max_declared_cells:
+        raise ReaderError("READER_BUDGET", subject, "the declared cell count is over budget")
+
+
+def encode_canonical_csv(
+    columns: Sequence[str],
+    rows: Iterable[Sequence[Any] | Mapping[str, Any]],
+    *,
+    budgets: ReaderBudgets | None = None,
+) -> bytes:
+    """Encode a table as the canonical sealed bytes: utf-8, ``\\n``, no byte-order mark.
+
+    Column order is exactly the stated order and never inferred from a row.  A row may be a
+    sequence matching that order exactly, or a mapping keyed by column name -- a mapping
+    naming a column the header does not state is refused rather than dropped, and a column
+    the mapping omits is the empty cell.
+
+    Budgets are enforced while reading the rows, so an over-budget input is refused before
+    it is materialised rather than after.
+    """
+
+    limits = budgets if budgets is not None else ReaderBudgets()
+    header = _validate_columns(columns, limits, "reader.output.columns")
+    # One scratch buffer holds one encoded line at a time, and the table accumulates as
+    # utf-8 bytes.  The whole output therefore exists once, not once as text and once as
+    # bytes -- a table near the output budget would otherwise cost twice its own size at
+    # the encode boundary.  Encoding line by line is byte-identical to encoding the joined
+    # text, because utf-8 carries no state between code points.
+    scratch = io.StringIO(newline="")
+    writer = csv.DictWriter(
+        scratch,
+        fieldnames=list(header),
+        lineterminator="\n",
+        extrasaction="raise",
+        restval="",
+        quotechar=_QUOTE_CHARACTER,
+        doublequote=True,
+    )
+    encoded = bytearray()
+    encoded_size = 0
+    encode_error: UnicodeEncodeError | None = None
+
+    def flush_scratch() -> None:
+        # The output-byte budget stays an end-of-input refusal, exactly where it has always
+        # been: rows keep streaming so an admission refusal later in the input still wins,
+        # but the buffer stops growing, so an over-budget input cannot cost its own size in
+        # memory before it is refused.  An unencodable cell -- a lone surrogate -- is held
+        # to the same order: the joined-text form could not crash on it until the loop had
+        # finished, so the first such error is recorded here and raised after the loop,
+        # before the budget comparison, exactly where the one-shot encode used to raise.
+        nonlocal encoded_size, encode_error
+        piece_text = scratch.getvalue()
+        scratch.seek(0)
+        scratch.truncate(0)
+        if encode_error is not None:
+            return
+        try:
+            piece = piece_text.encode("utf-8")
+        except UnicodeEncodeError as error:
+            encode_error = error
+            return
+        encoded_size += len(piece)
+        if encoded_size <= limits.max_output_bytes:
+            encoded.extend(piece)
+
+    writer.writeheader()
+    flush_scratch()
+
+    width = len(header)
+    written = 0
+    for row in rows:
+        if written >= limits.max_rows:
+            raise ReaderError("READER_BUDGET", "reader.output.rows", "exceeds the row budget")
+        if (written + 1) * width > limits.max_declared_cells:
+            raise ReaderError("READER_BUDGET", "reader.output.rows", "exceeds the cell budget")
+        subject = f"reader.output.rows[{written}]"
+        if isinstance(row, Mapping):
+            rendered = {key: _render_cell(value, subject) for key, value in row.items()}
+        elif isinstance(row, Sequence) and not isinstance(row, (str, bytes, bytearray)):
+            if len(row) != width:
+                raise ReaderError("READER_OUTPUT", subject, "row width differs from the header")
+            rendered = {
+                name: _render_cell(value, subject) for name, value in zip(header, row, strict=True)
+            }
+        else:
+            raise ReaderError("READER_OUTPUT", subject, "a row must be a sequence or a mapping")
+        try:
+            writer.writerow(rendered)
+        except ValueError as error:
+            raise ReaderError(
+                "READER_OUTPUT",
+                subject,
+                "row names a column the header does not state",
+            ) from error
+        flush_scratch()
+        written += 1
+
+    if encode_error is not None:
+        raise encode_error
+    if encoded_size > limits.max_output_bytes:
+        raise ReaderError(
+            "READER_BUDGET",
+            "reader.output.content",
+            "exceeds the output byte budget",
+        )
+    return bytes(encoded)
+
+
+def decode_delimited(
+    content: bytes,
+    *,
+    delimiter: str,
+    header: bool = True,
+    budgets: ReaderBudgets | None = None,
+) -> tuple[tuple[str, ...], tuple[tuple[str, ...], ...]]:
+    """Decode delimited text into a header and its rows, refusing what the parser refuses.
+
+    This is ``decode_delimited_stream`` drained into tuples, for the callers whose input is
+    small and whose next step needs the whole table.  The admissions, budgets, and refusal
+    order are the stream's own -- there is one delimited decoder, not a strict one and a
+    lenient one.
+    """
+
+    columns, rows = decode_delimited_stream(
+        content, delimiter=delimiter, header=header, budgets=budgets
+    )
+    return columns, tuple(rows)
+
+
+def decode_delimited_stream(
+    content: bytes,
+    *,
+    delimiter: str,
+    header: bool = True,
+    budgets: ReaderBudgets | None = None,
+) -> tuple[tuple[str, ...], Iterator[tuple[str, ...]]]:
+    """Decode delimited text into a header and an iterator of admitted rows.
+
+    The refusals are the ones the acquisition parser already makes, for the same reasons: a
+    NUL byte, a repeated column name, and a row whose width differs from its header are all
+    refused rather than repaired.  Text that is not strict UTF-8 is refused too; one leading
+    byte-order mark is removed first, because a mark is an encoding artefact of the file and
+    not the first character of the first column's name.
+
+    When the recipe says there is no header, columns are named by position rather than
+    guessed from the first row, so the first row stays data.
+
+    The header and every prologue refusal are settled before this function returns; each row
+    is admitted as it is pulled.  Streaming is what keeps a year-scale table from existing
+    as millions of live tuples at once: the caller that feeds rows straight into the
+    canonical encoder holds one row at a time, and the budgets refuse an over-budget input
+    at the same row they always have.
+
+    Which tokenizer splits the rows is chosen by the input rather than by an option.
+    ``fast_delimited`` states a closed subset of inputs for which splitting on the line
+    terminator and then on the delimiter is the same tokenizing the stdlib parser performs,
+    and inside that subset it does the splitting without decoding the whole input into one
+    string first.  Everything outside the subset -- a quote, a carriage return, a NUL byte,
+    a blank line, text that is not strict UTF-8 -- reaches the stdlib parser below and is
+    admitted or refused by it exactly as it always was.  The one refusal that belongs to the
+    tokenizing rather than to this function, a field longer than ``csv.field_size_limit()``,
+    the splitter raises itself as the same ``csv.Error``, so it arrives here as the malformed
+    refusal it has always arrived as.
+    """
+
+    limits = budgets if budgets is not None else ReaderBudgets()
+    if (
+        not isinstance(delimiter, str)
+        or len(delimiter) != 1
+        or delimiter == _QUOTE_CHARACTER
+        or delimiter in "\r\n"
+    ):
+        raise ReaderError(
+            "READER_OPTIONS",
+            "reader.decode.delimiter",
+            "must be exactly one character that is neither the quote nor a line terminator",
+        )
+    if not isinstance(content, (bytes, bytearray)):
+        raise ReaderError("READER_ADMISSION", "reader.decode.content", "must be exact bytes")
+    # Checked before decoding, so an over-budget input is refused without being materialised
+    # as text first.
+    if len(content) > limits.max_input_bytes:
+        raise ReaderError(
+            "READER_BUDGET",
+            "reader.decode.content",
+            "exceeds the input byte budget",
+        )
+    raw = bytes(content).removeprefix(b"\xef\xbb\xbf")
+    # The certified splitter is the same tokenizing for the inputs it admits, so which one ran
+    # is not observable in a row, a refusal, or a sealed byte -- only in what the decode cost.
+    # An input it declines never reaches it: the stdlib reader below runs from the first byte,
+    # including for text that is not strict UTF-8, which the subset excludes rather than
+    # refuses so that the refusal keeps its one home.
+    certified = certifies(raw, delimiter=delimiter)
+    reader: Iterator[list[str]]
+    if certified:
+        reader = certified_rows(raw, delimiter=delimiter)
+    else:
+        try:
+            text = raw.decode("utf-8", errors="strict")
+        except UnicodeDecodeError:
+            raise ReaderError(
+                "READER_ADMISSION",
+                "reader.decode.content",
+                "source text must be strict UTF-8",
+            ) from None
+
+        reader = csv.reader(
+            io.StringIO(text, newline=""),
+            delimiter=delimiter,
+            quotechar=_QUOTE_CHARACTER,
+            doublequote=True,
+            strict=True,
+        )
+    pending: list[list[str]] = []
+    try:
+        if header:
+            first = next(reader, None)
+            if first is None:
+                raise ReaderError(
+                    "READER_ADMISSION",
+                    "reader.decode.header",
+                    "delimited text has no header row",
+                )
+            columns = _decoded_columns(first, limits)
+        else:
+            first = next(reader, None)
+            if first is None:
+                raise ReaderError(
+                    "READER_ADMISSION",
+                    "reader.decode.content",
+                    "delimited text has no rows",
+                )
+            positional = [f"column_{index + 1}" for index in range(len(first))]
+            columns = _decoded_columns(positional, limits)
+            pending.append(first)
+    except csv.Error:
+        raise ReaderError(
+            "READER_ADMISSION",
+            "reader.decode.content",
+            "delimited text is malformed",
+        ) from None
+    return columns, _admitted_rows(
+        reader, pending, len(columns), limits, scan_for_nul=not certified
+    )
+
+
+def _admitted_rows(
+    reader: Iterator[list[str]],
+    pending: list[list[str]],
+    width: int,
+    limits: ReaderBudgets,
+    *,
+    scan_for_nul: bool = True,
+) -> Iterator[tuple[str, ...]]:
+    """Yield each admitted row, refusing exactly what the drained decoder refused.
+
+    ``scan_for_nul`` is not a policy switch and never widens what is admitted: it is off only
+    when the whole input has already been searched for a NUL byte and found not to carry one,
+    which is one of the conditions the certified subset is defined by.  A cell of an input
+    with no NUL byte in it cannot contain a NUL byte, so the scan below would spend a pass
+    over every cell to reach the answer that pass already has.
+    """
+
+    admitted = 0
+    try:
+        # Chained rather than unpacked: unpacking would read the whole input into a list
+        # before the first budget check, which is the allocation the budgets exist to
+        # prevent.
+        for raw_row in itertools.chain(pending, reader):
+            if admitted >= limits.max_rows:
+                raise ReaderError("READER_BUDGET", "reader.decode.rows", "exceeds the row budget")
+            if (admitted + 1) * width > limits.max_declared_cells:
+                raise ReaderError("READER_BUDGET", "reader.decode.rows", "exceeds the cell budget")
+            if len(raw_row) != width:
+                raise ReaderError(
+                    "READER_ADMISSION",
+                    f"reader.decode.rows[{admitted}]",
+                    "row width differs from its header",
+                )
+            if scan_for_nul:
+                for cell in raw_row:
+                    if "\x00" in cell:
+                        raise ReaderError(
+                            "READER_ADMISSION",
+                            f"reader.decode.rows[{admitted}]",
+                            "delimited text contains a NUL byte",
+                        )
+            yield tuple(raw_row)
+            admitted += 1
+    except csv.Error:
+        raise ReaderError(
+            "READER_ADMISSION",
+            "reader.decode.content",
+            "delimited text is malformed",
+        ) from None
+
+
+def _decoded_columns(values: Sequence[str], budgets: ReaderBudgets) -> tuple[str, ...]:
+    if not values:
+        raise ReaderError(
+            "READER_ADMISSION",
+            "reader.decode.header",
+            "table must have a nonempty column set",
+        )
+    if len(values) > budgets.max_columns:
+        raise ReaderError("READER_BUDGET", "reader.decode.header", "exceeds the column budget")
+    if any(_SAFE_COLUMN.fullmatch(name) is None for name in values):
+        raise ReaderError(
+            "READER_ADMISSION",
+            "reader.decode.header",
+            "column names must be bounded text without control characters",
+        )
+    if len(set(values)) != len(values):
+        raise ReaderError("READER_ADMISSION", "reader.decode.header", "column names must be unique")
+    return tuple(values)