mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/acquisition/ranges.py

@@ -0,0 +1,495 @@
+"""The content half of slice fetching: sidecar parsing, range planning, structural checks.
+
+Everything here is pure and I/O-free.  It parses an untrusted index sidecar into an ordered
+message table, turns a recipe-named selection into exact byte ranges emitted in index order,
+and proves a fetched slice is a well-formed GRIB message before anything else touches it.
+
+The division of labour with ``acquisition.http`` is deliberate and no single part is sufficient
+alone.  HTTP proves the *span*: exactly 206, a ``Content-Range`` parsed strictly and equal to
+what was ordered, three lengths that agree.  ``check_grib_message`` proves the delivered bytes
+are a whole, well-formed message rather than a shifted, truncated or spliced window of one --
+the shape a wrong offset produces.
+
+Neither proves *which object* the message came from.  Two well-formed messages of equal length
+are indistinguishable to both, and for GRIB2 at a fixed grid and a fixed bits-per-value, equal
+length across a re-issued or superseded run is the ordinary case rather than a contrived one.
+``check_grib_reference_time`` is the control for that case: it reads Section 1's reference time
+out of the delivered message and refuses anything that did not come from the run the order
+named.  It is in force only when the order names a run -- ``expected_reference_time`` on the
+range plan -- because a slice order that names no run has nothing to be held to.
+
+What remains uncovered, stated rather than implied, in two parts.
+
+First, a *different object belonging to the same run* -- a neighbouring field's file at the same
+cycle, of equal message length -- passes every check here.  Closing that would need per-object
+evidence the index sidecar does not carry.
+
+Second, and cheaper: *the index chooses which message answers a descriptor, and nothing verifies
+that pairing*.  ``plan_ranges`` turns a descriptor into a byte span using nothing but the
+sidecar's own descriptor->offset mapping, and no control anywhere binds a delivered message back
+to the descriptor that named it.  All three controls -- the span, ``check_grib_message`` and
+``check_grib_reference_time`` -- are properties of the bytes alone, so a sidecar that keeps the
+object's real, strictly ascending offsets and attaches each descriptor to its neighbour's span
+delivers a whole, well-formed message from the correct model run at exactly the span the index
+named.  The sidecar is untrusted input for its *shape*; it is trusted for the descriptor->offset
+mapping, and this is where that trust is spent.  Unlike the case above it needs no second file,
+no mirror lag and no redirect -- only the small text ``.idx`` beside the object, which is a
+separate, far cheaper cache or edge object than the binary it describes.  Both are pinned by
+``known_gap`` tests in ``tests/h3/test_ranges.py`` and ``tests/h3/test_sandbox.py``.
+
+This module is a leaf on purpose: it imports the transport's typed range and its range budget
+and nothing else from the package, so the sandbox worker can import it without dragging in a
+recipe, a pipeline or a source registry.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import date
+
+from mostlyright.data_harness.acquisition.http import ByteRange, require_range_plan_within_budget
+from mostlyright.data_harness.acquisition.url_policy import AcquisitionSecurityError
+
+# A message number or a byte offset, in one canonical spelling: no sign, no whitespace, no
+# leading zeros, and at most sixteen digits.  The same shape ``acquisition.http`` accepts in a
+# ``Content-Range``, for the same reason -- one delivered span must have one spelling, and a
+# sidecar must not be able to name an offset no object could have.
+_SIDECAR_DECIMAL = re.compile(r"0|[1-9][0-9]{0,15}")
+
+GRIB_MAGIC = b"GRIB"
+GRIB_EDITION = 2
+GRIB_EDITION_OFFSET = 7
+GRIB_TRAILER = b"7777"
+# Section 0 is sixteen bytes: ``GRIB``, two reserved bytes, the discipline, the edition, then
+# the whole message's length as a big-endian eight-byte integer.  With the four-byte trailer,
+# the shortest byte string that could be a message at all is twenty bytes.
+GRIB_SECTION_ZERO_BYTES = 16
+GRIB_LENGTH_SLICE = slice(8, 16)
+GRIB_MINIMUM_BYTES = GRIB_SECTION_ZERO_BYTES + len(GRIB_TRAILER)
+
+# Section 1, the identification section, begins where Section 0 ends and is at least twenty-one
+# octets.  Its octets are numbered from one, so octet ``n`` sits at index ``15 + n`` of the
+# message.  Only the four fields below are read and nothing here decodes: the reference time,
+# and the significance byte that says what the reference time means.
+GRIB_SECTION_ONE_OFFSET = GRIB_SECTION_ZERO_BYTES
+GRIB_SECTION_ONE_MINIMUM_BYTES = 21
+GRIB_SECTION_ONE_NUMBER = 1
+GRIB_SECTION_ONE_LENGTH_SLICE = slice(16, 20)
+GRIB_SECTION_ONE_NUMBER_OFFSET = 20
+# Code table 1.2.  Value 1 is "start of forecast", which is the model run itself.  Any other
+# significance names a different instant -- a verifying time, an observation time -- and is
+# refused rather than compared against a cycle it does not denote.
+GRIB_REFERENCE_SIGNIFICANCE_OFFSET = 27
+GRIB_REFERENCE_START_OF_FORECAST = 1
+GRIB_REFERENCE_YEAR_SLICE = slice(28, 30)
+GRIB_REFERENCE_MONTH_OFFSET = 30
+GRIB_REFERENCE_DAY_OFFSET = 31
+GRIB_REFERENCE_HOUR_OFFSET = 32
+GRIB_REFERENCE_MINUTE_OFFSET = 33
+GRIB_REFERENCE_SECOND_OFFSET = 34
+
+# One spelling for a model run, everywhere it crosses a boundary: a UTC instant to the second,
+# zero-padded, with the literal ``Z``.  The pattern is the shape; ``date`` below is what rejects
+# a shape-valid instant that is not a day, so ``2026-02-30T00:00:00Z`` cannot be ordered.
+_REFERENCE_TIME_SPELLING = re.compile(
+    r"([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z"
+)
+
+
+@dataclass(frozen=True)
+class SidecarLimits:
+    """Coordinator-owned ceilings for parsing one index sidecar.
+
+    A real NOAA ``.idx`` for a ~518 MiB GFS file is a few hundred lines of a few hundred bytes.
+    The defaults leave that an order of magnitude of headroom and refuse anything past it, so a
+    pathological or hostile sidecar is bounded before it is looked at rather than after.
+    """
+
+    max_bytes: int = 1024 * 1024
+    max_lines: int = 4096
+    max_line_bytes: int = 512
+
+    def __post_init__(self) -> None:
+        bounded = (
+            ("max_bytes", self.max_bytes, 1, 16 * 1024 * 1024),
+            ("max_lines", self.max_lines, 1, 65_536),
+            ("max_line_bytes", self.max_line_bytes, 1, 8192),
+        )
+        for name, value, minimum, maximum in bounded:
+            if type(value) is not int or not minimum <= value <= maximum:
+                raise AcquisitionSecurityError(
+                    "SIDECAR_LIMIT",
+                    f"{name} must be an integer in [{minimum}, {maximum}]",
+                )
+
+
+@dataclass(frozen=True)
+class SidecarMessage:
+    """One line of an index sidecar, after parsing: where a message starts and what it is.
+
+    A sidecar value is never allowed to become a length, an allocation size or a URL.  The
+    offset is a proposal about an object, checked against what actually arrives.
+    """
+
+    index: int
+    offset: int
+    descriptor: str
+
+    def __post_init__(self) -> None:
+        if type(self.index) is not int or self.index < 1:
+            raise AcquisitionSecurityError(
+                "SIDECAR_MESSAGE",
+                "message index must be an integer of at least 1",
+            )
+        if type(self.offset) is not int or self.offset < 0:
+            raise AcquisitionSecurityError(
+                "SIDECAR_MESSAGE",
+                "message offset must be a non-negative integer",
+            )
+        if not isinstance(self.descriptor, str):
+            raise AcquisitionSecurityError(
+                "SIDECAR_MESSAGE",
+                "message descriptor must be text",
+            )
+
+
+def parse_index_sidecar(content: bytes, *, limits: SidecarLimits) -> tuple[SidecarMessage, ...]:
+    """Parse an untrusted ``.idx`` sidecar into an ordered message table, or refuse it.
+
+    The sidecar is untrusted input and is treated as such throughout: it is bounded before it
+    is decoded, decoded strictly, and accepted only in one closed shape.  Message numbers must
+    run exactly ``1, 2, 3, ...`` and offsets must ascend strictly.  Strict ascent is what makes
+    each message's end computable -- a message ends where the next one begins -- and what stops
+    a crafted sidecar from proposing an inverted or overlapping span.
+
+    No refusal here quotes the sidecar's own text.  A refusal crosses the sandbox boundary and
+    must not become a channel for whatever the fetched bytes happened to contain.
+    """
+
+    if not isinstance(content, bytes):
+        raise AcquisitionSecurityError("SIDECAR_INPUT", "sidecar input must be exact bytes")
+    if len(content) > limits.max_bytes:
+        raise AcquisitionSecurityError(
+            "SIDECAR_BYTES",
+            f"sidecar exceeds the {limits.max_bytes} byte budget",
+        )
+    try:
+        text = content.decode("ascii", errors="strict")
+    except UnicodeDecodeError:
+        raise AcquisitionSecurityError("SIDECAR_ENCODING", "sidecar must be strict ASCII") from None
+
+    lines = text.split("\n")
+    # Exactly one trailing empty line is a line terminator on the final record.  Every other
+    # blank line is a malformed record and falls through to the field check below.
+    if lines and lines[-1] == "":
+        lines = lines[:-1]
+    if not lines:
+        raise AcquisitionSecurityError("SIDECAR_EMPTY", "sidecar names no messages")
+    if len(lines) > limits.max_lines:
+        raise AcquisitionSecurityError(
+            "SIDECAR_LINES",
+            f"sidecar exceeds the {limits.max_lines} line budget",
+        )
+
+    messages: list[SidecarMessage] = []
+    for line_number, line in enumerate(lines, start=1):
+        if len(line) > limits.max_line_bytes:
+            raise AcquisitionSecurityError(
+                "SIDECAR_LINE",
+                f"sidecar line {line_number} exceeds the {limits.max_line_bytes} byte budget",
+            )
+        fields = line.split(":")
+        if len(fields) < 3:
+            raise AcquisitionSecurityError(
+                "SIDECAR_FIELD",
+                f"sidecar line {line_number} has fewer than three fields",
+            )
+        if not _SIDECAR_DECIMAL.fullmatch(fields[0]) or not _SIDECAR_DECIMAL.fullmatch(fields[1]):
+            raise AcquisitionSecurityError(
+                "SIDECAR_FIELD",
+                f"sidecar line {line_number} does not begin with a plain number and offset",
+            )
+        index = int(fields[0])
+        offset = int(fields[1])
+        if index != line_number:
+            raise AcquisitionSecurityError(
+                "SIDECAR_ORDER",
+                f"sidecar line {line_number} is numbered {index}, breaking the message sequence",
+            )
+        if messages and offset <= messages[-1].offset:
+            raise AcquisitionSecurityError(
+                "SIDECAR_ORDER",
+                f"sidecar line {line_number} does not advance past the previous offset",
+            )
+        descriptor = ":".join(fields[2:])
+        if descriptor.endswith(":"):
+            descriptor = descriptor[:-1]
+        messages.append(SidecarMessage(index=index, offset=offset, descriptor=descriptor))
+
+    return tuple(messages)
+
+
+@dataclass(frozen=True)
+class PlannedRange:
+    """One message the acquisition ordered: which message it is and exactly which bytes."""
+
+    index: int
+    descriptor: str
+    byte_range: ByteRange
+
+    def __post_init__(self) -> None:
+        if type(self.index) is not int or self.index < 1:
+            raise AcquisitionSecurityError(
+                "PLANNED_RANGE",
+                "planned message index must be an integer of at least 1",
+            )
+        if not isinstance(self.descriptor, str):
+            raise AcquisitionSecurityError("PLANNED_RANGE", "planned descriptor must be text")
+        if not isinstance(self.byte_range, ByteRange):
+            raise AcquisitionSecurityError(
+                "PLANNED_RANGE",
+                "a planned range must carry the transport's own byte range type",
+            )
+
+
+def plan_ranges(
+    messages: tuple[SidecarMessage, ...],
+    *,
+    selectors: tuple[str, ...],
+    max_ranges: int,
+) -> tuple[PlannedRange, ...]:
+    """Turn a recipe-named selection into exact byte ranges, emitted in message-index order.
+
+    Matching is exact string equality against ``SidecarMessage.descriptor``.  There is no
+    substring match, no glob and no regular expression: a selector that matched by prefix would
+    silently pick up a different variable the day a producer adds a level to its descriptors,
+    and the acquisition would keep succeeding while fetching the wrong field.
+
+    The result is sorted by message index before it is returned, and that sort is what makes the
+    sealed digest reproducible.  Slices may be fetched with any concurrency, so responses arrive
+    in whatever order the network produced; emitting in arrival order would make the receipt a
+    function of network timing rather than of the recipe.
+
+    Selecting the last message in the sidecar is refused.  Its end is the end of the object and
+    this module does not learn object lengths -- a guessed end is exactly the "correct span,
+    wrong bytes" failure the structural check below exists to catch, and guessing it here would
+    manufacture that failure rather than detect it.
+
+    No refusal quotes a selector or a descriptor: both are text that came from outside.
+    """
+
+    if not isinstance(messages, tuple) or not messages:
+        raise AcquisitionSecurityError(
+            "SIDECAR_EMPTY",
+            "a range plan needs a non-empty parsed message table",
+        )
+    if not all(isinstance(message, SidecarMessage) for message in messages):
+        raise AcquisitionSecurityError(
+            "SIDECAR_MESSAGE",
+            "a range plan may only be built from parsed sidecar messages",
+        )
+    if not isinstance(selectors, tuple) or not all(
+        isinstance(selector, str) for selector in selectors
+    ):
+        raise AcquisitionSecurityError("SELECTOR_TYPE", "selectors must be a tuple of text")
+
+    # Bound the plan before doing any of its work, and bound it in exactly one place: the
+    # transport owns the range budget, so this module calls it rather than restating it.
+    require_range_plan_within_budget(len(selectors), max_ranges=max_ranges)
+
+    if len(set(selectors)) != len(selectors):
+        raise AcquisitionSecurityError(
+            "SELECTOR_DUPLICATE",
+            "a selector is named more than once; a repeated order is not a plan",
+        )
+
+    by_descriptor: dict[str, list[int]] = {}
+    for position, message in enumerate(messages):
+        by_descriptor.setdefault(message.descriptor, []).append(position)
+
+    planned: list[PlannedRange] = []
+    for selector in selectors:
+        positions = by_descriptor.get(selector, [])
+        if not positions:
+            raise AcquisitionSecurityError(
+                "SELECTOR_MISS",
+                "a selector names no message in this sidecar",
+            )
+        if len(positions) > 1:
+            raise AcquisitionSecurityError(
+                "SELECTOR_AMBIGUOUS",
+                f"a selector names {len(positions)} messages in this sidecar",
+            )
+        position = positions[0]
+        if position == len(messages) - 1:
+            raise AcquisitionSecurityError(
+                "SELECTOR_TERMINAL",
+                "the last message in a sidecar has no computable end and is not selectable",
+            )
+        message = messages[position]
+        planned.append(
+            PlannedRange(
+                index=message.index,
+                descriptor=message.descriptor,
+                byte_range=ByteRange(
+                    first_byte=message.offset,
+                    last_byte=messages[position + 1].offset - 1,
+                ),
+            )
+        )
+
+    return tuple(sorted(planned, key=lambda entry: entry.index))
+
+
+def check_grib_message(payload: bytes) -> None:
+    """Prove a fetched slice is one whole, well-formed GRIB message, or refuse it.
+
+    The division of labour, stated plainly and no wider than it is.  ``acquisition.http``
+    establishes that the server answered 206 with a ``Content-Range`` equal to what was ordered
+    and that three lengths agree; that proves the *span*.  This function proves the *framing*:
+    a message carries its own magic, its own edition, its own self-declared length and its own
+    trailer, so a window taken at the wrong offset, a window one message short or long, and a
+    body that is not a message at all each fail at least one of the four.
+
+    What this function does **not** prove is which object the bytes came from.  Every check
+    below is a property of the message alone, so two well-formed messages of equal length are
+    indistinguishable here however different their contents -- which is exactly the shape a
+    stale cache, a re-issued file or a mirror running behind produces.  The control for that is
+    ``check_grib_reference_time``, and it is in force only when the order named a model run.
+
+    Nothing here decodes.  Section 1 and beyond, the grid and packing allowlist and the
+    known-good samples belong to the decoding family.
+
+    A refusal names the check that failed and never the payload.  These bytes are untrusted and
+    the refusal crosses the sandbox boundary, so it must not become a channel out.
+    """
+
+    if not isinstance(payload, bytes):
+        raise AcquisitionSecurityError("GRIB_PAYLOAD", "a slice must be checked as exact bytes")
+    # Length first, so no comparison below can index past the end of a short payload.
+    if len(payload) < GRIB_MINIMUM_BYTES:
+        raise AcquisitionSecurityError(
+            "GRIB_TRUNCATED",
+            f"a GRIB message is at least {GRIB_MINIMUM_BYTES} bytes",
+        )
+    if payload[: len(GRIB_MAGIC)] != GRIB_MAGIC:
+        raise AcquisitionSecurityError("GRIB_MAGIC", "the slice does not begin with GRIB")
+    if payload[GRIB_EDITION_OFFSET] != GRIB_EDITION:
+        raise AcquisitionSecurityError(
+            "GRIB_EDITION",
+            f"only GRIB edition {GRIB_EDITION} is admitted",
+        )
+    if int.from_bytes(payload[GRIB_LENGTH_SLICE], "big") != len(payload):
+        raise AcquisitionSecurityError(
+            "GRIB_LENGTH",
+            "the message's self-declared length is not the number of bytes fetched",
+        )
+    if payload[-len(GRIB_TRAILER) :] != GRIB_TRAILER:
+        raise AcquisitionSecurityError("GRIB_TRAILER", "the slice does not end with the trailer")
+
+
+def require_reference_time_spelling(value: str) -> str:
+    """Return a model-run instant in the one spelling this package accepts, or refuse it.
+
+    Shared by the coordinator that builds an order and the worker that parses one, so a run can
+    have only one spelling on either side of the boundary.  The refusal never quotes the value:
+    it is text that came from outside this process and the refusal crosses the sandbox boundary.
+    """
+
+    if not isinstance(value, str):
+        raise AcquisitionSecurityError(
+            "REFERENCE_TIME",
+            "a model run must be named as text",
+        )
+    matched = _REFERENCE_TIME_SPELLING.fullmatch(value)
+    if matched is None:
+        raise AcquisitionSecurityError(
+            "REFERENCE_TIME",
+            "a model run must be spelled YYYY-MM-DDTHH:MM:SSZ, in UTC, zero-padded",
+        )
+    year, month, day = (int(part) for part in matched.group(1, 2, 3))
+    hour, minute, second = (int(part) for part in matched.group(4, 5, 6))
+    try:
+        date(year, month, day)
+    except ValueError:
+        raise AcquisitionSecurityError(
+            "REFERENCE_TIME",
+            "a model run must name a real calendar day",
+        ) from None
+    if hour > 23 or minute > 59 or second > 59:
+        raise AcquisitionSecurityError(
+            "REFERENCE_TIME",
+            "a model run must name a real time of day",
+        )
+    return value
+
+
+def check_grib_reference_time(payload: bytes, *, expected: str) -> None:
+    """Prove a delivered message came from the model run that was ordered, or refuse it.
+
+    This is the control ``check_grib_message`` cannot be: every check there is a property of
+    the message alone, so a re-issued run, a stale cache or a mirror running behind can answer
+    the ordered span with a well-formed message of the same length from a *different* run and
+    pass.  Section 1 carries the reference time, and for a forecast product that reference time
+    is the run -- so a message from another run says so about itself, in its own bytes.
+
+    Read, never decoded, and read defensively: ``check_grib_message`` runs first and has already
+    established framing, but the section header is re-measured here so a message whose Section 1
+    is shorter than its fields cannot index past its own end.
+
+    The significance byte is required to say "start of forecast".  A message whose reference
+    time means a verifying time or an observation time is refused rather than compared against a
+    cycle it does not denote -- an approximate comparison here is worse than none, because it
+    would report a run binding that is not one.
+
+    Scope, stated so it is not read wider: this binds a slice to a *run*, not to an object.  A
+    different object belonging to the same run still passes.
+
+    A refusal names the check that failed and quotes neither the payload nor the expected run.
+    """
+
+    expected = require_reference_time_spelling(expected)
+    if not isinstance(payload, bytes):
+        raise AcquisitionSecurityError("GRIB_PAYLOAD", "a slice must be checked as exact bytes")
+    if len(payload) < GRIB_SECTION_ONE_OFFSET + GRIB_SECTION_ONE_MINIMUM_BYTES + len(GRIB_TRAILER):
+        raise AcquisitionSecurityError(
+            "GRIB_SECTION_ONE",
+            "the slice is too short to carry an identification section",
+        )
+    section_length = int.from_bytes(payload[GRIB_SECTION_ONE_LENGTH_SLICE], "big")
+    if section_length < GRIB_SECTION_ONE_MINIMUM_BYTES:
+        raise AcquisitionSecurityError(
+            "GRIB_SECTION_ONE",
+            f"an identification section is at least {GRIB_SECTION_ONE_MINIMUM_BYTES} bytes",
+        )
+    if GRIB_SECTION_ONE_OFFSET + section_length > len(payload) - len(GRIB_TRAILER):
+        raise AcquisitionSecurityError(
+            "GRIB_SECTION_ONE",
+            "the identification section declares more bytes than the message holds",
+        )
+    if payload[GRIB_SECTION_ONE_NUMBER_OFFSET] != GRIB_SECTION_ONE_NUMBER:
+        raise AcquisitionSecurityError(
+            "GRIB_SECTION_ONE",
+            f"the section after the header is not section {GRIB_SECTION_ONE_NUMBER}",
+        )
+    if payload[GRIB_REFERENCE_SIGNIFICANCE_OFFSET] != GRIB_REFERENCE_START_OF_FORECAST:
+        raise AcquisitionSecurityError(
+            "GRIB_REFERENCE_SIGNIFICANCE",
+            "the message's reference time does not denote the start of a forecast run",
+        )
+    # Rendered into the one spelling and compared as text rather than built into an instant: a
+    # hostile month of 13 or day of 99 must fail this comparison, not raise out of it.
+    delivered = "{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}Z".format(
+        int.from_bytes(payload[GRIB_REFERENCE_YEAR_SLICE], "big"),
+        payload[GRIB_REFERENCE_MONTH_OFFSET],
+        payload[GRIB_REFERENCE_DAY_OFFSET],
+        payload[GRIB_REFERENCE_HOUR_OFFSET],
+        payload[GRIB_REFERENCE_MINUTE_OFFSET],
+        payload[GRIB_REFERENCE_SECOND_OFFSET],
+    )
+    if delivered != expected:
+        raise AcquisitionSecurityError(
+            "GRIB_REFERENCE_TIME",
+            "the delivered message belongs to a model run other than the one ordered",
+        )