mostlyright-data 0.18.0__tar.gz → 0.18.2__tar.gz

{mostlyright_data-0.18.0 → mostlyright_data-0.18.2}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.18.0
+Version: 0.18.2
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/
-Requires-Python: <3.14,>=3.13
+Requires-Python: >=3.13
 Provides-Extra: build
 Requires-Dist: hatchling==1.28.0; extra == 'build'
 Requires-Dist: uv-build==0.11.3; extra == 'build'
@@ -42,7 +42,7 @@ Requires-Dist: protobuf==7.35.1; extra == 'dev'
 Requires-Dist: pyarrow==23.0.1; extra == 'dev'
 Requires-Dist: pyclipper==1.4.0; extra == 'dev'
 Requires-Dist: pycparser==3.0; extra == 'dev'
-Requires-Dist: pyhdf==0.11.7; ((sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64')) and extra == 'dev'
+Requires-Dist: pyhdf==0.11.7; ((sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64' and python_full_version < '3.14')) and extra == 'dev'
 Requires-Dist: pypdfium2==5.13.0; extra == 'dev'
 Requires-Dist: pypublicdecompwt==2.8.1.6; extra == 'dev'
 Requires-Dist: pyreadstat==1.3.6; extra == 'dev'
@@ -155,7 +155,7 @@ Requires-Dist: eccodeslib==2.48.0.26; (sys_platform != 'win32') and extra == 'sc
 Requires-Dist: h5py==3.16.0; extra == 'scientific'
 Requires-Dist: netcdf4==1.7.4; extra == 'scientific'
 Requires-Dist: pillow==12.3.0; extra == 'scientific'
-Requires-Dist: pyhdf==0.11.7; ((sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64')) and extra == 'scientific'
+Requires-Dist: pyhdf==0.11.7; ((sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64' and python_full_version < '3.14')) and extra == 'scientific'
 Requires-Dist: pypublicdecompwt==2.8.1.6; extra == 'scientific'
 Requires-Dist: rasterio==1.5.1; extra == 'scientific'
 Requires-Dist: zarr==3.3.0; extra == 'scientific'
@@ -173,7 +173,7 @@ they finish.
 
 ## Requirements
 
-Use CPython 3.13.
+Use CPython 3.13 or newer. The retired local execution lane (`[local,scientific]` extras) still needs 3.13.
 
 ## Install
 

{mostlyright_data-0.18.0 → mostlyright_data-0.18.2}/README.md

@@ -10,7 +10,7 @@ they finish.
 
 ## Requirements
 
-Use CPython 3.13.
+Use CPython 3.13 or newer. The retired local execution lane (`[local,scientific]` extras) still needs 3.13.
 
 ## Install
 

{mostlyright_data-0.18.0 → mostlyright_data-0.18.2}/pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "mostlyright-data"
-version = "0.18.0"
+version = "0.18.2"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
-requires-python = ">=3.13,<3.14"
+requires-python = ">=3.13"
 # The base distribution IS the thin client (ADR 0021): pure Python, no compiled extension anywhere
 # in the install closure, and -- as it turns out -- no third-party package at all. Every runtime
 # dependency this project had was reached only from the local execution lane, so all of them moved
@@ -50,7 +50,7 @@ mostlyright-grib = [
 antlr4-python3-runtime = { path = "vendor/wheels/antlr4_python3_runtime-4.9.3-py3-none-any.whl" }
 mostlyright-studio-client-v3 = { path = "vendor/wheels/mostlyright_studio_client_v3-3.0.0-py3-none-any.whl" }
 pyhdf = [
-    { path = "vendor/wheels/pyhdf-0.11.7-cp313-cp313-macosx_15_0_arm64.whl", marker = "sys_platform == 'darwin' and platform_machine == 'arm64'" },
+    { path = "vendor/wheels/pyhdf-0.11.7-cp313-cp313-macosx_15_0_arm64.whl", marker = "sys_platform == 'darwin' and platform_machine == 'arm64' and python_full_version < '3.14'" },
 ]
 
 [project.scripts]
@@ -198,7 +198,7 @@ scientific = [
     "netCDF4==1.7.4",
     # Linux resolves the exact upstream wheel. macOS arm64 uses the reviewed source build,
     # because upstream's CP313 wheel is built with an unsupported macOS 26 deployment floor.
-    "pyhdf==0.11.7; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64')",
+    "pyhdf==0.11.7; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'darwin' and platform_machine == 'arm64' and python_full_version < '3.14')",
     "Pillow==12.3.0",
     "pypublicdecompwt==2.8.1.6",
     "rasterio==1.5.1",

{mostlyright_data-0.18.0 → mostlyright_data-0.18.2}/src/mostlyright/data_harness/sources/stream_connector.py

@@ -57,7 +57,6 @@ from mostlyright.data_harness.canonical import (
     CanonicalJSONError,
     canonical_json_bytes,
     canonical_sha256,
-    parse_json,
 )
 from mostlyright.data_harness.sources.authenticated_https import _canonical_origin
 from mostlyright.data_harness.sources.authenticated_https_v2 import (
@@ -69,7 +68,7 @@ from mostlyright.data_harness.sources.authenticated_https_v2 import (
     _parse_request,
 )
 from mostlyright.data_harness.sources.contracts import SourceContractError
-from mostlyright.data_harness.sources.stream_numbers import parse_json_numbers_as_text
+from mostlyright.data_harness.sources.stream_numbers import parse_frame_json
 
 STREAM_CONNECTOR_VERSION = "harness-stream-connector.v1"
 STREAM_CONNECTOR_VERSION_V2 = "harness-stream-connector.v2"
@@ -860,11 +859,21 @@ class StreamConnectorConfig:
         The single reader every caller that reads a frame of this venue picks up, so a capture, a
         credential acknowledgement and a probe cannot disagree about whether ``0.5`` is a message
         or a decode error.
+
+        BOTH READINGS GO THROUGH THE FRAME READER, which differs from the document reader in one
+        bound and one only: how many JSON members one frame may carry. A frame is bounded by bytes
+        where it was assembled, and a venue asked about a large roster answers the whole roster in
+        one array; the canonical member ceiling was written for what this build seals, and applying
+        it here refused an answer this build had asked for. Whether that array IS a batch is this
+        document's word (``frame_shape``), and the reader is told it, because a batch is sealed
+        element by element and a single message is sealed whole: each is held at the read to what
+        its seal will hold it to. See
+        :func:`~mostlyright.data_harness.sources.stream_numbers.parse_frame_json`.
         """
 
-        if self.numbers == NUMBERS_TEXT:
-            return parse_json_numbers_as_text(text)
-        return parse_json(text)
+        return parse_frame_json(
+            text, numbers_as_text=self.numbers == NUMBERS_TEXT, batches=self.admits_array_frames
+        )
 
     @property
     def ignores_untyped_messages(self) -> bool:

mostlyright_data-0.18.2/src/mostlyright/data_harness/sources/stream_numbers.py

@@ -0,0 +1,338 @@
+"""Read a venue frame, in either number reading, under the bounds a frame actually has.
+
+The text reading is a third reading beside :func:`parse_json` and :func:`parse_foreign_json`, for
+a document whose numbers have to survive being sealed.  A venue that writes a price as ``0.5``
+writes a fractional number, and :func:`parse_json` refuses one because nothing fractional may
+reach a digest;
+:func:`parse_foreign_json` reads it as a float, which may be inspected but never written, so a frame
+read that way could not become a recorded line.  Neither answer is available to a reader whose
+whole job is to keep the frame.
+
+So the number is kept as the literal its sender wrote.  ``parse_int`` and ``parse_float`` are
+handed the exact text JSON's own grammar matched -- ``0.5``, ``-3``, ``1e9`` -- and that text is the
+value.  No precision is chosen, because none is computed: the string that comes back is the string
+that went in, so ``0.10`` and ``0.1`` stay the two different things the venue said them as, and a
+seventy-seven-digit integer keeps all seventy-seven digits.  The result holds a string wherever a
+number stood and is therefore an ordinary value of the canonical data model:
+:func:`~mostlyright.data_harness.canonical.canonical_json_bytes` seals it and :func:`parse_json`
+reads it back unchanged.
+
+Everything else is exactly as strict as the canonical reader.  ``NaN`` and ``Infinity`` are still
+refused, and so are duplicate object keys, byte-order marks, non-strict UTF-8, unpaired surrogates,
+and the depth and string-length bounds.  The reader lives beside the canonical module rather than
+inside it because that module's bytes are a cross-system pin.
+
+THE MEMBER BOUND IS THE ONE THING A FRAME DOES NOT SHARE WITH A DOCUMENT, and
+:func:`parse_frame_json` is the reader that says so, for BOTH number readings.  A frame is already
+bounded, by bytes, at the transport that assembled it; the canonical member ceiling was written for
+what this build seals and digests, where a hundred thousand members is far past anything authored
+here.  A venue asked about a roster of sixteen hundred members answers all sixteen hundred in one
+array, and a count written for a different kind of document refused the answer it had been asked
+for.  See :data:`MAX_FRAME_MEMBERS` for where the frame's ceiling comes from instead, and
+:func:`parse_frame_json` for why nothing it admits is ever sealed as one value.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from collections.abc import Callable
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    MAX_CANONICAL_DEPTH,
+    MAX_CANONICAL_INTEGER,
+    MAX_CANONICAL_MEMBERS,
+    MAX_CANONICAL_STRING_LENGTH,
+    CanonicalJSONError,
+    _parse_integer,
+    _reject_constant,
+    _reject_duplicate_pairs,
+    _reject_fraction,
+    _validate_unicode,
+)
+
+__all__ = [
+    "MAX_FRAME_MEMBERS",
+    "MAX_MESSAGE_MEMBERS",
+    "RECORDED_LINE_WRAPPER_MEMBERS",
+    "parse_frame_json",
+]
+
+#: The largest number of JSON members one frame off a socket may be read as.
+#:
+#: A FRAME IS NOT A DOCUMENT THIS BUILD WRITES, and the canonical member ceiling was written for
+#: one that is. :data:`~mostlyright.data_harness.canonical.MAX_CANONICAL_MEMBERS` bounds what may
+#: be sealed, digested and read back, where a hundred thousand members is already far past
+#: anything this repository authors. A frame is a venue's own answer to a subscription, and a
+#: venue that is asked about sixteen hundred members may answer all sixteen hundred in one array,
+#: each an object of a few dozen members with nested sides of its own. That frame is bounded
+#: already -- by bytes, at the transport, where an assembled message past
+#: ``MAX_MESSAGE_BYTES`` is refused before a reader ever sees it -- and bounding it a second time
+#: by a count written for a different kind of document refuses an answer the venue was asked for.
+#:
+#: So the ceiling here is DERIVED FROM THE BOUND THAT ALREADY HOLDS rather than chosen: the
+#: smallest JSON member is two bytes (``[]`` is one member in one pair of bytes, and every member
+#: inside a container costs at least a delimiter and a character), so a frame of at most N bytes
+#: cannot hold more than N // 2 members. The count therefore adds no capacity the byte bound did
+#: not already grant -- it stays a real ceiling for a reader walking a value that is not bytes,
+#: while never being the bound that decides a frame. The relationship is asserted against the
+#: transport's own constant in the connector's tests; it is not imported, because the reader of a
+#: frame's bytes has no business reaching the socket that assembled them.
+MAX_FRAME_MEMBERS = (8 * 1024 * 1024) // 2
+
+#: How many JSON members the WIDEST line this build writes adds around the message it carries.
+#:
+#: A venue message is written down more than once on its way to a table. The capture session
+#: and the segment recorder write it as ``{"epoch": …, "message": …, "received_at": …}``, three
+#: members it did not bring; the Recorder then seals it as one line of exactly the members
+#: ``sources.stream.SEALED_FRAME_KEYS`` names -- the line object and seven scalar siblings
+#: beside ``message`` -- which is eight. The ceiling below is derived from the widest of those,
+#: because a message that fits the first line and not the last is admitted at the read and
+#: refused at the seal, which is the failure this module exists to remove. The number is stated
+#: here rather than imported, for the reason the frame ceiling states its own: this reader has
+#: no business reaching the Recorder, and the tests hold the two equal so a key added to the
+#: sealed line moves this or fails the build.
+RECORDED_LINE_WRAPPER_MEMBERS = 8
+
+#: The largest number of JSON members ONE MESSAGE may carry and still be sealed as one line.
+#:
+#: The frame ceiling above is about what a venue may answer with; this is about what this build
+#: may write down, and it is the canonical ceiling less the widest line's wrapper. A message is
+#: sealed by the canonical writer as one recorded line and read back by the strict reader as one,
+#: so a message past this count is one the seal refuses whatever the read admitted. THE READ
+#: THEREFORE HOLDS IT TOO: a frame that is one message is read under this ceiling exactly as it
+#: always was, and a batch frame's elements are each read under it, because each becomes one
+#: line. Only the batch as a whole may exceed it, up to the frame ceiling, because a batch is
+#: never sealed whole. Nothing this reader admits is refused one step later for its size.
+MAX_MESSAGE_MEMBERS = MAX_CANONICAL_MEMBERS - RECORDED_LINE_WRAPPER_MEMBERS
+
+
+def parse_frame_json(
+    value: str | bytes | bytearray | memoryview, *, numbers_as_text: bool, batches: bool
+) -> Any:
+    """Read ONE FRAME off a socket, in either number reading, bounded the way a frame is bounded.
+
+    The two readings are the same reader here on purpose. Whether ``0.5`` is a message or a decode
+    error is the venue document's statement and is answered by ``numbers_as_text``; how large an
+    answer a venue may give is not a number-reading question at all, and a frame ceiling that
+    applied to only one of the two readings would be a build where a venue's answer was admitted
+    or refused according to how its prices are spelled.
+
+    EVERY OTHER BOUND IS THE CANONICAL ONE, UNCHANGED: duplicate object keys, byte-order marks,
+    non-strict UTF-8, ``NaN``/``Infinity``, out-of-range integers, unpaired surrogates, the
+    sixty-four-level nesting bound and the string-length bound all refuse exactly what they refuse
+    for a document. Only the member count differs, and it differs by what will be sealed.
+
+    ``batches`` is the document's word on whether an array frame is a batch of messages, and it
+    decides which ceiling holds over what. A frame that is ONE MESSAGE is read under
+    :data:`MAX_MESSAGE_MEMBERS`, because it becomes one recorded line and a line past the canonical
+    ceiling is one the seal refuses -- which is the count a frame was always read under, less the
+    widest line's wrapper. A frame that is a BATCH is read under :data:`MAX_FRAME_MEMBERS` as a
+    whole and under :data:`MAX_MESSAGE_MEMBERS` per element, because ``_admit_batch``
+    and the hosted feed's ``_frame_messages`` write each element as its own line with its own
+    sequence and the frame's arrival time, and the strict reader that reads a recording back
+    (``stream_recorder``) reads those LINES, one message each. So a frame carrying a venue's whole
+    roster becomes a thousand-odd ordinary documents, every one of them under the canonical
+    member ceiling, and that ceiling goes on holding over everything this build writes and
+    digests. A refusal for size is raised HERE, at the read every lane already guards, and
+    never one step later at a write no lane guards; the transport bounds the bytes, and every
+    place a frame is written down afterwards holds whatever the transport admitted.
+    """
+
+    return _read(
+        value,
+        number=_number_literal if numbers_as_text else None,
+        max_members=MAX_MESSAGE_MEMBERS,
+        batch_ceiling=MAX_FRAME_MEMBERS if batches else None,
+    )
+
+
+def _read(
+    value: str | bytes | bytearray | memoryview,
+    *,
+    number: Callable[[str], str] | None,
+    max_members: int,
+    batch_ceiling: int | None = None,
+) -> Any:
+    """The one reading both entry points are, differing only in numbers and in member ceiling.
+
+    ``max_members`` bounds whatever is read as ONE VALUE: a document, a message, or one element
+    of a batch. With a ``batch_ceiling``, a top-level array is a batch -- walked one element at a
+    time, each under ``max_members`` with a count of its own, and the array as a whole held to
+    the batch ceiling beside them; anything that is not a top-level array is one value as
+    before. The paths and the depth are exactly what one walk over the whole array would have
+    named, so a refusal reads the same whichever way the array was walked.
+    """
+
+    if isinstance(value, str):
+        text = value
+    elif isinstance(value, (bytes, bytearray, memoryview)):
+        raw = bytes(value)
+        if raw.startswith(b"\xef\xbb\xbf"):
+            raise CanonicalJSONError("JSON_BOM", "$", "UTF-8 byte-order marks are not allowed")
+        try:
+            text = raw.decode("utf-8", errors="strict")
+        except UnicodeDecodeError as exc:
+            raise CanonicalJSONError(
+                "JSON_UTF8",
+                "$",
+                f"input is not strict UTF-8 at byte {exc.start}",
+            ) from None
+    else:
+        raise CanonicalJSONError(
+            "JSON_INPUT_TYPE",
+            "$",
+            "JSON input must be text or a bytes-like value",
+        )
+
+    if text.startswith("\ufeff"):
+        raise CanonicalJSONError("JSON_BOM", "$", "Unicode byte-order marks are not allowed")
+    try:
+        parsed = json.loads(
+            text,
+            object_pairs_hook=_reject_duplicate_pairs,
+            parse_int=number or _parse_integer,
+            parse_float=number or _reject_fraction,
+            parse_constant=_reject_constant,
+        )
+    except CanonicalJSONError:
+        raise
+    except json.JSONDecodeError as exc:
+        raise CanonicalJSONError(
+            "JSON_SYNTAX",
+            "$",
+            f"invalid JSON at line {exc.lineno}, column {exc.colno}",
+        ) from None
+    if batch_ceiling is None or not isinstance(parsed, list):
+        return _normalize(parsed, "$", depth=0, members=[0], max_members=max_members)
+    total = 1
+    for index, element in enumerate(parsed):
+        counted = [0]
+        _normalize(element, f"$[{index}]", depth=1, members=counted, max_members=max_members)
+        total += counted[0]
+        if total > batch_ceiling:
+            raise CanonicalJSONError("JSON_MEMBERS", "$", f"value exceeds {batch_ceiling} members")
+    return parsed
+
+
+def _normalize(
+    value: Any,
+    path: str,
+    *,
+    depth: int,
+    members: list[int],
+    max_members: int,
+) -> Any:
+    """:func:`~mostlyright.data_harness.canonical._normalize_json`, with the ceiling as an argument.
+
+    Written here rather than reached for there because the canonical module's BYTES ARE A
+    CROSS-SYSTEM PIN and may not move to grow a parameter. Every bound below is the canonical
+    one, spelled the same way and raising the same code at the same path; ``max_members`` is the
+    only thing this walk is told rather than knows. That the two agree is not asserted by reading
+    them side by side -- ``tests/test_canonical.py`` walks a corpus through both and requires the
+    same value or the same refusal from each.
+
+    IT VALIDATES IN PLACE AND BUILDS NOTHING. The canonical walk returns a fresh tree because it
+    is handed arbitrary Python values -- tuples, ordered mappings, whatever a caller composed --
+    and has to hand back the one shape it seals. This walk is handed only what ``json.loads``
+    just built from a frame, which is already that shape, so a second tree of it would be the
+    same value at twice the cost. The cost matters because the frame is a venue's: eight
+    mebibytes of two-byte containers is millions of objects, and the parse that builds them is
+    the peak a hostile frame can reach in this process -- the same peak the canonical reader has
+    always reached on such a frame before its own count could trip -- and this walk does not
+    raise it. The parse is bounded by the transport's byte bound and by nothing else; nothing
+    here pretends the member count bounds memory, because it never did.
+
+    The fractional-number branch is here for the canonical module's non-finite refusal only.
+    A frame is read either strictly, where a fraction is refused at the scanner before this walk
+    ever sees it, or as text, where it never became a float; no caller of this reader admits one,
+    so the foreign-document allowance has nothing to correspond to, but a float that did arrive
+    would be refused with the same code the canonical walk gives it.
+    """
+
+    if depth > MAX_CANONICAL_DEPTH:
+        raise CanonicalJSONError(
+            "JSON_DEPTH",
+            path,
+            f"nesting exceeds {MAX_CANONICAL_DEPTH} levels",
+        )
+    members[0] += 1
+    if members[0] > max_members:
+        raise CanonicalJSONError(
+            "JSON_MEMBERS",
+            path,
+            f"value exceeds {max_members} members",
+        )
+
+    if value is None or type(value) is bool:
+        return value
+    if type(value) is int:
+        if not -MAX_CANONICAL_INTEGER - 1 <= value <= MAX_CANONICAL_INTEGER:
+            raise CanonicalJSONError(
+                "JSON_INTEGER_RANGE",
+                path,
+                "integer is outside the signed 64-bit range",
+            )
+        return value
+    if type(value) is float:
+        if not math.isfinite(value):
+            raise CanonicalJSONError(
+                "JSON_NONFINITE",
+                path,
+                "non-finite JSON numbers are unsupported",
+            )
+        raise CanonicalJSONError(
+            "JSON_NONINTEGER_NUMBER",
+            path,
+            "non-integer JSON numbers are unsupported",
+        )
+    if isinstance(value, str):
+        if len(value) > MAX_CANONICAL_STRING_LENGTH:
+            raise CanonicalJSONError(
+                "JSON_STRING_LENGTH",
+                path,
+                f"string exceeds {MAX_CANONICAL_STRING_LENGTH} code points",
+            )
+        _validate_unicode(value, path)
+        return value
+    if isinstance(value, dict):
+        if any(not isinstance(key, str) for key in value):
+            raise CanonicalJSONError(
+                "JSON_OBJECT_KEY",
+                path,
+                "object keys must all be strings",
+            )
+        for key, item in value.items():
+            _validate_unicode(key, f"{path}.<key>")
+            child = f"{path}.{key}" if key else f"{path}['']"
+            _normalize(item, child, depth=depth + 1, members=members, max_members=max_members)
+        return value
+    if isinstance(value, list):
+        for index, item in enumerate(value):
+            _normalize(
+                item,
+                f"{path}[{index}]",
+                depth=depth + 1,
+                members=members,
+                max_members=max_members,
+            )
+        return value
+    # A tuple is the one shape the canonical walk copies into a list. A walk that copies nothing
+    # cannot, and ``json.loads`` never produces one, so it is refused rather than half-handled.
+    raise CanonicalJSONError(
+        "JSON_VALUE_TYPE",
+        path,
+        f"unsupported value type {type(value).__name__}",
+    )
+
+
+def _number_literal(value: str) -> str:
+    """Return a JSON number exactly as it was written, as text.
+
+    The hook JSON's own scanner hands the matched literal to, so this is the sender's spelling and
+    not a rendering of something parsed out of it.  Nothing is widened, narrowed or rounded on the
+    way through, because nothing is converted.
+    """
+
+    return value

{mostlyright_data-0.18.0 → mostlyright_data-0.18.2}/src/mostlyright/data_harness/stream_segment_spill.py

@@ -20,7 +20,11 @@ every frame to disk as it goes -- and this is what lets the pending prefix reach
 of waiting in RAM for the rest of the segment.
 
 The file is an ordinary append log, one canonical JSON object per line, raw frame bytes carried in
-base64 because a provider frame is bytes and a line is text.  It is written with the work root's
+base64 because a provider frame is bytes and a line is text -- in a LIST of pieces, because a
+canonical string is bounded at ``MAX_CANONICAL_STRING_LENGTH`` code points and a frame the
+transport admits may be eight mebibytes: every frame the feed accepted has to be one this file
+can hold, whatever its size, or a segment fails at its high-water mark for one frame that had
+nowhere else to go.  It is written with the work root's
 own permissions, it is never delivered anywhere, and it is discarded the moment the segment it
 belongs to is sealed or abandoned.  Nothing about it is evidence: the evidence is what the
 recorder seals from it.
@@ -40,6 +44,7 @@ from collections.abc import Iterator, Sequence
 from pathlib import Path
 
 from mostlyright.data_harness.canonical import (
+    MAX_CANONICAL_STRING_LENGTH,
     CanonicalJSONError,
     canonical_json_bytes,
     parse_json,
@@ -53,6 +58,10 @@ from mostlyright.data_harness.stream_subscription_worker import (
 #: The file one spilled segment lives in, beneath its own directory.
 SPILL_FILE_NAME = "frames.ndjson"
 
+#: How many raw bytes one piece of a spilled frame carries: what one canonical string holds in
+#: base64, which writes four characters for every three bytes. Derived so the two cannot drift.
+SPILL_PIECE_BYTES = (MAX_CANONICAL_STRING_LENGTH // 4) * 3
+
 #: The exact members one spilled record carries.  Equality in both directions, so a record written
 #: by a build that knew about one more field is refused rather than partly read.
 _RECORD_MEMBERS = frozenset(
@@ -149,7 +158,16 @@ class SegmentSpill:
         landed_bytes = 0
         try:
             for frame in frames:
-                _write(descriptor, canonical_json_bytes(_record(frame)) + b"\n")
+                try:
+                    line = canonical_json_bytes(_record(frame))
+                except CanonicalJSONError as error:
+                    # Unreachable for any frame the transport admits, since the pieces are cut to
+                    # what one canonical string holds; kept so a refusal here is this module's
+                    # own word rather than an escaping reader error.
+                    raise StreamSubscriptionError(
+                        "STREAM_SUBSCRIPTION_SEGMENT_INVALID", "spilled frame could not be written"
+                    ) from error
+                _write(descriptor, line + b"\n")
                 landed += 1
                 landed_bytes += len(frame.raw)
             # Once, after the batch rather than after every frame: the durability this needs is
@@ -204,10 +222,14 @@ _SAFE_ID_CHARACTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTU
 
 
 def _record(frame: StreamFrame) -> dict[str, object]:
+    pieces = [
+        base64.b64encode(frame.raw[start : start + SPILL_PIECE_BYTES]).decode("ascii")
+        for start in range(0, len(frame.raw), SPILL_PIECE_BYTES)
+    ]
     return {
         "provider_event_id": frame.provider_event_id,
         "cursor": frame.cursor,
-        "raw": base64.b64encode(frame.raw).decode("ascii"),
+        "raw": pieces,
         "received_at": frame.received_at,
         "connection_epoch": frame.connection_epoch,
         "snapshot": frame.snapshot,
@@ -226,12 +248,12 @@ def _frame(line: bytes) -> StreamFrame:
             "STREAM_SUBSCRIPTION_SEGMENT_INVALID", "spilled frame does not carry its exact members"
         )
     raw = value["raw"]
-    if not isinstance(raw, str):
+    if not isinstance(raw, list) or not all(isinstance(piece, str) for piece in raw):
         raise StreamSubscriptionError(
             "STREAM_SUBSCRIPTION_SEGMENT_INVALID", "spilled frame carries no readable payload"
         )
     try:
-        decoded = base64.b64decode(raw, validate=True)
+        decoded = b"".join(base64.b64decode(piece, validate=True) for piece in raw)
     except (binascii.Error, ValueError) as error:
         raise StreamSubscriptionError(
             "STREAM_SUBSCRIPTION_SEGMENT_INVALID", "spilled frame carries no readable payload"