mostlyright-data 0.15.1__tar.gz → 0.15.3__tar.gz

{mostlyright_data-0.15.1 → mostlyright_data-0.15.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.15.1
+Version: 0.15.3
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.15.1 → mostlyright_data-0.15.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.15.1"
+version = "0.15.3"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"
@@ -84,6 +84,9 @@ mr-data-generic-api = "mostlyright.data_harness.thin.workers:generic_api_probe_m
 mr-data-session-worker = "mostlyright.data_harness.thin.workers:session_worker_main"
 mr-data-session-worker-job = "mostlyright.data_harness.thin.workers:session_worker_job_main"
 mr-data-ingestion-job = "mostlyright.data_harness.thin.workers:ingestion_job_main"
+# The shared, persistent executor for bounded incremental refreshes.  Unlike the
+# segment-delivery Job above, it claims a run directly and has no task-specific argv.
+mr-data-ingestion-worker = "mostlyright.data_harness.thin.workers:ingestion_worker_main"
 # The hosted stream venue's recorder. It runs from the same worker runtime the producer and
 # the ingestion Job run from -- the install stage puts every console script in the image and
 # that stage deletes none of them -- so this entry is the whole of what a new job shape needs.

{mostlyright_data-0.15.1 → mostlyright_data-0.15.3}/skills/mr-data-build/SKILL.md

@@ -516,8 +516,7 @@ the ones above. Spell out the names a reader would search for -- the place, the
 every common form, the programme and the publisher -- once each, in the opening sentences and
 never as a keyword list. The opening sentences double as the summary cards and search engines
 show, so they must stand alone. A one-line description is a defect to fix in the revision, not a
-style choice. Give `table.description` the grain and the window, and give every column a
-description a reader can act on.
+style choice. Give `table.description` the grain and the window.
 
 **Every dataset gets a cover before its first run is presented.** Once the description is
 written, generate one with
@@ -545,7 +544,17 @@ years are already in their names.
 Leaving it out is a defect in the same way a one-line dataset description is: the source is then
 drawn with its endpoint alone, and a reader is left to guess from a query string.
 
-### Say what each column shows
+### Say what each column is, and what it shows
+
+**Every column carries a `description`, and every column that measures a physical quantity carries
+a `units` entry.** The description is one sentence saying what the column means, written for
+somebody who has never seen the source — `The highest air temperature recorded that day.` — and it
+is what the dataset page, the schema and everybody reading the table downstream go on. The
+units entry names the code that column's numbers are in, and a temperature, a distance, a duration
+or a rate that arrives without one is a number nobody can convert or compare. Neither member is
+work to leave for later: a recipe that leaves a column undescribed or a measurement unitless is not
+one to start a run on, and a run already built from one is not finished, however well its checks
+reported.
 
 Decide, per column, what a reader should see, and write it on the column as `presentation`. This is
 your judgement about the data and nobody else can make it: the Builder can see the values but not
@@ -808,6 +817,14 @@ required check that fails ends the run with `failure_code: CHECK_FAILED`.
 `mr-data diff RUN_A RUN_B` compares two runs — columns added, removed and retyped, row counts, the
 coverage window and the data fingerprint. `--columns-only` narrows what is compared to the schema.
 
+**Then read the columns against the document, because an undescribed column is a revision rather
+than a caveat.** `peek` names the columns the run sealed; the recipe is where their descriptions
+and units are, so this is a reading of the document rather than a judgement about the data: every
+column has a `description`, and every column that measures a physical quantity has a `units`
+entry. A run whose recipe leaves either out is not finished — the table arrives with nothing
+beside it to say what its numbers mean or what they are measured in — so it is a revision, which
+is stage 7, and it is registered again before anything is presented.
+
 ### 7. Fix — revise the document and register it again
 
 Change the document and send it again. Same dataset, same table, new digest:

{mostlyright_data-0.15.1 → mostlyright_data-0.15.3}/src/mostlyright/data_harness/acquisition/http.py

@@ -1088,6 +1088,34 @@ class TransportResponse:
         return values[0] if values else None
 
 
+@dataclass(frozen=True)
+class StreamedTransportResponse:
+    """A transport response whose body was copied to a coordinator-owned sink.
+
+    The body deliberately has no representation here.  Callers get the facts needed for the
+    same pinned-peer receipt as ``TransportResponse`` without an opportunity to re-materialise
+    a large source in coordinator memory.
+    """
+
+    status: int
+    headers: tuple[tuple[str, str], ...]
+    body_size_bytes: int
+    body_sha256: str
+    connected_peer: str
+    tls_verified: bool
+    hostname_verified: bool
+
+    def header(self, name: str) -> str | None:
+        return TransportResponse(
+            self.status,
+            self.headers,
+            b"",
+            self.connected_peer,
+            self.tls_verified,
+            self.hostname_verified,
+        ).header(name)
+
+
 class PinnedTransport(Protocol):
     def request(
         self,
@@ -1109,6 +1137,17 @@ class PinnedTransport(Protocol):
         can influence character by character.
         """
 
+    def request_to_sink(
+        self,
+        target: ValidatedTarget,
+        *,
+        approved_ip: str,
+        headers: Mapping[str, str],
+        limits: RetrievalLimits,
+        sink: Callable[[bytes], None],
+    ) -> StreamedTransportResponse:
+        """Stream one response into ``sink`` while retaining only bounded receipt facts."""
+
 
 class TransportFailureError(AcquisitionSecurityError):
     """Retryable transport failure with exact response-body bytes already consumed."""
@@ -1391,6 +1430,33 @@ class RetrievedBytes:
             )
 
 
+@dataclass(frozen=True)
+class RetrievedSink:
+    """Receipt facts for a retrieval copied directly into a caller-owned file descriptor."""
+
+    source_url: str
+    final_url: str
+    media_type: str
+    content_sha256: str
+    content_size_bytes: int
+    hops: tuple[RetrievalHop, ...]
+    transport_evidence_digest: str
+
+    @property
+    def total_response_body_size_bytes(self) -> int:
+        """Every response byte consumed, including failed peer attempts."""
+
+        return sum(
+            hop.response_body_size_bytes
+            + sum(
+                attempt.response_body_size_bytes
+                for attempt in hop.peer_attempts[:-1]
+                if attempt.outcome == "transport_failure"
+            )
+            for hop in self.hops
+        )
+
+
 def probe_transport_for(retrieved: RetrievedBytes) -> ProbeTransport:
     """Seal what the terminal response of one retrieval stated.
 
@@ -1653,6 +1719,209 @@ class PinnedHttpsRetriever:
             validators=validators,
         )
 
+    def retrieve_to_sink(
+        self,
+        url: str,
+        *,
+        sink: Callable[[bytes], None],
+        reset_sink: Callable[[], None] | None = None,
+    ) -> RetrievedSink:
+        """Fetch a public whole CSV while streaming its terminal body to ``sink``.
+
+        Redirect bodies are discarded after their digest is measured.  A peer retry starts a
+        fresh request only after the failed attempt records its consumed bytes; the caller must
+        therefore use a fresh staging file per acquisition and discard it on any exception.
+        """
+        if not callable(sink):
+            raise AcquisitionSecurityError("STREAM_SINK", "stream sink must be callable")
+        started = float(self._clock())
+        current_url = url
+        hops: list[RetrievalHop] = []
+        aggregate = 0
+        requests = 0
+        for redirect_count in range(self._limits.max_redirects + 1):
+            self._check_total_timeout(started)
+            target = validate_public_https_url(
+                current_url, resolver=self._resolver, egress_policy=self._egress_policy
+            )
+            headers = {
+                "Accept": _accept_header(self._limits),
+                "Accept-Encoding": "identity",
+                "Connection": "close",
+                "Host": urlsplit(target.normalized_url).netloc,
+                "User-Agent": self._limits.user_agent,
+            }
+            response: StreamedTransportResponse | None = None
+            attempts: list[PeerAttempt] = []
+            for ip in target.approved_ips:
+                if requests >= self._limits.max_requests:
+                    raise PeerAttemptError(
+                        "REQUEST_LIMIT",
+                        "retrieval exceeded its request budget",
+                        tuple(attempts),
+                        hops=tuple(hops),
+                    )
+                requests += 1
+                remaining = self._limits.max_aggregate_response_bytes - aggregate
+                try:
+                    limits = self._remaining_attempt_limits(
+                        started + self._limits.total_timeout_seconds,
+                        remaining_aggregate_bytes=remaining,
+                    )
+                    # Only the terminal body reaches the supplied fd. Redirect bytes are still
+                    # fully consumed and evidenced, but are never retained.
+                    with self._limiter.acquire(
+                        target.hostname, deadline=started + self._limits.total_timeout_seconds
+                    ):
+                        response = self._transport.request_to_sink(
+                            target, approved_ip=ip, headers=headers, limits=limits, sink=sink
+                        )
+                except AcquisitionSecurityError as exc:
+                    consumed = (
+                        exc.response_body_size_bytes
+                        if isinstance(exc, TransportFailureError)
+                        else 0
+                    )
+                    aggregate += consumed
+                    attempts.append(
+                        PeerAttempt(
+                            approved_ip=ip,
+                            outcome="transport_failure",
+                            failure_code=exc.code,
+                            response_body_size_bytes=consumed,
+                        )
+                    )
+                    if (
+                        exc.code == "TRANSPORT_FAILURE"
+                        and ip != target.approved_ips[-1]
+                        and aggregate < self._limits.max_aggregate_response_bytes
+                    ):
+                        # A failed peer may already have copied an unauthenticated prefix to
+                        # the caller-owned staging file.  A second peer answers a fresh whole
+                        # representation, so it can proceed only after that prefix is removed.
+                        if reset_sink is None:
+                            raise PeerAttemptError(
+                                "STREAM_SINK",
+                                "retrying a streamed retrieval requires a resettable sink",
+                                tuple(attempts),
+                                hops=tuple(hops),
+                            ) from None
+                        reset_sink()
+                        continue
+                    raise PeerAttemptError(
+                        exc.code, exc.detail, tuple(attempts), hops=tuple(hops)
+                    ) from None
+                attempts.append(
+                    PeerAttempt(
+                        approved_ip=ip,
+                        outcome="response",
+                        failure_code=None,
+                        response_body_size_bytes=response.body_size_bytes,
+                    )
+                )
+                break
+            if response is None:
+                raise PeerAttemptError(
+                    "TRANSPORT_FAILURE",
+                    "all resolver-approved peers failed",
+                    tuple(attempts),
+                    hops=tuple(hops),
+                )
+            aggregate += response.body_size_bytes
+            if aggregate > self._limits.max_aggregate_response_bytes:
+                raise PeerAttemptError(
+                    "AGGREGATE_RESPONSE_LIMIT",
+                    "response bodies exceeded aggregate budget",
+                    tuple(attempts),
+                    hops=tuple(hops),
+                )
+            if not response.tls_verified or not response.hostname_verified:
+                raise PeerAttemptError(
+                    "TLS_VERIFICATION",
+                    "transport did not prove TLS verification",
+                    tuple(attempts),
+                    hops=tuple(hops),
+                )
+            peer = require_approved_peer(target, response.connected_peer)
+            normalized = _validate_response_headers(response.headers)
+            content_length = response.header("content-length")
+            if content_length is not None and (
+                not _DECIMAL_ONLY.fullmatch(content_length)
+                or int(content_length) != response.body_size_bytes
+            ):
+                raise PeerAttemptError(
+                    "CONTENT_LENGTH",
+                    "response bytes do not match declared content length",
+                    tuple(attempts),
+                    hops=tuple(hops),
+                )
+            encoding = (response.header("content-encoding") or "").lower().strip()
+            if encoding not in _ALLOWED_RESPONSE_ENCODINGS:
+                raise PeerAttemptError(
+                    "CONTENT_ENCODING",
+                    "compressed network responses are disabled",
+                    tuple(attempts),
+                    hops=tuple(hops),
+                )
+            hop = RetrievalHop(
+                url=target.normalized_url,
+                resolution_digest=target.resolution_digest,
+                connected_peer=peer,
+                status=response.status,
+                response_headers_digest=canonical_sha256(normalized),
+                response_body_size_bytes=response.body_size_bytes,
+                response_body_sha256=response.body_sha256,
+                etag=response.header("etag"),
+                last_modified=response.header("last-modified"),
+                peer_attempts=tuple(attempts),
+                approved_ips=target.approved_ips,
+            )
+            hops.append(hop)
+            if response.status in REDIRECT_STATUSES:
+                location = response.header(_REDIRECT_HEADER)
+                if not location or redirect_count >= self._limits.max_redirects:
+                    raise PeerAttemptError(
+                        "REDIRECT_LIMIT",
+                        "response redirect is outside the admitted limit",
+                        tuple(attempts),
+                        hops=tuple(hops),
+                    )
+                current_url = urljoin(target.normalized_url, location)
+                # The redirect entity is evidence, never source content.  A file-backed caller
+                # truncates its coordinator staging descriptor here before the next hop.
+                if reset_sink is None:
+                    raise AcquisitionSecurityError(
+                        "STREAM_SINK", "redirecting streamed retrieval requires a resettable sink"
+                    )
+                reset_sink()
+                continue
+            if response.status == NOT_FOUND_STATUS or not 200 <= response.status <= 299:
+                raise PeerAttemptError(
+                    "HTTP_STATUS",
+                    f"source returned HTTP status {response.status}",
+                    tuple(attempts),
+                    hops=tuple(hops),
+                    http_status=response.status,
+                )
+            media_type = _media_type(response.header("content-type"))
+            if media_type not in self._limits.allowed_media_types:
+                raise PeerAttemptError(
+                    "MEDIA_TYPE",
+                    "response media type is not allowlisted",
+                    tuple(attempts),
+                    hops=tuple(hops),
+                )
+            return RetrievedSink(
+                source_url=url,
+                final_url=target.normalized_url,
+                media_type=media_type,
+                content_sha256=response.body_sha256,
+                content_size_bytes=response.body_size_bytes,
+                hops=tuple(hops),
+                transport_evidence_digest=canonical_sha256([hop.to_dict() for hop in hops]),
+            )
+        raise AcquisitionSecurityError("REDIRECT_LIMIT", "redirect loop did not terminate")
+
     def retrieve_datagov_v4(
         self,
         url: str,
@@ -2613,6 +2882,97 @@ class StdlibPinnedTransport:
         finally:
             connection.close()
 
+    def request_to_sink(
+        self,
+        target: ValidatedTarget,
+        *,
+        approved_ip: str,
+        headers: Mapping[str, str],
+        limits: RetrievalLimits,
+        sink: Callable[[bytes], None],
+    ) -> StreamedTransportResponse:
+        """The production no-buffer variant of :meth:`request`.
+
+        The size check happens before every write.  A rejected response can therefore leave a
+        partial staging file, but never an admitted result; its exact consumed byte count is
+        carried by ``TransportFailureError`` for retry accounting.
+        """
+        deadline = time.monotonic() + limits.total_timeout_seconds
+        connection = _PinnedHTTPSConnection(
+            hostname=target.hostname,
+            approved_ip=approved_ip,
+            port=target.port,
+            timeout=min(limits.connect_timeout_seconds, limits.total_timeout_seconds),
+            context=self._context,
+        )
+        total = 0
+        digest = hashlib.sha256()
+        try:
+            connection.request("GET", target.request_target, headers=dict(headers))
+            if connection.sock is not None:
+                connection.sock.settimeout(
+                    _remaining_socket_timeout(deadline, configured=limits.read_timeout_seconds)
+                )
+            response = connection.getresponse()
+            raw_headers = tuple(
+                (key.lower(), value.strip()) for key, value in response.getheaders()
+            )
+            _validate_response_headers(raw_headers)
+            content_length = next(
+                (value for key, value in raw_headers if key == "content-length"), None
+            )
+            if content_length is not None:
+                try:
+                    if int(content_length) > limits.max_response_bytes:
+                        raise AcquisitionSecurityError(
+                            "RESPONSE_LIMIT",
+                            "declared response length exceeds the configured limit",
+                        )
+                except ValueError:
+                    raise AcquisitionSecurityError(
+                        "CONTENT_LENGTH", "response content length is invalid"
+                    ) from None
+            while True:
+                if connection.sock is not None:
+                    connection.sock.settimeout(
+                        _remaining_socket_timeout(deadline, configured=limits.read_timeout_seconds)
+                    )
+                chunk = response.read(min(65_536, limits.max_response_bytes + 1 - total))
+                if not chunk:
+                    break
+                total += len(chunk)
+                if total > limits.max_response_bytes:
+                    raise AcquisitionSecurityError(
+                        "RESPONSE_LIMIT", "response exceeded the configured byte limit"
+                    )
+                digest.update(chunk)
+                sink(chunk)
+            peer = connection.connected_peer
+            if peer is None:
+                raise AcquisitionSecurityError(
+                    "PEER_ADDRESS", "transport did not record its connected peer"
+                )
+            return StreamedTransportResponse(
+                response.status, raw_headers, total, digest.hexdigest(), peer, True, True
+            )
+        except http.client.IncompleteRead as exc:
+            partial = exc.partial if isinstance(exc.partial, bytes) else bytes(exc.partial)
+            raise TransportFailureError(
+                f"HTTPS peer disconnected during a response: {type(exc).__name__}",
+                response_body_size_bytes=total + len(partial),
+            ) from None
+        except http.client.RemoteDisconnected as exc:
+            raise TransportFailureError(
+                f"HTTPS peer disconnected before a response: {type(exc).__name__}",
+                response_body_size_bytes=total,
+            ) from None
+        except OSError as exc:
+            raise TransportFailureError(
+                f"HTTPS transport failed: {type(exc).__name__}", response_body_size_bytes=total
+            ) from None
+        finally:
+            connection.close()
+
 
 def _remaining_socket_timeout(deadline: float, *, configured: float) -> float:
     remaining = deadline - time.monotonic()