mostlyright-data 0.15.2__tar.gz → 0.15.3__tar.gz

{mostlyright_data-0.15.2 → mostlyright_data-0.15.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.15.2
+Version: 0.15.3
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.15.2 → mostlyright_data-0.15.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.15.2"
+version = "0.15.3"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"
@@ -84,6 +84,9 @@ mr-data-generic-api = "mostlyright.data_harness.thin.workers:generic_api_probe_m
 mr-data-session-worker = "mostlyright.data_harness.thin.workers:session_worker_main"
 mr-data-session-worker-job = "mostlyright.data_harness.thin.workers:session_worker_job_main"
 mr-data-ingestion-job = "mostlyright.data_harness.thin.workers:ingestion_job_main"
+# The shared, persistent executor for bounded incremental refreshes.  Unlike the
+# segment-delivery Job above, it claims a run directly and has no task-specific argv.
+mr-data-ingestion-worker = "mostlyright.data_harness.thin.workers:ingestion_worker_main"
 # The hosted stream venue's recorder. It runs from the same worker runtime the producer and
 # the ingestion Job run from -- the install stage puts every console script in the image and
 # that stage deletes none of them -- so this entry is the whole of what a new job shape needs.

mostlyright_data-0.15.3/src/mostlyright/data_harness/hosted_ingestion_worker.py

@@ -0,0 +1,238 @@
+"""Persistent executor for Studio-admitted bounded direct refreshes.
+
+This worker deliberately owns no session, tenant, queue selector or long-lived Studio
+capability.  Each poll presents only its workload identity to the closed ingestion-pop
+route; Studio selects an already-admitted run and returns a fresh run-scoped producer
+capability.  ``run_v4`` still creates a fresh scratch root for every run, while every
+Reader remains a fresh networkless, address-space-bounded child.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import logging
+import os
+import signal
+import time
+from collections.abc import Sequence
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from threading import Event, Thread
+from typing import Any
+
+from mostlyright.data_harness.hosted_bootstrap import (
+    UrlLibBootstrapTransport,
+    _https_url,
+    _metadata_identity,
+)
+from mostlyright.data_harness.hosted_run_worker import (
+    StudioRunCallbacks,
+    _assert_bounded_direct_runtime_image,
+    _enforce_bounded_direct_runtime_resources,
+    _install_termination_handlers,
+    _restore_termination_handlers,
+    _StageCursor,
+    run_v4,
+)
+from mostlyright.data_harness.hosted_session_worker import (
+    _claim_poll_interval,
+    _claimed_or_empty,
+    _V4Internal,
+)
+from mostlyright.data_harness.hosted_worker import (
+    HostedWorkerError,
+    _new_transfer_client,
+    parse_capability,
+)
+from mostlyright.data_harness.run_execution_v4 import (
+    RunExecutionError,
+    failed_body_for_unparsed,
+    parse_run_execution_v4,
+)
+
+INGESTION_RUN_CLAIM_PATH = "/internal/v4/ingestion-worker/runs:claim"
+DEFAULT_POLL_SECONDS = 1.0
+MAX_POLL_SECONDS = 60.0
+_LOG = logging.getLogger(__name__)
+
+
+class _HealthHandler(BaseHTTPRequestHandler):
+    def do_GET(self) -> None:
+        if self.path not in {"/livez", "/healthz"}:
+            self.send_error(404)
+            return
+        self.send_response(200)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
+    def log_message(self, _format: str, *_args: object) -> None:
+        return
+
+
+class _HealthServer(HTTPServer):
+    """A serial probe listener whose slow client cannot retain the worker at shutdown."""
+
+    request_queue_size = 8
+
+    def get_request(self) -> tuple[Any, Any]:
+        request, address = super().get_request()
+        request.settimeout(2.0)
+        return request, address
+
+
+def _health_server() -> _HealthServer:
+    """Serve only Cloud Run liveness; it never reports queue or tenant state."""
+
+    port = int(os.environ.get("PORT", "8080"))
+    server = _HealthServer(("0.0.0.0", port), _HealthHandler)
+    Thread(target=server.serve_forever, daemon=True).start()
+    return server
+
+
+def _config_from_environment() -> tuple[str, str]:
+    origin = os.environ.get("STUDIO_API_ORIGIN", "")
+    audience = os.environ.get("STUDIO_INGESTION_WORKER_AUDIENCE", origin)
+    try:
+        return _https_url(origin, "STUDIO_API_ORIGIN").rstrip("/"), _https_url(
+            audience, "STUDIO_INGESTION_WORKER_AUDIENCE"
+        )
+    except HostedWorkerError as error:
+        raise HostedWorkerError("INGESTION_WORKER_CONFIG_INVALID", error.detail) from error
+
+
+def run_ingestion_claim_loop(
+    *,
+    surface: Any,
+    transport: Any,
+    origin: str,
+    stopped: Any,
+    sleep: Any = time.sleep,
+    execute: Any = run_v4,
+) -> None:
+    """Execute one claimed refresh at a time until the host requests shutdown.
+
+    Keeping the loop injectable makes its operational guarantees testable: identity/claim
+    failures back off, a claim can execute at most once at a time, and only the short-lived
+    claim capability reaches callbacks or ``run_v4``.  ``run_v4`` owns the per-run scratch
+    directory and removes it on both terminal paths.
+    """
+
+    interval = DEFAULT_POLL_SECONDS
+    while not stopped():
+        try:
+            answer = surface.post(
+                INGESTION_RUN_CLAIM_PATH,
+                {},
+                route=INGESTION_RUN_CLAIM_PATH,
+            )
+            interval = min(MAX_POLL_SECONDS, _claim_poll_interval(answer, interval))
+            claimed = _claimed_or_empty(answer, route=INGESTION_RUN_CLAIM_PATH, kind="run")
+            if claimed is None:
+                sleep(interval)
+                continue
+            raw_run = claimed.get("run")
+            token = claimed.get("capability_token")
+            if not isinstance(raw_run, dict) or not isinstance(token, str) or not token:
+                raise HostedWorkerError(
+                    "V4_CLAIM_ANSWER_INVALID",
+                    "ingestion claim omitted its run-scoped capability or run document",
+                )
+            # Do not let an arbitrary claim string reach Authorization.  Build callbacks from
+            # the raw document before parsing so a malformed claim can promptly terminalize its
+            # own lease, following the ordinary one-shot worker's failure path.
+            token = parse_capability(token.encode("ascii"))
+            raw_run_id = raw_run.get("run_id")
+            raw_attempt = raw_run.get("attempt")
+            callbacks = StudioRunCallbacks(
+                base_url=origin,
+                token=token,
+                transport=transport,
+                run_id=raw_run_id if isinstance(raw_run_id, str) else "",
+                attempt=raw_attempt if type(raw_attempt) is int and raw_attempt >= 1 else 1,
+            )
+            try:
+                job = parse_run_execution_v4(raw_run)
+                if job.source_history_protocol != "source-history.v2" or job.mode != "refresh":
+                    raise HostedWorkerError(
+                        "V4_CLAIM_ANSWER_INVALID",
+                        "ingestion claim named a run outside the bounded direct refresh contract",
+                    )
+                # These guards run before run_v4 installs its own failure handler.  Run them
+                # here as well so a rollout witness mismatch terminalizes promptly instead of
+                # waiting for lease recovery.
+                _assert_bounded_direct_runtime_image(job)
+                _enforce_bounded_direct_runtime_resources(job)
+            except Exception:
+                body = failed_body_for_unparsed(
+                    raw_run,
+                    failure_code="V4_CLAIM_ANSWER_INVALID",
+                    detail="the ingestion worker rejected its claimed run document",
+                    failed_stage="acquire",
+                )
+                if body is not None:
+                    with contextlib.suppress(Exception):
+                        callbacks.failed(body)
+                raise
+            # A platform SIGTERM during a run must enter run_v4's existing typed
+            # terminal-report path.  The idle handler in ``ingestion_worker_main``
+            # is temporarily replaced only for this fresh execution.
+            cursor = _StageCursor()
+            installed = _install_termination_handlers(cursor)
+            try:
+                execute(job, token, callbacks=callbacks, base_url=origin, cursor=cursor)
+            finally:
+                _restore_termination_handlers(installed)
+            interval = DEFAULT_POLL_SECONDS
+        except RunExecutionError as error:
+            if error.code == "WORKER_TERMINATED":
+                raise
+            _LOG.warning("ingestion run failed after terminal reporting: %s", error.code)
+            sleep(interval)
+            interval = min(MAX_POLL_SECONDS, max(DEFAULT_POLL_SECONDS, interval * 2))
+        except Exception as error:
+            # Studio's attempt lease/watchdog owns recovery after a worker disappears;
+            # backoff avoids a hot loop while a transient identity or control-plane error clears.
+            _LOG.warning("ingestion claim loop failed: %s", type(error).__name__)
+            sleep(interval)
+            interval = min(MAX_POLL_SECONDS, max(DEFAULT_POLL_SECONDS, interval * 2))
+
+
+def ingestion_worker_main(argv: Sequence[str] | None = None) -> int:
+    """Poll the closed queue and run one fresh deterministic execution per claim."""
+
+    parser = argparse.ArgumentParser(prog="mr-data-ingestion-worker")
+    parser.parse_args(argv)
+    origin, audience = _config_from_environment()
+    health = _health_server()
+    stop = Event()
+
+    def request_stop(_signum: int, _frame: Any) -> None:
+        stop.set()
+
+    installed: list[tuple[int, Any]] = []
+    for signum in (signal.SIGTERM, signal.SIGINT):
+        installed.append((signum, signal.signal(signum, request_stop)))
+    transport = _new_transfer_client()
+    surface = _V4Internal(
+        base_url=origin,
+        transport=transport,
+        identity=lambda: _metadata_identity(UrlLibBootstrapTransport(), audience),
+    )
+    try:
+        run_ingestion_claim_loop(
+            surface=surface,
+            transport=transport,
+            origin=origin,
+            stopped=stop.is_set,
+            # Event.wait is interrupted by request_stop, unlike time.sleep which
+            # resumes after a handled signal on modern Python.
+            sleep=stop.wait,
+        )
+    finally:
+        for signum, previous in installed:
+            signal.signal(signum, previous)
+        with contextlib.suppress(Exception):
+            transport.close()
+        health.shutdown()
+        health.server_close()
+    return 0

{mostlyright_data-0.15.2 → mostlyright_data-0.15.3}/src/mostlyright/data_harness/run_execution_v4.py

@@ -598,7 +598,7 @@ class RunSourceHistoryArtifact:
 
 @dataclass(frozen=True)
 class RunSourceHistory:
-    history_id: UUID
+    history_id: str
     run_id: UUID
     table_version_id: str
     candidate_digest: str
@@ -1278,6 +1278,10 @@ _PREDECESSOR_FIELD_SETS = frozenset(
         ("source_snapshot",),
         ("manifest_download_session", "source_snapshot"),
         ("manifest_download_session", "compact_inputs"),
+        # A source-history v1/v2 predecessor may be a table version with neither a
+        # manifest object nor mirrored parts: its retained index is the only
+        # predecessor payload the incremental reader restores.
+        ("source_history",),
         ("manifest_download_session", "source_history"),
         ("manifest_download_session", "compact_inputs", "source_history"),
         # ``parts`` IS THE MANIFEST'S STAND-IN AND NEVER ITS COMPANION. A version composed at a
@@ -1369,7 +1373,7 @@ def _parse_source_history(value: Any, predecessor: Mapping[str, Any]) -> RunSour
     except SourceStateIndexError as error:
         raise _refuse(code, "source history binding is invalid") from error
     return RunSourceHistory(
-        _typed(_uuid, value["history_id"], "source_history.history_id", code),
+        _typed(_identifier, value["history_id"], "source_history.history_id", code),
         _typed(_uuid, value["run_id"], "source_history.run_id", code),
         _typed(_identifier, value["table_version_id"], "source_history.table_version_id", code),
         _typed(_digest, value["candidate_digest"], "source_history.candidate_digest", code),
@@ -1960,6 +1964,20 @@ def parse_run_execution_v4(value: Any) -> RunExecution:
         raise _refuse(
             "RUN_PREDECESSOR_INVALID", "source history was not enabled by the job protocol"
         )
+    pure_source_history = has_block and set(value["predecessor"]) == {
+        "run_id",
+        "table_version_id",
+        "candidate_digest",
+        "artifact_download_session",
+        "source_history",
+    }
+    if pure_source_history and (
+        protocol not in {"source-history.v1", "source-history.v2"} or mode != "refresh"
+    ):
+        raise _refuse(
+            "RUN_PREDECESSOR_INVALID",
+            "a source-history-only predecessor is valid only for a refresh",
+        )
     if protocol == "source-history.v2" and "bounded_direct" not in value:
         raise _refuse(
             "BOUNDED_DIRECT_ROUTE_REFUSED", "source-history.v2 requires a bounded direct admission"

{mostlyright_data-0.15.2 → mostlyright_data-0.15.3}/src/mostlyright/data_harness/studio_boundary.py

@@ -85,7 +85,7 @@ PINNED_OPENAPI_SOURCE_SHA256 = "f2164517534e299f96449659332fc5bc0552fa7f94539c3d
 #: One line, and it stays one line: ``_replace_exactly`` in the repin matches the whole assignment
 #: as a literal and refuses on any count but one, so a wrapped constant is a constant the repin can
 #: no longer rewrite.
-PINNED_V4_OPENAPI_SOURCE_SHA256 = "6998ce6d23c893fb425c3589ecaa2daae54c82c1a70de2aafad54d61a0e6d6b0"
+PINNED_V4_OPENAPI_SOURCE_SHA256 = "5d812f2a3009e8206c5404e0c46295f4ce983272f59b45ac666c7efbc0f04f1b"
 
 _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
 _IMAGE_DIGEST = re.compile(r"^[a-z0-9][a-z0-9._/-]{0,254}@sha256:[0-9a-f]{64}$")

{mostlyright_data-0.15.2 → mostlyright_data-0.15.3}/src/mostlyright/data_harness/thin/vocabulary.py

@@ -362,6 +362,11 @@ WORKER_ENTRY_POINTS: dict[str, tuple[str, str, str]] = {
         "ingestion_job_main",
         "runs one fenced resumable-ingestion delivery inside a backend Cloud Run Job task",
     ),
+    "mr-data-ingestion-worker": (
+        "mostlyright.data_harness.hosted_ingestion_worker",
+        "ingestion_worker_main",
+        "claims and executes bounded direct refreshes from the persistent backend worker pool",
+    ),
     "mr-data-stream-recorder-job": (
         "mostlyright.data_harness.hosted_stream_recorder_job",
         "stream_recorder_job_main",

{mostlyright_data-0.15.2 → mostlyright_data-0.15.3}/src/mostlyright/data_harness/thin/workers.py

@@ -196,6 +196,10 @@ def ingestion_job_main(argv: Sequence[str] | None = None) -> int:
     return _dispatch("mr-data-ingestion-job", argv)
 
 
+def ingestion_worker_main(argv: Sequence[str] | None = None) -> int:
+    return _dispatch("mr-data-ingestion-worker", argv)
+
+
 def stream_recorder_job_main(argv: Sequence[str] | None = None) -> int:
     return _dispatch("mr-data-stream-recorder-job", argv)
 
@@ -219,6 +223,7 @@ SHIMS = {
     "mr-data-session-worker": session_worker_main,
     "mr-data-session-worker-job": session_worker_job_main,
     "mr-data-ingestion-job": ingestion_job_main,
+    "mr-data-ingestion-worker": ingestion_worker_main,
     "mr-data-stream-recorder-job": stream_recorder_job_main,
     "mr-data-stream-subscription-worker": stream_subscription_worker_main,
 }
@@ -234,6 +239,7 @@ __all__ = [
     "crawler_main",
     "generic_api_probe_main",
     "ingestion_job_main",
+    "ingestion_worker_main",
     "producer_main",
     "run_worker_main",
     "session_worker_job_main",

{mostlyright_data-0.15.2 → mostlyright_data-0.15.3}/src/mostlyright/data_harness/ux/remediation.py

@@ -5,7 +5,7 @@ for the commonest failure of all -- pointing a command at a folder that is not t
 engineering wording reads like a security incident. This module turns a code into the two or three
 plain sentences that say what happened and what to do next.
 
-There are two layers, because 1839 typed codes across 216 prefixes can reach a person at a
+There are two layers, because 1840 typed codes across 216 prefixes can reach a person at a
 workbench, and a hand-written sentence for each one would be neither affordable nor honest:
 
 * **Families.** Every code prefix is assigned to a family. A family is one paragraph: what this