mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/hosted_ingestion_job.py

@@ -0,0 +1,356 @@
+"""Capability-free Cloud Run Job bootstrap for one resumable ingestion delivery.
+
+This is deliberately a small front door, not another worker protocol.  The Job receives exactly
+an opaque dispatch id and the Studio endpoint/audience derived from its image configuration.  Its
+ambient workload identity redeems that id once.  All durable progress is represented by the
+generation- and digest-pinned manifest reference in :mod:`hosted_ingestion_contract`; a process
+that reaches its checkpoint reserve exits before credential expiry rather than trying to stretch a
+delivery lease.
+
+There is no generic source fetcher in this module.  The only admitted first-slice continuation is
+the immutable-public deterministic offset contract, and a reviewed connector injects its bounded
+processor through :func:`run_hosted_ingestion_delivery`.  This keeps an unwired Cloud Run Job from
+silently treating an arbitrary URL or cursor as executable work.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from http import HTTPStatus
+from typing import Protocol
+from uuid import UUID
+
+from mostlyright.data_harness.canonical import CanonicalJSONError, canonical_json_bytes, parse_json
+from mostlyright.data_harness.hosted_bootstrap import (
+    BootstrapTransport,
+    UrlLibBootstrapTransport,
+    _https_url,
+    _metadata_identity,
+)
+from mostlyright.data_harness.hosted_ingestion_contract import (
+    HOSTED_INGESTION_DISPATCH_SCHEMA,
+    MAX_DELIVERY_DEADLINE_SECONDS,
+    HostedIngestionDispatchV1,
+    parse_delivery_deadline,
+)
+from mostlyright.data_harness.ingestion.contracts import IngestionError
+
+INGESTION_DISPATCH_SCHEMA_VERSION = "3.0.0"
+INGESTION_DISPATCH_EXCHANGE_PATH_TEMPLATE = (
+    "/internal/v3/ingestion/dispatches/{dispatch_id}:exchange"
+)
+MAX_INGESTION_DISPATCH_RESPONSE_BYTES = 64 * 1024
+CHECKPOINT_RESERVE_SECONDS = 30.0
+MAX_JOB_DEADLINE_SECONDS = 15 * 60
+CONSUMED_INGESTION_DISPATCH_CODES = frozenset(
+    {"INGESTION_DISPATCH_ALREADY_CLAIMED", "INGESTION_DISPATCH_CLOSED"}
+)
+_CODE = re.compile(r"[A-Z][A-Z0-9_]{2,127}\Z")
+
+
+class HostedIngestionJobError(RuntimeError):
+    """A coded refusal which is safe to emit from the Job entrypoint."""
+
+    def __init__(self, code: str, detail: str) -> None:
+        super().__init__(f"{code}: {detail}")
+        self.code = code
+        self.detail = detail
+
+
+@dataclass(frozen=True)
+class IngestionDispatch:
+    """Exactly the three opaque/non-secret task arguments permitted by deployment."""
+
+    dispatch_id: str
+    exchange_url: str
+    audience: str
+
+
+@dataclass(frozen=True)
+class IngestionJobConfig:
+    """The pinned Studio origin; storage authority stays out of this bootstrap config."""
+
+    studio_origin: str
+
+    @classmethod
+    def from_environment(cls, environment: dict[str, str] | None = None) -> IngestionJobConfig:
+        active = os.environ if environment is None else environment
+        origin = active.get("STUDIO_API_ORIGIN")
+        if not isinstance(origin, str) or not origin:
+            raise HostedIngestionJobError(
+                "INGESTION_DISPATCH_CONFIG_INVALID", "STUDIO_API_ORIGIN is required"
+            )
+        try:
+            normalized = _https_url(origin, "STUDIO_API_ORIGIN")
+        except (RuntimeError, ValueError) as error:
+            raise HostedIngestionJobError(
+                "INGESTION_DISPATCH_CONFIG_INVALID", "STUDIO_API_ORIGIN is not HTTPS"
+            ) from error
+        return cls(studio_origin=normalized.rstrip("/"))
+
+
+def _uuid4(value: object) -> str:
+    if not isinstance(value, str):
+        raise HostedIngestionJobError("INGESTION_DISPATCH_INVALID", "dispatch id must be a UUID4")
+    try:
+        parsed = UUID(value)
+    except ValueError as error:
+        raise HostedIngestionJobError(
+            "INGESTION_DISPATCH_INVALID", "dispatch id must be a UUID4"
+        ) from error
+    if parsed.version != 4 or str(parsed) != value:
+        raise HostedIngestionJobError("INGESTION_DISPATCH_INVALID", "dispatch id must be a UUID4")
+    return value
+
+
+def _https(value: object, label: str) -> str:
+    try:
+        return _https_url(value, label)
+    except (RuntimeError, TypeError, ValueError) as error:
+        raise HostedIngestionJobError(
+            "INGESTION_DISPATCH_INVALID", f"{label} is not HTTPS"
+        ) from error
+
+
+def parse_ingestion_dispatch(argv: Sequence[str]) -> IngestionDispatch:
+    """Parse the only six argv cells deployment may put on an ingestion Job task."""
+
+    values = tuple(argv)
+    flags = (
+        "--ingestion-dispatch-id",
+        "--ingestion-dispatch-exchange-url",
+        "--ingestion-dispatch-audience",
+    )
+    if len(values) != 6 or values[::2] != flags or any(not value for value in values[1::2]):
+        raise HostedIngestionJobError(
+            "INGESTION_DISPATCH_ARGUMENTS_INVALID",
+            "expected --ingestion-dispatch-id, --ingestion-dispatch-exchange-url and "
+            "--ingestion-dispatch-audience in that order",
+        )
+    dispatch_id, exchange_url, audience = values[1::2]
+    return IngestionDispatch(
+        dispatch_id=_uuid4(dispatch_id),
+        exchange_url=_https(exchange_url, "ingestion dispatch exchange URL"),
+        audience=_https(audience, "ingestion dispatch audience"),
+    )
+
+
+def _validate_dispatch(config: IngestionJobConfig, dispatch: IngestionDispatch) -> None:
+    expected = config.studio_origin + INGESTION_DISPATCH_EXCHANGE_PATH_TEMPLATE.format(
+        dispatch_id=dispatch.dispatch_id
+    )
+    if dispatch.exchange_url != expected:
+        raise HostedIngestionJobError(
+            "INGESTION_DISPATCH_URL_UNRECOGNIZED",
+            "ingestion dispatch exchange URL is not this deployment's endpoint for this dispatch",
+        )
+    if dispatch.audience != config.studio_origin:
+        raise HostedIngestionJobError(
+            "INGESTION_DISPATCH_AUDIENCE_UNRECOGNIZED",
+            "ingestion dispatch audience is not this deployment's Studio audience",
+        )
+
+
+def _exchange_error(status: int, raw: bytearray) -> HostedIngestionJobError:
+    try:
+        document = parse_json(bytes(raw))
+    except (CanonicalJSONError, TypeError, ValueError):
+        document = None
+    if isinstance(document, dict) and isinstance(document.get("code"), str):
+        code = document["code"]
+        if _CODE.fullmatch(code) is not None:
+            return HostedIngestionJobError(
+                code, f"ingestion dispatch exchange returned HTTP {status}"
+            )
+    return HostedIngestionJobError(
+        "INGESTION_DISPATCH_EXCHANGE_FAILED",
+        f"ingestion dispatch exchange returned HTTP {status}",
+    )
+
+
+def exchange_ingestion_dispatch(
+    *,
+    config: IngestionJobConfig,
+    dispatch: IngestionDispatch,
+    transport: BootstrapTransport | None = None,
+) -> HostedIngestionDispatchV1:
+    """Redeem one opaque id through workload identity and parse the closed response document."""
+
+    _validate_dispatch(config, dispatch)
+    active = transport or UrlLibBootstrapTransport()
+    identity = _metadata_identity(active, dispatch.audience)
+    request = canonical_json_bytes(
+        {"schema_version": INGESTION_DISPATCH_SCHEMA_VERSION, "dispatch_id": dispatch.dispatch_id}
+    )
+    try:
+        try:
+            token = identity.decode("ascii")
+        except UnicodeDecodeError as error:  # pragma: no cover - metadata helper validates this
+            raise HostedIngestionJobError(
+                "INGESTION_DISPATCH_IDENTITY_INVALID", "workload identity was not ASCII"
+            ) from error
+        status, response = active.request(
+            "POST",
+            dispatch.exchange_url,
+            {"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+            request,
+            MAX_INGESTION_DISPATCH_RESPONSE_BYTES,
+        )
+    finally:
+        identity[:] = b"\x00" * len(identity)
+    raw = bytearray(response)
+    try:
+        if status != HTTPStatus.OK:
+            raise _exchange_error(status, raw)
+        try:
+            document = parse_json(bytes(raw))
+            result = HostedIngestionDispatchV1.from_dict(document)
+        except (ValueError, IngestionError, TypeError) as error:
+            raise HostedIngestionJobError(
+                "INGESTION_DISPATCH_RESPONSE_INVALID",
+                "ingestion dispatch exchange response is not the exact accepted schema",
+            ) from error
+        if result.schema_version != HOSTED_INGESTION_DISPATCH_SCHEMA:
+            raise AssertionError("HostedIngestionDispatchV1 accepted an unsupported schema")
+        return result
+    finally:
+        raw[:] = b"\x00" * len(raw)
+
+
+@dataclass(frozen=True)
+class DeliveryDeadline:
+    """A monotonic checkpoint boundary derived from Studio's absolute UTC deadline."""
+
+    deadline: datetime
+    wall_clock: Callable[[], datetime] = lambda: datetime.now(UTC)
+    reserve_seconds: float = CHECKPOINT_RESERVE_SECONDS
+
+    def __post_init__(self) -> None:
+        if self.deadline.tzinfo is None:
+            raise ValueError("deadline must be timezone aware")
+        if not 0 < self.reserve_seconds < MAX_DELIVERY_DEADLINE_SECONDS:
+            raise ValueError("checkpoint reserve is invalid")
+
+    @classmethod
+    def from_dispatch(
+        cls,
+        dispatch: HostedIngestionDispatchV1,
+        *,
+        wall_clock: Callable[[], datetime] = lambda: datetime.now(UTC),
+        reserve_seconds: float = CHECKPOINT_RESERVE_SECONDS,
+    ) -> DeliveryDeadline:
+        deadline = parse_delivery_deadline(dispatch.delivery.deadline)
+        remaining = (deadline - wall_clock().astimezone(UTC)).total_seconds()
+        if remaining <= reserve_seconds:
+            raise HostedIngestionJobError(
+                "INGESTION_DELIVERY_EXPIRED", "delivery has no safe checkpoint window remaining"
+            )
+        if remaining > MAX_JOB_DEADLINE_SECONDS:
+            raise HostedIngestionJobError(
+                "INGESTION_DELIVERY_DEADLINE_INVALID", "delivery deadline exceeds the Job ceiling"
+            )
+        return cls(deadline=deadline, wall_clock=wall_clock, reserve_seconds=reserve_seconds)
+
+    @property
+    def checkpoint_at(self) -> datetime:
+        from datetime import timedelta
+
+        return self.deadline - timedelta(seconds=self.reserve_seconds)
+
+    def checkpoint_due(self) -> bool:
+        return self.wall_clock().astimezone(UTC) >= self.checkpoint_at
+
+    def require_work_time(self) -> None:
+        if self.checkpoint_due():
+            raise CheckpointRequired()
+
+
+class CheckpointRequired(RuntimeError):
+    """Normal delivery exit: durable state must be handed back before capability expiry."""
+
+
+class DeliveryProcessor(Protocol):
+    """A reviewed connector's bounded execution hook, intentionally not a generic URL runner."""
+
+    def __call__(
+        self, dispatch: HostedIngestionDispatchV1, deadline: DeliveryDeadline
+    ) -> DeliveryOutcome: ...
+
+
+@dataclass(frozen=True)
+class DeliveryOutcome:
+    """A connector tells Studio's future callback layer whether it finalized or checkpointed."""
+
+    status: str
+
+    def __post_init__(self) -> None:
+        if self.status not in {"checkpointed", "finalized"}:
+            raise ValueError("delivery status must be checkpointed or finalized")
+
+
+def run_hosted_ingestion_delivery(
+    *, dispatch: HostedIngestionDispatchV1, processor: DeliveryProcessor, deadline: DeliveryDeadline
+) -> DeliveryOutcome:
+    """Run one reviewed processor but force a normal checkpoint exit before its hard deadline."""
+
+    try:
+        deadline.require_work_time()
+        outcome = processor(dispatch, deadline)
+    except CheckpointRequired:
+        return DeliveryOutcome("checkpointed")
+    if not isinstance(outcome, DeliveryOutcome):
+        raise HostedIngestionJobError(
+            "INGESTION_DELIVERY_PROCESSOR_INVALID", "connector returned no bounded delivery outcome"
+        )
+    if deadline.checkpoint_due() and outcome.status != "checkpointed":
+        raise HostedIngestionJobError(
+            "INGESTION_DELIVERY_DEADLINE_OVERRUN",
+            "connector returned after the checkpoint boundary without checkpointing",
+        )
+    return outcome
+
+
+def ingestion_job_main(argv: Sequence[str] | None = None) -> int:
+    """Cloud Run entrypoint.  It refuses safely until a reviewed connector is wired in."""
+
+    arguments = tuple(sys.argv[1:] if argv is None else argv)
+    try:
+        dispatch = parse_ingestion_dispatch(arguments)
+    except HostedIngestionJobError as error:
+        sys.stderr.write(f"{error.code}: {error.detail}\n")
+        return 2
+    try:
+        config = IngestionJobConfig.from_environment()
+        delivery = exchange_ingestion_dispatch(config=config, dispatch=dispatch)
+        DeliveryDeadline.from_dispatch(delivery)
+    except HostedIngestionJobError as error:
+        if error.code in CONSUMED_INGESTION_DISPATCH_CODES:
+            return 0
+        sys.stderr.write(f"{error.code}: {error.detail}\n")
+        return 1
+    # The deliberately closed dispatch contract has no source URL, generic callback endpoint or
+    # credential.  A later reviewed connector/callback slice must call run_hosted_ingestion_delivery
+    # instead of making this entrypoint silently invent one of those authorities.
+    sys.stderr.write("INGESTION_DELIVERY_PROCESSOR_UNWIRED: no reviewed connector is installed\n")
+    return 1
+
+
+__all__ = [
+    "CHECKPOINT_RESERVE_SECONDS",
+    "CONSUMED_INGESTION_DISPATCH_CODES",
+    "CheckpointRequired",
+    "DeliveryDeadline",
+    "DeliveryOutcome",
+    "HostedIngestionJobError",
+    "IngestionDispatch",
+    "IngestionJobConfig",
+    "exchange_ingestion_dispatch",
+    "ingestion_job_main",
+    "parse_ingestion_dispatch",
+    "run_hosted_ingestion_delivery",
+]

mostlyright/data_harness/hosted_ingestion_job_smoke.py

@@ -0,0 +1,40 @@
+"""Offline smoke for the identity-only resumable-ingestion Cloud Run Job image entrypoint."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+from mostlyright.data_harness.hosted_ingestion_job import parse_ingestion_dispatch
+
+_DISPATCH_ID = "99999999-9999-4999-8999-999999999999"
+_JOB_ARGS = (
+    "--ingestion-dispatch-id",
+    _DISPATCH_ID,
+    "--ingestion-dispatch-exchange-url",
+    f"https://studio.invalid/internal/v3/ingestion/dispatches/{_DISPATCH_ID}:exchange",
+    "--ingestion-dispatch-audience",
+    "https://studio.invalid",
+)
+
+
+def main() -> int:
+    dispatch = parse_ingestion_dispatch(_JOB_ARGS)
+    if dispatch.dispatch_id != _DISPATCH_ID:
+        raise SystemExit("ingestion Job parser did not preserve the dispatch id")
+    entrypoint = Path(sys.executable).with_name("mr-data-ingestion-job")
+    result = subprocess.run(
+        [entrypoint, "--not-an-ingestion-job-argument"],
+        check=False,
+        capture_output=True,
+        timeout=10,
+    )
+    if result.returncode != 2 or b"INGESTION_DISPATCH_ARGUMENTS_INVALID" not in result.stderr:
+        raise SystemExit("ingestion Job entrypoint does not enforce its exact argv contract")
+    print("hosted-ingestion-job-smoke-ok")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - executed in the built image
+    raise SystemExit(main())

mostlyright/data_harness/hosted_session_container_smoke.py

@@ -0,0 +1,194 @@
+"""Linux image smoke for the production research-session parser boundaries.
+
+This module is run inside the real session-worker image with Docker's memory ceiling set to the
+exact Cloud Run value. Unit tests can exercise every refusal, but only this lane proves the image's
+non-root entrypoint sees the real cgroup spelling and can install seccomp before the installed
+parser execs.
+"""
+
+from __future__ import annotations
+
+import os
+import signal
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+from mostlyright.data_harness.acquisition.parsing import ParseLimits
+from mostlyright.data_harness.acquisition.sandbox import (
+    HOSTED_SESSION_CONTAINER_MEMORY_BYTES,
+    _run_worker_command,
+)
+from mostlyright.data_harness.hosted_session_worker import SessionWorkerConfig, _default_sandbox
+
+_ADVERSARIAL_CHILD = r"""
+import ctypes
+import errno
+import os
+import socket
+
+status = open("/proc/self/status", encoding="ascii").read()
+if "NoNewPrivs:\t1" not in status or "Seccomp:\t2" not in status:
+    raise SystemExit(20)
+
+def memory_limit():
+    for candidate in (
+        "/sys/fs/cgroup/memory.max",
+        "/sys/fs/cgroup/memory/memory.limit_in_bytes",
+    ):
+        try:
+            return open(candidate, encoding="ascii").read().strip()
+        except OSError:
+            continue
+    raise SystemExit(21)
+
+if memory_limit() != "4294967296":
+    raise SystemExit(22)
+try:
+    open(f"/proc/{os.getppid()}/environ", "rb").read()
+except PermissionError:
+    pass
+else:
+    raise SystemExit(23)
+membership = open("/proc/self/cgroup", encoding="ascii").read()
+child = os.fork()
+if child == 0:
+    if open("/proc/self/cgroup", encoding="ascii").read() != membership:
+        os._exit(30)
+    if memory_limit() != "4294967296":
+        os._exit(31)
+    try:
+        socket.socket()
+    except OSError as error:
+        os._exit(0 if error.errno == errno.EPERM else 32)
+    os._exit(33)
+_, child_status = os.waitpid(child, 0)
+if os.waitstatus_to_exitcode(child_status) != 0:
+    raise SystemExit(34)
+
+libc = ctypes.CDLL(None, use_errno=True)
+for number, arguments in (
+    (101, (0, 0, 0, 0)),
+    (109, (0, 0)),
+    (112, ()),
+    (272, (0,)),
+    (308, (-1, 0)),
+    (310, (os.getppid(), 0, 0, 0, 0, 0)),
+    (311, (os.getppid(), 0, 0, 0, 0, 0)),
+    (434, (os.getppid(), 0)),
+    (438, (-1, 0, 0)),
+):
+    ctypes.set_errno(0)
+    outcome = libc.syscall(number, *arguments)
+    if outcome != -1 or ctypes.get_errno() != errno.EPERM:
+        raise SystemExit(35)
+ctypes.set_errno(0)
+if libc.syscall(435, 0, 0) != -1 or ctypes.get_errno() != errno.ENOSYS:
+    raise SystemExit(36)
+print("hosted-session-container-boundary-ok")
+"""
+
+_LINGERING_DESCENDANT_CHILD = r"""
+import errno
+import os
+import signal
+
+descendant = os.fork()
+if descendant == 0:
+    for detach in (os.setsid, lambda: os.setpgid(0, 0)):
+        try:
+            detach()
+        except OSError as error:
+            if error.errno != errno.EPERM:
+                os._exit(40)
+        else:
+            os._exit(41)
+    for descriptor in (0, 1, 2):
+        try:
+            os.close(descriptor)
+        except OSError:
+            pass
+    while True:
+        signal.pause()
+print(descendant, flush=True)
+os._exit(0)
+"""
+
+
+def _require_descendant_reaped(pid: int) -> None:
+    deadline = time.monotonic() + 5.0
+    while time.monotonic() < deadline:
+        try:
+            waited, status = os.waitpid(pid, os.WNOHANG)
+        except ChildProcessError:
+            waited = 0
+            status = 0
+        if waited == pid:
+            if not os.WIFSIGNALED(status) or os.WTERMSIG(status) != signal.SIGKILL:
+                raise SystemExit("hosted session descendant did not die with group cleanup")
+            return
+        try:
+            os.kill(pid, 0)
+        except ProcessLookupError:
+            return
+        time.sleep(0.01)
+    raise SystemExit("hosted session descendant survived its request")
+
+
+def main() -> int:
+    if sys.platform != "linux":
+        raise SystemExit("hosted session container smoke requires Linux")
+    config = SessionWorkerConfig(
+        studio_origin="https://studio.invalid",
+        lease_audience="https://studio.invalid",
+        warm_pool_size=0,
+    )
+    sandbox = _default_sandbox(config)
+    result = sandbox.parse(
+        request_id="session.container.smoke",
+        content=b"city,temp\noslo,1\n",
+        data_format="csv",
+        media_type="text/csv",
+        filename="source.csv",
+        limits=ParseLimits(),
+    )
+    if result.parsed.columns != ("city", "temp") or result.parsed.rows != (("oslo", "1"),):
+        raise SystemExit("installed session parser returned the wrong table")
+
+    with tempfile.TemporaryDirectory(prefix="mr-session-container-smoke-") as temporary:
+        stdout, stderr, returncode = _run_worker_command(
+            [sys.executable, "-I", "-c", _ADVERSARIAL_CHILD],
+            cwd=Path(temporary).resolve(),
+            env={"PATH": os.defpath, "PYTHONIOENCODING": "utf-8"},
+            request_bytes=b"",
+            timeout_seconds=10.0,
+            memory_cgroup_root_fd=None,
+            hosted_session_container_memory_bytes=HOSTED_SESSION_CONTAINER_MEMORY_BYTES,
+        )
+    if returncode != 0 or stderr or stdout.strip() != b"hosted-session-container-boundary-ok":
+        raise SystemExit("installed session child escaped or lacked its production boundaries")
+
+    with tempfile.TemporaryDirectory(prefix="mr-session-descendant-smoke-") as temporary:
+        stdout, stderr, returncode = _run_worker_command(
+            [sys.executable, "-I", "-c", _LINGERING_DESCENDANT_CHILD],
+            cwd=Path(temporary).resolve(),
+            env={"PATH": os.defpath, "PYTHONIOENCODING": "utf-8"},
+            request_bytes=b"",
+            timeout_seconds=10.0,
+            memory_cgroup_root_fd=None,
+            hosted_session_container_memory_bytes=HOSTED_SESSION_CONTAINER_MEMORY_BYTES,
+        )
+    try:
+        descendant_pid = int(stdout.strip())
+    except ValueError as error:
+        raise SystemExit("hosted session descendant did not report its pid") from error
+    if returncode != 0 or stderr:
+        raise SystemExit("hosted session descendant could detach before cleanup")
+    _require_descendant_reaped(descendant_pid)
+    print("hosted-session-container-smoke-ok")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())