launchhelm 0.1.0__py3-none-any.whl

launchhelm/runtime.py

@@ -0,0 +1,1348 @@
+"""Cooperative LaunchHelm runtime: telemetry spool plus safe-point commands.
+
+Contract:
+    * ``log`` admits a point into a bounded in-memory queue (volatile).
+    * ``flush(delivered=False)`` waits until the writer has persisted the
+      queue to SQLite (local durability).
+    * ``flush()`` waits until the uploader has a query-visible receipt.
+    * ``close`` persists the local queue but does not promise server delivery.
+    * Memory and spool limits are serialized-payload budgets.
+    * Command tombstones use a separate journal budget; exhaustion fails
+      closed via ``BufferFull``.
+    * Unknown effects are never automatically re-executed.
+
+Threading:
+    Shared mutable state lives behind ``self._lock`` (an ``RLock``).
+    ``self._space`` is a ``Condition`` on that same lock and is used for
+    memory-budget wait/notify. ``StateStore`` has its own ``_mutex``.
+    Lock ordering: never acquire ``Runtime._lock`` while holding
+    ``StateStore._mutex``. ``log`` holds ``Runtime._lock`` only for queue
+    admission and never calls the store. Background threads call the store
+    without the runtime lock.
+
+    Protected by ``Runtime._lock``: ``_session`` (epoch / state revision),
+    ``_fatal``, ``_accepting``, ``_closed``, ``_memory_used``, ``_handlers``,
+    ``_active``, ``_safe_points``, ``_safe_points_dirty``, ``dropped``,
+    ``last_transport_error``.
+    ``queue.Queue`` instances are themselves thread-safe.
+    ``httpx.Client`` request methods are thread-safe; the runtime issues
+    them from the caller, control, and uploader threads.
+
+Failure: ``log`` / ``flush`` / ``safe_point`` raise the first fatal
+background error, preferring ``RuntimeFenced`` if a fence is observed.
+"""
+
+from __future__ import annotations
+
+import copy
+import queue
+import re
+import threading
+import time
+import uuid
+import warnings
+from collections.abc import Callable, Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Literal, NoReturn
+
+from jsonschema import Draft202012Validator, FormatChecker
+from referencing import Registry
+from referencing.exceptions import NoSuchResource
+
+from .client import RuntimeClient
+from .errors import BufferFull, LaunchHelmError, RuntimeFenced
+from .models import Capability, CommandContext, Document, Reference, Scope, timestamp
+from .store import OversizePoint, StateStore, TelemetryBatch
+from .values import (
+    JSONObject,
+    JSONSchema,
+    JSONValue,
+    encode,
+    fingerprint,
+    json_object,
+    string_object,
+)
+
+if TYPE_CHECKING:
+    from .rich import PendingBlob
+
+OverflowPolicy = Literal["drop_new", "block", "raise"]
+CapabilityHandler = Callable[[JSONValue, CommandContext], JSONValue]
+CapabilityValidator = Callable[[JSONValue], None]
+
+MIN_MEMORY_BYTES = 1 << 20
+MAX_VOLATILE_POINT_BYTES = (1 << 19) - 128
+HEARTBEAT_INTERVAL_SECONDS = 15.0
+CONTROL_POLL_SECONDS = 0.25
+CLAIM_WAIT_MS = 19_000
+WRITER_POLL_SECONDS = 0.1
+WRITER_BATCH_MAX = 512
+WRITER_BATCH_BYTES = 1 << 20
+UPLOAD_IDLE_SECONDS = 0.1
+UPLOAD_MAX_BACKOFF_SECONDS = 10.0
+FLUSH_POLL_SECONDS = 0.02
+FENCE_WAIT_SECONDS = 1.0
+DEFAULT_FLUSH_TIMEOUT_SECONDS = 30.0
+DEFAULT_CLOSE_TIMEOUT_SECONDS = 35.0
+CONTROL_MAX_BACKOFF_SECONDS = 10.0
+MAX_SAFE_POINTS = 64
+_SAFE_POINT_NAME = re.compile(r"^[a-z][a-z0-9_.:-]{0,127}$")
+_DECIMAL = re.compile(r"^-?(0|[1-9][0-9]*)(\.[0-9]+)?$")
+
+_COORDINATES: JSONSchema = {
+    "type": "array",
+    "minItems": 1,
+    "maxItems": 32,
+    "items": {
+        "type": "object",
+        "required": ["axis", "value"],
+        "properties": {
+            "axis": {"type": "string", "pattern": "^[a-z][a-z0-9_.:-]{0,127}$"},
+            "value": {
+                "type": "string",
+                "maxLength": 128,
+                "pattern": r"^-?(0|[1-9][0-9]*)(\.[0-9]+)?$",
+            },
+            "unit": {"type": "string", "maxLength": 64},
+        },
+        "additionalProperties": False,
+    },
+}
+
+
+def _no_remote(uri: str) -> NoReturn:
+    # referencing.NoSuchResource is an attrs frozen exception; the runtime
+    # stubs do not expose the ``ref`` keyword even though the library API does.
+    raise NoSuchResource(ref=uri)  # type: ignore[call-arg]
+
+
+def _validate(schema: JSONSchema, value: object) -> None:
+    # Registry stores retrieve= as a private attrs field ``_retrieve``.
+    registry = Registry(retrieve=_no_remote)  # type: ignore[call-arg]
+    Draft202012Validator(schema, registry=registry, format_checker=FormatChecker()).validate(value)
+
+
+def _require_semantic(schema: JSONValue, what: str) -> None:
+    root = schema.get("root") if isinstance(schema, dict) else None
+    if not isinstance(root, dict) or not isinstance(root.get("kind"), str):
+        raise ValueError(f"{what} must be a semantic schema")
+    structure = root.get("structure")
+    if isinstance(structure, dict):
+        Draft202012Validator.check_schema(structure)
+
+
+def _validate_semantic(schema: JSONValue, value: object) -> None:
+    root = schema.get("root") if isinstance(schema, dict) else None
+    if not isinstance(root, dict):
+        raise ValueError("input schema must be a semantic schema")
+    _check_semantic(root, value)
+
+
+def _check_semantic(node: JSONObject, value: object) -> None:
+    kind = node.get("kind")
+    if kind == "opaque":
+        return
+    if kind == "scalar":
+        _check_scalar(node, value)
+        return
+    if kind == "record":
+        if not isinstance(value, dict):
+            raise ValueError("record value must be an object")
+        fields = node.get("fields") or []
+        if not isinstance(fields, list):
+            raise ValueError("record fields must be a list")
+        names: set[str] = set()
+        for raw in fields:
+            if not isinstance(raw, dict):
+                raise ValueError("record field is invalid")
+            name = raw.get("name")
+            if not isinstance(name, str):
+                raise ValueError("record field is invalid")
+            names.add(name)
+            child = raw.get("type")
+            if name not in value:
+                if raw.get("optional") is True:
+                    continue
+                raise ValueError(f"field {name} is required")
+            if not isinstance(child, dict):
+                raise ValueError(f"field {name} has no type")
+            _check_semantic(child, value[name])
+        for key in value:
+            if key not in names:
+                raise ValueError(f"field {key} is not in the type")
+        return
+    if kind == "list":
+        if not isinstance(value, list):
+            raise ValueError("list value must be an array")
+        item = node.get("item")
+        if not isinstance(item, dict):
+            raise ValueError("list requires an item type")
+        for entry in value:
+            _check_semantic(item, entry)
+        return
+    if kind == "enum":
+        allowed = node.get("values")
+        if not isinstance(allowed, list) or any(not isinstance(item, str) for item in allowed):
+            raise ValueError("enum type requires string values")
+        if not isinstance(value, str) or value not in allowed:
+            raise ValueError("value is not in the enum")
+        return
+    if kind == "schedule":
+        points = value.get("points") if isinstance(value, dict) else None
+        if not isinstance(points, list):
+            raise ValueError("schedule value must contain points")
+        for point in points:
+            if not isinstance(point, dict):
+                raise ValueError("schedule point must be an object")
+            at = point.get("at")
+            raw = point.get("value")
+            if not isinstance(at, str) or not _DECIMAL.fullmatch(at):
+                raise ValueError("point axis must be a decimal")
+            if not isinstance(raw, str) or not _DECIMAL.fullmatch(raw):
+                raise ValueError("point value must be a decimal")
+        return
+    raise ValueError(f"unsupported semantic kind {kind!r}")
+
+
+def _check_scalar(node: JSONObject, value: object) -> None:
+    scalar_kind = node.get("scalar")
+    if scalar_kind == "number" and isinstance(value, int | float) and not isinstance(value, bool):
+        _check_bounds(node, value)
+        return
+    if scalar_kind == "integer" and isinstance(value, int) and not isinstance(value, bool):
+        _check_bounds(node, value)
+        return
+    if scalar_kind == "string" and isinstance(value, str):
+        return
+    if scalar_kind == "boolean" and isinstance(value, bool):
+        return
+    raise ValueError("value does not match the scalar type")
+
+
+def _check_bounds(node: JSONObject, value: int | float) -> None:
+    bounds = node.get("bounds")
+    if not isinstance(bounds, dict):
+        return
+    if "min" in bounds and float(value) < float(str(bounds["min"])):
+        raise ValueError("value is below the minimum")
+    if "max" in bounds and float(value) > float(str(bounds["max"])):
+        raise ValueError("value is above the maximum")
+
+
+class Runtime:
+    """Cooperative participant runtime.
+
+    Contract: see the module docstring for ``log`` / ``flush`` / ``close``
+    semantics, budgets, and the command journal. Construct with an explicit
+    overflow policy and positive bounded queue budgets.
+
+    Failure: ``ValueError`` if budgets or overflow are invalid, or if the
+    participant resource is the wrong kind. Background fatals surface on
+    the next ``log`` / ``flush`` / ``safe_point``.
+    """
+
+    def __init__(
+        self,
+        client: RuntimeClient,
+        participant_id: str,
+        *,
+        state_dir: str | Path,
+        overflow: OverflowPolicy,
+        max_spool_bytes: int = 64 << 20,
+        media_spool_bytes: int = 256 << 20,
+        memory_records: int = 1024,
+        max_memory_bytes: int = 8 << 20,
+        safe_points: tuple[str, ...] = (),
+        attempt_id: str = "",
+        serve_commands: bool = True,
+    ) -> None:
+        if (
+            overflow not in {"drop_new", "block", "raise"}
+            or memory_records < 1
+            or max_memory_bytes < MIN_MEMORY_BYTES
+            or media_spool_bytes < 1
+        ):
+            raise ValueError("explicit overflow policy and positive bounded queue budgets required")
+        self.client = client
+        self.last_result: JSONValue | None = None
+        self.participant = client.get_document(participant_id)
+        if self.participant.kind != "participant":
+            raise ValueError("runtime requires a participant")
+        self.scope = self.participant.scope
+        self.overflow = overflow
+        self.store = StateStore(
+            state_dir,
+            f"{client.base_url}|{client.organization_id}|{participant_id}",
+            max_spool_bytes,
+        )
+        self._queue: queue.Queue[tuple[str, str, str, bytes]] = queue.Queue(memory_records)
+        self._commands: queue.Queue[JSONObject] = queue.Queue(1)
+        self._handlers: dict[
+            str, tuple[Capability, CapabilityHandler, CapabilityValidator | None, str]
+        ] = {}
+        self._stop = threading.Event()
+        self._lock = threading.RLock()
+        self._space = threading.Condition(self._lock)
+        self._media_space = threading.Condition(self._lock)
+        self._media_wake = threading.Event()
+        self._media_max = media_spool_bytes
+        self._memory_used = 0
+        self._max_memory_bytes = max_memory_bytes
+        self._active: JSONObject | None = None
+        self._fatal: BaseException | None = None
+        self._accepting = True
+        self._closed = False
+        self._ended = False
+        self._session: JSONObject = {}
+        self._instance_id = str(uuid.uuid4())
+        self._safe_points: list[str] = []
+        self._safe_points_dirty = False
+        self.dropped = 0
+        self.last_transport_error: str | None = None
+        self._envelope_supported = True
+        self._serve_commands = serve_commands
+        try:
+            for name in safe_points:
+                self._add_safe_point(name)
+            self._session = dict(
+                client.connect(
+                    participant_id,
+                    self._instance_id,
+                    safe_points=tuple(self._safe_points),
+                    attempt_id=attempt_id,
+                )
+            )
+            self._safe_points_dirty = False
+            self._recover()
+        except BaseException:
+            self.store.close()
+            raise
+        self._threads = [
+            threading.Thread(target=target, daemon=True, name=name)
+            for target, name in [
+                (self._writer, "launchhelm-spool"),
+                (self._uploader, "launchhelm-upload"),
+                (self._media_worker, "launchhelm-media"),
+                (self._control, "launchhelm-control"),
+            ]
+        ]
+        for thread in self._threads:
+            thread.start()
+
+    @property
+    def session(self) -> JSONObject:
+        """Copy of the current execution session (epoch and state revision).
+
+        Contract: the copy is consistent with the last connect/heartbeat
+        installed under the runtime lock.
+
+        Failure: none.
+        """
+        with self._lock:
+            return dict(self._session)
+
+    def __enter__(self) -> Runtime:
+        """Return ``self`` for ``with Runtime(...)`` usage.
+
+        Contract: ``__exit__`` calls ``close``.
+
+        Failure: none.
+        """
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        """Close the runtime, ignoring the block exception.
+
+        Contract: does not suppress exceptions from the ``with`` body.
+
+        Failure: propagates ``close`` errors.
+        """
+        self.close()
+
+    def register(
+        self,
+        capability: Capability,
+        handler: CapabilityHandler,
+        *,
+        validate: CapabilityValidator | None = None,
+    ) -> Document:
+        """Publish ``capability`` and bind ``handler`` to its resource id.
+
+        Contract: schemas are checked locally before the server create. The
+        handler runs only at a matching ``safe_point``, never automatically
+        on restart.
+
+        Failure: ``jsonschema.SchemaError`` for an invalid schema.
+        ``LaunchHelmError`` from the server.
+        """
+        capability = copy.deepcopy(capability)
+        _require_semantic(capability.input_schema, "input_schema")
+        _require_semantic(capability.output_schema, "output_schema")
+        _require_semantic(capability.presentation, "presentation")
+        document = self.client.create_document(
+            "capability", self.scope, capability.wire(self.participant.id)
+        )
+        added = False
+        with self._lock:
+            self._serve_commands = True
+            self._handlers[document.id] = (
+                capability,
+                handler,
+                validate,
+                document.revision,
+            )
+            for name in capability.safe_points:
+                added = self._add_safe_point(name) or added
+        if added:
+            self._advertise_safe_points()
+        return document
+
+    def stream(
+        self,
+        name: str,
+        *,
+        value_schema: JSONSchema,
+        axes: tuple[str, ...] = ("step",),
+        unit: str = "",
+    ) -> Document:
+        """Create a telemetry stream owned by this participant.
+
+        Contract: ``value_schema`` is checked locally. Points are admitted
+        with ``log``.
+
+        Failure: ``jsonschema.SchemaError`` for an invalid schema.
+        ``LaunchHelmError`` from the server.
+        """
+        Draft202012Validator.check_schema(value_schema)
+        spec: JSONObject = {
+            "name": name,
+            "participant_id": self.participant.id,
+            "value_schema": value_schema,
+            "axes": list(axes),
+            "labels": {},
+        }
+        if unit:
+            spec["unit"] = unit
+        return self.client.create_document("stream", self.scope, spec)
+
+    def log(
+        self,
+        stream: Document,
+        value: JSONValue,
+        *,
+        coordinates: list[dict[str, str]],
+        observed_at: str | None = None,
+        timeout: float | None = None,
+    ) -> bool:
+        """Admit one telemetry point into the volatile memory queue.
+
+        Contract: success means the point is in process memory, not durable
+        and not query-visible. ``True`` means accepted; ``False`` means
+        dropped under ``overflow="drop_new"``. The session epoch is
+        snapshotted under the runtime lock at enqueue so a fence cannot
+        attach a stale epoch. The point is canonicalized exactly once here
+        (``encode``); later stages store and concatenate those bytes.
+
+        Failure:
+            ValueError: stream identity/scope mismatch, or the encoded point
+                is too large.
+            RuntimeError: the runtime is closed.
+            BufferFull: memory budget exhausted with overflow ``raise``, or
+                ``block`` timed out.
+            RuntimeFenced / LaunchHelmError / other fatal: a background
+                thread has failed closed.
+        """
+        if (
+            stream.kind != "stream"
+            or stream.spec.get("participant_id") != self.participant.id
+            or stream.scope != self.scope
+        ):
+            raise ValueError("stream belongs to another participant or scope")
+        coords: list[JSONValue] = [string_object(item) for item in coordinates]
+        payload: JSONObject = {
+            "observed_at": observed_at or timestamp(),
+            "coordinates": coords,
+            "value": value,
+        }
+        raw = encode(payload)
+        if len(raw) > MAX_VOLATILE_POINT_BYTES:
+            raise ValueError("point is too large; use an artifact")
+        deadline = time.monotonic() + timeout if timeout is not None else None
+        with self._space:
+            while True:
+                if not self._accepting:
+                    raise RuntimeError("runtime is closed")
+                if self._fatal is not None:
+                    raise self._fatal
+                epoch = str(self._session["epoch"])
+                if (
+                    not self._queue.full()
+                    and self._memory_used + len(raw) <= self._max_memory_bytes
+                ):
+                    self._queue.put_nowait((stream.id, stream.revision, epoch, raw))
+                    self._memory_used += len(raw)
+                    return True
+                if self.overflow == "drop_new":
+                    self._dropped()
+                    return False
+                if self.overflow == "raise":
+                    raise BufferFull()
+                remaining = None if deadline is None else deadline - time.monotonic()
+                if remaining is not None and remaining <= 0:
+                    raise BufferFull()
+                self._space.wait(remaining)
+
+    def log_media(
+        self,
+        stream: Document,
+        value: JSONValue,
+        *,
+        coordinates: list[dict[str, str]],
+        blobs: Sequence[PendingBlob],
+        timeout: float | None = None,
+    ) -> bool:
+        """Admit a point whose media items are still local blobs.
+
+        Contract: each ``PendingBlob`` in ``blobs`` is written
+        content-addressed into the state directory and the encoded point
+        is recorded in the media spool — durable locally before this
+        returns, so ``flush(delivered=False)`` has nothing further to do.
+        The background media worker uploads each blob as a ``media``
+        artifact (idempotent on run and content hash) and admits the
+        point only after every reference resolves. Pending media bytes
+        are bounded by ``media_spool_bytes`` under the runtime overflow
+        policy: ``drop_new`` returns ``False``, ``raise`` raises
+        ``BufferFull``, ``block`` waits. ``True`` means accepted.
+
+        Failure: same as ``log`` plus ``BufferFull`` for the media budget
+        under ``raise``, or ``block`` timing out.
+        """
+        if (
+            stream.kind != "stream"
+            or stream.spec.get("participant_id") != self.participant.id
+            or stream.scope != self.scope
+        ):
+            raise ValueError("stream belongs to another participant or scope")
+        coords: list[JSONValue] = [string_object(item) for item in coordinates]
+        payload: JSONObject = {
+            "observed_at": timestamp(),
+            "coordinates": coords,
+            "value": value,
+        }
+        raw = encode(payload)
+        if len(raw) > MAX_VOLATILE_POINT_BYTES:
+            raise ValueError("point is too large; use an artifact")
+        sized: dict[str, tuple[str, bytes]] = {}
+        for blob in blobs:
+            hash = str(blob.hash)
+            media_type = str(blob.media_type)
+            data = bytes(blob.data)
+            sized[hash] = (media_type, data)
+        deadline = time.monotonic() + timeout if timeout is not None else None
+        with self._media_space:
+            while True:
+                if not self._accepting:
+                    raise RuntimeError("runtime is closed")
+                if self._fatal is not None:
+                    raise self._fatal
+                epoch = str(self._session["epoch"])
+                added = self.store.media_new_bytes(
+                    [(hash, len(data)) for hash, (_mt, data) in sized.items()]
+                )
+                if self.store.media_used() + added + len(raw) <= self._media_max:
+                    break
+                if self.overflow == "drop_new":
+                    self._dropped()
+                    return False
+                if self.overflow == "raise":
+                    raise BufferFull()
+                remaining = None if deadline is None else deadline - time.monotonic()
+                if remaining is not None and remaining <= 0:
+                    raise BufferFull()
+                self._media_space.wait(remaining)
+        self.store.enqueue_media(
+            stream.id,
+            stream.revision,
+            epoch,
+            raw,
+            [(hash, media_type, data) for hash, (media_type, data) in sized.items()],
+        )
+        self._media_wake.set()
+        return True
+
+    def _dropped(self) -> None:
+        with self._lock:
+            self.dropped += 1
+            count = self.dropped
+        if count == 1 or count % 100 == 0:
+            warnings.warn(
+                f"LaunchHelm telemetry overflow: {count} volatile records dropped",
+                RuntimeWarning,
+                stacklevel=2,
+            )
+
+    def _set_fatal(self, error: BaseException) -> None:
+        with self._space:
+            if isinstance(self._fatal, RuntimeFenced):
+                self._space.notify_all()
+                return
+            if self._fatal is None or isinstance(error, RuntimeFenced):
+                self._fatal = error
+            self._space.notify_all()
+
+    def _set_transport_error(self, code: str | None) -> None:
+        with self._lock:
+            self.last_transport_error = code
+
+    def _raise_if_fatal(self) -> None:
+        with self._lock:
+            if self._fatal is not None:
+                raise self._fatal
+
+    def _add_safe_point(self, name: str) -> bool:
+        if not _SAFE_POINT_NAME.fullmatch(name):
+            raise ValueError("invalid safe point name")
+        with self._lock:
+            if name in self._safe_points:
+                return False
+            if len(self._safe_points) >= MAX_SAFE_POINTS:
+                raise ValueError("at most 64 safe points may be declared")
+            self._safe_points.append(name)
+            self._safe_points_dirty = True
+            return True
+
+    def _advertise_safe_points(self) -> None:
+        # The control loop cannot heartbeat while its claim is parked, so a
+        # newly declared point would stay invisible to the server until the
+        # wait elapsed. Advertise it now; a claim already parked re-reads the
+        # session each poll slice and claims the command within that slice.
+        with self._lock:
+            if self._ended or isinstance(self._fatal, RuntimeFenced):
+                return
+            points = tuple(self._safe_points)
+            epoch = str(self._session["epoch"])
+        try:
+            session = self.client.heartbeat(self.participant.id, epoch, safe_points=points)
+        except LaunchHelmError as error:
+            self._set_transport_error(error.code)
+            self._heartbeat_fenced(error)
+            return
+        with self._lock:
+            if not isinstance(self._fatal, RuntimeFenced):
+                self._session = dict(session)
+                self._safe_points_dirty = False
+
+    def _writer(self) -> None:
+        while not self._stop.is_set() or not self._queue.empty():
+            try:
+                first = self._queue.get(timeout=WRITER_POLL_SECONDS)
+            except queue.Empty:
+                continue
+            batch = [first]
+            size = len(first[3])
+            while len(batch) < WRITER_BATCH_MAX and size < WRITER_BATCH_BYTES:
+                try:
+                    item = self._queue.get_nowait()
+                except queue.Empty:
+                    break
+                batch.append(item)
+                size += len(item[3])
+            self._persist(batch)
+
+    def _persist(self, batch: list[tuple[str, str, str, bytes]]) -> None:
+        """Persist a drained batch, applying the per-point overflow contract.
+
+        ``enqueue_many`` admits a prefix inside one transaction. The point
+        after it is handled exactly as the serial writer handled a
+        ``BufferFull`` for that point: ``drop_new`` drops it and moves on;
+        ``raise`` records a fatal and keeps retrying; ``block`` retries
+        until the spool has room; on stop the point is dropped after a
+        fatal is set. A point that fails the per-point size cap records a
+        fatal and is dropped; the rest of the batch still persists.
+        """
+        pending = batch
+        while pending:
+            try:
+                admitted = self.store.enqueue_many(pending)
+            except OversizePoint as error:
+                self._release(pending[: error.index])
+                self._set_fatal(error)
+                self._dropped()
+                self._release(pending[error.index : error.index + 1])
+                pending = pending[error.index + 1 :]
+                continue
+            except Exception as error:
+                self._set_fatal(error)
+                self._dropped()
+                self._release(pending[:1])
+                pending = pending[1:]
+                continue
+            self._release(pending[:admitted])
+            pending = pending[admitted:]
+            if not pending:
+                return
+            if self.overflow == "drop_new":
+                self._dropped()
+                self._release(pending[:1])
+                pending = pending[1:]
+                continue
+            if self.overflow == "raise":
+                self._set_fatal(BufferFull())
+            if self._stop.is_set():
+                self._set_fatal(BufferFull())
+                self._dropped()
+                self._release(pending[:1])
+                pending = pending[1:]
+                continue
+            self._stop.wait(WRITER_POLL_SECONDS)
+
+    def _release(self, items: Sequence[tuple[str, str, str, bytes]]) -> None:
+        """Account for dequeued points: ``task_done`` and memory release.
+
+        Contract: every item the writer dequeued is released exactly once,
+        after it is persisted or dropped, so ``flush(delivered=False)``
+        keeps its local-durability contract.
+        """
+        freed = 0
+        for _stream, _revision, _epoch, raw in items:
+            self._queue.task_done()
+            freed += len(raw)
+        with self._space:
+            self._memory_used -= freed
+            self._space.notify_all()
+
+    def _uploader(self) -> None:
+        delay = UPLOAD_IDLE_SECONDS
+        while not self._stop.is_set():
+            try:
+                if self._upload_grouped():
+                    delay = UPLOAD_IDLE_SECONDS
+                    continue
+                batch = self.store.batch(self.participant.id)
+                if batch is None:
+                    self._stop.wait(UPLOAD_IDLE_SECONDS)
+                    continue
+                receipt = self.client.ingest(batch)
+                if (
+                    receipt.get("batch_id") != batch.batch_id
+                    or receipt.get("durability") != "query_visible"
+                ):
+                    raise LaunchHelmError(
+                        "invalid_receipt", "Server did not confirm batch durability"
+                    )
+                self.store.acknowledge(batch.batch_id)
+                self._set_transport_error(None)
+                delay = UPLOAD_IDLE_SECONDS
+            except LaunchHelmError as error:
+                self._set_transport_error(error.code)
+                if not error.retryable:
+                    self._set_fatal(error)
+                self._stop.wait(delay)
+                delay = min(delay * 2, UPLOAD_MAX_BACKOFF_SECONDS)
+            except Exception as error:
+                self._set_fatal(error)
+                self._stop.wait(1)
+
+    def _media_worker(self) -> None:
+        """Upload pending media blobs and release their points.
+
+        Each loop claims one un-uploaded blob; uploads are idempotent on
+        ``media:<run>:<hash>`` so a crash mid-upload resumes to the same
+        artifact, and ``resolve_media`` admits every point whose last
+        reference resolved. Resolved-but-budget-blocked points are
+        retried through ``flush_ready_media`` each pass.
+        """
+        delay = UPLOAD_IDLE_SECONDS
+        while not self._stop.is_set():
+            try:
+                if self._media_once():
+                    delay = UPLOAD_IDLE_SECONDS
+                    continue
+                self._media_wake.wait(UPLOAD_IDLE_SECONDS)
+                self._media_wake.clear()
+            except LaunchHelmError as error:
+                self._set_transport_error(error.code)
+                if not error.retryable:
+                    self._set_fatal(error)
+                self._stop.wait(delay)
+                delay = min(delay * 2, UPLOAD_MAX_BACKOFF_SECONDS)
+            except Exception as error:
+                self._set_fatal(error)
+                self._stop.wait(1)
+
+    def _media_once(self) -> bool:
+        """Make one unit of media progress; True when any work happened."""
+        progressed = False
+        admitted, freed = self.store.flush_ready_media()
+        if admitted > 0:
+            progressed = True
+        for hash in freed:
+            self.store.release_media_blob_file(hash)
+        pending = self.store.next_pending_media()
+        if pending is None:
+            return progressed
+        hash, media_type = pending
+        upload = getattr(self.client, "upload_file", None)
+        if not callable(upload):
+            raise LaunchHelmError("protocol", "client cannot upload media")
+        document = upload(
+            self.store.media_blob_path(hash),
+            self.scope,
+            idempotency_key=f"media:{self.scope.run_id}:{hash}",
+            media_type=media_type,
+            purpose="media",
+        )
+        if not isinstance(document, Document):
+            raise LaunchHelmError("protocol", "media upload did not return an artifact document")
+        for blob_hash in self.store.resolve_media(hash, document.id):
+            self.store.release_media_blob_file(blob_hash)
+        with self._media_space:
+            self._media_space.notify_all()
+        return True
+
+    def _upload_grouped(self) -> bool:
+        """Upload pending batches through the grouped endpoint.
+
+        Returns True when the envelope path handled this tick (including a
+        discovered fallback to single-batch ingest). Returns False when the
+        caller should use the single-batch endpoint: the client lacks
+        ``ingest_many`` or an earlier probe found the endpoint missing.
+        """
+        if not self._envelope_supported:
+            return False
+        ingest_many = getattr(self.client, "ingest_many", None)
+        if not callable(ingest_many):
+            self._envelope_supported = False
+            return False
+        batches = self.store.batches(self.participant.id)
+        if not batches:
+            self._stop.wait(UPLOAD_IDLE_SECONDS)
+            return True
+        try:
+            response = ingest_many(batches)
+        except LaunchHelmError as error:
+            if error.code != "not_found" or error.status != 404:
+                raise
+            # The endpoint may not exist on this server. Distinguish a
+            # missing route from a missing organization by sending the first
+            # batch through the original endpoint: if it also misses, the
+            # envelope failure was not about routing.
+            single = self.client.ingest(batches[0])
+            if single.get("batch_id") != batches[0].batch_id:
+                raise error from None
+            if single.get("durability") != "query_visible":
+                raise LaunchHelmError(
+                    "invalid_receipt", "Server did not confirm batch durability"
+                ) from None
+            self.store.acknowledge(batches[0].batch_id)
+            self._envelope_supported = False
+            return True
+        self._reconcile_receipts(batches, response)
+        self._set_transport_error(None)
+        return True
+
+    def _reconcile_receipts(self, batches: list[TelemetryBatch], response: JSONObject) -> None:
+        """Apply per-batch outcomes from a grouped ingest response.
+
+        Contract: every receipt must name a submitted ``batch_id`` and every
+        submitted batch must be acknowledged, reported duplicate, or
+        rejected. Violations are protocol errors. Accepted and duplicate
+        batches leave the spool; rejected batches stay spooled so a fatal
+        stop preserves them for inspection, and a retryable rejection is
+        re-sent with the next envelope.
+        """
+        receipts = response.get("receipts")
+        if not isinstance(receipts, list) or len(receipts) != len(batches):
+            raise LaunchHelmError("invalid_receipt", "Server did not return one receipt per batch")
+        pending = {batch.batch_id: batch for batch in batches}
+        first_rejected: LaunchHelmError | None = None
+        for entry in receipts:
+            receipt = json_object(entry, what="ingestion receipt")
+            batch_id = receipt.get("batch_id")
+            batch = pending.pop(batch_id, None) if isinstance(batch_id, str) else None
+            if batch is None:
+                raise LaunchHelmError(
+                    "invalid_receipt", "Server returned a receipt for an unknown batch"
+                )
+            status = receipt.get("status")
+            if status in ("accepted", "duplicate"):
+                if receipt.get("durability") != "query_visible":
+                    raise LaunchHelmError(
+                        "invalid_receipt", "Server did not confirm batch durability"
+                    )
+                self.store.acknowledge(batch.batch_id)
+            elif status == "rejected":
+                if first_rejected is None:
+                    code = receipt.get("code")
+                    message = receipt.get("message")
+                    first_rejected = LaunchHelmError(
+                        code if isinstance(code, str) else "rejected",
+                        message if isinstance(message, str) else "Server rejected the batch",
+                        retryable=bool(receipt.get("retryable")),
+                    )
+            else:
+                raise LaunchHelmError(
+                    "invalid_receipt", "Server returned an unknown receipt status"
+                )
+        if pending:
+            raise LaunchHelmError(
+                "invalid_receipt", "Server omitted receipts for submitted batches"
+            )
+        if first_rejected is not None:
+            raise first_rejected
+
+    def _result(
+        self,
+        command: JSONObject,
+        outcome: str,
+        result: JSONValue = None,
+        coordinates: list[dict[str, str]] | None = None,
+        failure: str = "",
+    ) -> JSONObject:
+        applied: list[JSONValue] = []
+        for item in coordinates or []:
+            applied.append(string_object(item))
+        request = json_object(command.get("request") or {}, what="command request")
+        return {
+            "lease_id": command["lease_id"],
+            "epoch": request["epoch"],
+            "outcome": outcome,
+            "result": result,
+            "applied_at": applied,
+            "failure_code": failure,
+        }
+
+    def _publish_pending(self) -> None:
+        for command, phase, result in self.store.recoverable():
+            if phase != "completed" or result is None:
+                continue
+            try:
+                reported = self.client.report(command, result)
+            except LaunchHelmError as error:
+                if error.code != "conflict" or result["outcome"] != "rejected":
+                    raise
+                reported = self.client.get_command(str(command["id"]))
+                if reported["execution"] not in {"rejected", "cancelled"}:
+                    raise
+            self.store.journal_phase(str(command["id"]), "reported")
+            revision = reported.get("resulting_state_revision")
+            if revision:
+                with self._lock:
+                    session = dict(self._session)
+                    session["state_revision"] = revision
+                    self._session = session
+            with self._lock:
+                active = self._active
+                if active and active["id"] == command["id"]:
+                    self._active = None
+
+    def _recover(self) -> None:
+        for command, phase, result in self.store.recoverable():
+            if phase == "received":
+                current = self.client.get_command(str(command["id"]))
+                if current["execution"] in {
+                    "rejected",
+                    "cancelled",
+                    "applied",
+                    "failed",
+                }:
+                    self.store.journal_phase(str(command["id"]), "reported")
+                    continue
+            if result is None:
+                outcome = "rejected" if phase == "received" else "outcome_unknown"
+                self.store.journal_phase(
+                    str(command["id"]),
+                    "completed",
+                    self._result(command, outcome, failure="runtime_restart"),
+                )
+        try:
+            self._publish_pending()
+        except LaunchHelmError:
+            self._set_transport_error("reconciliation_required")
+
+    def end_session(self, *, close_run: str = "never") -> JSONObject | None:
+        """End the participant's live session epoch on the server.
+
+        Contract: heartbeats and claims stop at once; a heartbeat or claim
+        already in flight when the end lands is expected to conflict and is
+        not reported as fencing. Returns the server response, or ``None``
+        when the client does not implement ``end_session`` — the session
+        then simply expires. The epoch sent is the last one the server
+        confirmed.
+
+        Failure: ``LaunchHelmError`` from the server, including ``conflict``
+        when the recorded epoch is no longer live.
+        """
+        with self._lock:
+            self._ended = True
+            epoch = str(self._session.get("epoch") or "")
+        end = getattr(self.client, "end_session", None)
+        if not callable(end) or not epoch:
+            return None
+        return json_object(
+            end(self.participant.id, epoch, close_run=close_run),
+            what="session end",
+        )
+
+    def _heartbeat_fenced(self, error: LaunchHelmError) -> None:
+        if error.code != "conflict":
+            return
+        with self._lock:
+            ended = self._ended
+        if not ended:
+            self._set_fatal(RuntimeFenced())
+
+    def _control(self) -> None:
+        heartbeat_at = 0.0
+        delay = CONTROL_POLL_SECONDS
+        while not self._stop.is_set():
+            parked = False
+            try:
+                self._publish_pending()
+                with self._lock:
+                    if self._ended:
+                        return
+                    fenced = isinstance(self._fatal, RuntimeFenced)
+                if fenced:
+                    self._stop.wait(FENCE_WAIT_SECONDS)
+                    continue
+                heartbeated = False
+                with self._lock:
+                    dirty = self._safe_points_dirty
+                    points = tuple(self._safe_points)
+                    epoch = str(self._session["epoch"])
+                if dirty:
+                    try:
+                        session = self.client.heartbeat(
+                            self.participant.id, epoch, safe_points=points
+                        )
+                    except LaunchHelmError as error:
+                        self._heartbeat_fenced(error)
+                        raise
+                    with self._lock:
+                        if not isinstance(self._fatal, RuntimeFenced):
+                            self._session = dict(session)
+                            self._safe_points_dirty = False
+                    heartbeat_at = time.monotonic() + HEARTBEAT_INTERVAL_SECONDS
+                    heartbeated = True
+                elif time.monotonic() >= heartbeat_at:
+                    try:
+                        session = self.client.heartbeat(self.participant.id, epoch)
+                    except LaunchHelmError as error:
+                        self._heartbeat_fenced(error)
+                        raise
+                    with self._lock:
+                        if not isinstance(self._fatal, RuntimeFenced):
+                            self._session = dict(session)
+                    heartbeat_at = time.monotonic() + HEARTBEAT_INTERVAL_SECONDS
+                    heartbeated = True
+                if heartbeated:
+                    with self._lock:
+                        active = self._active
+                    if active:
+                        try:
+                            self.client.renew(active)
+                        except LaunchHelmError as error:
+                            if error.code != "conflict":
+                                raise
+                            self._set_transport_error("lease_reconciliation")
+                with self._lock:
+                    available = (
+                        self._active is None and self._fatal is None and self._serve_commands
+                    )
+                    epoch = str(self._session["epoch"])
+                if available and self._commands.empty():
+                    command = self.client.claim(self.participant.id, epoch, wait_ms=CLAIM_WAIT_MS)
+                    parked = True
+                    if command:
+                        phase = self.store.remember(command)
+                        if phase != "received":
+                            self._set_transport_error("reconciliation_required")
+                        else:
+                            with self._lock:
+                                self._active = command
+                            self._commands.put_nowait(command)
+                delay = 0 if parked else CONTROL_POLL_SECONDS
+            except LaunchHelmError as error:
+                with self._lock:
+                    ended = self._ended
+                if self._stop.is_set() or ended:
+                    return
+                self._set_transport_error(error.code)
+                if error.code in {"unauthorized", "forbidden"}:
+                    self._set_fatal(error)
+                delay = min(max(1, delay * 2), CONTROL_MAX_BACKOFF_SECONDS)
+            except Exception as error:
+                if self._stop.is_set():
+                    return
+                self._set_fatal(error)
+                delay = 1
+            if delay:
+                self._stop.wait(delay)
+
+    def package_capture(
+        self,
+        path: str | Path,
+        scope: Scope,
+        *,
+        idempotency_key: str,
+        completeness: str = "partial",
+        provenance: JSONObject | None = None,
+    ) -> JSONObject:
+        """Upload one capture artifact and return the command result body.
+
+        Contract: called from a safe-point handler. The returned object is the
+        capture command result: artifact reference, completeness, and the
+        digest the server must match before commit. ``completeness`` is
+        ``partial`` or ``complete``.
+
+        Failure: ``ValueError`` for an unknown completeness. ``LaunchHelmError``
+        when the client cannot upload or the artifact has no sha256.
+        """
+        if completeness not in {"partial", "complete"}:
+            raise ValueError("completeness must be partial or complete")
+        upload = getattr(self.client, "upload_file", None)
+        if not callable(upload):
+            raise LaunchHelmError("protocol", "runtime client cannot upload artifacts")
+        artifact = upload(path, scope, idempotency_key=idempotency_key, provenance=provenance)
+        if not isinstance(artifact, Document):
+            raise LaunchHelmError("protocol", "upload did not return an artifact document")
+        digest = str(artifact.spec.get("sha256") or "")
+        if len(digest) != 64:
+            raise LaunchHelmError("protocol", "artifact digest missing")
+        return {
+            "artifact": {
+                "id": artifact.id,
+                "kind": "artifact",
+                "revision": artifact.revision,
+            },
+            "completeness": completeness,
+            "sha256": digest,
+        }
+
+    def pending(self, name: str) -> bool:
+        """Return ``True`` when a claimed command waits locally for ``name``.
+
+        Contract: reads the local claimed-command queue only; no network, no
+        state change. A ``True`` result means the next ``safe_point(name)``
+        call has work. ``False`` means nothing claimed for ``name`` has
+        arrived locally — the control thread may still be claiming.
+
+        Failure: none.
+        """
+        with self._commands.mutex:
+            for command in self._commands.queue:
+                request = command.get("request")
+                if isinstance(request, dict) and request.get("safe_point") == name:
+                    return True
+        return False
+
+    def safe_point(self, name: str, *, coordinates: list[dict[str, str]]) -> int:
+        """Run at most one claimed command whose safe-point name is ``name``.
+
+        Contract: returns ``1`` if a handler ran, ``0`` if no matching
+        command was waiting or the command was rejected locally. The first
+        call for ``name`` advertises it on a heartbeat immediately (Connect
+        still declares the constructor set) so a parked Claim can deliver
+        commands for that point without waiting out its park. A claimed
+        command whose
+        ``safe_point`` does not match ``name`` is a protocol violation: the
+        command is journaled as rejected and the runtime fails closed. A
+        handler exception is recorded as ``outcome_unknown`` and is never
+        retried automatically. Begin failures that are not definitive also
+        become ``outcome_unknown``.
+
+        Failure: ``ValueError`` for invalid or duplicate coordinates, an
+        invalid safe-point name, or a fingerprint mismatch. ``LaunchHelmError``
+        with code ``protocol`` when the claimed command does not match
+        ``name``. ``LaunchHelmError`` from a definitive ``begin`` rejection
+        is raised after journaling. Background fatals are raised before
+        touching the command queue.
+        """
+        _validate(_COORDINATES, coordinates)
+        if len({item["axis"] for item in coordinates}) != len(coordinates):
+            raise ValueError("duplicate coordinate axis")
+        with self._lock:
+            self._serve_commands = True
+        if self._add_safe_point(name):
+            self._advertise_safe_points()
+        self._raise_if_fatal()
+        try:
+            command = self._commands.get_nowait()
+        except queue.Empty:
+            return 0
+        request = json_object(command.get("request") or {}, what="command request")
+        if request.get("safe_point") != name:
+            encoded = self._result(command, "rejected", failure="undeclared_safe_point")
+            self.store.journal_phase(str(command["id"]), "completed", encoded)
+            error = LaunchHelmError(
+                "protocol",
+                f"claimed command safe_point {request.get('safe_point')!r} does not match {name!r}",
+            )
+            self._set_fatal(error)
+            raise error
+        if fingerprint(request) != command["fingerprint"]:
+            self.store.journal_phase(
+                str(command["id"]),
+                "completed",
+                self._result(command, "rejected", failure="fingerprint_mismatch"),
+            )
+            raise ValueError("command fingerprint does not match payload")
+        capability_ref = json_object(request.get("capability") or {}, what="capability ref")
+        with self._lock:
+            registered = self._handlers.get(str(capability_ref.get("id")))
+        if registered is None or capability_ref.get("revision") != registered[3]:
+            self.store.journal_phase(
+                str(command["id"]),
+                "completed",
+                self._result(command, "rejected", failure="handler_unavailable_or_changed"),
+            )
+            return 0
+        capability, handler, validate, _ = registered
+        try:
+            payload = request["input"]
+            _validate_semantic(capability.input_schema, payload)
+            if validate is not None:
+                validate(payload)
+        except Exception:
+            self.store.journal_phase(
+                str(command["id"]),
+                "completed",
+                self._result(command, "rejected", failure="local_validation"),
+            )
+            return 0
+        self.store.journal_phase(str(command["id"]), "starting")
+        try:
+            self.client.begin(command)
+        except LaunchHelmError as error:
+            definitive = error.code in {
+                "validation",
+                "conflict",
+                "not_found",
+                "forbidden",
+                "unauthorized",
+            }
+            self.store.journal_phase(
+                str(command["id"]),
+                "completed",
+                self._result(
+                    command,
+                    "rejected" if definitive else "outcome_unknown",
+                    failure="begin_rejected" if definitive else "begin_uncertain",
+                ),
+            )
+            raise
+        self.store.journal_phase(str(command["id"]), "applying")
+        targets_raw = request.get("targets")
+        targets_list = targets_raw if isinstance(targets_raw, list) else []
+        context = CommandContext(
+            str(command["id"]),
+            tuple(
+                Reference(str(item["id"]), str(item["kind"]), str(item.get("revision") or ""))
+                for item in targets_list
+                if isinstance(item, dict)
+            ),
+            str(request["epoch"]),
+            tuple(coordinates),
+        )
+        try:
+            result = handler(payload, context)
+            _validate(capability.output_schema, result)
+            self.last_result = result
+            encoded = self._result(command, "applied", result, coordinates)
+            encode(encoded)
+        except Exception as error:
+            encoded = self._result(command, "outcome_unknown", failure=type(error).__name__[:128])
+        self.store.journal_phase(str(command["id"]), "completed", encoded)
+        return 1
+
+    def flush(
+        self, *, timeout: float = DEFAULT_FLUSH_TIMEOUT_SECONDS, delivered: bool = True
+    ) -> None:
+        """Wait until admitted points reach the requested durability.
+
+        Contract: ``delivered=False`` is local SQLite durability (the writer
+        has dequeued every volatile record; pending media is already durable
+        at ``log_media`` so there is nothing further to wait for).
+        ``delivered=True`` additionally waits until media uploads resolve
+        and the spool has no unacknowledged points (query-visible).
+        Does not promise delivery after ``close``.
+
+        The media count is probed before the point count: a media point
+        enters ``points`` atomically with its media row's removal, so a
+        ``pending_count`` read can pass just before that commit and a
+        ``media_pending_count`` read just after — reading them in this
+        order would let ``flush`` return with the point still unsent.
+        Points admitted concurrently with the wait itself are outside the
+        contract; ``stop_accepting`` fences them for callers like
+        ``Run.finish`` that need a fixed drain set.
+
+        Failure: raises the background fatal if one is set, including when
+        the queues are empty. ``TimeoutError`` if the deadline expires with
+        work remaining.
+        """
+        deadline = time.monotonic() + timeout
+        while True:
+            self._raise_if_fatal()
+            if not self._queue.unfinished_tasks and (
+                not delivered
+                or (not self.store.media_pending_count() and not self.store.pending_count())
+            ):
+                return
+            if time.monotonic() >= deadline:
+                raise TimeoutError(
+                    "LaunchHelm flush timed out; unacknowledged durable records remain in the spool"
+                )
+            time.sleep(FLUSH_POLL_SECONDS)
+
+    def stop_accepting(self) -> None:
+        """Stop point admission without stopping delivery.
+
+        Contract: idempotent. ``log`` and ``log_media`` calls — including
+        ones already waiting on memory or media budget — raise
+        ``RuntimeError`` instead of admitting another point. Uploads,
+        media resolution, heartbeats, and the spool keep running so a
+        following ``flush(delivered=True)`` drains a fixed set; ``close``
+        performs the full stop.
+
+        Failure: none.
+        """
+        with self._space:
+            self._accepting = False
+            self._space.notify_all()
+            self._media_space.notify_all()
+            self._media_wake.set()
+
+    def close(self, *, timeout: float = DEFAULT_CLOSE_TIMEOUT_SECONDS) -> None:
+        """Stop admission, persist the volatile queue, and join workers.
+
+        Contract: local durability is attempted via
+        ``flush(delivered=False)``. Server delivery is not promised. Idempotent.
+
+        Failure: ``TimeoutError`` if background threads outlive ``timeout``.
+        Re-raises a flush error after joining threads and closing the store.
+        """
+        with self._space:
+            if self._closed:
+                return
+        self.stop_accepting()
+        failure: BaseException | None = None
+        deadline = time.monotonic() + timeout
+        try:
+            self.flush(timeout=timeout, delivered=False)
+        except BaseException as error:
+            failure = error
+        finally:
+            self._stop.set()
+            abort = getattr(self.client, "abort", None)
+            if callable(abort):
+                abort()
+            for thread in self._threads:
+                thread.join(max(0, deadline - time.monotonic()))
+        if any(thread.is_alive() for thread in self._threads):
+            raise TimeoutError("LaunchHelm background operations have not stopped")
+        self.store.close()
+        with self._lock:
+            self._closed = True
+        if failure is not None:
+            raise failure