pi-durable-python 1.3.0__py3-none-any.whl

pi_durable/harness/submissions.py

@@ -0,0 +1,357 @@
+"""Submission admission, waits, and withdrawal (spec section 6).
+
+``Conversation.submit()`` durably admits an input or passive write in one commit; the returned
+handle tracks its settlement. Admission decides between immediate placement and the inbox; a
+boundary later places queued items, and run control lives in ``pi.live``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable, Mapping
+from typing import Any, Literal, cast
+
+from ..context import Context
+from ..errors import ConversationBusy
+from ..ids import ConversationId, SubmissionId
+from ..json import JsonValue, copy_json
+from ..session.drafts import DocumentDraft
+from ..session.transaction import Transaction
+from ..types import EntryDraft, SubmissionCreate, SubmissionRecord
+from .inbox import (
+    InboxDoc,
+    QueueModes,
+    append_input,
+    apply_boundary,
+    has_queued_reset,
+    is_display_only,
+    is_stale,
+    pick_input_entry,
+    prepare_boundary,
+    remove_inbox_item,
+)
+from .live import LiveDoc
+from .types import InputEntry, SettledSubmissionRecord, Submission, SubmissionDraft
+from .util import Waiters, closed_error, resolved
+
+AbortResult = Literal["aborted", "already_placed", "settled"]
+
+StartRun = Callable[[Transaction, ConversationId, DocumentDraft, list[SubmissionId]], Awaitable[None]]
+
+
+def _type(draft: Mapping[str, Any]) -> Any:
+    return draft.get("type")
+
+
+def _request_id(draft: Mapping[str, Any]) -> str | None:
+    value = draft.get("request_id", draft.get("requestId"))
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        raise TypeError("A submission request ID must be a string")
+    return value
+
+
+def _when_busy(draft: Mapping[str, Any]) -> str | None:
+    return draft.get("when_busy", draft.get("whenBusy"))
+
+
+def is_settled(record: SubmissionRecord) -> bool:
+    """Whether a submission reached ``done`` or ``unanswered``."""
+
+    return record["status"] in ("done", "unanswered")
+
+
+def check_input_entry(entry: InputEntry) -> None:
+    """Reject an input entry whose kind is not a non-empty string outside the ``pi.`` prefix."""
+
+    kind = entry.get("kind")
+    if not isinstance(kind, str) or not kind or kind.startswith("pi."):
+        raise TypeError(
+            "An input entry's kind must be a non-empty string not starting with \"pi.\", "
+            f"got {kind!r}"
+        )
+
+
+def _clean_entry(entry: Mapping[str, Any]) -> dict[str, JsonValue]:
+    """Store the fields the caller supplied as strict JSON, preserving explicit JSON nulls."""
+
+    return {key: copy_json(value) for key, value in entry.items()}
+
+
+class Submissions:
+    """Admission, waits, and withdrawal of the durable submissions of one Harness."""
+
+    def __init__(
+        self,
+        session: Any,
+        now: Callable[[], int],
+        queue_modes: Callable[[], QueueModes],
+        resume: Callable[[], None],
+        start_run: StartRun,
+    ) -> None:
+        self._session = session
+        self._now = now
+        self._queue_modes = queue_modes
+        self._resume = resume
+        self._start_run = start_run
+        self._waiters: Waiters[SubmissionId, SettledSubmissionRecord] = Waiters()
+        self._closed = False
+        session.observe_commits(self._observe)
+        session.subscribe_close(self._close)
+
+    async def submit(
+        self, conversation_id: ConversationId, draft: SubmissionDraft, context: Context
+    ) -> Submission:
+        self._resume()
+        value = cast(Mapping[str, Any], draft)
+
+        async def admit(tx: Transaction) -> SubmissionId:
+            return await admit_submission(
+                tx,
+                conversation_id,
+                value,
+                self._now(),
+                self._queue_modes(),
+                self._start_run,
+            )
+
+        id = await self._session.commit_with(admit, context)
+        return SubmissionHandle(cast(SubmissionId, id), self)
+
+    async def _record(self, id: SubmissionId, context: Context) -> SubmissionRecord | None:
+        return await self._session.read_on_line(
+            lambda: self._session.storage.submission(id, context)
+        )
+
+    async def get(self, id: SubmissionId, context: Context) -> Submission | None:
+        record = await self._record(id, context)
+        return None if record is None else SubmissionHandle(record["id"], self)
+
+    async def status(self, id: SubmissionId, context: Context) -> SubmissionRecord:
+        record = await self._record(id, context)
+        if record is None:
+            raise ValueError(f"Submission {id} does not exist")
+        return record
+
+    async def wait(self, id: SubmissionId, context: Context) -> SettledSubmissionRecord:
+        self._resume()
+
+        async def find() -> dict[str, Any]:
+            record = await self._session.storage.submission(id, context)
+            if record is None:
+                raise ValueError(f"Submission {id} does not exist")
+            if is_settled(record):
+                return {"promise": resolved(cast(SettledSubmissionRecord, record))}
+            if self._closed:
+                raise closed_error(self._session)
+            return {"promise": self._waiters.add(id, context)}
+
+        found = await self._session.read_on_line(find)
+        return cast(SettledSubmissionRecord, await found["promise"])
+
+    async def abort(
+        self, id: SubmissionId, context: Context, conversation_id: ConversationId | None = None
+    ) -> AbortResult | Literal["not_found"]:
+        async def withdraw(tx: Transaction) -> AbortResult | Literal["not_found"]:
+            record = await tx.submission(id)
+            if record is None or (
+                conversation_id is not None and record["conversationId"] != conversation_id
+            ):
+                return "not_found"
+            if record["status"] == "queued":
+                tx.settle_submission(id, {"status": "unanswered", "reason": "aborted"})
+                await remove_inbox_item(tx, record["conversationId"], id)
+                return "aborted"
+            return "already_placed" if record["status"] == "placed" else "settled"
+
+        return await self._session.commit_with(withdraw, context)
+
+    def _observe(self, publication: Any, _context: Context) -> None:
+        for change in publication["changes"]:
+            if change["type"] != "submission":
+                continue
+            value = change["value"]
+            if is_settled(value):
+                self._waiters.resolve(value["id"], cast(SettledSubmissionRecord, value))
+
+    def _close(self) -> None:
+        self._closed = True
+        self._waiters.reject_all(closed_error(self._session))
+
+
+class SubmissionHandle:
+    """One admitted submission, tracked by the Harness that returned it."""
+
+    __slots__ = ("id", "_submissions")
+
+    def __init__(self, id: SubmissionId, submissions: Submissions) -> None:
+        self.id = id
+        self._submissions = submissions
+
+    async def status(self, context: Context) -> SubmissionRecord:
+        return await self._submissions.status(self.id, context)
+
+    async def wait(self, context: Context) -> SettledSubmissionRecord:
+        return await self._submissions.wait(self.id, context)
+
+    async def abort(self, context: Context) -> AbortResult:
+        result = await self._submissions.abort(self.id, context)
+        if result == "not_found":
+            raise ValueError(f"Submission {self.id} does not exist")
+        return result
+
+
+async def admit_submission(
+    tx: Transaction,
+    conversation_id: ConversationId,
+    draft: Mapping[str, Any],
+    now: int,
+    queue_modes: QueueModes,
+    start_run: StartRun,
+) -> SubmissionId:
+    """Admit one submission inside a commit (spec section 6)."""
+
+    kind = _type(draft)
+    if kind not in ("input", "write"):
+        raise TypeError(f"Submission draft type must be 'input' or 'write', got {kind!r}")
+    entry = draft.get("entry")
+    if kind == "input" and entry is not None:
+        if not isinstance(entry, Mapping):
+            raise TypeError("An input entry must be an object")
+        check_input_entry(cast(InputEntry, entry))
+    request_id = _request_id(draft)
+    if request_id is not None:
+        existing = await tx.submission_by_request(conversation_id, request_id)
+        if existing is not None:
+            if existing["type"] != kind:
+                raise ValueError(
+                    f"Request {request_id} already identifies a submission of type {existing['type']}"
+                )
+            return existing["id"]
+    request = {} if request_id is None else {"requestId": request_id}
+
+    display_only = False
+    if kind == "write":
+        if not isinstance(entry, Mapping):
+            raise TypeError("A write submission requires an entry draft")
+        inbox = await tx.doc(InboxDoc, conversation_id)
+        display_only = is_display_only(cast(Any, entry))
+        if display_only and not has_queued_reset(inbox["items"]):
+            return await _place_write(tx, conversation_id, request, entry)
+
+    live = await tx.doc(LiveDoc, conversation_id)
+    busy = live.get("run") is not None
+    when_busy = _when_busy(draft)
+    if busy and kind == "input" and when_busy == "reject":
+        raise ConversationBusy(conversation_id)
+
+    boundary = None if busy else await prepare_boundary(tx, conversation_id, queue_modes)
+    if boundary is None or len(boundary.inbox["items"]) > 0:
+        inbox = boundary.inbox if boundary is not None else await tx.doc(InboxDoc, conversation_id)
+        record = await tx.create_submission(
+            cast(
+                SubmissionCreate,
+                {
+                    "conversationId": conversation_id,
+                    **request,
+                    "type": kind,
+                    "status": "queued",
+                },
+            )
+        )
+        id = record["id"]
+        if kind == "write":
+            inbox["items"].append(
+                {"id": id, "mode": "write", "entry": _clean_entry(cast(Mapping[str, Any], entry))}
+            )
+        else:
+            mode = "steer" if when_busy == "steer" else "followUp"
+            item: dict[str, Any] = {
+                "id": id,
+                "mode": mode,
+                "content": copy_json(draft.get("content")),
+            }
+            if entry is not None:
+                item["entry"] = pick_input_entry(cast(InputEntry, entry))
+            inbox["items"].append(item)
+        if boundary is None or display_only:
+            return id
+        placed = await apply_boundary(tx, boundary, "final", now)
+        if placed.users:
+            await start_run(tx, conversation_id, live, placed.users)
+        return id
+
+    entry = cast(Mapping[str, Any], entry) if entry is not None else None
+    if kind == "write":
+        assert entry is not None
+        if is_stale(boundary, cast(Any, entry)):
+            record = await tx.create_submission(
+                cast(
+                    SubmissionCreate,
+                    {
+                        "conversationId": conversation_id,
+                        **request,
+                        "type": "write",
+                        "status": "unanswered",
+                        "reason": "stale",
+                    },
+                )
+            )
+            return record["id"]
+        return await _place_write(tx, conversation_id, request, entry)
+
+    placed_entry = await append_input(
+        tx,
+        conversation_id,
+        copy_json(draft.get("content")),
+        None if entry is None else cast(InputEntry, entry),
+        now,
+    )
+    record = await tx.create_submission(
+        cast(
+            SubmissionCreate,
+            {
+                "conversationId": conversation_id,
+                **request,
+                "type": "input",
+                "status": "placed",
+                "entry": placed_entry["id"],
+            },
+        )
+    )
+    await start_run(tx, conversation_id, live, [record["id"]])
+    return record["id"]
+
+
+async def _place_write(
+    tx: Transaction,
+    conversation_id: ConversationId,
+    request: Mapping[str, str],
+    entry: Mapping[str, Any],
+) -> SubmissionId:
+    """Append a write that a boundary or display-only admission places at once."""
+
+    appended = await tx.append_entry(conversation_id, cast(EntryDraft, _clean_entry(entry)))
+    record = await tx.create_submission(
+        cast(
+            SubmissionCreate,
+            {
+                "conversationId": conversation_id,
+                **request,
+                "type": "write",
+                "status": "done",
+                "entry": appended["id"],
+            },
+        )
+    )
+    return record["id"]
+
+
+__all__ = [
+    "AbortResult",
+    "SubmissionHandle",
+    "Submissions",
+    "admit_submission",
+    "check_input_entry",
+    "is_settled",
+]

pi_durable/harness/task_graph.py

@@ -0,0 +1,334 @@
+"""Task graph view: every live task, its waits, and the conversations it owns (spec §9.5).
+
+The graph is to the Session's tasks what the conversation view is to one conversation: a
+structural mount built on the Session line by its first observer, advanced only from committed
+publications, and dropped with its last observer. A node holds the committed task record without
+its input, checkpoint payload, outcome payload, or memos.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from functools import partial
+from typing import Any, Literal, NotRequired, TypeAlias, TypedDict, cast
+
+from ..context import Context, without_abort_signal
+from ..delta import Op
+from ..ids import ConversationId, TaskId
+from ..json import JsonValue
+from ..session.observation import CommittedState, CommittedWatch
+from ..session.session import SessionImpl, contained
+from ..types import (
+    CommitPublication,
+    JoinPolicy,
+    Storage,
+    TaskOutcome,
+    TaskRecord,
+)
+from .scheduler import json_equal
+from .util import cancellation_error, closed_error, scan_all
+
+LIVE_STATUSES: tuple[Literal["pending", "running", "waiting", "completing"], ...] = (
+    "pending",
+    "running",
+    "waiting",
+    "completing",
+)
+"""Durable statuses of a live task; terminal tasks leave the graph."""
+
+SCAN_PAGE_SIZE = 256
+
+
+class TaskGraphStatePending(TypedDict):
+    status: Literal["pending", "running"]
+    phase: str
+
+
+class TaskGraphStateWaiting(TypedDict):
+    status: Literal["waiting"]
+    phase: str
+    on: list[TaskId]
+    policy: JoinPolicy
+
+
+class TaskGraphStateCompleting(TypedDict):
+    """Outcome held until its ordinary owned work drains (spec §5.5)."""
+
+    status: Literal["completing"]
+    outcome: str
+
+
+TaskGraphState = TaskGraphStatePending | TaskGraphStateWaiting | TaskGraphStateCompleting
+"""One live task's durable state without its checkpoint or outcome payloads."""
+
+
+class TaskGraphNode(TypedDict):
+    id: TaskId
+    kind: str
+    conversationId: ConversationId
+    owner: NotRequired[TaskId]
+    background: bool
+    abortRequested: bool
+    state: TaskGraphState
+    conversations: list[ConversationId]
+
+
+class TaskGraph(TypedDict):
+    """Every live task of the Session, keyed by its decimal ID."""
+
+    tasks: dict[str, TaskGraphNode]
+
+
+TaskGraphWatch: TypeAlias = CommittedWatch
+"""A serialized exact-frame watch of the task graph."""
+
+
+@dataclass(slots=True)
+class GraphMount:
+    value: TaskGraph
+    observers: set[Any] = field(default_factory=set)
+
+
+class TaskGraphView:
+    """The Harness's single task-graph mount, alive while observed."""
+
+    def __init__(self, session: SessionImpl, storage: Storage) -> None:
+        self._session = session
+        self._storage = storage
+        self._mount: GraphMount | None = None
+        self._closed = False
+        session.observe_commits(self._on_publication)
+        session.subscribe_close(self._on_close)
+
+    def report(self, error: BaseException) -> None:
+        self._session.report(error)
+
+    async def state(self, context: Context) -> CommittedState:
+        """A disposable read-only state of the graph."""
+
+        return await self._attach(_state_observer, context)
+
+    async def watch(self, context: Context) -> CommittedWatch:
+        """A serialized exact-frame watch of the graph; cancelling ``context`` stops it."""
+
+        observer = await self._attach(_watch_observer, context)
+        signal = context.abort_signal
+        if signal is not None and signal.aborted:
+            observer.cancel()
+            raise cancellation_error(signal)
+        if signal is not None:
+            observer.observe_cancellation(signal)
+        return observer
+
+    async def _attach(
+        self,
+        create: Callable[[TaskGraph, Callable[[], None], TaskGraphView], Any],
+        context: Context,
+    ) -> Any:
+        async def job() -> Any:
+            self._session.assert_usable()
+            mount = self._mount
+            if mount is None:
+                mount = GraphMount(value=await self._build(context))
+            detached = False
+
+            def detach() -> None:
+                nonlocal detached
+                if detached:
+                    return
+                detached = True
+                mount.observers.discard(observer)
+                if not mount.observers and self._mount is mount:
+                    self._mount = None
+
+            observer = create(mount.value, detach, self)
+            if asyncio.iscoroutine(observer):
+                observer = await observer
+            if self._closed:
+                raise closed_error(self._session)
+            self._session.assert_usable()
+            signal = context.abort_signal
+            if signal is not None and signal.aborted:
+                raise cancellation_error(signal)
+            self._mount = mount
+            mount.observers.add(observer)
+            return observer
+
+        return await self._session.read_on_line(job)
+
+    async def _build(self, context: Context) -> TaskGraph:
+        records: list[TaskRecord] = []
+        for status in LIVE_STATUSES:
+            records.extend(
+                await scan_all(
+                    partial(
+                        self._storage.scan_tasks,
+                        {"status": status},
+                        SCAN_PAGE_SIZE,
+                        context=context,
+                    )
+                )
+            )
+        records.sort(key=lambda record: record["id"])
+        tasks: dict[str, TaskGraphNode] = {}
+        for record in records:
+            owned = await scan_all(
+                partial(
+                    self._storage.scan_conversations,
+                    {"ownerTaskId": record["id"]},
+                    SCAN_PAGE_SIZE,
+                    context=context,
+                )
+            )
+            conversations = sorted(conversation["id"] for conversation in owned)
+            tasks[str(record["id"])] = _node_of(record, conversations)
+        return {"tasks": tasks}
+
+    def _on_publication(self, publication: CommitPublication, context: Context) -> None:
+        if self._mount is not None:
+            advance_graph(self._mount, publication, context, self.report)
+
+    def _on_close(self) -> None:
+        self._closed = True
+        failure = self._session.failure
+        mount = self._mount
+        if mount is not None:
+            for observer in list(mount.observers):
+                contained(partial(observer.close_session, failure), self.report)
+        self._mount = None
+
+
+def _state_observer(
+    value: TaskGraph, release: Callable[[], None], view: TaskGraphView
+) -> CommittedState:
+    return CommittedState(value, release, view.report)
+
+
+def _watch_observer(
+    value: TaskGraph, release: Callable[[], None], view: TaskGraphView
+) -> CommittedWatch:
+    return CommittedWatch(value, release, view.report)
+
+
+def advance_graph(
+    mount: GraphMount,
+    publication: CommitPublication,
+    context: Context,
+    report: Callable[[BaseException], None],
+) -> None:
+    """Derive the mount's operations from one publication and publish the new revision."""
+
+    ops: list[Op] = []
+    changed: dict[str, TaskGraphNode | None] = {}
+
+    def node(key: str) -> TaskGraphNode | None:
+        if key in changed:
+            return changed[key]
+        return mount.value["tasks"].get(key)
+
+    for change in publication["changes"]:
+        if change["type"] != "task":
+            continue
+        record = change["value"]
+        key = str(record["id"])
+        previous = node(key)
+        if record["state"]["status"] == "terminal":
+            if previous is None:
+                continue
+            ops.append(["d", ["tasks", key]])
+            changed[key] = None
+            continue
+        next_node = _node_of(record, previous["conversations"] if previous is not None else [])
+        if previous is not None and json_equal(next_node, previous):
+            continue
+        ops.append(["s", ["tasks", key], next_node])
+        changed[key] = next_node
+
+    # After the tasks, so a conversation created with its owner task in one commit finds the owner's node.
+    created: dict[str, list[ConversationId]] = {}
+    for change in publication["changes"]:
+        if change["type"] != "conversation":
+            continue
+        owner = change["value"].get("owner")
+        if owner is None:
+            continue
+        key = str(owner["taskId"])
+        if node(key) is not None:
+            created.setdefault(key, []).append(change["value"]["id"])
+    for key, ids in created.items():
+        current = node(key)
+        assert current is not None
+        conversations = sorted([*current["conversations"], *ids])
+        ops.append(["s", ["tasks", key, "conversations"], conversations])
+        changed[key] = {**current, "conversations": conversations}
+
+    if not ops:
+        return
+    tasks = dict(mount.value["tasks"])
+    for op in ops:
+        path = op[1]
+        if op[0] == "d":
+            tasks.pop(path[1], None)
+        elif len(path) == 2:
+            tasks[path[1]] = op[2]
+        else:
+            current = tasks[path[1]]
+            tasks[path[1]] = {**current, "conversations": op[2]}
+    mount.value = {"tasks": tasks}
+    frame_context = without_abort_signal(context)
+    for observer in list(mount.observers):
+        advance = getattr(observer, "advance_frame", None) or observer.advance
+        contained(partial(advance, mount.value, ops, frame_context), report)
+
+
+def _node_of(record: TaskRecord, conversations: list[ConversationId]) -> TaskGraphNode:
+    node = TaskGraphNode(
+        id=record["id"],
+        kind=record["kind"],
+        conversationId=record["conversationId"],
+        background=record["background"],
+        abortRequested=record["abortRequested"],
+        state=_state_of(record),
+        conversations=list(conversations),
+    )
+    owner = record.get("owner")
+    if owner is not None:
+        node["owner"] = owner
+    return node
+
+
+def _state_of(record: TaskRecord) -> TaskGraphState:
+    state = cast(dict[str, Any], record["state"])
+    status = state["status"]
+    if status in ("pending", "running"):
+        return {"status": status, "phase": _phase_of(state["checkpoint"])}
+    if status == "waiting":
+        return {
+            "status": "waiting",
+            "phase": _phase_of(state["checkpoint"]),
+            "on": list(state["on"]),
+            "policy": state["policy"],
+        }
+    outcome = _outcome_status(state["outcome"])
+    return {"status": "completing", "outcome": outcome}
+
+
+def _phase_of(checkpoint: JsonValue) -> str:
+    return str(cast(dict[str, Any], checkpoint)["phase"])
+
+
+def _outcome_status(outcome: TaskOutcome) -> str:
+    return outcome["status"]
+
+
+__all__ = [
+    "LIVE_STATUSES",
+    "TaskGraph",
+    "TaskGraphNode",
+    "TaskGraphState",
+    "TaskGraphView",
+    "TaskGraphWatch",
+    "advance_graph",
+]