shikiru-cli 0.2.0__py3-none-any.whl

shikiru/engine/lifecycle.py

@@ -0,0 +1,60 @@
+"""Shared terminal task cleanup."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from shikiru.domain.tasks import Status
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from shikiru.domain.tasks import Task
+    from shikiru.engine.feedback_audit import FeedbackWorkAudit
+    from shikiru.engine.layout_guard import LayoutGuard
+    from shikiru.engine.pull_request_closing import PullRequestClosing
+    from shikiru.engine.sessions import SessionManager
+    from shikiru.workspace.roots import TaskRootBuilder
+
+
+class TaskFinisher:
+    """Close task sessions and window, queue a cancelled task's PR close, remove its worktree."""
+
+    def __init__(
+        self,
+        sessions: SessionManager,
+        layout: LayoutGuard,
+        roots: TaskRootBuilder,
+        pull_requests: PullRequestClosing,
+        audit: FeedbackWorkAudit,
+    ) -> None:
+        self._sessions = sessions
+        self._layout = layout
+        self._roots = roots
+        self._pull_requests = pull_requests
+        self._audit = audit
+        self._finished_tasks: set[str] = set()
+
+    async def finish_task(self, task: Task) -> Path | None:
+        """Release task resources after it reaches a terminal lifecycle state.
+
+        A cancelled task's open pull request is queued for closing off the calling loop, never
+        awaited here. The worktree goes last, so a task root left behind marks unfinished
+        cleanup.
+        A done task whose branch holds feedback commits the pull request never received keeps
+        its worktree and branch; the merged evidence still completes it.
+        Returns the task root when unintegrated work kept it in place.
+        """
+        if task.id in self._finished_tasks:
+            return None
+        self._sessions.close_task(task.id)
+        await self._layout.close(task)
+        if task.status is Status.CANCELLED:
+            self._pull_requests.request(task.id)
+        integrated = task.status is Status.DONE
+        keep_work = integrated and await self._audit.holds_unpublished_work(task)
+        kept = await self._roots.cleanup(
+            task.id, task.repo, integrated=integrated and not keep_work, keep_work=keep_work
+        )
+        self._finished_tasks.add(task.id)
+        return kept

shikiru/engine/monitor.py

@@ -0,0 +1,317 @@
+"""Consumes one headless turn stream and records its facts and live view."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from dataclasses import asdict
+from datetime import datetime
+from enum import Enum
+from functools import partial
+from typing import TYPE_CHECKING
+
+from shikiru.agents.contracts import (
+    AgentSignal,
+    Compacted,
+    Message,
+    QuotaReported,
+    SessionStarted,
+    ToolFinished,
+    ToolStarted,
+    Turn,
+    TurnEnded,
+    TurnFailed,
+    UsageReported,
+)
+from shikiru.domain.activity import is_stalled, seconds_until_stall
+from shikiru.domain.events import EventKind
+from shikiru.domain.recovery import ProtocolError, Stalled
+from shikiru.private import append_private_text, make_private_dir
+from shikiru.ui.glyphs import GLYPH_STALLED, ROLE_GLYPHS
+from shikiru.ui.tokens import HOLD_COLOR, ROLE_COLORS
+from shikiru.view_log import (
+    VIEW_RUNG_KEY,
+    VIEW_SIGNAL_KEY,
+    VIEW_TIME_KEY,
+    VIEW_TURN_KEY,
+    view_log_path,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Awaitable, Callable
+    from pathlib import Path
+
+    from shikiru.agents.contracts import Agent, Rung
+    from shikiru.clock import Clock
+    from shikiru.config.settings import Settings
+    from shikiru.domain.tasks import Role, Task
+    from shikiru.engine.quota_reconciler import QuotaReconciler
+    from shikiru.engine.sessions import SessionManager
+    from shikiru.engine.turn_processes import TurnProcesses
+    from shikiru.store.events import EventLog
+    from shikiru.store.runs import RunStore
+    from shikiru.tmux.decorator import PaneDecorator
+
+_LINE_ENDING = "\n"
+
+
+class TurnMonitor:
+    """Read a turn's signals once, persist run facts and stop a stalled process."""
+
+    def __init__(
+        self,
+        *,
+        sessions: SessionManager,
+        runs: RunStore,
+        quotas: QuotaReconciler,
+        events: EventLog,
+        clock: Clock,
+        settings: Settings,
+        decorator: PaneDecorator,
+        refresh_titles: Callable[[Task], Awaitable[None]],
+        processes: TurnProcesses,
+    ) -> None:
+        self._sessions = sessions
+        self._runs = runs
+        self._quotas = quotas
+        self._events = events
+        self._clock = clock
+        self._settings = settings
+        self._decorator = decorator
+        self._refresh_titles = refresh_titles
+        self._processes = processes
+
+    def process_recorder(self, run_dir: Path, role: Role) -> Callable[[int], None]:
+        """Return the callback recording each process of `role`'s turn as it is spawned."""
+        return partial(self._processes.record, run_dir, role)
+
+    async def run(
+        self,
+        turn: Turn,
+        *,
+        agent: Agent,
+        task: Task,
+        role: Role,
+        rung: Rung,
+        run_id: int,
+        turn_dir: Path,
+        run_dir: Path,
+    ) -> TurnEnded | TurnFailed | Stalled | ProtocolError:
+        """Consume `turn` through its terminal signal, or stop it when it stalls."""
+        iterator = turn.signals()
+        last_signal_at = self._clock.now()
+        tool_running = False
+        pending_signal = asyncio.create_task(_next_signal(iterator))
+        try:
+            await self._decorator.set_pane_state(
+                task.panes[role], color=ROLE_COLORS[role], glyph=ROLE_GLYPHS[role]
+            )
+            while True:
+                now = self._clock.now()
+                timeouts = self._settings.orchestration.timeouts
+                if is_stalled(last_signal_at, tool_running, now, timeouts):
+                    await turn.stop()
+                    tool_running, pending_signal = await self._drain_stopped_turn(
+                        pending_signal,
+                        iterator,
+                        turn_dir=turn_dir,
+                        run_dir=run_dir,
+                        agent=agent,
+                        task=task,
+                        role=role,
+                        rung=rung,
+                        run_id=run_id,
+                        tool_running=tool_running,
+                    )
+                    await self._decorator.set_pane_state(
+                        task.panes[role], color=HOLD_COLOR, glyph=GLYPH_STALLED
+                    )
+                    return Stalled()
+                poll_timeout = min(
+                    self._settings.orchestration.poll_seconds,
+                    seconds_until_stall(last_signal_at, tool_running, now, timeouts),
+                )
+                signal_ready = await self._wait_for_signal(pending_signal, poll_timeout)
+                if not signal_ready:
+                    continue
+                signal = pending_signal.result()
+                pending_signal = asyncio.create_task(_next_signal(iterator))
+                last_signal_at = self._clock.now()
+                self._append_view(run_dir, turn_dir, role, rung, signal)
+                tool_running, terminal = await self._record_signal(
+                    signal,
+                    agent=agent,
+                    task=task,
+                    role=role,
+                    rung=rung,
+                    run_id=run_id,
+                    run_dir=run_dir,
+                    tool_running=tool_running,
+                )
+                if terminal is not None:
+                    return terminal
+        except StopAsyncIteration:
+            return ProtocolError()
+        except asyncio.CancelledError:
+            await turn.stop()
+            _, pending_signal = await asyncio.shield(
+                self._drain_stopped_turn(
+                    pending_signal,
+                    iterator,
+                    turn_dir=turn_dir,
+                    run_dir=run_dir,
+                    agent=agent,
+                    task=task,
+                    role=role,
+                    rung=rung,
+                    run_id=run_id,
+                    tool_running=tool_running,
+                )
+            )
+            raise
+        except BaseException:
+            await turn.stop()
+            raise
+        finally:
+            if not pending_signal.done():
+                pending_signal.cancel()
+            await asyncio.gather(pending_signal, return_exceptions=True)
+            self._processes.clear(run_dir, role)
+
+    async def _drain_stopped_turn(
+        self,
+        pending_signal: asyncio.Task[AgentSignal],
+        iterator: AsyncIterator[AgentSignal],
+        *,
+        turn_dir: Path,
+        run_dir: Path,
+        agent: Agent,
+        task: Task,
+        role: Role,
+        rung: Rung,
+        run_id: int,
+        tool_running: bool,
+    ) -> tuple[bool, asyncio.Task[AgentSignal]]:
+        try:
+            async with asyncio.timeout(self._settings.orchestration.shutdown_seconds):
+                while True:
+                    try:
+                        signal = await pending_signal
+                    except StopAsyncIteration:
+                        return tool_running, pending_signal
+                    self._append_view(run_dir, turn_dir, role, rung, signal)
+                    tool_running, terminal = await self._record_signal(
+                        signal,
+                        agent=agent,
+                        task=task,
+                        role=role,
+                        rung=rung,
+                        run_id=run_id,
+                        run_dir=run_dir,
+                        tool_running=tool_running,
+                    )
+                    if terminal is not None:
+                        return tool_running, pending_signal
+                    pending_signal = asyncio.create_task(_next_signal(iterator))
+        except TimeoutError:
+            if not pending_signal.done():
+                pending_signal.cancel()
+            await asyncio.gather(pending_signal, return_exceptions=True)
+            return tool_running, pending_signal
+
+    async def _record_signal(
+        self,
+        signal: AgentSignal,
+        *,
+        agent: Agent,
+        task: Task,
+        role: Role,
+        rung: Rung,
+        run_id: int,
+        run_dir: Path,
+        tool_running: bool,
+    ) -> tuple[bool, TurnEnded | TurnFailed | None]:
+        if isinstance(signal, SessionStarted):
+            self._sessions.record_session_started(
+                run_id=run_id,
+                agent=agent,
+                session_id=signal.session_id,
+                run_dir=run_dir,
+                rung=rung,
+            )
+            await self._refresh_titles(task)
+        elif isinstance(signal, Message):
+            self._runs.set_last_message(run_id, signal.text)
+        elif isinstance(signal, UsageReported):
+            self._runs.record_usage(run_id, signal.usage)
+            await self._refresh_titles(task)
+        elif isinstance(signal, ToolStarted):
+            tool_running = True
+        elif isinstance(signal, ToolFinished):
+            tool_running = False
+        elif isinstance(signal, TurnEnded):
+            self._runs.set_last_message(run_id, signal.last_message)
+            return tool_running, signal
+        elif isinstance(signal, TurnFailed):
+            return tool_running, signal
+        else:
+            await self._record_auxiliary_signal(signal, agent=agent, task=task, role=role)
+        return tool_running, None
+
+    async def _record_auxiliary_signal(
+        self, signal: AgentSignal, *, agent: Agent, task: Task, role: Role
+    ) -> None:
+        if isinstance(signal, QuotaReported):
+            self._quotas.observe(agent.name, signal.windows)
+        elif isinstance(signal, Compacted):
+            self._events.append(
+                EventKind.COMPACTED,
+                task_id=task.id,
+                message=f"{task.id} {role.value} compacted context",
+                data={
+                    "role": role.value,
+                    "pre_tokens": signal.pre_tokens,
+                    "post_tokens": signal.post_tokens,
+                },
+            )
+
+    async def _wait_for_signal(
+        self, signal_task: asyncio.Task[AgentSignal], timeout: float
+    ) -> bool:
+        timer_task = asyncio.create_task(self._clock.sleep(timeout))
+        try:
+            done, _ = await asyncio.wait(
+                {signal_task, timer_task}, return_when=asyncio.FIRST_COMPLETED
+            )
+            return signal_task in done
+        finally:
+            if not timer_task.done():
+                timer_task.cancel()
+            await asyncio.gather(timer_task, return_exceptions=True)
+
+    def _append_view(
+        self, run_dir: Path, turn_dir: Path, role: Role, rung: Rung, signal: AgentSignal
+    ) -> None:
+        path = view_log_path(run_dir, role.value)
+        make_private_dir(path.parent)
+        row = {
+            VIEW_TIME_KEY: self._clock.now().isoformat(),
+            VIEW_TURN_KEY: turn_dir.name,
+            VIEW_RUNG_KEY: asdict(rung),
+            VIEW_SIGNAL_KEY: type(signal).__name__,
+            **asdict(signal),
+        }
+        append_private_text(path, json.dumps(row, default=_json_value) + _LINE_ENDING)
+
+
+def _json_value(value: object) -> str:
+    if isinstance(value, datetime):
+        return value.isoformat()
+    if isinstance(value, Enum):
+        return str(value.value)
+    message = f"cannot encode {type(value).__name__} in a turn view"
+    raise TypeError(message)
+
+
+async def _next_signal(signals: AsyncIterator[AgentSignal]) -> AgentSignal:
+    return await signals.__anext__()

shikiru/engine/notification_delivery.py

@@ -0,0 +1,124 @@
+"""Off-loop, prioritised and bounded notification delivery for the engine daemon."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections import deque
+from contextlib import suppress
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+    from shikiru.domain.events import Event
+    from shikiru.engine.notifier import Notifier
+
+INFORMATIONAL_QUEUE_SIZE = 64
+
+
+class NotificationDelivery:
+    """Delivers events on two tasks, so a slow hook never blocks the engine loop or an escalation.
+
+    Urgent events (escalations, engine faults, fault stops) have a delivery task of their own, so
+    a stuck informational hook never delays one by more than the urgent hook in flight; they are
+    never dropped, and only the informational backlog is bounded. Each lane delivers one event at
+    a time. An urgent event leaves its queue only after its delivery, so
+    `oldest_undelivered_urgent_id` covers the one in flight.
+    """
+
+    def __init__(self, notifier: Notifier) -> None:
+        self._notifier = notifier
+        self._urgent: deque[Event] = deque()
+        self._informational: deque[Event] = deque(maxlen=INFORMATIONAL_QUEUE_SIZE)
+        self._urgent_pending = asyncio.Event()
+        self._informational_pending = asyncio.Event()
+        self._urgent_task: asyncio.Task[None] | None = None
+        self._informational_task: asyncio.Task[None] | None = None
+        self._logger = logging.getLogger(__name__)
+
+    def start(self) -> None:
+        """Start the urgent and the informational delivery tasks."""
+        self._urgent_task = asyncio.create_task(self._deliver_urgent())
+        self._informational_task = asyncio.create_task(self._deliver_informational())
+
+    def enqueue(self, event: Event) -> None:
+        """Queue a notifiable `event`; a full informational backlog drops its oldest entry."""
+        if not self._notifier.wants(event):
+            return
+        if self._notifier.is_urgent(event):
+            self._urgent.append(event)
+            self._urgent_pending.set()
+        else:
+            if len(self._informational) == INFORMATIONAL_QUEUE_SIZE:
+                self._logger.warning(
+                    "notification queue full; dropped event %s for event %s",
+                    self._informational[0].id,
+                    event.id,
+                )
+            self._informational.append(event)
+            self._informational_pending.set()
+
+    def replay(self, event: Event) -> None:
+        """Queue `event` after a restart when it is urgent; informational ones are not replayed."""
+        if self._notifier.is_urgent(event):
+            self.enqueue(event)
+
+    @property
+    def oldest_undelivered_urgent_id(self) -> int | None:
+        """Return the id of the oldest urgent event not yet delivered, if any."""
+        return self._urgent[0].id if self._urgent else None
+
+    def supervise(self, logger: Logger) -> None:
+        """Log a delivery task failure and start a replacement task."""
+        if _failed(self._urgent_task, logger):
+            self._urgent_task = asyncio.create_task(self._deliver_urgent())
+        if _failed(self._informational_task, logger):
+            self._informational_task = asyncio.create_task(self._deliver_informational())
+
+    async def shutdown(self) -> None:
+        """Cancel the delivery tasks, terminating running hooks, without draining the queues."""
+        tasks = (self._urgent_task, self._informational_task)
+        self._urgent_task = self._informational_task = None
+        for task in tasks:
+            if task is not None:
+                task.cancel()
+        for task in tasks:
+            if task is not None:
+                with suppress(asyncio.CancelledError):
+                    await task
+
+    async def deliver_final(self, event: Event, timeout_seconds: float) -> None:
+        """Deliver `event` directly once the loop is over, giving up after `timeout_seconds`."""
+        try:
+            await asyncio.wait_for(self._notifier.notify(event), timeout_seconds)
+        except TimeoutError:
+            self._logger.warning("final notification for event %s timed out", event.id)
+
+    async def _deliver_urgent(self) -> None:
+        while True:
+            if self._urgent:
+                await self._notifier.notify(self._urgent[0])
+                self._urgent.popleft()
+            else:
+                self._urgent_pending.clear()
+                await self._urgent_pending.wait()
+
+    async def _deliver_informational(self) -> None:
+        while True:
+            if self._informational:
+                await self._notifier.notify(self._informational.popleft())
+            else:
+                self._informational_pending.clear()
+                await self._informational_pending.wait()
+
+
+def _failed(task: asyncio.Task[None] | None, logger: Logger) -> bool:
+    if task is None or not task.done():
+        return False
+    if not task.cancelled() and (error := task.exception()) is not None:
+        logger.error(
+            "notification delivery failed",
+            exc_info=(type(error), error, error.__traceback__),
+        )
+    return True

shikiru/engine/notification_feed.py

@@ -0,0 +1,57 @@
+"""Feeds new events to notification delivery and keeps the delivered-up-to cursor durable."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from shikiru.engine.notification_delivery import NotificationDelivery
+    from shikiru.store.events import EventLog
+    from shikiru.store.notification_cursor import NotificationCursor
+
+
+class NotificationFeed:
+    """Queues events for delivery and persists the id every urgent event up to is delivered.
+
+    The cursor trails the oldest undelivered urgent event, so a crash or kill replays exactly the
+    urgent events (escalations, engine faults, fault stops) that were not yet delivered.
+    Informational events are never replayed. The cursor is written on the next engine poll after
+    a delivery, so an event delivered in that short window before a kill is delivered again.
+    """
+
+    def __init__(
+        self, events: EventLog, cursor: NotificationCursor, delivery: NotificationDelivery
+    ) -> None:
+        """Read events from `events`, store the cursor in `cursor`, queue into `delivery`."""
+        self._events = events
+        self._cursor = cursor
+        self._delivery = delivery
+        self._scanned = 0
+        self._persisted: int | None = None
+
+    def restore(self) -> None:
+        """Load the cursor and re-queue the urgent events it has not covered.
+
+        The first start of a project has no cursor and begins at the latest event.
+        """
+        self._persisted = self._cursor.read()
+        self._scanned = self._events.latest_id() if self._persisted is None else self._persisted
+        for event in self._events.since(self._scanned):
+            self._scanned = event.id
+            self._delivery.replay(event)
+        self.persist()
+
+    def scan(self) -> None:
+        """Queue the events appended since the last scan and persist the cursor."""
+        for event in self._events.since(self._scanned):
+            self._scanned = event.id
+            self._delivery.enqueue(event)
+        self.persist()
+
+    def persist(self) -> None:
+        """Write the cursor when it moved."""
+        oldest = self._delivery.oldest_undelivered_urgent_id
+        delivered_up_to = self._scanned if oldest is None else oldest - 1
+        if delivered_up_to != self._persisted:
+            self._cursor.write(delivered_up_to)
+            self._persisted = delivered_up_to

shikiru/engine/notifier.py

@@ -0,0 +1,91 @@
+"""tmux and external command notifications."""
+
+from __future__ import annotations
+
+import json
+import logging
+import shlex
+from typing import TYPE_CHECKING
+
+from shikiru.domain.events import STOP_REASON_KEY, Event, EventKind
+
+if TYPE_CHECKING:
+    from shikiru.engine.fault_reporter import FaultReporter
+    from shikiru.process import SubprocessRunner
+    from shikiru.tmux.client import Tmux
+
+_URGENT = frozenset({EventKind.ESCALATION, EventKind.ENGINE_FAULT})
+_NOTIFIABLE = frozenset(
+    {
+        EventKind.ESCALATION,
+        EventKind.TASK_DONE,
+        EventKind.ENGINE_FAULT,
+        EventKind.INTEGRATION_WAITING,
+        EventKind.TAKEOVER_LOST,
+    }
+)
+HOOK_FAULT_SCOPE = "notification hook"
+
+
+class Notifier:
+    """Displays escalation/completion events in tmux and invokes `notify.command`."""
+
+    def __init__(
+        self,
+        tmux: Tmux,
+        runner: SubprocessRunner,
+        reporter: FaultReporter,
+        *,
+        session_target: str,
+        command: str,
+        timeout_seconds: float,
+    ) -> None:
+        """Use `tmux`, `runner`, the project session and an optional argv command.
+
+        A failing hook is recorded through `reporter`, once per streak.
+        """
+        self._tmux = tmux
+        self._runner = runner
+        self._reporter = reporter
+        self._session_target = session_target
+        self._command = command
+        self._timeout_seconds = timeout_seconds
+        self._logger = logging.getLogger(__name__)
+
+    @staticmethod
+    def wants(event: Event) -> bool:
+        """Return whether `event` is an escalation, completion, fault, merge wait or fault stop."""
+        return event.kind in _NOTIFIABLE or (
+            event.kind is EventKind.ENGINE_STOPPED and STOP_REASON_KEY in event.data
+        )
+
+    @staticmethod
+    def is_urgent(event: Event) -> bool:
+        """Return whether `event` is an escalation, an engine fault or a fault stop."""
+        return event.kind in _URGENT or (
+            event.kind is EventKind.ENGINE_STOPPED and STOP_REASON_KEY in event.data
+        )
+
+    async def notify(self, event: Event) -> None:
+        """Deliver `event` when it is an escalation or task completion."""
+        if not self.wants(event):
+            return
+        try:
+            await self._tmux.display_message(self._session_target, event.message)
+        except Exception:
+            self._logger.exception("tmux notification failed for event %s", event.id)
+        if self._command.strip():
+            try:
+                argv = tuple(shlex.split(self._command))
+                payload = json.dumps(event.model_dump(mode="json"))
+                await self._runner.run(
+                    argv,
+                    timeout=self._timeout_seconds,
+                    input_text=payload,
+                    check=True,
+                )
+            except Exception as error:
+                self._logger.exception("external notification failed for event %s", event.id)
+                self._reporter.report(error, scope=HOOK_FAULT_SCOPE)
+            else:
+                self._reporter.resolve(HOOK_FAULT_SCOPE)