shikiru-cli 0.2.0__py3-none-any.whl

shikiru/engine/supervisor.py

@@ -0,0 +1,101 @@
+"""Supervise concurrent task flows for the engine daemon."""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import suppress
+from typing import TYPE_CHECKING
+
+from shikiru.domain.tasks import Status, StatusReason
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+    from shikiru.engine.flow import TaskFlow
+    from shikiru.engine.lifecycle import TaskFinisher
+    from shikiru.store.tasks import TaskStore
+
+_IN_FLIGHT = (Status.RUNNING, Status.INTEGRATING)
+
+
+class TaskSupervisor:
+    """Start, interrupt and reap the task flows owned by one engine daemon."""
+
+    def __init__(self, tasks: TaskStore, flow: TaskFlow, finisher: TaskFinisher) -> None:
+        self._tasks = tasks
+        self._flow = flow
+        self._finisher = finisher
+        self._stop = asyncio.Event()
+        self._flow_tasks: dict[str, asyncio.Task[None]] = {}
+
+    @property
+    def stop_requested(self) -> bool:
+        return self._stop.is_set()
+
+    def request_stop(self) -> None:
+        """Ask the daemon command loop to stop after acknowledging the current command."""
+        self._stop.set()
+
+    async def start_task(self, task_id: str) -> None:
+        """Schedule a task flow unless one is still live; `ensure_flows` covers a finishing one."""
+        self._start(task_id, resumed=False)
+
+    async def resume_task(self, task_id: str) -> None:
+        """Schedule the flow of a task whose step an engine restart interrupted."""
+        self._start(task_id, resumed=True)
+
+    async def ensure_flows(self) -> None:
+        """Start a resumed flow for each unheld running or integrating task without one."""
+        for task in self._tasks.list_active():
+            if task.status in _IN_FLIGHT and not task.held:
+                self._start(task.id, resumed=True)
+
+    def _start(self, task_id: str, *, resumed: bool) -> None:
+        existing = self._flow_tasks.get(task_id)
+        if existing is not None and not existing.done():
+            return
+        self._flow_tasks[task_id] = asyncio.create_task(self._run_flow(task_id, resumed=resumed))
+
+    async def cancel_task(self, task_id: str) -> None:
+        """Interrupt and await one supervised flow coroutine, if present."""
+        current = self._flow_tasks.get(task_id)
+        if current is None:
+            return
+        current.cancel()
+        with suppress(asyncio.CancelledError):
+            await current
+        self._flow_tasks.pop(task_id, None)
+
+    async def supervise(self, logger: Logger) -> None:
+        """Log task flow failures and remove completed flows from the active set."""
+        for task_id, flow_task in tuple(self._flow_tasks.items()):
+            if not flow_task.done():
+                continue
+            self._flow_tasks.pop(task_id, None)
+            if flow_task.cancelled():
+                continue
+            error = flow_task.exception()
+            if error is not None:
+                logger.error(
+                    "task flow %s failed",
+                    task_id,
+                    exc_info=(type(error), error, error.__traceback__),
+                )
+
+    async def shutdown(self) -> None:
+        """Cancel and await all task flows before the daemon closes its resources."""
+        for task in tuple(self._flow_tasks.values()):
+            task.cancel()
+        if self._flow_tasks:
+            await asyncio.gather(*self._flow_tasks.values(), return_exceptions=True)
+        self._flow_tasks.clear()
+
+    async def _run_flow(self, task_id: str, *, resumed: bool) -> None:
+        try:
+            await self._flow.run(task_id, resumed=resumed)
+        finally:
+            current = self._tasks.get(task_id)
+            if current is not None and (
+                current.status is Status.DONE or current.status_reason is StatusReason.SUPERSEDED
+            ):
+                await self._finisher.finish_task(current)

shikiru/engine/takeover.py

@@ -0,0 +1,223 @@
+"""Operator take over and hand back of one task role's session."""
+
+from __future__ import annotations
+
+import shlex
+from typing import TYPE_CHECKING, NoReturn
+
+from shikiru.agents.contracts import Rung
+from shikiru.domain.commands import COMMAND_ID_KEY
+from shikiru.domain.events import EventKind
+from shikiru.domain.ladder import pick_rung
+from shikiru.domain.tasks import Hold, Role, Status, Task
+from shikiru.engine.turns import session_environment
+from shikiru.errors import PROCESS_FAILURES, CommandRejectedError
+from shikiru.paths import run_dir, task_root
+from shikiru.tmux.client import FORMAT_PANE_PID
+from shikiru.workspace.messages import command_message_key
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+    from contextlib import AbstractContextManager
+    from pathlib import Path
+
+    from shikiru.config.settings import Settings
+    from shikiru.domain.runs import Run
+    from shikiru.engine.agents import AgentPool
+    from shikiru.engine.layout_guard import LayoutGuard
+    from shikiru.store.events import EventLog
+    from shikiru.store.runs import RunStore
+    from shikiru.store.tasks import TaskStore
+    from shikiru.tmux.client import Tmux
+    from shikiru.workspace.messages import OperatorMessages
+
+HAND_BACK_MESSAGE = (
+    "The Operator worked in this session since your last turn; take their changes into account."
+)
+_ACTIVE_STATUSES = (Status.RUNNING, Status.INTEGRATING)
+TAKEOVER_PANE_OPTION = "@shk_takeover"
+_FORMAT_TAKEOVER = f"#{{{TAKEOVER_PANE_OPTION}}}"
+
+
+class Takeover:
+    """Moves a role's session between the engine's headless turns and the Operator's pane."""
+
+    def __init__(
+        self,
+        *,
+        tmux: Tmux,
+        layout: LayoutGuard,
+        agents: AgentPool,
+        tasks: TaskStore,
+        runs: RunStore,
+        messages: OperatorMessages,
+        events: EventLog,
+        project_root: Path,
+        project_name: str,
+        settings: Settings,
+        cancel_task: Callable[[str], Awaitable[None]],
+        start_task: Callable[[str], Awaitable[None]],
+        refresh_titles: Callable[[Task], Awaitable[None]],
+        atomically: Callable[[], AbstractContextManager[object]],
+    ) -> None:
+        """Drive panes through `tmux`/`layout` and task flows through the two callbacks."""
+        self._tmux = tmux
+        self._layout = layout
+        self._agents = agents
+        self._tasks = tasks
+        self._runs = runs
+        self._messages = messages
+        self._events = events
+        self._project_root = project_root
+        self._project_name = project_name
+        self._settings = settings
+        self._cancel_task = cancel_task
+        self._start_task = start_task
+        self._refresh_titles = refresh_titles
+        self._atomically = atomically
+
+    async def take_over(self, task: Task, role: Role, command_id: int) -> None:
+        """Stop `task`'s turn, hold it and run `role`'s session interactively in its pane.
+
+        The run mark, hold and event (naming `command_id`) commit together before the pane is
+        respawned; a replay after that commit skips them and only respawns the pane, which is
+        repeatable, while the role is still taken over.
+        """
+        if not self._events.command_applied(EventKind.TAKEN_OVER, command_id):
+            if self._runs.get_taken_over(task.id, role) is not None:
+                message = f"{task.id} {role.value} is already taken over"
+                raise CommandRejectedError(message)
+            run = self._runs.get_live(task.id, role)
+            if run is None or run.session_id is None:
+                message = f"{task.id} has no {role.value} session to take over"
+                raise CommandRejectedError(message)
+            _role_pane(task, role)
+            if task.status in _ACTIVE_STATUSES:
+                await self._cancel_task(task.id)
+            with self._atomically():
+                self._runs.mark_taken_over(run.id)
+                self._tasks.set_holds(task.id, self._current(task.id).holds | {Hold.INTERVENTION})
+                self._events.append(
+                    EventKind.TAKEN_OVER,
+                    task_id=task.id,
+                    message=f"{task.id} {role.value} taken over",
+                    data={
+                        "role": role.value,
+                        "session_id": run.session_id,
+                        COMMAND_ID_KEY: command_id,
+                    },
+                )
+        owned = self._runs.get_taken_over(task.id, role)
+        if owned is None or owned.session_id is None:
+            message = f"{task.id} {role.value} take over did not hold"
+            raise CommandRejectedError(message)
+        try:
+            await self._ensure_interactive(
+                self._current(task.id), role, owned, owned.session_id, command_id
+            )
+        except PROCESS_FAILURES as error:
+            await self._release_unstarted(task, role, owned.id, error)
+        await self._refresh_titles(self._current(task.id))
+
+    async def _ensure_interactive(
+        self, task: Task, role: Role, run: Run, session_id: str, command_id: int
+    ) -> None:
+        pane = _role_pane(task, role)
+        if await self._interactive_is_live(pane, command_id):
+            return
+        configured_rung = pick_rung(task, role, self._settings)
+        rung = (
+            configured_rung
+            if configured_rung.agent == run.agent
+            else Rung(agent=run.agent, model=run.model, effort=run.effort)
+        )
+        root = task_root(self._project_root, task.id)
+        task_run_dir = run_dir(self._project_root, task.id)
+        argv = self._agents.get(run.agent).interactive_command(session_id, task_run_dir, rung)
+        env = session_environment(task, role, task_run_dir, self._project_name)
+        await self._tmux.respawn_pane(pane, argv, env=env, cwd=root)
+        pid = await self._tmux.display(pane, FORMAT_PANE_PID)
+        with self._atomically():
+            if (run.model, run.effort) != (rung.model, rung.effort):
+                self._runs.set_rung(run.id, rung)
+            self._runs.set_session(run.id, session_id, shlex.join(argv))
+        await self._tmux.set_pane_option(pane, TAKEOVER_PANE_OPTION, _live_marker(command_id, pid))
+
+    async def _interactive_is_live(self, pane: str, command_id: int) -> bool:
+        marker = await self._tmux.display(pane, _FORMAT_TAKEOVER)
+        pid = await self._tmux.display(pane, FORMAT_PANE_PID)
+        return marker == _live_marker(command_id, pid) and not await self._tmux.is_pane_dead(pane)
+
+    async def _release_unstarted(
+        self, task: Task, role: Role, run_id: int, error: Exception
+    ) -> NoReturn:
+        with self._atomically():
+            self._runs.mark_live(run_id)
+            if not self._runs.has_taken_over(task.id):
+                holds = self._current(task.id).holds - {Hold.INTERVENTION}
+                self._tasks.set_holds(task.id, holds)
+            self._events.append(
+                EventKind.TAKEOVER_LOST,
+                task_id=task.id,
+                message=f"{task.id} {role.value} take over released: its pane could not be started",
+                data={"role": role.value},
+            )
+        current = self._current(task.id)
+        if current.status in _ACTIVE_STATUSES and not current.held:
+            await self._start_task(task.id)
+        await self._refresh_titles(current)
+        message = (
+            f"{task.id} {role.value} was not taken over: its pane could not be started ({error}); "
+            "nothing is held and the task continues, take over again to retry"
+        )
+        raise CommandRejectedError(message) from error
+
+    async def hand_back(self, task: Task, role: Role, command_id: int) -> None:
+        """Return `role`'s taken-over session to the engine and re-run the current step.
+
+        The run, hold and event changes commit together, and the event names `command_id`; a
+        replay after that commit skips them and only repeats the follow-up.
+        """
+        if not self._events.command_applied(EventKind.HANDED_BACK, command_id):
+            run = self._runs.get_taken_over(task.id, role)
+            if run is None:
+                message = f"{task.id} {role.value} is not taken over"
+                raise CommandRejectedError(message)
+            await self._layout.start_view(task.id, role, _role_pane(task, role))
+            path = self._messages.queue(
+                task.id, role, HAND_BACK_MESSAGE, command_message_key(command_id)
+            )
+            with self._atomically():
+                self._runs.mark_live(run.id)
+                if not self._runs.has_taken_over(task.id):
+                    holds = self._current(task.id).holds - {Hold.INTERVENTION}
+                    self._tasks.set_holds(task.id, holds)
+                self._events.append(
+                    EventKind.HANDED_BACK,
+                    task_id=task.id,
+                    message=f"{task.id} {role.value} handed back",
+                    data={"role": role.value, "path": str(path), COMMAND_ID_KEY: command_id},
+                )
+        current = self._current(task.id)
+        if current.status in _ACTIVE_STATUSES and not current.held:
+            await self._start_task(task.id)
+        await self._refresh_titles(self._current(task.id))
+
+    def _current(self, task_id: str) -> Task:
+        task = self._tasks.get(task_id)
+        if task is None:
+            message = f"task {task_id} disappeared during take over"
+            raise CommandRejectedError(message)
+        return task
+
+
+def _live_marker(command_id: int, pid: str) -> str:
+    return f"{command_id}:{pid}"
+
+
+def _role_pane(task: Task, role: Role) -> str:
+    pane_id = task.panes.get(role)
+    if pane_id is None:
+        message = f"{task.id} has no {role.value} pane"
+        raise CommandRejectedError(message)
+    return pane_id

shikiru/engine/task_facts.py

@@ -0,0 +1,46 @@
+"""Git facts of a task's worktree, shared by prompt, evidence and acceptance code."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from shikiru.errors import InvariantError
+from shikiru.paths import task_root
+from shikiru.workspace.facts import render_facts
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from shikiru.config.settings import Settings
+    from shikiru.domain.tasks import RepoState, Task
+    from shikiru.workspace.facts import GitFacts, WorkspaceFacts
+
+
+def require_repo(task: Task) -> RepoState:
+    """Return the task's repo state or raise when its task root does not exist yet."""
+    if task.repo is None:
+        message = f"{task.id} has no task root yet"
+        raise InvariantError(message)
+    return task.repo
+
+
+class TaskFacts:
+    """Collects and renders the facts of one task's worktree."""
+
+    def __init__(self, *, git_facts: GitFacts, project_root: Path, settings: Settings) -> None:
+        """Collect facts from `git_facts` for worktrees under `project_root`."""
+        self._git_facts = git_facts
+        self._project_root = project_root
+        self._settings = settings
+
+    async def collect(self, task: Task) -> WorkspaceFacts:
+        """Collect the raw facts of `task`'s worktree."""
+        return await self._git_facts.collect(
+            task_root(self._project_root, task.id),
+            require_repo(task),
+            self._settings.project.trackers,
+        )
+
+    async def text(self, task: Task) -> str:
+        """Render the facts of `task`'s worktree as prompt markdown."""
+        return render_facts(await self.collect(task))

shikiru/engine/tick_guard.py

@@ -0,0 +1,95 @@
+"""The fault barrier around one engine tick or command pass."""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import TYPE_CHECKING
+
+from shikiru.errors import ShikiruError
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+    from datetime import datetime
+    from logging import Logger
+
+    from shikiru.clock import Clock
+    from shikiru.config.settings import FaultsConfig
+    from shikiru.engine.fault_reporter import FaultReporter
+
+_BACKOFF_FACTOR = 2
+
+
+class EngineFaultLimitError(ShikiruError):
+    """Unexpected (programming) faults repeated on every tick up to the configured limit."""
+
+
+class TickGuard:
+    """Runs a tick, retrying it with bounded backoff after a fault instead of ending the engine.
+
+    Recoverable faults (`fault_reporter.RECOVERABLE_FAULTS`) are retried without limit. Any other
+    exception is a programming error: it is recorded like the rest, and `unexpected_fault_limit`
+    of them in a row end the engine with `EngineFaultLimitError`.
+    """
+
+    def __init__(
+        self,
+        *,
+        reporter: FaultReporter,
+        clock: Clock,
+        settings: FaultsConfig,
+        logger: Logger,
+        scope: str,
+    ) -> None:
+        """Record faults through `reporter` under `scope` and pace retries with `clock`."""
+        self._reporter = reporter
+        self._clock = clock
+        self._settings = settings
+        self._logger = logger
+        self._scope = scope
+        self._streak = 0
+        self._unexpected = 0
+        self._backoff = 0.0
+        self._retry_at: datetime | None = None
+
+    async def attempt[T](self, tick: Callable[[], Awaitable[T]]) -> T | None:
+        """Run `tick` unless a backoff is pending, absorbing its fault.
+
+        Returns what `tick` returned, or `None` when it was skipped or failed.
+        """
+        if self._retry_at is not None and self._clock.now() < self._retry_at:
+            return None
+        try:
+            result = await tick()
+        except Exception as error:
+            self._logger.exception("%s failed", self._scope)
+            self._record(error)
+            return None
+        self._recover()
+        return result
+
+    def _record(self, error: Exception) -> None:
+        recoverable = self._reporter.report(error, scope=self._scope)
+        self._streak += 1
+        self._unexpected = 0 if recoverable else self._unexpected + 1
+        grown = (
+            self._settings.backoff_initial_seconds
+            if self._streak == 1
+            else self._backoff * _BACKOFF_FACTOR
+        )
+        self._backoff = min(grown, self._settings.backoff_max_seconds)
+        self._retry_at = self._clock.now() + timedelta(seconds=self._backoff)
+        self._logger.warning(
+            "%s attempt %d failed; retry in %.0fs", self._scope, self._streak, self._backoff
+        )
+        if self._unexpected >= self._settings.unexpected_fault_limit:
+            kind = type(error).__name__
+            message = f"{self._unexpected} consecutive unexpected faults, last {kind}: {error}"
+            raise EngineFaultLimitError(message) from error
+
+    def _recover(self) -> None:
+        self._reporter.resolve(self._scope)
+        if self._streak:
+            self._logger.info("%s recovered after %d failed attempts", self._scope, self._streak)
+        self._streak = 0
+        self._unexpected = 0
+        self._retry_at = None