shikiru-cli 0.2.0__py3-none-any.whl

shikiru/engine/events.py

@@ -0,0 +1,285 @@
+"""Shared event construction for engine lifecycle boundaries."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from shikiru.agents.contracts import COST_BASIS_UNKNOWN
+from shikiru.domain.events import EventKind
+from shikiru.domain.ladder import pick_rung
+from shikiru.domain.recovery import Failed, ProtocolError, Stalled
+from shikiru.domain.reports import (
+    DECISION_ANSWER,
+    STATUS_COMPLETED,
+    ConsultAnswer,
+    FeedbackAnalysis,
+    ImplementationReport,
+    PlanReport,
+    TriageReport,
+    Verdict,
+)
+from shikiru.domain.steps import ACTING_ROLE
+
+if TYPE_CHECKING:
+    from shikiru.agents.contracts import Rung, Usage
+    from shikiru.config.settings import Settings
+    from shikiru.domain.recovery import TurnResult
+    from shikiru.domain.reports import ReportKind
+    from shikiru.domain.steps import Outcome
+    from shikiru.domain.tasks import Role, Step, Task
+    from shikiru.engine.turn_execution import TurnExecution
+    from shikiru.store.events import EventLog
+
+TURN_FIELD = "turn"
+ROLE_FIELD = "role"
+ITERATION_FIELD = "iteration"
+REPORT_KIND_FIELD = "report_kind"
+
+
+def emit_step_started(events: EventLog, task: Task, settings: Settings) -> None:
+    """Record the selected role and rung when a task step starts."""
+    if task.step is None:
+        return
+    role = ACTING_ROLE[task.step]
+    rung = pick_rung(task, role, settings)
+    events.append(
+        EventKind.STEP_STARTED,
+        task_id=task.id,
+        message=f"{task.id} {task.step.value} started",
+        data={
+            "step": task.step.value,
+            "role": role.value,
+            "iteration": task.iteration,
+            "agent": rung.agent,
+            "model": rung.model,
+        },
+    )
+
+
+def emit_turn_started(
+    events: EventLog,
+    task: Task,
+    step: Step,
+    role: Role,
+    rung: Rung,
+    *,
+    kind: ReportKind,
+    run_id: int | None = None,
+    turn: str | None = None,
+) -> None:
+    """Record that `role` started its `step` turn on `rung`, owing a report of `kind`."""
+    events.append(
+        EventKind.TURN_STARTED,
+        task_id=task.id,
+        message=f"{task.id} {role.value} turn started",
+        data={
+            "step": step.value,
+            "role": role.value,
+            "iteration": task.iteration,
+            "agent": rung.agent,
+            "model": rung.model,
+            "effort": rung.effort,
+            REPORT_KIND_FIELD: kind.value,
+            **_turn_data(turn),
+            **({"run_id": run_id} if run_id is not None else {}),
+        },
+    )
+
+
+def emit_turn_stopped(
+    events: EventLog,
+    task: Task,
+    step: Step,
+    role: Role,
+    *,
+    rung: Rung,
+    run_id: int,
+    turn: str | None = None,
+) -> None:
+    """Record that `role`'s `step` turn ended without a result because the task was stopped."""
+    events.append(
+        EventKind.TURN_STOPPED,
+        task_id=task.id,
+        message=f"{task.id} {role.value} turn stopped",
+        data={
+            "step": step.value,
+            "role": role.value,
+            "iteration": task.iteration,
+            "agent": rung.agent,
+            "model": rung.model,
+            "effort": rung.effort,
+            "run_id": run_id,
+            **_turn_data(turn),
+        },
+    )
+
+
+def emit_nudge(
+    events: EventLog, task: Task, step: Step, role: Role, *, turn: str | None = None
+) -> None:
+    """Record that the engine nudged `role`'s stalled `step` turn."""
+    events.append(
+        EventKind.NUDGE,
+        task_id=task.id,
+        message=f"{task.id} {role.value} nudge",
+        data={"step": step.value, "role": role.value, **_turn_data(turn)},
+    )
+
+
+def emit_turn_failed(
+    events: EventLog,
+    task: Task,
+    step: Step,
+    result: TurnResult,
+    *,
+    rung: Rung,
+    run_id: int,
+    turn: str | None = None,
+    role: Role | None = None,
+) -> None:
+    """Record a turn that ended without a valid report."""
+    if isinstance(result, Failed):
+        message = result.message
+        data: dict[str, object] = {"kind": result.kind.value, "resets_at": result.resets_at}
+    elif isinstance(result, Stalled):
+        message = "agent stalled"
+        data = {}
+    elif isinstance(result, ProtocolError):
+        message = "agent protocol error"
+        data = {}
+    else:
+        message = type(result).__name__
+        data = {}
+    data.update(
+        {
+            "step": step.value,
+            "result": type(result).__name__,
+            **_provenance_data(role, task.iteration),
+            "agent": rung.agent,
+            "model": rung.model,
+            "effort": rung.effort,
+            "run_id": run_id,
+            **_turn_data(turn),
+        }
+    )
+    events.append(EventKind.TURN_FAILED, task_id=task.id, message=message, data=data)
+
+
+def emit_turn_completed(
+    events: EventLog, task: Task, outcome: Outcome, execution: TurnExecution
+) -> None:
+    """Record a completed turn and, for a verdict, the verdict itself."""
+    summary = _turn_summary(outcome)
+    events.append(
+        EventKind.TURN_COMPLETED,
+        task_id=task.id,
+        message=summary,
+        data={
+            "step": task.step.value if task.step is not None else None,
+            "iteration": task.iteration,
+            "role": execution.role.value,
+            "agent": execution.rung.agent,
+            "model": execution.rung.model,
+            "effort": execution.rung.effort,
+            "run_id": execution.run_id,
+            REPORT_KIND_FIELD: execution.kind.value,
+            "outcome": _turn_outcome(outcome),
+            "summary": summary,
+            "question": _turn_question(outcome),
+            "usage_delta": _usage_delta(execution.usage_before, execution.usage_after),
+            **_turn_data(execution.turn_dir.name),
+        },
+    )
+    if isinstance(outcome, Verdict):
+        emit_verdict(
+            events,
+            task,
+            outcome,
+            summary,
+            turn=execution.turn_dir.name,
+            role=execution.role,
+        )
+
+
+def emit_verdict(
+    events: EventLog,
+    task: Task,
+    verdict: Verdict,
+    summary: str,
+    *,
+    turn: str | None = None,
+    role: Role | None = None,
+) -> None:
+    """Record the Leader's (or the engine's) verdict on a task."""
+    events.append(
+        EventKind.VERDICT,
+        task_id=task.id,
+        message=summary,
+        data={
+            "decision": verdict.decision,
+            "summary": summary,
+            "issues": [issue.model_dump() for issue in verdict.issues],
+            "residual_risks": list(verdict.residual_risks),
+            **_provenance_data(role, task.iteration),
+            **_turn_data(turn),
+        },
+    )
+
+
+def _turn_data(turn: str | None) -> dict[str, str]:
+    return {TURN_FIELD: turn} if turn is not None else {}
+
+
+def _provenance_data(role: Role | None, iteration: int) -> dict[str, str | int]:
+    data: dict[str, str | int] = {ITERATION_FIELD: iteration}
+    if role is not None:
+        data[ROLE_FIELD] = role.value
+    return data
+
+
+def _turn_summary(outcome: Outcome) -> str:
+    if isinstance(outcome, TriageReport):
+        return outcome.title
+    if isinstance(outcome, ConsultAnswer):
+        return outcome.answer if outcome.decision == DECISION_ANSWER else "escalate"
+    return outcome.summary
+
+
+def _turn_question(outcome: Outcome) -> str:
+    return outcome.question if isinstance(outcome, ImplementationReport) else ""
+
+
+def _turn_outcome(outcome: Outcome) -> str:
+    if isinstance(outcome, TriageReport):
+        return outcome.mode.value
+    if isinstance(outcome, PlanReport | FeedbackAnalysis):
+        return STATUS_COMPLETED
+    if isinstance(outcome, ConsultAnswer):
+        return outcome.decision
+    if isinstance(outcome, ImplementationReport):
+        return outcome.status
+    return outcome.decision
+
+
+def _usage_delta(before: Usage, after: Usage) -> dict[str, int | float | str | None]:
+    cost_delta, cost_basis = _comparable_cost_delta(before, after)
+    return {
+        "input_tokens": after.input_tokens - before.input_tokens,
+        "output_tokens": after.output_tokens - before.output_tokens,
+        "cache_read_tokens": after.cache_read_tokens - before.cache_read_tokens,
+        "cache_write_tokens": after.cache_write_tokens - before.cache_write_tokens,
+        "cost_usd": cost_delta,
+        "cost_basis": cost_basis,
+    }
+
+
+def _comparable_cost_delta(before: Usage, after: Usage) -> tuple[float | None, str]:
+    if (
+        before.cost_usd is None
+        or after.cost_usd is None
+        or before.cost_basis != after.cost_basis
+        or after.cost_basis == COST_BASIS_UNKNOWN
+        or after.cost_usd < before.cost_usd
+    ):
+        return None, COST_BASIS_UNKNOWN
+    return after.cost_usd - before.cost_usd, after.cost_basis

shikiru/engine/evidence.py

@@ -0,0 +1,81 @@
+"""Gathers the evidence of a task escalation: reports, workspace facts and the turn tail."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from shikiru.domain.steps import ACTING_ROLE
+from shikiru.errors import ShikiruError
+from shikiru.paths import run_dir
+from shikiru.view_log import read_tail_lines, view_log_path
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from shikiru.domain.tasks import Role, Step, Task
+    from shikiru.engine.history import ReportHistory
+    from shikiru.engine.task_facts import TaskFacts
+
+_LOGGER = logging.getLogger(__name__)
+
+ESCALATION_PANE_TAIL_LINES = 60
+"""Lines of pane scrollback captured as escalation evidence."""
+
+FACTS_UNAVAILABLE = "(facts unavailable: {error})"
+HISTORY_UNAVAILABLE = "(report history unavailable, so no reports are shown: {error})"
+
+
+@dataclass(frozen=True)
+class Evidence:
+    """The recorded reports, rendered facts and recent turn signals of one task."""
+
+    reports: list[dict[str, str]]
+    facts: str
+    view_tail: str
+
+
+class EvidenceCollector:
+    """Collects the evidence that still exists for a task from its files and git."""
+
+    def __init__(self, *, history: ReportHistory, facts: TaskFacts, project_root: Path) -> None:
+        """Read reports through `history`, facts through `facts` and turn tails under the root."""
+        self._history = history
+        self._facts = facts
+        self._project_root = project_root
+
+    async def collect(self, task: Task, step: Step | None, role: Role | None = None) -> Evidence:
+        """Gather `task`'s evidence; the turn tail is that of `role` or of `step`'s acting role."""
+        history_note = ""
+        try:
+            history = self._history.load(task.id)
+        except (OSError, ValueError) as error:
+            _LOGGER.exception("could not load report history for escalation of %s", task.id)
+            history = []
+            history_note = HISTORY_UNAVAILABLE.format(error=error)
+        facts = await self._workspace_facts(task)
+        return Evidence(
+            reports=self._history.context(history),
+            facts="\n\n".join(part for part in (history_note, facts) if part),
+            view_tail=self._view_tail(task, step, role),
+        )
+
+    async def _workspace_facts(self, task: Task) -> str:
+        if task.repo is None:
+            return ""
+        try:
+            return await self._facts.text(task)
+        except ShikiruError as error:
+            return FACTS_UNAVAILABLE.format(error=error)
+
+    def _view_tail(self, task: Task, step: Step | None, role_override: Role | None) -> str:
+        role = role_override
+        if role is None and step is not None:
+            role = ACTING_ROLE[step]
+        if role is None:
+            return ""
+        path = view_log_path(run_dir(self._project_root, task.id), role.value)
+        if not path.is_file():
+            return ""
+        return "\n".join(read_tail_lines(path, ESCALATION_PANE_TAIL_LINES))

shikiru/engine/fault_reporter.py

@@ -0,0 +1,144 @@
+"""Records engine faults and their recovery as `engine_fault` and `engine_recovered` events."""
+
+from __future__ import annotations
+
+import traceback
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import TYPE_CHECKING, Any
+
+from shikiru.domain.events import EventKind
+from shikiru.domain.faults import (
+    COMMAND_KEY,
+    DETAIL_KEY,
+    ERROR_KEY,
+    EXIT_CODE_KEY,
+    RECOVERABLE_KEY,
+    REPEAT_KEY,
+    SCOPE_KEY,
+    STDERR_KEY,
+    TRACEBACK_KEY,
+)
+from shikiru.errors import CommandFailed, CommandTimeoutError, InvariantError, ShikiruError
+from shikiru.store.database import OperationalDatabaseError
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator
+    from datetime import datetime
+    from logging import Logger
+
+    from shikiru.clock import Clock
+    from shikiru.config.settings import FaultsConfig
+    from shikiru.store.events import EventLog
+
+RECOVERABLE_FAULTS = (ShikiruError, OSError, OperationalDatabaseError)
+"""Environmental faults (a failed tmux, git, file or database call), retried while they last."""
+
+
+def is_recoverable(error: Exception) -> bool:
+    """Return whether `error` is environmental; an `InvariantError` is a logic bug, never one."""
+    return isinstance(error, RECOVERABLE_FAULTS) and not isinstance(error, InvariantError)
+
+
+def _evidence(error: Exception) -> dict[str, Any]:
+    evidence: dict[str, Any] = {
+        DETAIL_KEY: str(error),
+        TRACEBACK_KEY: "".join(traceback.format_exception(error)),
+    }
+    if isinstance(error, CommandFailed | CommandTimeoutError):
+        evidence[COMMAND_KEY] = list(error.argv)
+    if isinstance(error, CommandFailed):
+        evidence[EXIT_CODE_KEY] = error.returncode
+        evidence[STDERR_KEY] = error.stderr
+    return evidence
+
+
+@dataclass
+class _Streak:
+    recorded_at: datetime | None = None
+    occurrences: int = 0
+
+
+class FaultReporter:
+    """Logs every fault and records one `engine_fault` event per scope, task and error class.
+
+    A repeat of the same fault is counted and recorded again only after
+    `repeat_notify_seconds`, so a persistent or flapping fault stays visible without flooding
+    the log or the Operator, and a new error class is recorded at once. Each record carries the
+    full error, its traceback and, for a failed command, the command, exit code and stderr.
+    """
+
+    def __init__(
+        self, *, events: EventLog, clock: Clock, settings: FaultsConfig, logger: Logger
+    ) -> None:
+        """Record in `events`, rate-limited with `clock`."""
+        self._events = events
+        self._clock = clock
+        self._settings = settings
+        self._logger = logger
+        self._streaks: dict[tuple[str, str | None, str], _Streak] = {}
+        self._open: set[tuple[str, str | None]] = set()
+
+    def report(self, error: Exception, *, scope: str, task_id: str | None = None) -> bool:
+        """Record `error` raised in `scope` unless just recorded, and classify it.
+
+        Returns whether it is a recoverable fault.
+        """
+        recoverable = isinstance(error, RECOVERABLE_FAULTS) and not isinstance(
+            error, InvariantError
+        )
+        kind = type(error).__name__
+        now = self._clock.now()
+        key = (scope, task_id, kind)
+        streak = self._streaks.setdefault(key, _Streak())
+        streak.occurrences += 1
+        interval = timedelta(seconds=self._settings.repeat_notify_seconds)
+        if streak.recorded_at is None or now - streak.recorded_at >= interval:
+            try:
+                self._events.append(
+                    EventKind.ENGINE_FAULT,
+                    task_id=task_id,
+                    message=f"{scope} failed: {kind}: {error}",
+                    data={
+                        SCOPE_KEY: scope,
+                        ERROR_KEY: kind,
+                        RECOVERABLE_KEY: recoverable,
+                        REPEAT_KEY: streak.occurrences,
+                        **_evidence(error),
+                    },
+                )
+            except OperationalDatabaseError:
+                self._logger.exception("could not record the fault of %s", scope)
+                return recoverable
+            streak.recorded_at = now
+            streak.occurrences = 0
+            self._open.add((scope, task_id))
+        return recoverable
+
+    def resolve(self, scope: str, *, task_id: str | None = None) -> None:
+        """Record an `engine_recovered` event when `scope` failed, was recorded, and now works."""
+        if (scope, task_id) not in self._open:
+            return
+        try:
+            self._events.append(
+                EventKind.ENGINE_RECOVERED,
+                task_id=task_id,
+                message=f"{scope} recovered",
+                data={SCOPE_KEY: scope},
+            )
+        except OperationalDatabaseError:
+            self._logger.exception("could not record the recovery of %s", scope)
+            return
+        self._open.remove((scope, task_id))
+
+    @asynccontextmanager
+    async def isolate(self, scope: str, *, task_id: str | None = None) -> AsyncGenerator[None]:
+        """Absorb any fault of the block: log it with its traceback and record it for `task_id`."""
+        try:
+            yield
+        except Exception as error:
+            self._logger.exception("%s failed", scope)
+            self.report(error, scope=scope, task_id=task_id)
+        else:
+            self.resolve(scope, task_id=task_id)