caudate-cli 0.1.0__py3-none-any.whl

planning/orchestrator.py

@@ -0,0 +1,1363 @@
+"""Forge orchestrator — pick the next ready feature, run an agent on it.
+
+Python port of LocalForge's lib/agent/orchestrator.ts (1099 lines TS),
+adapted to Cognos's idioms:
+  - In-process asyncio instead of detached child processes (LocalForge
+    uses Node's spawn(); we keep Caudate observation, permissions,
+    hooks, and LLM connection pooling shared with the parent).
+  - SQLAlchemy state instead of Drizzle (planning/forge_models.py).
+  - asyncio.Event broadcasting instead of EventEmitter (subscribers get
+    an async generator they can iterate over for SSE).
+
+Lifecycle (per session):
+  start_orchestrator(project_id)
+    → pick highest-priority ready feature whose deps are completed
+    → flip it to in_progress, create forge_sessions row
+    → spawn an asyncio task running CognosAgent on the feature task
+    → emit log+status events as work progresses
+  finalize_session(session_id, outcome)
+    → success: feature → completed; auto-continue with next ready
+    → failure: feature → backlog with priority demoted (-1)
+    → terminated: feature → backlog (no demotion, user stopped manually)
+
+Concurrency:
+  Per-project max set by `forge.max_concurrent` setting (default 3).
+  start_orchestrator returns 409 if the cap is hit. Auto-continue fills
+  empty slots after each finalize.
+
+Watchdog:
+  Each session has a deadline (`forge.session_timeout_s` setting,
+  default 30 min). When it fires, the asyncio task is cancelled and
+  the session finalises as `terminated`.
+
+Reconciliation:
+  On startup, any sessions whose row says in_progress but isn't in
+  the running map are reaped to `terminated`, and their features
+  reset to backlog. This handles cognos-serve restarts.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any, AsyncIterator, Callable
+
+from planning.forge_models import (
+    ForgeFeature,
+    ForgeFeatureDep,
+    ForgeLog,
+    ForgeProject,
+    ForgeSession,
+    ForgeSetting,
+    init,
+    session_scope,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ───────────────────────── Configuration knobs ────────────────────────
+
+
+def _get_forge_setting(key: str, default: str) -> str:
+    """Read a forge setting from the existing core/settings.py hierarchy.
+    Settings live under the `forge.<key>` namespace so they don't
+    collide with top-level cognos settings."""
+    try:
+        from core.settings import Settings
+        s = Settings.load()
+        forge_block = s.get("forge") or {}
+        return str(forge_block.get(key, default))
+    except Exception:
+        return default
+
+
+# A feature that fails this many sessions in a row is parked in
+# `status="failed"` rather than bouncing back to the backlog. Stops
+# the auto-continue loop from hammering the same broken feature
+# forever. The user can manually flip status back to "backlog" once
+# the underlying cause is fixed.
+_MAX_FEATURE_FAILURES = 3
+
+
+def _max_concurrent_for_project(project_id: int) -> int:
+    """Hard cap is 6. Priority chain:
+       1. per-project override (forge_settings.max_concurrent_agents)
+       2. global cognos forge.max_concurrent
+       3. default 3
+    """
+    # Per-project override
+    try:
+        with session_scope() as sess:
+            row = (
+                sess.query(ForgeSetting)
+                .filter_by(project_id=project_id, key="max_concurrent_agents")
+                .first()
+            )
+            if row and row.value:
+                try:
+                    return max(1, min(6, int(row.value)))
+                except ValueError:
+                    pass
+    except Exception:
+        pass
+    raw = _get_forge_setting("max_concurrent", "3")
+    try:
+        n = int(raw)
+    except ValueError:
+        n = 3
+    return max(1, min(6, n))
+
+
+def _session_timeout_s() -> int:
+    raw = _get_forge_setting("session_timeout_s", str(30 * 60))
+    try:
+        return int(raw)
+    except ValueError:
+        return 30 * 60
+
+
+# ───────────────────────────── Errors ────────────────────────────────
+
+
+class OrchestratorError(Exception):
+    """Carries an HTTP-style status so api/server.py can translate."""
+    def __init__(self, message: str, status: int = 500):
+        super().__init__(message)
+        self.status = status
+
+
+# ─────────────────────── In-memory running state ──────────────────────
+
+
+class _RunningSession:
+    """Live state for one in-flight session. Persisted state is in DB."""
+
+    def __init__(self, session_id: int, project_id: int, feature_id: int,
+                 task: asyncio.Task[Any], started_at: float):
+        self.session_id = session_id
+        self.project_id = project_id
+        self.feature_id = feature_id
+        self.task = task
+        self.started_at = started_at
+
+
+class _OrchestratorState:
+    """Process-wide singleton holding live sessions + subscribers."""
+
+    def __init__(self):
+        self.running: dict[int, _RunningSession] = {}  # session_id -> rs
+        # session_id -> list of asyncio.Queue (one per SSE subscriber)
+        self.subscribers: dict[int, list[asyncio.Queue]] = {}
+        # listeners that get every event regardless of session
+        self.global_subscribers: list[asyncio.Queue] = []
+
+
+_STATE: _OrchestratorState | None = None
+
+
+def _state() -> _OrchestratorState:
+    global _STATE
+    if _STATE is None:
+        _STATE = _OrchestratorState()
+    return _STATE
+
+
+# ─────────────────────────── Event types ──────────────────────────────
+# Plain dicts (JSON-serialisable for SSE). Same shape as LocalForge's
+# OrchestratorEvent union.
+
+
+def _emit_event(event: dict[str, Any]) -> None:
+    """Push an event to all matching subscribers (non-blocking)."""
+    sid = event.get("session_id")
+    state = _state()
+    if isinstance(sid, int):
+        for q in state.subscribers.get(sid, []):
+            try:
+                q.put_nowait(event)
+            except asyncio.QueueFull:
+                pass
+    for q in state.global_subscribers:
+        try:
+            q.put_nowait(event)
+        except asyncio.QueueFull:
+            pass
+
+
+def _persist_log(session_id: int, feature_id: int | None, message: str,
+                 message_type: str = "info") -> int:
+    """Append a log row, return its id."""
+    with session_scope() as sess:
+        row = ForgeLog(
+            session_id=session_id,
+            feature_id=feature_id,
+            message=message,
+            message_type=message_type,
+        )
+        sess.add(row)
+        sess.flush()
+        log_id = row.id
+    _emit_event({
+        "type": "log",
+        "session_id": session_id,
+        "feature_id": feature_id,
+        "message": message,
+        "message_type": message_type,
+        "log_id": log_id,
+        "created_at": datetime.utcnow().isoformat(),
+    })
+    return log_id
+
+
+def _emit_status(session_id: int, feature_id: int | None,
+                 session_status: str, feature_status: str | None = None,
+                 feature_title: str | None = None) -> None:
+    _emit_event({
+        "type": "status",
+        "session_id": session_id,
+        "feature_id": feature_id,
+        "session_status": session_status,
+        "feature_status": feature_status,
+        "feature_title": feature_title,
+    })
+
+
+# ─────────────────── DB queries (next ready, etc.) ─────────────────────
+
+
+def find_next_ready_feature(project_id: int) -> dict[str, Any] | None:
+    """Highest-priority backlog feature whose deps are all completed.
+    Returns a plain dict (so we don't hold an ORM session open across
+    awaits).
+
+    ADR 0006 Phase C: when multiple features tie on the lowest
+    priority, ask the forge_advisor to break the tie. Order modes
+    via ``forge.backlog_order`` setting:
+      - ``caudate_easy_first`` (default): pick highest success_prob
+      - ``caudate_hard_first``: pick lowest success_prob
+      - ``priority_only``: legacy behaviour, first by id ascending
+    """
+    with session_scope() as sess:
+        # Pull all backlog features for this project, ordered by priority.
+        rows = (
+            sess.query(ForgeFeature)
+            .filter(
+                ForgeFeature.project_id == project_id,
+                ForgeFeature.status == "backlog",
+            )
+            .order_by(ForgeFeature.priority.asc(), ForgeFeature.id.asc())
+            .all()
+        )
+        # Compute readiness for each (deps all completed).
+        ready: list[ForgeFeature] = []
+        for r in rows:
+            deps = (
+                sess.query(ForgeFeatureDep.depends_on_feature_id)
+                .filter(ForgeFeatureDep.feature_id == r.id)
+                .all()
+            )
+            dep_ids = [d[0] for d in deps]
+            if not dep_ids:
+                ready.append(r)
+                continue
+            completed = (
+                sess.query(ForgeFeature)
+                .filter(
+                    ForgeFeature.id.in_(dep_ids),
+                    ForgeFeature.status == "completed",
+                )
+                .count()
+            )
+            if completed == len(dep_ids):
+                ready.append(r)
+        if not ready:
+            return None
+        # Group by priority (asc); within the lowest-priority group
+        # apply Caudate-driven tiebreak.
+        lowest_prio = ready[0].priority
+        head = [r for r in ready if r.priority == lowest_prio]
+        if len(head) == 1:
+            return _row_to_dict(head[0])
+        chosen = _caudate_tiebreak(head, project_id)
+        return _row_to_dict(chosen)
+
+
+def _caudate_tiebreak(
+    candidates: list[ForgeFeature], project_id: int,
+) -> ForgeFeature:
+    """ADR 0006 Phase C tiebreaker. Defaults to easy-first; honours
+    the per-project ``forge.backlog_order`` override. Falls back to
+    id-asc when the advisor is unavailable."""
+    mode = _get_forge_setting("backlog_order", "caudate_easy_first")
+    if mode == "priority_only":
+        return min(candidates, key=lambda r: r.id)
+    try:
+        from nn.forge_advisor import predict_feature_difficulty
+    except Exception:
+        return min(candidates, key=lambda r: r.id)
+    # Resolve a model id for the prediction (per-project effective
+    # model or the global default). Best-effort.
+    try:
+        from core.settings import Settings
+        s = Settings.load()
+        model_id = (
+            s.get("system1") or s.get("system2") or s.get("model")
+        )
+    except Exception:
+        model_id = None
+    scored = []
+    for r in candidates:
+        text = f"{r.title}\n\n{r.description or ''}"
+        try:
+            p = predict_feature_difficulty(text, model_id).success_prob
+        except Exception:
+            p = 0.5
+        scored.append((p, r))
+    reverse = (mode != "caudate_hard_first")  # easy_first → highest p first
+    scored.sort(key=lambda x: x[0], reverse=reverse)
+    return scored[0][1]
+
+
+def _row_to_dict(r: ForgeFeature) -> dict[str, Any]:
+    return {
+        "id": r.id,
+        "project_id": r.project_id,
+        "title": r.title,
+        "description": r.description,
+        "acceptance_criteria": r.acceptance_criteria,
+        "status": r.status,
+        "priority": r.priority,
+        "category": r.category,
+        "verify_command": r.verify_command,
+    }
+
+
+def get_kanban_state(project_id: int) -> dict[str, Any]:
+    """Return projects/features/running info for the UI in one shot."""
+    with session_scope() as sess:
+        proj = sess.get(ForgeProject, project_id)
+        if not proj:
+            raise OrchestratorError(f"project {project_id} not found", 404)
+        feats = (
+            sess.query(ForgeFeature)
+            .filter(ForgeFeature.project_id == project_id)
+            .order_by(ForgeFeature.priority.asc(), ForgeFeature.id.asc())
+            .all()
+        )
+        feat_dicts = []
+        for r in feats:
+            deps = (
+                sess.query(ForgeFeatureDep.depends_on_feature_id)
+                .filter(ForgeFeatureDep.feature_id == r.id)
+                .all()
+            )
+            d = _row_to_dict(r)
+            d["depends_on"] = [x[0] for x in deps]
+            feat_dicts.append(d)
+
+        running_feature_ids = {
+            rs.feature_id for rs in _state().running.values()
+            if rs.project_id == project_id
+        }
+        return {
+            "project": {
+                "id": proj.id,
+                "name": proj.name,
+                "description": proj.description,
+                "folder_path": proj.folder_path,
+                "status": proj.status,
+            },
+            "features": feat_dicts,
+            "running_feature_ids": sorted(running_feature_ids),
+            "max_concurrent": _max_concurrent_for_project(project_id),
+        }
+
+
+def get_running_for_project(project_id: int) -> list[int]:
+    return [
+        rs.session_id for rs in _state().running.values()
+        if rs.project_id == project_id
+    ]
+
+
+# ─────────────────────── Reconciliation on startup ─────────────────────
+
+
+def reconcile_orphaned_sessions() -> int:
+    """Sweep DB for sessions marked in_progress that have no live task,
+    flip them to terminated, reset their features to backlog. Called
+    once at process start. Returns count reclaimed."""
+    init()
+    n = 0
+    state = _state()
+    with session_scope() as sess:
+        rows = (
+            sess.query(ForgeSession)
+            .filter(ForgeSession.status == "in_progress")
+            .all()
+        )
+        for r in rows:
+            if r.id in state.running:
+                continue  # actually live in this process
+            r.status = "terminated"
+            r.ended_at = datetime.utcnow()
+            if r.feature_id is not None:
+                feat = sess.get(ForgeFeature, r.feature_id)
+                if feat and feat.status == "in_progress":
+                    feat.status = "backlog"
+            n += 1
+    if n:
+        logger.info(f"reconciled {n} orphaned forge sessions")
+    return n
+
+
+# ───────────────────────── Start a session ────────────────────────────
+
+
+async def start_orchestrator(project_id: int) -> dict[str, Any]:
+    """Pick the next ready feature, create a session, kick off the
+    runner. Returns {session_id, feature_id, started: True}.
+
+    Raises OrchestratorError(409) if the project has hit max_concurrent
+    or there's nothing ready to work on.
+    """
+    init()
+    reconcile_orphaned_sessions()
+
+    state = _state()
+    running = [
+        rs for rs in state.running.values() if rs.project_id == project_id
+    ]
+    cap = _max_concurrent_for_project(project_id)
+    if len(running) >= cap:
+        raise OrchestratorError(
+            f"max concurrent agents reached ({len(running)}/{cap})", 409,
+        )
+
+    feature = find_next_ready_feature(project_id)
+    if feature is None:
+        raise OrchestratorError("no ready features (backlog empty or blocked)", 409)
+
+    # Flip feature → in_progress + create session row in one transaction.
+    with session_scope() as sess:
+        feat = sess.get(ForgeFeature, feature["id"])
+        if feat is None or feat.status != "backlog":
+            raise OrchestratorError("feature changed underneath us; retry", 409)
+        feat.status = "in_progress"
+        sess.flush()
+
+        session_row = ForgeSession(
+            project_id=project_id,
+            feature_id=feature["id"],
+            session_type="coding",
+            status="in_progress",
+        )
+        sess.add(session_row)
+        sess.flush()
+        session_id = session_row.id
+
+        proj = sess.get(ForgeProject, project_id)
+        project_dir = proj.folder_path if proj else None
+
+    _persist_log(
+        session_id, feature["id"],
+        f"Orchestrator starting coding agent for \"{feature['title']}\"",
+        "info",
+    )
+    _emit_status(
+        session_id, feature["id"], "in_progress",
+        feature_status="in_progress", feature_title=feature["title"],
+    )
+
+    loop = asyncio.get_event_loop()
+    task = loop.create_task(
+        _run_session_with_watchdog(
+            session_id=session_id,
+            project_id=project_id,
+            feature=feature,
+            project_dir=project_dir,
+        )
+    )
+    state.running[session_id] = _RunningSession(
+        session_id=session_id,
+        project_id=project_id,
+        feature_id=feature["id"],
+        task=task,
+        started_at=loop.time(),
+    )
+    return {
+        "session_id": session_id,
+        "feature_id": feature["id"],
+        "feature_title": feature["title"],
+        "started": True,
+    }
+
+
+# ───────────────────────── Per-session runner ─────────────────────────
+
+
+async def _run_session_with_watchdog(
+    *,
+    session_id: int,
+    project_id: int,
+    feature: dict[str, Any],
+    project_dir: str | None,
+) -> None:
+    """Wrap _run_feature in an asyncio timeout. On timeout/cancel,
+    finalise as `terminated`."""
+    timeout_s = _session_timeout_s()
+    try:
+        # asyncio.timeout() was added in 3.11; use wait_for for 3.10
+        # compatibility. Same semantics — TimeoutError on deadline.
+        outcome, reason = await asyncio.wait_for(
+            _run_feature(
+                session_id=session_id,
+                feature=feature,
+                project_dir=project_dir,
+            ),
+            timeout=timeout_s,
+        )
+    except asyncio.TimeoutError:
+        _persist_log(
+            session_id, feature["id"],
+            f"Watchdog: session exceeded {timeout_s // 60}min timeout",
+            "error",
+        )
+        outcome, reason = "terminated", "timeout"
+    except asyncio.CancelledError:
+        _persist_log(
+            session_id, feature["id"],
+            "Session cancelled by orchestrator", "info",
+        )
+        outcome, reason = "terminated", "cancelled"
+        finalize_session(session_id, outcome, reason)
+        raise
+    except Exception as e:
+        logger.exception(f"forge runner crashed for session {session_id}")
+        _persist_log(
+            session_id, feature["id"], f"Runner crashed: {e}", "error",
+        )
+        outcome, reason = "failed", str(e)
+
+    finalize_session(session_id, outcome, reason)
+
+
+async def _run_feature(
+    *,
+    session_id: int,
+    feature: dict[str, Any],
+    project_dir: str | None,
+) -> tuple[str, str | None]:
+    """Run one feature to completion. Returns (outcome, reason)."""
+    # Build a CognosAgent for the project. We lazy-import here so the
+    # orchestrator module can be imported without yanking the world in.
+    from core.agent import CognosAgent
+    import os
+
+    prev_cwd = os.getcwd() if project_dir else None
+    if project_dir:
+        Path(project_dir).mkdir(parents=True, exist_ok=True)
+        os.chdir(project_dir)
+
+    try:
+        # ADR 0006 Phase B — Caudate-biased model selection per
+        # feature. If the advisor predicts this feature has a low
+        # success probability AND a system2 is configured, override
+        # the agent's primary model to system2 from turn one. Default
+        # behaviour (no override) keeps ADR 0005's local-first chain.
+        agent_model_override: str | None = None
+        try:
+            from nn.forge_advisor import predict_feature_difficulty
+            from core.settings import Settings
+            s = Settings.load()
+            sys1 = s.get("system1")
+            sys2 = s.get("system2")
+            promote_threshold = float(
+                _get_forge_setting("caudate_promote_threshold", "0.4")
+            )
+            text = f"{feature.get('title','')}\n\n{feature.get('description','')}"
+            pred = predict_feature_difficulty(text, sys1)
+            if pred.success_prob < promote_threshold and sys2:
+                agent_model_override = sys2
+                logger.info(
+                    f"forge: promoting feature #{feature['id']} to "
+                    f"{sys2!r} (predicted success {pred.success_prob:.2f} "
+                    f"below threshold {promote_threshold:.2f})"
+                )
+        except Exception as e:
+            logger.debug(f"caudate model bias skipped: {e}")
+
+        # Forge feature runs are autonomous — no human is waiting to
+        # answer "may I write this file?" prompts. Use BYPASS mode so
+        # the agent can actually do its job. The project's folder is a
+        # sandbox dir we already chdir'd into, which keeps the blast
+        # radius local.
+        agent_kwargs: dict[str, Any] = {"permission_mode": "bypass"}
+        if agent_model_override:
+            agent_kwargs["model"] = agent_model_override
+        agent = CognosAgent(**agent_kwargs)
+
+        # Pipe every tool call into the Forge live-log panel so the
+        # user can see what the agent is actually doing without
+        # tailing journalctl. PRE_TOOL_USE fires before the call,
+        # POST_TOOL_USE after success, POST_TOOL_USE_FAILURE on error.
+        # Each handler persists a forge_logs row which SSE-broadcasts
+        # to the UI in real time.
+        from core.hooks import HookEvent as _HE
+
+        def _short(value: Any, n: int = 220) -> str:
+            s = str(value).replace("\n", " ")
+            return s if len(s) <= n else s[:n] + "…"
+
+        async def _on_pre(_event: str, payload: dict[str, Any]):
+            tool = payload.get("tool_name", "?")
+            args = _short(payload.get("args", {}))
+            _persist_log(
+                session_id, feature["id"],
+                f"→ {tool}({args})", "action",
+            )
+            return None
+
+        async def _on_post(_event: str, payload: dict[str, Any]):
+            tool = payload.get("tool_name", "?")
+            result = _short(payload.get("result", ""), n=320)
+            _persist_log(
+                session_id, feature["id"],
+                f"✓ {tool} → {result}", "info",
+            )
+            return None
+
+        async def _on_post_fail(_event: str, payload: dict[str, Any]):
+            tool = payload.get("tool_name", "?")
+            err = _short(payload.get("error", "?"), n=320)
+            _persist_log(
+                session_id, feature["id"],
+                f"✗ {tool} failed: {err}", "error",
+            )
+            return None
+
+        # AgenticLoop's HookManager (always present, distinct from
+        # CognosAgent.hooks which is None when no external manager
+        # was passed in).
+        hooks_mgr = getattr(agent, "agentic", None) and agent.agentic.hooks
+        if hooks_mgr is not None:
+            hooks_mgr.register(_HE.PRE_TOOL_USE, _on_pre)
+            hooks_mgr.register(_HE.POST_TOOL_USE, _on_post)
+            hooks_mgr.register(_HE.POST_TOOL_USE_FAILURE, _on_post_fail)
+
+        prompt = _build_feature_prompt(feature)
+        _persist_log(session_id, feature["id"],
+                     f"Agent starting on: {feature['title']}", "action")
+
+        # Use the agent's chat path — agentic loop, full tool suite,
+        # sessions, hooks, Caudate observation. Returns the final text.
+        # Wrap in subscription_auth_scope so Anthropic-routed System-2
+        # calls find the Claude Code OAuth token instead of crashing
+        # on missing ANTHROPIC_API_KEY.
+        # Also bind the Forge project so the forge_* feature-CRUD tools
+        # know which backlog they operate on.
+        from core.anthropic_auth import subscription_auth_scope
+        from execution.tools.forge_feature_tools import forge_project_scope
+        try:
+            with subscription_auth_scope(), \
+                 forge_project_scope(feature["project_id"]):
+                response = await agent.chat(prompt)
+        except Exception as e:
+            return "failed", f"agent.chat raised: {e}"
+
+        # The chat response shape varies by build; just stringify.
+        result_text = str(getattr(response, "content", response) or "")
+        _persist_log(
+            session_id, feature["id"],
+            f"Agent finished: {result_text[:300]}", "info",
+        )
+
+        # ADR 0001: agent attestation is one half of the done contract.
+        # Scan for give-up phrases — if any appear, the feature is NOT
+        # done regardless of what the verify_command says. The verify
+        # half is checked below this block.
+        give_up = _detect_give_up(result_text)
+        if give_up:
+            _persist_log(
+                session_id, feature["id"],
+                f"Agent attestation failed — found give-up phrase: "
+                f"{give_up!r}", "error",
+            )
+            return "failed", f"agent gave up: {give_up}"
+
+        # Optional verify_command — must exit 0 to count as success.
+        verify = feature.get("verify_command")
+        if verify:
+            ok, output = await _run_verify(verify, project_dir)
+            _persist_log(
+                session_id, feature["id"],
+                f"verify: {verify}\n{output[:1500]}",
+                "test_result" if ok else "error",
+            )
+            if not ok:
+                return "failed", "verify_command failed"
+        return "success", None
+    finally:
+        if prev_cwd:
+            try:
+                os.chdir(prev_cwd)
+            except Exception:
+                pass
+
+
+def _build_feature_prompt(feature: dict[str, Any]) -> str:
+    """Render the feature into a coder prompt the agent can act on."""
+    parts = [
+        f"# Feature: {feature['title']}",
+        "",
+        feature.get("description") or "",
+        "",
+        "## Acceptance criteria",
+        feature.get("acceptance_criteria") or "(none specified)",
+        "",
+        "## Rules",
+        "- Implement this feature in the current working directory.",
+        "- Use the tools available (Read, Write, Edit, Bash, Grep, "
+        "Glob, etc.) — DO NOT use code_interpreter or any browser-side "
+        "Python shell, those will fail.",
+        "- DO NOT write README.md, CHECKLIST.md, USAGE_EXAMPLES.md, "
+        "TECHNICAL_SPEC.md, DEPLOYMENT.md, or any other freeform "
+        "documentation files unless an acceptance criterion explicitly "
+        "requires them. Focus on the code and tests called for above.",
+        "- Run the verify_command yourself first if you can, before "
+        "declaring done — it'll save a retry cycle.",
+        "- When done, respond with a short summary of what changed "
+        "(one paragraph maximum).",
+    ]
+    return "\n".join(parts)
+
+
+# ADR 0001 — give-up phrase detection for the attestation half of the
+# done contract. Conservative: only matches phrases that unambiguously
+# signal the agent stopped short. Lower-cases the response so e.g.
+# "Max iterations reached" matches "max iterations reached".
+_GIVE_UP_PHRASES = (
+    "i couldn't",
+    "i could not",
+    "i can't",
+    "i cannot",
+    "i gave up",
+    "i give up",
+    "once permissions are restored",
+    "blocked by",
+    "permission denied",
+    "tools are being denied",
+    "max iterations reached",
+    "max_iterations reached",
+    "i was unable to",
+    "unable to complete",
+    "stopping here",
+)
+
+
+def _detect_give_up(text: str) -> str | None:
+    """Return the matching phrase if the agent appears to have given
+    up, else None. Used by _run_feature to enforce ADR 0001's
+    attestation half of the done contract."""
+    if not text:
+        # An empty response from a coding agent is itself a give-up.
+        return "(empty agent response)"
+    lowered = text.lower()
+    for phrase in _GIVE_UP_PHRASES:
+        if phrase in lowered:
+            return phrase
+    return None
+
+
+async def _run_verify(cmd: str, project_dir: str | None) -> tuple[bool, str]:
+    """Run the verify_command. Returns (success, combined-output).
+
+    Compatibility shim: many bootstrapper outputs use ``python`` even
+    when the host only has ``python3`` on PATH (most modern Linux
+    distros). If ``python`` isn't installed but ``python3`` is, we
+    rewrite the command so the verify still works without re-running
+    the bootstrapper.
+    """
+    import os
+    import shutil
+
+    fixed = cmd
+    if shutil.which("python") is None and shutil.which("python3") is not None:
+        # Replace `python ` at start or after whitespace; leave
+        # `python3`, `python.x`, and quoted "python" strings alone.
+        import re as _re
+        fixed = _re.sub(r"(?<![\w.])python(?=\s|$)", "python3", cmd)
+
+    proc = await asyncio.create_subprocess_shell(
+        fixed,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.STDOUT,
+        cwd=project_dir or os.getcwd(),
+    )
+    try:
+        stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=300)
+    except asyncio.TimeoutError:
+        proc.kill()
+        return False, "[verify] timed out after 300s"
+    text = stdout.decode("utf-8", errors="replace")
+    rc = proc.returncode
+    # pytest exit codes:
+    #   0 — all tests passed
+    #   5 — no tests collected (treat as success: a foundation feature
+    #       may exist without any tests yet; failing the whole feature
+    #       on a missing test file is too strict)
+    # See https://docs.pytest.org/en/stable/reference/exit-codes.html
+    is_pytest = "pytest" in fixed.split()[0:2] or " pytest " in (" " + fixed + " ")
+    success = rc == 0 or (is_pytest and rc == 5)
+    note = ""
+    if fixed != cmd:
+        note += f"[verify cmd: {fixed!r}]\n"
+    if is_pytest and rc == 5:
+        note += "[pytest: no tests collected — treating as PASS for now]\n"
+    return success, note + text
+
+
+# ───────────────────────── Finalise + auto-continue ────────────────────
+
+
+def finalize_session(session_id: int, outcome: str,
+                     reason: str | None = None) -> None:
+    """Idempotent finalise. Called from the runner OR from stop_session.
+
+    outcome ∈ {success, failed, terminated}.
+    """
+    state = _state()
+    rs = state.running.pop(session_id, None)
+
+    # Hold all the values we need *out* of the session before it closes
+    # — otherwise we get DetachedInstanceError when reading later.
+    new_session_status: str | None = None
+    feature_title: str | None = None
+    feature_status: str | None = None
+    feature_id: int | None = None
+    project_id: int | None = None
+    # ADR 0003: when the failure cap fires and revision_count==0, the
+    # 3-fail branch sets this to the feature id; we trigger the
+    # revision attempt after the DB transaction closes so it isn't
+    # mixed with the finalize write.
+    revision_due_for_feature_id: int | None = None
+
+    with session_scope() as sess:
+        srow = sess.get(ForgeSession, session_id)
+        if srow is None or srow.status != "in_progress":
+            return  # already finalised
+        new_session_status = ({"success": "completed",
+                               "failed": "failed",
+                               "terminated": "terminated"}[outcome])
+        srow.status = new_session_status
+        srow.ended_at = datetime.utcnow()
+        project_id = srow.project_id
+
+        if srow.feature_id is not None:
+            feat = sess.get(ForgeFeature, srow.feature_id)
+            if feat is not None:
+                if outcome == "success":
+                    feat.status = "completed"
+                elif outcome == "failed":
+                    # Cap consecutive failures so a broken feature can't
+                    # trap auto-continue in a tight retry loop. We count
+                    # how many sessions for THIS feature have ended with
+                    # status='failed' since the last 'completed' (or
+                    # since the feature was created). After N in a row,
+                    # park the feature in 'failed' instead of bouncing
+                    # back to the backlog.
+                    fail_streak = (
+                        sess.query(ForgeSession)
+                        .filter_by(feature_id=feat.id, status="failed")
+                        .count()
+                    )
+                    # ADR 0006 Phase D — early-revision when retry is
+                    # likely doomed. After the FIRST failure, ask
+                    # forge_advisor whether the next retry is likely
+                    # to fail; if so, skip straight to revision now.
+                    early_revise = False
+                    if (fail_streak >= 1
+                            and fail_streak < _MAX_FEATURE_FAILURES
+                            and (feat.revision_count or 0) < 1):
+                        try:
+                            from nn.forge_advisor import (
+                                predict_failure_repeats,
+                            )
+                            text = (
+                                f"{feat.title}\n\n{feat.description or ''}"
+                            )
+                            p_fail = predict_failure_repeats(
+                                text, fail_streak,
+                            )
+                            threshold = float(
+                                _get_forge_setting("caudate_skip_threshold", "0.7")
+                            )
+                            if p_fail > threshold:
+                                early_revise = True
+                        except Exception:
+                            pass
+
+                    if fail_streak >= _MAX_FEATURE_FAILURES or early_revise:
+                        # ADR 0003: try a single bootstrapper revision
+                        # before parking. revision_count tracks whether
+                        # the revision shot has been spent.
+                        if (feat.revision_count or 0) < 1:
+                            # Reset for the revision attempt; orchestrator
+                            # will replace this feature out-of-band.
+                            feat.status = "backlog"
+                            feat.priority = max(0, (feat.priority or 0) + 1)
+                            revision_due_for_feature_id = feat.id
+                        else:
+                            feat.status = "failed"
+                            feat.priority = max(0, (feat.priority or 0) + 1)
+                    else:
+                        feat.status = "backlog"
+                        feat.priority = max(0, (feat.priority or 0) + 1)
+                else:  # terminated
+                    feat.status = "backlog"
+                feature_title = feat.title
+                feature_status = feat.status
+                feature_id = feat.id
+
+    if outcome == "success":
+        _persist_log(session_id, feature_id,
+                     f"Feature marked completed", "info")
+        # Promote project status when every feature finishes — mirrors
+        # LocalForge's markProjectCompletedIfAllDone, fires the
+        # celebration overlay in the UI.
+        if project_id is not None:
+            from planning.forge_models import (
+                mark_project_completed_if_all_done,
+            )
+            promoted = mark_project_completed_if_all_done(project_id)
+            if promoted:
+                _emit_event({
+                    "type": "status",
+                    "session_id": session_id,
+                    "feature_id": None,
+                    "session_status": "completed",
+                    "feature_status": None,
+                    "feature_title": None,
+                    "project_id": project_id,
+                })
+    elif outcome == "failed":
+        _persist_log(session_id, feature_id,
+                     f"Feature returned to backlog (priority demoted). "
+                     f"reason={reason or 'unknown'}",
+                     "error")
+    else:
+        _persist_log(session_id, feature_id,
+                     f"Session stopped — feature returned to backlog. "
+                     f"reason={reason or 'manual'}",
+                     "info")
+
+    _emit_status(
+        session_id, feature_id, new_session_status or "terminated",
+        feature_status=feature_status, feature_title=feature_title,
+    )
+
+    if rs is not None and not rs.task.done():
+        rs.task.cancel()
+
+    # Caudate hook — observe the feature outcome.
+    try:
+        from nn.observer import observe_feature_outcome
+        from core.settings import Settings
+        # CognosAgent uses DualLLMProvider when both system1 and system2
+        # are configured — each turn is routed by Caudate between the
+        # fast (local) and slow (cloud) brain. Previously we recorded
+        # only system2 here, which made the outcome log claim every
+        # feature was run by claude-haiku even when most turns hit the
+        # local Ollama model. Record the actual configuration so the
+        # log reflects what was available; per-turn routing detail
+        # lives in the agent's own session messages.
+        try:
+            settings = Settings.load()
+            s1 = settings.get("system1")
+            s2 = settings.get("system2")
+            single = settings.get("model")
+            if s1 and s2:
+                model_used = f"cognos-dual:{s1}+{s2}"
+            else:
+                model_used = s1 or s2 or single or "unknown"
+        except Exception:
+            model_used = "unknown"
+
+        # Pull session timing + log count for richer signals.
+        feature_text = f"{feature_title or ''}".strip()
+        n_logs = 0
+        duration_s = 0.0
+        try:
+            with session_scope() as sess:
+                if feature_id:
+                    feat = sess.get(ForgeFeature, feature_id)
+                    if feat:
+                        feature_text = (
+                            f"{feat.title}\n\n{feat.description or ''}".strip()
+                        )
+                if session_id:
+                    srow = sess.get(ForgeSession, session_id)
+                    if srow and srow.started_at and srow.ended_at:
+                        duration_s = (
+                            srow.ended_at - srow.started_at
+                        ).total_seconds()
+                if session_id:
+                    from sqlalchemy import func
+                    n_logs = (
+                        sess.query(func.count(ForgeLog.id))
+                        .filter_by(session_id=session_id)
+                        .scalar()
+                    ) or 0
+        except Exception:
+            pass
+
+        observe_feature_outcome(
+            feature_text=feature_text or f"feature#{feature_id}",
+            model_used=str(model_used),
+            n_turns=int(n_logs),
+            n_tool_calls=int(n_logs),
+            success=(outcome == "success"),
+            duration_s=float(duration_s),
+            project_id=project_id,
+            feature_id=feature_id,
+            session_id=session_id,
+            extras={"outcome": outcome, "reason": reason},
+        )
+    except (ImportError, AttributeError):
+        # Observer hook not yet implemented; that's fine.
+        pass
+    except Exception as e:
+        logger.debug(f"Caudate hook failed: {e}")
+
+    # ADR 0003: revision attempt before parking.
+    if revision_due_for_feature_id is not None and project_id is not None:
+        try:
+            loop = asyncio.get_running_loop()
+            loop.create_task(
+                _try_revise_feature(project_id, revision_due_for_feature_id)
+            )
+        except RuntimeError:
+            logger.debug("no running loop; revision deferred")
+
+    # Auto-continue: fill empty agent slots after every finalise.
+    # We need to schedule on the running event loop (we're called from
+    # inside a coroutine, but `finalize_session` itself is sync).
+    if outcome in ("success", "failed") and project_id is not None:
+        try:
+            loop = asyncio.get_running_loop()
+            loop.create_task(_maybe_auto_continue(project_id))
+        except RuntimeError:
+            # No running loop — happens in CLI sync paths or tests
+            # that called finalize_session directly. Ignore; the next
+            # explicit `cognos forge start` will pick up.
+            logger.debug("no running loop; skipping auto-continue")
+
+
+async def _try_revise_feature(project_id: int, feature_id: int) -> None:
+    """ADR 0003: kick off a single bootstrapper revision pass for a
+    feature that hit the 3-failure cap.
+
+    Pulls the feature row + the failure logs from the last few
+    sessions, calls ``Planner.revise_feature``, and either replaces
+    the original feature in-place (single revision) or splits it
+    into multiple new features (and parks the original as 'failed'
+    with revision_count=1).
+    """
+    try:
+        # 1. Snapshot the failed feature + collect failure logs
+        with session_scope() as sess:
+            feat = sess.get(ForgeFeature, feature_id)
+            if feat is None:
+                return
+            failed_dict = {
+                "title": feat.title,
+                "description": feat.description,
+                "acceptance_criteria": feat.acceptance_criteria,
+                "verify_command": feat.verify_command,
+                "category": feat.category,
+                "priority": feat.priority,
+            }
+            # Pull logs from this feature's last 3 failed sessions
+            fail_sids = [
+                r.id for r in (
+                    sess.query(ForgeSession)
+                    .filter_by(feature_id=feature_id, status="failed")
+                    .order_by(ForgeSession.id.desc())
+                    .limit(3)
+                    .all()
+                )
+            ]
+            log_rows = (
+                sess.query(ForgeLog)
+                .filter(ForgeLog.session_id.in_(fail_sids))
+                .order_by(ForgeLog.id.asc())
+                .all()
+                if fail_sids else []
+            )
+            failure_logs = "\n".join(
+                f"[s{r.session_id}][{r.message_type}] {r.message}"
+                for r in log_rows
+            )
+            project_name = ""
+            project_desc = ""
+            proj = sess.get(ForgeProject, project_id)
+            if proj is not None:
+                project_name = proj.name
+                project_desc = proj.description or ""
+
+        _persist_log(
+            fail_sids[0] if fail_sids else 0, feature_id,
+            f"ADR 0003 revision: re-decomposing feature #{feature_id}",
+            "action",
+        )
+
+        # 2. Ask the planner for a revised backlog
+        from planning.planner import Planner
+        from llm.provider import LLMProvider
+        from core.settings import Settings
+        from core.anthropic_auth import subscription_auth_scope
+        from config import SYSTEM_1_MODEL, SYSTEM_2_MODEL, LLM_MODEL
+
+        settings = Settings.load()
+        chosen = (
+            settings.get("system1") or SYSTEM_1_MODEL
+            or settings.get("system2") or SYSTEM_2_MODEL
+            or LLM_MODEL
+        )
+        planner = Planner(LLMProvider(model=chosen))
+        try:
+            with subscription_auth_scope():
+                replacements = await planner.revise_feature(
+                    failed_feature=failed_dict,
+                    failure_logs=failure_logs or "(no logs available)",
+                    project_goal=(
+                        f"{project_name}\n{project_desc}".strip()
+                        or None
+                    ),
+                )
+        except Exception as e:
+            logger.warning(f"revision LLM call failed: {e}")
+            # Fall back to permanent park
+            with session_scope() as sess:
+                feat = sess.get(ForgeFeature, feature_id)
+                if feat is not None:
+                    feat.status = "failed"
+                    feat.revision_count = 1
+            return
+
+        if not replacements:
+            with session_scope() as sess:
+                feat = sess.get(ForgeFeature, feature_id)
+                if feat is not None:
+                    feat.status = "failed"
+                    feat.revision_count = 1
+            return
+
+        # 3. Apply the revision to the DB.
+        # Strategy:
+        # - revision_count incremented to 1 on the original feature.
+        # - If exactly one replacement: rewrite the row in-place
+        #   (preserves dependents).
+        # - If multiple: rewrite the original to be the FIRST sub-feature
+        #   and create the rest as new features. Each subsequent
+        #   sub-feature depends on the previous. Original's existing
+        #   dependents (features that pointed at it) continue to point
+        #   at the original — which is now the first link in the chain.
+        with session_scope() as sess:
+            feat = sess.get(ForgeFeature, feature_id)
+            if feat is None:
+                return
+            first = replacements[0]
+            feat.title = first.title
+            feat.description = first.description
+            feat.acceptance_criteria = first.acceptance_criteria
+            feat.category = first.category if first.category in (
+                "functional", "style"
+            ) else "functional"
+            if first.verify_command:
+                feat.verify_command = first.verify_command
+            feat.status = "backlog"
+            feat.priority = 0
+            feat.revision_count = 1
+
+            new_ids: list[int] = [feature_id]
+            prev_id = feature_id
+            for rep in replacements[1:]:
+                cat = rep.category if rep.category in (
+                    "functional", "style"
+                ) else "functional"
+                new_row = ForgeFeature(
+                    project_id=project_id,
+                    title=rep.title,
+                    description=rep.description,
+                    acceptance_criteria=rep.acceptance_criteria,
+                    status="backlog",
+                    priority=(rep.priority or 0) + 100,  # after originals
+                    category=cat,
+                    verify_command=rep.verify_command,
+                    revision_count=0,
+                )
+                sess.add(new_row)
+                sess.flush()
+                # Wire the new sub-feature to depend on the previous.
+                sess.add(ForgeFeatureDep(
+                    feature_id=new_row.id,
+                    depends_on_feature_id=prev_id,
+                ))
+                new_ids.append(new_row.id)
+                prev_id = new_row.id
+
+        # 4. Broadcast a status event so the kanban refreshes.
+        _emit_event({
+            "type": "status",
+            "session_id": fail_sids[0] if fail_sids else 0,
+            "feature_id": feature_id,
+            "session_status": "revised",
+            "feature_status": "backlog",
+            "feature_title": failed_dict["title"],
+            "project_id": project_id,
+            "revision": {"new_feature_ids": new_ids},
+        })
+        logger.info(
+            f"revised feature #{feature_id} into {len(new_ids)} "
+            f"feature(s): {new_ids}"
+        )
+    except Exception:
+        logger.exception("revision crashed")
+
+
+async def _maybe_auto_continue(project_id: int) -> None:
+    """If slots are open and there's a ready feature, start it."""
+    try:
+        cap = _max_concurrent_for_project(project_id)
+        running = sum(
+            1 for rs in _state().running.values()
+            if rs.project_id == project_id
+        )
+        slots = cap - running
+        if slots <= 0:
+            return
+        for _ in range(slots):
+            try:
+                await start_orchestrator(project_id)
+            except OrchestratorError as e:
+                if e.status == 409:
+                    return  # nothing more to do
+                logger.warning(f"auto-continue failed: {e}")
+                return
+    except Exception:
+        logger.exception("auto-continue crashed")
+
+
+# ───────────────────────────── Stop ──────────────────────────────────
+
+
+def stop_session(session_id: int) -> bool:
+    """Cancel the live task. The runner's finalise handler will mark
+    it terminated. Returns True if a live task was cancelled."""
+    rs = _state().running.get(session_id)
+    if rs is None:
+        # Already done; just make sure DB row is consistent.
+        finalize_session(session_id, "terminated", "stop called after exit")
+        return False
+    rs.task.cancel()
+    return True
+
+
+def stop_all_for_project(project_id: int) -> int:
+    """Cancel every live session for this project. Returns count."""
+    n = 0
+    for sid in list(_state().running.keys()):
+        rs = _state().running[sid]
+        if rs.project_id == project_id:
+            stop_session(sid)
+            n += 1
+    return n
+
+
+# ─────────────────────────── SSE subscriptions ──────────────────────────
+
+
+async def subscribe_session(
+    session_id: int,
+    queue_size: int = 128,
+) -> AsyncIterator[dict[str, Any]]:
+    """Yield events for one session until it ends. Caller is expected
+    to consume in an async-for; on cancellation we unhook cleanly."""
+    q: asyncio.Queue = asyncio.Queue(maxsize=queue_size)
+    _state().subscribers.setdefault(session_id, []).append(q)
+    try:
+        # Replay last 50 logs first so a late subscriber sees context.
+        with session_scope() as sess:
+            recent = (
+                sess.query(ForgeLog)
+                .filter(ForgeLog.session_id == session_id)
+                .order_by(ForgeLog.id.desc())
+                .limit(50)
+                .all()
+            )
+            for r in reversed(recent):
+                yield {
+                    "type": "log",
+                    "session_id": session_id,
+                    "feature_id": r.feature_id,
+                    "message": r.message,
+                    "message_type": r.message_type,
+                    "log_id": r.id,
+                    "created_at": r.created_at.isoformat() if r.created_at else None,
+                    "replay": True,
+                }
+        while True:
+            try:
+                ev = await asyncio.wait_for(q.get(), timeout=30)
+            except asyncio.TimeoutError:
+                yield {"type": "heartbeat"}
+                continue
+            yield ev
+            # Terminal event ends the stream.
+            if ev.get("type") == "status" and ev.get("session_status") in (
+                "completed", "failed", "terminated",
+            ):
+                return
+    finally:
+        try:
+            _state().subscribers.get(session_id, []).remove(q)
+        except ValueError:
+            pass
+
+
+async def subscribe_global(queue_size: int = 256) -> AsyncIterator[dict[str, Any]]:
+    """Yield every event from every session. Used by toast notifiers."""
+    q: asyncio.Queue = asyncio.Queue(maxsize=queue_size)
+    _state().global_subscribers.append(q)
+    try:
+        while True:
+            try:
+                ev = await asyncio.wait_for(q.get(), timeout=30)
+                yield ev
+            except asyncio.TimeoutError:
+                yield {"type": "heartbeat"}
+    finally:
+        try:
+            _state().global_subscribers.remove(q)
+        except ValueError:
+            pass
+
+
+# Public surface
+__all__ = [
+    "OrchestratorError",
+    "find_next_ready_feature",
+    "get_kanban_state",
+    "get_running_for_project",
+    "reconcile_orphaned_sessions",
+    "start_orchestrator",
+    "stop_session",
+    "stop_all_for_project",
+    "finalize_session",
+    "subscribe_session",
+    "subscribe_global",
+]