polymath-agent 0.4.0__py3-none-any.whl

polymath/pipeline.py

@@ -0,0 +1,700 @@
+"""
+Task pipeline with agentic loop.
+
+  SIMPLE:  PLAN → PLAN_APPROVAL → CLARIFY → EXECUTE → REVIEW → OUTPUT
+  MEDIUM:  CLARIFY → PLAN → PLAN_APPROVAL → CLARIFY → EXECUTE → REVIEW → OUTPUT
+  COMPLEX: UNDERSTAND → CLARIFY → PLAN → PLAN_APPROVAL → CLARIFY
+           → EXECUTE (REASON/ACT/OBSERVE) → REVIEW → CORRECTIONS → RE-REVIEW → OUTPUT
+
+Plan mode is on for ALL complexities by default — this is Polymath's
+headline UX. The only path that skips plan is `--ask` (direct one-shot),
+handled outside this pipeline by run_ask_flow in main.py.
+
+Override via cfg["plan_mode"]:
+  "always"     → plan for every session (default)
+  "complexity" → plan only for MEDIUM + COMPLEX (legacy behaviour)
+  "off"        → never plan (use sparingly; loses the moat)
+"""
+from __future__ import annotations
+
+import inspect
+import json
+import logging
+import re
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, AsyncIterator, Dict, List
+
+from polymath.adapters.base import BaseAdapter, Message, ToolCall
+
+logger = logging.getLogger(__name__)
+from polymath.compressor import build_context
+from polymath.config import ModelInfo, TaskComplexity, TaskType, memory_setting
+from polymath.domain import PermissionRequest
+from polymath.permissions import build_permission_request
+from polymath.context_store import add_message, export_session_md, get_messages
+from polymath.context_manager import build_context_injection, export_session_to_project
+from polymath.memory.retriever import retrieve_or_fallback
+from polymath.memory.writer import harvest_session as memory_write_back
+from polymath.orchestrator.race import (
+    AllCandidatesFailed,
+    RaceCandidate,
+    StreamCandidate,
+    race_streams,
+    with_failover,
+)
+from polymath.orchestrator.speculative_review import SpeculativeReviewer
+from polymath.tools import registry as tool_registry
+from polymath.workspace import get_workspace_context
+
+
+class Step(Enum):
+    UNDERSTAND   = "understand"
+    CLARIFY_1    = "clarify_1"
+    PLAN         = "plan"
+    CLARIFY_2    = "clarify_2"
+    EXECUTE      = "execute"
+    REASON       = "reason"
+    ACT          = "act"
+    OBSERVE      = "observe"
+    PREVIEW      = "preview"
+    REVIEW       = "review"
+    CORRECTIONS  = "corrections"
+    RE_REVIEW    = "re_review"
+    OUTPUT       = "output"
+
+
+@dataclass
+class PipelineResult:
+    output: str = ""
+    plan: str = ""
+    review_notes: list[str] = field(default_factory=list)
+    primary_model: str = ""
+    reviewer_model: str = ""
+    steps_run: list[str] = field(default_factory=list)
+
+
+# ── System prompts ────────────────────────────────────────────────────────────
+
+UNDERSTAND_SYSTEM = (
+    "You are a thorough analyst. When given a task:\n"
+    "1. State your understanding of what's being asked.\n"
+    "2. Identify any ambiguities or missing information.\n"
+    "3. List any assumptions you're making.\n"
+    "Keep this concise — 3–5 sentences total."
+)
+
+CLARIFY_SYSTEM = (
+    "Based on your understanding, generate up to 3 clarifying questions that would "
+    "meaningfully improve your response. If no clarification is needed, say exactly: "
+    "NO_CLARIFICATION_NEEDED. Otherwise list questions numbered 1, 2, 3."
+)
+
+PLAN_SYSTEM = (
+    "Create a clear, numbered step-by-step plan to accomplish this task. "
+    "Be specific. Each step should be actionable. No commentary — just the plan."
+)
+
+AGENT_SYSTEM = (
+    "You are an autonomous agent capable of using tools to interact with your environment. "
+    "When a task requires information you don't have, use a tool to get it. "
+    "Always reason out loud before taking an action. "
+    "If you have all the information needed, provide a final answer. "
+    "Be precise and thorough."
+)
+
+REVIEW_SYSTEM = (
+    "You are a critical reviewer. You will receive a task and a response to it.\n"
+    "Review the response for:\n"
+    "- Correctness and completeness\n"
+    "- Missing edge cases or errors\n"
+    "- Clarity and quality\n\n"
+    "Format your review as:\n"
+    "VERDICT: APPROVED | NEEDS_WORK\n"
+    "ISSUES:\n- issue 1\n- issue 2\n"
+    "SUGGESTIONS:\n- suggestion 1\n"
+    "If APPROVED with no issues, just say: VERDICT: APPROVED"
+)
+
+CORRECTIONS_SYSTEM = (
+    "You are revising your previous response based on reviewer feedback. "
+    "Apply all suggested corrections. Output the complete corrected response only — "
+    "no meta-commentary about what changed."
+)
+
+
+# ── Pipeline ──────────────────────────────────────────────────────────────────
+
+class Pipeline:
+    def __init__(
+        self,
+        session_id: str,
+        primary_adapter: BaseAdapter,
+        primary_model: ModelInfo,
+        reviewer_adapter: BaseAdapter | None,
+        reviewer_model: ModelInfo | None,
+        summarizer_adapter: BaseAdapter | None,
+        summarizer_model: str,
+        review_rounds: int = 2,
+        active_project: str = "",
+        cfg: dict | None = None,
+        failover_adapter: BaseAdapter | None = None,
+        failover_model: ModelInfo | None = None,
+        # Callbacks for UI
+        on_step: Callable[[Step, str], None] | None = None,
+        on_token: Callable[[str], None] | None = None,
+        on_ask: Callable[[str], Any] | None = None,    # returns or awaits user answer
+        on_permission: Callable[[PermissionRequest], Any] | None = None,
+        on_plan_approval: Callable[[str], Any] | None = None,
+    ) -> None:
+        self.session_id = session_id
+        self.primary = primary_adapter
+        self.primary_model = primary_model
+        self.reviewer = reviewer_adapter
+        self.reviewer_model = reviewer_model
+        self.summarizer = summarizer_adapter
+        self.summarizer_model = summarizer_model
+        self.review_rounds = review_rounds
+        self.active_project = active_project
+        self.cfg = cfg or {}
+        self.failover_adapter = failover_adapter
+        self.failover_model = failover_model
+        self._speculative_reviewer: SpeculativeReviewer | None = None
+        self.on_step = on_step or (lambda s, t: None)
+        self.on_token = on_token or (lambda t: None)
+        self.on_ask = on_ask or (lambda q: "")
+        self.on_permission = on_permission or (lambda q: True)
+        # Plan approval: receives plan text, returns "proceed"|"cancel"|"refine:<text>"
+        self.on_plan_approval = on_plan_approval or (lambda plan: "proceed")
+
+    async def _resolve_callback(self, callback_result: Any) -> Any:
+        if inspect.isawaitable(callback_result):
+            return await callback_result
+        return callback_result
+
+    async def run(
+        self,
+        user_input: str,
+        complexity: TaskComplexity,
+        task_type: TaskType,
+        active_project: str = "",
+    ) -> PipelineResult:
+        # active_project may be passed directly; fall back to instance attribute
+        effective_project = active_project or self.active_project
+        result = PipelineResult(
+            primary_model=self.primary_model.display_name,
+            reviewer_model=self.reviewer_model.display_name if self.reviewer_model else "",
+        )
+
+        # Save user message
+        add_message(self.session_id, "user", user_input, step=Step.EXECUTE.value)
+
+        # Optional speculative reviewer — observes agent loop, primes final review
+        if (
+            self.reviewer is not None
+            and self.reviewer_model is not None
+            and memory_setting(self.cfg, "speculative_review")
+        ):
+            self._speculative_reviewer = SpeculativeReviewer(
+                adapter=self.reviewer,
+                model_id=self.reviewer_model.id,
+                task=user_input,
+            )
+
+        # ── Pipeline logic ──────────────────────────────────────────────────
+        plan_mode = (self.cfg.get("plan_mode") or "always").lower()
+        plan_for_simple = plan_mode in ("always",)
+        plan_for_medium = plan_mode in ("always", "complexity")
+        plan_for_complex = plan_mode in ("always", "complexity")
+
+        if complexity == TaskComplexity.SIMPLE:
+            # Plan first (when on), approve, clarify, then execute
+            if plan_for_simple:
+                await self._plan(user_input, result)
+                if not await self._await_plan_approval(user_input, result):
+                    return result
+            await self._clarify(user_input, result, round_num=1)
+            await self._agentic_execute(user_input, result, task_type=task_type, active_project=effective_project)
+        elif complexity == TaskComplexity.MEDIUM:
+            # Clarify intent → plan → approve → clarify again with refined context → execute
+            await self._clarify(user_input, result, round_num=1)
+            if plan_for_medium:
+                await self._plan(user_input, result)
+                if not await self._await_plan_approval(user_input, result):
+                    return result
+            await self._clarify(user_input, result, round_num=2)
+            await self._agentic_execute(user_input, result, task_type=task_type, active_project=effective_project)
+        else:  # COMPLEX
+            # Understand deeply → clarify → plan → approve → clarify again → execute
+            await self._understand(user_input, result)
+            await self._clarify(user_input, result, round_num=1)
+            if plan_for_complex:
+                await self._plan(user_input, result)
+                if not await self._await_plan_approval(user_input, result):
+                    return result
+            await self._clarify(user_input, result, round_num=2)
+            await self._agentic_execute(user_input, result, task_type=task_type, active_project=effective_project)
+
+        # Review loop
+        if self.reviewer and self.reviewer_model:
+            await self._review_loop(user_input, result)
+
+        result.steps_run.append(Step.OUTPUT.value)
+        self.on_step(Step.OUTPUT, result.output)
+        
+        # Export session for cross-model unified context persistence
+        try:
+            export_session_md(self.session_id)
+        except Exception as e:
+            logger.warning("export_session_md failed for session %s: %s", self.session_id, e)
+
+        # Export to project folder if a project is active
+        if effective_project:
+            try:
+                messages = get_messages(self.session_id)
+                export_session_to_project(
+                    project=effective_project,
+                    session_id=self.session_id,
+                    messages=messages,
+                    meta={"primary_model": result.primary_model, "steps_run": result.steps_run},
+                )
+            except Exception as e:
+                logger.warning("export_session_to_project failed for %s: %s", effective_project, e)
+
+        # Auto-append key decisions to decisions.md for COMPLEX tasks
+        if effective_project and complexity == TaskComplexity.COMPLEX:
+            try:
+                await self._maybe_append_decisions(result, effective_project)
+            except Exception as e:
+                logger.warning("decisions append failed for %s: %s", effective_project, e)
+
+        # Memory writeback: extract durable facts from this run as candidate chunks.
+        # OFF by default — prefer the deliberate `/learn-from <correction>` flow,
+        # which is Boris-style (manual, 100% reliable, no candidate→confirmed graveyard).
+        # Opt in by setting cfg["memory"]["auto_writeback"] = True.
+        if effective_project and self.cfg.get("memory", {}).get("auto_writeback", False):
+            try:
+                await memory_write_back(
+                    session_id=self.session_id,
+                    project=effective_project,
+                    primary_adapter=self.primary,
+                    primary_model_id=self.primary_model.id,
+                    cfg=self.cfg,
+                )
+            except Exception as e:
+                logger.warning("memory writeback failed for %s: %s", effective_project, e)
+
+        return result
+
+    # ── Steps ─────────────────────────────────────────────────────────────────
+
+    async def _agentic_execute(
+        self,
+        user_input: str,
+        result: PipelineResult,
+        task_type: TaskType = TaskType.GENERAL,
+        active_project: str = "",
+    ) -> None:
+        """The main agentic loop: REASON -> ACT -> OBSERVE."""
+        self.on_step(Step.EXECUTE, "")
+
+        # Build initial context with workspace awareness + typed project context
+        context = await self._get_context()
+        ws_context = get_workspace_context()
+        effective_project = active_project or self.active_project
+        project_ctx = (
+            await retrieve_or_fallback(effective_project, user_input, task_type, self.cfg)
+            if effective_project else ""
+        )
+        system_prompt = AGENT_SYSTEM
+        if project_ctx:
+            system_prompt += f"\n\n{project_ctx}"
+        system_prompt += f"\n\n{ws_context}"
+        
+        context.append(Message(role="user", content=user_input))
+        
+        max_turns = 10
+        for turn in range(max_turns):
+            self.on_step(Step.REASON, f"Turn {turn + 1}")
+            
+            # 1. Complete with tool support
+            response = await self.primary.complete(
+                messages=context,
+                model_id=self.primary_model.id,
+                system=system_prompt,
+                tools=tool_registry.list_tools()
+            )
+            
+            if isinstance(response, str):
+                # Final answer
+                result.output = response
+                self.on_step(Step.PREVIEW, response)
+                add_message(
+                    self.session_id, "assistant", response,
+                    model=self.primary_model.id, step=Step.EXECUTE.value
+                )
+                if self._speculative_reviewer is not None:
+                    self._speculative_reviewer.kick_off(response)
+                break
+
+            # Handle tool calls
+            assistant_msg = response
+            context.append(assistant_msg)
+            add_message(
+                self.session_id, "assistant", assistant_msg.content,
+                model=self.primary_model.id, step=Step.ACT.value
+            )
+
+            if assistant_msg.content:
+                self.on_step(Step.REASON, assistant_msg.content)
+                if self._speculative_reviewer is not None and len(assistant_msg.content) > 200:
+                    self._speculative_reviewer.kick_off(assistant_msg.content)
+
+            for tc in assistant_msg.tool_calls:
+                self.on_step(Step.ACT, f"Calling {tc.name}({tc.arguments})")
+
+                # Check permission if needed
+                tool = tool_registry.get(tc.name)
+                if tool and tool.needs_permission(tc.arguments):
+                    if not await self._resolve_callback(
+                        self.on_permission(
+                            build_permission_request(tc.name, tc.arguments)
+                        )
+                    ):
+                        obs = "Error: Permission denied by user."
+                    else:
+                        obs = await tool_registry.execute(tc.name, tc.arguments)
+                else:
+                    obs = await tool_registry.execute(tc.name, tc.arguments)
+
+                self.on_step(Step.OBSERVE, obs[:500] + "..." if len(obs) > 500 else obs)
+
+                # Add observation to context
+                context.append(Message(role="tool", content=obs, model=tc.name, tool_call_id=tc.id))
+                add_message(self.session_id, "tool", obs, model=tc.name, step=Step.OBSERVE.value)
+        
+        result.steps_run.append(Step.EXECUTE.value)
+
+    async def _understand(self, user_input: str, result: PipelineResult) -> None:
+        self.on_step(Step.UNDERSTAND, "")
+        response = await self._call_primary(
+            extra_user=f"Task: {user_input}",
+            system=UNDERSTAND_SYSTEM,
+            step=Step.UNDERSTAND,
+        )
+        result.steps_run.append(Step.UNDERSTAND.value)
+
+    async def _clarify(self, user_input: str, result: PipelineResult, round_num: int) -> None:
+        step = Step.CLARIFY_1 if round_num == 1 else Step.CLARIFY_2
+        self.on_step(step, "")
+
+        questions_raw = await self._call_primary(
+            extra_user=f"Original task: {user_input}\nDo you need clarification?",
+            system=CLARIFY_SYSTEM,
+            step=step,
+        )
+
+        if "NO_CLARIFICATION_NEEDED" in questions_raw.upper():
+            return
+
+        questions = _extract_questions(questions_raw)
+        if not questions:
+            return
+
+        for i, q in enumerate(questions[:3], 1):
+            answer = await self._resolve_callback(
+                self.on_ask(f"[Clarification {i}/{len(questions[:3])}] {q}")
+            )
+            if answer:
+                add_message(self.session_id, "user", answer, step=step.value)
+
+        result.steps_run.append(step.value)
+
+    async def _plan(self, user_input: str, result: PipelineResult) -> None:
+        self.on_step(Step.PLAN, "")
+        plan = await self._call_primary(
+            extra_user=f"Create a plan for: {user_input}",
+            system=PLAN_SYSTEM,
+            step=Step.PLAN,
+        )
+        result.plan = plan
+        result.steps_run.append(Step.PLAN.value)
+
+    async def _await_plan_approval(self, user_input: str, result: PipelineResult) -> bool:
+        """
+        Show plan to user, loop until approved or cancelled.
+        Returns True to proceed, False to cancel.
+        Accepts up to 3 refinement rounds before auto-proceeding.
+        """
+        for _ in range(3):
+            decision = await self._resolve_callback(self.on_plan_approval(result.plan))
+            d = decision.strip().lower()
+            if d in ("", "y", "yes", "proceed"):
+                return True
+            if d == "cancel":
+                result.steps_run.append("cancelled")
+                return False
+            if d.startswith("refine:"):
+                feedback = decision[7:].strip()
+                await self._plan(f"{user_input}\n\nFeedback on previous plan: {feedback}", result)
+            else:
+                # Unrecognised input — treat as proceed
+                return True
+        return True  # auto-proceed after max refinements
+
+    async def _maybe_append_decisions(self, result: PipelineResult, project: str) -> None:
+        """
+        Extract up to 5 key decisions from a COMPLEX task output and append to decisions.md.
+        Only runs when output is substantial (>200 chars).
+        """
+        if len(result.output) < 200:
+            return
+        prompt = (
+            "Extract up to 5 key decisions or conclusions from this AI session output "
+            "as brief bullet points (start each with '- '). Only include genuinely "
+            "decision-worthy items (architectural choices, important findings, chosen "
+            "approaches, tradeoffs accepted). If none exist, respond with exactly: "
+            "NO_DECISIONS\n\n"
+            f"Output:\n{result.output[:4000]}"
+        )
+        resp = await self.primary.complete(
+            messages=[Message(role="user", content=prompt)],
+            model_id=self.primary_model.id,
+            system="Extract decisions only. Be terse.",
+            temperature=0.2,
+        )
+        text = resp if isinstance(resp, str) else resp.content
+        if "NO_DECISIONS" in text.upper():
+            return
+        from polymath.context_manager import read_context, write_context
+        from datetime import datetime, timezone
+        timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+        existing = read_context(project, "decisions")
+        if existing and not existing.endswith("\n"):
+            existing += "\n"
+        entry = f"\n### {timestamp}\n{text.strip()}\n"
+        write_context(project, "decisions", existing + entry)
+
+    async def _review_loop(self, user_input: str, result: PipelineResult) -> None:
+        for round_num in range(self.review_rounds):
+            step = Step.REVIEW if round_num == 0 else Step.RE_REVIEW
+            self.on_step(step, "")
+
+            if round_num == 0 and self._speculative_reviewer is not None:
+                review = await self._speculative_reviewer.finalize(result.output)
+            else:
+                review_prompt = (
+                    f"ORIGINAL TASK:\n{user_input}\n\n"
+                    f"RESPONSE TO REVIEW:\n{result.output}"
+                )
+                review = await self.reviewer.complete(
+                    messages=[Message(role="user", content=review_prompt)],
+                    model_id=self.reviewer_model.id,
+                    system=REVIEW_SYSTEM,
+                    temperature=0.3,
+                )
+            add_message(
+                self.session_id, "assistant", review,
+                model=self.reviewer_model.id, step=step.value,
+            )
+            result.review_notes.append(review)
+            self.on_step(step, review)
+
+            if "VERDICT: APPROVED" in review and "NEEDS_WORK" not in review:
+                break
+
+            # Apply corrections
+            self.on_step(Step.CORRECTIONS, "")
+            correction_prompt = (
+                f"Your previous response:\n{result.output}\n\n"
+                f"Reviewer feedback:\n{review}\n\n"
+                "Apply all corrections and output the complete revised response."
+            )
+            corrected = await self._call_primary(
+                extra_user=correction_prompt,
+                system=CORRECTIONS_SYSTEM,
+                step=Step.CORRECTIONS,
+                stream=True,
+            )
+            if isinstance(corrected, str):
+                result.output = corrected
+            else:
+                result.output = corrected.content
+
+            result.steps_run.append(step.value)
+            result.steps_run.append(Step.CORRECTIONS.value)
+
+    # ── Helpers ───────────────────────────────────────────────────────────────
+
+    async def _call_primary(
+        self,
+        extra_user: str,
+        system: str,
+        step: Step,
+        stream: bool = False,
+    ) -> str | Message:
+        context = await self._get_context()
+        context.append(Message(role="user", content=extra_user))
+
+        if stream:
+            output = ""
+            async for token in self.primary.stream(
+                messages=context, model_id=self.primary_model.id, system=system
+            ):
+                if isinstance(token, str):
+                    output += token
+                    self.on_token(token)
+                else:
+                    # Tool calls not expected in simple calls, but just in case
+                    pass
+            add_message(
+                self.session_id, "assistant", output,
+                model=self.primary_model.id, step=step.value,
+            )
+            return output
+
+        resp, winner_model = await self._race_complete(
+            context=context, system=system, temperature=0.5,
+        )
+        content = resp if isinstance(resp, str) else resp.content
+        add_message(
+            self.session_id, "assistant", content,
+            model=winner_model.id, step=step.value,
+        )
+        return resp
+
+    async def _race_complete(
+        self,
+        context: list[Message],
+        system: str,
+        temperature: float = 0.5,
+    ) -> tuple[Any, ModelInfo]:
+        """If a failover adapter is configured, run primary + failover in parallel
+        with delay; first success wins. Returns (response, winning_model_info)."""
+        delay = float(memory_setting(self.cfg, "race_delay_seconds"))
+
+        async def call(adapter: BaseAdapter, model_id: str):
+            return await adapter.complete(
+                messages=context, model_id=model_id, system=system, temperature=temperature,
+            )
+
+        primary = RaceCandidate(
+            id=self.primary_model.id,
+            factory=lambda: call(self.primary, self.primary_model.id),
+        )
+        if self.failover_adapter is None or self.failover_model is None:
+            return await primary.factory(), self.primary_model
+
+        fallback = RaceCandidate(
+            id=self.failover_model.id,
+            factory=lambda: call(self.failover_adapter, self.failover_model.id),
+        )
+        try:
+            outcome = await with_failover(primary, fallback, delay_seconds=delay)
+        except AllCandidatesFailed as e:
+            primary_exc = e.failures.get(primary.id)
+            if primary_exc:
+                raise primary_exc
+            raise
+
+        winner_model = self.primary_model if outcome.winner_id == primary.id else self.failover_model
+        return outcome.result, winner_model
+
+    async def _get_context(self) -> list[Message]:
+        summarizer_adapter = self.summarizer
+        summarizer_model_id = self.summarizer_model
+        return await build_context(
+            self.session_id,
+            summarizer=summarizer_adapter,
+            summarizer_model=summarizer_model_id,
+        )
+
+
+async def quick_answer(
+    session_id: str,
+    question: str,
+    primary_adapter: BaseAdapter,
+    primary_model: ModelInfo,
+    active_project: str = "",
+    task_type: TaskType = TaskType.GENERAL,
+    summarizer_adapter: BaseAdapter | None = None,
+    summarizer_model: str = "",
+    on_token: Callable[[str], None] | None = None,
+    cfg: dict | None = None,
+    failover_adapter: BaseAdapter | None = None,
+    failover_model: ModelInfo | None = None,
+) -> str:
+    """
+    Lightweight path — no pipeline, no review, no clarify.
+    Injects typed project context + recent history and answers directly.
+    Used by --ask flag and `polymath ask` CLI.
+
+    If a failover adapter is provided, races primary + failover at the stream
+    level: first to emit a token wins the terminal, loser is silently cancelled.
+    """
+    on_token = on_token or (lambda t: None)
+    cfg = cfg or {}
+
+    context = await build_context(
+        session_id,
+        summarizer=summarizer_adapter,
+        summarizer_model=summarizer_model,
+    )
+    context.append(Message(role="user", content=question))
+
+    project_ctx = (
+        await retrieve_or_fallback(active_project, question, task_type, cfg)
+        if active_project else ""
+    )
+    system = "Answer concisely and directly."
+    if project_ctx:
+        system = f"{project_ctx}\n\n{system}"
+
+    if failover_adapter is not None and failover_model is not None:
+        def make_stream(adapter: BaseAdapter, model_id: str):
+            def factory():
+                return adapter.stream(
+                    messages=context,
+                    model_id=model_id,
+                    system=system,
+                )
+            return factory
+
+        outcome = await race_streams(
+            candidates=[
+                StreamCandidate(id=primary_model.id, factory=make_stream(primary_adapter, primary_model.id)),
+                StreamCandidate(id=failover_model.id, factory=make_stream(failover_adapter, failover_model.id)),
+            ],
+            on_token=on_token,
+        )
+        output = outcome.output
+        winner_id = outcome.winner_id
+    else:
+        output = ""
+        async for token in primary_adapter.stream(
+            messages=context,
+            model_id=primary_model.id,
+            system=system,
+        ):
+            if isinstance(token, str):
+                output += token
+                on_token(token)
+        winner_id = primary_model.id
+
+    add_message(session_id, "user", question, step="ask")
+    add_message(session_id, "assistant", output, model=winner_id, step="ask")
+    export_session_md(session_id)
+    return output
+
+
+def _extract_questions(text: str) -> list[str]:
+    lines = text.strip().splitlines()
+    questions = []
+    for line in lines:
+        line = line.strip()
+        match = re.match(r"^[1-3][.)]\s*(.+)", line)
+        if match:
+            questions.append(match.group(1))
+        elif line.endswith("?") and len(line) > 10:
+            questions.append(line)
+    return questions[:3]