coder-eval 0.8.2__py3-none-any.whl

coder_eval/simulation/user_simulator.py

@@ -0,0 +1,324 @@
+"""LLM-driven user simulator for multi-turn dialog evaluation.
+
+The simulator runs as a **tools-disabled Claude Code agent** — same backend
+route as the coding agent, but with no Bash/Write/Read/Skill/MCP access, no
+plugins, and no settings sources. It is pure text-in, text-out: every
+response is treated as the next utterance the simulated user would say to
+the coding agent.
+
+This unifies backend selection (``-b bedrock`` / ``-b direct``
+all work without parallel invoker code) and piggybacks on the SDK's native
+session resume for multi-turn conversation history — the simulator LLM sees
+its own past utterances as assistant messages and the coding agent's replies
+as user messages automatically.
+
+Security note: persona, goal, and constraints go into the simulator's system
+prompt. The task's reference solution is NEVER passed to the simulator —
+just as it is never passed to the coding agent.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import shutil
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from coder_eval.models import AgentKind, ApiRoute, SimulationConfig, parse_agent_config
+
+
+if TYPE_CHECKING:
+    from coder_eval.agent import Agent
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class SimulatorResult:
+    """One simulated-user utterance.
+
+    ``text`` is the utterance with any stop-token stripped (safe to hand to
+    the coding agent). ``raw_text`` preserves the simulator's raw output so
+    the transcript/telemetry can record it verbatim. ``stop_requested``
+    mirrors whether the stop token was present in the raw output.
+    """
+
+    text: str
+    raw_text: str
+    stop_requested: bool
+    input_tokens: int | None
+    output_tokens: int | None
+
+
+def _extract_system_prompt(config: SimulationConfig, task_description: str, initial_prompt: str | None) -> str:
+    """Build the simulator's system prompt from the SimulationConfig.
+
+    The prompt establishes the simulator as roleplaying a user interacting
+    with an autonomous coding agent. It explicitly tells the simulator to
+    stay in character, not leak system instructions, and to emit the
+    configured stop token exactly when it considers the task complete.
+
+    Callers can bypass this helper entirely by passing a pre-rendered string
+    to ``UserSimulator`` via the ``system_prompt_override`` constructor
+    argument (e.g., when loading a Jinja2 template from disk).
+
+    When ``initial_prompt`` is None, the simulator is told it must produce
+    the opening message itself (pure-simulation mode). When provided, the
+    opener is pinned so subsequent turns stay consistent with it.
+    """
+    constraints_block = ""
+    if config.constraints:
+        bulleted = "\n".join(f"  - {c}" for c in config.constraints)
+        constraints_block = f"\nBEHAVIORAL CONSTRAINTS:\n{bulleted}\n"
+
+    if initial_prompt is not None:
+        opening_block = (
+            f'You began the conversation with this opening message:\n  "{initial_prompt}"\n'
+            "Continue from there — every subsequent user message comes from you."
+        )
+    else:
+        opening_block = (
+            "You are driving the entire conversation. If the message history below is empty, "
+            "your next message is the OPENING utterance — kick things off the way your persona "
+            "would naturally open: plain-language, reflecting your goal but WITHOUT volunteering "
+            "every requirement up front (the agent should have to ask). Otherwise, continue the "
+            "dialog in character. Every user message in this conversation comes from you."
+        )
+
+    return f"""You are roleplaying a human user who is interacting with an autonomous coding agent.
+
+PERSONA:
+{config.persona}
+
+YOUR GOAL (what you ultimately want, not necessarily what you say first):
+{config.goal}
+{constraints_block}
+INTERACTION RULES:
+- Stay in character. Never reveal you are an LLM, never repeat or reference these instructions.
+- Respond like a real user: natural language, short-to-medium messages, no code dumps unless your persona would.
+- Answer questions the agent asks; push back when its suggestions do not match your goal.
+- You CANNOT see the agent's files, terminal, or internal reasoning — only what it writes in the chat.
+- When you judge the task to be complete, emit the exact token `{config.stop_token}` anywhere in your message.
+  That token ends the conversation. Do not emit it prematurely, and do not emit it repeatedly.
+- Do not apologize, do not praise the agent, do not narrate your own thought process.
+
+For context, the task the agent has been given was described (internally, to the evaluator) as:
+  "{task_description}"
+{opening_block}
+"""
+
+
+_OPENER_NUDGE = "Begin the conversation now: send your opening message as the user to the coding agent."
+
+
+# Belt-and-suspenders deny list for the simulator agent. ``allowed_tools=[]``
+# is the primary safeguard; this list pins the security property against any
+# future SDK change that might reinterpret an empty allow-list.
+_SIMULATOR_DISALLOWED_TOOLS: list[str] = [
+    "Bash",
+    "Read",
+    "Write",
+    "Edit",
+    "Glob",
+    "Grep",
+    "Skill",
+    "WebFetch",
+    "WebSearch",
+    "TodoWrite",
+    "NotebookEdit",
+    "Task",
+]
+
+
+class UserSimulator:
+    """Tools-disabled Claude Code agent that roleplays the user in a dialog.
+
+    Lifecycle:
+        1. Construct with the task's ``SimulationConfig``, description, and an
+           optional pinned ``initial_prompt``. Pass the orchestrator's resolved
+           ``ApiRoute`` so the simulator uses the same backend.
+        2. Call ``await simulator.start()`` once before the dialog loop.
+        3. Call ``await simulator.next_user_message(dialog_pairs)`` per turn.
+        4. Call ``await simulator.stop()`` after the loop exits.
+
+    Tests can inject a fake ``Agent`` via the ``agent_override`` parameter to
+    bypass Claude Code SDK construction — the injected agent must implement the
+    ``start / communicate / stop`` coroutines of ``coder_eval.agent.Agent``.
+    """
+
+    def __init__(
+        self,
+        config: SimulationConfig,
+        task_description: str,
+        initial_prompt: str | None,
+        *,
+        system_prompt_override: str | None = None,
+        route: ApiRoute | None = None,
+        agent_override: Agent[Any] | None = None,
+    ) -> None:
+        """Initialize the simulator.
+
+        Args:
+            config: Simulation configuration (persona, goal, model, etc.).
+            task_description: The task's human-readable description.
+            initial_prompt: The first user message sent to the coding agent,
+                if the task pinned one. When ``None`` (pure-simulation mode),
+                the simulator generates the opening utterance itself on the
+                first ``next_user_message([])`` call.
+            system_prompt_override: Pre-rendered system prompt — used when
+                the caller loaded a custom Jinja2 template.
+            route: ``ApiRoute`` the orchestrator resolved for the coding
+                agent — the simulator agent uses the same one.
+            agent_override: Test-only — a pre-built ``Agent`` (typically a fake)
+                used instead of constructing a Claude Code agent at ``start()``.
+        """
+        self.config = config
+        self.task_description = task_description
+        self.initial_prompt: str | None = initial_prompt
+        self._system_prompt = (
+            system_prompt_override
+            if system_prompt_override is not None
+            else _extract_system_prompt(config, task_description, initial_prompt)
+        )
+
+        self._route: ApiRoute | None = route
+        self._agent_override: Agent[Any] | None = agent_override
+        self._agent: Agent[Any] | None = None
+        self._scratch_dir: Path | None = None
+
+        # model is intentionally left to the route: ClaudeCodeAgent._build_sdk_env
+        # maps BedrockRoute.model → ANTHROPIC_MODEL env, so pinning a Gateway-style
+        # name here (e.g. "anthropic.claude-sonnet-4-6") would break Bedrock runs.
+        # For direct routes, None lets the SDK pick its default.
+        #
+        # allowed_tools=[] is the primary guarantee that the simulator cannot
+        # touch files or run commands. The disallowed_tools list below is
+        # belt-and-suspenders against a future SDK change where an empty
+        # allow-list silently means "allow everything" — every common tool is
+        # named explicitly so a regression surfaces as a deny rather than as
+        # a security failure.
+        from coder_eval.models import ClaudeCodeAgentConfig
+
+        agent_config = parse_agent_config(
+            type=AgentKind.CLAUDE_CODE,
+            model=None,
+            allowed_tools=[],
+            disallowed_tools=_SIMULATOR_DISALLOWED_TOOLS,
+            plugins=None,
+            setting_sources=[],
+            permission_mode="default",
+            system_prompt=self._system_prompt,
+        )
+        # parse_agent_config returns a union, but type=CLAUDE_CODE guarantees ClaudeCodeAgentConfig
+        assert isinstance(agent_config, ClaudeCodeAgentConfig)
+        self._agent_config = agent_config
+
+        if route is not None:
+            logger.info("User simulator: Claude Code agent backend (route=%s)", type(route).__name__)
+        else:
+            logger.info("User simulator: Claude Code agent backend (default route)")
+
+    @property
+    def system_prompt(self) -> str:
+        """The fully-rendered system prompt given to the simulator LLM."""
+        return self._system_prompt
+
+    async def _remove_scratch_dir(self) -> None:
+        """Remove the ephemeral scratch dir if present and reset the handle.
+
+        Shared by stop() and start()'s failure path so the rmtree idiom lives in
+        one place. Best-effort (ignore_errors=True); idempotent (safe to call twice).
+        """
+        if self._scratch_dir is not None and self._scratch_dir.exists():
+            await asyncio.to_thread(shutil.rmtree, self._scratch_dir, ignore_errors=True)
+        self._scratch_dir = None
+
+    async def start(self) -> None:
+        """Spin up the underlying Claude Code agent in an ephemeral scratch dir.
+
+        The scratch dir stays empty — the simulator has no tools and therefore
+        no way to touch it — but Claude Code requires a ``cwd``.
+        """
+        if not self.config.enabled:
+            return
+        self._scratch_dir = Path(tempfile.mkdtemp(prefix="sim-"))
+        try:
+            if self._agent_override is not None:
+                self._agent = self._agent_override
+            else:
+                from coder_eval.agents.claude_code_agent import ClaudeCodeAgent
+
+                self._agent = ClaudeCodeAgent(self._agent_config, route=self._route, instance_name="simulator")
+            await self._agent.start(str(self._scratch_dir))
+        except BaseException:
+            # Agent construction or _agent.start() failed (SDK/transport
+            # startup, missing CLI, bad config, or cancellation). The dialog
+            # loop's finally (its only caller of stop()) is never entered on
+            # this path, so clean up our own scratch dir here to avoid a sim-*
+            # tempdir leak that compounds across a batch of simulation tasks.
+            # Wrapping construction too (not just start) closes the leak for
+            # every failure after mkdtemp. Re-raise to preserve the original
+            # failure (incl. cancellation/interrupt) semantics.
+            await self._remove_scratch_dir()
+            self._agent = None
+            raise
+
+    async def stop(self) -> None:
+        """Tear down the simulator agent and remove its scratch dir."""
+        if self._agent is not None:
+            try:
+                await self._agent.stop()
+            except Exception:
+                logger.exception("Error stopping simulator agent")
+            self._agent = None
+        await self._remove_scratch_dir()
+
+    async def next_user_message(self, dialog_pairs: list[tuple[str, str]]) -> SimulatorResult:
+        """Generate the next simulated-user utterance.
+
+        Sends the coding agent's latest reply (or an opener-nudge on turn 1
+        in pure-sim mode) to the simulator Claude Code agent and returns its
+        response.
+
+        Args:
+            dialog_pairs: The dialog history so far, ordered oldest-first.
+                Each entry is ``(clean_user_text, agent_text)``. Only the most
+                recent agent reply is sent — conversation state is carried by
+                the SDK's session resume.
+
+        Returns:
+            A ``SimulatorResult`` with the next user utterance. When the
+            simulator emits the configured stop token, ``stop_requested``
+            is True and ``text`` has the token stripped.
+        """
+        from coder_eval.simulation.termination import strip_stop_token
+
+        assert self._agent is not None, "UserSimulator.start() must be called before next_user_message()"
+        prompt = dialog_pairs[-1][1] if dialog_pairs else _OPENER_NUDGE
+        # Simulator emits one user utterance per call, so cap the inner loop at 1 turn.
+        turn = await self._agent.communicate(prompt, max_turns=1)
+        raw = turn.agent_output or ""
+        usage = turn.token_usage
+        input_tokens = usage.uncached_input_tokens if usage is not None else None
+        output_tokens = usage.output_tokens if usage is not None else None
+
+        stop_requested = self.config.stop_token in raw
+        cleaned = strip_stop_token(raw, self.config.stop_token) if stop_requested else raw.strip()
+
+        # Guard against empty cleaned text after stripping the stop token —
+        # the agent still needs *something* to react to, and the dialog
+        # terminates on this turn anyway.
+        if stop_requested and not cleaned:
+            cleaned = "(the user indicated the task is complete)"
+
+        return SimulatorResult(
+            text=cleaned,
+            raw_text=raw,
+            stop_requested=stop_requested,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+        )

coder_eval/streaming/__init__.py

@@ -0,0 +1,44 @@
+"""Real-time streaming of agent events to the terminal."""
+
+from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback, TaskScopedCallback, safe_emit
+from coder_eval.streaming.collector import EventCollector
+from coder_eval.streaming.events import (
+    AgentEndEvent,
+    AgentEndStatus,
+    AgentStartEvent,
+    CriteriaCheckEvent,
+    CriterionSummary,
+    StreamEvent,
+    TextChunkEvent,
+    ToolEndEvent,
+    ToolEndStatus,
+    ToolStartEvent,
+    TurnEndEvent,
+    TurnEndStatus,
+    TurnStartEvent,
+)
+from coder_eval.streaming.renderers import LoggingStreamRenderer, RichStreamRenderer
+
+
+__all__ = [
+    "AgentEndEvent",
+    "AgentEndStatus",
+    "AgentStartEvent",
+    "CompositeStreamCallback",
+    "CriteriaCheckEvent",
+    "CriterionSummary",
+    "EventCollector",
+    "LoggingStreamRenderer",
+    "RichStreamRenderer",
+    "StreamCallback",
+    "StreamEvent",
+    "TaskScopedCallback",
+    "TextChunkEvent",
+    "ToolEndEvent",
+    "ToolEndStatus",
+    "ToolStartEvent",
+    "TurnEndEvent",
+    "TurnEndStatus",
+    "TurnStartEvent",
+    "safe_emit",
+]

coder_eval/streaming/callbacks.py

@@ -0,0 +1,57 @@
+"""Streaming callback protocol and helpers."""
+
+import logging
+from typing import Protocol
+
+from coder_eval.streaming.events import StreamEvent
+
+
+logger = logging.getLogger(__name__)
+
+
+class StreamCallback(Protocol):
+    """Protocol for receiving streaming events."""
+
+    def on_event(self, event: StreamEvent) -> None:
+        """Handle a streaming event."""
+        pass
+
+
+class TaskScopedCallback:
+    """Wrapper that overrides task_id on all events before forwarding to the inner callback.
+
+    Used by the orchestrator to ensure agent-emitted events carry the correct
+    task ID, since agents don't know about evaluation-level task identity.
+    """
+
+    def __init__(self, inner: StreamCallback, task_id: str) -> None:
+        self._inner = inner
+        self._task_id = task_id
+
+    def on_event(self, event: StreamEvent) -> None:
+        event.task_id = self._task_id
+        self._inner.on_event(event)
+
+
+class CompositeStreamCallback:
+    """Composite callback that forwards events to multiple handlers.
+
+    Useful for dispatching events to both logging and display renderers.
+    """
+
+    def __init__(self, callbacks: list[StreamCallback]) -> None:
+        self._callbacks = callbacks
+
+    def on_event(self, event: StreamEvent) -> None:
+        for callback in self._callbacks:
+            safe_emit(callback, event)
+
+
+def safe_emit(callback: StreamCallback | None, event: StreamEvent) -> None:
+    """Emit an event to the callback, catching and logging any exceptions."""
+    if callback is None:
+        return
+    try:
+        callback.on_event(event)
+    except Exception:
+        logger.warning("Stream callback failed (ignored)", exc_info=True)

coder_eval/streaming/collector.py

@@ -0,0 +1,193 @@
+"""EventCollector: reduce a standardized event stream into a TurnRecord.
+
+This is the single, agent-agnostic place where the persisted ``TurnRecord``
+(and therefore ``task.json``) is assembled from the event stream — so adding a
+new agent means emitting the standard events, with capture coming for free
+(no per-agent telemetry-assembly code).
+
+Reduction split:
+
+- ``commands`` are derived from the ``ToolEndEvent`` stream (every tool call,
+  including crash-orphaned ones force-closed as ``unresolved``), ordered by the
+  tool's ``sequence_number``. This is the genuine "events are the source of
+  truth" path for tool telemetry.
+- The per-message telemetry / token payload (the intricate, SDK-specific token
+  machinery the plan defers) rides on the terminal ``AgentEndEvent`` and is read
+  back verbatim — no re-derivation, so token correctness is untouched.
+
+An agent attaches its own ``EventCollector`` alongside the caller's callback and
+returns ``build_turn_record()`` from ``communicate()``; the orchestrator keeps
+reading the return value (and ``pending_turn`` on crash), now event-derived.
+"""
+
+from __future__ import annotations
+
+from coder_eval.models import (
+    AssistantMessage,
+    CommandTelemetry,
+    ReconciliationMessage,
+    TokenUsage,
+    TranscriptMessage,
+    TurnRecord,
+)
+from coder_eval.streaming.events import (
+    AgentEndEvent,
+    AgentStartEvent,
+    StreamEvent,
+    ToolEndEvent,
+    TurnStartEvent,
+)
+
+
+class EventCollector:
+    """A ``StreamCallback`` that accumulates events and builds a ``TurnRecord``.
+
+    Tolerant by construction: ``build_turn_record()`` can be called at any point
+    (including mid-stream after a crash) and returns the best record derivable
+    from the events seen so far. Sub-agent activity is captured as
+    ``parent_tool_use_id``-tagged messages in the transcript; per-sub-agent
+    attribution is derived by grouping those messages, not from a separate field.
+    """
+
+    def __init__(self) -> None:
+        self._iteration: int = 0
+        self._user_input: str = ""
+        self._model: str | None = None
+        self._turn_starts: int = 0
+        # tool_id -> finalized telemetry (last ToolEnd wins, mirroring last-result-wins).
+        self._commands: dict[str, CommandTelemetry] = {}
+        self._agent_end: AgentEndEvent | None = None
+
+    def on_event(self, event: StreamEvent) -> None:
+        # Only the main agent's own events shape its TurnRecord. This guard is
+        # forward-looking: nested sub-agent events (parent_thread_id set) are NOT
+        # emitted by any agent yet (sub-agent nesting is deferred), so this branch
+        # is currently never taken. It's here so that when nesting lands, child
+        # events are skipped here and attributed via the finalization payload
+        # rather than corrupting the main-agent record.
+        if event.parent_thread_id is not None:
+            return
+
+        if isinstance(event, AgentStartEvent):
+            self._iteration = event.iteration
+            self._user_input = event.prompt
+            if event.model:
+                self._model = event.model
+        elif isinstance(event, TurnStartEvent):
+            self._turn_starts += 1
+            if event.model:
+                self._model = event.model
+        elif isinstance(event, ToolEndEvent):
+            self._commands[event.tool.tool_id] = event.tool
+        elif isinstance(event, AgentEndEvent):
+            self._agent_end = event
+
+    def _ordered_commands(self) -> list[CommandTelemetry]:
+        return sorted(self._commands.values(), key=lambda c: c.sequence_number)
+
+    @staticmethod
+    def _reconciled_messages(messages: list[TranscriptMessage], usage: TokenUsage) -> list[TranscriptMessage]:
+        """Append a ``ReconciliationMessage`` so the transcript's token buckets
+        sum to ``usage`` (the authoritative turn total).
+
+        The per-``AssistantMessage`` stream consistently under-reports the bill —
+        a fixed prompt slice (~512 input tokens on Claude) is billed on no
+        SDK-emitted message, and sub-agent input/cache only partially bubbles up.
+        We book that residual once, explicitly, as a synthetic entry rather than
+        smearing fabricated tokens across real generations. After this, any
+        consumer that sums the four token buckets across the transcript reproduces
+        ``usage`` exactly — no separate aggregate needed. Only assistant
+        generations carry agent-billed tokens, so the residual is measured against
+        them (simulator ``UserMessage`` tokens are a separate bill). Emitted only
+        when some bucket actually diverges.
+        """
+        in_sum = out_sum = cw_sum = cr_sum = 0
+        for m in messages:
+            if isinstance(m, AssistantMessage):
+                in_sum += m.input_tokens
+                out_sum += m.output_tokens
+                cw_sum += m.cache_creation_tokens
+                cr_sum += m.cache_read_tokens
+        d_in = usage.uncached_input_tokens - in_sum
+        d_out = usage.output_tokens - out_sum
+        d_cw = usage.cache_creation_input_tokens - cw_sum
+        d_cr = usage.cache_read_input_tokens - cr_sum
+        if d_in == 0 and d_out == 0 and d_cw == 0 and d_cr == 0:
+            return messages
+        # The residual is almost always positive (tokens billed but not streamed).
+        # A negative residual means the captured generations OVER-report the turn
+        # total for some bucket; word the note for that case so a "-512" entry
+        # doesn't read as "billed but not surfaced".
+        positive = d_in >= 0 and d_out >= 0 and d_cw >= 0 and d_cr >= 0
+        note = (
+            "Tokens the agent billed but never surfaced as a generation "
+            + "(fixed prompt overhead + sub-agent input/cache the stream doesn't bubble up). "
+            + "Booked here so the transcript reconciles to the turn total."
+            if positive
+            else (
+                "Per-bucket residual reconciling the captured generations to the turn total "
+                + "(negative where the stream over-reports a bucket). "
+                + "Booked here so the transcript sums to the authoritative usage."
+            )
+        )
+        return [
+            *messages,
+            ReconciliationMessage(
+                input_tokens=d_in,
+                output_tokens=d_out,
+                cache_creation_tokens=d_cw,
+                cache_read_tokens=d_cr,
+                note=note,
+            ),
+        ]
+
+    def build_turn_record(self) -> TurnRecord:
+        """Assemble the ``TurnRecord`` from the events observed so far."""
+        end = self._agent_end
+        commands = self._ordered_commands()
+
+        if end is None:
+            # No terminal event yet (e.g. mid-stream snapshot). Return a minimal
+            # record from the granular events we have.
+            return TurnRecord(
+                iteration=self._iteration,
+                user_input=self._user_input,
+                agent_output="",
+                commands=commands,
+                token_usage=None,
+                model_used=self._model,
+                assistant_turn_count=self._turn_starts,
+            )
+
+        # Treat an all-zero, costless usage as "no usage reported" (None) so the
+        # record matches agents that surfaced nothing; otherwise carry it through.
+        tokens = end.usage
+        token_usage: TokenUsage | None = (
+            tokens if (not tokens.is_empty() or tokens.total_cost_usd is not None) else None
+        )
+
+        # The authoritative turn total (token_usage) is the source of truth, but
+        # the per-message stream under-reports it. Book the residual as a single
+        # synthetic ReconciliationMessage so the transcript's token buckets sum
+        # to the total — making the stream self-reconciling for any downstream
+        # consumer (e.g. the evalboard) without a competing aggregate.
+        messages: list[TranscriptMessage] = list(end.messages)
+        if token_usage is not None:
+            messages = self._reconciled_messages(messages, token_usage)
+
+        return TurnRecord(
+            iteration=end.iteration or self._iteration,
+            user_input=end.user_input or self._user_input,
+            agent_output=end.agent_output,
+            commands=commands,
+            duration_seconds=end.duration_seconds,
+            token_usage=token_usage,
+            model_used=end.model_used or self._model,
+            assistant_turn_count=end.assistant_turn_count,
+            messages=messages,
+            num_turns=end.num_turns,
+            max_turns_exhausted=end.max_turns_exhausted,
+            result_summary=end.result_summary,
+            crashed=end.crashed,
+            crash_reason=end.crash_reason,
+        )