coder-eval 0.8.2__py3-none-any.whl

coder_eval/streaming/events.py

@@ -0,0 +1,198 @@
+"""Standardized streaming events for agent execution.
+
+Every event originates inside ``Agent.communicate()`` (the agent is the *single*
+emitter — the orchestrator is a pure consumer). Events form a tree mirroring
+execution and close every Start with a matching End on every exit path:
+
+    AgentStartEvent(thread_id=A)
+      TurnStartEvent(thread_id=A, turn_id=t1)
+        TextChunkEvent(turn_id=t1)
+        ToolStartEvent(turn_id=t1, tool)
+        ToolEndEvent(turn_id=t1, tool, status)
+      TurnEndEvent(thread_id=A, turn_id=t1, status, tokens)
+    AgentEndEvent(thread_id=A, status, usage)
+
+Self-describing via ``thread_id`` / ``parent_thread_id`` / ``turn_id`` so a
+consumer can place each event in the tree without keeping its own state. Sub-agents
+are *not* a separate event type — a sub-agent is a nested ``AgentStart``/``AgentEnd``
+pair linked by ``parent_thread_id``.
+
+Events are Pydantic ``BaseModel``s (keyword-only construction) and reuse the leaf
+telemetry models (``TokenUsage``, ``CommandTelemetry``) rather than re-declaring
+fields.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from coder_eval.models import (
+    CommandTelemetry,
+    ResultSummary,
+    TokenUsage,
+    TranscriptMessage,
+)
+
+
+# --- Status codes -----------------------------------------------------------
+#
+# One enum per End-event level. Every End event carries the appropriate one so
+# success, tool error, permission denial, crash, timeout and orphaned tool calls
+# are a single mechanism (no ToolErrorEvent subclass + string scans).
+
+
+class ToolEndStatus(StrEnum):
+    """Terminal status of a single tool call."""
+
+    OK = "ok"  # tool ran, no error
+    ERROR = "error"  # tool ran and returned an error
+    PERMISSION_DENIED = "permission_denied"  # blocked by permission/sandbox
+    UNRESOLVED = "unresolved"  # ToolStart emitted, no result observed (crash/timeout/orphan)
+
+
+class TurnEndStatus(StrEnum):
+    """Terminal status of one inner turn (one API response / one Codex turn)."""
+
+    COMPLETED = "completed"
+    CRASHED = "crashed"
+    TIMEOUT = "timeout"
+    MAX_TURNS_EXHAUSTED = "max_turns_exhausted"
+
+
+class AgentEndStatus(StrEnum):
+    """Terminal status of one agent invocation (a whole ``communicate()`` call)."""
+
+    COMPLETED = "completed"
+    CRASHED = "crashed"
+    TIMEOUT = "timeout"
+    MAX_TURNS_EXHAUSTED = "max_turns_exhausted"
+
+
+# Reuse the canonical TranscriptMessage union (defined once in telemetry.py) so
+# AgentEndEvent carries per-message telemetry losslessly and stays in lock-step
+# with TurnRecord.messages (the deferred token path rides here, not on granular events).
+_MessageList = list[TranscriptMessage]
+
+
+class StreamEvent(BaseModel):
+    """Base class for all streaming events."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    task_id: str
+    timestamp: datetime = Field(default_factory=datetime.now)
+    thread_id: str | None = Field(
+        default=None,
+        description="Agent-context id. Main agent: SDK session id (None until assigned). Sub-agent: spawning tool id.",
+    )
+    parent_thread_id: str | None = Field(
+        default=None,
+        description="None for the main agent; the parent's thread_id for a sub-agent.",
+    )
+
+
+class AgentStartEvent(StreamEvent):
+    """Emitted on the first line of ``communicate()`` (replaces the orchestrator's TurnStartEvent)."""
+
+    kind: Literal["agent_start"] = "agent_start"
+    prompt: str = ""
+    iteration: int = 0
+    model: str | None = None
+
+
+class TurnStartEvent(StreamEvent):
+    """Emitted when one inner turn begins (Claude: per API call; Codex: once per communicate())."""
+
+    kind: Literal["turn_start"] = "turn_start"
+    turn_id: str = ""
+    model: str | None = None
+
+
+class TextChunkEvent(StreamEvent):
+    """Emitted when the agent produces visible text output within a turn."""
+
+    kind: Literal["text_chunk"] = "text_chunk"
+    turn_id: str = ""
+    text: str = ""
+
+
+class ToolStartEvent(StreamEvent):
+    """Emitted when the agent invokes a tool (the tool's result fields are pending)."""
+
+    kind: Literal["tool_start"] = "tool_start"
+    turn_id: str = ""
+    tool: CommandTelemetry
+
+
+class ToolEndEvent(StreamEvent):
+    """Emitted when a tool returns its result (or is force-closed as unresolved on crash)."""
+
+    kind: Literal["tool_end"] = "tool_end"
+    turn_id: str = ""
+    tool: CommandTelemetry
+    status: ToolEndStatus = ToolEndStatus.OK
+
+
+class TurnEndEvent(StreamEvent):
+    """Emitted at each inner-turn boundary; carries best-effort per-turn tokens."""
+
+    kind: Literal["turn_end"] = "turn_end"
+    turn_id: str = ""
+    status: TurnEndStatus = TurnEndStatus.COMPLETED
+    tokens: TokenUsage | None = None
+
+
+class AgentEndEvent(StreamEvent):
+    """Emitted from ``finally`` at the end of ``communicate()`` — the turn finalization payload.
+
+    Carries the cumulative, authoritative ``usage`` plus the fields the
+    ``EventCollector`` cannot losslessly derive from granular events (the per-message
+    telemetry / token path the plan defers). ``EventCollector`` reduces commands from
+    the ``ToolEnd`` stream and reads everything else here.
+    """
+
+    kind: Literal["agent_end"] = "agent_end"
+    status: AgentEndStatus = AgentEndStatus.COMPLETED
+    usage: TokenUsage = Field(default_factory=TokenUsage)
+
+    # Finalization payload (the deferred token/messages path rides here).
+    iteration: int = 0
+    user_input: str = ""
+    agent_output: str = ""
+    model_used: str | None = None
+    assistant_turn_count: int = 0
+    messages: _MessageList = Field(default_factory=list)
+    num_turns: int | None = None
+    max_turns_exhausted: bool = False
+    result_summary: ResultSummary | None = None
+    crashed: bool = False
+    crash_reason: str | None = None
+    duration_seconds: float = 0.0
+
+
+# --- Post-evaluation events (orchestrator-owned, not part of the agent lifecycle) ---
+
+
+class CriterionSummary(BaseModel):
+    """Summary of a single criterion check for display."""
+
+    criterion_type: str = ""
+    description: str = ""
+    score: float = 0.0
+    passed: bool = False
+    failure_reason: str | None = None
+
+
+class CriteriaCheckEvent(StreamEvent):
+    """Emitted after success criteria are evaluated."""
+
+    kind: Literal["criteria_check"] = "criteria_check"
+    passed: int = 0
+    total: int = 0
+    weighted_score: float = 0.0
+    details: list[str] = Field(default_factory=list)
+    criteria: list[CriterionSummary] = Field(default_factory=list)

coder_eval/streaming/renderers.py

@@ -0,0 +1,248 @@
+"""Rich terminal renderer for streaming events."""
+
+import logging
+import threading
+
+from rich.console import Console
+from rich.markup import escape
+
+from coder_eval.formatting import format_payload, format_token_usage
+from coder_eval.models import ResultSummary
+from coder_eval.streaming.events import (
+    AgentEndEvent,
+    AgentStartEvent,
+    CriteriaCheckEvent,
+    CriterionSummary,
+    StreamEvent,
+    TextChunkEvent,
+    ToolEndEvent,
+    ToolEndStatus,
+    ToolStartEvent,
+    TurnEndEvent,
+    TurnStartEvent,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+# Budgets tuned so that a typical JSON payload (agent params + one tool
+# result) fits without truncation but runaway stdout still stays bounded.
+_MAX_PARAMS_LEN = 800
+_MAX_RESULT_LEN = 800
+
+
+def _truncate(text: str, max_len: int) -> str:
+    """Truncate text with ellipsis if it exceeds max_len."""
+    if len(text) <= max_len:
+        return text
+    return text[: max_len - 3] + "..."
+
+
+def _format_result_error(summary: ResultSummary | None) -> str | None:
+    """Compact SDK error detail from a ResultSummary, or None when not an error.
+
+    Surfaces the diagnostic detail of an ``is_error`` ResultMessage even on
+    exits the agent treats as clean (a benign error-subtype result, or a
+    ``max_turns`` short-circuit) — paths where ``crash_reason`` is unset
+    because the turn never raised. Both renderers share this so the detail is
+    rendered identically on the console and in task.log.
+    """
+    if summary is None or not summary.is_error:
+        return None
+    parts: list[str] = []
+    if summary.subtype:
+        parts.append(f"subtype={summary.subtype}")
+    if summary.stop_reason:
+        parts.append(f"stop_reason={summary.stop_reason}")
+    if summary.result:
+        parts.append(f"result={_truncate(summary.result, _MAX_RESULT_LEN)}")
+    return ", ".join(parts) if parts else "(no detail)"
+
+
+class RichStreamRenderer:
+    """Renders streaming events to a Rich console."""
+
+    def __init__(
+        self,
+        console: Console | None = None,
+        verbosity: str = "full",
+        batch_mode: bool = False,
+    ) -> None:
+        self._console = console or Console(stderr=True)
+        self._verbosity = verbosity
+        self._batch_mode = batch_mode
+        self._lock = threading.Lock()
+
+    def on_event(self, event: StreamEvent) -> None:
+        """Render a streaming event to the console."""
+        if self._verbosity == "minimal" and isinstance(event, (ToolStartEvent, ToolEndEvent, TextChunkEvent)):
+            return
+
+        line = self._format_event(event)
+        if line is None:
+            return
+
+        if self._batch_mode:
+            line = f"[dim]\\[{escape(event.task_id)}][/dim] {line}"
+
+        with self._lock:
+            self._console.print(line, highlight=False)
+
+    def _format_event(self, event: StreamEvent) -> str | None:
+        """Format a single event into a Rich markup string."""
+        if isinstance(event, AgentStartEvent):
+            model = f" (model={escape(event.model)})" if event.model else ""
+            return f"[bold]--- Iteration {event.iteration}{model} ---[/bold]"
+
+        # TurnStartEvent is intentionally not rendered on the console: the Rich
+        # view is a terse live feed and Claude emits one per API call (noisy).
+        # The full turn tree lives in task.log via LoggingStreamRenderer.
+
+        if isinstance(event, ToolStartEvent):
+            params_str = escape(format_payload(event.tool.parameters, max_chars=_MAX_PARAMS_LEN))
+            return f"[cyan]>>> TOOL: {escape(event.tool.tool_name)}[/cyan] | {params_str}"
+
+        if isinstance(event, ToolEndEvent):
+            preview = escape(event.tool.result_summary or "")
+            if event.status == ToolEndStatus.OK:
+                tag = "[green]<<< OK[/green]"
+            elif event.status == ToolEndStatus.UNRESOLVED:
+                tag = "[yellow]<<< UNRESOLVED:[/yellow]"
+            else:
+                tag = f"[red]<<< {escape(event.status.value.upper())}:[/red]"
+            return f"{tag} ({len(preview)} chars) {preview}"
+
+        if isinstance(event, TextChunkEvent):
+            return f"[dim]{escape(event.text)}[/dim]"
+
+        if isinstance(event, AgentEndEvent):
+            usage_str = escape(format_token_usage(event.usage))
+            line = (
+                f"[bold]--- Turn complete: {len(event.messages)} msgs, "
+                f"{event.duration_seconds:.1f}s, {usage_str} ---[/bold]"
+            )
+            if event.max_turns_exhausted:
+                line += " [yellow](max_turns exhausted)[/yellow]"
+            if event.crashed and event.crash_reason:
+                line += f"\n[red]    reason: {escape(event.crash_reason)}[/red]"
+            error_detail = _format_result_error(event.result_summary)
+            if error_detail is not None:
+                line += f"\n[red]    detail: {escape(error_detail)}[/red]"
+            return line
+
+        if isinstance(event, CriteriaCheckEvent):
+            score_color = "green" if event.passed == event.total else "yellow"
+            header = (
+                f"[{score_color}]Criteria: {event.passed}/{event.total} passed"
+                + f" (score: {event.weighted_score:.3f})[/{score_color}]"
+            )
+            if event.criteria:
+                return self._format_criteria_details(header, event.criteria)
+            # Fallback to legacy flat details
+            if event.details:
+                header += f" \\[{escape(' | '.join(event.details))}]"
+            return header
+
+        return None
+
+    @staticmethod
+    def _format_criteria_details(header: str, criteria: list[CriterionSummary]) -> str:
+        """Format criteria with per-criterion lines and failure reasons.
+
+        For failed criteria, the first line of the failure reason is shown
+        at normal brightness; subsequent lines are dimmed.
+        """
+        lines = [header]
+        for c in criteria:
+            if c.passed:
+                lines.append(f"  [green]PASS[/green]  {escape(c.criterion_type)}  {escape(c.description)}")
+            else:
+                lines.append(f"  [red]FAIL[/red]  {escape(c.criterion_type)}  {escape(c.description)}")
+                if c.failure_reason:
+                    reason_lines = c.failure_reason.splitlines()
+                    lines.append(f"        > {escape(reason_lines[0])}")
+                    for extra in reason_lines[1:]:
+                        lines.append(f"        [dim]{escape(extra)}[/dim]")
+        return "\n".join(lines)
+
+
+class LoggingStreamRenderer:
+    """Logs streaming events to the task logger (for task.log file capture).
+
+    Converts stream events into DEBUG log lines, making them available for
+    task.log persistence. This is the single, agent-agnostic place where the
+    event stream becomes task.log lines — agents emit events and get logging
+    for free, with no per-agent message-dumping.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+
+    def on_event(self, event: StreamEvent) -> None:
+        """Log a streaming event to the task logger."""
+        line = self._format_event(event)
+        if line is None:
+            return
+
+        with self._lock:
+            logger.debug(line)
+
+    def _format_event(self, event: StreamEvent) -> str | None:
+        """Format a single event into a log line (plain text, no Rich markup).
+
+        Logs lifecycle + tool boundaries; skips TextChunkEvent to avoid
+        cluttering task.log with streaming text chunks.
+        """
+        if isinstance(event, AgentStartEvent):
+            model = f" (model={event.model})" if event.model else ""
+            return f"[{event.task_id}] --- Iteration {event.iteration}{model} ---"
+
+        if isinstance(event, TurnStartEvent):
+            model = f" model={event.model}" if event.model else ""
+            return f"[{event.task_id}] >>> Turn start: id={event.turn_id}{model}"
+
+        if isinstance(event, ToolStartEvent):
+            params_str = format_payload(event.tool.parameters, max_chars=_MAX_PARAMS_LEN)
+            return (
+                f"[{event.task_id}] >>> TOOL CALL: {event.tool.tool_name} "
+                f"| id={event.tool.tool_id} | params={params_str}"
+            )
+
+        if isinstance(event, ToolEndEvent):
+            return (
+                f"[{event.task_id}] <<< TOOL RESULT [{event.status.value.upper()}]: "
+                f"id={event.tool.tool_id} | {event.tool.result_summary or ''}"
+            )
+
+        if isinstance(event, TextChunkEvent):
+            # Skip text chunks to avoid cluttering logs with streaming text
+            return None
+
+        if isinstance(event, TurnEndEvent):
+            tok = format_token_usage(event.tokens) if event.tokens is not None else "n/a"
+            return f"[{event.task_id}] --- Turn end [{event.status.value}]: {tok} ---"
+
+        if isinstance(event, AgentEndEvent):
+            usage_str = format_token_usage(event.usage)
+            line = (
+                f"[{event.task_id}] --- Agent complete [{event.status.value}]: "
+                f"{len(event.messages)} msgs, {event.duration_seconds:.1f}s, {usage_str} ---"
+            )
+            if event.max_turns_exhausted:
+                line += " (max_turns exhausted)"
+            if event.crashed and event.crash_reason:
+                line += f"\n[{event.task_id}]     reason: {event.crash_reason}"
+            error_detail = _format_result_error(event.result_summary)
+            if error_detail is not None:
+                line += f"\n[{event.task_id}]     detail: {error_detail}"
+            return line
+
+        if isinstance(event, CriteriaCheckEvent):
+            details = " | ".join(event.details) if event.details else f"{event.passed}/{event.total}"
+            return (
+                f"[{event.task_id}] Criteria: {event.passed}/{event.total} passed "
+                f"(score: {event.weighted_score:.3f}) [{details}]"
+            )
+
+        return None

coder_eval/streaming/wire.py

@@ -0,0 +1,115 @@
+"""Line-based wire format for streaming events across a process boundary.
+
+Used by the Docker isolation path: the in-container CLI installs a callback
+that writes prefixed lines to stdout; ``DockerRunner`` on the host parses them
+and forwards to the original host callback. Plain log lines without the prefix
+pass through to docker.log unchanged.
+
+Each line is ``LINE_PREFIX`` (the RS-framed sentinel ``"\x1ecoder-eval-stream\x1e:"``)
+followed by ``{"cls": <event class name>, "data": <event.model_dump(mode="json")>}``.
+The control-char sentinel (not a plain ``STREAM_EVENT:`` token) avoids collisions
+with ordinary log output. Datetimes are encoded as ISO strings by Pydantic.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+
+from coder_eval.streaming.events import (
+    AgentEndEvent,
+    AgentStartEvent,
+    CriteriaCheckEvent,
+    StreamEvent,
+    TextChunkEvent,
+    ToolEndEvent,
+    ToolStartEvent,
+    TurnEndEvent,
+    TurnStartEvent,
+)
+
+
+logger = logging.getLogger(__name__)
+
+# ASCII Record Separator (U+001E) framing the sentinel makes collision
+# with real agent / tool / pytest output effectively impossible: control
+# chars below 0x20 don't appear in normal stdout. A line that starts with
+# this exact byte sequence is a streamed event by construction.
+LINE_PREFIX = "\x1ecoder-eval-stream\x1e:"
+
+_EVENT_CLASSES: dict[str, type[StreamEvent]] = {
+    cls.__name__: cls
+    for cls in [
+        AgentStartEvent,
+        TurnStartEvent,
+        ToolStartEvent,
+        ToolEndEvent,
+        TextChunkEvent,
+        TurnEndEvent,
+        AgentEndEvent,
+        CriteriaCheckEvent,
+    ]
+}
+
+
+def serialize_event(event: StreamEvent) -> str:
+    """Render a single event as a one-line wire string (no trailing newline)."""
+    return LINE_PREFIX + json.dumps({"cls": type(event).__name__, "data": event.model_dump(mode="json")})
+
+
+def has_prefix(line: str) -> bool:
+    """Cheap check for whether a line *claims* to be an event.
+
+    Lets callers distinguish "not an event, log as-is" from "parse
+    failed, also log as-is but warn" -- the latter is a wire-format
+    bug worth surfacing.
+    """
+    return line.startswith(LINE_PREFIX)
+
+
+def deserialize_event(line: str) -> StreamEvent | None:
+    """Parse a streamed-event line back into a StreamEvent.
+
+    Returns ``None`` for any line without the prefix OR a prefixed line
+    that fails to parse. The control-char prefix makes false positives
+    practically impossible; a malformed prefixed line is logged at WARN
+    so the caller can preserve the raw line in docker.log.
+    """
+    if not line.startswith(LINE_PREFIX):
+        return None
+    try:
+        payload = json.loads(line[len(LINE_PREFIX) :])
+        cls_name = payload["cls"]
+        data = payload["data"]
+        cls = _EVENT_CLASSES.get(cls_name)
+        if cls is None:
+            logger.warning("Unknown stream event class %r; dropping", cls_name)
+            return None
+        # Pydantic handles datetime + nested-model reconstruction from the dict.
+        return cls.model_validate(data)
+    except Exception as exc:
+        # Prefix matched but JSON / class / construction failed. Log loud --
+        # this is a wire-format bug, not benign noise.
+        logger.warning("Failed to deserialize stream event %r: %s", line[:80], exc)
+        return None
+
+
+class StdoutNDJsonCallback:
+    """StreamCallback that writes each event as a ``STREAM_EVENT:`` line to stdout.
+
+    Used inside the Docker container; the host's DockerRunner picks the
+    lines back up and forwards them to the host-side callback. Lines are
+    flushed immediately so the host sees events in real time.
+    """
+
+    def on_event(self, event: StreamEvent) -> None:
+        # Plain print + flush -- the in-container Python is line-buffered
+        # under non-tty stdout, so explicit flush matters.
+        # BrokenPipeError guard: if the host got SIGKILL'd or docker-kill'd
+        # the container mid-run, our stdout pipe peer is gone. Letting
+        # BrokenPipeError propagate would crash whatever in-container code
+        # path emitted the event (typically the orchestrator's run loop)
+        # and prevent task.json from being written for partial results.
+        with contextlib.suppress(BrokenPipeError, OSError):
+            print(serialize_event(event), flush=True)