coder-eval 0.8.2__py3-none-any.whl

coder_eval/agents/noop_agent.py

@@ -0,0 +1,100 @@
+"""No-op ("agentless") agent — the Null Object of the Agent hierarchy.
+
+``NoOpAgent`` binds to ``AgentKind.NONE`` (selected via ``agent: {type: none}``)
+for system / canary checks that reuse the eval infrastructure (sandbox,
+``pre_run``, reports, evalboard, ADX) without running a coding agent. Its
+``start`` / ``communicate`` / ``stop`` are no-ops and it makes no model API
+call; ``communicate`` emits the standardized event protocol for a single empty
+turn and returns the ``EventCollector``'s reduction (an empty
+:class:`~coder_eval.models.results.TurnRecord`), so the orchestrator's normal
+lifecycle runs unmodified and then checks the success criteria directly against
+the sandbox.
+
+See ``docs/TASK_DEFINITION_GUIDE.md`` (No-op / System Tasks) and issue #203.
+"""
+
+from __future__ import annotations
+
+from coder_eval.agent import Agent, AgentState
+from coder_eval.agents.registry import AgentRegistry
+from coder_eval.models import AgentKind, ApiRoute, NoneAgentConfig, TurnRecord
+from coder_eval.streaming.callbacks import CompositeStreamCallback, StreamCallback
+from coder_eval.streaming.collector import EventCollector
+from coder_eval.streaming.events import (
+    AgentEndEvent,
+    AgentEndStatus,
+    AgentStartEvent,
+    TurnEndEvent,
+    TurnEndStatus,
+    TurnStartEvent,
+)
+
+
+@AgentRegistry.register(AgentKind.NONE, NoneAgentConfig)
+class NoOpAgent(Agent[NoneAgentConfig]):
+    """Agent that does nothing — every lifecycle method is a no-op.
+
+    Created and driven by the orchestrator exactly like any other agent, so no
+    ``agentless`` branching is needed: the single signal is ``agent.type ==
+    AgentKind.NONE``. ``communicate`` is the SOLE emitter of one clean, balanced
+    event tree (``AgentStart`` -> ``TurnStart`` -> ``TurnEnd`` -> ``AgentEnd``,
+    all ``COMPLETED``) and returns the empty turn the ``EventCollector`` reduces
+    from it.
+    """
+
+    def __init__(self, config: NoneAgentConfig, route: ApiRoute | None = None) -> None:
+        self.config = config
+        self.route = route
+
+    async def start(
+        self,
+        working_directory: str,
+        *,
+        env_path_prepend: list[str] | None = None,
+        plugin_tools_dir: str | None = None,
+    ) -> None:
+        """No-op: there is no agent process to launch."""
+        self._state = AgentState.WORKING
+
+    async def communicate(
+        self,
+        user_input: str,
+        *,
+        stream_callback: StreamCallback | None = None,
+        timeout: float | None = None,
+        max_turns: int | None = None,
+    ) -> TurnRecord:
+        """Return an empty turn without contacting any model.
+
+        Honors the streaming contract — sole emitter of a balanced event tree
+        (``AgentStart`` -> ``TurnStart`` -> ``TurnEnd`` -> ``AgentEnd``) — so the
+        task-log handler and renderers see a clean turn boundary. The returned
+        ``TurnRecord`` is the ``EventCollector``'s reduction of those events.
+        """
+        self._begin_turn()
+
+        task_id = str(self.config.type)  # str() so a plugin subclass with a non-enum kind also works
+        turn_id = f"none-{self._iteration}"
+        collector = EventCollector()
+        emit = CompositeStreamCallback([c for c in (collector, stream_callback) if c is not None])
+
+        emit.on_event(AgentStartEvent(task_id=task_id, prompt=user_input, iteration=self._iteration))
+        emit.on_event(TurnStartEvent(task_id=task_id, turn_id=turn_id))
+        emit.on_event(TurnEndEvent(task_id=task_id, turn_id=turn_id, status=TurnEndStatus.COMPLETED))
+        emit.on_event(
+            AgentEndEvent(
+                task_id=task_id,
+                status=AgentEndStatus.COMPLETED,
+                iteration=self._iteration,
+                user_input=user_input,
+                agent_output="",
+                assistant_turn_count=0,
+            )
+        )
+
+        self._end_turn_ok()
+        return collector.build_turn_record()
+
+    async def stop(self) -> None:
+        """No-op: nothing to tear down."""
+        self._mark_stopped()

coder_eval/agents/registry.py

@@ -0,0 +1,163 @@
+"""Agent registration and factory pattern for BYOA (bring-your-own-agent) support."""
+
+# by-design model-hub ↔ registry type-level cycle; runtime imports are lazy per CE017
+# pyright: reportImportCycles=false
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast
+
+
+# Imports kept TYPE_CHECKING-only (with future annotations) so this module imports
+# nothing from coder_eval at runtime. That keeps the dependency one-way — the
+# plugin loader and models layer import the registry, never the reverse — so there
+# is no import cycle (CodeQL py/cyclic-import).
+if TYPE_CHECKING:
+    from coder_eval.agent import Agent
+    from coder_eval.models import AgentKind, ApiRoute, BaseAgentConfig
+
+MethodConfigT = TypeVar("MethodConfigT", bound="BaseAgentConfig")
+AgentClassT = TypeVar("AgentClassT")
+
+
+@dataclass
+class AgentRegistration[ConfigT: BaseAgentConfig]:
+    """Metadata for a registered agent.
+
+    Stores the agent class and its expected config class for runtime validation.
+    """
+
+    agent_class: type[Agent[Any]]
+    config_class: type[ConfigT]
+
+
+class AgentRegistry:
+    """Global registry for custom agents.
+
+    Provides a decorator-based registration pattern that decouples agent
+    implementations from the orchestrator factory. Keyed by the agent *kind
+    string* so a built-in :class:`AgentKind` member and a plugin-supplied raw
+    string collide on the same key (``AgentKind`` is a ``StrEnum``): an external
+    plugin can register a brand-new kind that is not an enum member.
+    """
+
+    _registry: ClassVar[dict[str, AgentRegistration[Any]]] = {}
+
+    @classmethod
+    def register(
+        cls, agent_kind: str | AgentKind, config_class: type[MethodConfigT]
+    ) -> Callable[[type[AgentClassT]], type[AgentClassT]]:
+        """Decorator to register an agent class (identity-preserving).
+
+        Usage:
+            @AgentRegistry.register(AgentKind.CLAUDE_CODE, ClaudeCodeAgentConfig)
+            class ClaudeCodeAgent(Agent[ClaudeCodeAgentConfig]):
+                ...
+
+        Args:
+            agent_kind: The agent kind this agent implements — an ``AgentKind``
+                member (built-ins) or a raw kind string (plugins).
+            config_class: The config class this agent expects (e.g., ClaudeCodeAgentConfig)
+
+        Returns:
+            A decorator that registers and returns the agent class unchanged (preserves type)
+        """
+
+        def decorator(agent_cls: type[AgentClassT]) -> type[AgentClassT]:
+            kind = str(agent_kind)
+            existing = cls._registry.get(kind)
+            # Re-registering the SAME classes is legitimate (idempotent built-in
+            # reload via load_plugins(force=True)). Re-registering a kind with a
+            # DIFFERENT implementation is a silent shadow: which agent runs would
+            # depend on entry-point discovery order, which isn't stable across
+            # environments — a reproducibility hole. Reject it loudly.
+            if existing is not None and (existing.agent_class, existing.config_class) != (agent_cls, config_class):
+                raise ValueError(
+                    f"Agent kind {kind!r} is already registered to "
+                    + f"{existing.agent_class.__name__} ({existing.config_class.__name__}); "
+                    + f"{agent_cls.__name__} ({config_class.__name__}) cannot shadow it. "
+                    + "Two plugins must not claim the same agent.type."
+                )
+            cls._registry[kind] = AgentRegistration(
+                agent_class=agent_cls,  # type: ignore[arg-type]
+                config_class=config_class,
+            )
+            return agent_cls
+
+        return decorator
+
+    @classmethod
+    def get(cls, agent_kind: str | AgentKind) -> AgentRegistration[Any] | None:
+        """Look up a registered agent by kind.
+
+        Args:
+            agent_kind: The agent kind to look up (``AgentKind`` member or raw string)
+
+        Returns:
+            AgentRegistration if found, None otherwise
+        """
+        return cls._registry.get(str(agent_kind))
+
+    @classmethod
+    def list_kinds(cls) -> list[str]:
+        """Registered agent kind strings (sorted for stable error messages)."""
+        return sorted(cls._registry)
+
+    @classmethod
+    def registrations(cls) -> list[AgentRegistration[Any]]:
+        """All registered agent registrations (for config-class enumeration)."""
+        return list(cls._registry.values())
+
+    @classmethod
+    def unregistered_kind_error(cls, agent_kind: str | AgentKind) -> ValueError:
+        """The single ``ValueError`` for an unknown kind (shared by the factory and
+        ``parse_agent_config``) so both report identically and list valid kinds."""
+        return ValueError(f"No agent registered for type {str(agent_kind)!r}. Registered kinds: {cls.list_kinds()}")
+
+
+def create_agent(
+    agent_kind: str | AgentKind,
+    config: BaseAgentConfig,
+    route: ApiRoute | None = None,
+    **kwargs: Any,
+) -> Agent[Any]:
+    """Factory function to create an agent by kind.
+
+    Validates that the config matches the agent's registered config class.
+
+    Args:
+        agent_kind: The agent kind to instantiate (``AgentKind`` member or raw string)
+        config: Configuration object (must match the registered agent's config class)
+        route: Optional API routing configuration
+        **kwargs: Additional arguments passed to the agent constructor
+
+    Returns:
+        An instance of the requested agent type
+
+    Plugins must already be loaded: callers reach a config object through
+    ``parse_agent_config`` (which loads plugins), and the orchestrator / CLI also
+    load them up-front. ``create_agent`` deliberately does NOT import
+    ``coder_eval.plugins`` itself, so ``plugins`` -> ``agents.registry`` stays a
+    one-way edge (no import cycle).
+
+    Raises:
+        ValueError: If the agent_kind is not registered
+        TypeError: If the config type doesn't match the agent's expected config class
+    """
+    registration = AgentRegistry.get(agent_kind)
+    if not registration:
+        raise AgentRegistry.unregistered_kind_error(agent_kind)
+
+    # Type check: ensure config matches the registered agent's config class
+    if not isinstance(config, registration.config_class):
+        raise TypeError(
+            f"Agent {agent_kind!r} expects {registration.config_class.__name__} "
+            + f"but received {type(config).__name__}. "
+            + f"Did you pass --type {agent_kind} with mismatched config?"
+        )
+
+    # Instantiate the agent with the typed config
+    # Cast to Any to allow pyright to resolve the generic class instantiation
+    return cast(Any, registration.agent_class)(config, route=route, **kwargs)

coder_eval/agents/watchdog.py

@@ -0,0 +1,116 @@
+"""OS-thread-based deadline enforcer for async code.
+
+Replaces ``asyncio.sleep``/``asyncio.wait_for`` as the primitive for enforcing
+wall-clock timeouts on code that may block the event loop or be wrapped in
+anyio cancel scopes that swallow asyncio.CancelledError. The timer runs on a
+daemon OS thread, so it fires even when the event loop is starved or stuck
+on subprocess I/O.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import threading
+from collections.abc import Callable
+from typing import Self
+
+
+logger = logging.getLogger(__name__)
+
+
+class ThreadedWatchdog:
+    """OS-thread-based deadline enforcer.
+
+    A ``threading.Timer`` fires at ``timeout_seconds`` and invokes ``on_timeout``
+    from the timer thread (NOT the asyncio event loop). After it fires,
+    ``fired`` is True (readable from any thread).
+
+    ``on_timeout`` MUST be synchronous and thread-safe. Exceptions it raises
+    are logged and swallowed so one bad callback never kills the timer thread
+    without a trace.
+
+    Typical use in async code:
+
+        def _on_timeout() -> None:
+            kill_subprocess_by_pid(pid)  # sync, thread-safe
+
+        with ThreadedWatchdog(
+            timeout_seconds=1200,
+            on_timeout=_on_timeout,
+            asyncio_task_to_cancel=asyncio.current_task(),
+            label="turn timeout",
+        ) as wd:
+            async for message in query(...):
+                ...
+        if wd.fired:
+            raise TurnTimeoutError(...)
+
+    Notes:
+    - ``timeout_seconds`` of None or <= 0 → no-op watchdog (no timer started).
+      This lets callers write a uniform ``with`` block regardless of whether
+      a timeout is configured.
+    - When ``asyncio_task_to_cancel`` is provided, the timer thread also
+      calls ``loop.call_soon_threadsafe(task.cancel)`` after ``on_timeout``.
+      This delivers cancellation across the thread boundary so mock-based
+      tests (no real subprocess) still unwind at the deadline.
+    - Each instance is single-use. Re-entering the ``with`` block after exit
+      is not supported.
+    """
+
+    def __init__(
+        self,
+        *,
+        timeout_seconds: float | None,
+        on_timeout: Callable[[], None],
+        asyncio_task_to_cancel: asyncio.Task[object] | None = None,
+        label: str = "watchdog",
+    ) -> None:
+        self._timeout = timeout_seconds
+        self._on_timeout = on_timeout
+        self._task_to_cancel = asyncio_task_to_cancel
+        self._label = label
+        self._timer: threading.Timer | None = None
+        self._lock = threading.Lock()
+        self._fired = False
+
+    @property
+    def fired(self) -> bool:
+        """True if the timer has fired (thread-safe)."""
+        with self._lock:
+            return self._fired
+
+    def _fire(self) -> None:
+        """Timer-thread callback. Short, exception-safe."""
+        with self._lock:
+            if self._fired:
+                return
+            self._fired = True
+        logger.warning("%s fired after %.1fs — hard-killing subprocess", self._label, self._timeout or 0)
+        try:
+            self._on_timeout()
+        except Exception:
+            # Log-and-swallow: a bad on_timeout callback must not crash the
+            # timer thread silently. `logger.exception` captures the traceback.
+            logger.exception("%s on_timeout callback raised", self._label)
+        if self._task_to_cancel is not None:
+            # Loop may already be closed (e.g. interpreter shutdown), in
+            # which case call_soon_threadsafe raises RuntimeError — we've
+            # already done the kill, the cancel is a best-effort secondary.
+            with contextlib.suppress(RuntimeError):
+                loop = self._task_to_cancel.get_loop()
+                loop.call_soon_threadsafe(self._task_to_cancel.cancel)
+
+    def __enter__(self) -> Self:
+        if self._timeout is None or self._timeout <= 0:
+            return self
+        self._timer = threading.Timer(self._timeout, self._fire)
+        self._timer.daemon = True
+        self._timer.start()
+        return self
+
+    def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
+        if self._timer is not None:
+            self._timer.cancel()
+            self._timer = None

coder_eval/analysis.py

@@ -0,0 +1,88 @@
+"""Analysis tools for evaluation results.
+
+This module provides utilities for analyzing and aggregating evaluation data,
+particularly command telemetry statistics.
+
+Separation of Concerns:
+- Orchestrator: Coordinates evaluation execution
+- Analysis: Processes results into insights
+- Reports: Formats insights for human consumption
+"""
+
+from collections import Counter
+
+from .models import CommandStatistics, CommandTelemetry, SlowestCommandInfo, TurnRecord
+
+
+def calculate_command_statistics(turns: list[TurnRecord]) -> CommandStatistics:
+    """Aggregate command telemetry across all turns, including crashed partials.
+
+    Partial-record commands are real executed work (sandbox + SDK session persist on retry).
+    """
+    all_commands: list[CommandTelemetry] = []
+    for turn in turns:
+        all_commands.extend(turn.commands)
+
+    if not all_commands:
+        return CommandStatistics(total_commands=0)
+
+    # Count by tool type
+    commands_by_tool: dict[str, int] = {}
+    for cmd in all_commands:
+        commands_by_tool[cmd.tool_name] = commands_by_tool.get(cmd.tool_name, 0) + 1
+
+    # Timing statistics (only for commands with duration data)
+    # Use `is not None` to include valid 0.0ms durations
+    total_time = sum(cmd.duration_ms for cmd in all_commands if cmd.duration_ms is not None)
+    timed_count = sum(1 for cmd in all_commands if cmd.duration_ms is not None)
+    avg_time = total_time / timed_count if timed_count > 0 else 0
+
+    # Find slowest commands (type-safe using SlowestCommandInfo model)
+    commands_with_timing = [c for c in all_commands if c.duration_ms is not None]
+    slowest = sorted(commands_with_timing, key=lambda x: x.duration_ms or 0, reverse=True)[:5]
+
+    slowest_info = [
+        SlowestCommandInfo(
+            tool=cmd.tool_name,
+            duration_ms=cmd.duration_ms if cmd.duration_ms is not None else 0.0,
+            parameters=cmd.parameters,
+            tool_id=cmd.tool_id,
+        )
+        for cmd in slowest
+    ]
+
+    # Success/failure/unknown rates
+    successful = sum(1 for c in all_commands if c.result_status == "success")
+    failed = sum(1 for c in all_commands if c.result_status == "error")
+    unknown = sum(1 for c in all_commands if c.result_status == "unknown" or c.result_status is None)
+
+    # Calculate success rate excluding unknown (avoid division by zero)
+    known_commands = successful + failed
+    success_rate = (successful / known_commands * 100) if known_commands > 0 else 0.0
+
+    # Most common command sequence (3-grams)
+    sequences = []
+    for turn in turns:
+        cmds = [c.tool_name for c in turn.commands]
+        # Build 3-command sequences
+        for i in range(len(cmds) - 2):
+            sequences.append(f"{cmds[i]} → {cmds[i + 1]} → {cmds[i + 2]}")
+
+    # Find most frequent sequence (handle edge case of no sequences)
+    most_common = None
+    if sequences:
+        counter = Counter(sequences)
+        most_common = counter.most_common(1)[0][0]
+
+    return CommandStatistics(
+        total_commands=len(all_commands),
+        commands_by_tool=commands_by_tool,
+        total_command_time_ms=total_time,
+        avg_command_time_ms=avg_time,
+        slowest_commands=slowest_info,
+        successful_commands=successful,
+        failed_commands=failed,
+        unknown_commands=unknown,
+        success_rate=success_rate,
+        most_common_sequence=most_common,
+    )

coder_eval/cli/__init__.py

@@ -0,0 +1,87 @@
+"""Command-line interface for coder_eval."""
+
+import typer
+
+from coder_eval.telemetry import track_command
+
+from .aggregate_command import aggregate_command
+from .console import console
+from .evaluate_command import evaluate_command
+from .plan_command import plan_command
+from .report_command import report_command
+from .run_command import run_command
+from .run_task_internal_command import run_task_internal_command
+
+
+# Create the Typer app
+app = typer.Typer(
+    name="coder-eval",
+    help="A framework for evaluating AI coding agents",
+    add_completion=False,
+)
+
+
+def _version_callback(value: bool) -> None:
+    """Print the installed coder-eval version and exit (eager `--version`)."""
+    if value:
+        from coder_eval import __version__
+
+        console.print(__version__)
+        raise typer.Exit(0)
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    ctx: typer.Context,
+    version: bool = typer.Option(
+        False,
+        "--version",
+        help="Show the installed coder-eval version and exit.",
+        callback=_version_callback,
+        is_eager=True,
+    ),
+) -> None:
+    """A framework for evaluating AI coding agents.
+
+    Run 'coder-eval COMMAND --help' for help on a specific command.
+
+    Available commands:
+    - run: Execute evaluation tasks
+    - plan: Validate task files (dry-run)
+    - evaluate: Run criteria against a directory without an agent
+    - report: Display or export evaluation reports
+    - aggregate: Rebuild run.json/run.md from finalized task.json files
+    """
+    # Discover and register agents (built-in + third-party plugins) before any
+    # subcommand resolves a task or builds an agent.
+    from coder_eval.plugins import load_plugins
+
+    load_plugins()
+
+    # One-time telemetry init (no-op when disabled / no connection string).
+    # Runs before the no-subcommand early-exit so even `--help` inits harmlessly.
+    from coder_eval import __version__
+    from coder_eval.telemetry import init_telemetry
+
+    init_telemetry(version=__version__)
+
+    # If no subcommand was invoked, show help and exit
+    if ctx.invoked_subcommand is None:
+        console.print(ctx.get_help())
+        raise typer.Exit(0)
+
+
+# Register core commands. Each public command is wrapped with track_command so it
+# emits a CoderEval.Cli.<name> event (Status/DurationMs/ErrorType) on completion;
+# functools.wraps preserves the signature so Typer still parses each command's flags.
+app.command(name="run")(track_command("run")(run_command))
+app.command(name="plan")(track_command("plan")(plan_command))
+app.command(name="evaluate")(track_command("evaluate")(evaluate_command))
+app.command(name="report")(track_command("report")(report_command))
+app.command(name="aggregate")(track_command("aggregate")(aggregate_command))
+# Hidden internal command invoked inside the Docker container only — UNWRAPPED
+# (it runs inside the run-task subprocess and would double-count / pollute events).
+app.command(name="_run-task-internal", hidden=True)(run_task_internal_command)
+
+
+__all__ = ["app"]