coder-eval 0.8.2__py3-none-any.whl

coder_eval/cli/aggregate_command.py

@@ -0,0 +1,153 @@
+"""Aggregate command — (re)build run.json + run.md from finalized task.json files.
+
+The standalone aggregation step: where `coder-eval run` folds the live batch into
+run.json/run.md at the end of execution, this re-aggregates the same artifacts
+afterwards from the finalized task.json files already on disk — for a run dir
+whose top-level run.json is missing or stale (e.g. after recovering or combining
+run dirs). It reuses the exact builder a live run uses (`build_run_summary`), so
+the counts and version chip are identical.
+"""
+
+import json
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import typer
+from pydantic import ValidationError
+
+from ..models import SkippedTask, TaskResult
+from .console import console
+
+
+def aggregate_command(
+    run_dir: Path = typer.Argument(  # noqa: B008
+        ...,
+        help="Run directory holding finalized task.json files (e.g. runs/2026-06-22_14-32-27).",
+        exists=True,
+        file_okay=False,
+    ),
+    output_dir: Path | None = typer.Option(  # noqa: B008
+        None,
+        "--output",
+        "-o",
+        help="Directory to write run.json/run.md into instead of run_dir (e.g. a merged output dir).",
+        file_okay=False,
+    ),
+) -> None:
+    """(Re)build run.json + run.md by aggregating the finalized task.json files under a run dir.
+
+    Rebuilds the run-level summary only. Per-suite rollups (suite.json/suite.md) and
+    experiment reports (experiment.json/experiment.md) that a live ``run`` produces are
+    NOT rebuilt — the per-row suite/variant grouping they need is not recoverable from
+    task.json alone.
+
+    Examples:
+        # Rebuild a run's summary in place
+        coder-eval aggregate runs/2026-06-22_14-32-27
+
+        # Aggregate a combined dir's task results into a fresh summary
+        coder-eval aggregate runs/combined -o runs/combined
+    """
+    from ..orchestration.batch import build_run_summary, recover_task_results, write_run_summary
+
+    results = recover_task_results(run_dir)
+    if not results:
+        console.print(f"[red]Error: no finalized task.json files found under {run_dir}[/red]")
+        console.print("\n[dim]Hint: this aggregates a finished run — use 'coder-eval run' to create one.[/dim]")
+        raise typer.Exit(1)
+
+    out_dir = output_dir or run_dir
+
+    # tags / source-path / run-level fields are inputs (static task metadata), not
+    # results, so carry them from an existing run.json when present — reading them
+    # from a stale summary is safe. Absent → empty maps + a window derived from the
+    # recovered results.
+    task_tags, task_paths, prior = _read_prior_metadata(run_dir)
+    start_time, end_time = _resolve_window(results, prior)
+    skipped = _recover_skipped_tasks(prior)
+
+    summary = build_run_summary(
+        out_dir.name,
+        results,
+        start_time,
+        end_time,
+        task_tags,
+        task_paths=task_paths,
+        max_parallel=int(prior.get("max_parallel", 1) or 1),
+        skipped_tasks=skipped,
+    )
+    write_run_summary(summary, out_dir)
+    console.print(
+        f"[green][OK][/green] Aggregated {summary.tasks_run} task(s) "
+        + f"({summary.tasks_succeeded} ok / {summary.tasks_failed} fail / {summary.tasks_error} err) "
+        + f"→ {out_dir / 'run.json'}"
+    )
+    console.print(
+        "[dim]Note: run-level summary only — per-suite (suite.json/suite.md) and "
+        + "experiment (experiment.json/experiment.md) rollups are not rebuilt.[/dim]"
+    )
+
+
+def _read_prior_metadata(run_dir: Path) -> tuple[dict[str, list[str]], dict[str, str], dict[str, Any]]:
+    """Pull per-task tags/source-paths + run-level fields from an existing run.json.
+
+    Returns ``(task_tags, task_paths, run_meta)`` — empty maps and ``{}`` when there
+    is no readable run.json (a fresh dir, or one assembled purely from task.json).
+    """
+    try:
+        prior = json.loads((run_dir / "run.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}, {}, {}
+    if not isinstance(prior, dict):
+        return {}, {}, {}
+    task_tags: dict[str, list[str]] = {}
+    task_paths: dict[str, str] = {}
+    for row in prior.get("task_results", []):
+        if not isinstance(row, dict):
+            continue
+        task_id = row.get("task_id")
+        if not task_id:
+            continue
+        if isinstance(row.get("tags"), list):
+            task_tags[task_id] = row["tags"]
+        if isinstance(row.get("task_path"), str):
+            task_paths[task_id] = row["task_path"]
+    return task_tags, task_paths, prior
+
+
+def _recover_skipped_tasks(prior: dict[str, Any]) -> list[SkippedTask]:
+    """Reconstruct the skipped-task carry-over from a prior run.json, per entry.
+
+    The prior summary is untrusted (possibly stale / hand-edited / older-schema), so a
+    malformed entry must drop rather than abort the rebuild — mirroring the rest of
+    ``_read_prior_metadata``'s degrade-to-empty stance.
+    """
+    recovered: list[SkippedTask] = []
+    for entry in prior.get("skipped_tasks", []):
+        if not isinstance(entry, dict):
+            continue
+        try:
+            recovered.append(SkippedTask.model_validate(entry))
+        except ValidationError:
+            console.print(f"[yellow]Dropping malformed skipped_tasks entry from prior run.json: {entry}[/yellow]")
+    return recovered
+
+
+def _resolve_window(results: list[TaskResult], prior: dict[str, Any]) -> tuple[datetime, datetime]:
+    """Determine the run's (start, end) for total_duration_seconds.
+
+    Prefer the prior run.json's timestamps (the real wall-clock); otherwise derive a
+    best-effort window from the recovered results' start times + durations. ``results``
+    is always non-empty here (the command errors out earlier on an empty recovery) and
+    ``EvaluationResult.started_at`` is a required field, so a start time always exists.
+    """
+    start_raw, end_raw = prior.get("start_time"), prior.get("end_time")
+    if isinstance(start_raw, str) and isinstance(end_raw, str):
+        try:
+            return datetime.fromisoformat(start_raw), datetime.fromisoformat(end_raw)
+        except ValueError:
+            pass
+    start = min(r.result.started_at for r in results)
+    end = max(r.result.started_at + timedelta(seconds=r.duration) for r in results)
+    return start, max(end, start)

coder_eval/cli/console.py

@@ -0,0 +1,7 @@
+"""Shared console instance for CLI commands."""
+
+from rich.console import Console
+
+
+# Single console instance shared across all CLI commands
+console = Console()

coder_eval/cli/evaluate_command.py

@@ -0,0 +1,164 @@
+"""Evaluate command - run criteria against a directory without an agent."""
+
+import asyncio
+from pathlib import Path
+
+import typer
+
+from ..logging_config import setup_logging
+from ..models import (
+    AgentKind,
+    EvaluationResult,
+    FinalStatus,
+    PreservationMode,
+    TemplateDirSource,
+    parse_agent_config,
+)
+from ..orchestration.task_loader import load_task
+from ..orchestrator import Orchestrator
+from ..sandbox import Sandbox
+from .console import console
+from .run_helpers import prepare_run_directory
+
+
+def evaluate_command(
+    task_file: Path = typer.Argument(  # noqa: B008
+        ...,
+        help="Path to task YAML file",
+        exists=True,
+    ),
+    work_dir: Path = typer.Argument(  # noqa: B008
+        ...,
+        help="Directory containing the code to evaluate",
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Enable verbose (DEBUG level) logging",
+    ),
+    preserve: bool = typer.Option(
+        True,
+        "--preserve/--no-preserve",
+        "-p/-P",
+        help="Move sandbox artifacts to run directory (default: preserve). The temp sandbox is always removed.",
+    ),
+    run_dir: Path | None = typer.Option(  # noqa: B008
+        None,
+        "--run-dir",
+        help="Custom run directory (default: auto-generated timestamped directory in runs/)",
+    ),
+) -> None:
+    """Evaluate criteria against a directory without running an agent.
+
+    Runs the success criteria defined in a task against a work directory.
+    Artifacts are saved to a run directory when --preserve is used.
+
+    Examples:
+        coder-eval evaluate tasks/hello.yaml ./my_solution
+        coder-eval evaluate tasks/test.yaml /path/to/code --preserve
+        coder-eval evaluate tasks/test.yaml /path/to/code --run-dir ./my_eval_run
+    """
+    setup_logging(verbose=verbose)
+
+    console.print("\n[bold]Evaluating Criteria[/bold]\n")
+
+    try:
+        task, source_yaml = load_task(task_file)
+    except Exception as e:
+        console.print(f"[red]✗ Failed to load task:[/red] {e}")
+        raise typer.Exit(1) from e
+
+    # Evaluate-only mode bypasses experiment resolution + CLI overrides, so
+    # `agent` may be None or `agent.type` may be unset for tasks that defer
+    # those to the experiment / CLI layers. The orchestrator only uses
+    # `agent.type` for result labeling here (no agent is created), so a
+    # default is safe.
+
+    if task.agent is None:
+        task.agent = parse_agent_config(type=AgentKind.CLAUDE_CODE)
+    elif task.agent.type is None:
+        task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE})
+
+    if not work_dir.is_dir():
+        console.print(f"[red]✗ Work directory is not a directory:[/red] {work_dir}")
+        raise typer.Exit(1)
+
+    try:
+        prepared_run_dir = prepare_run_directory(run_dir)
+    except Exception as e:
+        console.print(f"[red]✗ Failed to prepare run directory:[/red] {e}")
+        raise typer.Exit(1) from e
+
+    # Build a sandbox pre-loaded with the work_dir contents, then run evaluate-only
+    sandbox_config = task.sandbox.model_copy(deep=True)
+    template_source = TemplateDirSource(path=str(work_dir.resolve()))
+    if sandbox_config.template_sources:
+        sandbox_config.template_sources = [template_source, *sandbox_config.template_sources]
+    else:
+        sandbox_config.template_sources = [template_source]
+
+    task_dir = task_file.parent.resolve()
+    sandbox = Sandbox(sandbox_config, task_id=task.task_id, task_dir=task_dir)
+
+    async def _setup_and_run() -> EvaluationResult:
+        await asyncio.to_thread(sandbox.setup)
+        orchestrator = Orchestrator(
+            task=task,
+            run_dir=prepared_run_dir,
+            preservation_mode=PreservationMode.MOVE_ON_WRITE if preserve else PreservationMode.NONE,
+            task_file=task_file,
+            sandbox=sandbox,
+            variant_id="evaluate",
+            source_yaml=source_yaml,
+        )
+        return await orchestrator.run()
+
+    result = asyncio.run(_setup_and_run())
+
+    # Display results
+    console.print("[bold]Criteria Results:[/bold]\n")
+
+    criteria_results = result.success_criteria_results or []
+    if len(criteria_results) != len(task.success_criteria):
+        console.print(
+            f"[red]✗ Result count mismatch: got {len(criteria_results)}, expected {len(task.success_criteria)}[/red]"
+        )
+        raise typer.Exit(1)
+
+    for criterion, cr in zip(task.success_criteria, criteria_results, strict=True):
+        status = "[green]✓[/green]" if cr.score >= criterion.pass_threshold else "[red]✗[/red]"
+        console.print(f"{status} {cr.criterion_type}")
+        console.print(f"  [dim]{cr.description}[/dim]")
+        console.print(f"  [dim]Score: {cr.score:.2f}[/dim]")
+        if cr.details:
+            console.print(f"  [dim]Details: {cr.details}[/dim]")
+        if cr.error:
+            console.print(f"  [red]Error: {cr.error}[/red]")
+        console.print()
+
+    passed = sum(
+        1 for cr, c in zip(criteria_results, task.success_criteria, strict=True) if cr.score >= c.pass_threshold
+    )
+    total = len(task.success_criteria)
+    failed = total - passed
+
+    console.print("[bold]Summary:[/bold]")
+    console.print(f"  Passed: {passed}/{total}")
+    console.print(f"  Failed: {failed}/{total}")
+    console.print(f"\n[dim]Run directory: {prepared_run_dir}[/dim]")
+    if result.sandbox_path:
+        console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]")
+
+    if result.final_status == FinalStatus.ERROR:
+        console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]")
+        raise typer.Exit(1)
+    elif failed == 0:
+        console.print("\n[green]All criteria passed! ✓[/green]")
+        raise typer.Exit(0)
+    else:
+        console.print(f"\n[red]{failed} criterion/criteria failed.[/red]")
+        raise typer.Exit(1)

coder_eval/cli/plan_command.py

@@ -0,0 +1,152 @@
+"""Plan command - validate task files without executing."""
+
+import warnings
+from pathlib import Path
+
+import typer
+
+from ..models.tasks import UnknownTaskFieldWarning
+from ..orchestration.task_loader import load_task
+from .console import console
+from .run_helpers import discover_default_tasks
+from .utils import check_api_keys, check_tools
+
+
+def plan_command(
+    task_files: list[Path] | None = typer.Argument(  # noqa: B008
+        None,
+        help="Path(s) to task YAML file(s) to validate. Defaults to all tasks/ recursively.",
+    ),
+    experiment: Path | None = typer.Option(  # noqa: B008
+        None,
+        "--experiment",
+        "-e",
+        help="Experiment definition YAML (default: experiments/default.yaml)",
+    ),
+) -> None:
+    """Validate task files without executing (dry-run).
+
+    When no TASK_FILES are provided, all .yaml files under tasks/ are discovered recursively.
+
+    This command checks:
+    - Task file syntax and schema validity
+    - Required CLI tools are available (claude, uv)
+    - API keys are configured
+    - Task configuration is reasonable
+
+    When an experiment is provided (or experiments/default.yaml exists),
+    also shows experiment info and resolved agent configs per variant.
+
+    Unknown top-level fields on TaskDefinition currently soft-warn
+    (see _warn_on_unknown_fields in models/tasks.py) — those warnings
+    surface inline under each task as ⚠ notices, without failing the run.
+
+    Examples:
+        coder-eval plan
+        coder-eval plan tasks/hello_date.yaml
+        coder-eval plan tasks/*.yaml
+        coder-eval plan tasks/*.yaml -e experiments/model-comparison.yaml
+    """
+    # Default to discovering all tasks under tasks/ when none provided
+    resolved_task_files = task_files if task_files else discover_default_tasks()
+
+    console.print("\n[bold]Task Validation (Dry-Run)[/bold]\n")
+
+    # Check required tools
+    check_tools()
+
+    # Check API keys
+    check_api_keys()
+
+    # Lazy import to avoid circular dependency at module level
+    from ..orchestration.experiment import DEFAULT_EXPERIMENT_PATH, load_experiment, resolve_task_for_variant
+
+    # Always load experiment (defaults to experiments/default.yaml)
+    exp_path = experiment if isinstance(experiment, Path) else DEFAULT_EXPERIMENT_PATH
+    try:
+        exp_def = load_experiment(exp_path)
+        if exp_path == DEFAULT_EXPERIMENT_PATH:
+            default_exp = exp_def
+        elif DEFAULT_EXPERIMENT_PATH.exists():
+            default_exp = load_experiment(DEFAULT_EXPERIMENT_PATH)
+        else:
+            default_exp = exp_def  # fall back to custom as its own baseline
+    except Exception as e:
+        console.print(f"[red]Failed to load experiment ({exp_path}): {e}[/red]")
+        raise typer.Exit(1) from e
+
+    # Show experiment info
+    console.print("[bold cyan]Experiment[/bold cyan]")
+    console.print(f"  [dim]ID: {exp_def.experiment_id}[/dim]")
+    if exp_def.description:
+        console.print(f"  [dim]Description: {exp_def.description}[/dim]")
+    console.print(
+        f"  [dim]Variants ({len(exp_def.variants)}): {', '.join(v.variant_id for v in exp_def.variants)}[/dim]"
+    )
+    if exp_def.defaults and exp_def.defaults.agent:
+        console.print(f"  [dim]Default agent config: {exp_def.defaults.agent}[/dim]")
+    console.print()
+
+    # Validate each task file
+    all_valid = True
+    for task_file in resolved_task_files:
+        try:
+            # Capture warnings so unknown-field UnknownTaskFieldWarnings
+            # (emitted by TaskDefinition._warn_on_unknown_fields while the
+            # top-level schema stays in soft-launch mode) surface inline
+            # below \u2014 they don't fail the run, but they're visible to the
+            # author and to any CI log scraper. Other DeprecationWarnings
+            # raised during load (legacy-timing migrations, pydantic,
+            # transitive libs) are re-emitted through warnings.showwarning
+            # so they still reach stderr instead of getting swallowed.
+            with warnings.catch_warnings(record=True) as caught:
+                warnings.simplefilter("always", DeprecationWarning)
+                task, _source_yaml = load_task(task_file)
+
+            console.print(f"[green]\u2713[/green] {task_file.name}")
+            console.print(f"  [dim]Task ID: {task.task_id}[/dim]")
+
+            # Handle optional agent field. Phase 3 made agent.type optional —
+            # tasks may now defer it to experiment defaults / --type.
+            if task.agent is None:
+                console.print("  [dim]Agent: N/A (resolved from experiment)[/dim]")
+            elif task.agent.type is None:
+                console.print("  [dim]Agent type: (deferred to experiment / --type)[/dim]")
+            else:
+                console.print(f"  [dim]Agent: {task.agent.type}[/dim]")
+
+            console.print(f"  [dim]Success criteria: {len(task.success_criteria)}[/dim]")
+
+            # Surface unknown-field warnings as inline notices (non-blocking;
+            # catches stale top-level fields like max_iterations / llm_reviewer
+            # that the soft-launch validator otherwise drops silently). Match
+            # by category, not message text, so a reworded warning string
+            # doesn't silently break this rendering. Anything else captured
+            # gets re-emitted to stderr so non-target deprecations stay visible.
+            for w in caught:
+                if issubclass(w.category, UnknownTaskFieldWarning):
+                    console.print(f"  [yellow]⚠[/yellow] [yellow]{w.message}[/yellow]")
+                else:
+                    warnings.showwarning(w.message, w.category, w.filename, w.lineno)
+
+            # Show resolved agent per variant
+            for variant in exp_def.variants:
+                try:
+                    resolved, _lineage, _ = resolve_task_for_variant(default_exp, task, exp_def, variant)
+                    agent_type = str(resolved.agent.type) if resolved.agent else "unknown"
+                    agent_model = resolved.agent.model if resolved.agent else None
+                    model_str = f" ({agent_model})" if agent_model else ""
+                    console.print(f"    [dim]Variant '{variant.variant_id}': {agent_type}{model_str}[/dim]")
+                except Exception as e:
+                    console.print(f"    [red]Variant '{variant.variant_id}': resolution failed - {e}[/red]")
+
+        except Exception as e:
+            console.print(f"[red]\u2717[/red] {task_file.name}")
+            console.print(f"  [red]Error: {e}[/red]")
+            all_valid = False
+
+    if all_valid:
+        console.print("\n[green]All tasks are valid![/green]")
+    else:
+        console.print("\n[red]Some tasks have errors.[/red]")
+        raise typer.Exit(1)

coder_eval/cli/report_command.py

@@ -0,0 +1,121 @@
+"""Report command - display or export evaluation results."""
+
+from pathlib import Path
+
+import typer
+from rich.markdown import Markdown
+
+from ..models import EvaluationResult
+from ..reports import ReportGenerator
+from ..reports_html import write_task_html
+from .console import console
+
+
+def report_command(
+    run_dir: Path = typer.Argument(  # noqa: B008
+        ...,
+        help="Path to a run directory (e.g., runs/latest or runs/2025-10-10_20-57-30)",
+        exists=True,
+    ),
+    output_file: Path | None = typer.Option(  # noqa: B008
+        None,
+        "--output",
+        "-o",
+        help="Output file (default: display markdown to stdout).",
+    ),
+    report_format: str = typer.Option(
+        "md",
+        "--format",
+        "-f",
+        help="Output format: 'md' (default markdown), 'html' (render task.json files as HTML).",
+    ),
+) -> None:
+    """Display or export a run report.
+
+    The run command automatically generates a report during execution.
+    This command allows you to view or export that report later.
+
+    Examples:
+        # Display latest run report
+        coder-eval report runs/latest
+
+        # Export specific run to markdown file
+        coder-eval report runs/2025-10-10_20-57-30 -o summary.md
+
+        # Re-generate HTML reports for every task.json under a run dir
+        coder-eval report runs/latest --format html
+    """
+    fmt = report_format.lower()
+    if fmt not in ("md", "html"):
+        console.print(f"[red]Error: unknown --format '{report_format}' (expected 'md' or 'html')[/red]")
+        raise typer.Exit(1)
+
+    if fmt == "html":
+        _regenerate_html_reports(run_dir, output_file)
+        return
+
+    try:
+        report_md, source_path = ReportGenerator.load_from_run_dir(run_dir)
+        console.print(f"[dim]Reading report from {source_path}[/dim]\n")
+
+    except FileNotFoundError as e:
+        console.print(f"[red]Error: {e}[/red]")
+        console.print("\n[dim]Hint: Use 'coder-eval run' to create evaluation runs.[/dim]")
+        raise typer.Exit(1) from e
+
+    # Output report
+    if output_file:
+        output_file.write_text(report_md, encoding="utf-8")
+        console.print(f"[green][OK]Report saved to {output_file}[/green]")
+    else:
+        console.print(Markdown(report_md))
+
+
+def _regenerate_html_reports(run_dir: Path, output_file: Path | None) -> None:
+    """Regenerate task-level HTML reports from every task.json under run_dir.
+
+    If `output_file` is provided and exactly one task.json is found, the
+    report is written to that file. Otherwise each task.html is written
+    next to its task.json.
+    """
+    task_json_paths = sorted(run_dir.rglob("task.json"))
+    if not task_json_paths:
+        console.print(f"[red]Error: no task.json files found under {run_dir}[/red]")
+        raise typer.Exit(1)
+
+    if output_file and len(task_json_paths) > 1:
+        msg = (
+            f"[red]Error: --output targets a single file but {len(task_json_paths)} task.json files "
+            + "were found. Omit --output to regenerate all reports in place.[/red]"
+        )
+        console.print(msg)
+        raise typer.Exit(1)
+
+    from ..evaluation.judge_persistence import load_judge_transcripts
+
+    written: list[Path] = []
+    failed: list[Path] = []
+    for task_json in task_json_paths:
+        try:
+            result = EvaluationResult.model_validate_json(task_json.read_text(encoding="utf-8"))
+        except Exception as e:
+            console.print(f"[yellow]Skipping {task_json}: {e}[/yellow]")
+            failed.append(task_json)
+            continue
+        # Pull any spilled judge transcripts back onto the in-memory result so
+        # the HTML re-render shows the same disclosures the original run produced.
+        load_judge_transcripts(result, task_json.parent)
+        target = output_file if output_file else task_json.with_name("task.html")
+        written_path = write_task_html(result, target)
+        if written_path is None:
+            console.print(f"[red]Failed to write HTML for {task_json}[/red]")
+            failed.append(task_json)
+            continue
+        written.append(written_path)
+
+    for p in written:
+        console.print(f"[green][OK]Wrote {p}[/green]")
+    console.print(f"[green]Generated {len(written)} HTML report(s)[/green]")
+    if failed:
+        console.print(f"[red]{len(failed)} task(s) failed[/red]")
+        raise typer.Exit(1)