lightcone-cli 0.2.0__py3-none-any.whl

lightcone/eval/harness.py

@@ -0,0 +1,265 @@
+"""Core eval loop — runs trials with ThreadPoolExecutor concurrency."""
+
+from __future__ import annotations
+
+import logging
+import signal
+import threading
+import time
+import uuid
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from lightcone.eval.build import build_eval_wheels
+from lightcone.eval.graders import compute_composite_score, run_graders
+from lightcone.eval.models import (
+    EvalRun,
+    EvalRunConfig,
+    IterationResult,
+    TaskSpec,
+    TrialResult,
+)
+from lightcone.eval.sandbox import BUILD_COMPLETE_MARKER, EvalSandbox
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_LOOP_PROMPT = """\
+/lc-build this analysis and make sure to cover universe {{UNIVERSE}}.
+Do NOT ask for plan approval — skip straight to building. This is an automated eval run.
+"""
+
+
+def load_task(evals_dir: Path, task_id: str) -> TaskSpec:
+    """Load a TaskSpec from evals/tasks/<task_id>/task.yaml."""
+    task_file = evals_dir / "tasks" / task_id / "task.yaml"
+    try:
+        data = yaml.safe_load(task_file.read_text())
+    except FileNotFoundError:
+        raise FileNotFoundError(f"Task not found: {task_file}") from None
+    return TaskSpec(**data)
+
+
+def load_run_config(config_path: Path) -> EvalRunConfig:
+    """Load an EvalRunConfig from a YAML file."""
+    data = yaml.safe_load(config_path.read_text())
+    return EvalRunConfig(**data)
+
+
+def _get_loop_prompt(evals_dir: Path, task_id: str) -> str:
+    """Get loop prompt template: task-specific or default."""
+    task_prompt = evals_dir / "tasks" / task_id / "loop-prompt.md"
+    if task_prompt.exists():
+        return task_prompt.read_text()
+    return DEFAULT_LOOP_PROMPT
+
+
+def run_trial(
+    task: TaskSpec,
+    trial_number: int,
+    *,
+    evals_dir: Path,
+    config: EvalRunConfig,
+    run_id: str,
+    wheels: list[Path],
+    sidecar_dir: Path | None = None,
+) -> TrialResult:
+    """Run a single trial: create sandbox -> run /lc-build -> grade -> teardown."""
+    trial_id = f"{run_id}-{task.id}-{trial_number}"
+    trial = TrialResult(
+        trial_id=trial_id,
+        task_id=task.id,
+        trial_number=trial_number,
+        started_at=datetime.now(UTC),
+    )
+
+    env_vars = {
+        "LIGHTCONE_EVAL_RUN_ID": run_id,
+        "CLAUDE_CODE_SESSION_ID": f"eval-{trial_id}",
+    }
+
+    sandbox = EvalSandbox(
+        task_id=task.id,
+        trial_id=trial_id,
+        sandbox_image=config.sandbox_image,
+        env_vars=env_vars,
+    )
+
+    try:
+        sandbox.create()
+
+        seed_dir = evals_dir / "tasks" / task.id
+        loop_prompt = _get_loop_prompt(evals_dir, task.id)
+
+        sandbox.setup(
+            seed_dir=seed_dir,
+            universe=task.universe,
+            loop_prompt_template=loop_prompt,
+            wheels=wheels,
+        )
+
+        # Single invocation: /lc-build handles its own loop internally
+        start = time.monotonic()
+        try:
+            claude_result = sandbox.exec_claude(
+                max_turns=task.max_turns,
+                timeout=task.trial_timeout,
+            )
+            duration = time.monotonic() - start
+
+            build_complete = BUILD_COMPLETE_MARKER in claude_result.result_text
+            iteration = IterationResult(
+                iteration=0,
+                cost_usd=claude_result.cost_usd,
+                num_turns=claude_result.num_turns,
+                duration_seconds=duration,
+                build_complete=build_complete,
+                output_summary=(
+                    "" if claude_result.is_error else claude_result.result_text[:500]
+                ),
+                error=claude_result.result_text[:500] if claude_result.is_error else None,
+            )
+
+            # Save transcript sidecar
+            if sidecar_dir is not None and claude_result.raw_jsonl:
+                trial_log_dir = sidecar_dir / trial_id
+                trial_log_dir.mkdir(parents=True, exist_ok=True)
+                jsonl_path = trial_log_dir / "transcript.jsonl"
+                jsonl_path.write_text(claude_result.raw_jsonl)
+                iteration.transcript_path = str(
+                    jsonl_path.relative_to(sidecar_dir.parent)
+                )
+        except Exception as exc:
+            duration = time.monotonic() - start
+            iteration = IterationResult(
+                iteration=0,
+                duration_seconds=duration,
+                error=str(exc),
+            )
+
+        trial.iterations.append(iteration)
+        trial.build_complete = iteration.build_complete
+
+        # Run graders
+        trial.grader_results = run_graders(sandbox, task.graders, evals_dir, task.id)
+        trial.composite_score = compute_composite_score(trial.grader_results)
+
+        # Aggregate metrics
+        trial.total_cost_usd = sum(it.cost_usd for it in trial.iterations)
+        trial.total_turns = sum(it.num_turns for it in trial.iterations)
+        trial.total_duration_seconds = sum(it.duration_seconds for it in trial.iterations)
+
+    except Exception as exc:
+        logger.error("Trial %s failed: %s", trial_id, exc, exc_info=True)
+        trial.error = str(exc)
+    finally:
+        sandbox.teardown()
+
+    trial.finished_at = datetime.now(UTC)
+    return trial
+
+
+def run_eval(
+    config: EvalRunConfig,
+    evals_dir: Path,
+    *,
+    progress_callback: Callable[[TrialResult], None] | None = None,
+    dry_run: bool = False,
+) -> EvalRun:
+    """Run all trials: tasks x num_trials with ThreadPoolExecutor."""
+    run_id = config.id or str(uuid.uuid4())[:8]
+
+    # Load all tasks
+    tasks = [load_task(evals_dir, tid) for tid in config.tasks]
+
+    # Build trial schedule
+    schedule: list[dict[str, Any]] = []
+    for task in tasks:
+        for n in range(config.num_trials):
+            schedule.append({"task": task, "trial_number": n})
+
+    if dry_run:
+        return EvalRun(
+            config=config,
+            started_at=datetime.now(UTC),
+            finished_at=datetime.now(UTC),
+            summary={"dry_run": True, "total_trials": len(schedule), "schedule": [
+                {"task": s["task"].id, "trial": s["trial_number"]}
+                for s in schedule
+            ]},
+        )
+
+    # Build lightcone-cli wheel from current working tree + collect ASTRA wheel
+    version_info, wheels = build_eval_wheels(evals_dir)
+
+    # Compute run stem using git commit for traceability
+    commit_short = version_info.lightcone_commit[:8] or "unknown"
+    run_stem = f"{run_id}-{commit_short}"
+    output_base = Path(config.output_dir)
+    sidecar_dir = output_base / run_stem / "logs"
+
+    eval_run = EvalRun(
+        config=config,
+        version=version_info,
+        started_at=datetime.now(UTC),
+        run_stem=run_stem,
+        transcript_dir=str(sidecar_dir),
+    )
+
+    # Handle SIGINT: save partial results
+    interrupted = False
+
+    def _signal_handler(signum: int, frame: Any) -> None:
+        nonlocal interrupted
+        interrupted = True
+        logger.warning("SIGINT received — finishing current trials and saving partial results")
+
+    is_main_thread = threading.current_thread() is threading.main_thread()
+    if is_main_thread:
+        original_handler = signal.getsignal(signal.SIGINT)
+        signal.signal(signal.SIGINT, _signal_handler)
+
+    try:
+        with ThreadPoolExecutor(max_workers=config.max_concurrency) as pool:
+            futures = {
+                pool.submit(
+                    run_trial,
+                    s["task"],
+                    s["trial_number"],
+                    evals_dir=evals_dir,
+                    config=config,
+                    run_id=run_id,
+                    wheels=wheels,
+                    sidecar_dir=sidecar_dir,
+                ): s
+                for s in schedule
+            }
+
+            for future in as_completed(futures):
+                if interrupted:
+                    break
+
+                try:
+                    trial = future.result()
+                except Exception as exc:
+                    s = futures[future]
+                    trial = TrialResult(
+                        trial_id=f"{run_id}-error",
+                        task_id=s["task"].id,
+                        trial_number=s["trial_number"],
+                        error=str(exc),
+                    )
+
+                eval_run.trials.append(trial)
+                if progress_callback:
+                    progress_callback(trial)
+    finally:
+        if is_main_thread:
+            signal.signal(signal.SIGINT, original_handler)
+
+    eval_run.finished_at = datetime.now(UTC)
+    return eval_run

lightcone/eval/models.py

@@ -0,0 +1,117 @@
+"""Pydantic data models for the eval harness."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class GraderType(StrEnum):
+    """Supported grader types."""
+
+    command = "command"
+    status = "status"
+    script = "script"
+
+
+class GraderSpec(BaseModel):
+    """Specification for a single grader."""
+
+    name: str
+    type: GraderType
+    command: str | None = None
+    script: str | None = None
+    weight: float = 1.0
+    timeout: int = 120
+
+
+class TaskSpec(BaseModel):
+    """Specification for an eval task (loaded from task.yaml)."""
+
+    id: str
+    description: str = ""
+    seed_project: str = ""
+    universe: str = "baseline"
+    max_turns: int = 200
+    trial_timeout: int = 7200
+    graders: list[GraderSpec] = Field(default_factory=list)
+
+
+class IterationResult(BaseModel):
+    """Result from a single build-loop iteration."""
+
+    iteration: int
+    cost_usd: float = 0.0
+    num_turns: int = 0
+    duration_seconds: float = 0.0
+    build_complete: bool = False
+    output_summary: str = ""
+    error: str | None = None
+    transcript_path: str | None = None
+
+
+class GraderResult(BaseModel):
+    """Result from a single grader."""
+
+    name: str
+    type: GraderType
+    passed: bool = False
+    score: float = 0.0
+    weight: float = 1.0
+    details: str = ""
+    error: str | None = None
+
+
+class TrialResult(BaseModel):
+    """Result of a single trial (one task x repetition)."""
+
+    trial_id: str
+    task_id: str
+    trial_number: int = 0
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    iterations: list[IterationResult] = Field(default_factory=list)
+    grader_results: list[GraderResult] = Field(default_factory=list)
+    composite_score: float = 0.0
+    build_complete: bool = False
+    total_cost_usd: float = 0.0
+    total_turns: int = 0
+    total_duration_seconds: float = 0.0
+    error: str | None = None
+
+
+class EvalRunConfig(BaseModel):
+    """Configuration for an eval run (loaded from run config YAML)."""
+
+    id: str = ""
+    tasks: list[str] = Field(default_factory=list)
+    num_trials: int = 3
+    max_concurrency: int = 4
+    sandbox_image: str | None = None
+    output_dir: str = "eval-results"
+
+
+class VersionInfo(BaseModel):
+    """Git and wheel version metadata for reproducibility."""
+
+    lightcone_commit: str = ""
+    lightcone_branch: str = ""
+    lightcone_dirty: bool = False
+    lightcone_version: str = ""
+    astra_version: str = ""
+
+
+class EvalRun(BaseModel):
+    """Complete results of an eval run."""
+
+    config: EvalRunConfig
+    version: VersionInfo = Field(default_factory=VersionInfo)
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    trials: list[TrialResult] = Field(default_factory=list)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    transcript_dir: str | None = None
+    run_stem: str | None = None

lightcone/eval/report.py

@@ -0,0 +1,214 @@
+"""Aggregation, display, and persistence for eval results."""
+
+from __future__ import annotations
+
+import json
+import math
+from collections import defaultdict
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.table import Table
+
+from lightcone.eval.models import EvalRun, TrialResult
+
+
+def compute_summary(eval_run: EvalRun) -> dict[str, Any]:
+    """Group trials by task and compute aggregate statistics."""
+    groups: dict[str, list[TrialResult]] = defaultdict(list)
+    for trial in eval_run.trials:
+        groups[trial.task_id].append(trial)
+
+    summary: dict[str, Any] = {"groups": {}, "totals": {}}
+
+    all_costs: list[float] = []
+    all_durations: list[float] = []
+
+    for task_id, trials in groups.items():
+        scores: list[float] = []
+        costs: list[float] = []
+        durations: list[float] = []
+        completions = 0
+        errors = 0
+
+        for t in trials:
+            if t.error is not None:
+                errors += 1
+                continue
+            scores.append(t.composite_score)
+            costs.append(t.total_cost_usd)
+            durations.append(t.total_duration_seconds)
+            if t.build_complete:
+                completions += 1
+
+        n = len(scores)
+        mean_score = sum(scores) / n if n > 0 else 0.0
+        stderr_score = (
+            math.sqrt(sum((s - mean_score) ** 2 for s in scores) / (n - 1)) / math.sqrt(n)
+            if n > 1
+            else 0.0
+        )
+        mean_cost = sum(costs) / n if n > 0 else 0.0
+        mean_duration = sum(durations) / n if n > 0 else 0.0
+
+        summary["groups"][task_id] = {
+            "task_id": task_id,
+            "num_trials": len(trials),
+            "num_errors": errors,
+            "mean_score": round(mean_score, 4),
+            "stderr_score": round(stderr_score, 4),
+            "pass_at_k": completions / len(trials) if trials else 0.0,
+            "mean_cost_usd": round(mean_cost, 4),
+            "mean_duration_seconds": round(mean_duration, 1),
+        }
+
+        all_costs.extend(costs)
+        all_durations.extend(durations)
+
+    summary["totals"] = {
+        "total_trials": len(eval_run.trials),
+        "total_cost_usd": round(sum(all_costs), 4),
+        "total_duration_seconds": round(sum(all_durations), 1),
+    }
+
+    return summary
+
+
+def _score_cell(g: dict[str, Any]) -> str:
+    """Format a score cell with color coding."""
+    score = g["mean_score"]
+    stderr = g["stderr_score"]
+    completion = g["pass_at_k"]
+
+    if score >= 0.8:
+        color = "green"
+    elif score >= 0.5:
+        color = "yellow"
+    else:
+        color = "red"
+
+    cell = f"[{color}]{score:.2f}[/{color}] +/- {stderr:.2f}\npass@k: {completion:.0%}"
+    if g["num_errors"] > 0:
+        cell += f"\n[red]{g['num_errors']} errors[/red]"
+    return cell
+
+
+def _cost_cell(g: dict[str, Any]) -> str:
+    """Format a cost/duration cell."""
+    cost = g["mean_cost_usd"]
+    dur = g["mean_duration_seconds"]
+    return f"${cost:.2f}\n{dur:.0f}s"
+
+
+def print_comparison_table(
+    eval_run: EvalRun,
+    console: Console | None = None,
+) -> None:
+    """Print a Rich summary table of eval results."""
+    if console is None:
+        console = Console()
+
+    summary = eval_run.summary or compute_summary(eval_run)
+    groups = summary.get("groups", {})
+
+    if not groups:
+        console.print("[yellow]No results to display.[/yellow]")
+        return
+
+    # Scores table
+    score_table = Table(title="Eval Results: Scores", show_lines=True)
+    score_table.add_column("Task", style="bold")
+    score_table.add_column("Score", justify="center")
+    for task_id, g in groups.items():
+        score_table.add_row(task_id, _score_cell(g))
+    console.print(score_table)
+
+    # Cost table
+    console.print()
+    cost_table = Table(title="Eval Results: Cost & Duration", show_lines=True)
+    cost_table.add_column("Task", style="bold")
+    cost_table.add_column("Cost / Duration", justify="center")
+    for task_id, g in groups.items():
+        cost_table.add_row(task_id, _cost_cell(g))
+    console.print(cost_table)
+
+    # Totals
+    totals = summary.get("totals", {})
+    if totals:
+        console.print(
+            f"\n[bold]Total:[/bold] {totals.get('total_trials', 0)} trials, "
+            f"${totals.get('total_cost_usd', 0):.2f}, "
+            f"{totals.get('total_duration_seconds', 0):.0f}s"
+        )
+
+
+def print_comparison_between(
+    run1: EvalRun,
+    run2: EvalRun,
+    console: Console | None = None,
+) -> None:
+    """Print a comparison between two eval runs."""
+    if console is None:
+        console = Console()
+
+    s1 = run1.summary or compute_summary(run1)
+    s2 = run2.summary or compute_summary(run2)
+
+    g1 = s1.get("groups", {})
+    g2 = s2.get("groups", {})
+
+    all_keys = sorted(set(g1.keys()) | set(g2.keys()))
+    if not all_keys:
+        console.print("[yellow]No results to compare.[/yellow]")
+        return
+
+    table = Table(title="Eval Comparison", show_lines=True)
+    table.add_column("Task", style="bold")
+    table.add_column("Run 1 Score", justify="center")
+    table.add_column("Run 2 Score", justify="center")
+    table.add_column("Delta", justify="center")
+
+    for key in all_keys:
+        r1 = g1.get(key)
+        r2 = g2.get(key)
+
+        s1_score = f"{r1['mean_score']:.2f} +/- {r1['stderr_score']:.2f}" if r1 else "-"
+        s2_score = f"{r2['mean_score']:.2f} +/- {r2['stderr_score']:.2f}" if r2 else "-"
+
+        if r1 and r2:
+            delta = r2["mean_score"] - r1["mean_score"]
+            color = "green" if delta > 0 else ("red" if delta < 0 else "white")
+            delta_str = f"[{color}]{delta:+.2f}[/{color}]"
+        else:
+            delta_str = "-"
+
+        table.add_row(key, s1_score, s2_score, delta_str)
+
+    console.print(table)
+
+
+def save_results(eval_run: EvalRun, output_dir: str | Path) -> Path:
+    """Save full EvalRun to JSON inside the run's sidecar directory."""
+    output_dir = Path(output_dir)
+
+    if eval_run.run_stem:
+        run_dir = output_dir / eval_run.run_stem
+    else:
+        timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
+        run_id = eval_run.config.id or "eval"
+        run_dir = output_dir / f"{run_id}-{timestamp}"
+
+    run_dir.mkdir(parents=True, exist_ok=True)
+    output_path = run_dir / "results.json"
+    data = eval_run.model_dump(mode="json")
+    output_path.write_text(json.dumps(data, indent=2, default=str))
+
+    return output_path
+
+
+def load_results(path: Path) -> EvalRun:
+    """Load an EvalRun from a JSON file."""
+    data = json.loads(path.read_text())
+    return EvalRun.model_validate(data)