lightcone-cli 0.2.0__py3-none-any.whl

lightcone/eval/build.py

@@ -0,0 +1,148 @@
+"""Auto-build wheels and collect version metadata for eval runs."""
+
+from __future__ import annotations
+
+import logging
+import re
+import subprocess
+import tempfile
+from pathlib import Path
+
+from lightcone.eval.models import VersionInfo
+
+logger = logging.getLogger(__name__)
+
+
+def _get_repo_root() -> Path:
+    """Find the lightcone-cli repo root via git."""
+    result = subprocess.run(
+        ["git", "rev-parse", "--show-toplevel"],
+        capture_output=True, text=True, check=True,
+    )
+    return Path(result.stdout.strip())
+
+
+def _get_git_info(repo_root: Path) -> VersionInfo:
+    """Collect git metadata from the repo."""
+    info = VersionInfo()
+
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            capture_output=True, text=True, check=True, cwd=repo_root,
+        )
+        info.lightcone_commit = result.stdout.strip()
+    except subprocess.CalledProcessError:
+        pass
+
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+            capture_output=True, text=True, check=True, cwd=repo_root,
+        )
+        info.lightcone_branch = result.stdout.strip()
+    except subprocess.CalledProcessError:
+        pass
+
+    try:
+        result = subprocess.run(
+            ["git", "diff", "--quiet", "HEAD"],
+            capture_output=True, cwd=repo_root,
+        )
+        info.lightcone_dirty = result.returncode != 0
+    except subprocess.CalledProcessError:
+        info.lightcone_dirty = True
+
+    return info
+
+
+def _build_wheel(repo_root: Path, outdir: Path) -> Path:
+    """Build a wheel from the repo and return the wheel path."""
+    result = subprocess.run(
+        ["python", "-m", "build", "--wheel", "--outdir", str(outdir)],
+        capture_output=True, text=True, cwd=repo_root,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"Wheel build failed:\n{result.stderr}")
+
+    wheels = list(outdir.glob("lightcone_cli-*.whl"))
+    if not wheels:
+        raise RuntimeError(f"No lightcone_cli wheel found in {outdir} after build")
+    return wheels[0]
+
+
+def _extract_version(wheel_path: Path) -> str:
+    """Extract version string from a wheel filename."""
+    # Wheel filenames: {name}-{version}-{tags}.whl
+    match = re.match(r"[^-]+-([^-]+)-", wheel_path.name)
+    return match.group(1) if match else wheel_path.name
+
+
+def _build_astra_wheel(outdir: Path) -> Path:
+    """Build an ASTRA wheel from the version installed in the current environment.
+
+    Uses importlib.metadata to find the git URL and commit of the installed astra
+    package, then builds a wheel pinned to that exact commit via pip wheel.
+    """
+    import importlib.metadata
+    import json
+
+    dist = importlib.metadata.distribution("astra")
+    url_text = dist.read_text("direct_url.json")
+    if not url_text:
+        raise RuntimeError(
+            "Cannot determine ASTRA source — installed package has no direct_url.json. "
+            "Ensure astra is installed from its git repo (not a plain wheel)."
+        )
+    url_info = json.loads(url_text)
+    git_url = url_info["url"]
+    commit = url_info.get("vcs_info", {}).get("commit_id", "")
+    spec = f"astra @ git+{git_url}@{commit}" if commit else f"astra @ git+{git_url}"
+
+    logger.info("Building ASTRA wheel from %s (commit %s) ...", git_url, commit[:8] or "HEAD")
+    result = subprocess.run(
+        ["pip", "wheel", "--no-deps", "--wheel-dir", str(outdir), spec],
+        capture_output=True, text=True,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"ASTRA wheel build failed:\n{result.stderr}")
+
+    wheels = list(outdir.glob("astra-*.whl"))
+    if not wheels:
+        raise RuntimeError(f"No astra wheel found in {outdir} after build")
+    return wheels[0]
+
+
+def build_eval_wheels(evals_dir: Path) -> tuple[VersionInfo, list[Path]]:
+    """Build the lightcone-cli and ASTRA wheels for sandbox injection.
+
+    Returns (version_info, [wheel_paths]).
+    Both wheels are built fresh from the current environment:
+    - lightcone-cli: built from the local git working tree
+    - ASTRA: built from the git URL/commit recorded in the installed package metadata
+    """
+    repo_root = _get_repo_root()
+    version_info = _get_git_info(repo_root)
+
+    # Build lightcone-cli wheel into a temp dir
+    tmpdir = Path(tempfile.mkdtemp(prefix="lightcone-eval-wheels-"))
+    logger.info("Building lightcone-cli wheel from %s ...", repo_root)
+    lightcone_wheel = _build_wheel(repo_root, tmpdir)
+    version_info.lightcone_version = _extract_version(lightcone_wheel)
+    logger.info("Built %s (commit %s%s)",
+                lightcone_wheel.name,
+                version_info.lightcone_commit[:8],
+                " dirty" if version_info.lightcone_dirty else "")
+
+    wheels: list[Path] = [lightcone_wheel]
+
+    # Build ASTRA wheel from the installed package's git source
+    try:
+        astra_wheel = _build_astra_wheel(tmpdir)
+        version_info.astra_version = _extract_version(astra_wheel)
+        wheels.append(astra_wheel)
+        logger.info("Built %s", astra_wheel.name)
+    except Exception as exc:
+        logger.warning("Failed to build ASTRA wheel: %s — sandbox may not have astra CLI", exc)
+
+    return version_info, wheels

lightcone/eval/cli.py

@@ -0,0 +1,176 @@
+"""CLI commands for the eval harness: lc eval {run, report, compare}."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+from rich.console import Console
+
+console = Console()
+
+
+@click.group()
+def eval_group() -> None:
+    """Evaluate the lightcone-cli build loop against seed tasks."""
+    import logging
+
+    from dotenv import load_dotenv
+
+    load_dotenv()
+
+    # Configure logging so sandbox build logs and harness progress are visible
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(name)s %(message)s",
+        datefmt="%H:%M:%S",
+    )
+
+
+@eval_group.command("run")
+@click.argument("config_path", type=click.Path(exists=True, path_type=Path))
+@click.option("--concurrency", "-c", type=int, default=None, help="Max parallel sandboxes")
+@click.option("--num-trials", "-n", type=int, default=None, help="Override number of trials")
+@click.option("--dry-run", is_flag=True, help="Print trial schedule without running")
+@click.option(
+    "--evals-dir",
+    type=click.Path(exists=True, path_type=Path),
+    default=None,
+    help="Path to evals/ directory (default: evals/ in current dir)",
+)
+def run_cmd(
+    config_path: Path,
+    concurrency: int | None,
+    num_trials: int | None,
+    dry_run: bool,
+    evals_dir: Path | None,
+) -> None:
+    """Run an eval suite from a config file.
+
+    Examples:
+        lc eval run evals/example-run.yaml
+        lc eval run evals/example-run.yaml --dry-run
+        lc eval run evals/example-run.yaml --num-trials 1 --concurrency 2
+    """
+    from lightcone.eval.harness import load_run_config, run_eval
+    from lightcone.eval.report import compute_summary, print_comparison_table, save_results
+
+    config = load_run_config(config_path)
+
+    # Apply CLI overrides
+    if concurrency is not None:
+        config.max_concurrency = concurrency
+    if num_trials is not None:
+        config.num_trials = num_trials
+
+    if evals_dir is None:
+        evals_dir = Path.cwd() / "evals"
+
+    if not evals_dir.exists():
+        console.print(f"[red]Error:[/red] Evals directory not found: {evals_dir}")
+        raise SystemExit(1)
+
+    if dry_run:
+        console.print("[bold]Dry run — trial schedule:[/bold]\n")
+
+    def _on_trial_complete(trial: object) -> None:
+        from lightcone.eval.models import TrialResult
+
+        assert isinstance(trial, TrialResult)
+        if trial.build_complete:
+            status = "[green]complete[/green]"
+        else:
+            status = "[yellow]incomplete[/yellow]"
+        if trial.error:
+            status = f"[red]error: {trial.error[:60]}[/red]"
+        console.print(
+            f"  {trial.task_id} "
+            f"trial {trial.trial_number}: "
+            f"score={trial.composite_score:.2f} {status}"
+        )
+
+    eval_run = run_eval(
+        config,
+        evals_dir,
+        progress_callback=_on_trial_complete,
+        dry_run=dry_run,
+    )
+
+    if dry_run:
+        schedule = eval_run.summary.get("schedule", [])
+        for s in schedule:
+            console.print(f"  {s['task']} trial {s['trial']}")
+        console.print(f"\n[bold]Total: {eval_run.summary.get('total_trials', 0)} trials[/bold]")
+        return
+
+    # Display version info
+    v = eval_run.version
+    dirty = " [yellow](dirty)[/yellow]" if v.lightcone_dirty else ""
+    console.print(
+        f"\n[bold]lightcone-cli:[/bold] {v.lightcone_version} "
+        f"({v.lightcone_branch} {v.lightcone_commit[:8]}){dirty}"
+    )
+    if v.astra_version:
+        console.print(f"[bold]ASTRA:[/bold] {v.astra_version}")
+
+    # Compute summary and display
+    eval_run.summary = compute_summary(eval_run)
+    console.print()
+    print_comparison_table(eval_run, console=console)
+
+    # Save results
+    output_path = save_results(eval_run, config.output_dir)
+    console.print(f"\n[bold]Results saved to:[/bold] {output_path}")
+
+
+@eval_group.command("report")
+@click.argument("results_path", type=click.Path(exists=True, path_type=Path))
+@click.option("--json", "as_json", is_flag=True, help="Output raw JSON summary")
+def report_cmd(results_path: Path, as_json: bool) -> None:
+    """Display results from a previous eval run.
+
+    Examples:
+        lc eval report eval-results/my-run-20260315.json
+        lc eval report eval-results/my-run-20260315.json --json
+    """
+    import json
+
+    from lightcone.eval.report import compute_summary, load_results, print_comparison_table
+
+    eval_run = load_results(results_path)
+
+    if not eval_run.summary:
+        eval_run.summary = compute_summary(eval_run)
+
+    if as_json:
+        console.print(json.dumps(eval_run.summary, indent=2, default=str))
+    else:
+        print_comparison_table(eval_run, console=console)
+
+
+@eval_group.command("compare")
+@click.argument("results1", type=click.Path(exists=True, path_type=Path))
+@click.argument("results2", type=click.Path(exists=True, path_type=Path))
+def compare_cmd(results1: Path, results2: Path) -> None:
+    """Compare two eval runs side by side.
+
+    Examples:
+        lc eval compare eval-results/run1.json eval-results/run2.json
+    """
+    from lightcone.eval.report import (
+        compute_summary,
+        load_results,
+        print_comparison_between,
+    )
+
+    run1 = load_results(results1)
+    run2 = load_results(results2)
+
+    if not run1.summary:
+        run1.summary = compute_summary(run1)
+    if not run2.summary:
+        run2.summary = compute_summary(run2)
+
+    print_comparison_between(run1, run2, console=console)
+
+

lightcone/eval/graders.py

@@ -0,0 +1,192 @@
+"""Grader dispatch for eval trials."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from lightcone.eval.models import GraderResult, GraderSpec, GraderType
+
+if TYPE_CHECKING:
+    from lightcone.eval.sandbox import EvalSandbox
+
+logger = logging.getLogger(__name__)
+
+
+def _error_result(grader: GraderSpec, error: str, **kwargs: object) -> GraderResult:
+    """Create a failed GraderResult with score 0."""
+    return GraderResult(
+        name=grader.name,
+        type=grader.type,
+        passed=False,
+        score=0.0,
+        weight=grader.weight,
+        error=error,
+        **kwargs,  # type: ignore[arg-type]
+    )
+
+
+def run_graders(
+    sandbox: EvalSandbox,
+    graders: list[GraderSpec],
+    evals_dir: Path,
+    task_id: str,
+) -> list[GraderResult]:
+    """Run all graders for a trial and return results."""
+    results: list[GraderResult] = []
+    for grader in graders:
+        try:
+            result = _run_single_grader(sandbox, grader, evals_dir, task_id)
+        except Exception as exc:
+            result = _error_result(grader, str(exc))
+        results.append(result)
+    return results
+
+
+def _run_single_grader(
+    sandbox: EvalSandbox,
+    grader: GraderSpec,
+    evals_dir: Path,
+    task_id: str,
+) -> GraderResult:
+    """Run a single grader and return its result."""
+    if grader.type == GraderType.command:
+        return _grade_command(sandbox, grader)
+    elif grader.type == GraderType.status:
+        return _grade_status(sandbox, grader)
+    elif grader.type == GraderType.script:
+        return _grade_script(sandbox, grader, evals_dir, task_id)
+    else:
+        return _error_result(grader, f"Unknown grader type: {grader.type}")
+
+
+def _grade_command(sandbox: EvalSandbox, grader: GraderSpec) -> GraderResult:
+    """Run a shell command; exit 0 = pass."""
+    if grader.command is None:
+        raise ValueError("command grader requires 'command' field")
+
+    cmd = f"cd {sandbox.WORK_DIR} && {grader.command}"
+    result = sandbox.exec(cmd, timeout=grader.timeout)
+    passed = result.exit_code == 0
+    return GraderResult(
+        name=grader.name,
+        type=grader.type,
+        passed=passed,
+        score=1.0 if passed else 0.0,
+        weight=grader.weight,
+        details=result.output[:2000] if result.output else "",
+    )
+
+
+def _grade_status(sandbox: EvalSandbox, grader: GraderSpec) -> GraderResult:
+    """Check output materialization status via lc status.
+
+    Parses the text table output, counting 'ok' vs 'pending'/'no_recipe' entries.
+    """
+    cmd = f"cd {sandbox.WORK_DIR} && lc status"
+    result = sandbox.exec(cmd, timeout=grader.timeout)
+
+    if result.exit_code != 0:
+        return _error_result(
+            grader,
+            f"lc status exited with code {result.exit_code}",
+            details=result.output[:2000],
+        )
+
+    # Parse the Rich table output for status values
+    materialized = 0
+    total = 0
+    for line in result.output.splitlines():
+        # Table rows start with │ and contain status words
+        if "│" not in line:
+            continue
+        cells = [c.strip() for c in line.split("│") if c.strip()]
+        if len(cells) < 2:
+            continue
+        # Skip header row
+        if cells[0].lower() in ("output", ""):
+            continue
+        # Each non-header cell after the first is a status value
+        for cell in cells[1:]:
+            if cell in ("ok", "pending", "no_recipe"):
+                total += 1
+                if cell == "ok":
+                    materialized += 1
+
+    if total == 0:
+        return GraderResult(
+            name=grader.name,
+            type=grader.type,
+            passed=False,
+            score=0.0,
+            weight=grader.weight,
+            details="No outputs found in lc status output",
+        )
+
+    score = materialized / total
+    return GraderResult(
+        name=grader.name,
+        type=grader.type,
+        passed=score == 1.0,
+        score=score,
+        weight=grader.weight,
+        details=f"{materialized}/{total} outputs materialized",
+    )
+
+
+
+def _grade_script(
+    sandbox: EvalSandbox,
+    grader: GraderSpec,
+    evals_dir: Path,
+    task_id: str,
+) -> GraderResult:
+    """Upload and run a custom grading script; exit 0 = pass, last stdout line as score."""
+    if grader.script is None:
+        raise ValueError("script grader requires 'script' field")
+
+    # Look for script in task graders dir
+    script_path = evals_dir / "tasks" / task_id / "graders" / grader.script
+    if not script_path.exists():
+        return _error_result(grader, f"Grader script not found: {script_path}")
+
+    remote_script = f"/tmp/grader_{grader.name}.py"
+    sandbox.upload_file(remote_script, script_path.read_bytes())
+
+    result = sandbox.exec(
+        f"cd {sandbox.WORK_DIR} && python {remote_script}",
+        timeout=grader.timeout,
+    )
+
+    passed = result.exit_code == 0
+    score = 1.0 if passed else 0.0
+
+    # Try to parse last stdout line as a float score
+    if result.output:
+        lines = result.output.strip().splitlines()
+        if lines:
+            try:
+                parsed = float(lines[-1].strip())
+                if 0.0 <= parsed <= 1.0:
+                    score = parsed
+                    passed = score > 0.0
+            except ValueError:
+                pass
+
+    return GraderResult(
+        name=grader.name,
+        type=grader.type,
+        passed=passed,
+        score=score,
+        weight=grader.weight,
+        details=result.output[:2000] if result.output else "",
+    )
+
+
+def compute_composite_score(results: list[GraderResult]) -> float:
+    """Compute weighted mean of grader scores."""
+    total_weight = sum(r.weight for r in results)
+    if total_weight == 0:
+        return 0.0
+    return sum(r.score * r.weight for r in results) / total_weight