latch-eval-tools 0.1.0__py3-none-any.whl

latch_eval_tools/graders/spatial.py

@@ -0,0 +1,93 @@
+from .base import BinaryGrader, GraderResult
+
+
+class SpatialAdjacencyGrader(BinaryGrader):
+    def evaluate_answer(self, agent_answer: dict, config: dict) -> GraderResult:
+        scoring = config.get("scoring", {})
+        thresholds = scoring.get("pass_thresholds", {})
+
+        max_median_ic_to_pc = thresholds.get("max_median_ic_to_pc_um", 25.0)
+        max_p90_ic_to_pc = thresholds.get("max_p90_ic_to_pc_um", 80.0)
+        min_pct_within_15um = thresholds.get("min_pct_ic_within_15um", 60.0)
+        min_pct_mixed_within_55um = thresholds.get("min_pct_ic_mixed_within_55um", 60.0)
+
+        required_fields = [
+            "median_ic_to_pc_um",
+            "p90_ic_to_pc_um",
+            "pct_ic_within_15um",
+            "pct_ic_mixed_within_55um",
+            "adjacency_pass"
+        ]
+
+        missing = [f for f in required_fields if f not in agent_answer]
+        if missing:
+            return GraderResult(
+                passed=False,
+                metrics={},
+                reasoning=f"Agent answer missing required fields: {missing}",
+                agent_answer=agent_answer
+            )
+
+        median_ic_to_pc = agent_answer["median_ic_to_pc_um"]
+        p90_ic_to_pc = agent_answer["p90_ic_to_pc_um"]
+        pct_within_15um = agent_answer["pct_ic_within_15um"]
+        pct_mixed_within_55um = agent_answer["pct_ic_mixed_within_55um"]
+        adjacency_pass = agent_answer["adjacency_pass"]
+
+        median_pass = median_ic_to_pc <= max_median_ic_to_pc
+        p90_pass = p90_ic_to_pc <= max_p90_ic_to_pc
+        within_15um_pass = pct_within_15um >= min_pct_within_15um
+        mixed_55um_pass = pct_mixed_within_55um >= min_pct_mixed_within_55um
+
+        passed = median_pass and p90_pass and within_15um_pass and mixed_55um_pass and adjacency_pass
+
+        metrics = {
+            "median_ic_to_pc_um": median_ic_to_pc,
+            "p90_ic_to_pc_um": p90_ic_to_pc,
+            "pct_ic_within_15um": pct_within_15um,
+            "pct_ic_mixed_within_55um": pct_mixed_within_55um,
+            "adjacency_pass": adjacency_pass,
+            "max_median_threshold": max_median_ic_to_pc,
+            "max_p90_threshold": max_p90_ic_to_pc,
+            "min_pct_15um_threshold": min_pct_within_15um,
+            "min_pct_55um_threshold": min_pct_mixed_within_55um,
+            "median_pass": median_pass,
+            "p90_pass": p90_pass,
+            "within_15um_pass": within_15um_pass,
+            "mixed_55um_pass": mixed_55um_pass,
+        }
+
+        lines = [
+            f"Spatial Adjacency Analysis: {'PASS' if passed else 'FAIL'}",
+            "",
+            "IC->PC Distance Metrics:",
+            f"  {'+'if median_pass else 'x'} Median distance: {median_ic_to_pc:.2f} um (threshold: <={max_median_ic_to_pc:.2f} um)",
+            f"  {'+'if p90_pass else 'x'} 90th percentile: {p90_ic_to_pc:.2f} um (threshold: <={max_p90_ic_to_pc:.2f} um)",
+            "",
+            "IC Proximity to PC:",
+            f"  {'+'if within_15um_pass else 'x'} IC within 15 um: {pct_within_15um:.1f}% (threshold: >={min_pct_within_15um:.1f}%)",
+            f"  {'+'if mixed_55um_pass else 'x'} IC with PC within 55 um: {pct_mixed_within_55um:.1f}% (threshold: >={min_pct_mixed_within_55um:.1f}%)",
+            "",
+            f"Agent adjacency assessment: {'+'if adjacency_pass else 'x'} {adjacency_pass}",
+        ]
+
+        if not passed:
+            failures = []
+            if not median_pass:
+                failures.append(f"Median {median_ic_to_pc:.2f} > {max_median_ic_to_pc:.2f} um")
+            if not p90_pass:
+                failures.append(f"P90 {p90_ic_to_pc:.2f} > {max_p90_ic_to_pc:.2f} um")
+            if not within_15um_pass:
+                failures.append(f"Within 15 um {pct_within_15um:.1f}% < {min_pct_within_15um:.1f}%")
+            if not mixed_55um_pass:
+                failures.append(f"Within 55 um {pct_mixed_within_55um:.1f}% < {min_pct_mixed_within_55um:.1f}%")
+            if not adjacency_pass:
+                failures.append("Agent marked adjacency_pass as false")
+            lines.append(f"\nFailure: {'; '.join(failures)}")
+
+        return GraderResult(
+            passed=passed,
+            metrics=metrics,
+            reasoning="\n".join(lines),
+            agent_answer=agent_answer
+        )

latch_eval_tools/harness/__init__.py

@@ -0,0 +1,27 @@
+from latch_eval_tools.harness.runner import EvalRunner
+from latch_eval_tools.harness.utils import (
+    get_project_root,
+    get_cache_dir,
+    download_single_dataset,
+    download_data,
+    batch_download_datasets,
+    setup_workspace,
+    cleanup_workspace,
+)
+from latch_eval_tools.harness.minisweagent import run_minisweagent_task
+from latch_eval_tools.harness.claudecode import run_claudecode_task
+from latch_eval_tools.harness.plotsagent import run_plotsagent_task
+
+__all__ = [
+    "EvalRunner",
+    "get_project_root",
+    "get_cache_dir",
+    "download_single_dataset",
+    "download_data",
+    "batch_download_datasets",
+    "setup_workspace",
+    "cleanup_workspace",
+    "run_minisweagent_task",
+    "run_claudecode_task",
+    "run_plotsagent_task",
+]

latch_eval_tools/harness/claudecode.py

@@ -0,0 +1,212 @@
+import json
+import os
+import re
+import subprocess
+import time
+from pathlib import Path
+
+EVAL_TIMEOUT = 600
+
+MODEL_MAP = {
+    "anthropic/claude-opus-4-5": "opus",
+    "anthropic/claude-sonnet-4-5": "sonnet",
+    "anthropic/claude-sonnet-4": "claude-sonnet-4-20250514",
+    "anthropic/claude-opus-4": "claude-opus-4-20250514",
+    "anthropic/claude-haiku-3-5": "haiku",
+}
+
+
+def run_claudecode_task(
+    task_prompt: str,
+    work_dir: Path,
+    model_name: str | None = None,
+    eval_timeout: int = EVAL_TIMEOUT,
+) -> dict:
+    """Run Claude Code agent on a task.
+    
+    Args:
+        task_prompt: Task description for the agent
+        work_dir: Working directory for the agent
+        model_name: Optional model name (e.g., "anthropic/claude-sonnet-4")
+        eval_timeout: Timeout for entire evaluation (seconds)
+    
+    Returns:
+        dict with keys "answer" (parsed JSON or None) and "metadata"
+    """
+    agent_log_file = work_dir / "agent_output.log"
+    if agent_log_file.exists():
+        agent_log_file.unlink()
+
+    enhanced_prompt = _enhance_prompt_with_local_files(task_prompt, work_dir)
+    enhanced_prompt += """
+
+CRITICAL: You must write eval_answer.json BEFORE signaling completion.
+Correct order: 1) Perform analysis 2) Write eval_answer.json with your answer 3) Exit"""
+
+    cmd = ["claude", "--print", "--dangerously-skip-permissions", "--verbose", "--output-format", "stream-json"]
+
+    if model_name:
+        claude_model = MODEL_MAP.get(model_name, model_name)
+        cmd.extend(["--model", claude_model])
+
+    run_as_claude_user = os.geteuid() == 0
+    if run_as_claude_user:
+        import pwd
+        import shutil
+        import stat
+        try:
+            pwd.getpwnam("claude")
+            home_dir = Path.home()
+            current_mode = home_dir.stat().st_mode
+            home_dir.chmod(current_mode | stat.S_IXOTH)
+            eval_cache_dir = home_dir / ".eval_cache"
+            if eval_cache_dir.exists():
+                shutil.chown(eval_cache_dir, user="claude", group="claude")
+                for item in eval_cache_dir.rglob("*"):
+                    try:
+                        shutil.chown(item, user="claude", group="claude")
+                    except PermissionError:
+                        pass
+        except KeyError:
+            run_as_claude_user = False
+
+    env = os.environ.copy()
+
+    start_time = time.time()
+    timed_out = False
+    claude_result = None
+    trajectory = []
+
+    try:
+        if run_as_claude_user:
+            env_vars = [f"{k}={v}" for k, v in env.items() if k.endswith("_API_KEY")]
+            cmd = ["runuser", "-u", "claude", "--", "env"] + env_vars + cmd
+
+        process = subprocess.Popen(
+            cmd,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=str(work_dir),
+            env=env,
+            text=True,
+        )
+
+        try:
+            stdout, stderr = process.communicate(
+                input=enhanced_prompt,
+                timeout=eval_timeout
+            )
+
+            with open(agent_log_file, 'w') as log_file:
+                log_file.write(stdout)
+                if stderr:
+                    log_file.write(f"\n\nSTDERR:\n{stderr}")
+
+            for line in stdout.strip().split('\n'):
+                if line:
+                    try:
+                        event = json.loads(line)
+                        trajectory.append(event)
+                        if event.get("type") == "result":
+                            claude_result = event
+                    except json.JSONDecodeError:
+                        pass
+
+        except subprocess.TimeoutExpired:
+            timed_out = True
+            process.kill()
+            stdout, stderr = process.communicate()
+            with open(agent_log_file, 'w') as log_file:
+                log_file.write(stdout)
+                log_file.write(f"\n\nAgent timed out after {eval_timeout} seconds")
+
+    except Exception as e:
+        with open(agent_log_file, 'a') as f:
+            f.write(f"\nError running claude: {e}")
+
+    duration = time.time() - start_time
+    print(f"Agent output saved to: {agent_log_file}")
+
+    if trajectory:
+        trajectory_file = work_dir / "trajectory.json"
+        trajectory_file.write_text(json.dumps(trajectory, indent=2))
+        print(f"Trajectory saved to: {trajectory_file}")
+
+    eval_answer_file = work_dir / "eval_answer.json"
+    agent_answer = None
+    error_details = None
+
+    if not eval_answer_file.exists():
+        log_tail = ""
+        if agent_log_file.exists():
+            log_content = agent_log_file.read_text()
+            log_tail = log_content[-1000:]
+
+        error_msg = "Agent timed out" if timed_out else "Agent did not create eval_answer.json"
+        error_details = {
+            "error": error_msg,
+            "timed_out": timed_out,
+            "log_tail": log_tail
+        }
+        print(f"\nWarning: {error_msg}")
+    else:
+        try:
+            agent_answer = json.loads(eval_answer_file.read_text())
+        except json.JSONDecodeError as e:
+            error_details = {
+                "error": f"Failed to parse eval_answer.json: {e}",
+                "file_contents": eval_answer_file.read_text()[:500]
+            }
+            print(f"\nWarning: Failed to parse eval_answer.json: {e}")
+
+    metadata = {
+        "duration_s": round(duration, 2),
+        "model": model_name,
+    }
+    if claude_result:
+        metadata["total_cost"] = claude_result.get("total_cost_usd")
+        metadata["n_turns"] = claude_result.get("num_turns")
+        metadata["session_id"] = claude_result.get("session_id")
+        metadata["usage"] = claude_result.get("usage")
+    if timed_out:
+        metadata["timed_out"] = True
+        metadata["eval_timeout_seconds"] = eval_timeout
+    if error_details:
+        metadata["error_details"] = error_details
+
+    return {"answer": agent_answer, "metadata": metadata}
+
+
+def _enhance_prompt_with_local_files(task_prompt: str, work_dir: Path) -> str:
+    """Extract <ContextualNodeData> and add local file list to prompt."""
+    contextual_data_match = re.search(
+        r'<ContextualNodeData>(.*?)</ContextualNodeData>',
+        task_prompt,
+        re.DOTALL
+    )
+
+    if not contextual_data_match:
+        return task_prompt
+
+    try:
+        contextual_data = json.loads(contextual_data_match.group(1))
+    except json.JSONDecodeError:
+        return task_prompt
+
+    local_files = []
+    for item in contextual_data:
+        if 'local_path' in item:
+            local_files.append(item['local_path'])
+
+    if not local_files:
+        return task_prompt
+
+    file_list = "\n".join([f"- {f}" for f in local_files])
+    enhancement = f"\n\nThe following data files are available in your current working directory:\n{file_list}\n\nUse these local filenames to access the data.\n"
+
+    parts = task_prompt.split('<ContextualNodeData>')
+    if len(parts) == 2:
+        return parts[0] + enhancement + '<ContextualNodeData>' + parts[1]
+
+    return task_prompt

latch_eval_tools/harness/minisweagent.py

@@ -0,0 +1,265 @@
+import io
+import json
+import os
+import re
+import signal
+import sys
+from pathlib import Path
+
+OPERATION_TIMEOUT = 300
+EVAL_TIMEOUT = 600
+
+
+class AgentTimeoutError(Exception):
+    pass
+
+
+def _timeout_handler(signum, frame):
+    raise AgentTimeoutError("Agent exceeded time limit")
+
+
+class StreamingLogFile:
+    def __init__(self, file_path):
+        self.file_path = file_path
+        self.buffer = io.StringIO()
+
+    def write(self, data):
+        self.buffer.write(data)
+        with open(self.file_path, 'a') as f:
+            f.write(data)
+            f.flush()
+
+    def flush(self):
+        pass
+
+    def getvalue(self):
+        return self.buffer.getvalue()
+
+
+def _patch_agent_for_progress(log_file, agent_class):
+    original_add_message = agent_class.add_message
+
+    def patched_add_message(self, role, content, **kwargs):
+        original_add_message(self, role, content, **kwargs)
+
+        with open(log_file, 'a') as f:
+            if role == "assistant":
+                step_num = len([m for m in self.messages if m.get("role") == "assistant"])
+                f.write(f"\n[Step {step_num}]\n")
+                f.write(f"Assistant: {content}\n")
+            elif role == "user" and len(self.messages) > 2:
+                f.write(f"Observation: {content}\n")
+            f.flush()
+
+    agent_class.add_message = patched_add_message
+
+
+def run_minisweagent_task(
+    task_prompt: str,
+    work_dir: Path,
+    model_name: str | None = None,
+    agent_config: dict | None = None,
+    operation_timeout: int = OPERATION_TIMEOUT,
+    eval_timeout: int = EVAL_TIMEOUT,
+) -> dict:
+    """Run MiniSWE agent on a task.
+    
+    Args:
+        task_prompt: Task description for the agent
+        work_dir: Working directory for the agent
+        model_name: Optional model name (e.g., "anthropic/claude-sonnet-4")
+        agent_config: Optional agent configuration dict
+        operation_timeout: Timeout for individual operations (seconds)
+        eval_timeout: Timeout for entire evaluation (seconds)
+    
+    Returns:
+        dict with keys "answer" (parsed JSON or None) and "metadata"
+    """
+    from minisweagent.agents.default import DefaultAgent, AgentConfig, FormatError
+    from minisweagent.environments.local import LocalEnvironment
+    from minisweagent.models import get_model
+    import re
+
+    class FlexibleAgent(DefaultAgent):
+        def parse_action(self, response: dict) -> dict:
+            content = response["content"]
+            actions = re.findall(r"```(?:bash|sh|shell)?\s*\n(.*?)\n?```", content, re.DOTALL)
+            if len(actions) == 1:
+                return {"action": actions[0].strip(), **response}
+            raise FormatError(self.render_template(self.config.format_error_template, actions=actions))
+
+        def has_finished(self, output: dict):
+            from minisweagent.agents.default import Submitted
+            full_output = output.get("output", "")
+            for marker in ["COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT", "MINI_SWE_AGENT_FINAL_OUTPUT"]:
+                if marker in full_output:
+                    idx = full_output.find(marker)
+                    rest = full_output[idx + len(marker):].strip()
+                    raise Submitted(rest)
+
+    original_dir = os.getcwd()
+
+    agent_log_file = work_dir / "agent_output.log"
+    _patch_agent_for_progress(agent_log_file, FlexibleAgent)
+    if agent_log_file.exists():
+        agent_log_file.unlink()
+
+    captured_output = StreamingLogFile(agent_log_file)
+    original_stdout = sys.stdout
+    original_stderr = sys.stderr
+
+    class TeeOutput:
+        def __init__(self, *streams):
+            self.streams = streams
+
+        def write(self, data):
+            for stream in self.streams:
+                stream.write(data)
+                if hasattr(stream, 'flush'):
+                    stream.flush()
+
+        def flush(self):
+            for stream in self.streams:
+                if hasattr(stream, 'flush'):
+                    stream.flush()
+
+    agent = None
+    timed_out = False
+    try:
+        os.chdir(str(work_dir))
+
+        sys.stdout = TeeOutput(original_stdout, captured_output)
+        sys.stderr = TeeOutput(original_stderr, captured_output)
+
+        enhanced_prompt = _enhance_prompt_with_local_files(task_prompt, work_dir)
+
+        enhanced_prompt += """
+
+CRITICAL INSTRUCTIONS:
+1. Do NOT wrap your code in try/except blocks. Let errors propagate so you can see them and fix them in subsequent steps.
+2. You must write eval_answer.json BEFORE printing the completion signal.
+3. Correct order: Perform analysis -> Write eval_answer.json -> Print 'COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT' as your FINAL line of output."""
+
+        if model_name:
+            os.environ['MSWEA_MODEL_NAME'] = model_name
+
+        model = get_model()
+        env = LocalEnvironment(timeout=operation_timeout)
+
+        if agent_config:
+            agent = FlexibleAgent(model, env, step_limit=100, **agent_config)
+        else:
+            agent = FlexibleAgent(model, env, step_limit=100)
+
+        old_handler = signal.signal(signal.SIGALRM, _timeout_handler)
+        signal.alarm(eval_timeout)
+
+        try:
+            agent.run(enhanced_prompt)
+        except AgentTimeoutError:
+            timed_out = True
+            print(f"\nAgent timed out after {eval_timeout} seconds")
+        except Exception as e:
+            if "Submitted" in str(type(e).__name__):
+                pass
+            else:
+                raise
+        finally:
+            signal.alarm(0)
+            signal.signal(signal.SIGALRM, old_handler)
+
+            sys.stdout = original_stdout
+            sys.stderr = original_stderr
+
+            print(f"Agent output saved to: {agent_log_file}")
+
+            if hasattr(agent, "messages"):
+                trajectory_file = work_dir / "trajectory.json"
+                trajectory_data = {
+                    "messages": agent.messages,
+                    "actions": getattr(agent, "actions", [])
+                }
+                trajectory_file.write_text(json.dumps(trajectory_data, indent=2))
+                print(f"Agent trajectory saved to: {trajectory_file}")
+                print(f"  Total message exchanges: {len(agent.messages)}")
+
+        eval_answer_file = work_dir / "eval_answer.json"
+        agent_answer = None
+        error_details = None
+
+        if not eval_answer_file.exists():
+            agent_log_file = work_dir / "agent_output.log"
+            log_tail = ""
+            if agent_log_file.exists():
+                log_content = agent_log_file.read_text()
+                log_tail = log_content[-1000:]
+
+            trajectory_info = ""
+            if hasattr(agent, "messages"):
+                trajectory_info = f"Agent had {len(agent.messages)} message exchanges."
+
+            error_msg = "Agent timed out" if timed_out else "Agent did not create eval_answer.json"
+            error_details = {
+                "error": error_msg,
+                "timed_out": timed_out,
+                "trajectory_info": trajectory_info,
+                "log_tail": log_tail
+            }
+            print(f"\nWarning: {error_msg}. {trajectory_info}")
+        else:
+            try:
+                agent_answer = json.loads(eval_answer_file.read_text())
+            except json.JSONDecodeError as e:
+                error_details = {
+                    "error": f"Failed to parse eval_answer.json: {e}",
+                    "file_contents": eval_answer_file.read_text()[:500]
+                }
+                print(f"\nWarning: Failed to parse eval_answer.json: {e}")
+
+        metadata = {}
+        if hasattr(agent, "model"):
+            metadata["total_cost"] = getattr(agent.model, "cost", None)
+            metadata["n_steps"] = getattr(agent.model, "n_calls", None)
+        if hasattr(agent, "messages"):
+            metadata["n_messages"] = len(agent.messages)
+        if timed_out:
+            metadata["timed_out"] = True
+            metadata["eval_timeout_seconds"] = eval_timeout
+        if error_details:
+            metadata["error_details"] = error_details
+
+        return {"answer": agent_answer, "metadata": metadata}
+
+    finally:
+        os.chdir(original_dir)
+
+
+def _enhance_prompt_with_local_files(task_prompt: str, work_dir: Path) -> str:
+    """Extract <ContextualNodeData> and add local file list to prompt."""
+    contextual_data_match = re.search(r'<ContextualNodeData>(.*?)</ContextualNodeData>', task_prompt, re.DOTALL)
+
+    if not contextual_data_match:
+        return task_prompt
+
+    try:
+        contextual_data = json.loads(contextual_data_match.group(1))
+    except json.JSONDecodeError:
+        return task_prompt
+
+    local_files = []
+    for item in contextual_data:
+        if 'local_path' in item:
+            local_files.append(item['local_path'])
+
+    if not local_files:
+        return task_prompt
+
+    file_list = "\n".join([f"- {f}" for f in local_files])
+
+    enhancement = f"\n\nThe following data files are available in your current working directory:\n{file_list}\n\nUse these local filenames to access the data.\n"
+
+    parts = task_prompt.split('<ContextualNodeData>')
+    if len(parts) == 2:
+        return parts[0] + enhancement + '<ContextualNodeData>' + parts[1]
+
+    return task_prompt