ssebench 1.0.0__py3-none-any.whl

ssebench/__init__.py

@@ -0,0 +1,5 @@
+"""SSEBench: run AI agents against real vulnerabilities and grade their patches."""
+
+from ssebench.version import __version__
+
+__all__ = ["__version__"]

ssebench/__main__.py

@@ -0,0 +1,7 @@
+import logging
+
+from ssebench.cli import main
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()

ssebench/_data/.env.example

@@ -0,0 +1,33 @@
+# Local settings. `ssebench init` and `just setup` copy this file to .env and
+# fill in the two generated secrets. .env is ignored by git; never commit it.
+
+# Admin key of the local LiteLLM proxy. The CLI uses it to create a key with a
+# budget for each run.
+LITELLM_MASTER_KEY=
+# Password of the proxy's Postgres database; letters and digits only, as it is
+# part of a URL. Postgres keeps the password it was created with: after you
+# change it, remove the stack's volume (`docker compose -p <project> down -v`).
+POSTGRES_PASSWORD=
+
+# Keys of the model providers you use. models/*.yaml refers to them as
+# os.environ/<NAME>; a model without its key fails when it is called. After a
+# change, restart the proxy with `ssebench proxy up` (`just launch`).
+ANTHROPIC_API_KEY=
+OPENAI_API_KEY=
+GOOGLE_API_KEY=
+
+# Optional settings, shown with their defaults.
+# Host port of the LiteLLM proxy.
+# LITELLM_PORT=4000
+# Host address the proxy's port is published on. It holds the master key and the
+# provider keys, so it listens on loopback only.
+# LITELLM_BIND=127.0.0.1
+# Compose project of the proxy stack; stacks with different names (and ports)
+# run side by side.
+# COMPOSE_PROJECT_NAME=ssebench
+# Registry prefix of every image SSEBench builds or pulls.
+# SSEBENCH_REGISTRY=ghcr.io/42-b3yond-6ug/ssebench
+# Task catalog of `ssebench run` and the web UI: the path or URL of a
+# manifest.json, a dataset directory, or the URL of a catalog service. Unset,
+# it is the bundled datasets/pilot/manifest.json.
+# SSEBENCH_CATALOG=

ssebench/_data/agents/README.md

@@ -0,0 +1,26 @@
+# agents
+
+One folder per agent. An agent is the layer that goes on top of the tool layer
+of a run; `ssebench run --agent <name>` builds it from its folder.
+
+| Agent | What it is |
+|---|---|
+| [`claude-code/`](claude-code/) | Anthropic's Claude Code. |
+| [`codex/`](codex/) | OpenAI's Codex CLI. |
+| [`opencode/`](opencode/) | The open-source OpenCode agent. |
+| [`dummy/`](dummy/) | Does nothing and makes no model calls; a run with it grades the untouched source, so its patch fails. It tests the pipeline. |
+| [`reference/`](reference/) | Applies the task's known fix and makes no model calls; a sound task passes every check. It checks a task, and powers `just demo`. Its runs never count as a model's score. |
+
+Each folder has an `agent.yaml` (the agent's name), a `Dockerfile` and, for
+every agent but `dummy`, a wrapper `<name>-sse/`. The wrapper is a Python project
+that starts the agent, reads the task through the `sse` SDK and writes
+`dialog.jsonl`; every wrapper is a member of the repository's uv workspace. An
+agent never talks to a model provider directly: all LLM traffic goes through the
+LiteLLM proxy, at the endpoint the run passes in `SSE_BASE_URL`.
+
+To add an agent, follow [Add an agent](../docs/guides/add-an-agent.md).
+
+- [Dialog protocol](../docs/reference/dialog-protocol.md): the format of
+  `dialog.jsonl`
+- [Reference runs](../docs/reference/cli.md#reference-runs)
+- [Image layers](../docs/concepts/image-layers.md)

ssebench/_data/agents/claude-code/.dockerignore

@@ -0,0 +1,11 @@
+# Python virtual environments
+**/.venv
+**/__pycache__
+**/*.pyc
+
+# IDE
+**/.idea
+**/.vscode
+
+# Cache
+**/.cache

ssebench/_data/agents/claude-code/Dockerfile

@@ -0,0 +1,64 @@
+# Stage 1: Install Claude Code (cached separately from agent code changes)
+FROM ubuntu:22.04 AS claude-installer
+RUN apt-get update && apt-get install -y curl ca-certificates && rm -rf /var/lib/apt/lists/*
+RUN useradd -m -u 1000 model
+USER model
+ENV HOME=/home/model
+ARG TARGETARCH
+ARG CLAUDE_CODE_VERSION=2.1.277
+# Checksums of the linux-x64 and linux-arm64 builds, from the release's
+# manifest.json; update both when bumping the version.
+ARG CLAUDE_CODE_SHA256_AMD64=722210f05ba494d8f6df69423c4d4f2960900f7a007d0532851c7a36e375cab7
+ARG CLAUDE_CODE_SHA256_ARM64=242c4d743beabc822edd8f247101bb800b4036c69e74b9e2a1adb120dfe46f5d
+# `claude install` sets up the launcher in ~/.local/bin for this version.
+RUN case "${TARGETARCH}" in \
+      amd64) platform=linux-x64; sha256="${CLAUDE_CODE_SHA256_AMD64}" ;; \
+      arm64) platform=linux-arm64; sha256="${CLAUDE_CODE_SHA256_ARM64}" ;; \
+      *) echo "no Claude Code release for ${TARGETARCH}" >&2; exit 1 ;; \
+    esac && \
+    curl -fsSLo /tmp/claude \
+        "https://downloads.claude.ai/claude-code-releases/${CLAUDE_CODE_VERSION}/${platform}/claude" && \
+    echo "${sha256}  /tmp/claude" | sha256sum -c - && \
+    chmod +x /tmp/claude && \
+    /tmp/claude install "${CLAUDE_CODE_VERSION}" && \
+    rm /tmp/claude
+
+# Stage 2: Final image
+FROM ssebench-agent
+
+# The wrapper is a member of the uv workspace at the repository root, so /app
+# holds the workspace files it needs (from the `workspace` build context) in the
+# repository layout.
+# Create /app owned by model user (WORKDIR creates as root)
+RUN mkdir -p /app/agents/claude-code/claude-code-sse && chown -R model:model /app
+WORKDIR /app/agents/claude-code/claude-code-sse
+
+# Copy Claude Code installation from stage 1
+COPY --from=claude-installer --chown=model:model /home/model/.local /home/model/.local
+COPY --from=claude-installer --chown=model:model /home/model/.claude /home/model/.claude
+
+# Ensure model user's home and cache directories have correct ownership
+# This prevents "Permission denied" errors when uv runs as model user
+RUN mkdir -p /home/model/.cache && \
+    rm -rf /home/model/.cache/uv && \
+    chown -R model:model /home/model
+
+# Only set CLAUDE path - HOME is set by entrypoint when running as model user
+# Setting HOME globally causes MCP server (running as root) to pollute model's cache
+ENV CLAUDE=/home/model/.local/bin/claude
+
+COPY --from=workspace --chown=model:model pyproject.toml uv.lock /app/
+COPY --from=workspace --chown=model:model sdk/python/pyproject.toml sdk/python/README.md /app/sdk/python/
+COPY --from=workspace --chown=model:model sdk/python/sse/ /app/sdk/python/sse/
+
+# Copy agent code (changes frequently, but doesn't invalidate cached stages above)
+# .dockerignore excludes .venv, __pycache__, etc.
+COPY --chown=model:model claude-code-sse/ /app/agents/claude-code/claude-code-sse
+
+# Install the wrapper's environment, Python included, now: at run time the
+# container reaches the LiteLLM proxy only.
+USER model
+RUN HOME=/home/model uv sync --frozen --no-dev --package claude-code-sse
+USER root
+
+CMD ["./run.sh"]

ssebench/_data/agents/claude-code/agent.yaml

@@ -0,0 +1 @@
+name: claude-code

ssebench/_data/agents/claude-code/claude-code-sse/main.py

@@ -0,0 +1,441 @@
+import asyncio
+import json
+import os
+import subprocess
+import sys
+import time
+from asyncio.streams import StreamReader
+from datetime import UTC, datetime
+from io import TextIOWrapper
+from pathlib import Path
+from typing import Any
+
+from sse import project
+from sse.prompt import task_prompt
+
+# =============================================================================
+# Dialog Writer - Converts Claude Code stream-json to SSEBench dialog format
+# =============================================================================
+
+
+class DialogWriter:
+    """
+    Writes agent dialog entries to $SSE_ARCHIVE/dialog.jsonl for WebUI consumption.
+
+    Converts Claude Code's stream-json output to our simplified dialog format:
+    - init: Session start metadata
+    - prompt: User task given to agent
+    - message: Assistant text responses
+    - thinking: Reasoning blocks (extended thinking)
+    - tool: Tool calls with status (running/success/error)
+    - complete: Session end with stats
+    """
+
+    def __init__(self, log_path: str | None = None):
+        # Default to $SSE_ARCHIVE/dialog.jsonl, in the agent's archive directory
+        if log_path is None:
+            archive = os.environ.get("SSE_ARCHIVE", "/tmp/sse-archive")
+            log_path = f"{archive}/dialog.jsonl"
+        self.log_path: Path = Path(log_path)
+        self.log_path.parent.mkdir(parents=True, exist_ok=True)
+        self.seq: int = 0
+        self.start_time: float = time.time()
+        self.turns: int = 0
+        self.total_input_tokens: int = 0
+        self.total_output_tokens: int = 0
+        self.pending_tools: dict[str, dict[str, str]] = {}  # tool_id -> entry data
+        self.completed: bool = False
+        self.file: TextIOWrapper = open(self.log_path, "w")
+
+    def _write_entry(self, entry: dict[str, Any]):
+        """Write a single entry to the JSONL file."""
+        _ = self.file.write(json.dumps(entry) + "\n")
+        self.file.flush()
+
+    def _now(self) -> str:
+        """Get current timestamp in ISO format."""
+        return datetime.now(UTC).isoformat()
+
+    def _next_seq(self) -> int:
+        """Get next sequence number."""
+        seq = self.seq
+        self.seq += 1
+        return seq
+
+    def write_init(self, task: str, cwd: str, model: str, agent: str = "claude-code"):
+        """Write session init entry."""
+        self._write_entry(
+            {
+                "seq": self._next_seq(),
+                "ts": self._now(),
+                "type": "init",
+                "data": {
+                    "task": task,
+                    "cwd": cwd,
+                    "model": model,
+                    "agent": agent,
+                },
+            }
+        )
+
+    def write_prompt(self, content: str):
+        """Write user prompt entry."""
+        self._write_entry(
+            {
+                "seq": self._next_seq(),
+                "ts": self._now(),
+                "type": "prompt",
+                "content": content,
+            }
+        )
+
+    def write_message(
+        self, content: str, input_tokens: int = 0, output_tokens: int = 0
+    ):
+        """Write assistant message entry."""
+        entry: dict[str, Any] = {
+            "seq": self._next_seq(),
+            "ts": self._now(),
+            "type": "message",
+            "role": "assistant",
+            "content": content,
+        }
+        if input_tokens > 0 or output_tokens > 0:
+            entry["tokens"] = {"in": input_tokens, "out": output_tokens}
+            self.total_input_tokens += input_tokens
+            self.total_output_tokens += output_tokens
+        self._write_entry(entry)
+
+    def write_thinking(self, content: str):
+        """Write thinking/reasoning entry."""
+        self._write_entry(
+            {
+                "seq": self._next_seq(),
+                "ts": self._now(),
+                "type": "thinking",
+                "content": content,
+            }
+        )
+
+    def write_tool_start(
+        self, tool_id: str, name: str, args: dict[str, Any] | None = None
+    ):
+        """Write tool call start entry (status=running)."""
+        entry: dict[str, Any] = {
+            "seq": self._next_seq(),
+            "ts": self._now(),
+            "type": "tool",
+            "tool_id": tool_id,
+            "name": name,
+            "status": "running",
+        }
+        if args:
+            entry["args"] = args
+        self._write_entry(entry)
+        self.pending_tools[tool_id] = {"name": name}
+
+    def write_tool_result(
+        self, tool_id: str, result: str | None = None, error: str | None = None
+    ):
+        """Write tool call result entry (status=success or error)."""
+        name = self.pending_tools.pop(tool_id, {}).get("name", "unknown")
+        entry = {
+            "seq": self._next_seq(),
+            "ts": self._now(),
+            "type": "tool",
+            "tool_id": tool_id,
+            "name": name,
+            "status": "error" if error else "success",
+        }
+        if result:
+            entry["result"] = result
+        if error:
+            entry["error"] = error
+        self._write_entry(entry)
+
+    def write_complete(self, status: str = "success", message: str | None = None):
+        """Write session completion entry. Guarded against double-writes."""
+        if self.completed:
+            return
+        self.completed = True
+        duration_ms = int((time.time() - self.start_time) * 1000)
+        entry = {
+            "seq": self._next_seq(),
+            "ts": self._now(),
+            "type": "complete",
+            "status": status,
+            "turns": self.turns,
+            "duration_ms": duration_ms,
+            "total_tokens": {
+                "in": self.total_input_tokens,
+                "out": self.total_output_tokens,
+            },
+        }
+        if message:
+            entry["message"] = message
+        self._write_entry(entry)
+
+    def process_claude_event(self, event: dict[str, Any]):
+        """
+        Process a single Claude Code stream-json event and convert to dialog format.
+
+        Claude Code stream-json events include:
+        - {"type": "assistant", "message": {...}}
+        - {"type": "user", "message": {...}}
+        - {"type": "result", ...}
+        - {"type": "system", ...}
+        """
+        event_type = event.get("type", "")
+
+        if event_type == "assistant":
+            self.turns += 1
+            message = event.get("message", {})
+            content_blocks = message.get("content", [])
+
+            # Extract per-turn token usage
+            usage = message.get("usage", {})
+            input_tokens = usage.get("input_tokens", 0)
+            output_tokens = usage.get("output_tokens", 0)
+            tokens_attributed = False
+
+            for block in content_blocks:
+                block_type = block.get("type", "")
+
+                if block_type == "text":
+                    text = block.get("text", "")
+                    if text:
+                        if not tokens_attributed:
+                            self.write_message(text, input_tokens, output_tokens)
+                            tokens_attributed = True
+                        else:
+                            self.write_message(text)
+
+                elif block_type == "thinking":
+                    thinking = block.get("thinking", "")
+                    if thinking:
+                        self.write_thinking(thinking)
+
+                elif block_type == "tool_use":
+                    tool_id = block.get("id", str(self._next_seq()))
+                    tool_name = block.get("name", "unknown")
+                    tool_input = block.get("input", {})
+                    self.write_tool_start(tool_id, tool_name, tool_input)
+
+            # If no message was written, still track token usage globally
+            if not tokens_attributed:
+                self.total_input_tokens += input_tokens
+                self.total_output_tokens += output_tokens
+
+        elif event_type == "user":
+            message = event.get("message", {})
+            content_blocks = message.get("content", [])
+
+            for block in content_blocks:
+                block_type = block.get("type", "")
+
+                if block_type == "tool_result":
+                    tool_id = block.get("tool_use_id", "")
+                    is_error = block.get("is_error", False)
+                    content = block.get("content", "")
+
+                    # Content can be string or list of content blocks
+                    if isinstance(content, list):
+                        text_parts = []
+                        for c in content:
+                            if isinstance(c, dict) and c.get("type") == "text":
+                                text_parts.append(c.get("text", ""))
+                            elif isinstance(c, str):
+                                text_parts.append(c)
+                        content = "\n".join(text_parts)
+
+                    if is_error:
+                        self.write_tool_result(tool_id, error=content)
+                    else:
+                        self.write_tool_result(tool_id, result=content)
+
+        elif event_type == "result":
+            # Session ended. `result` holds the final text; `subtype` says how it ended.
+            subtype = event.get("subtype", "")
+            if subtype == "success" and not event.get("is_error", False):
+                self.write_complete("success")
+            else:
+                self.write_complete(
+                    "error", message=f"Session ended: {subtype or 'unknown'}"
+                )
+
+    def close(self):
+        """Close the log file."""
+        self.file.close()
+
+
+# =============================================================================
+# Process Output Handling
+# =============================================================================
+
+
+async def stream_output_with_dialog(
+    stream: StreamReader, prefix: str = "", dialog: DialogWriter | None = None
+):
+    """
+    Reads from a stream line-by-line, prints it with a prefix,
+    and processes JSONL events for dialog logging.
+    """
+    buf = b""
+
+    while True:
+        try:
+            chunk = await stream.read(4096)
+            buf += chunk
+            line_sep = b"\n"
+            while line_sep in buf:
+                first_line, buf = buf.split(line_sep, maxsplit=1)
+                line_str = first_line.decode().strip()
+                print(f"{prefix} {line_str}", flush=True)
+
+                # Try to parse as JSON for dialog processing
+                if dialog and prefix == "[STDOUT]" and line_str:
+                    try:
+                        event: dict[str, Any] = json.loads(line_str)
+                        dialog.process_claude_event(event)
+                    except json.JSONDecodeError:
+                        pass  # Not JSON, skip
+
+            if len(chunk) == 0:
+                # EOF, prints whatever we have in buf
+                remaining = buf.decode().strip()
+                if remaining:
+                    print(f"{prefix} {remaining}", flush=True)
+                break
+
+        except Exception as e:
+            print(f"Error reading stream {prefix}: {e}", file=sys.stderr)
+            continue
+
+
+async def run_claude(dialog: DialogWriter):
+    try:
+        claude_executable = os.environ["CLAUDE"]
+        model_name = os.environ["SSE_MODEL_NAME"]
+        base_url = os.environ["SSE_BASE_URL"]
+        api_key = os.environ["SSE_API_KEY"]
+    except KeyError as _:
+        dialog.write_complete("error", message="Missing environment variables")
+        return
+
+    # Write init entry
+    dialog.write_init(
+        task=project.metadata.id,
+        cwd=str(project.source),
+        model=model_name,
+        agent="claude-code",
+    )
+
+    prompt = task_prompt()
+    dialog.write_prompt(prompt)
+
+    # Build Claude command arguments
+    # --dangerously-skip-permissions requires non-root user (handled by entrypoint)
+    command_args = [
+        claude_executable,
+        "--verbose",
+        "--max-turns",
+        "500",
+        "-p",
+        "--model",
+        model_name,
+        "--output-format",
+        "stream-json",
+        "--dangerously-skip-permissions",
+        # Passed here rather than written to the project's .claude/, which
+        # would become part of the agent's patch.
+        "--settings",
+        json.dumps({"permissions": {"defaultMode": "bypassPermissions"}}),
+    ]
+
+    # Set up environment for Claude Code
+    env = os.environ.copy()
+    env["ANTHROPIC_BASE_URL"] = base_url
+    env["ANTHROPIC_AUTH_TOKEN"] = api_key
+    env["ANTHROPIC_DEFAULT_HAIKU_MODEL"] = model_name  # redirects the haiku slot
+    env["ANTHROPIC_DEFAULT_SONNET_MODEL"] = model_name  # redirects the sonnet slot
+    env["ANTHROPIC_DEFAULT_OPUS_MODEL"] = model_name  # redirects the opus slot
+
+    process = await asyncio.create_subprocess_exec(
+        *command_args,
+        cwd=project.source,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        stdin=asyncio.subprocess.PIPE,
+        env=env,
+    )
+
+    if process.stdin:
+        process.stdin.write(prompt.encode("utf-8"))
+        await process.stdin.drain()
+        process.stdin.close()
+
+    assert process.stdout is not None
+    assert process.stderr is not None
+
+    stdout_task = asyncio.create_task(
+        stream_output_with_dialog(process.stdout, prefix="[STDOUT]", dialog=dialog)
+    )
+    stderr_task = asyncio.create_task(
+        stream_output_with_dialog(process.stderr, prefix="[STDERR]", dialog=None)
+    )
+
+    return_code = await process.wait()
+    _ = await asyncio.gather(stdout_task, stderr_task)
+
+    # Write completion if not already written by a result event
+    if not dialog.completed:
+        if return_code == 0:
+            dialog.write_complete("success")
+        else:
+            dialog.write_complete(
+                "error", message=f"Process exited with code {return_code}"
+            )
+
+    print(f"[claude] exited with code {return_code}")
+
+
+async def main():
+    """
+    Main function that configures and launches the Claude Code agent.
+    Assumes MCP server is already running on http://localhost:3000/mcp
+    """
+    claude_executable = os.environ["CLAUDE"]
+
+    # Register the MCP server with Claude Code
+    _ = subprocess.run(
+        [
+            claude_executable,
+            "mcp",
+            "add",
+            "ssebench",
+            "http://localhost:3000/mcp",
+            "--transport",
+            "http",
+            "--scope",
+            "user",
+        ]
+    )
+
+    # Initialize dialog writer
+    dialog = DialogWriter()
+
+    try:
+        # Launch Claude Code agent
+        process_task = asyncio.create_task(run_claude(dialog))
+        await process_task
+    except Exception as e:
+        dialog.write_complete("error", message=str(e))
+    finally:
+        dialog.close()
+
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("killed")

ssebench/_data/agents/claude-code/claude-code-sse/pyproject.toml

@@ -0,0 +1,13 @@
+[project]
+name = "claude-code-sse"
+version = "1.0.0"
+description = "Claude"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+  "fastmcp",
+  "ssebench-sdk",
+]
+
+[tool.uv.sources]
+ssebench-sdk = { workspace = true }

ssebench/_data/agents/claude-code/claude-code-sse/run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -euo pipefail
+
+# change cwd to current folder
+SCRIPT_PATH="$(realpath "${BASH_SOURCE[0]}")"
+SCRIPT_DIR="$(dirname "$SCRIPT_PATH")"
+cd "$SCRIPT_DIR" || exit 1
+
+# The run container reaches the LiteLLM proxy only: turn off Claude Code's
+# update checks, telemetry, error reports and plugin marketplace installs.
+export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1
+export CLAUDE_CODE_DISABLE_OFFICIAL_MARKETPLACE_AUTOINSTALL=1
+
+# Launch the agent (runs as model user via entrypoint)
+echo "[run.sh] Launching Claude Code agent..."
+# The image holds the environment, and the run container has no internet.
+exec uv run --offline --no-sync main.py

ssebench/_data/agents/codex/.dockerignore

@@ -0,0 +1,11 @@
+# Python virtual environments
+**/.venv
+**/__pycache__
+**/*.pyc
+
+# IDE
+**/.idea
+**/.vscode
+
+# Cache
+**/.cache

ssebench/_data/agents/codex/Dockerfile

@@ -0,0 +1,26 @@
+FROM ssebench-agent
+
+# The wrapper is a member of the uv workspace at the repository root, so /app
+# holds the workspace files it needs (from the `workspace` build context) in the
+# repository layout.
+RUN mkdir -p /app/agents/codex/codex-sse && chown -R model:model /app
+WORKDIR /app/agents/codex/codex-sse
+
+RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \
+    apt-get install -y nodejs
+
+ARG CODEX_VERSION=0.159.0
+RUN npm install -g "@openai/codex@${CODEX_VERSION}"
+
+COPY --from=workspace --chown=model:model pyproject.toml uv.lock /app/
+COPY --from=workspace --chown=model:model sdk/python/pyproject.toml sdk/python/README.md /app/sdk/python/
+COPY --from=workspace --chown=model:model sdk/python/sse/ /app/sdk/python/sse/
+COPY --chown=model:model codex-sse/ /app/agents/codex/codex-sse
+
+# Install the wrapper's environment, Python included, now: at run time the
+# container reaches the LiteLLM proxy only.
+USER model
+RUN HOME=/home/model uv sync --frozen --no-dev --package codex-sse
+USER root
+
+CMD ["./run.sh"]

ssebench/_data/agents/codex/agent.yaml

@@ -0,0 +1 @@
+name: codex