self-bench 0.3.5 → 0.3.8

package/.dockerignore

@@ -2,8 +2,18 @@
 .github
 .selfbench
 .venv
-dist
+.env
+.env.*
+*.key
+*.pem
 jobs
 node_modules
 tests
 *.md
+!src/skills/selfbench/SKILL.md
+infra
+*.tfstate*
+*.tfplan*
+*.tfvars*
+**/.terraform
+gha-creds-*.json

package/Dockerfile

@@ -4,22 +4,21 @@ WORKDIR /app
 ARG SELFBENCH_BUILD_COMMIT
 COPY package.json bun.lock ./
 RUN bun install --frozen-lockfile
-COPY biome.json tsconfig.json tsconfig.build.json ./
-COPY src ./src
-COPY review ./review
-COPY scripts ./scripts
-RUN bun run build
+COPY . .
+RUN if [ -f tsconfig.build.json ]; then bun run build; else test -f dist/api-main.js; fi
+RUN rm -rf node_modules && bun install --frozen-lockfile --production
 
 FROM docker:29.4.0-cli AS docker-cli
 
 FROM node:22-bookworm
 
-ARG HARBOR_VERSION=0.20.1.dev202608040148
+ARG HARBOR_VERSION=0.23.0
 RUN apt-get update \
   && apt-get install -y --no-install-recommends bash ca-certificates curl gh git jq ripgrep tar \
   && rm -rf /var/lib/apt/lists/* \
   && curl -LsSf https://astral.sh/uv/0.11.3/install.sh | env UV_INSTALL_DIR=/usr/local/bin UV_NO_MODIFY_PATH=1 sh \
-  && env UV_PYTHON_INSTALL_DIR=/opt/uv-python UV_TOOL_DIR=/opt/uv-tools UV_TOOL_BIN_DIR=/usr/local/bin uv tool install --python 3.12 "harbor[modal]==${HARBOR_VERSION}" \
+  && env UV_PYTHON_INSTALL_DIR=/opt/uv-python UV_TOOL_DIR=/opt/uv-tools UV_TOOL_BIN_DIR=/usr/local/bin uv tool install --python 3.12 --with-executables-from modal "harbor[e2b,daytona,modal,vercel]==${HARBOR_VERSION}" \
+  && modal --version \
   && chmod -R a+rX /opt/uv-python /opt/uv-tools
 COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
 COPY --from=docker-cli /usr/local/libexec/docker/cli-plugins /usr/local/libexec/docker/cli-plugins
@@ -30,8 +29,9 @@ WORKDIR /app
 RUN mkdir -p /var/lib/selfbench/artifacts && chown -R node:node /var/lib/selfbench
 COPY --from=build /app/node_modules ./node_modules
 COPY --from=build /app/dist ./dist
-COPY src/extensions ./src/extensions
-COPY src/skills ./src/skills
+COPY --from=build /app/drizzle ./drizzle
+COPY --from=build /app/src/extensions ./src/extensions
+COPY --from=build /app/src/skills ./src/skills
 COPY package.json ./package.json
 
 USER node

package/Dockerfile.sandbox

@@ -1,12 +1,13 @@
 FROM node:22-bookworm@sha256:0557ac14e0d45d02ed563067b82856ca5e7aa3437fa28d98d4350ea9c3d9494a
 
 ARG PI_VERSION=0.84.0
-ARG CODEX_VERSION=0.146.1
 ARG GH_VERSION=2.89.0
-ARG TARGETARCH
+# Docker BuildKit overrides this automatic platform arg; E2B's Dockerfile
+# template builder uses the amd64 default.
+ARG TARGETARCH=amd64
 
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends bash ca-certificates curl git jq ripgrep unzip xz-utils \
+  && apt-get install -y --no-install-recommends bash ca-certificates curl git jq ripgrep skopeo unzip xz-utils \
   && rm -rf /var/lib/apt/lists/*
 RUN case "${TARGETARCH}" in \
     amd64) GH_SHA256=d0422caade520530e76c1c558da47daebaa8e1203d6b7ff10ad7d6faba3490d8 ;; \
@@ -19,6 +20,6 @@ RUN case "${TARGETARCH}" in \
   && mv "/tmp/gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh" /usr/local/bin/gh \
   && rm -rf /tmp/gh.tar.gz "/tmp/gh_${GH_VERSION}_linux_${TARGETARCH}"
 RUN npm install --global --ignore-scripts "@earendil-works/pi-coding-agent@${PI_VERSION}"
-RUN npm install --global "@openai/codex@${CODEX_VERSION}"
+RUN mkdir -p /work && chmod 0777 /work
 
 WORKDIR /work

package/README.md

@@ -10,6 +10,8 @@
 
 It finds completed requests from local coding sessions and merged GitHub pull requests, then reconstructs each task from the commit before the change. For every accepted task, self-bench creates hidden tests and a reference solution, proves that the task fails without a solution and passes with the original implementation, and exports a native task for [Harbor](https://harborframework.com/), a runner for coding-agent evaluations.
 
+SelfBench is sandbox-agnostic: run generation locally with Docker or on Modal, Vercel Sandbox, or E2B. Generation and Harbor validation are configured independently, so Modal is optional.
+
 The result is a private `.tar.gz` benchmark that you can run against multiple models:
 
 ```text
@@ -24,22 +26,21 @@ gpt-5.6-luna vs gpt-5.6-terra vs gpt-5.6-sol
 
 ## Quickstart
 
-This path runs the self-bench API, worker, and [Temporal](https://temporal.io/) workflow state locally while using [Modal](https://modal.com/) for disposable task-generation and validation sandboxes.
+This path runs the self-bench API, worker, [Temporal](https://temporal.io/), and all generation and validation sandboxes locally with Docker. It does not require an account with a hosted sandbox provider.
 
 ### Prerequisites
 
 - [Bun](https://bun.sh/) 1.3.14 or newer
+- [uv](https://docs.astral.sh/uv/) for installing Harbor
 - Docker with Compose
-- [Modal](https://modal.com/) CLI, account, and token (`pip install modal`)
 - [`gh`](https://cli.github.com/), authenticated with read access to the repository
 - An OpenAI API key with access to `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna`
 - A Git checkout with a GitHub `origin` and completed work in its history
 
-Install self-bench and authenticate Modal and GitHub:
+Install self-bench and authenticate GitHub:
 
 ```bash
 bun add --global self-bench
-modal token new
 gh auth login
 ```
 
@@ -54,11 +55,11 @@ export SELFBENCH_API_TOKEN="$(openssl rand -hex 24)"
 ### 1. Start self-bench
 
 ```bash
-self-bench up --backend modal
+self-bench up --backend docker
 export SELFBENCH_API_URL=http://127.0.0.1:8080
 ```
 
-This starts Postgres, Temporal, the self-bench API, and a worker in Docker. The worker sends sandbox work to Modal; `SELFBENCH_API_URL` tells subsequent CLI commands where to reach the local API.
+This starts Postgres, Temporal, the self-bench API, and a worker. The worker creates disposable local Docker sandboxes; `SELFBENCH_API_URL` tells subsequent CLI commands where to reach the local API. From a git worktree or any checkout not named `self-bench`, `up` gives the stack its own Compose project and ports and prints the URL, so several checkouts run side by side (see [Operations](docs/operations.md#one-stack-per-checkout)).
 
 ### 2. Build a benchmark
 
@@ -75,12 +76,12 @@ Easy, medium, and hard candidates require at least 20, 50, and 100 changed imple
 
 The repository must be a Git checkout with a GitHub `origin`. self-bench pins its current `HEAD`, ignores uncommitted changes, and may take hours to author, validate, review, and export the accepted tasks. `--output` waits for completion and verifies the downloaded archive with SHA-256.
 
-### 3. Compare models with Harbor
+### 3. Evaluate with Harbor
 
 Install Harbor and extract the generated tasks:
 
 ```bash
-uv tool install --python 3.12 'harbor[modal]==0.20.1.dev202608040148'
+uv tool install --python 3.12 'harbor==0.23.0'
 
 mkdir -p ./self-bench-export ./self-bench-tasks
 tar -xzf ./self-bench-evals.tar.gz -C ./self-bench-export
@@ -91,7 +92,7 @@ for archive in ./self-bench-export/tasks/*.tar.gz; do
 done
 ```
 
-Run Harbor's Codex agent adapter once for all three models. Harbor evaluates every extracted task at high reasoning and keeps the model results in one job directory:
+Run Harbor's Codex agent adapter directly. Harbor evaluates every extracted task at high reasoning and keeps the model results in one job directory. Repeat with additional `--model` values when comparing models:
 
 ```bash
 harbor run \
@@ -102,15 +103,17 @@ harbor run \
   --model gpt-5.6-sol \
   --ak version=0.146.1 \
   --ak reasoning_effort=high \
-  --env modal \
+  --env docker \
   --jobs-dir ./harbor-jobs \
-  --n-concurrent 20 \
+  --n-concurrent 1 \
   --yes
 ```
 
 The evaluated agent receives the base repository and task instruction, but not the hidden tests or reference solution.
 
-See [Running self-bench evaluations](docs/evaluations.md) for running one task and using the optional resumable matrix helper.
+See [Running self-bench evaluations](docs/evaluations.md) for running tasks directly with Harbor.
+
+The signed-in site also supports repository-scoped solver runs with live output and completed results. See [Run tasks from the site](docs/solver-evaluations.md) for worker configuration and supported choices.
 
 ## Run management
 
@@ -131,34 +134,47 @@ self-bench down
 
 Named Docker volumes retain Temporal history and generated artifacts.
 
-## Other deployments
+## Choose a sandbox backend
 
-The quickstart is the recommended setup: a local stack with Modal sandboxes.
+The quickstart uses local Docker sandboxes so it works without a hosted sandbox account. For more concurrency or unattended runs, generation can use Modal, Vercel Sandbox, or E2B instead:
 
-- **Local stack + Docker sandboxes:** use `self-bench up --backend docker` when you want all execution on your machine.
-- **Local stack + Vercel Sandbox:** run `self-bench setup vercel`, then explicitly choose Docker or Modal for Harbor validation.
-- **Temporal Cloud + Modal:** use this for persistent unattended workers and large repositories.
+- **Docker:** `self-bench up --backend docker` keeps generation and validation on your machine.
+- **Modal:** authenticate with `modal token new`, then run `self-bench up --backend modal`.
+- **Vercel Sandbox:** run `self-bench setup vercel`; Harbor validation defaults to Vercel too.
+- **E2B:** build the pinned SelfBench runtime with `self-bench setup e2b --name NAME[:TAG]`; Harbor validation defaults to E2B too.
+- **Temporal Cloud + Modal/E2B:** use a persistent worker for unattended runs and large repositories.
 
-See [Operations and deployment](docs/operations.md) for backend configuration, credentials, persistence, object storage, and the complete Temporal Cloud deployment.
+See [Operations and deployment](docs/operations.md) for provider setup, credentials, persistence, object storage, and Temporal Cloud deployment.
 
-Generation sandboxes and Harbor validation are independent choices. Docker and Modal retain their matching defaults; Vercel must name a Harbor environment because Harbor does not currently provide a Vercel environment:
+Generation sandboxes and Harbor validation are independent choices. Every generation backend defaults to the matching Harbor environment, and `--harbor-environment` picks a different one. Daytona is available for Harbor only:
 
 ```bash
 # Matching defaults
 self-bench up --backend docker
 self-bench up --backend modal
 
-# Vercel generation with either supported Harbor environment
+# Vercel generation, with Vercel Harbor by default or any other environment
 self-bench setup vercel
+self-bench up --backend vercel                           # Vercel + Vercel
 self-bench up --backend vercel --harbor-environment docker
 self-bench up --backend vercel --harbor-environment modal
+self-bench up --backend vercel --harbor-environment e2b
+DAYTONA_API_KEY=... self-bench up --backend vercel --harbor-environment daytona
+
+# E2B generation uses a required prebuilt template; setup never installs at runtime
+export E2B_API_KEY=...
+self-bench setup e2b --name selfbench-runtime:v1
+export SELFBENCH_E2B_TEMPLATE=selfbench-runtime:v1
+self-bench up --backend e2b                              # E2B + E2B
+# self-bench up --backend e2b --harbor-environment docker
+# self-bench up --backend e2b --harbor-environment modal
 
 # Explicit cross-provider combinations are also supported
 self-bench up --backend docker --harbor-environment modal
 self-bench up --backend modal --harbor-environment docker
 ```
 
-Provider selection belongs to the worker, so all runs on one task queue use the same pairing. See [Vercel Sandbox setup](docs/operations.md#vercel-sandbox) for project authentication, publishing the runtime image, limits, cleanup, and troubleshooting.
+Provider selection belongs to the worker, so all runs on one task queue use the same pairing. See [Operations and deployment](docs/operations.md) for Vercel and [E2B setup](docs/operations.md#e2b) for template builds, credentials, plan limits, resources, cleanup, and unattended deployment.
 
 ## How tasks are validated
 
@@ -201,6 +217,7 @@ bun run dev:review
 
 - [Task construction and validation](docs/task-construction.md)
 - [Running evaluations](docs/evaluations.md)
+- [HTTP API reference](docs/api.md)
 - [Operations and deployment](docs/operations.md)
 
 ## License