evalsi-harness 0.1.0__tar.gz

evalsi_harness-0.1.0/.gitignore

@@ -0,0 +1,30 @@
+# Go
+/evalsid
+/bin/
+# `go build ./cmd/...` without -o drops a binary in the working directory
+/evalsi-guest
+/evalsi-operator
+/evalsi-collector
+
+# Python
+__pycache__/
+*.py[cod]
+.venv/
+*.egg-info/
+dist/
+build/
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+# Editors and OS
+.idea/
+.vscode/
+.DS_Store
+
+# Server data directories (examples use .evalsi and .evalsi-auth-demo)
+.evalsi*/
+examples/demo/terminal-bench-2/
+examples/demo/_local-*.yaml
+
+node_modules/

evalsi_harness-0.1.0/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.5
+Name: evalsi-harness
+Version: 0.1.0
+Summary: The light built-in agent harness for Evals.si: tool loop, MCP tools, sandboxes, user simulator, record and replay.
+Project-URL: Homepage, https://abhishek-rnjn.github.io/Evals.si/
+Project-URL: Documentation, https://abhishek-rnjn.github.io/Evals.si/
+Project-URL: Repository, https://github.com/Abhishek-Rnjn/Evals.si
+Project-URL: Issues, https://github.com/Abhishek-Rnjn/Evals.si/issues
+Project-URL: Changelog, https://github.com/Abhishek-Rnjn/Evals.si/blob/main/CHANGELOG.md
+License-Expression: Apache-2.0
+Requires-Python: >=3.11
+Requires-Dist: evalsi[server]==0.1.0
+Requires-Dist: httpx>=0.27
+Description-Content-Type: text/markdown
+
+# evalsi-harness
+
+The light built-in agent harness for [Evals.si](https://github.com/Abhishek-Rnjn/Evals.si).
+It drives an agent through a task in an Evals.si sandbox and grades the end state.
+
+It runs either of these:
+- the built-in tool-calling agent on any OpenAI-compatible or Anthropic model;
+- a bring-your-own agent over A2A, MCP, OpenAI Responses or HTTP, or as a CLI run inside the sandbox.
+
+```bash
+pip install evalsi-harness
+evalsi run -f agent-run.yaml
+```
+
+What it provides:
+- **Sandbox tools:** `bash`, `read_file` and `write_file`, plus an escalation request the harness policy decides on.
+- **Other tools:** MCP tools over streamable HTTP or stdio; mocked tools; deterministic fault injection.
+- **Budgets:** steps, tokens, spend and wall-clock time.
+- **Simulated user:** an LLM user for multi-turn tasks.
+- **Record and replay:** model calls and tool I/O are captured, replay can branch from step N, and sandbox commands are re-executed so the environment is rebuilt.
+- **Environment checkers:** exit code, JSON, JUnit or a Python function.
+- **OTel spans:** every step becomes a span.
+- **The agent's diff:** recorded for the `code-quality` evaluator when the workdir is a git repository.
+- **Benchmark importers:** `harbor://` (Terminal-Bench 2 and other Harbor datasets) and `terminal-bench://` (Terminal-Bench 1) task directories, with their Dockerfiles translated into environments.
+
+See [docs/guides/agent-runs.md](../../docs/guides/agent-runs.md).

evalsi_harness-0.1.0/README.md

@@ -0,0 +1,26 @@
+# evalsi-harness
+
+The light built-in agent harness for [Evals.si](https://github.com/Abhishek-Rnjn/Evals.si).
+It drives an agent through a task in an Evals.si sandbox and grades the end state.
+
+It runs either of these:
+- the built-in tool-calling agent on any OpenAI-compatible or Anthropic model;
+- a bring-your-own agent over A2A, MCP, OpenAI Responses or HTTP, or as a CLI run inside the sandbox.
+
+```bash
+pip install evalsi-harness
+evalsi run -f agent-run.yaml
+```
+
+What it provides:
+- **Sandbox tools:** `bash`, `read_file` and `write_file`, plus an escalation request the harness policy decides on.
+- **Other tools:** MCP tools over streamable HTTP or stdio; mocked tools; deterministic fault injection.
+- **Budgets:** steps, tokens, spend and wall-clock time.
+- **Simulated user:** an LLM user for multi-turn tasks.
+- **Record and replay:** model calls and tool I/O are captured, replay can branch from step N, and sandbox commands are re-executed so the environment is rebuilt.
+- **Environment checkers:** exit code, JSON, JUnit or a Python function.
+- **OTel spans:** every step becomes a span.
+- **The agent's diff:** recorded for the `code-quality` evaluator when the workdir is a git repository.
+- **Benchmark importers:** `harbor://` (Terminal-Bench 2 and other Harbor datasets) and `terminal-bench://` (Terminal-Bench 1) task directories, with their Dockerfiles translated into environments.
+
+See [docs/guides/agent-runs.md](../../docs/guides/agent-runs.md).

evalsi_harness-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "evalsi-harness"
+dynamic = ["version"]
+description = "The light built-in agent harness for Evals.si: tool loop, MCP tools, sandboxes, user simulator, record and replay."
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.11"
+dependencies = [
+    # The harness speaks evalsi's protocol: the same release only. The
+    # release workflow sets this with the version.
+    "evalsi[server]==0.1.0",
+    "httpx>=0.27",
+]
+
+[project.entry-points."evalsi.importers"]
+harbor = "evalsi_harness.benchmarks.harbor:load_harbor"
+terminal-bench = "evalsi_harness.benchmarks.harbor:load_terminal_bench"
+
+[project.urls]
+Homepage = "https://abhishek-rnjn.github.io/Evals.si/"
+Documentation = "https://abhishek-rnjn.github.io/Evals.si/"
+Repository = "https://github.com/Abhishek-Rnjn/Evals.si"
+Issues = "https://github.com/Abhishek-Rnjn/Evals.si/issues"
+Changelog = "https://github.com/Abhishek-Rnjn/Evals.si/blob/main/CHANGELOG.md"
+
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/evalsi_harness"]
+
+[tool.hatch.version]
+path = "src/evalsi_harness/_version.py"

evalsi_harness-0.1.0/src/evalsi_harness/__init__.py

@@ -0,0 +1,39 @@
+"""evalsi-harness: the light built-in agent harness for Evals.si (design §11).
+
+It drives an agent through a task in an Evals.si sandbox: the built-in
+tool-calling agent on any OpenAI-compatible or Anthropic model, or a
+bring-your-own agent (A2A, MCP, OpenAI Responses, HTTP, or a CLI run in the
+sandbox). It provides sandbox and MCP tools, budgets, a simulated user,
+mocked and faulty tools, record and replay, sandbox policy events and OTel
+spans, and grades the end state with the environment's checker.
+
+Most users never import it: ``evalsi run`` and the server's worker call
+:func:`run_task` for every record of an agent run.
+"""
+
+from evalsi_harness._version import __version__
+from evalsi_harness.events import FinalEvent, PolicyEvent, StepEvent
+from evalsi_harness.harness import BuiltinHarness, Harness, HarnessContext, HarnessManifest
+from evalsi_harness.runner import TaskOutcome, load_harness, run_task
+from evalsi_harness.task import EnvironmentConfig, Task, TaskError
+from evalsi_harness.tools import FunctionTool, Tool, ToolResult
+
+__all__ = [
+    "BuiltinHarness",
+    "EnvironmentConfig",
+    "FinalEvent",
+    "FunctionTool",
+    "Harness",
+    "HarnessContext",
+    "HarnessManifest",
+    "PolicyEvent",
+    "StepEvent",
+    "Task",
+    "TaskError",
+    "TaskOutcome",
+    "Tool",
+    "ToolResult",
+    "__version__",
+    "load_harness",
+    "run_task",
+]

evalsi_harness-0.1.0/src/evalsi_harness/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

evalsi_harness-0.1.0/src/evalsi_harness/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark task formats as Evals.si agent tasks."""

evalsi_harness-0.1.0/src/evalsi_harness/benchmarks/fixtures.py

@@ -0,0 +1,111 @@
+"""Tiny Harbor and Terminal-Bench 1 tasks, for tests and smoke runs: each
+asks for /app/hello.txt on alpine:3.22, copies a binary file in, and grades
+offline (no package installs), so they run in any sandbox that can pull the
+image.
+
+    python -m evalsi_harness.benchmarks.fixtures <dir>
+
+writes ``<dir>/harbor/hello`` and ``<dir>/terminal-bench/hello``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+INSTRUCTION = 'Create a file called /app/hello.txt with "Hello, world!" as the content.'
+BLOB = bytes(range(256)) * 4
+
+DOCKERFILE = """\
+FROM alpine:3.22
+ARG GREETING_FILE=greeting.txt
+ENV GREETING="Hello, world!"
+WORKDIR /app
+COPY blob.bin /opt/data/
+RUN printf '%s' "$GREETING" > /opt/data/${GREETING_FILE} \\
+    && echo ready > state.txt
+CMD ["sh"]
+"""
+
+# Checks the agent's file, and that the image setup (ENV, ARG, RUN in the
+# WORKDIR) and the binary copy came through intact.
+CHECK = """\
+test "$(cat /app/hello.txt)" = "$(cat /opt/data/greeting.txt)" && \\
+test "$(cat /app/state.txt)" = ready && \\
+cmp -s /opt/data/blob.bin /tests/blob.bin"""
+
+TASK_TOML = """\
+schema_version = "1.1"
+
+[task]
+name = "evalsi/hello"
+
+[metadata]
+difficulty = "easy"
+
+[verifier]
+timeout_sec = 60.0
+
+[agent]
+timeout_sec = 120.0
+
+[environment]
+network_mode = "no-network"
+"""
+
+HARBOR_TEST = f"""#!/bin/sh
+if {CHECK}; then
+  echo 1 > /logs/verifier/reward.txt
+else
+  echo 0 > /logs/verifier/reward.txt
+fi
+"""
+
+TASK_YAML = f"""\
+instruction: |-
+  {INSTRUCTION}
+difficulty: easy
+category: file-operations
+parser_name: pytest
+max_agent_timeout_sec: 120.0
+max_test_timeout_sec: 60.0
+"""
+
+# A pytest-shaped report without pytest, so the task needs no network.
+RUN_TESTS = f"""#!/bin/sh
+echo "============================= test session starts =============================="
+if {CHECK}; then hello=PASSED; else hello=FAILED; fi
+echo "=========================== short test summary info ============================"
+echo "PASSED tests/test_outputs.py::test_setup"
+echo "$hello tests/test_outputs.py::test_hello_file"
+"""
+
+
+def write(directory: str | Path) -> Path:
+    root = Path(directory)
+    h = root / "harbor" / "hello"
+    (h / "environment").mkdir(parents=True, exist_ok=True)
+    (h / "tests").mkdir(exist_ok=True)
+    (h / "solution").mkdir(exist_ok=True)
+    (h / "task.toml").write_text(TASK_TOML)
+    (h / "instruction.md").write_text(INSTRUCTION + "\n")
+    (h / "environment" / "Dockerfile").write_text(DOCKERFILE)
+    (h / "environment" / "blob.bin").write_bytes(BLOB)
+    (h / "tests" / "test.sh").write_text(HARBOR_TEST)
+    (h / "tests" / "blob.bin").write_bytes(BLOB)
+    (h / "solution" / "solve.sh").write_text("#!/bin/sh\nprintf 'Hello, world!' > /app/hello.txt\n")
+
+    t = root / "terminal-bench" / "hello"
+    (t / "tests").mkdir(parents=True, exist_ok=True)
+    (t / "task.yaml").write_text(TASK_YAML)
+    (t / "Dockerfile").write_text(DOCKERFILE)
+    (t / "blob.bin").write_bytes(BLOB)
+    (t / "run-tests.sh").write_text(RUN_TESTS)
+    (t / "tests" / "blob.bin").write_bytes(BLOB)
+    (t / "solution.sh").write_text("#!/bin/sh\nprintf 'Hello, world!' > /app/hello.txt\n")
+    return root
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import sys
+
+    write(sys.argv[1] if len(sys.argv) > 1 else "benchmark-fixtures")