meridian-eval 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meridian/__init__.py +15 -0
- meridian/_assets/agents/terminal_agent/__init__.py +0 -0
- meridian/_assets/agents/terminal_agent/agent.py +209 -0
- meridian/_assets/agents/terminal_agent/prompts/system.md +11 -0
- meridian/_assets/agents/terminal_agent/tools.py +97 -0
- meridian/_assets/agents/terminal_agent/transcript.py +38 -0
- meridian/_assets/proxy.Dockerfile +43 -0
- meridian/adapters/__init__.py +7 -0
- meridian/adapters/base.py +60 -0
- meridian/adapters/entrypoint.py +174 -0
- meridian/adapters/langgraph_adapter.py +79 -0
- meridian/adapters/subprocess_adapter.py +80 -0
- meridian/assets.py +69 -0
- meridian/cache/__init__.py +5 -0
- meridian/cache/keys.py +96 -0
- meridian/cli.py +980 -0
- meridian/config.py +199 -0
- meridian/docker_client.py +33 -0
- meridian/exit_codes.py +15 -0
- meridian/gate/__init__.py +1 -0
- meridian/gate/baseline.py +91 -0
- meridian/gate/comment.py +291 -0
- meridian/gate/decide.py +189 -0
- meridian/grading/__init__.py +6 -0
- meridian/grading/graders/__init__.py +1 -0
- meridian/grading/graders/base.py +78 -0
- meridian/grading/graders/filesystem.py +64 -0
- meridian/grading/graders/json_state.py +113 -0
- meridian/grading/graders/process.py +44 -0
- meridian/grading/graders/pytest_grader.py +70 -0
- meridian/grading/graders/script_grader.py +83 -0
- meridian/grading/graders/sqlite.py +49 -0
- meridian/grading/pipeline.py +98 -0
- meridian/hashing.py +68 -0
- meridian/importers/__init__.py +1 -0
- meridian/importers/harbor.py +568 -0
- meridian/manifest/__init__.py +1 -0
- meridian/manifest/build.py +146 -0
- meridian/manifest/replay.py +144 -0
- meridian/manifest/store.py +107 -0
- meridian/models/__init__.py +6 -0
- meridian/models/adapter.py +83 -0
- meridian/models/manifest.py +95 -0
- meridian/models/run.py +229 -0
- meridian/models/suite.py +77 -0
- meridian/models/task.py +328 -0
- meridian/report/__init__.py +1 -0
- meridian/report/html.py +225 -0
- meridian/report/table.py +85 -0
- meridian/runtime/__init__.py +1 -0
- meridian/runtime/isolation.py +141 -0
- meridian/runtime/orchestrator.py +338 -0
- meridian/runtime/payload.py +174 -0
- meridian/runtime/proxy/__init__.py +17 -0
- meridian/runtime/proxy/budget.py +234 -0
- meridian/runtime/proxy/cassette.py +236 -0
- meridian/runtime/proxy/openai_compat.py +191 -0
- meridian/runtime/proxy/retry.py +141 -0
- meridian/runtime/proxy/server.py +374 -0
- meridian/runtime/proxy/session.py +325 -0
- meridian/runtime/proxy/stub_provider.py +277 -0
- meridian/runtime/pytest_runner.py +244 -0
- meridian/runtime/scheduler.py +114 -0
- meridian/runtime/secrets.py +110 -0
- meridian/runtime/trial_runner.py +763 -0
- meridian/snapshots/__init__.py +1 -0
- meridian/snapshots/build.py +151 -0
- meridian/snapshots/registry.py +101 -0
- meridian/stats/__init__.py +1 -0
- meridian/stats/bootstrap.py +69 -0
- meridian/stats/intervals.py +180 -0
- meridian/stats/passk.py +53 -0
- meridian/stats/power.py +144 -0
- meridian/stats/significance.py +70 -0
- meridian/store/__init__.py +6 -0
- meridian/store/migrations/env.py +48 -0
- meridian/store/migrations/script.py.mako +22 -0
- meridian/store/migrations/versions/0001_initial_schema.py +46 -0
- meridian/store/repo.py +251 -0
- meridian/store/schema.sql +124 -0
- meridian/suites/__init__.py +1 -0
- meridian/suites/audit.py +137 -0
- meridian/suites/loader.py +174 -0
- meridian/suites/validate.py +115 -0
- meridian/version.py +9 -0
- meridian_eval-0.2.0.dist-info/METADATA +475 -0
- meridian_eval-0.2.0.dist-info/RECORD +91 -0
- meridian_eval-0.2.0.dist-info/WHEEL +4 -0
- meridian_eval-0.2.0.dist-info/entry_points.txt +2 -0
- meridian_eval-0.2.0.dist-info/licenses/LICENSE +21 -0
- meridian_eval-0.2.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +36 -0
meridian/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Meridian — an agent evaluation harness.
|
|
2
|
+
|
|
3
|
+
Four rules govern everything in this package:
|
|
4
|
+
|
|
5
|
+
1. Every trial starts from a clean, isolated environment.
|
|
6
|
+
2. Grade outcomes, not trajectories.
|
|
7
|
+
3. Report pass^k, not a mean score.
|
|
8
|
+
4. Every run is reproducible from its manifest.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from meridian.version import __version__
|
|
14
|
+
|
|
15
|
+
__all__ = ["__version__"]
|
|
File without changes
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""A general terminal agent: a LangGraph loop over one `bash` tool.
|
|
2
|
+
|
|
3
|
+
plan → act ⇄ tools → finish
|
|
4
|
+
|
|
5
|
+
Unlike the checkout fixture, nothing here knows the task. It reads the prompt,
|
|
6
|
+
works in the container's working directory through a shell, and stops when the
|
|
7
|
+
model stops calling tools. That makes it the system under test for imported
|
|
8
|
+
benchmark tasks (see `meridian import-harbor`).
|
|
9
|
+
|
|
10
|
+
It calls the model through the Meridian proxy, never a provider, with no credential
|
|
11
|
+
of its own.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import time
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Annotated, Any, TypedDict
|
|
21
|
+
|
|
22
|
+
import httpx
|
|
23
|
+
from langgraph.graph import END, StateGraph
|
|
24
|
+
|
|
25
|
+
from terminal_agent.tools import TOOL_SCHEMAS, format_result, run_bash
|
|
26
|
+
from terminal_agent.transcript import record
|
|
27
|
+
|
|
28
|
+
# The proxy substitutes the model the operator chose, so this name is only a label.
|
|
29
|
+
MODEL = os.environ.get("MERIDIAN_FIXTURE_MODEL", "meridian-model")
|
|
30
|
+
MAX_TURNS = 30
|
|
31
|
+
REQUEST_TIMEOUT_SECONDS = 180.0
|
|
32
|
+
# The trial token budget is a ceiling on the whole trial, not on one reply.
|
|
33
|
+
MAX_TOKENS_PER_REPLY = 4096
|
|
34
|
+
|
|
35
|
+
PROMPTS = Path(__file__).resolve().parent / "prompts"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _append(existing: list[Any], incoming: list[Any]) -> list[Any]:
|
|
39
|
+
return [*existing, *incoming]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class AgentState(TypedDict, total=False):
|
|
43
|
+
context: dict[str, Any]
|
|
44
|
+
messages: Annotated[list[dict[str, Any]], _append]
|
|
45
|
+
system: str
|
|
46
|
+
turns: int
|
|
47
|
+
tool_calls: Annotated[list[dict[str, Any]], _append]
|
|
48
|
+
input_tokens: int
|
|
49
|
+
output_tokens: int
|
|
50
|
+
error: str | None
|
|
51
|
+
completed: bool
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def plan(state: AgentState) -> AgentState:
|
|
55
|
+
context = state["context"]
|
|
56
|
+
return {
|
|
57
|
+
"system": (PROMPTS / "system.md").read_text(encoding="utf-8"),
|
|
58
|
+
"messages": [{"role": "user", "content": context["prompt"]}],
|
|
59
|
+
"turns": 0,
|
|
60
|
+
"input_tokens": 0,
|
|
61
|
+
"output_tokens": 0,
|
|
62
|
+
"tool_calls": [],
|
|
63
|
+
"completed": False,
|
|
64
|
+
"error": None,
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def act(state: AgentState) -> AgentState:
|
|
69
|
+
"""One model turn, through the proxy."""
|
|
70
|
+
context = state["context"]
|
|
71
|
+
body = {
|
|
72
|
+
"model": MODEL,
|
|
73
|
+
"max_tokens": min(
|
|
74
|
+
int(context.get("max_tokens", MAX_TOKENS_PER_REPLY)), MAX_TOKENS_PER_REPLY
|
|
75
|
+
),
|
|
76
|
+
"system": state.get("system", ""),
|
|
77
|
+
"messages": state.get("messages", []),
|
|
78
|
+
"tools": TOOL_SCHEMAS,
|
|
79
|
+
}
|
|
80
|
+
headers = {
|
|
81
|
+
"x-meridian-task": str(context.get("task_slug", "")),
|
|
82
|
+
"x-meridian-trial": str(context.get("trial_index", 0)),
|
|
83
|
+
"content-type": "application/json",
|
|
84
|
+
}
|
|
85
|
+
url = f"{context['model_base_url']}/v1/messages"
|
|
86
|
+
workdir = str(context.get("workdir", "."))
|
|
87
|
+
asked = time.monotonic()
|
|
88
|
+
record(workdir, "model_request", turn=state.get("turns", 0) + 1)
|
|
89
|
+
try:
|
|
90
|
+
response = httpx.post(url, json=body, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)
|
|
91
|
+
except httpx.HTTPError as exc:
|
|
92
|
+
record(workdir, "model_error", error=f"{type(exc).__name__}: {exc}")
|
|
93
|
+
return {"error": f"model call failed: {exc}", "completed": False}
|
|
94
|
+
if response.status_code != 200:
|
|
95
|
+
record(workdir, "model_error", status=response.status_code, body=response.text)
|
|
96
|
+
return {
|
|
97
|
+
"error": f"model returned {response.status_code}: {response.text[:400]}",
|
|
98
|
+
"completed": False,
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
payload = response.json()
|
|
102
|
+
usage = payload.get("usage") or {}
|
|
103
|
+
reply = payload.get("content") or []
|
|
104
|
+
record(
|
|
105
|
+
workdir,
|
|
106
|
+
"model_reply",
|
|
107
|
+
seconds=round(time.monotonic() - asked, 1),
|
|
108
|
+
stop=payload.get("stop_reason"),
|
|
109
|
+
text=" ".join(
|
|
110
|
+
b.get("text", "") for b in reply if isinstance(b, dict) and b.get("type") == "text"
|
|
111
|
+
),
|
|
112
|
+
commands=[
|
|
113
|
+
(b.get("input") or {}).get("command")
|
|
114
|
+
for b in reply
|
|
115
|
+
if isinstance(b, dict) and b.get("type") == "tool_use"
|
|
116
|
+
],
|
|
117
|
+
)
|
|
118
|
+
return {
|
|
119
|
+
"messages": [{"role": "assistant", "content": payload.get("content") or []}],
|
|
120
|
+
"turns": state.get("turns", 0) + 1,
|
|
121
|
+
"input_tokens": state.get("input_tokens", 0) + int(usage.get("input_tokens", 0)),
|
|
122
|
+
"output_tokens": state.get("output_tokens", 0) + int(usage.get("output_tokens", 0)),
|
|
123
|
+
"completed": payload.get("stop_reason") != "tool_use",
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def tools(state: AgentState) -> AgentState:
|
|
128
|
+
"""Run every tool call the model made, in order."""
|
|
129
|
+
workdir = str(state["context"]["workdir"])
|
|
130
|
+
results: list[dict[str, Any]] = []
|
|
131
|
+
recorded: list[dict[str, Any]] = []
|
|
132
|
+
for block in state["messages"][-1].get("content", []):
|
|
133
|
+
if not isinstance(block, dict) or block.get("type") != "tool_use":
|
|
134
|
+
continue
|
|
135
|
+
arguments = dict(block.get("input") or {})
|
|
136
|
+
if block.get("name") != "bash" or not isinstance(arguments.get("command"), str):
|
|
137
|
+
text, ok = f"[unknown tool or bad arguments: {json.dumps(arguments)[:200]}]", False
|
|
138
|
+
else:
|
|
139
|
+
began = time.monotonic()
|
|
140
|
+
result = run_bash(
|
|
141
|
+
arguments["command"],
|
|
142
|
+
workdir=workdir,
|
|
143
|
+
timeout_seconds=arguments.get("timeout_seconds"),
|
|
144
|
+
)
|
|
145
|
+
text, ok = format_result(result), result["error"] is None
|
|
146
|
+
record(
|
|
147
|
+
workdir,
|
|
148
|
+
"command",
|
|
149
|
+
command=arguments["command"],
|
|
150
|
+
exit_code=result["exit_code"],
|
|
151
|
+
error=result["error"],
|
|
152
|
+
seconds=round(time.monotonic() - began, 1),
|
|
153
|
+
output=result["output"],
|
|
154
|
+
)
|
|
155
|
+
recorded.append({"name": str(block.get("name")), "ok": ok})
|
|
156
|
+
results.append(
|
|
157
|
+
{
|
|
158
|
+
"type": "tool_result",
|
|
159
|
+
"tool_use_id": block.get("id"),
|
|
160
|
+
"content": text,
|
|
161
|
+
"is_error": not ok,
|
|
162
|
+
}
|
|
163
|
+
)
|
|
164
|
+
return {"messages": [{"role": "user", "content": results}], "tool_calls": recorded}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def route(state: AgentState) -> str:
|
|
168
|
+
if state.get("error") or state.get("completed"):
|
|
169
|
+
return "finish"
|
|
170
|
+
if state.get("turns", 0) >= MAX_TURNS:
|
|
171
|
+
return "out_of_turns"
|
|
172
|
+
last = state["messages"][-1]
|
|
173
|
+
wants_tools = any(
|
|
174
|
+
isinstance(block, dict) and block.get("type") == "tool_use"
|
|
175
|
+
for block in last.get("content", [])
|
|
176
|
+
)
|
|
177
|
+
return "tools" if wants_tools else "finish"
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def out_of_turns(state: AgentState) -> AgentState:
|
|
181
|
+
# An agent failure, not a harness one: it had its turns and did not finish.
|
|
182
|
+
return {"error": f"gave up after {MAX_TURNS} turns", "completed": False}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def finish(state: AgentState) -> AgentState:
|
|
186
|
+
return {"completed": state.get("error") is None}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def build_graph() -> Any:
|
|
190
|
+
builder: StateGraph[AgentState] = StateGraph(AgentState)
|
|
191
|
+
builder.add_node("plan", plan)
|
|
192
|
+
builder.add_node("act", act)
|
|
193
|
+
builder.add_node("tools", tools)
|
|
194
|
+
builder.add_node("out_of_turns", out_of_turns)
|
|
195
|
+
builder.add_node("finish", finish)
|
|
196
|
+
builder.set_entry_point("plan")
|
|
197
|
+
builder.add_edge("plan", "act")
|
|
198
|
+
builder.add_conditional_edges(
|
|
199
|
+
"act",
|
|
200
|
+
route,
|
|
201
|
+
{"tools": "tools", "finish": "finish", "out_of_turns": "out_of_turns"},
|
|
202
|
+
)
|
|
203
|
+
builder.add_edge("tools", "act")
|
|
204
|
+
builder.add_edge("out_of_turns", "finish")
|
|
205
|
+
builder.add_edge("finish", END)
|
|
206
|
+
return builder.compile()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
graph = build_graph()
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
You are an autonomous software engineer working in a Linux container. You have one tool, `bash`, which runs a shell command in your working directory and returns its output and exit code.
|
|
2
|
+
|
|
3
|
+
Work toward the task you are given:
|
|
4
|
+
|
|
5
|
+
- Look before you change things: list files and read what is relevant.
|
|
6
|
+
- Make the change, then check your own work by running it. Do not assume it worked.
|
|
7
|
+
- Commands have a time limit and large outputs are truncated, so prefer targeted commands.
|
|
8
|
+
- You have no internet access and no root privileges. Use only what is installed.
|
|
9
|
+
- When the task is complete, stop calling tools and reply with a one-line summary.
|
|
10
|
+
|
|
11
|
+
Your result is judged only by the final state of the container, so make sure the files and outputs the task asks for exist and are correct before you stop.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""The terminal agent's one tool: run a shell command.
|
|
2
|
+
|
|
3
|
+
Standard library only, so it can be tested on the host without LangGraph.
|
|
4
|
+
|
|
5
|
+
The command runs inside the trial container, which is already the sandbox: a
|
|
6
|
+
read-only root filesystem, an unprivileged user, no capabilities and no network.
|
|
7
|
+
This module adds only what any shell tool needs -- a time limit and bounded output.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import subprocess
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
MAX_OUTPUT_CHARS = 8000
|
|
17
|
+
DEFAULT_TIMEOUT_SECONDS = 60
|
|
18
|
+
MAX_TIMEOUT_SECONDS = 300
|
|
19
|
+
|
|
20
|
+
TOOL_SCHEMAS: list[dict[str, Any]] = [
|
|
21
|
+
{
|
|
22
|
+
"name": "bash",
|
|
23
|
+
"description": (
|
|
24
|
+
"Run a shell command in the working directory. Returns the combined "
|
|
25
|
+
"stdout/stderr and the exit code. Commands that run longer than the "
|
|
26
|
+
"timeout are killed."
|
|
27
|
+
),
|
|
28
|
+
"input_schema": {
|
|
29
|
+
"type": "object",
|
|
30
|
+
"properties": {
|
|
31
|
+
"command": {"type": "string", "description": "The command to run."},
|
|
32
|
+
"timeout_seconds": {
|
|
33
|
+
"type": "integer",
|
|
34
|
+
"description": f"Time limit, at most {MAX_TIMEOUT_SECONDS} (default "
|
|
35
|
+
f"{DEFAULT_TIMEOUT_SECONDS}).",
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
"required": ["command"],
|
|
39
|
+
},
|
|
40
|
+
}
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def truncate(text: str, limit: int = MAX_OUTPUT_CHARS) -> str:
|
|
45
|
+
"""Keep the start and the end of long output; the middle is rarely the point."""
|
|
46
|
+
if len(text) <= limit:
|
|
47
|
+
return text
|
|
48
|
+
head = limit // 2
|
|
49
|
+
tail = limit - head
|
|
50
|
+
omitted = len(text) - limit
|
|
51
|
+
return f"{text[:head]}\n... [{omitted} characters omitted] ...\n{text[-tail:]}"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def run_bash(command: str, *, workdir: str, timeout_seconds: int | None = None) -> dict[str, Any]:
|
|
55
|
+
"""Run `command` with bash in `workdir`. Never raises; failures are results."""
|
|
56
|
+
timeout = DEFAULT_TIMEOUT_SECONDS if timeout_seconds is None else timeout_seconds
|
|
57
|
+
timeout = max(1, min(int(timeout), MAX_TIMEOUT_SECONDS))
|
|
58
|
+
env = {
|
|
59
|
+
"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
|
|
60
|
+
"HOME": "/tmp",
|
|
61
|
+
"LANG": "C.UTF-8",
|
|
62
|
+
"TERM": "dumb",
|
|
63
|
+
}
|
|
64
|
+
try:
|
|
65
|
+
completed = subprocess.run(
|
|
66
|
+
["bash", "-c", command],
|
|
67
|
+
cwd=workdir,
|
|
68
|
+
env=env,
|
|
69
|
+
capture_output=True,
|
|
70
|
+
timeout=timeout,
|
|
71
|
+
stdin=subprocess.DEVNULL,
|
|
72
|
+
check=False,
|
|
73
|
+
)
|
|
74
|
+
except subprocess.TimeoutExpired as exc:
|
|
75
|
+
partial = (exc.stdout or b"") + (exc.stderr or b"")
|
|
76
|
+
output = partial.decode("utf-8", errors="replace")
|
|
77
|
+
return {
|
|
78
|
+
"exit_code": None,
|
|
79
|
+
"output": truncate(output),
|
|
80
|
+
"error": f"timed out after {timeout}s",
|
|
81
|
+
}
|
|
82
|
+
except OSError as exc:
|
|
83
|
+
return {"exit_code": None, "output": "", "error": f"{type(exc).__name__}: {exc}"}
|
|
84
|
+
output = (completed.stdout + completed.stderr).decode("utf-8", errors="replace")
|
|
85
|
+
return {"exit_code": completed.returncode, "output": truncate(output), "error": None}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def format_result(result: dict[str, Any]) -> str:
|
|
89
|
+
"""The text the model sees for one command."""
|
|
90
|
+
lines = []
|
|
91
|
+
if result["error"]:
|
|
92
|
+
lines.append(f"[{result['error']}]")
|
|
93
|
+
elif result["exit_code"] != 0:
|
|
94
|
+
lines.append(f"[exit code {result['exit_code']}]")
|
|
95
|
+
if result["output"]:
|
|
96
|
+
lines.append(result["output"])
|
|
97
|
+
return "\n".join(lines) if lines else "[no output]"
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""An append-only transcript of what the agent did, written as it goes.
|
|
2
|
+
|
|
3
|
+
Written line by line and flushed immediately, so that when the harness kills the
|
|
4
|
+
agent at its deadline the record of what it was doing survives. The harness reads
|
|
5
|
+
the tail of this file into a timeout verdict.
|
|
6
|
+
|
|
7
|
+
Standard library only.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import time
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
TRANSCRIPT_RELATIVE = ".meridian/transcript.jsonl"
|
|
18
|
+
FIELD_LIMIT = 400
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _clip(value: Any) -> Any:
|
|
22
|
+
if isinstance(value, str) and len(value) > FIELD_LIMIT:
|
|
23
|
+
return value[:FIELD_LIMIT] + f"... [{len(value) - FIELD_LIMIT} more]"
|
|
24
|
+
return value
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def record(workdir: str, event: str, **fields: Any) -> None:
|
|
28
|
+
"""Append one event. Never raises: a transcript must not be able to fail a trial."""
|
|
29
|
+
line = {"t": round(time.time(), 2), "event": event}
|
|
30
|
+
line.update({key: _clip(value) for key, value in fields.items()})
|
|
31
|
+
try:
|
|
32
|
+
path = Path(workdir) / TRANSCRIPT_RELATIVE
|
|
33
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
34
|
+
with path.open("a", encoding="utf-8") as handle:
|
|
35
|
+
handle.write(json.dumps(line, sort_keys=True) + "\n")
|
|
36
|
+
handle.flush()
|
|
37
|
+
except OSError:
|
|
38
|
+
pass
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# The model proxy image.
|
|
2
|
+
#
|
|
3
|
+
# Built with the repository root as context because it ships a slice of the
|
|
4
|
+
# harness. Deliberately minimal: this container holds the only provider
|
|
5
|
+
# credential in the system, so the less it contains the better.
|
|
6
|
+
FROM python:3.13-slim
|
|
7
|
+
|
|
8
|
+
ENV PYTHONDONTWRITEBYTECODE=1 \
|
|
9
|
+
PYTHONUNBUFFERED=1 \
|
|
10
|
+
PIP_NO_CACHE_DIR=1 \
|
|
11
|
+
PIP_DISABLE_PIP_VERSION_CHECK=1
|
|
12
|
+
|
|
13
|
+
RUN pip install --no-cache-dir \
|
|
14
|
+
"starlette==0.45.3" \
|
|
15
|
+
"uvicorn==0.34.0" \
|
|
16
|
+
"httpx==0.28.1" \
|
|
17
|
+
"pydantic==2.10.6" \
|
|
18
|
+
"pyyaml==6.0.2"
|
|
19
|
+
|
|
20
|
+
COPY src/meridian/__init__.py /opt/meridian/meridian/__init__.py
|
|
21
|
+
COPY src/meridian/version.py /opt/meridian/meridian/version.py
|
|
22
|
+
COPY src/meridian/hashing.py /opt/meridian/meridian/hashing.py
|
|
23
|
+
COPY src/meridian/config.py /opt/meridian/meridian/config.py
|
|
24
|
+
# cassette.py validates tape names against the task schema's slug alphabet, so
|
|
25
|
+
# the proxy needs that one model module (pydantic only, no further imports).
|
|
26
|
+
COPY src/meridian/models/__init__.py /opt/meridian/meridian/models/__init__.py
|
|
27
|
+
COPY src/meridian/models/task.py /opt/meridian/meridian/models/task.py
|
|
28
|
+
COPY src/meridian/runtime/__init__.py /opt/meridian/meridian/runtime/__init__.py
|
|
29
|
+
COPY src/meridian/runtime/secrets.py /opt/meridian/meridian/runtime/secrets.py
|
|
30
|
+
COPY src/meridian/runtime/proxy/ /opt/meridian/meridian/runtime/proxy/
|
|
31
|
+
|
|
32
|
+
ENV PYTHONPATH=/opt/meridian \
|
|
33
|
+
MERIDIAN_CASSETTE_DIR=/cassettes \
|
|
34
|
+
MERIDIAN_PROXY_PORT=8080
|
|
35
|
+
|
|
36
|
+
# Named `meridian`, not `proxy`: Debian base images already ship a `proxy` user
|
|
37
|
+
# and groupadd exits 9 on the collision.
|
|
38
|
+
RUN groupadd --gid 10002 meridian && useradd --uid 10002 --gid 10002 --no-create-home meridian \
|
|
39
|
+
&& mkdir -p /cassettes && chown 10002:10002 /cassettes
|
|
40
|
+
USER 10002:10002
|
|
41
|
+
|
|
42
|
+
EXPOSE 8080
|
|
43
|
+
ENTRYPOINT ["python3", "-m", "meridian.runtime.proxy.server"]
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Adapters run **inside** the trial container, never in the harness process.
|
|
2
|
+
|
|
3
|
+
The harness never imports the system under test — it ships this package into the
|
|
4
|
+
container and lets the entrypoint resolve the adapter spec there. That is what
|
|
5
|
+
makes "framework-agnostic" a structural property rather than a claim: the harness
|
|
6
|
+
has no way to depend on a framework it cannot import.
|
|
7
|
+
"""
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Adapter spec strings: `kind:module:attribute`.
|
|
2
|
+
|
|
3
|
+
`langgraph:checkout_agent.agent:graph` says *which adapter* to use, *which
|
|
4
|
+
module* holds the system under test, and *which attribute* of it to invoke. The
|
|
5
|
+
harness only ever parses this; it resolves it inside the container.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import importlib
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
KNOWN_KINDS = ("langgraph", "subprocess")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class AdapterSpecError(ValueError):
|
|
18
|
+
"""A malformed or unknown adapter spec. A configuration error, not a verdict."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class AdapterSpec:
|
|
23
|
+
"""A parsed `kind:module:attribute` triple."""
|
|
24
|
+
|
|
25
|
+
kind: str
|
|
26
|
+
module: str
|
|
27
|
+
attribute: str
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def parse(cls, raw: str) -> AdapterSpec:
|
|
31
|
+
parts = raw.split(":")
|
|
32
|
+
if len(parts) != 3 or not all(part.strip() for part in parts):
|
|
33
|
+
raise AdapterSpecError(
|
|
34
|
+
f"adapter spec {raw!r} must be 'kind:module:attribute', "
|
|
35
|
+
f"e.g. 'langgraph:checkout_agent.agent:graph'"
|
|
36
|
+
)
|
|
37
|
+
kind, module, attribute = (part.strip() for part in parts)
|
|
38
|
+
if kind not in KNOWN_KINDS:
|
|
39
|
+
raise AdapterSpecError(
|
|
40
|
+
f"unknown adapter kind {kind!r}; Meridian ships {', '.join(KNOWN_KINDS)}"
|
|
41
|
+
)
|
|
42
|
+
return cls(kind=kind, module=module, attribute=attribute)
|
|
43
|
+
|
|
44
|
+
def __str__(self) -> str:
|
|
45
|
+
return f"{self.kind}:{self.module}:{self.attribute}"
|
|
46
|
+
|
|
47
|
+
def resolve_target(self) -> Any:
|
|
48
|
+
"""Import the module and return the attribute. Container-side only."""
|
|
49
|
+
try:
|
|
50
|
+
module = importlib.import_module(self.module)
|
|
51
|
+
except ImportError as exc:
|
|
52
|
+
raise AdapterSpecError(
|
|
53
|
+
f"cannot import {self.module!r} for adapter spec {self}: {exc}"
|
|
54
|
+
) from exc
|
|
55
|
+
try:
|
|
56
|
+
return getattr(module, self.attribute)
|
|
57
|
+
except AttributeError as exc:
|
|
58
|
+
raise AdapterSpecError(
|
|
59
|
+
f"module {self.module!r} has no attribute {self.attribute!r}"
|
|
60
|
+
) from exc
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""The in-container entrypoint. PID 1's child; the harness never runs this.
|
|
2
|
+
|
|
3
|
+
Reads a serialized `TrialContext`, resolves the adapter spec, runs the agent,
|
|
4
|
+
and writes `result.json` and `transcript.json` next to the payload. It exits 0
|
|
5
|
+
whenever it *ran the agent to a conclusion* — including a conclusion the agent
|
|
6
|
+
did not like — because the trial's verdict is decided outside, by grading the
|
|
7
|
+
final state. A non-zero exit here means the entrypoint itself could not run,
|
|
8
|
+
which is a harness error.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import json
|
|
15
|
+
import sys
|
|
16
|
+
import traceback
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from meridian.adapters.base import AdapterSpec, AdapterSpecError
|
|
21
|
+
from meridian.models.adapter import AdapterResult, TrialContext
|
|
22
|
+
|
|
23
|
+
EXIT_OK = 0
|
|
24
|
+
EXIT_ENTRYPOINT_FAILED = 70
|
|
25
|
+
|
|
26
|
+
# Written when the entrypoint itself could not run. The harness reads it so the
|
|
27
|
+
# trial's detail says *what* was misconfigured rather than only that something
|
|
28
|
+
# was.
|
|
29
|
+
HARNESS_ERROR_FILE = "harness_error.txt"
|
|
30
|
+
|
|
31
|
+
# Short: this only ever runs after the agent has already failed, and the answer
|
|
32
|
+
# is "is anything listening", not "is it fast".
|
|
33
|
+
PROXY_HEALTH_TIMEOUT_SECONDS = 5.0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def build_adapter(spec: AdapterSpec) -> Any:
|
|
37
|
+
"""Pick the adapter implementation for a spec kind.
|
|
38
|
+
|
|
39
|
+
Imported lazily and per-kind: the LangGraph adapter imports LangGraph, and a
|
|
40
|
+
subprocess trial should not pay for — or fail on — a framework it never uses.
|
|
41
|
+
"""
|
|
42
|
+
if spec.kind == "subprocess":
|
|
43
|
+
from meridian.adapters.subprocess_adapter import SubprocessAdapter
|
|
44
|
+
|
|
45
|
+
return SubprocessAdapter(spec)
|
|
46
|
+
if spec.kind == "langgraph":
|
|
47
|
+
from meridian.adapters.langgraph_adapter import LangGraphAdapter
|
|
48
|
+
|
|
49
|
+
return LangGraphAdapter(spec)
|
|
50
|
+
raise ValueError(f"no adapter implementation for kind {spec.kind!r}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def record_harness_error(payload_dir: Path, message: str) -> int:
|
|
54
|
+
"""Report a Meridian misconfiguration, not an agent failure."""
|
|
55
|
+
(payload_dir / HARNESS_ERROR_FILE).write_text(message, encoding="utf-8")
|
|
56
|
+
(payload_dir / "traceback.txt").write_text(traceback.format_exc(), encoding="utf-8")
|
|
57
|
+
return EXIT_ENTRYPOINT_FAILED
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def proxy_is_unreachable(base_url: str) -> bool:
|
|
61
|
+
"""Ask the proxy whether it is alive. Used only to explain a failure.
|
|
62
|
+
|
|
63
|
+
This is the seam that separates "the agent failed" from "the agent was never
|
|
64
|
+
given a model". It asks the proxy directly rather than matching on the text of
|
|
65
|
+
the agent's error, because the wording of that error belongs to whatever
|
|
66
|
+
framework raised it and will not survive an upgrade — and a classification
|
|
67
|
+
rule built on someone else's error string fails silently and in the expensive
|
|
68
|
+
direction.
|
|
69
|
+
|
|
70
|
+
Unreachable means *refused or unanswered*, not "returned an error". A proxy
|
|
71
|
+
that answers with a budget refusal or a cassette miss is working exactly as
|
|
72
|
+
designed, and those are genuine agent-visible failures.
|
|
73
|
+
"""
|
|
74
|
+
if not base_url:
|
|
75
|
+
return False
|
|
76
|
+
try:
|
|
77
|
+
import httpx
|
|
78
|
+
except ImportError:
|
|
79
|
+
# The trial image supplies httpx, not the payload. An environment without
|
|
80
|
+
# it must not turn every agent failure into an unanswerable question:
|
|
81
|
+
# raising here would escape `run()`, exit EXIT_ENTRYPOINT_FAILED, and be
|
|
82
|
+
# classified as a harness error — which is *retried*. Every genuine agent
|
|
83
|
+
# `fail` on such an image would then get a second attempt, which is the
|
|
84
|
+
# one thing the retry policy exists to forbid.
|
|
85
|
+
return False
|
|
86
|
+
|
|
87
|
+
try:
|
|
88
|
+
response = httpx.get(
|
|
89
|
+
f"{base_url.rstrip('/')}/healthz", timeout=PROXY_HEALTH_TIMEOUT_SECONDS
|
|
90
|
+
)
|
|
91
|
+
except Exception:
|
|
92
|
+
# Deliberately broad. `httpx.InvalidURL` is not an `HTTPError`, so a
|
|
93
|
+
# malformed base URL would otherwise crash the entrypoint rather than
|
|
94
|
+
# answer the question. Anything that prevents an answer means the proxy
|
|
95
|
+
# could not be confirmed alive, which is what the caller asked.
|
|
96
|
+
return True
|
|
97
|
+
return response.status_code >= 500
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
async def run(context_path: Path) -> int:
|
|
101
|
+
payload_dir = context_path.parent
|
|
102
|
+
raw = json.loads(context_path.read_text(encoding="utf-8"))
|
|
103
|
+
|
|
104
|
+
# Everything up to and including resolving the adapter is configuration. A
|
|
105
|
+
# failure here says the harness was set up wrong, and calling it an agent
|
|
106
|
+
# failure would put a configuration mistake into the score.
|
|
107
|
+
try:
|
|
108
|
+
adapter_spec = AdapterSpec.parse(raw.pop("adapter_spec"))
|
|
109
|
+
ctx = TrialContext.model_validate(raw)
|
|
110
|
+
adapter = build_adapter(adapter_spec)
|
|
111
|
+
adapter_spec.resolve_target()
|
|
112
|
+
except (AdapterSpecError, ValueError) as exc:
|
|
113
|
+
return record_harness_error(payload_dir, f"{type(exc).__name__}: {exc}")
|
|
114
|
+
|
|
115
|
+
try:
|
|
116
|
+
result = await adapter.invoke(ctx)
|
|
117
|
+
except AdapterSpecError as exc:
|
|
118
|
+
return record_harness_error(payload_dir, f"{type(exc).__name__}: {exc}")
|
|
119
|
+
except Exception as exc: # the agent blew up: an agent failure, not a harness one
|
|
120
|
+
result = AdapterResult(
|
|
121
|
+
completed=False,
|
|
122
|
+
error=f"{type(exc).__name__}: {exc}",
|
|
123
|
+
transcript_path=str(payload_dir / "transcript.json"),
|
|
124
|
+
)
|
|
125
|
+
(payload_dir / "traceback.txt").write_text(traceback.format_exc(), encoding="utf-8")
|
|
126
|
+
finally:
|
|
127
|
+
await adapter.teardown()
|
|
128
|
+
|
|
129
|
+
# An agent that could not reach the model has not been evaluated, so its
|
|
130
|
+
# failure is Meridian's and not the agent's. Checked here rather than at the
|
|
131
|
+
# exception site because an agent may *return* the failure instead of raising
|
|
132
|
+
# it -- the checkout fixture catches httpx errors and reports them as an
|
|
133
|
+
# AdapterResult -- and both paths must classify the same way.
|
|
134
|
+
#
|
|
135
|
+
# Getting this wrong is expensive in the direction that wastes a person's
|
|
136
|
+
# afternoon: a dead proxy fails every trial of every task, the gate reports
|
|
137
|
+
# the largest regression it has ever seen, and the developer reverts a change
|
|
138
|
+
# that was never the cause.
|
|
139
|
+
if result.error and proxy_is_unreachable(ctx.model_base_url):
|
|
140
|
+
return record_harness_error(
|
|
141
|
+
payload_dir,
|
|
142
|
+
f"the model proxy at {ctx.model_base_url} did not answer its health "
|
|
143
|
+
f"check, so the agent was never given a model. The agent reported: "
|
|
144
|
+
f"{result.error}",
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
(payload_dir / "result.json").write_text(
|
|
148
|
+
result.model_dump_json(indent=2),
|
|
149
|
+
encoding="utf-8",
|
|
150
|
+
)
|
|
151
|
+
transcript = payload_dir / "transcript.json"
|
|
152
|
+
if not transcript.exists():
|
|
153
|
+
transcript.write_text(
|
|
154
|
+
json.dumps({"task": ctx.task_slug, "trial": ctx.trial_index, "turns": result.turns}),
|
|
155
|
+
encoding="utf-8",
|
|
156
|
+
)
|
|
157
|
+
return EXIT_OK
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def main(argv: list[str]) -> int:
|
|
161
|
+
if len(argv) != 2:
|
|
162
|
+
print("usage: entrypoint.py <context.json>", file=sys.stderr)
|
|
163
|
+
return EXIT_ENTRYPOINT_FAILED
|
|
164
|
+
try:
|
|
165
|
+
return asyncio.run(run(Path(argv[1])))
|
|
166
|
+
except Exception:
|
|
167
|
+
# The entrypoint itself failed. This is Meridian's problem, and the
|
|
168
|
+
# non-zero exit is what tells the runner to classify it as one.
|
|
169
|
+
traceback.print_exc()
|
|
170
|
+
return EXIT_ENTRYPOINT_FAILED
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
if __name__ == "__main__": # pragma: no cover - exercised inside the container
|
|
174
|
+
raise SystemExit(main(sys.argv))
|