meridian-eval 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. meridian/__init__.py +15 -0
  2. meridian/_assets/agents/terminal_agent/__init__.py +0 -0
  3. meridian/_assets/agents/terminal_agent/agent.py +209 -0
  4. meridian/_assets/agents/terminal_agent/prompts/system.md +11 -0
  5. meridian/_assets/agents/terminal_agent/tools.py +97 -0
  6. meridian/_assets/agents/terminal_agent/transcript.py +38 -0
  7. meridian/_assets/proxy.Dockerfile +43 -0
  8. meridian/adapters/__init__.py +7 -0
  9. meridian/adapters/base.py +60 -0
  10. meridian/adapters/entrypoint.py +174 -0
  11. meridian/adapters/langgraph_adapter.py +79 -0
  12. meridian/adapters/subprocess_adapter.py +80 -0
  13. meridian/assets.py +69 -0
  14. meridian/cache/__init__.py +5 -0
  15. meridian/cache/keys.py +96 -0
  16. meridian/cli.py +980 -0
  17. meridian/config.py +199 -0
  18. meridian/docker_client.py +33 -0
  19. meridian/exit_codes.py +15 -0
  20. meridian/gate/__init__.py +1 -0
  21. meridian/gate/baseline.py +91 -0
  22. meridian/gate/comment.py +291 -0
  23. meridian/gate/decide.py +189 -0
  24. meridian/grading/__init__.py +6 -0
  25. meridian/grading/graders/__init__.py +1 -0
  26. meridian/grading/graders/base.py +78 -0
  27. meridian/grading/graders/filesystem.py +64 -0
  28. meridian/grading/graders/json_state.py +113 -0
  29. meridian/grading/graders/process.py +44 -0
  30. meridian/grading/graders/pytest_grader.py +70 -0
  31. meridian/grading/graders/script_grader.py +83 -0
  32. meridian/grading/graders/sqlite.py +49 -0
  33. meridian/grading/pipeline.py +98 -0
  34. meridian/hashing.py +68 -0
  35. meridian/importers/__init__.py +1 -0
  36. meridian/importers/harbor.py +568 -0
  37. meridian/manifest/__init__.py +1 -0
  38. meridian/manifest/build.py +146 -0
  39. meridian/manifest/replay.py +144 -0
  40. meridian/manifest/store.py +107 -0
  41. meridian/models/__init__.py +6 -0
  42. meridian/models/adapter.py +83 -0
  43. meridian/models/manifest.py +95 -0
  44. meridian/models/run.py +229 -0
  45. meridian/models/suite.py +77 -0
  46. meridian/models/task.py +328 -0
  47. meridian/report/__init__.py +1 -0
  48. meridian/report/html.py +225 -0
  49. meridian/report/table.py +85 -0
  50. meridian/runtime/__init__.py +1 -0
  51. meridian/runtime/isolation.py +141 -0
  52. meridian/runtime/orchestrator.py +338 -0
  53. meridian/runtime/payload.py +174 -0
  54. meridian/runtime/proxy/__init__.py +17 -0
  55. meridian/runtime/proxy/budget.py +234 -0
  56. meridian/runtime/proxy/cassette.py +236 -0
  57. meridian/runtime/proxy/openai_compat.py +191 -0
  58. meridian/runtime/proxy/retry.py +141 -0
  59. meridian/runtime/proxy/server.py +374 -0
  60. meridian/runtime/proxy/session.py +325 -0
  61. meridian/runtime/proxy/stub_provider.py +277 -0
  62. meridian/runtime/pytest_runner.py +244 -0
  63. meridian/runtime/scheduler.py +114 -0
  64. meridian/runtime/secrets.py +110 -0
  65. meridian/runtime/trial_runner.py +763 -0
  66. meridian/snapshots/__init__.py +1 -0
  67. meridian/snapshots/build.py +151 -0
  68. meridian/snapshots/registry.py +101 -0
  69. meridian/stats/__init__.py +1 -0
  70. meridian/stats/bootstrap.py +69 -0
  71. meridian/stats/intervals.py +180 -0
  72. meridian/stats/passk.py +53 -0
  73. meridian/stats/power.py +144 -0
  74. meridian/stats/significance.py +70 -0
  75. meridian/store/__init__.py +6 -0
  76. meridian/store/migrations/env.py +48 -0
  77. meridian/store/migrations/script.py.mako +22 -0
  78. meridian/store/migrations/versions/0001_initial_schema.py +46 -0
  79. meridian/store/repo.py +251 -0
  80. meridian/store/schema.sql +124 -0
  81. meridian/suites/__init__.py +1 -0
  82. meridian/suites/audit.py +137 -0
  83. meridian/suites/loader.py +174 -0
  84. meridian/suites/validate.py +115 -0
  85. meridian/version.py +9 -0
  86. meridian_eval-0.2.0.dist-info/METADATA +475 -0
  87. meridian_eval-0.2.0.dist-info/RECORD +91 -0
  88. meridian_eval-0.2.0.dist-info/WHEEL +4 -0
  89. meridian_eval-0.2.0.dist-info/entry_points.txt +2 -0
  90. meridian_eval-0.2.0.dist-info/licenses/LICENSE +21 -0
  91. meridian_eval-0.2.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +36 -0
meridian/__init__.py ADDED
@@ -0,0 +1,15 @@
1
+ """Meridian — an agent evaluation harness.
2
+
3
+ Four rules govern everything in this package:
4
+
5
+ 1. Every trial starts from a clean, isolated environment.
6
+ 2. Grade outcomes, not trajectories.
7
+ 3. Report pass^k, not a mean score.
8
+ 4. Every run is reproducible from its manifest.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from meridian.version import __version__
14
+
15
+ __all__ = ["__version__"]
File without changes
@@ -0,0 +1,209 @@
1
+ """A general terminal agent: a LangGraph loop over one `bash` tool.
2
+
3
+ plan → act ⇄ tools → finish
4
+
5
+ Unlike the checkout fixture, nothing here knows the task. It reads the prompt,
6
+ works in the container's working directory through a shell, and stops when the
7
+ model stops calling tools. That makes it the system under test for imported
8
+ benchmark tasks (see `meridian import-harbor`).
9
+
10
+ It calls the model through the Meridian proxy, never a provider, with no credential
11
+ of its own.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ import os
18
+ import time
19
+ from pathlib import Path
20
+ from typing import Annotated, Any, TypedDict
21
+
22
+ import httpx
23
+ from langgraph.graph import END, StateGraph
24
+
25
+ from terminal_agent.tools import TOOL_SCHEMAS, format_result, run_bash
26
+ from terminal_agent.transcript import record
27
+
28
+ # The proxy substitutes the model the operator chose, so this name is only a label.
29
+ MODEL = os.environ.get("MERIDIAN_FIXTURE_MODEL", "meridian-model")
30
+ MAX_TURNS = 30
31
+ REQUEST_TIMEOUT_SECONDS = 180.0
32
+ # The trial token budget is a ceiling on the whole trial, not on one reply.
33
+ MAX_TOKENS_PER_REPLY = 4096
34
+
35
+ PROMPTS = Path(__file__).resolve().parent / "prompts"
36
+
37
+
38
+ def _append(existing: list[Any], incoming: list[Any]) -> list[Any]:
39
+ return [*existing, *incoming]
40
+
41
+
42
+ class AgentState(TypedDict, total=False):
43
+ context: dict[str, Any]
44
+ messages: Annotated[list[dict[str, Any]], _append]
45
+ system: str
46
+ turns: int
47
+ tool_calls: Annotated[list[dict[str, Any]], _append]
48
+ input_tokens: int
49
+ output_tokens: int
50
+ error: str | None
51
+ completed: bool
52
+
53
+
54
+ def plan(state: AgentState) -> AgentState:
55
+ context = state["context"]
56
+ return {
57
+ "system": (PROMPTS / "system.md").read_text(encoding="utf-8"),
58
+ "messages": [{"role": "user", "content": context["prompt"]}],
59
+ "turns": 0,
60
+ "input_tokens": 0,
61
+ "output_tokens": 0,
62
+ "tool_calls": [],
63
+ "completed": False,
64
+ "error": None,
65
+ }
66
+
67
+
68
+ def act(state: AgentState) -> AgentState:
69
+ """One model turn, through the proxy."""
70
+ context = state["context"]
71
+ body = {
72
+ "model": MODEL,
73
+ "max_tokens": min(
74
+ int(context.get("max_tokens", MAX_TOKENS_PER_REPLY)), MAX_TOKENS_PER_REPLY
75
+ ),
76
+ "system": state.get("system", ""),
77
+ "messages": state.get("messages", []),
78
+ "tools": TOOL_SCHEMAS,
79
+ }
80
+ headers = {
81
+ "x-meridian-task": str(context.get("task_slug", "")),
82
+ "x-meridian-trial": str(context.get("trial_index", 0)),
83
+ "content-type": "application/json",
84
+ }
85
+ url = f"{context['model_base_url']}/v1/messages"
86
+ workdir = str(context.get("workdir", "."))
87
+ asked = time.monotonic()
88
+ record(workdir, "model_request", turn=state.get("turns", 0) + 1)
89
+ try:
90
+ response = httpx.post(url, json=body, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)
91
+ except httpx.HTTPError as exc:
92
+ record(workdir, "model_error", error=f"{type(exc).__name__}: {exc}")
93
+ return {"error": f"model call failed: {exc}", "completed": False}
94
+ if response.status_code != 200:
95
+ record(workdir, "model_error", status=response.status_code, body=response.text)
96
+ return {
97
+ "error": f"model returned {response.status_code}: {response.text[:400]}",
98
+ "completed": False,
99
+ }
100
+
101
+ payload = response.json()
102
+ usage = payload.get("usage") or {}
103
+ reply = payload.get("content") or []
104
+ record(
105
+ workdir,
106
+ "model_reply",
107
+ seconds=round(time.monotonic() - asked, 1),
108
+ stop=payload.get("stop_reason"),
109
+ text=" ".join(
110
+ b.get("text", "") for b in reply if isinstance(b, dict) and b.get("type") == "text"
111
+ ),
112
+ commands=[
113
+ (b.get("input") or {}).get("command")
114
+ for b in reply
115
+ if isinstance(b, dict) and b.get("type") == "tool_use"
116
+ ],
117
+ )
118
+ return {
119
+ "messages": [{"role": "assistant", "content": payload.get("content") or []}],
120
+ "turns": state.get("turns", 0) + 1,
121
+ "input_tokens": state.get("input_tokens", 0) + int(usage.get("input_tokens", 0)),
122
+ "output_tokens": state.get("output_tokens", 0) + int(usage.get("output_tokens", 0)),
123
+ "completed": payload.get("stop_reason") != "tool_use",
124
+ }
125
+
126
+
127
+ def tools(state: AgentState) -> AgentState:
128
+ """Run every tool call the model made, in order."""
129
+ workdir = str(state["context"]["workdir"])
130
+ results: list[dict[str, Any]] = []
131
+ recorded: list[dict[str, Any]] = []
132
+ for block in state["messages"][-1].get("content", []):
133
+ if not isinstance(block, dict) or block.get("type") != "tool_use":
134
+ continue
135
+ arguments = dict(block.get("input") or {})
136
+ if block.get("name") != "bash" or not isinstance(arguments.get("command"), str):
137
+ text, ok = f"[unknown tool or bad arguments: {json.dumps(arguments)[:200]}]", False
138
+ else:
139
+ began = time.monotonic()
140
+ result = run_bash(
141
+ arguments["command"],
142
+ workdir=workdir,
143
+ timeout_seconds=arguments.get("timeout_seconds"),
144
+ )
145
+ text, ok = format_result(result), result["error"] is None
146
+ record(
147
+ workdir,
148
+ "command",
149
+ command=arguments["command"],
150
+ exit_code=result["exit_code"],
151
+ error=result["error"],
152
+ seconds=round(time.monotonic() - began, 1),
153
+ output=result["output"],
154
+ )
155
+ recorded.append({"name": str(block.get("name")), "ok": ok})
156
+ results.append(
157
+ {
158
+ "type": "tool_result",
159
+ "tool_use_id": block.get("id"),
160
+ "content": text,
161
+ "is_error": not ok,
162
+ }
163
+ )
164
+ return {"messages": [{"role": "user", "content": results}], "tool_calls": recorded}
165
+
166
+
167
+ def route(state: AgentState) -> str:
168
+ if state.get("error") or state.get("completed"):
169
+ return "finish"
170
+ if state.get("turns", 0) >= MAX_TURNS:
171
+ return "out_of_turns"
172
+ last = state["messages"][-1]
173
+ wants_tools = any(
174
+ isinstance(block, dict) and block.get("type") == "tool_use"
175
+ for block in last.get("content", [])
176
+ )
177
+ return "tools" if wants_tools else "finish"
178
+
179
+
180
+ def out_of_turns(state: AgentState) -> AgentState:
181
+ # An agent failure, not a harness one: it had its turns and did not finish.
182
+ return {"error": f"gave up after {MAX_TURNS} turns", "completed": False}
183
+
184
+
185
+ def finish(state: AgentState) -> AgentState:
186
+ return {"completed": state.get("error") is None}
187
+
188
+
189
+ def build_graph() -> Any:
190
+ builder: StateGraph[AgentState] = StateGraph(AgentState)
191
+ builder.add_node("plan", plan)
192
+ builder.add_node("act", act)
193
+ builder.add_node("tools", tools)
194
+ builder.add_node("out_of_turns", out_of_turns)
195
+ builder.add_node("finish", finish)
196
+ builder.set_entry_point("plan")
197
+ builder.add_edge("plan", "act")
198
+ builder.add_conditional_edges(
199
+ "act",
200
+ route,
201
+ {"tools": "tools", "finish": "finish", "out_of_turns": "out_of_turns"},
202
+ )
203
+ builder.add_edge("tools", "act")
204
+ builder.add_edge("out_of_turns", "finish")
205
+ builder.add_edge("finish", END)
206
+ return builder.compile()
207
+
208
+
209
+ graph = build_graph()
@@ -0,0 +1,11 @@
1
+ You are an autonomous software engineer working in a Linux container. You have one tool, `bash`, which runs a shell command in your working directory and returns its output and exit code.
2
+
3
+ Work toward the task you are given:
4
+
5
+ - Look before you change things: list files and read what is relevant.
6
+ - Make the change, then check your own work by running it. Do not assume it worked.
7
+ - Commands have a time limit and large outputs are truncated, so prefer targeted commands.
8
+ - You have no internet access and no root privileges. Use only what is installed.
9
+ - When the task is complete, stop calling tools and reply with a one-line summary.
10
+
11
+ Your result is judged only by the final state of the container, so make sure the files and outputs the task asks for exist and are correct before you stop.
@@ -0,0 +1,97 @@
1
+ """The terminal agent's one tool: run a shell command.
2
+
3
+ Standard library only, so it can be tested on the host without LangGraph.
4
+
5
+ The command runs inside the trial container, which is already the sandbox: a
6
+ read-only root filesystem, an unprivileged user, no capabilities and no network.
7
+ This module adds only what any shell tool needs -- a time limit and bounded output.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import os
13
+ import subprocess
14
+ from typing import Any
15
+
16
+ MAX_OUTPUT_CHARS = 8000
17
+ DEFAULT_TIMEOUT_SECONDS = 60
18
+ MAX_TIMEOUT_SECONDS = 300
19
+
20
+ TOOL_SCHEMAS: list[dict[str, Any]] = [
21
+ {
22
+ "name": "bash",
23
+ "description": (
24
+ "Run a shell command in the working directory. Returns the combined "
25
+ "stdout/stderr and the exit code. Commands that run longer than the "
26
+ "timeout are killed."
27
+ ),
28
+ "input_schema": {
29
+ "type": "object",
30
+ "properties": {
31
+ "command": {"type": "string", "description": "The command to run."},
32
+ "timeout_seconds": {
33
+ "type": "integer",
34
+ "description": f"Time limit, at most {MAX_TIMEOUT_SECONDS} (default "
35
+ f"{DEFAULT_TIMEOUT_SECONDS}).",
36
+ },
37
+ },
38
+ "required": ["command"],
39
+ },
40
+ }
41
+ ]
42
+
43
+
44
+ def truncate(text: str, limit: int = MAX_OUTPUT_CHARS) -> str:
45
+ """Keep the start and the end of long output; the middle is rarely the point."""
46
+ if len(text) <= limit:
47
+ return text
48
+ head = limit // 2
49
+ tail = limit - head
50
+ omitted = len(text) - limit
51
+ return f"{text[:head]}\n... [{omitted} characters omitted] ...\n{text[-tail:]}"
52
+
53
+
54
+ def run_bash(command: str, *, workdir: str, timeout_seconds: int | None = None) -> dict[str, Any]:
55
+ """Run `command` with bash in `workdir`. Never raises; failures are results."""
56
+ timeout = DEFAULT_TIMEOUT_SECONDS if timeout_seconds is None else timeout_seconds
57
+ timeout = max(1, min(int(timeout), MAX_TIMEOUT_SECONDS))
58
+ env = {
59
+ "PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
60
+ "HOME": "/tmp",
61
+ "LANG": "C.UTF-8",
62
+ "TERM": "dumb",
63
+ }
64
+ try:
65
+ completed = subprocess.run(
66
+ ["bash", "-c", command],
67
+ cwd=workdir,
68
+ env=env,
69
+ capture_output=True,
70
+ timeout=timeout,
71
+ stdin=subprocess.DEVNULL,
72
+ check=False,
73
+ )
74
+ except subprocess.TimeoutExpired as exc:
75
+ partial = (exc.stdout or b"") + (exc.stderr or b"")
76
+ output = partial.decode("utf-8", errors="replace")
77
+ return {
78
+ "exit_code": None,
79
+ "output": truncate(output),
80
+ "error": f"timed out after {timeout}s",
81
+ }
82
+ except OSError as exc:
83
+ return {"exit_code": None, "output": "", "error": f"{type(exc).__name__}: {exc}"}
84
+ output = (completed.stdout + completed.stderr).decode("utf-8", errors="replace")
85
+ return {"exit_code": completed.returncode, "output": truncate(output), "error": None}
86
+
87
+
88
+ def format_result(result: dict[str, Any]) -> str:
89
+ """The text the model sees for one command."""
90
+ lines = []
91
+ if result["error"]:
92
+ lines.append(f"[{result['error']}]")
93
+ elif result["exit_code"] != 0:
94
+ lines.append(f"[exit code {result['exit_code']}]")
95
+ if result["output"]:
96
+ lines.append(result["output"])
97
+ return "\n".join(lines) if lines else "[no output]"
@@ -0,0 +1,38 @@
1
+ """An append-only transcript of what the agent did, written as it goes.
2
+
3
+ Written line by line and flushed immediately, so that when the harness kills the
4
+ agent at its deadline the record of what it was doing survives. The harness reads
5
+ the tail of this file into a timeout verdict.
6
+
7
+ Standard library only.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import time
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ TRANSCRIPT_RELATIVE = ".meridian/transcript.jsonl"
18
+ FIELD_LIMIT = 400
19
+
20
+
21
+ def _clip(value: Any) -> Any:
22
+ if isinstance(value, str) and len(value) > FIELD_LIMIT:
23
+ return value[:FIELD_LIMIT] + f"... [{len(value) - FIELD_LIMIT} more]"
24
+ return value
25
+
26
+
27
+ def record(workdir: str, event: str, **fields: Any) -> None:
28
+ """Append one event. Never raises: a transcript must not be able to fail a trial."""
29
+ line = {"t": round(time.time(), 2), "event": event}
30
+ line.update({key: _clip(value) for key, value in fields.items()})
31
+ try:
32
+ path = Path(workdir) / TRANSCRIPT_RELATIVE
33
+ path.parent.mkdir(parents=True, exist_ok=True)
34
+ with path.open("a", encoding="utf-8") as handle:
35
+ handle.write(json.dumps(line, sort_keys=True) + "\n")
36
+ handle.flush()
37
+ except OSError:
38
+ pass
@@ -0,0 +1,43 @@
1
+ # The model proxy image.
2
+ #
3
+ # Built with the repository root as context because it ships a slice of the
4
+ # harness. Deliberately minimal: this container holds the only provider
5
+ # credential in the system, so the less it contains the better.
6
+ FROM python:3.13-slim
7
+
8
+ ENV PYTHONDONTWRITEBYTECODE=1 \
9
+ PYTHONUNBUFFERED=1 \
10
+ PIP_NO_CACHE_DIR=1 \
11
+ PIP_DISABLE_PIP_VERSION_CHECK=1
12
+
13
+ RUN pip install --no-cache-dir \
14
+ "starlette==0.45.3" \
15
+ "uvicorn==0.34.0" \
16
+ "httpx==0.28.1" \
17
+ "pydantic==2.10.6" \
18
+ "pyyaml==6.0.2"
19
+
20
+ COPY src/meridian/__init__.py /opt/meridian/meridian/__init__.py
21
+ COPY src/meridian/version.py /opt/meridian/meridian/version.py
22
+ COPY src/meridian/hashing.py /opt/meridian/meridian/hashing.py
23
+ COPY src/meridian/config.py /opt/meridian/meridian/config.py
24
+ # cassette.py validates tape names against the task schema's slug alphabet, so
25
+ # the proxy needs that one model module (pydantic only, no further imports).
26
+ COPY src/meridian/models/__init__.py /opt/meridian/meridian/models/__init__.py
27
+ COPY src/meridian/models/task.py /opt/meridian/meridian/models/task.py
28
+ COPY src/meridian/runtime/__init__.py /opt/meridian/meridian/runtime/__init__.py
29
+ COPY src/meridian/runtime/secrets.py /opt/meridian/meridian/runtime/secrets.py
30
+ COPY src/meridian/runtime/proxy/ /opt/meridian/meridian/runtime/proxy/
31
+
32
+ ENV PYTHONPATH=/opt/meridian \
33
+ MERIDIAN_CASSETTE_DIR=/cassettes \
34
+ MERIDIAN_PROXY_PORT=8080
35
+
36
+ # Named `meridian`, not `proxy`: Debian base images already ship a `proxy` user
37
+ # and groupadd exits 9 on the collision.
38
+ RUN groupadd --gid 10002 meridian && useradd --uid 10002 --gid 10002 --no-create-home meridian \
39
+ && mkdir -p /cassettes && chown 10002:10002 /cassettes
40
+ USER 10002:10002
41
+
42
+ EXPOSE 8080
43
+ ENTRYPOINT ["python3", "-m", "meridian.runtime.proxy.server"]
@@ -0,0 +1,7 @@
1
+ """Adapters run **inside** the trial container, never in the harness process.
2
+
3
+ The harness never imports the system under test — it ships this package into the
4
+ container and lets the entrypoint resolve the adapter spec there. That is what
5
+ makes "framework-agnostic" a structural property rather than a claim: the harness
6
+ has no way to depend on a framework it cannot import.
7
+ """
@@ -0,0 +1,60 @@
1
+ """Adapter spec strings: `kind:module:attribute`.
2
+
3
+ `langgraph:checkout_agent.agent:graph` says *which adapter* to use, *which
4
+ module* holds the system under test, and *which attribute* of it to invoke. The
5
+ harness only ever parses this; it resolves it inside the container.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import importlib
11
+ from dataclasses import dataclass
12
+ from typing import Any
13
+
14
+ KNOWN_KINDS = ("langgraph", "subprocess")
15
+
16
+
17
+ class AdapterSpecError(ValueError):
18
+ """A malformed or unknown adapter spec. A configuration error, not a verdict."""
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class AdapterSpec:
23
+ """A parsed `kind:module:attribute` triple."""
24
+
25
+ kind: str
26
+ module: str
27
+ attribute: str
28
+
29
+ @classmethod
30
+ def parse(cls, raw: str) -> AdapterSpec:
31
+ parts = raw.split(":")
32
+ if len(parts) != 3 or not all(part.strip() for part in parts):
33
+ raise AdapterSpecError(
34
+ f"adapter spec {raw!r} must be 'kind:module:attribute', "
35
+ f"e.g. 'langgraph:checkout_agent.agent:graph'"
36
+ )
37
+ kind, module, attribute = (part.strip() for part in parts)
38
+ if kind not in KNOWN_KINDS:
39
+ raise AdapterSpecError(
40
+ f"unknown adapter kind {kind!r}; Meridian ships {', '.join(KNOWN_KINDS)}"
41
+ )
42
+ return cls(kind=kind, module=module, attribute=attribute)
43
+
44
+ def __str__(self) -> str:
45
+ return f"{self.kind}:{self.module}:{self.attribute}"
46
+
47
+ def resolve_target(self) -> Any:
48
+ """Import the module and return the attribute. Container-side only."""
49
+ try:
50
+ module = importlib.import_module(self.module)
51
+ except ImportError as exc:
52
+ raise AdapterSpecError(
53
+ f"cannot import {self.module!r} for adapter spec {self}: {exc}"
54
+ ) from exc
55
+ try:
56
+ return getattr(module, self.attribute)
57
+ except AttributeError as exc:
58
+ raise AdapterSpecError(
59
+ f"module {self.module!r} has no attribute {self.attribute!r}"
60
+ ) from exc
@@ -0,0 +1,174 @@
1
+ """The in-container entrypoint. PID 1's child; the harness never runs this.
2
+
3
+ Reads a serialized `TrialContext`, resolves the adapter spec, runs the agent,
4
+ and writes `result.json` and `transcript.json` next to the payload. It exits 0
5
+ whenever it *ran the agent to a conclusion* — including a conclusion the agent
6
+ did not like — because the trial's verdict is decided outside, by grading the
7
+ final state. A non-zero exit here means the entrypoint itself could not run,
8
+ which is a harness error.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import asyncio
14
+ import json
15
+ import sys
16
+ import traceback
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ from meridian.adapters.base import AdapterSpec, AdapterSpecError
21
+ from meridian.models.adapter import AdapterResult, TrialContext
22
+
23
+ EXIT_OK = 0
24
+ EXIT_ENTRYPOINT_FAILED = 70
25
+
26
+ # Written when the entrypoint itself could not run. The harness reads it so the
27
+ # trial's detail says *what* was misconfigured rather than only that something
28
+ # was.
29
+ HARNESS_ERROR_FILE = "harness_error.txt"
30
+
31
+ # Short: this only ever runs after the agent has already failed, and the answer
32
+ # is "is anything listening", not "is it fast".
33
+ PROXY_HEALTH_TIMEOUT_SECONDS = 5.0
34
+
35
+
36
+ def build_adapter(spec: AdapterSpec) -> Any:
37
+ """Pick the adapter implementation for a spec kind.
38
+
39
+ Imported lazily and per-kind: the LangGraph adapter imports LangGraph, and a
40
+ subprocess trial should not pay for — or fail on — a framework it never uses.
41
+ """
42
+ if spec.kind == "subprocess":
43
+ from meridian.adapters.subprocess_adapter import SubprocessAdapter
44
+
45
+ return SubprocessAdapter(spec)
46
+ if spec.kind == "langgraph":
47
+ from meridian.adapters.langgraph_adapter import LangGraphAdapter
48
+
49
+ return LangGraphAdapter(spec)
50
+ raise ValueError(f"no adapter implementation for kind {spec.kind!r}")
51
+
52
+
53
+ def record_harness_error(payload_dir: Path, message: str) -> int:
54
+ """Report a Meridian misconfiguration, not an agent failure."""
55
+ (payload_dir / HARNESS_ERROR_FILE).write_text(message, encoding="utf-8")
56
+ (payload_dir / "traceback.txt").write_text(traceback.format_exc(), encoding="utf-8")
57
+ return EXIT_ENTRYPOINT_FAILED
58
+
59
+
60
+ def proxy_is_unreachable(base_url: str) -> bool:
61
+ """Ask the proxy whether it is alive. Used only to explain a failure.
62
+
63
+ This is the seam that separates "the agent failed" from "the agent was never
64
+ given a model". It asks the proxy directly rather than matching on the text of
65
+ the agent's error, because the wording of that error belongs to whatever
66
+ framework raised it and will not survive an upgrade — and a classification
67
+ rule built on someone else's error string fails silently and in the expensive
68
+ direction.
69
+
70
+ Unreachable means *refused or unanswered*, not "returned an error". A proxy
71
+ that answers with a budget refusal or a cassette miss is working exactly as
72
+ designed, and those are genuine agent-visible failures.
73
+ """
74
+ if not base_url:
75
+ return False
76
+ try:
77
+ import httpx
78
+ except ImportError:
79
+ # The trial image supplies httpx, not the payload. An environment without
80
+ # it must not turn every agent failure into an unanswerable question:
81
+ # raising here would escape `run()`, exit EXIT_ENTRYPOINT_FAILED, and be
82
+ # classified as a harness error — which is *retried*. Every genuine agent
83
+ # `fail` on such an image would then get a second attempt, which is the
84
+ # one thing the retry policy exists to forbid.
85
+ return False
86
+
87
+ try:
88
+ response = httpx.get(
89
+ f"{base_url.rstrip('/')}/healthz", timeout=PROXY_HEALTH_TIMEOUT_SECONDS
90
+ )
91
+ except Exception:
92
+ # Deliberately broad. `httpx.InvalidURL` is not an `HTTPError`, so a
93
+ # malformed base URL would otherwise crash the entrypoint rather than
94
+ # answer the question. Anything that prevents an answer means the proxy
95
+ # could not be confirmed alive, which is what the caller asked.
96
+ return True
97
+ return response.status_code >= 500
98
+
99
+
100
+ async def run(context_path: Path) -> int:
101
+ payload_dir = context_path.parent
102
+ raw = json.loads(context_path.read_text(encoding="utf-8"))
103
+
104
+ # Everything up to and including resolving the adapter is configuration. A
105
+ # failure here says the harness was set up wrong, and calling it an agent
106
+ # failure would put a configuration mistake into the score.
107
+ try:
108
+ adapter_spec = AdapterSpec.parse(raw.pop("adapter_spec"))
109
+ ctx = TrialContext.model_validate(raw)
110
+ adapter = build_adapter(adapter_spec)
111
+ adapter_spec.resolve_target()
112
+ except (AdapterSpecError, ValueError) as exc:
113
+ return record_harness_error(payload_dir, f"{type(exc).__name__}: {exc}")
114
+
115
+ try:
116
+ result = await adapter.invoke(ctx)
117
+ except AdapterSpecError as exc:
118
+ return record_harness_error(payload_dir, f"{type(exc).__name__}: {exc}")
119
+ except Exception as exc: # the agent blew up: an agent failure, not a harness one
120
+ result = AdapterResult(
121
+ completed=False,
122
+ error=f"{type(exc).__name__}: {exc}",
123
+ transcript_path=str(payload_dir / "transcript.json"),
124
+ )
125
+ (payload_dir / "traceback.txt").write_text(traceback.format_exc(), encoding="utf-8")
126
+ finally:
127
+ await adapter.teardown()
128
+
129
+ # An agent that could not reach the model has not been evaluated, so its
130
+ # failure is Meridian's and not the agent's. Checked here rather than at the
131
+ # exception site because an agent may *return* the failure instead of raising
132
+ # it -- the checkout fixture catches httpx errors and reports them as an
133
+ # AdapterResult -- and both paths must classify the same way.
134
+ #
135
+ # Getting this wrong is expensive in the direction that wastes a person's
136
+ # afternoon: a dead proxy fails every trial of every task, the gate reports
137
+ # the largest regression it has ever seen, and the developer reverts a change
138
+ # that was never the cause.
139
+ if result.error and proxy_is_unreachable(ctx.model_base_url):
140
+ return record_harness_error(
141
+ payload_dir,
142
+ f"the model proxy at {ctx.model_base_url} did not answer its health "
143
+ f"check, so the agent was never given a model. The agent reported: "
144
+ f"{result.error}",
145
+ )
146
+
147
+ (payload_dir / "result.json").write_text(
148
+ result.model_dump_json(indent=2),
149
+ encoding="utf-8",
150
+ )
151
+ transcript = payload_dir / "transcript.json"
152
+ if not transcript.exists():
153
+ transcript.write_text(
154
+ json.dumps({"task": ctx.task_slug, "trial": ctx.trial_index, "turns": result.turns}),
155
+ encoding="utf-8",
156
+ )
157
+ return EXIT_OK
158
+
159
+
160
+ def main(argv: list[str]) -> int:
161
+ if len(argv) != 2:
162
+ print("usage: entrypoint.py <context.json>", file=sys.stderr)
163
+ return EXIT_ENTRYPOINT_FAILED
164
+ try:
165
+ return asyncio.run(run(Path(argv[1])))
166
+ except Exception:
167
+ # The entrypoint itself failed. This is Meridian's problem, and the
168
+ # non-zero exit is what tells the runner to classify it as one.
169
+ traceback.print_exc()
170
+ return EXIT_ENTRYPOINT_FAILED
171
+
172
+
173
+ if __name__ == "__main__": # pragma: no cover - exercised inside the container
174
+ raise SystemExit(main(sys.argv))