verifiers 0.3.2.dev115__py3-none-any.whl → 0.3.2.dev117__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- verifiers/v1/harnesses/browser_use/program.py +1 -1
- verifiers/v1/harnesses/claude_code/harness.py +4 -2
- verifiers/v1/harnesses/codex/harness.py +2 -2
- verifiers/v1/harnesses/hermes_agent/harness.py +34 -9
- verifiers/v1/harnesses/hermes_agent/program.py +2 -2
- verifiers/v1/harnesses/kimi_code/harness.py +1 -1
- verifiers/v1/harnesses/mini_swe_agent/program.py +1 -1
- verifiers/v1/harnesses/openclaw/harness.py +3 -2
- verifiers/v1/harnesses/pi/harness.py +2 -2
- verifiers/v1/harnesses/prime_agent/harness.py +9 -9
- verifiers/v1/harnesses/rlm/harness.py +1 -1
- verifiers/v1/harnesses/terminus_2/harness.py +1 -1
- verifiers/v1/rollout.py +3 -1
- verifiers/v1/runtimes/prime.py +19 -8
- verifiers/v1/task.py +3 -1
- verifiers/v1/tasksets/harbor/taskset.py +7 -2
- verifiers/v1/utils/artifacts.py +6 -3
- {verifiers-0.3.2.dev115.dist-info → verifiers-0.3.2.dev117.dist-info}/METADATA +1 -1
- {verifiers-0.3.2.dev115.dist-info → verifiers-0.3.2.dev117.dist-info}/RECORD +22 -22
- {verifiers-0.3.2.dev115.dist-info → verifiers-0.3.2.dev117.dist-info}/WHEEL +0 -0
- {verifiers-0.3.2.dev115.dist-info → verifiers-0.3.2.dev117.dist-info}/entry_points.txt +0 -0
- {verifiers-0.3.2.dev115.dist-info → verifiers-0.3.2.dev117.dist-info}/licenses/LICENSE +0 -0
|
@@ -11,7 +11,7 @@ from verifiers.v1.trace import Trace
|
|
|
11
11
|
|
|
12
12
|
CLAUDE_ACP_DIR = "/var/tmp/vf-claude-agent-acp-{version}-{acp_version}"
|
|
13
13
|
PACKAGES_DIR = f"{CLAUDE_ACP_DIR}/packages"
|
|
14
|
-
ACP_VERSION = "0.
|
|
14
|
+
ACP_VERSION = "0.79.0"
|
|
15
15
|
CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude"
|
|
16
16
|
ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp"
|
|
17
17
|
CLAUDE_CONFIG_ROOT = ".vf-claude"
|
|
@@ -28,7 +28,7 @@ touch {ready}
|
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
class ClaudeCodeHarnessConfig(HarnessConfig):
|
|
31
|
-
version: PinnedVersion = "2.1.
|
|
31
|
+
version: PinnedVersion = "2.1.278"
|
|
32
32
|
"""Claude Code release to install, pinned for reproducibility."""
|
|
33
33
|
|
|
34
34
|
|
|
@@ -79,6 +79,8 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]):
|
|
|
79
79
|
"options": {
|
|
80
80
|
"strictMcpConfig": True,
|
|
81
81
|
"disallowedTools": self.config.disabled_tools or [],
|
|
82
|
+
# A supplied title prevents the adapter's background title-model call.
|
|
83
|
+
"extraArgs": {"name": trace.id},
|
|
82
84
|
}
|
|
83
85
|
},
|
|
84
86
|
**({"systemPrompt": {"append": system_prompt}} if system_prompt else {}),
|
|
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
|
|
|
19
19
|
|
|
20
20
|
CODEX_DIR = "/var/tmp/vf-codex-{version}-{acp_version}"
|
|
21
21
|
PACKAGES_DIR = f"{CODEX_DIR}/acp"
|
|
22
|
-
ACP_VERSION = "1.
|
|
22
|
+
ACP_VERSION = "1.12.0"
|
|
23
23
|
CODEX_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex"
|
|
24
24
|
ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp"
|
|
25
25
|
INSTALL = r"""
|
|
@@ -35,7 +35,7 @@ touch {ready}
|
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
class CodexHarnessConfig(HarnessConfig):
|
|
38
|
-
version: PinnedVersion = "0.
|
|
38
|
+
version: PinnedVersion = "0.155.1"
|
|
39
39
|
"""Codex release to install, pinned for reproducibility."""
|
|
40
40
|
multi_agent: bool = False
|
|
41
41
|
"""Enable Codex's native multi-agent v2 tools."""
|
|
@@ -6,17 +6,28 @@ from pathlib import Path
|
|
|
6
6
|
from verifiers.v1.acp import ACPConfig, ACPHarness
|
|
7
7
|
from verifiers.v1.clients import ModelContext
|
|
8
8
|
from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
|
|
9
|
-
from verifiers.v1.harnesses.utils.install import remove_dir
|
|
9
|
+
from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
|
|
10
10
|
from verifiers.v1.runtimes import Runtime
|
|
11
|
+
from verifiers.v1.runtimes.base import _ENSURE_UV
|
|
11
12
|
from verifiers.v1.task import TaskData
|
|
12
13
|
from verifiers.v1.trace import Trace
|
|
13
14
|
|
|
14
15
|
PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
|
|
16
|
+
HERMES_DIR = "/var/tmp/vf-hermes-agent-{version}"
|
|
17
|
+
INSTALL = f"""
|
|
18
|
+
set -e
|
|
19
|
+
{_ENSURE_UV}
|
|
20
|
+
command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null)
|
|
21
|
+
curl -fsSL "https://github.com/NousResearch/hermes-agent/archive/refs/tags/$VF_HERMES_VERSION.tar.gz" \\
|
|
22
|
+
| tar -xz --strip-components=1 -C "$VF_HERMES_DIR"
|
|
23
|
+
uv sync --project "$VF_HERMES_DIR" --locked --no-dev --extra acp --extra mcp
|
|
24
|
+
touch "$VF_HERMES_DIR/.ready"
|
|
25
|
+
"""
|
|
15
26
|
|
|
16
27
|
|
|
17
28
|
class HermesAgentHarnessConfig(HarnessConfig):
|
|
18
|
-
version: PinnedVersion = "
|
|
19
|
-
"""Hermes Agent release to install, pinned for reproducibility."""
|
|
29
|
+
version: PinnedVersion = "v2026.9.14"
|
|
30
|
+
"""Hermes Agent Git release tag to install, pinned for reproducibility."""
|
|
20
31
|
use_bundled_skill: bool = False
|
|
21
32
|
"""Enable Hermes Agent's bundled skill catalog in addition to uploaded skills."""
|
|
22
33
|
|
|
@@ -27,9 +38,20 @@ class HermesAgentHarness(ACPHarness[HermesAgentHarnessConfig]):
|
|
|
27
38
|
SUPPORTS_SKILLS = True
|
|
28
39
|
|
|
29
40
|
async def setup(self, runtime: Runtime) -> None:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
41
|
+
# Hermes needs its source-tree assets and supports editable installs only.
|
|
42
|
+
directory = HERMES_DIR.format(version=self.config.version)
|
|
43
|
+
await ensure_installed(
|
|
44
|
+
runtime,
|
|
45
|
+
directory=directory,
|
|
46
|
+
ready=f"test -f {directory}/.ready",
|
|
47
|
+
install=INSTALL,
|
|
48
|
+
env={
|
|
49
|
+
**self.config.resolved_env,
|
|
50
|
+
"VF_HERMES_DIR": directory,
|
|
51
|
+
"VF_HERMES_VERSION": self.config.version,
|
|
52
|
+
},
|
|
53
|
+
label="Hermes Agent",
|
|
54
|
+
shell=("bash", "-o", "pipefail", "-c"),
|
|
33
55
|
)
|
|
34
56
|
await super().setup(runtime)
|
|
35
57
|
|
|
@@ -87,9 +109,12 @@ class HermesAgentHarness(ACPHarness[HermesAgentHarnessConfig]):
|
|
|
87
109
|
system_prompt, prompt = self.resolve_prompt(data)
|
|
88
110
|
return ACPConfig(
|
|
89
111
|
env=env,
|
|
90
|
-
command=
|
|
91
|
-
|
|
92
|
-
|
|
112
|
+
command=[
|
|
113
|
+
f"{HERMES_DIR.format(version=self.config.version)}/.venv/bin/python",
|
|
114
|
+
"-P", # Keep task files from shadowing installed Hermes modules.
|
|
115
|
+
"-c",
|
|
116
|
+
PROGRAM_SOURCE,
|
|
117
|
+
],
|
|
93
118
|
prompt=prompt,
|
|
94
119
|
system_prompt=system_prompt,
|
|
95
120
|
)
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# /// script
|
|
2
2
|
# requires-python = ">=3.11,<3.14"
|
|
3
|
-
# dependencies = [
|
|
3
|
+
# dependencies = []
|
|
4
4
|
# ///
|
|
5
|
-
"""Start Hermes Agent's native ACP server."""
|
|
5
|
+
"""Start Hermes Agent's native ACP server in its installed project environment."""
|
|
6
6
|
|
|
7
7
|
import os
|
|
8
8
|
|
|
@@ -37,7 +37,7 @@ env \
|
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
class KimiCodeHarnessConfig(HarnessConfig):
|
|
40
|
-
version: PinnedVersion = "0.
|
|
40
|
+
version: PinnedVersion = "2.0.2"
|
|
41
41
|
"""Kimi Code release to install, pinned for reproducibility."""
|
|
42
42
|
transport: Literal["chat_completions", "responses", "anthropic_messages"] = (
|
|
43
43
|
"chat_completions"
|
|
@@ -23,7 +23,7 @@ if [ ! -x "$VF_OPENCLAW_BIN" ]; then
|
|
|
23
23
|
curl -fsSL --proto '=https' --tlsv1.2 https://openclaw.ai/install-cli.sh \
|
|
24
24
|
| bash -s -- --prefix "$VF_OPENCLAW_DIR" --version "$VF_OPENCLAW_VERSION" --no-onboard
|
|
25
25
|
fi
|
|
26
|
-
set -- $(find "$VF_OPENCLAW_DIR/tools" -type f -path '*/openclaw/dist/
|
|
26
|
+
set -- $(find "$VF_OPENCLAW_DIR/tools" -type f -path '*/openclaw/dist/transcript-redact-*.mjs')
|
|
27
27
|
[ "$#" -eq 1 ] || { echo "could not locate OpenClaw's transcript sanitizer for replay patch" >&2; exit 1; }
|
|
28
28
|
"$VF_OPENCLAW_DIR/tools/node/bin/node" - "$1" <<'NODE'
|
|
29
29
|
const fs = require("node:fs");
|
|
@@ -116,7 +116,7 @@ wait "$acp_pid"
|
|
|
116
116
|
|
|
117
117
|
|
|
118
118
|
class OpenClawHarnessConfig(HarnessConfig):
|
|
119
|
-
version: PinnedVersion = "2026.
|
|
119
|
+
version: PinnedVersion = "2026.9.5"
|
|
120
120
|
"""OpenClaw release to install, pinned for reproducibility."""
|
|
121
121
|
use_bundled_skill: bool = True
|
|
122
122
|
"""Enable OpenClaw's bundled skill catalog in addition to uploaded harness skills."""
|
|
@@ -180,6 +180,7 @@ class OpenClawHarness(ACPHarness[OpenClawHarnessConfig]):
|
|
|
180
180
|
"heartbeat": {"every": "0m"},
|
|
181
181
|
"sandbox": {"mode": "off"},
|
|
182
182
|
"model": {"primary": ctx.model},
|
|
183
|
+
"utilityModel": "", # Disable background recap model calls.
|
|
183
184
|
}
|
|
184
185
|
},
|
|
185
186
|
"tools": {
|
|
@@ -21,7 +21,7 @@ KEY_VAR = "PI_INTERCEPT_KEY"
|
|
|
21
21
|
PI_DIR = "/var/tmp/vf-pi"
|
|
22
22
|
PACKAGES_DIR = f"{PI_DIR}/mcp"
|
|
23
23
|
PI_BIN = f"{PACKAGES_DIR}/node_modules/.bin/pi"
|
|
24
|
-
MCP_VERSION = "2.
|
|
24
|
+
MCP_VERSION = "2.34.0"
|
|
25
25
|
ACP_VERSION = "0.0.33"
|
|
26
26
|
MCP_ADAPTER = f"{PACKAGES_DIR}/node_modules/pi-mcp-adapter/index.ts"
|
|
27
27
|
ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/pi-acp"
|
|
@@ -44,7 +44,7 @@ fi
|
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
class PiHarnessConfig(HarnessConfig):
|
|
47
|
-
version: PinnedVersion = "0.
|
|
47
|
+
version: PinnedVersion = "0.86.1"
|
|
48
48
|
"""Pi release to install, pinned for reproducibility."""
|
|
49
49
|
transport: Literal["chat_completions", "responses", "anthropic_messages"] = (
|
|
50
50
|
"chat_completions"
|
|
@@ -19,10 +19,10 @@ logger = logging.getLogger(__name__)
|
|
|
19
19
|
GITHUB_RELEASE_URL = (
|
|
20
20
|
"https://github.com/PrimeIntellect-ai/prime-agent/releases/download"
|
|
21
21
|
)
|
|
22
|
-
PRIME_AGENT_COMMIT: Literal["
|
|
23
|
-
"
|
|
22
|
+
PRIME_AGENT_COMMIT: Literal["a7d791bc1be09793ed5f3ec05bf4cccbc60679ea"] = (
|
|
23
|
+
"a7d791bc1be09793ed5f3ec05bf4cccbc60679ea"
|
|
24
24
|
)
|
|
25
|
-
PRIME_AGENT_VERSION = "0.9.
|
|
25
|
+
PRIME_AGENT_VERSION = "0.9.5"
|
|
26
26
|
PRIME_AGENT_DIR = "/var/tmp/vf-prime-agent"
|
|
27
27
|
STATE_ROOT = "/tmp/vf-prime-agent-runs"
|
|
28
28
|
PROVIDER = "intercept"
|
|
@@ -50,10 +50,10 @@ for tarball in "$agent_tarball" "$ai_tarball" "$core_tarball" "$tui_tarball"; do
|
|
|
50
50
|
"$release_url/$tarball" -o "$download_dir/$tarball"
|
|
51
51
|
done
|
|
52
52
|
printf '%s %s\n' \
|
|
53
|
-
'
|
|
54
|
-
'
|
|
55
|
-
'
|
|
56
|
-
'
|
|
53
|
+
'349f1682c7909550842f1b04a71ba95814341b136474ade736df93f8ec006876' "$agent_tarball" \
|
|
54
|
+
'9ad0184b7b5f3d5c3b1677c0663335de32c092cbdb72697aeaf997f17910bb92' "$ai_tarball" \
|
|
55
|
+
'59aafeffb4b64eb997399d56a03f529b2f6604b228e5674372195277db52f04a' "$core_tarball" \
|
|
56
|
+
'e49f41170edfd1d0e72418729d2c117969ccf8df784dc55fe8c1a9d91778e103' "$tui_tarball" \
|
|
57
57
|
> "$download_dir/SHA256SUMS"
|
|
58
58
|
(cd "$download_dir" && sha256sum -c SHA256SUMS)
|
|
59
59
|
mkdir "$download_dir/package-root"
|
|
@@ -87,8 +87,8 @@ PRIME_AGENT_BOOTSTRAP_TOOLS_ON_INSTALL=1 npm install -g \
|
|
|
87
87
|
|
|
88
88
|
|
|
89
89
|
class PrimeAgentHarnessConfig(HarnessConfig):
|
|
90
|
-
commit: Literal["
|
|
91
|
-
"""Prime Agent
|
|
90
|
+
commit: Literal["a7d791bc1be09793ed5f3ec05bf4cccbc60679ea"] = PRIME_AGENT_COMMIT
|
|
91
|
+
"""Prime Agent release commit to install."""
|
|
92
92
|
|
|
93
93
|
autonomous: bool = False
|
|
94
94
|
"""Enable Prime Agent's autonomous continuation loop."""
|
|
@@ -69,7 +69,7 @@ class CompactionConfig(BaseConfig):
|
|
|
69
69
|
|
|
70
70
|
class RLMHarnessConfig(HarnessConfig):
|
|
71
71
|
version: str = Field(
|
|
72
|
-
default="
|
|
72
|
+
default="fc5734d20c5fe74f6ab527ecb7ab10299f3bb965", min_length=1
|
|
73
73
|
)
|
|
74
74
|
"""Git ref (branch, tag, or commit) of nano-rlm to install. Must know every
|
|
75
75
|
field this harness puts on the wire, i.e. be at least the default ref."""
|
verifiers/v1/rollout.py
CHANGED
|
@@ -485,7 +485,9 @@ class Rollout:
|
|
|
485
485
|
)
|
|
486
486
|
if self._collect_artifacts and not trace.state.artifacts:
|
|
487
487
|
trace.state.artifacts = await collect(
|
|
488
|
-
runtime,
|
|
488
|
+
runtime,
|
|
489
|
+
self.task.data.artifacts,
|
|
490
|
+
max_bytes=self.task.data.artifact_max_bytes,
|
|
489
491
|
)
|
|
490
492
|
now = time.time()
|
|
491
493
|
trace.timing.finalize.end = now
|
verifiers/v1/runtimes/prime.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"""Remote Prime VM sandbox runtime."""
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
-
import base64
|
|
5
4
|
import contextlib
|
|
5
|
+
import io
|
|
6
6
|
import logging
|
|
7
7
|
import math
|
|
8
8
|
import shlex
|
|
@@ -329,19 +329,30 @@ class PrimeRuntime(Runtime):
|
|
|
329
329
|
async def _read(self, path: str, max_bytes: int | None = None) -> bytes:
|
|
330
330
|
if max_bytes is not None:
|
|
331
331
|
try:
|
|
332
|
-
#
|
|
333
|
-
|
|
332
|
+
# Stream binary output: execute_command buffers base64 text for the
|
|
333
|
+
# entire file. Bound the source read and the host buffer independently.
|
|
334
|
+
process = await self._client.open_process(
|
|
334
335
|
self.info.id,
|
|
335
|
-
f"
|
|
336
|
+
f"head -c {max_bytes} -- {shlex.quote(path)}",
|
|
336
337
|
working_dir=self.config.workdir,
|
|
337
338
|
env=self.process_env({}),
|
|
338
|
-
timeout=EFFECTIVELY_UNBOUNDED_SECONDS,
|
|
339
339
|
)
|
|
340
|
+
async with contextlib.aclosing(process):
|
|
341
|
+
with io.BytesIO() as data:
|
|
342
|
+
async for chunk in process.stdout:
|
|
343
|
+
if data.tell() + len(chunk) > max_bytes:
|
|
344
|
+
raise SandboxError(
|
|
345
|
+
"read stream exceeded its byte limit"
|
|
346
|
+
)
|
|
347
|
+
data.write(chunk)
|
|
348
|
+
stderr = b""
|
|
349
|
+
async for chunk in process.stderr:
|
|
350
|
+
stderr = (stderr + chunk)[-500:]
|
|
351
|
+
if await process.wait():
|
|
352
|
+
raise SandboxError(stderr.decode(errors="replace").strip())
|
|
353
|
+
return data.getvalue()
|
|
340
354
|
except Exception as exc:
|
|
341
355
|
raise SandboxError(f"read {path!r}: {exc}") from exc
|
|
342
|
-
if result.exit_code:
|
|
343
|
-
raise SandboxError(f"read {path!r}: {result.stderr.strip()[-500:]}")
|
|
344
|
-
return base64.b64decode(result.stdout)
|
|
345
356
|
# Avoid background-job log limits and base64 overhead by downloading binary data directly.
|
|
346
357
|
# The temporary file is removed on every exit, and its byte read stays off the event loop.
|
|
347
358
|
target = (
|
verifiers/v1/task.py
CHANGED
|
@@ -25,7 +25,7 @@ from verifiers.v1.configs.task import TaskConfig
|
|
|
25
25
|
from verifiers.v1.errors import TaskError, boundary
|
|
26
26
|
from verifiers.v1.state import StateT
|
|
27
27
|
from verifiers.v1.types import Messages, content_text
|
|
28
|
-
from verifiers.v1.utils.artifacts import Artifact
|
|
28
|
+
from verifiers.v1.utils.artifacts import MAX_ARTIFACT_BYTES, Artifact
|
|
29
29
|
from verifiers.v1.utils.decorators import (
|
|
30
30
|
discover_decorated,
|
|
31
31
|
invoke_all,
|
|
@@ -115,6 +115,8 @@ class TaskData(BaseModel):
|
|
|
115
115
|
on top of the implicitly collected `/logs/artifacts/` convention dir. Declare
|
|
116
116
|
runtime outputs that must cross that boundary. A declared path that is missing at
|
|
117
117
|
collection time fails the rollout."""
|
|
118
|
+
artifact_max_bytes: int = Field(MAX_ARTIFACT_BYTES, gt=0)
|
|
119
|
+
"""Total byte limit for collected artifact archives, including the convention dir."""
|
|
118
120
|
|
|
119
121
|
timeout: TaskTimeout = TaskTimeout()
|
|
120
122
|
resources: TaskResources = TaskResources()
|
|
@@ -40,7 +40,7 @@ from verifiers.v1.state import State
|
|
|
40
40
|
from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout
|
|
41
41
|
from verifiers.v1.taskset import Taskset
|
|
42
42
|
from verifiers.v1.trace import Trace
|
|
43
|
-
from verifiers.v1.utils.artifacts import Artifact, collect
|
|
43
|
+
from verifiers.v1.utils.artifacts import MAX_ARTIFACT_BYTES, Artifact, collect
|
|
44
44
|
from verifiers.v1.utils.decorators import reward
|
|
45
45
|
|
|
46
46
|
logger = logging.getLogger(__name__)
|
|
@@ -61,6 +61,8 @@ class HarborTaskConfig(TaskConfig):
|
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
class HarborConfig(TasksetConfig):
|
|
64
|
+
artifact_max_bytes: int = Field(MAX_ARTIFACT_BYTES, gt=0)
|
|
65
|
+
"""Total byte limit for artifact archives transferred out of each solver runtime."""
|
|
64
66
|
task: HarborTaskConfig = HarborTaskConfig()
|
|
65
67
|
dataset: str = "harbor/hello-world"
|
|
66
68
|
"""A Harbor Hub package id ("org/name" or "org/name@ref"), where ref is a
|
|
@@ -272,7 +274,9 @@ class HarborTask(Task[HarborData, State, HarborTaskConfig]):
|
|
|
272
274
|
f"{hook.command}\n{detail}"
|
|
273
275
|
)
|
|
274
276
|
if not self.scoring_deferred:
|
|
275
|
-
trace.state.artifacts = await collect(
|
|
277
|
+
trace.state.artifacts = await collect(
|
|
278
|
+
runtime, self.data.artifacts, max_bytes=self.data.artifact_max_bytes
|
|
279
|
+
)
|
|
276
280
|
|
|
277
281
|
async def stage_verifier(self, trace: Trace, runtime: Runtime) -> None:
|
|
278
282
|
if any(
|
|
@@ -582,6 +586,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
|
|
|
582
586
|
)
|
|
583
587
|
return HarborData(
|
|
584
588
|
idx=idx,
|
|
589
|
+
artifact_max_bytes=harbor_config.artifact_max_bytes,
|
|
585
590
|
name=harbor_task.name,
|
|
586
591
|
description=task.description if task else None,
|
|
587
592
|
prompt=harbor_task.instruction.strip(),
|
verifiers/v1/utils/artifacts.py
CHANGED
|
@@ -21,7 +21,7 @@ ARTIFACTS_DIR = "/logs/artifacts"
|
|
|
21
21
|
"""Implicit artifact directory; tasks that write here need no declaration."""
|
|
22
22
|
|
|
23
23
|
MAX_ARTIFACT_BYTES = 32 * 1024 * 1024
|
|
24
|
-
"""
|
|
24
|
+
"""Default ceiling per collection. Sized for a delta, not a tree: the grading box boots from the
|
|
25
25
|
agent's image, so the repo is already there and only its output has to travel."""
|
|
26
26
|
|
|
27
27
|
|
|
@@ -35,7 +35,10 @@ class Artifact(BaseModel):
|
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
async def collect(
|
|
38
|
-
runtime: Runtime,
|
|
38
|
+
runtime: Runtime,
|
|
39
|
+
artifacts: list[Artifact] | None = None,
|
|
40
|
+
*,
|
|
41
|
+
max_bytes: int = MAX_ARTIFACT_BYTES,
|
|
39
42
|
) -> dict[str, bytes | None]:
|
|
40
43
|
"""Tar the convention dir and every declared path out of `runtime`.
|
|
41
44
|
|
|
@@ -114,7 +117,7 @@ async def collect(
|
|
|
114
117
|
)
|
|
115
118
|
existence.extend(output.splitlines())
|
|
116
119
|
collected: dict[str, bytes | None] = {}
|
|
117
|
-
budget =
|
|
120
|
+
budget = max_bytes
|
|
118
121
|
for artifact, exists in zip(entries, existence, strict=True):
|
|
119
122
|
source = artifact.source
|
|
120
123
|
if exists != "1":
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: verifiers
|
|
3
|
-
Version: 0.3.2.
|
|
3
|
+
Version: 0.3.2.dev117
|
|
4
4
|
Summary: Verifiers: Environments for LLM Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
|
|
6
6
|
Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
|
|
@@ -8,11 +8,11 @@ verifiers/v1/errors.py,sha256=uQySmCUrjDYFkBIii1vBFPiEcPFPQZ9EGpas_6HvcQI,5279
|
|
|
8
8
|
verifiers/v1/graph.py,sha256=b3Z3lCnsALnAvY82pDnjhODFYakkeLV_hXgGOvbryrQ,40737
|
|
9
9
|
verifiers/v1/harness.py,sha256=EmOmoen9ACsEgoUoS9-A7jpS2RyDg6ZlDbGo4APHU8E,15367
|
|
10
10
|
verifiers/v1/judge.py,sha256=uqn-djkK2_7Q3kOjkDo6ipOvV_HOvuLa9y6AoDjRux8,9618
|
|
11
|
-
verifiers/v1/rollout.py,sha256=
|
|
11
|
+
verifiers/v1/rollout.py,sha256=yN6MoQ3ZiXoLUhg4MxKAmM1zDpukyqyGZHsc0tushfc,24515
|
|
12
12
|
verifiers/v1/semantic.py,sha256=VQ_buw6Ya6QtRoh_fiYI5Avs5RmIQCfWH0H0bQc4aoc,4704
|
|
13
13
|
verifiers/v1/session.py,sha256=7iarEDRH2epID_O8ai-1HwveXze_o8shnLw9pFMFX5o,18900
|
|
14
14
|
verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
|
|
15
|
-
verifiers/v1/task.py,sha256=
|
|
15
|
+
verifiers/v1/task.py,sha256=j7pX-MMZwixSIp4344V5KDI443VGRL4YEs8S6Ukwmr8,12599
|
|
16
16
|
verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
|
|
17
17
|
verifiers/v1/trace.py,sha256=q88vHOr3ZwvhzXOneBx00ZQ82fry7UT4DCwidQnbk3Q,32120
|
|
18
18
|
verifiers/v1/types.py,sha256=PH1SxRWXOjPCgQD9aZgM18_bECvFSNQ25gkCbulmTKU,9857
|
|
@@ -89,31 +89,31 @@ verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFB
|
|
|
89
89
|
verifiers/v1/harnesses/bash/harness.py,sha256=nsjLKFMGFSya4s9fMHiNVrO92RRCTS3VHqTK3rv3AFA,5233
|
|
90
90
|
verifiers/v1/harnesses/browser_use/__init__.py,sha256=yxm2IG5TDLadxZEkX8SL26XWbcPcODXcrJfPfm1GqwU,171
|
|
91
91
|
verifiers/v1/harnesses/browser_use/harness.py,sha256=sIPRxF-dxiB5xXddCCCOMP6BF1kO01Azyr9AR16x4aw,5582
|
|
92
|
-
verifiers/v1/harnesses/browser_use/program.py,sha256=
|
|
92
|
+
verifiers/v1/harnesses/browser_use/program.py,sha256=hhQSU1-F3BtkBkHcddl_K7HvsfnVHjU-FGtietZeamg,10052
|
|
93
93
|
verifiers/v1/harnesses/claude_code/__init__.py,sha256=JkZQylMSA3o1fvBx_NM41y_umuahmOSqFjTrJ2hryTw,171
|
|
94
|
-
verifiers/v1/harnesses/claude_code/harness.py,sha256=
|
|
94
|
+
verifiers/v1/harnesses/claude_code/harness.py,sha256=T0y0xB1_Pc_oAUo0s7WEwUJPttpi9pv45JxL2p-eS8w,4245
|
|
95
95
|
verifiers/v1/harnesses/codex/__init__.py,sha256=ocyBvlpSO8c3pT6D1ebrQohoXYESQcCyPT5kN3j1-N4,132
|
|
96
|
-
verifiers/v1/harnesses/codex/harness.py,sha256=
|
|
96
|
+
verifiers/v1/harnesses/codex/harness.py,sha256=ijd_GbLp1tRLKJmYnKYnKCdy4gEFw6GWQczfjEuvy2k,7493
|
|
97
97
|
verifiers/v1/harnesses/hermes_agent/__init__.py,sha256=k84Y8UTTf1KOw5LGqPb2dk-hYFagPevDJBOF7qsR_vo,176
|
|
98
|
-
verifiers/v1/harnesses/hermes_agent/harness.py,sha256=
|
|
99
|
-
verifiers/v1/harnesses/hermes_agent/program.py,sha256=
|
|
98
|
+
verifiers/v1/harnesses/hermes_agent/harness.py,sha256=iD8Ol5zRn17jjL3sITHRPzxzmboPiZ44un-XDM3avk4,4710
|
|
99
|
+
verifiers/v1/harnesses/hermes_agent/program.py,sha256=txA-Enc-bUZ9fYBuW6ImOywiTquQUgH5iCTX1Lit2H4,573
|
|
100
100
|
verifiers/v1/harnesses/kimi_code/__init__.py,sha256=1BEotSfuzmVcE7UrxHeAREM5d7J20cGkonHuowlrzu4,161
|
|
101
|
-
verifiers/v1/harnesses/kimi_code/harness.py,sha256=
|
|
101
|
+
verifiers/v1/harnesses/kimi_code/harness.py,sha256=f_cagQif4IJ2gw7n65oU0oe0FdPNt64U1t6zOk00WbU,4332
|
|
102
102
|
verifiers/v1/harnesses/mini_swe_agent/__init__.py,sha256=1OBu2Y-XqL72-dGhFdgB6sr5uX_peEEXT9vEjPy1ovU,182
|
|
103
103
|
verifiers/v1/harnesses/mini_swe_agent/harness.py,sha256=HKGtlDyhXDJqNtDFeb3D4JYUerIgXK6HAmyqOm1mkRo,2473
|
|
104
|
-
verifiers/v1/harnesses/mini_swe_agent/program.py,sha256=
|
|
104
|
+
verifiers/v1/harnesses/mini_swe_agent/program.py,sha256=VvF0q-0DD3_h_KFmMBP8zINQ8I7_w7R7BlmUi-AV4j4,168
|
|
105
105
|
verifiers/v1/harnesses/null/__init__.py,sha256=XDeKOPeoXUQi9US9_Z3_yH9YaZ4EiOMKJmUtNNdtD7s,127
|
|
106
106
|
verifiers/v1/harnesses/null/harness.py,sha256=Bq19j8Bh_d4VvpFx9nwv6BKnG1tKweEk-2BSmaXWLkU,1330
|
|
107
107
|
verifiers/v1/harnesses/openclaw/__init__.py,sha256=kNyfPB1sGwEf4K3mTfKN0VEyxun3SGvNpvArVKvH-x8,160
|
|
108
|
-
verifiers/v1/harnesses/openclaw/harness.py,sha256=
|
|
108
|
+
verifiers/v1/harnesses/openclaw/harness.py,sha256=NrvZw2v2QSDQIVfZ-m9Vunvs18mZfOSw_m7HKbcxUmg,9432
|
|
109
109
|
verifiers/v1/harnesses/pi/__init__.py,sha256=1mSnzOf-RdEchdeg6IHtcdobG5KIpAhuTdUIgsCIn74,117
|
|
110
|
-
verifiers/v1/harnesses/pi/harness.py,sha256=
|
|
110
|
+
verifiers/v1/harnesses/pi/harness.py,sha256=DtTD-hnGtBdqbsmbPwrb1I-euZfczCCnCleh2C0smsU,7030
|
|
111
111
|
verifiers/v1/harnesses/prime_agent/__init__.py,sha256=xfPC4RrlnyYWaf38tPAOZq4YL6M55lMpj5BoRLmaK8I,171
|
|
112
|
-
verifiers/v1/harnesses/prime_agent/harness.py,sha256=
|
|
112
|
+
verifiers/v1/harnesses/prime_agent/harness.py,sha256=Zs_f5H2hQgDzUS4zwArM5UxCYJI5Zo0ELmayn1oBkK0,10663
|
|
113
113
|
verifiers/v1/harnesses/rlm/__init__.py,sha256=hwx51xTdEWzzYgMj-p4ETNcptoub3IPMDbEdJ4uB8Ns,122
|
|
114
|
-
verifiers/v1/harnesses/rlm/harness.py,sha256=
|
|
114
|
+
verifiers/v1/harnesses/rlm/harness.py,sha256=R5ztVFy6PFPTn8MifnHhjBw0TzdjHTnumnh6Vlg2B1w,13223
|
|
115
115
|
verifiers/v1/harnesses/terminus_2/__init__.py,sha256=l1pwfg1HyVDEqT9efdnFWcXqaZwtKwe7TrplkyxZSA0,166
|
|
116
|
-
verifiers/v1/harnesses/terminus_2/harness.py,sha256=
|
|
116
|
+
verifiers/v1/harnesses/terminus_2/harness.py,sha256=cKQJluW01mkxEccTLFwSWsVMavyiwq_w-NA4Hb_JkGU,3052
|
|
117
117
|
verifiers/v1/harnesses/terminus_2/program.py,sha256=X8MZdyItU3gbXGXnGOlJ9LWcUTnqIDaxlhkZvoRiqKY,2703
|
|
118
118
|
verifiers/v1/harnesses/utils/__init__.py,sha256=eTS-Ft2HwoLiYn5u1t_KueLmaZB_W12cdYmcVsE-G5A,364
|
|
119
119
|
verifiers/v1/harnesses/utils/compaction.py,sha256=DeY6CYLEsv7-vJm26sT2mRaPHu3W2ywiZhvFr_pi6Hg,10495
|
|
@@ -145,7 +145,7 @@ verifiers/v1/runtimes/base.py,sha256=IKmrJSJltWX9M1nPUZExKluPJM4N82VsnqQGFz0ECV8
|
|
|
145
145
|
verifiers/v1/runtimes/container.py,sha256=puQD8X9acP6GefoCNEnq40POi-nXPD-729bUjTVShfM,9554
|
|
146
146
|
verifiers/v1/runtimes/limiters.py,sha256=C6cWDD4pwyFwUY4QFke3oiMXwydSNIfSbJSIglUC2kE,3062
|
|
147
147
|
verifiers/v1/runtimes/modal.py,sha256=rFbxdDBi_VIiDA67XyZg-AOsc_XkV8aQWSKnTUAgCmU,17506
|
|
148
|
-
verifiers/v1/runtimes/prime.py,sha256=
|
|
148
|
+
verifiers/v1/runtimes/prime.py,sha256=BXwDK-eurEzOKLY1ezuC0UYZdEoI0Y9Cfcd0IhKjMPc,17586
|
|
149
149
|
verifiers/v1/runtimes/subprocess.py,sha256=QmpQ235_xrX6gowrr8557VgbydOXpLbfFq0KKGfJThk,8774
|
|
150
150
|
verifiers/v1/runtimes/docker/__init__.py,sha256=2wceP-8V-w0dQBEeLtoJFfCZ135DxP2rcdcUBmSWiNU,18543
|
|
151
151
|
verifiers/v1/runtimes/docker/egress.py,sha256=wtKWTL2_zFidxMR7lDRmfevQlOsl2Iyk88ebdO4Kqqw,24566
|
|
@@ -159,7 +159,7 @@ verifiers/v1/serve/types.py,sha256=Qw5IjiJhsZz4giSXUBi9dfInBnQZ5_xiUmgH2qp8jPI,1
|
|
|
159
159
|
verifiers/v1/tasksets/__init__.py,sha256=5hTKSPLXSZTgKBbeKQI7FEp06c0ZsaQfuwvrvVFLgb8,712
|
|
160
160
|
verifiers/v1/tasksets/harbor/__init__.py,sha256=JxThzHFYEkuSfQbVf3pEnjordgdfe4MU-QOsSSbyovU,326
|
|
161
161
|
verifiers/v1/tasksets/harbor/env.py,sha256=ATMkfwmrE_2jpOOffbjLQ91feG7TOtefBGlpwWbRq8c,3604
|
|
162
|
-
verifiers/v1/tasksets/harbor/taskset.py,sha256=
|
|
162
|
+
verifiers/v1/tasksets/harbor/taskset.py,sha256=35ljhNapn1EOlaYFTuVZxAMVU0KDfmFxQOGOcUk2Ow0,34082
|
|
163
163
|
verifiers/v1/tasksets/harbor/toolset.py,sha256=_7HXu-uuvqzhmJ1EX6c3KWLg0g0eRrLjCq1kBsZhIwo,3159
|
|
164
164
|
verifiers/v1/tasksets/lean/__init__.py,sha256=oyv-GbJQBvCyGwo7Ysxm2NjZW-aX4j1uDoHU0f-E7xQ,767
|
|
165
165
|
verifiers/v1/tasksets/lean/scoring.py,sha256=sfzsT6MUK0zdMgAQ57_P_0CBaPHy3M7go71QMr5hDrs,10599
|
|
@@ -175,7 +175,7 @@ verifiers/v1/tasksets/textarena/__init__.py,sha256=Os2OlBY_pSH0B1DP32RitumgSLpq2
|
|
|
175
175
|
verifiers/v1/tasksets/textarena/taskset.py,sha256=9fa_unlFiFZyuN4I02rW3x4qyRnFWXqwTUZTloQefu0,4433
|
|
176
176
|
verifiers/v1/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
177
177
|
verifiers/v1/utils/aio.py,sha256=0yNFOqB2oO6Qktc6NI_ZSAGuo1aHsIK_wuT1BtmoZ3I,1532
|
|
178
|
-
verifiers/v1/utils/artifacts.py,sha256=
|
|
178
|
+
verifiers/v1/utils/artifacts.py,sha256=3so4KLKlrtF2sM_jMZUAyjTHWnfuUltGX1MrrLns-tI,9315
|
|
179
179
|
verifiers/v1/utils/compile.py,sha256=5pxy31wx2Q-3pQjOO5B9SH6OT-wwb8r4F1LAl6HMeQA,5835
|
|
180
180
|
verifiers/v1/utils/decorators.py,sha256=vhoN6YoKb15-Fkqmc9RaNlUIbt9WRvNK4zAn2bAxfS4,5367
|
|
181
181
|
verifiers/v1/utils/format.py,sha256=yX160sGewMMZn9iJqTHRbYYt4Z9pHasjMlFT1dXrKD8,2446
|
|
@@ -192,8 +192,8 @@ verifiers/v1/utils/prime.py,sha256=UTYRjp9cbjNb6CVmBHda-1wWZAIIfxNOmTyNuT7_wL4,9
|
|
|
192
192
|
verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
|
|
193
193
|
verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
|
|
194
194
|
verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
|
|
195
|
-
verifiers-0.3.2.
|
|
196
|
-
verifiers-0.3.2.
|
|
197
|
-
verifiers-0.3.2.
|
|
198
|
-
verifiers-0.3.2.
|
|
199
|
-
verifiers-0.3.2.
|
|
195
|
+
verifiers-0.3.2.dev117.dist-info/METADATA,sha256=LIgiQlG8mlmytpzxUpHQpGLzD7F-bOpNG8AIKhhGyFA,4237
|
|
196
|
+
verifiers-0.3.2.dev117.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
197
|
+
verifiers-0.3.2.dev117.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
|
|
198
|
+
verifiers-0.3.2.dev117.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
|
|
199
|
+
verifiers-0.3.2.dev117.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|