reeval 0.6.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +265 -0
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +30 -307
- package/dist/{agent-DKuFnZPM.js → agent-B72lZ6w5.js} +6 -6
- package/dist/{agent-DKuFnZPM.js.map → agent-B72lZ6w5.js.map} +1 -1
- package/dist/{agent-oirN_8IS.js → agent-Cfj1wD4D.js} +5 -5
- package/dist/{agent-oirN_8IS.js.map → agent-Cfj1wD4D.js.map} +1 -1
- package/dist/{analysis-BV3TCznv.js → analysis-BTNnKh1w.js} +4 -4
- package/dist/{analysis-BV3TCznv.js.map → analysis-BTNnKh1w.js.map} +1 -1
- package/dist/{assemble-DYCDgbTq.js → assemble-BbFVjHeS.js} +125 -14
- package/dist/assemble-BbFVjHeS.js.map +1 -0
- package/dist/assemble-CXiTVyHD.js +2 -0
- package/dist/bridge/index.mjs +66 -7
- package/dist/bridge/package.json +3 -3
- package/dist/bridge/pnpm-lock.yaml +84 -89
- package/dist/capture-BXhBfmsM.js +1206 -0
- package/dist/capture-BXhBfmsM.js.map +1 -0
- package/dist/capture-N7r3yeak.js +2 -0
- package/dist/{cli-BIhzl_v_.js → cli-BMjTKY-D.js} +151 -193
- package/dist/cli-BMjTKY-D.js.map +1 -0
- package/dist/{cli-spec-DUeyuSIQ.js → cli-spec-DSvKsOK4.js} +25 -11
- package/dist/cli-spec-DSvKsOK4.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-CwzU9CiW.js +256 -0
- package/dist/client-CwzU9CiW.js.map +1 -0
- package/dist/{command-B8tcwUJk.js → command-BK4kOgkH.js} +15 -15
- package/dist/{command-B8tcwUJk.js.map → command-BK4kOgkH.js.map} +1 -1
- package/dist/{command-fJpZK205.js → command-BdstSEHu.js} +7 -7
- package/dist/{command-fJpZK205.js.map → command-BdstSEHu.js.map} +1 -1
- package/dist/{command-DuJBqcwu.js → command-BqJtFiPy.js} +153 -36
- package/dist/command-BqJtFiPy.js.map +1 -0
- package/dist/command-CTVGuH8y.js +2 -0
- package/dist/{command-KoOJ0ntD.js → command-JPGNzFsA.js} +9 -9
- package/dist/{command-KoOJ0ntD.js.map → command-JPGNzFsA.js.map} +1 -1
- package/dist/{commands-CHdTfPB6.js → commands-B618jg3C.js} +6 -6
- package/dist/{commands-CHdTfPB6.js.map → commands-B618jg3C.js.map} +1 -1
- package/dist/{commands-DE604ldB.js → commands-Duo2ur3r.js} +1 -1
- package/dist/{config-7-gXOgQf.js → config-B-E0xRkf.js} +4 -4
- package/dist/{config-7-gXOgQf.js.map → config-B-E0xRkf.js.map} +1 -1
- package/dist/db-CnoUy-FZ.js +3 -0
- package/dist/{db-DvkzBhM6.js → db-DFbtGIYy.js} +84 -37
- package/dist/db-DFbtGIYy.js.map +1 -0
- package/dist/{dist-DsZNsoD3.js → dist-CTzThfu_.js} +3 -3
- package/dist/{dist-DsZNsoD3.js.map → dist-CTzThfu_.js.map} +1 -1
- package/dist/{docker-BBOMThRi.js → docker-2qns7hJ2.js} +4 -4
- package/dist/{docker-BBOMThRi.js.map → docker-2qns7hJ2.js.map} +1 -1
- package/dist/{docker-CiNVc7di.js → docker-D6t23fZl.js} +1 -1
- package/dist/{docker-CzFiN2AZ.js → docker-IFRW8K5f.js} +2 -2
- package/dist/{docker-CzFiN2AZ.js.map → docker-IFRW8K5f.js.map} +1 -1
- package/dist/{doctor-C06kYs2r.js → doctor-B6oYTPoR.js} +11 -11
- package/dist/{doctor-C06kYs2r.js.map → doctor-B6oYTPoR.js.map} +1 -1
- package/dist/{egress-De64jpeQ.js → egress-IUKdeayc.js} +17 -10
- package/dist/egress-IUKdeayc.js.map +1 -0
- package/dist/fake-instructions-CUI33I_F.js +32 -0
- package/dist/fake-instructions-CUI33I_F.js.map +1 -0
- package/dist/{files-BqIF7iu4.js → files-B6aVVGrk.js} +3 -3
- package/dist/{files-BqIF7iu4.js.map → files-B6aVVGrk.js.map} +1 -1
- package/dist/files-BpYgF2z0.js +2 -0
- package/dist/{files-Cq99wif-.js → files-CMSUXAEg.js} +16 -5
- package/dist/{files-Cq99wif-.js.map → files-CMSUXAEg.js.map} +1 -1
- package/dist/files-YCCVaTQC.js +2 -0
- package/dist/{git-Dnl-kXgD.js → git-EmdFJ4DY.js} +2 -2
- package/dist/{git-Dnl-kXgD.js.map → git-EmdFJ4DY.js.map} +1 -1
- package/dist/{harness-3QUEWx8G.js → harness-iqWeJQMM.js} +35 -12
- package/dist/harness-iqWeJQMM.js.map +1 -0
- package/dist/harness.d.ts +22 -11
- package/dist/harness.js +1 -1
- package/dist/{header-IE-uIvX0.js → header-_F0EWob3.js} +2 -2
- package/dist/{header-IE-uIvX0.js.map → header-_F0EWob3.js.map} +1 -1
- package/dist/{images-B8Mp_VXZ.js → images-CscddrJU.js} +6 -6
- package/dist/{images-B8Mp_VXZ.js.map → images-CscddrJU.js.map} +1 -1
- package/dist/index.d.ts +13 -8
- package/dist/index.js +99 -24
- package/dist/index.js.map +1 -1
- package/dist/init-DBHyi92L.js +2 -0
- package/dist/{init-B8mw86E6.js → init-NQfIvi-0.js} +4 -4
- package/dist/{init-B8mw86E6.js.map → init-NQfIvi-0.js.map} +1 -1
- package/dist/{invocations-DHC946b-.js → invocations-nPvrlZPj.js} +3 -3
- package/dist/{invocations-DHC946b-.js.map → invocations-nPvrlZPj.js.map} +1 -1
- package/dist/{judge-7WjNdU9z.js → judge-CeYPN8P_.js} +2 -2
- package/dist/{judge-7WjNdU9z.js.map → judge-CeYPN8P_.js.map} +1 -1
- package/dist/judge.d.ts +5 -4
- package/dist/judge.js +4 -3
- package/dist/judge.js.map +1 -1
- package/dist/{live-DZkCtmNP.js → live-S0qODb6o.js} +2 -2
- package/dist/{live-DZkCtmNP.js.map → live-S0qODb6o.js.map} +1 -1
- package/dist/model-BbLGRtOs.js +2 -0
- package/dist/{model-CtrJdFi6.js → model-BndnmcLm.js} +241 -26
- package/dist/model-BndnmcLm.js.map +1 -0
- package/dist/{model-Cp6NoAT3.js → model-Cl507trk.js} +8 -7
- package/dist/model-Cl507trk.js.map +1 -0
- package/dist/{model-Bv8uUZ07.js → model-DEQo3FkZ.js} +6 -6
- package/dist/model-DEQo3FkZ.js.map +1 -0
- package/dist/{model-BeoRxCcJ.js → model-DMOraCcm.js} +1 -1
- package/dist/{opencode-providers-DjlA5fuC.js → opencode-providers-ByD85Q0l.js} +21 -7
- package/dist/{opencode-providers-DjlA5fuC.js.map → opencode-providers-ByD85Q0l.js.map} +1 -1
- package/dist/{options-CWYB613o.js → options-DPUlm4FI.js} +152 -29
- package/dist/options-DPUlm4FI.js.map +1 -0
- package/dist/pairs-DDsjgm5F.js +149 -0
- package/dist/pairs-DDsjgm5F.js.map +1 -0
- package/dist/{plan-CsPE24pp.js → plan-PPc4-6xd.js} +35 -18
- package/dist/plan-PPc4-6xd.js.map +1 -0
- package/dist/{preflight-r0DgibQV.js → preflight-CddycK92.js} +3 -3
- package/dist/{preflight-r0DgibQV.js.map → preflight-CddycK92.js.map} +1 -1
- package/dist/{preflight-DKZjb7ZH.js → preflight-Zumy0adT.js} +1 -1
- package/dist/{prepare-DOkvAa9D.js → prepare-CKdMoC7e.js} +2 -2
- package/dist/{prepare-CTLg3nuk.js → prepare-CsP-057p.js} +4 -4
- package/dist/{prepare-CTLg3nuk.js.map → prepare-CsP-057p.js.map} +1 -1
- package/dist/{provenance--cVNo80L.js → provenance-CsX6PHDR.js} +2 -2
- package/dist/{provenance--cVNo80L.js.map → provenance-CsX6PHDR.js.map} +1 -1
- package/dist/{providers-DNLQzVnc.js → providers-DkwiLKDS.js} +3 -3
- package/dist/{providers-DNLQzVnc.js.map → providers-DkwiLKDS.js.map} +1 -1
- package/dist/{providers-B2cSfewm.js → providers-yXZZcPfx.js} +1 -1
- package/dist/{query-DTtIZaua.js → query-BUcwCYsI.js} +2 -2
- package/dist/{query-DTtIZaua.js.map → query-BUcwCYsI.js.map} +1 -1
- package/dist/query-noGbekGF.js +2 -0
- package/dist/{questions-ZdT3JDyQ.js → questions-BEPj98fZ.js} +27 -118
- package/dist/questions-BEPj98fZ.js.map +1 -0
- package/dist/{record-CbuQH4l1.js → record-BRP9vZrj.js} +21 -1
- package/dist/record-BRP9vZrj.js.map +1 -0
- package/dist/results-BcSHWbgB.js +52 -0
- package/dist/results-BcSHWbgB.js.map +1 -0
- package/dist/resume-DZ9AmfPv.js +406 -0
- package/dist/resume-DZ9AmfPv.js.map +1 -0
- package/dist/{matchers-Bp-OH1vj.js → runtime-Dfb2r-cS.js} +2 -179
- package/dist/runtime-Dfb2r-cS.js.map +1 -0
- package/dist/sandbox.d.ts +1 -1
- package/dist/{server-BFlKK2j6.js → server-Da60oe4b.js} +9 -9
- package/dist/{server-BFlKK2j6.js.map → server-Da60oe4b.js.map} +1 -1
- package/dist/server-DiGR4JxK.js +2 -0
- package/dist/{source-B2ASbTdc.js → source-BVOnL3sn.js} +2 -2
- package/dist/{source-B2ASbTdc.js.map → source-BVOnL3sn.js.map} +1 -1
- package/dist/source-Dqu0jKwx.js +2 -0
- package/dist/src-BHNiP-5A.js +4 -0
- package/dist/{src-CDiGyoDt.js → src-BT4dB5CO.js} +1 -1
- package/dist/src-BT5pwco0.js +3 -0
- package/dist/src-BrMbt1B2.js +3 -0
- package/dist/{src-s-VffoYK.js → src-C1E9HDWO.js} +5 -3
- package/dist/{src-s-VffoYK.js.map → src-C1E9HDWO.js.map} +1 -1
- package/dist/{src-CqsERnCZ.js → src-CxTC5kNI.js} +10 -6
- package/dist/{src-CqsERnCZ.js.map → src-CxTC5kNI.js.map} +1 -1
- package/dist/src-D13vpMh0.js +3 -0
- package/dist/{src-DxFJrMUA.js → src-D95O-Smv.js} +12 -7
- package/dist/{src-DxFJrMUA.js.map → src-D95O-Smv.js.map} +1 -1
- package/dist/{src-CVoCJjEe.js → src-DDNXsD9H.js} +29 -5
- package/dist/{src-CVoCJjEe.js.map → src-DDNXsD9H.js.map} +1 -1
- package/dist/{src-DSE0FWaU.js → src-DVy58-zZ.js} +2 -2
- package/dist/{src-DSE0FWaU.js.map → src-DVy58-zZ.js.map} +1 -1
- package/dist/{src-BHFNhTTQ.js → src-D_cXqmnI.js} +59 -49
- package/dist/{src-BHFNhTTQ.js.map → src-D_cXqmnI.js.map} +1 -1
- package/dist/src-DdRlaiRc.js +3 -0
- package/dist/{src-zOeHvrzR.js → src-TdJuDYqB.js} +13 -13
- package/dist/{src-zOeHvrzR.js.map → src-TdJuDYqB.js.map} +1 -1
- package/dist/src-xczuQbE6.js +3 -0
- package/dist/{state-DBKcUJ2u.js → state-B72h_jEw.js} +14 -14
- package/dist/{state-DBKcUJ2u.js.map → state-B72h_jEw.js.map} +1 -1
- package/dist/state-Bf_Vq_pV.js +2 -0
- package/dist/studio-app/assets/{index-WQPq5K3E.js → index-Bgr-efZm.js} +9 -9
- package/dist/studio-app/index.html +1 -1
- package/dist/{summary-BBABbon9.js → summary-CJdxaRdM.js} +7 -7
- package/dist/{summary-BBABbon9.js.map → summary-CJdxaRdM.js.map} +1 -1
- package/dist/{summary-CahgSiVt.js → summary-k3mhWWXv.js} +8 -4
- package/dist/summary-k3mhWWXv.js.map +1 -0
- package/dist/testing.d.ts +14 -0
- package/dist/testing.js +2 -0
- package/dist/tool-target-BU1JnSE9.js +34 -0
- package/dist/tool-target-BU1JnSE9.js.map +1 -0
- package/dist/{tools-53YHomyk.js → tools-D0gF2QoV.js} +3 -3
- package/dist/{tools-53YHomyk.js.map → tools-D0gF2QoV.js.map} +1 -1
- package/dist/{types-CoAfC-_k.d.ts → types-_LOFdPUf.d.ts} +266 -36
- package/dist/{usage-DsYUqIUM.js → usage-DDF_6tkV.js} +4 -2
- package/dist/{usage-DsYUqIUM.js.map → usage-DDF_6tkV.js.map} +1 -1
- package/dist/view/index.d.ts +19 -6
- package/dist/view/index.js +6 -6
- package/dist/{views-Drl2OXuj.js → views-DZ1YDqey.js} +4 -4
- package/dist/{views-Drl2OXuj.js.map → views-DZ1YDqey.js.map} +1 -1
- package/dist/views-Ds0GXWtX.js +3 -0
- package/dist/{worker-entry-mCctDzig.js → worker-entry-DXeyWY-m.js} +73 -2
- package/dist/worker-entry-DXeyWY-m.js.map +1 -0
- package/dist/{workspace-BT4jd04k.js → workspace-DpXdrslo.js} +3 -3
- package/dist/{workspace-BT4jd04k.js.map → workspace-DpXdrslo.js.map} +1 -1
- package/dist/{workspace-U6M1gKzo.js → workspace-DzNL-3iO.js} +1 -1
- package/package.json +14 -8
- package/dist/assemble-DYCDgbTq.js.map +0 -1
- package/dist/assemble-DykFydSU.js +0 -2
- package/dist/capture-CisAmy-G.js +0 -587
- package/dist/capture-CisAmy-G.js.map +0 -1
- package/dist/capture-DquyqAHC.js +0 -2
- package/dist/cli-BIhzl_v_.js.map +0 -1
- package/dist/cli-spec-DUeyuSIQ.js.map +0 -1
- package/dist/command-CwuwuKJD.js +0 -2
- package/dist/command-DuJBqcwu.js.map +0 -1
- package/dist/db-BVE1kZ2w.js +0 -2
- package/dist/db-DvkzBhM6.js.map +0 -1
- package/dist/egress-De64jpeQ.js.map +0 -1
- package/dist/files-Cs8tbYQY.js +0 -2
- package/dist/files-wUtoO5aA.js +0 -2
- package/dist/harness-3QUEWx8G.js.map +0 -1
- package/dist/init-DDEpamaf.js +0 -2
- package/dist/matchers-Bp-OH1vj.js.map +0 -1
- package/dist/model-Bv8uUZ07.js.map +0 -1
- package/dist/model-C_B5JrhL.js +0 -2
- package/dist/model-Cp6NoAT3.js.map +0 -1
- package/dist/model-CtrJdFi6.js.map +0 -1
- package/dist/options-CWYB613o.js.map +0 -1
- package/dist/plan-CsPE24pp.js.map +0 -1
- package/dist/query-BWc_5Lhe.js +0 -2
- package/dist/questions-ZdT3JDyQ.js.map +0 -1
- package/dist/record-CbuQH4l1.js.map +0 -1
- package/dist/results-CrFs7_In.js +0 -37
- package/dist/results-CrFs7_In.js.map +0 -1
- package/dist/server-HRUHVcHc.js +0 -2
- package/dist/source-Co7h89WK.js +0 -2
- package/dist/src-BE19tji4.js +0 -3
- package/dist/src-Bo8GFUXg.js +0 -3
- package/dist/src-BoziVhKa.js +0 -3
- package/dist/src-CFxHPwQA.js +0 -3
- package/dist/src-CPkDaNxp.js +0 -3
- package/dist/src-D-go5-qz.js +0 -4
- package/dist/state-BaHjYXr1.js +0 -2
- package/dist/stats-BNAHlNMD.js +0 -114
- package/dist/stats-BNAHlNMD.js.map +0 -1
- package/dist/summary-CahgSiVt.js.map +0 -1
- package/dist/views-BhGinAz_.js +0 -3
- package/dist/worker-entry-mCctDzig.js.map +0 -1
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
import { n as UsageError } from "./errors-yDAEwmZo.js";
|
|
2
|
+
import { t as RUN_ID } from "./paths-BmR-7tx3.js";
|
|
3
|
+
import { s as recordFileName } from "./ids-Bp6ut7ZD.js";
|
|
4
|
+
import { c as readableLiveProgress, n as LIVE_FILE, o as liveCounts } from "./live-S0qODb6o.js";
|
|
5
|
+
import { p as scratchDir } from "./files-CMSUXAEg.js";
|
|
6
|
+
import { t as RefError } from "./views-DZ1YDqey.js";
|
|
7
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
|
8
|
+
import { hostname } from "node:os";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { readFile, readdir, stat } from "node:fs/promises";
|
|
11
|
+
function finished(result) {
|
|
12
|
+
const outcome = result.meta?.outcome;
|
|
13
|
+
if (result.state === "skipped") return outcome === "fail" || outcome === "infra_error" ? outcome : "skipped";
|
|
14
|
+
if (outcome === "infra_error") return "infra_error";
|
|
15
|
+
return result.state === "passed" ? "pass" : "fail";
|
|
16
|
+
}
|
|
17
|
+
/** Sharing-violation codes Windows uses while a reader has `to` open. */
|
|
18
|
+
const BUSY = /* @__PURE__ */ new Set([
|
|
19
|
+
"EPERM",
|
|
20
|
+
"EACCES",
|
|
21
|
+
"EBUSY"
|
|
22
|
+
]);
|
|
23
|
+
/** How long a busy rename is retried. A studio poll holds the file for a few milliseconds. */
|
|
24
|
+
const BUSY_MS = 1e3;
|
|
25
|
+
function pause(ms) {
|
|
26
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Renames `from` over `to`. Windows refuses while a reader (`reeval studio`) has `to` open, for a
|
|
30
|
+
* moment: try again for up to a second rather than drop the update.
|
|
31
|
+
*/
|
|
32
|
+
function replace(from, to, rename = renameSync, hooks = {}) {
|
|
33
|
+
const now = hooks.now ?? Date.now;
|
|
34
|
+
const wait = hooks.wait ?? pause;
|
|
35
|
+
const deadline = now() + (hooks.busyMs ?? BUSY_MS);
|
|
36
|
+
for (let attempt = 1;; attempt++) try {
|
|
37
|
+
rename(from, to);
|
|
38
|
+
return;
|
|
39
|
+
} catch (error) {
|
|
40
|
+
const code = error.code;
|
|
41
|
+
if (!code || !BUSY.has(code) || now() >= deadline) throw error;
|
|
42
|
+
wait(Math.min(attempt, 32));
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** Writes the file at once, then on every change and heartbeat. Write errors never fail the run. */
|
|
46
|
+
function writeLiveProgress(options) {
|
|
47
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
48
|
+
const file = join(options.dir, LIVE_FILE);
|
|
49
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
50
|
+
const attempts = new Map(options.planned.map((p) => [p.key, {
|
|
51
|
+
key: p.key,
|
|
52
|
+
evalId: p.id,
|
|
53
|
+
name: p.attempt.name,
|
|
54
|
+
file: p.file,
|
|
55
|
+
cell: p.attempt.cell.key,
|
|
56
|
+
n: p.attempt.n,
|
|
57
|
+
status: p.attempt.skip || p.attempt.plan.skip ? "skipped" : "queued",
|
|
58
|
+
startedAt: null,
|
|
59
|
+
durationMs: null
|
|
60
|
+
}]));
|
|
61
|
+
let phase = "running";
|
|
62
|
+
let stopped = false;
|
|
63
|
+
const read = () => {
|
|
64
|
+
const list = [...attempts.values()];
|
|
65
|
+
return {
|
|
66
|
+
schemaVersion: 1,
|
|
67
|
+
runId: options.runId,
|
|
68
|
+
startedAt: options.startedAt.toISOString(),
|
|
69
|
+
pid: process.pid,
|
|
70
|
+
host: hostname(),
|
|
71
|
+
heartbeatAt: now().toISOString(),
|
|
72
|
+
phase,
|
|
73
|
+
planned: list.length,
|
|
74
|
+
counts: liveCounts(list),
|
|
75
|
+
attempts: list
|
|
76
|
+
};
|
|
77
|
+
};
|
|
78
|
+
const write = () => {
|
|
79
|
+
if (stopped) return;
|
|
80
|
+
try {
|
|
81
|
+
writeFileSync(tmp, JSON.stringify(read()));
|
|
82
|
+
replace(tmp, file);
|
|
83
|
+
} catch {}
|
|
84
|
+
};
|
|
85
|
+
try {
|
|
86
|
+
mkdirSync(options.dir, { recursive: true });
|
|
87
|
+
} catch {}
|
|
88
|
+
write();
|
|
89
|
+
const heartbeat = setInterval(write, options.heartbeatMs ?? 2e3);
|
|
90
|
+
heartbeat.unref();
|
|
91
|
+
const stop = () => {
|
|
92
|
+
stopped = true;
|
|
93
|
+
clearInterval(heartbeat);
|
|
94
|
+
};
|
|
95
|
+
return {
|
|
96
|
+
start(key, at) {
|
|
97
|
+
const a = attempts.get(key);
|
|
98
|
+
if (!a) return;
|
|
99
|
+
a.status = "running";
|
|
100
|
+
a.startedAt = new Date(at).toISOString();
|
|
101
|
+
write();
|
|
102
|
+
},
|
|
103
|
+
end(result) {
|
|
104
|
+
const a = attempts.get(result.key);
|
|
105
|
+
if (!a) return;
|
|
106
|
+
a.status = finished(result);
|
|
107
|
+
a.durationMs = result.durationMs;
|
|
108
|
+
write();
|
|
109
|
+
},
|
|
110
|
+
phase(next) {
|
|
111
|
+
phase = next;
|
|
112
|
+
write();
|
|
113
|
+
},
|
|
114
|
+
stop,
|
|
115
|
+
remove() {
|
|
116
|
+
stop();
|
|
117
|
+
rmSync(file, { force: true });
|
|
118
|
+
rmSync(tmp, { force: true });
|
|
119
|
+
},
|
|
120
|
+
read
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
//#endregion
|
|
124
|
+
//#region src/resume.ts
|
|
125
|
+
/** Other-host / studio: a heartbeat older than this is quiet. A live pid on this host is not. */
|
|
126
|
+
const STALE_MS = 15e3;
|
|
127
|
+
/** Exclusive claim in staging so two CLIs cannot resume the same id through the plan window. */
|
|
128
|
+
const HOLDER_FILE = "holder";
|
|
129
|
+
/** `.reeval/current`: the run id, written when the run starts (after the plan). */
|
|
130
|
+
const currentPath = (root) => join(root, ".reeval", "current");
|
|
131
|
+
const stagingOf = (root, runId) => join(root, ".reeval", "staging", runId);
|
|
132
|
+
function writeCurrent(root, runId) {
|
|
133
|
+
const file = currentPath(root);
|
|
134
|
+
mkdirSync(join(root, ".reeval"), { recursive: true });
|
|
135
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
136
|
+
writeFileSync(tmp, `${runId}\n`);
|
|
137
|
+
try {
|
|
138
|
+
replace(tmp, file);
|
|
139
|
+
} catch (error) {
|
|
140
|
+
rmSync(tmp, { force: true });
|
|
141
|
+
throw error;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
const stillRunning = (runId) => new UsageError(`run ${runId} is still running. Wait for it to finish or stop, then reeval run --resume ${runId}.`);
|
|
145
|
+
/** Same host only: `kill(pid, 0)`. Another host cannot be probed. */
|
|
146
|
+
function processAlive(pid, host) {
|
|
147
|
+
if (host !== hostname()) return false;
|
|
148
|
+
try {
|
|
149
|
+
process.kill(pid, 0);
|
|
150
|
+
return true;
|
|
151
|
+
} catch (error) {
|
|
152
|
+
return error.code !== "ESRCH";
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function readHolder(dir) {
|
|
156
|
+
try {
|
|
157
|
+
const [pidLine, hostLine] = readFileSync(join(dir, HOLDER_FILE), "utf8").split("\n");
|
|
158
|
+
const pid = Number(pidLine);
|
|
159
|
+
if (!Number.isInteger(pid) || pid <= 0 || !hostLine) return null;
|
|
160
|
+
return {
|
|
161
|
+
pid,
|
|
162
|
+
host: hostLine
|
|
163
|
+
};
|
|
164
|
+
} catch {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
const holderLive = (dir) => {
|
|
169
|
+
const holder = readHolder(dir);
|
|
170
|
+
return holder ? processAlive(holder.pid, holder.host) : false;
|
|
171
|
+
};
|
|
172
|
+
function acquireHolder(staging, runId) {
|
|
173
|
+
const file = join(staging, HOLDER_FILE);
|
|
174
|
+
const body = `${process.pid}\n${hostname()}\n`;
|
|
175
|
+
const create = () => writeFileSync(file, body, { flag: "wx" });
|
|
176
|
+
try {
|
|
177
|
+
create();
|
|
178
|
+
return;
|
|
179
|
+
} catch (error) {
|
|
180
|
+
if (error.code !== "EEXIST") throw error;
|
|
181
|
+
}
|
|
182
|
+
if (holderLive(staging)) throw stillRunning(runId);
|
|
183
|
+
try {
|
|
184
|
+
unlinkSync(file);
|
|
185
|
+
} catch {
|
|
186
|
+
throw stillRunning(runId);
|
|
187
|
+
}
|
|
188
|
+
try {
|
|
189
|
+
create();
|
|
190
|
+
} catch (error) {
|
|
191
|
+
if (error.code === "EEXIST") throw stillRunning(runId);
|
|
192
|
+
throw error;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
function readCurrent(root) {
|
|
196
|
+
try {
|
|
197
|
+
const id = readFileSync(currentPath(root), "utf8").trim();
|
|
198
|
+
return RUN_ID.test(id) ? id : null;
|
|
199
|
+
} catch {
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
/** Finished with a real outcome: not queued, not skipped, not a cancel abort. */
|
|
204
|
+
function isCompletedAttempt(outcome, error) {
|
|
205
|
+
if (outcome === "skipped") return false;
|
|
206
|
+
if (error?.kind === "aborted") return false;
|
|
207
|
+
return outcome === "pass" || outcome === "fail" || outcome === "infra_error";
|
|
208
|
+
}
|
|
209
|
+
const completedFromRecord = (record) => record.evals.flatMap((e) => e.attempts.filter((a) => isCompletedAttempt(a.outcome, a.error)).map((a) => a.key));
|
|
210
|
+
function readEvalRecord(path) {
|
|
211
|
+
try {
|
|
212
|
+
return JSON.parse(readFileSync(path, "utf8"));
|
|
213
|
+
} catch {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
function stagingRecordComplete(staging, key) {
|
|
218
|
+
const record = readEvalRecord(join(staging, "evals", recordFileName(key)));
|
|
219
|
+
if (!record) return false;
|
|
220
|
+
return isCompletedAttempt(record.outcome, record.error);
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Attempt keys that already finished in `staging` or in an interrupted/budget scratch, so a
|
|
224
|
+
* resume skips them.
|
|
225
|
+
*/
|
|
226
|
+
function completedKeysFor(planned, staging, previous) {
|
|
227
|
+
const keys = /* @__PURE__ */ new Set();
|
|
228
|
+
const fromPrevious = new Set(previous ? completedFromRecord(previous) : []);
|
|
229
|
+
for (const { key } of planned) if (stagingRecordComplete(staging, key) || fromPrevious.has(key)) keys.add(key);
|
|
230
|
+
return keys;
|
|
231
|
+
}
|
|
232
|
+
function progressLive(progress, now) {
|
|
233
|
+
if (processAlive(progress.pid, progress.host)) return true;
|
|
234
|
+
if (now - Date.parse(progress.heartbeatAt) > STALE_MS) return false;
|
|
235
|
+
return progress.host !== hostname();
|
|
236
|
+
}
|
|
237
|
+
async function readScratch(root, runId) {
|
|
238
|
+
try {
|
|
239
|
+
const text = await readFile(join(scratchDir(root), `${runId}.json`), "utf8");
|
|
240
|
+
const record = JSON.parse(text);
|
|
241
|
+
return record.kind === "es-run" && record.runId === runId ? record : null;
|
|
242
|
+
} catch {
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
async function listCandidates(root, now = Date.now()) {
|
|
247
|
+
const found = /* @__PURE__ */ new Map();
|
|
248
|
+
const stagingRoot = join(root, ".reeval", "staging");
|
|
249
|
+
const names = await readdir(stagingRoot).catch(() => []);
|
|
250
|
+
for (const runId of names.filter((n) => RUN_ID.test(n))) {
|
|
251
|
+
const dir = join(stagingRoot, runId);
|
|
252
|
+
const info = await stat(dir).catch(() => null);
|
|
253
|
+
if (!info?.isDirectory()) continue;
|
|
254
|
+
const text = await readFile(join(dir, LIVE_FILE), "utf8").catch(() => null);
|
|
255
|
+
const progress = text ? readableLiveProgress(JSON.parse(text)) : null;
|
|
256
|
+
const live = (progress ? progressLive(progress, now) : false) || holderLive(dir);
|
|
257
|
+
const startedAt = progress?.startedAt ?? new Date(info.mtimeMs).toISOString();
|
|
258
|
+
found.set(runId, {
|
|
259
|
+
runId,
|
|
260
|
+
startedAt,
|
|
261
|
+
kind: "staging",
|
|
262
|
+
live,
|
|
263
|
+
complete: false,
|
|
264
|
+
previous: null,
|
|
265
|
+
previousArtifacts: null
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
const scratch = scratchDir(root);
|
|
269
|
+
const records = (await readdir(scratch).catch(() => [])).filter((n) => n.endsWith(".json") && RUN_ID.test(n.slice(0, -5)));
|
|
270
|
+
for (const name of records) {
|
|
271
|
+
const runId = name.slice(0, -5);
|
|
272
|
+
const record = await readScratch(root, runId);
|
|
273
|
+
if (!record) continue;
|
|
274
|
+
const artifacts = join(scratch, runId);
|
|
275
|
+
const previousArtifacts = existsSync(artifacts) ? artifacts : null;
|
|
276
|
+
const complete = record.status === "complete";
|
|
277
|
+
const existing = found.get(runId);
|
|
278
|
+
if (existing) {
|
|
279
|
+
existing.previous = record;
|
|
280
|
+
existing.previousArtifacts = previousArtifacts;
|
|
281
|
+
existing.complete = complete;
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
found.set(runId, {
|
|
285
|
+
runId,
|
|
286
|
+
startedAt: record.startedAt,
|
|
287
|
+
kind: "scratch",
|
|
288
|
+
live: false,
|
|
289
|
+
complete,
|
|
290
|
+
previous: complete ? null : record,
|
|
291
|
+
previousArtifacts: complete ? null : previousArtifacts
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
return [...found.values()].sort((a, b) => a.runId < b.runId ? -1 : 1);
|
|
295
|
+
}
|
|
296
|
+
const resumable = (c) => !c.live && !c.complete && (c.kind === "staging" || c.previous?.status === "interrupted" || c.previous?.status === "budget");
|
|
297
|
+
function toTarget(c) {
|
|
298
|
+
return {
|
|
299
|
+
runId: c.runId,
|
|
300
|
+
startedAt: new Date(c.startedAt),
|
|
301
|
+
kind: c.kind,
|
|
302
|
+
previous: c.previous,
|
|
303
|
+
previousArtifacts: c.previousArtifacts
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
function matchRef(candidates, ref) {
|
|
307
|
+
const exact = candidates.find((c) => c.runId === ref);
|
|
308
|
+
if (exact) return exact;
|
|
309
|
+
const byId = candidates.filter((c) => c.runId.startsWith(ref));
|
|
310
|
+
if (byId.length === 1) return byId[0];
|
|
311
|
+
if (byId.length > 1) throw new RefError(`"${ref}" matches ${byId.length} runs; use more of the id`);
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* `--resume` with no ref (or `latest`): the newest Stopped staging directory, or an
|
|
315
|
+
* `interrupted` / `budget` scratch. A pointer at `.reeval/current` wins when it still names
|
|
316
|
+
* one of those. A live pid (or a holder this process did not steal) is still running: omitted
|
|
317
|
+
* `--resume` names that id instead of "No resumable run". `--resume <ref>` is a run id or
|
|
318
|
+
* unique prefix, not today's assembled `latest` and not a git SHA.
|
|
319
|
+
*/
|
|
320
|
+
async function resolveResume(root, ref) {
|
|
321
|
+
const candidates = await listCandidates(root);
|
|
322
|
+
const open = candidates.filter(resumable);
|
|
323
|
+
if (ref === void 0 || ref === "" || ref === "latest") {
|
|
324
|
+
const pointer = readCurrent(root);
|
|
325
|
+
const pick = (pointer ? open.find((c) => c.runId === pointer) : void 0) ?? open.at(-1);
|
|
326
|
+
if (!pick) {
|
|
327
|
+
const liveOnes = candidates.filter((c) => c.live);
|
|
328
|
+
const livePick = (pointer ? liveOnes.find((c) => c.runId === pointer) : void 0) ?? liveOnes.at(-1);
|
|
329
|
+
if (livePick) throw stillRunning(livePick.runId);
|
|
330
|
+
throw new UsageError("No resumable run. A stopped staging directory, or an interrupted or budget-stopped scratch record, can be resumed.");
|
|
331
|
+
}
|
|
332
|
+
return toTarget(pick);
|
|
333
|
+
}
|
|
334
|
+
const back = /^latest~(\d+)$/.exec(ref);
|
|
335
|
+
if (back) {
|
|
336
|
+
const entry = open[open.length - 1 - Number(back[1])];
|
|
337
|
+
if (!entry) throw new RefError(`There are only ${open.length} resumable runs`);
|
|
338
|
+
return toTarget(entry);
|
|
339
|
+
}
|
|
340
|
+
const found = matchRef(candidates, ref);
|
|
341
|
+
if (!found) throw new RefError(`No run matches "${ref}"`);
|
|
342
|
+
if (found.live) throw stillRunning(found.runId);
|
|
343
|
+
if (found.complete || !resumable(found)) throw new UsageError(`run ${found.runId} is complete and cannot be resumed. reeval run starts a new run.`);
|
|
344
|
+
return toTarget(found);
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* For a scratch resume: move the assembled record aside so the continuation can write the same
|
|
348
|
+
* id. Staging already in place is left alone.
|
|
349
|
+
*/
|
|
350
|
+
async function prepareResume(target, root) {
|
|
351
|
+
const staging = stagingOf(root, target.runId);
|
|
352
|
+
mkdirSync(join(staging, "evals"), { recursive: true });
|
|
353
|
+
acquireHolder(staging, target.runId);
|
|
354
|
+
if (!target.previous) return;
|
|
355
|
+
const scratch = scratchDir(root);
|
|
356
|
+
const recordPath = join(scratch, `${target.runId}.json`);
|
|
357
|
+
const aside = join(staging, "previous.json");
|
|
358
|
+
if (existsSync(recordPath) && !existsSync(aside)) renameSync(recordPath, aside);
|
|
359
|
+
if (target.previousArtifacts && existsSync(target.previousArtifacts)) {
|
|
360
|
+
const dest = join(staging, "previous-artifacts");
|
|
361
|
+
if (!existsSync(dest)) renameSync(target.previousArtifacts, dest);
|
|
362
|
+
}
|
|
363
|
+
for (const e of target.previous.evals) for (const a of e.attempts) {
|
|
364
|
+
if (!isCompletedAttempt(a.outcome, a.error)) continue;
|
|
365
|
+
const path = join(staging, "evals", recordFileName(a.key));
|
|
366
|
+
if (existsSync(path)) continue;
|
|
367
|
+
const record = {
|
|
368
|
+
outcome: a.outcome === "skipped" ? "fail" : a.outcome,
|
|
369
|
+
error: a.error,
|
|
370
|
+
startedAt: a.startedAt ?? void 0,
|
|
371
|
+
turns: [],
|
|
372
|
+
diff: null,
|
|
373
|
+
checks: [],
|
|
374
|
+
judgements: [],
|
|
375
|
+
timings: {
|
|
376
|
+
setupMs: 0,
|
|
377
|
+
agentMs: 0,
|
|
378
|
+
diffMs: 0,
|
|
379
|
+
checksMs: 0,
|
|
380
|
+
judgeMs: 0
|
|
381
|
+
}
|
|
382
|
+
};
|
|
383
|
+
writeFileSync(path, `${JSON.stringify(record, null, 2)}\n`);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
async function loadPrevious(staging) {
|
|
387
|
+
const file = join(staging, "previous.json");
|
|
388
|
+
const artifacts = join(staging, "previous-artifacts");
|
|
389
|
+
try {
|
|
390
|
+
const record = JSON.parse(await readFile(file, "utf8"));
|
|
391
|
+
return {
|
|
392
|
+
record: record.kind === "es-run" ? record : null,
|
|
393
|
+
artifacts: existsSync(artifacts) ? artifacts : null
|
|
394
|
+
};
|
|
395
|
+
} catch {
|
|
396
|
+
return {
|
|
397
|
+
record: null,
|
|
398
|
+
artifacts: existsSync(artifacts) ? artifacts : null
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
const previousAttempt = (previous, key) => previous?.evals.flatMap((e) => e.attempts).find((a) => a.key === key);
|
|
403
|
+
//#endregion
|
|
404
|
+
export { previousAttempt as a, writeLiveProgress as c, prepareResume as i, isCompletedAttempt as n, resolveResume as o, loadPrevious as r, writeCurrent as s, completedKeysFor as t };
|
|
405
|
+
|
|
406
|
+
//# sourceMappingURL=resume-DZ9AmfPv.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resume-DZ9AmfPv.js","names":[],"sources":["../src/live.ts","../src/resume.ts"],"mappings":";;;;;;;;;;AAwCA,SAAS,SAAS,QAAgC;CAChD,MAAM,UAAU,OAAO,MAAM;CAC7B,IAAI,OAAO,UAAU,WACnB,OAAO,YAAY,UAAU,YAAY,gBAAgB,UAAU;CACrE,IAAI,YAAY,eAAe,OAAO;CACtC,OAAO,OAAO,UAAU,WAAW,SAAS;AAC9C;;AAGA,MAAM,uBAAO,IAAI,IAAI;CAAC;CAAS;CAAU;AAAO,CAAC;;AAGjD,MAAM,UAAU;AAQhB,SAAS,MAAM,IAAkB;CAC/B,QAAQ,KAAK,IAAI,WAAW,IAAI,kBAAkB,CAAC,CAAC,GAAG,GAAG,GAAG,EAAE;AACjE;;;;;AAMA,SAAgB,QACd,MACA,IACA,SAAS,YACT,QAAsB,CAAC,GACjB;CACN,MAAM,MAAM,MAAM,OAAO,KAAK;CAC9B,MAAM,OAAO,MAAM,QAAQ;CAC3B,MAAM,WAAW,IAAI,KAAK,MAAM,UAAU;CAC1C,KAAK,IAAI,UAAU,IAAK,WACtB,IAAI;EACF,OAAO,MAAM,EAAE;EACf;CACF,SAAS,OAAO;EACd,MAAM,OAAQ,MAAgC;EAC9C,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI,KAAK,IAAI,KAAK,UAAU,MAAM;EACzD,KAAK,KAAK,IAAI,SAAS,EAAE,CAAC;CAC5B;AAEJ;;AAGA,SAAgB,kBAAkB,SAAkD;CAClF,MAAM,MAAM,QAAQ,8BAAc,IAAI,KAAK;CAC3C,MAAM,OAAO,KAAK,QAAQ,KAAK,SAAS;CACxC,MAAM,MAAM,GAAG,KAAK,GAAG,QAAQ,IAAI;CACnC,MAAM,WAAW,IAAI,IACnB,QAAQ,QAAQ,KAAK,MAAM,CACzB,EAAE,KACF;EACE,KAAK,EAAE;EACP,QAAQ,EAAE;EACV,MAAM,EAAE,QAAQ;EAChB,MAAM,EAAE;EACR,MAAM,EAAE,QAAQ,KAAK;EACrB,GAAG,EAAE,QAAQ;EACb,QAAQ,EAAE,QAAQ,QAAQ,EAAE,QAAQ,KAAK,OAAO,YAAY;EAC5D,WAAW;EACX,YAAY;CACd,CACF,CAAC,CACH;CACA,IAAI,QAAmB;CACvB,IAAI,UAAU;CAEd,MAAM,aAA2B;EAC/B,MAAM,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC;EAClC,OAAO;GACL,eAAA;GACA,OAAO,QAAQ;GACf,WAAW,QAAQ,UAAU,YAAY;GACzC,KAAK,QAAQ;GACb,MAAM,SAAS;GACf,aAAa,IAAI,CAAC,CAAC,YAAY;GAC/B;GACA,SAAS,KAAK;GACd,QAAQ,WAAW,IAAI;GACvB,UAAU;EACZ;CACF;CACA,MAAM,cAAc;EAClB,IAAI,SAAS;EACb,IAAI;GACF,cAAc,KAAK,KAAK,UAAU,KAAK,CAAC,CAAC;GACzC,QAAQ,KAAK,IAAI;EACnB,QAAQ,CAAC;CACX;CAEA,IAAI;EACF,UAAU,QAAQ,KAAK,EAAE,WAAW,KAAK,CAAC;CAC5C,QAAQ,CAAC;CACT,MAAM;CACN,MAAM,YAAY,YAAY,OAAO,QAAQ,eAAA,GAA2B;CACxE,UAAU,MAAM;CAEhB,MAAM,aAAa;EACjB,UAAU;EACV,cAAc,SAAS;CACzB;CACA,OAAO;EACL,MAAM,KAAK,IAAI;GACb,MAAM,IAAI,SAAS,IAAI,GAAG;GAC1B,IAAI,CAAC,GAAG;GACR,EAAE,SAAS;GACX,EAAE,YAAY,IAAI,KAAK,EAAE,CAAC,CAAC,YAAY;GACvC,MAAM;EACR;EACA,IAAI,QAAQ;GACV,MAAM,IAAI,SAAS,IAAI,OAAO,GAAG;GACjC,IAAI,CAAC,GAAG;GACR,EAAE,SAAS,SAAS,MAAM;GAC1B,EAAE,aAAa,OAAO;GACtB,MAAM;EACR;EACA,MAAM,MAAM;GACV,QAAQ;GACR,MAAM;EACR;EACA;EACA,SAAS;GACP,KAAK;GACL,OAAO,MAAM,EAAE,OAAO,KAAK,CAAC;GAC5B,OAAO,KAAK,EAAE,OAAO,KAAK,CAAC;EAC7B;EACA;CACF;AACF;;;;ACtJA,MAAM,WAAW;;AAGjB,MAAa,cAAc;;AAG3B,MAAa,eAAe,SAAiB,KAAK,MAAM,WAAW,SAAS;AAE5E,MAAM,aAAa,MAAc,UAAkB,KAAK,MAAM,WAAW,WAAW,KAAK;AAEzF,SAAgB,aAAa,MAAc,OAAqB;CAC9D,MAAM,OAAO,YAAY,IAAI;CAC7B,UAAU,KAAK,MAAM,SAAS,GAAG,EAAE,WAAW,KAAK,CAAC;CACpD,MAAM,MAAM,GAAG,KAAK,GAAG,QAAQ,IAAI;CACnC,cAAc,KAAK,GAAG,MAAM,GAAG;CAC/B,IAAI;EACF,QAAQ,KAAK,IAAI;CACnB,SAAS,OAAO;EACd,OAAO,KAAK,EAAE,OAAO,KAAK,CAAC;EAC3B,MAAM;CACR;AACF;AAEA,MAAM,gBAAgB,UACpB,IAAI,WACF,OAAO,MAAM,6EAA6E,MAAM,EAClG;;AAGF,SAAgB,aAAa,KAAa,MAAuB;CAC/D,IAAI,SAAS,SAAS,GAAG,OAAO;CAChC,IAAI;EACF,QAAQ,KAAK,KAAK,CAAC;EACnB,OAAO;CACT,SAAS,OAAO;EACd,OAAQ,MAAgC,SAAS;CACnD;AACF;AAEA,SAAS,WAAW,KAAmD;CACrE,IAAI;EACF,MAAM,CAAC,SAAS,YAAY,aAAa,KAAK,KAAK,WAAW,GAAG,MAAM,CAAC,CAAC,MAAM,IAAI;EACnF,MAAM,MAAM,OAAO,OAAO;EAC1B,IAAI,CAAC,OAAO,UAAU,GAAG,KAAK,OAAO,KAAK,CAAC,UAAU,OAAO;EAC5D,OAAO;GAAE;GAAK,MAAM;EAAS;CAC/B,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,cAAc,QAAgB;CAClC,MAAM,SAAS,WAAW,GAAG;CAC7B,OAAO,SAAS,aAAa,OAAO,KAAK,OAAO,IAAI,IAAI;AAC1D;AAEA,SAAS,cAAc,SAAiB,OAAqB;CAC3D,MAAM,OAAO,KAAK,SAAS,WAAW;CACtC,MAAM,OAAO,GAAG,QAAQ,IAAI,IAAI,SAAS,EAAE;CAC3C,MAAM,eAAe,cAAc,MAAM,MAAM,EAAE,MAAM,KAAK,CAAC;CAC7D,IAAI;EACF,OAAO;EACP;CACF,SAAS,OAAO;EACd,IAAK,MAAgC,SAAS,UAAU,MAAM;CAChE;CACA,IAAI,WAAW,OAAO,GAAG,MAAM,aAAa,KAAK;CACjD,IAAI;EACF,WAAW,IAAI;CACjB,QAAQ;EACN,MAAM,aAAa,KAAK;CAC1B;CACA,IAAI;EACF,OAAO;CACT,SAAS,OAAO;EACd,IAAK,MAAgC,SAAS,UAAU,MAAM,aAAa,KAAK;EAChF,MAAM;CACR;AACF;AAEA,SAAgB,YAAY,MAA6B;CACvD,IAAI;EACF,MAAM,KAAK,aAAa,YAAY,IAAI,GAAG,MAAM,CAAC,CAAC,KAAK;EACxD,OAAO,OAAO,KAAK,EAAE,IAAI,KAAK;CAChC,QAAQ;EACN,OAAO;CACT;AACF;;AAGA,SAAgB,mBACd,SACA,OACS;CACT,IAAI,YAAY,WAAW,OAAO;CAClC,IAAI,OAAO,SAAS,WAAW,OAAO;CACtC,OAAO,YAAY,UAAU,YAAY,UAAU,YAAY;AACjE;AAEA,MAAM,uBAAuB,WAC3B,OAAO,MAAM,SAAS,MACpB,EAAE,SAAS,QAAQ,MAAM,mBAAmB,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,GAAG,CACnF;AAEF,SAAS,eAAe,MAAiC;CACvD,IAAI;EACF,OAAO,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;CAC9C,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,sBAAsB,SAAiB,KAAsB;CACpE,MAAM,SAAS,eAAe,KAAK,SAAS,SAAS,eAAe,GAAG,CAAC,CAAC;CACzE,IAAI,CAAC,QAAQ,OAAO;CACpB,OAAO,mBAAmB,OAAO,SAAS,OAAO,KAAK;AACxD;;;;;AAMA,SAAgB,iBACd,SACA,SACA,UACa;CACb,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,eAAe,IAAI,IAAI,WAAW,oBAAoB,QAAQ,IAAI,CAAC,CAAC;CAC1E,KAAK,MAAM,EAAE,SAAS,SACpB,IAAI,sBAAsB,SAAS,GAAG,KAAK,aAAa,IAAI,GAAG,GAAG,KAAK,IAAI,GAAG;CAEhF,OAAO;AACT;AAoBA,SAAS,aACP,UACA,KACS;CAET,IAAI,aAAa,SAAS,KAAK,SAAS,IAAI,GAAG,OAAO;CAEtD,IADc,MAAM,KAAK,MAAM,SAAS,WAAW,IAAI,UAC5C,OAAO;CAClB,OAAO,SAAS,SAAS,SAAS;AACpC;AAEA,eAAe,YAAY,MAAc,OAA0C;CACjF,IAAI;EACF,MAAM,OAAO,MAAM,SAAS,KAAK,WAAW,IAAI,GAAG,GAAG,MAAM,MAAM,GAAG,MAAM;EAC3E,MAAM,SAAS,KAAK,MAAM,IAAI;EAC9B,OAAO,OAAO,SAAS,YAAY,OAAO,UAAU,QAAQ,SAAS;CACvE,QAAQ;EACN,OAAO;CACT;AACF;AAEA,eAAe,eAAe,MAAc,MAAM,KAAK,IAAI,GAAyB;CAClF,MAAM,wBAAQ,IAAI,IAAuB;CACzC,MAAM,cAAc,KAAK,MAAM,WAAW,SAAS;CACnD,MAAM,QAAQ,MAAM,QAAQ,WAAW,CAAC,CAAC,YAAY,CAAC,CAAa;CACnE,KAAK,MAAM,SAAS,MAAM,QAAQ,MAAM,OAAO,KAAK,CAAC,CAAC,GAAG;EACvD,MAAM,MAAM,KAAK,aAAa,KAAK;EACnC,MAAM,OAAO,MAAM,KAAK,GAAG,CAAC,CAAC,YAAY,IAAI;EAC7C,IAAI,CAAC,MAAM,YAAY,GAAG;EAC1B,MAAM,OAAO,MAAM,SAAS,KAAK,KAAK,SAAS,GAAG,MAAM,CAAC,CAAC,YAAY,IAAI;EAC1E,MAAM,WAAW,OAAO,qBAAqB,KAAK,MAAM,IAAI,CAAY,IAAI;EAC5E,MAAM,QAAQ,WAAW,aAAa,UAAU,GAAG,IAAI,UAAU,WAAW,GAAG;EAC/E,MAAM,YAAY,UAAU,aAAa,IAAI,KAAK,KAAK,OAAO,CAAC,CAAC,YAAY;EAC5E,MAAM,IAAI,OAAO;GACf;GACA;GACA,MAAM;GACN;GACA,UAAU;GACV,UAAU;GACV,mBAAmB;EACrB,CAAC;CACH;CACA,MAAM,UAAU,WAAW,IAAI;CAC/B,MAAM,WAAW,MAAM,QAAQ,OAAO,CAAC,CAAC,YAAY,CAAC,CAAa,EAAA,CAAG,QAClE,MAAM,EAAE,SAAS,OAAO,KAAK,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE,CAAC,CAC1D;CACA,KAAK,MAAM,QAAQ,SAAS;EAC1B,MAAM,QAAQ,KAAK,MAAM,GAAG,EAAE;EAC9B,MAAM,SAAS,MAAM,YAAY,MAAM,KAAK;EAC5C,IAAI,CAAC,QAAQ;EACb,MAAM,YAAY,KAAK,SAAS,KAAK;EACrC,MAAM,oBAAoB,WAAW,SAAS,IAAI,YAAY;EAC9D,MAAM,WAAW,OAAO,WAAW;EACnC,MAAM,WAAW,MAAM,IAAI,KAAK;EAChC,IAAI,UAAU;GAEZ,SAAS,WAAW;GACpB,SAAS,oBAAoB;GAC7B,SAAS,WAAW;GACpB;EACF;EACA,MAAM,IAAI,OAAO;GACf;GACA,WAAW,OAAO;GAClB,MAAM;GACN,MAAM;GACN;GACA,UAAU,WAAW,OAAO;GAC5B,mBAAmB,WAAW,OAAO;EACvC,CAAC;CACH;CACA,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAO,EAAE,QAAQ,EAAE,QAAQ,KAAK,CAAE;AACxE;AAEA,MAAM,aAAa,MACjB,CAAC,EAAE,QACH,CAAC,EAAE,aACF,EAAE,SAAS,aAAa,EAAE,UAAU,WAAW,iBAAiB,EAAE,UAAU,WAAW;AAE1F,SAAS,SAAS,GAA4B;CAC5C,OAAO;EACL,OAAO,EAAE;EACT,WAAW,IAAI,KAAK,EAAE,SAAS;EAC/B,MAAM,EAAE;EACR,UAAU,EAAE;EACZ,mBAAmB,EAAE;CACvB;AACF;AAEA,SAAS,SAAS,YAAyB,KAAoC;CAC7E,MAAM,QAAQ,WAAW,MAAM,MAAM,EAAE,UAAU,GAAG;CACpD,IAAI,OAAO,OAAO;CAClB,MAAM,OAAO,WAAW,QAAQ,MAAM,EAAE,MAAM,WAAW,GAAG,CAAC;CAC7D,IAAI,KAAK,WAAW,GAAG,OAAO,KAAK;CACnC,IAAI,KAAK,SAAS,GAChB,MAAM,IAAI,SAAS,IAAI,IAAI,YAAY,KAAK,OAAO,0BAA0B;AAGjF;;;;;;;;AASA,eAAsB,cAAc,MAAc,KAAqC;CACrF,MAAM,aAAa,MAAM,eAAe,IAAI;CAC5C,MAAM,OAAO,WAAW,OAAO,SAAS;CAExC,IADgB,QAAQ,KAAA,KAAa,QAAQ,MAAM,QAAQ,UAC9C;EACX,MAAM,UAAU,YAAY,IAAI;EAEhC,MAAM,QADU,UAAU,KAAK,MAAM,MAAM,EAAE,UAAU,OAAO,IAAI,KAAA,MAC1C,KAAK,GAAG,EAAE;EAClC,IAAI,CAAC,MAAM;GACT,MAAM,WAAW,WAAW,QAAQ,MAAM,EAAE,IAAI;GAEhD,MAAM,YADc,UAAU,SAAS,MAAM,MAAM,EAAE,UAAU,OAAO,IAAI,KAAA,MAC1C,SAAS,GAAG,EAAE;GAC9C,IAAI,UAAU,MAAM,aAAa,SAAS,KAAK;GAC/C,MAAM,IAAI,WACR,oHACF;EACF;EACA,OAAO,SAAS,IAAI;CACtB;CACA,MAAM,OAAO,iBAAiB,KAAK,GAAG;CACtC,IAAI,MAAM;EACR,MAAM,QAAQ,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE;EACnD,IAAI,CAAC,OAAO,MAAM,IAAI,SAAS,kBAAkB,KAAK,OAAO,gBAAgB;EAC7E,OAAO,SAAS,KAAK;CACvB;CACA,MAAM,QAAQ,SAAS,YAAY,GAAG;CACtC,IAAI,CAAC,OAAO,MAAM,IAAI,SAAS,mBAAmB,IAAI,EAAE;CACxD,IAAI,MAAM,MAAM,MAAM,aAAa,MAAM,KAAK;CAC9C,IAAI,MAAM,YAAY,CAAC,UAAU,KAAK,GACpC,MAAM,IAAI,WACR,OAAO,MAAM,MAAM,iEACrB;CAEF,OAAO,SAAS,KAAK;AACvB;;;;;AAMA,eAAsB,cAAc,QAAsB,MAA6B;CACrF,MAAM,UAAU,UAAU,MAAM,OAAO,KAAK;CAC5C,UAAU,KAAK,SAAS,OAAO,GAAG,EAAE,WAAW,KAAK,CAAC;CACrD,cAAc,SAAS,OAAO,KAAK;CACnC,IAAI,CAAC,OAAO,UAAU;CACtB,MAAM,UAAU,WAAW,IAAI;CAC/B,MAAM,aAAa,KAAK,SAAS,GAAG,OAAO,MAAM,MAAM;CACvD,MAAM,QAAQ,KAAK,SAAS,eAAe;CAC3C,IAAI,WAAW,UAAU,KAAK,CAAC,WAAW,KAAK,GAAG,WAAW,YAAY,KAAK;CAC9E,IAAI,OAAO,qBAAqB,WAAW,OAAO,iBAAiB,GAAG;EACpE,MAAM,OAAO,KAAK,SAAS,oBAAoB;EAC/C,IAAI,CAAC,WAAW,IAAI,GAAG,WAAW,OAAO,mBAAmB,IAAI;CAClE;CAEA,KAAK,MAAM,KAAK,OAAO,SAAS,OAC9B,KAAK,MAAM,KAAK,EAAE,UAAU;EAC1B,IAAI,CAAC,mBAAmB,EAAE,SAAS,EAAE,KAAK,GAAG;EAC7C,MAAM,OAAO,KAAK,SAAS,SAAS,eAAe,EAAE,GAAG,CAAC;EACzD,IAAI,WAAW,IAAI,GAAG;EACtB,MAAM,SAAqB;GACzB,SAAS,EAAE,YAAY,YAAY,SAAS,EAAE;GAC9C,OAAO,EAAE;GACT,WAAW,EAAE,aAAa,KAAA;GAC1B,OAAO,CAAC;GACR,MAAM;GACN,QAAQ,CAAC;GACT,YAAY,CAAC;GACb,SAAS;IAAE,SAAS;IAAG,SAAS;IAAG,QAAQ;IAAG,UAAU;IAAG,SAAS;GAAE;EACxE;EACA,cAAc,MAAM,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,EAAE,GAAG;CAC5D;AAEJ;AAEA,eAAsB,aAAa,SAGhC;CACD,MAAM,OAAO,KAAK,SAAS,eAAe;CAC1C,MAAM,YAAY,KAAK,SAAS,oBAAoB;CACpD,IAAI;EACF,MAAM,SAAS,KAAK,MAAM,MAAM,SAAS,MAAM,MAAM,CAAC;EACtD,OAAO;GACL,QAAQ,OAAO,SAAS,WAAW,SAAS;GAC5C,WAAW,WAAW,SAAS,IAAI,YAAY;EACjD;CACF,QAAQ;EACN,OAAO;GAAE,QAAQ;GAAM,WAAW,WAAW,SAAS,IAAI,YAAY;EAAK;CAC7E;AACF;AAQA,MAAa,mBACX,UACA,QAEA,UAAU,MAAM,SAAS,MAAM,EAAE,QAAQ,CAAC,CAAC,MAAM,MAAM,EAAE,QAAQ,GAAG"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-yDAEwmZo.js";
|
|
2
|
-
import { t as Scheduler } from "./core-tLN-z1Be.js";
|
|
3
2
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
4
3
|
import { ASYMMETRIC_MATCHERS_OBJECT, GLOBAL_EXPECT, JestAsymmetricMatchers, JestChaiExpect, JestExtend, addCustomEqualityTesters, chai, customMatchers, getState, setState } from "@vitest/expect";
|
|
5
4
|
//#region src/engine/expect.ts
|
|
@@ -198,182 +197,6 @@ const registerGroup = (name, mode, body) => active().registerGroup(name, mode, b
|
|
|
198
197
|
const getRunContext = () => active().getRunContext();
|
|
199
198
|
const collectingFile = () => active().collectingFile();
|
|
200
199
|
//#endregion
|
|
201
|
-
|
|
202
|
-
var LocalTransport = class {
|
|
203
|
-
#pending = /* @__PURE__ */ new Map();
|
|
204
|
-
#scheduler = new Scheduler((r) => {
|
|
205
|
-
this.#pending.get(r.id)?.resolve();
|
|
206
|
-
this.#pending.delete(r.id);
|
|
207
|
-
});
|
|
208
|
-
request(id, limits, rank, pending) {
|
|
209
|
-
this.#pending.set(id, pending);
|
|
210
|
-
this.#scheduler.request({
|
|
211
|
-
id,
|
|
212
|
-
owner: "local",
|
|
213
|
-
limits,
|
|
214
|
-
...rank ? { rank } : {}
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
|
-
release(id) {
|
|
218
|
-
this.#pending.delete(id);
|
|
219
|
-
this.#scheduler.release(id);
|
|
220
|
-
}
|
|
221
|
-
};
|
|
222
|
-
/** Over the worker's IPC channel to reeval's own engine: requests out, grants back. */
|
|
223
|
-
var ChannelTransport = class {
|
|
224
|
-
#pending = /* @__PURE__ */ new Map();
|
|
225
|
-
#send;
|
|
226
|
-
constructor(send) {
|
|
227
|
-
this.#send = send;
|
|
228
|
-
}
|
|
229
|
-
grant(id) {
|
|
230
|
-
const pending = this.#pending.get(id);
|
|
231
|
-
this.#pending.delete(id);
|
|
232
|
-
pending?.resolve();
|
|
233
|
-
}
|
|
234
|
-
request(id, limits, rank, pending) {
|
|
235
|
-
this.#pending.set(id, pending);
|
|
236
|
-
this.#send({
|
|
237
|
-
t: "request",
|
|
238
|
-
id,
|
|
239
|
-
limits,
|
|
240
|
-
...rank ? { rank } : {}
|
|
241
|
-
});
|
|
242
|
-
}
|
|
243
|
-
release(id) {
|
|
244
|
-
this.#pending.delete(id);
|
|
245
|
-
this.#send({
|
|
246
|
-
t: "release",
|
|
247
|
-
id
|
|
248
|
-
});
|
|
249
|
-
}
|
|
250
|
-
};
|
|
251
|
-
let transport;
|
|
252
|
-
let ids = 0;
|
|
253
|
-
/** Points this process at the run's scheduler over `send`; returns what delivers its grants. */
|
|
254
|
-
function useSchedulerChannel(send) {
|
|
255
|
-
const channel = new ChannelTransport(send);
|
|
256
|
-
transport = channel;
|
|
257
|
-
return (id) => channel.grant(id);
|
|
258
|
-
}
|
|
259
|
-
/**
|
|
260
|
-
* Waits until every resource in `limits` has room, then holds a lease on each until released.
|
|
261
|
-
* Rejects with the signal's reason if it aborts first.
|
|
262
|
-
*/
|
|
263
|
-
function acquire(limits, options = {}) {
|
|
264
|
-
const { rank, signal } = options;
|
|
265
|
-
if (signal?.aborted) return Promise.reject(signal.reason);
|
|
266
|
-
transport ??= new LocalTransport();
|
|
267
|
-
const via = transport;
|
|
268
|
-
const id = String(++ids);
|
|
269
|
-
let released = false;
|
|
270
|
-
const lease = { release() {
|
|
271
|
-
if (released) return;
|
|
272
|
-
released = true;
|
|
273
|
-
via.release(id);
|
|
274
|
-
} };
|
|
275
|
-
return new Promise((resolve, reject) => {
|
|
276
|
-
const onAbort = () => {
|
|
277
|
-
lease.release();
|
|
278
|
-
reject(signal?.reason);
|
|
279
|
-
};
|
|
280
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
281
|
-
via.request(id, limits, rank, {
|
|
282
|
-
resolve: () => {
|
|
283
|
-
signal?.removeEventListener("abort", onAbort);
|
|
284
|
-
resolve(lease);
|
|
285
|
-
},
|
|
286
|
-
reject: (error) => {
|
|
287
|
-
signal?.removeEventListener("abort", onAbort);
|
|
288
|
-
reject(error);
|
|
289
|
-
}
|
|
290
|
-
});
|
|
291
|
-
});
|
|
292
|
-
}
|
|
293
|
-
/** Runs `fn` while holding leases on `limits`. */
|
|
294
|
-
async function withLease(limits, fn, signal) {
|
|
295
|
-
const lease = await acquire(limits, signal ? { signal } : {});
|
|
296
|
-
try {
|
|
297
|
-
return await fn();
|
|
298
|
-
} finally {
|
|
299
|
-
lease.release();
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
//#endregion
|
|
303
|
-
//#region src/matchers.ts
|
|
304
|
-
/** A check the turn's harness can't answer either way, e.g. a web fetch it never reports. */
|
|
305
|
-
var HarnessCapabilityError = class extends Error {
|
|
306
|
-
name = "HarnessCapabilityError";
|
|
307
|
-
};
|
|
308
|
-
const isTurn = (value) => typeof value === "object" && value !== null && Array.isArray(value.transcript) && Array.isArray(value.unreported);
|
|
309
|
-
const matches = (value, pattern) => pattern === void 0 || (typeof pattern === "string" ? value.includes(pattern) : pattern.test(value));
|
|
310
|
-
/** Passes when the turn has a `type` event whose `field` matches; refuses when it can't know. */
|
|
311
|
-
function webEvent(matcher, actual, type, field, pattern) {
|
|
312
|
-
if (!isTurn(actual)) throw new TypeError(`${matcher}() expects a turn from c.send()`);
|
|
313
|
-
const what = type === "web-search" ? "web searches" : "web fetches";
|
|
314
|
-
if (actual.unreported.includes(type)) throw new HarnessCapabilityError(`${matcher}() can't check this turn: its harness doesn't report ${what} ('${type}' is in turn.unreported), so the agent may have made them with no event. Check the reply or files instead, or use a harness that reports them.`);
|
|
315
|
-
const seen = actual.transcript.filter((e) => e.type === type).map(field);
|
|
316
|
-
const pass = seen.some((v) => matches(v, pattern));
|
|
317
|
-
const target = pattern === void 0 ? "" : ` matching ${pattern}`;
|
|
318
|
-
return {
|
|
319
|
-
pass,
|
|
320
|
-
message: () => pass ? `expected no ${what}${target}, got ${JSON.stringify(seen)}` : `expected ${what}${target}, got ${seen.length ? JSON.stringify(seen) : "none"}`
|
|
321
|
-
};
|
|
322
|
-
}
|
|
323
|
-
/** The last `lines` lines, and at most `chars` characters of them: one line can be megabytes. */
|
|
324
|
-
const tail = (s, lines = 20, chars = 2e3) => {
|
|
325
|
-
const last = s.trimEnd().split("\n").slice(-lines).join("\n");
|
|
326
|
-
return last.length <= chars ? last : `[… ${last.length - chars} earlier characters]${last.slice(-chars)}`;
|
|
327
|
-
};
|
|
328
|
-
const hasFiles = (value) => typeof value === "object" && value !== null && typeof value.readFile === "function";
|
|
329
|
-
const matchers = {
|
|
330
|
-
async toHaveFile(actual, path, pattern) {
|
|
331
|
-
if (!hasFiles(actual)) throw new TypeError("toHaveFile() expects a turn from c.send(), or c");
|
|
332
|
-
const content = await actual.readFile(path);
|
|
333
|
-
if (content === null) return {
|
|
334
|
-
pass: false,
|
|
335
|
-
message: () => `expected file ${path} to exist`
|
|
336
|
-
};
|
|
337
|
-
if (pattern === void 0) return {
|
|
338
|
-
pass: true,
|
|
339
|
-
message: () => `expected no file at ${path}`
|
|
340
|
-
};
|
|
341
|
-
const pass = typeof pattern === "string" ? content.includes(pattern) : pattern.test(content);
|
|
342
|
-
return {
|
|
343
|
-
pass,
|
|
344
|
-
message: () => `expected ${path} ${pass ? "not " : ""}to match ${pattern}\n--- ${path} ---\n${tail(content)}`
|
|
345
|
-
};
|
|
346
|
-
},
|
|
347
|
-
async toSucceed(actual) {
|
|
348
|
-
const r = await actual;
|
|
349
|
-
if (typeof r?.exitCode !== "number") throw new TypeError("toSucceed() expects run.exec(...)");
|
|
350
|
-
return {
|
|
351
|
-
pass: r.exitCode === 0,
|
|
352
|
-
message: () => r.exitCode === 0 ? "expected command to fail" : `expected command to succeed, exit code ${r.exitCode}\n--- stderr ---\n${tail(r.stderr)}\n--- stdout ---\n${tail(r.stdout)}`
|
|
353
|
-
};
|
|
354
|
-
},
|
|
355
|
-
toHaveSearched(actual, query) {
|
|
356
|
-
return webEvent("toHaveSearched", actual, "web-search", (e) => "query" in e ? e.query : "", query);
|
|
357
|
-
},
|
|
358
|
-
toHaveFetched(actual, url) {
|
|
359
|
-
return webEvent("toHaveFetched", actual, "web-fetch", (e) => "url" in e ? e.url : "", url);
|
|
360
|
-
},
|
|
361
|
-
toPass(actual) {
|
|
362
|
-
const failure = actual;
|
|
363
|
-
if (failure?.verdict === "error" && typeof failure.error === "string") return {
|
|
364
|
-
pass: !!this?.isNot,
|
|
365
|
-
message: () => `judge failed for "${failure.question}" (${failure.model}): ${failure.error}`
|
|
366
|
-
};
|
|
367
|
-
const v = actual;
|
|
368
|
-
if (typeof v?.score !== "number") throw new TypeError("toPass() expects a c.judge() verdict");
|
|
369
|
-
const score = v.score.toFixed(2);
|
|
370
|
-
return {
|
|
371
|
-
pass: v.verdict === "pass",
|
|
372
|
-
message: () => v.verdict === "pass" ? `expected judge to fail: score ${score} ≥ ${v.threshold} for "${v.question}"` : `judge score ${score} is below ${v.threshold} for "${v.question}" (${v.model})`
|
|
373
|
-
};
|
|
374
|
-
}
|
|
375
|
-
};
|
|
376
|
-
//#endregion
|
|
377
|
-
export { withLease as a, registerGroup as c, attemptExpectError as d, createExpect as f, withAttemptExpect as g, unawaitedError as h, useSchedulerChannel as i, registerTest as l, extendExpect as m, matchers as n, collectingFile as o, expect as p, acquire as r, getRunContext as s, HarnessCapabilityError as t, useEngine as u };
|
|
200
|
+
export { useEngine as a, expect as c, withAttemptExpect as d, registerTest as i, extendExpect as l, getRunContext as n, attemptExpectError as o, registerGroup as r, createExpect as s, collectingFile as t, unawaitedError as u };
|
|
378
201
|
|
|
379
|
-
//# sourceMappingURL=
|
|
202
|
+
//# sourceMappingURL=runtime-Dfb2r-cS.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime-Dfb2r-cS.js","names":[],"sources":["../src/engine/expect.ts","../src/engine/runtime.ts"],"mappings":";;;;AA2DA,KAAK,IAAI,UAAwB;AACjC,KAAK,IAAI,cAA4B;AACrC,KAAK,IAAI,sBAAoC;;AAkB7C,SAAgB,aAAa,MAAiC;CAC5D,MAAM,WAAW,OAAgB,YAAqB;EACpD,MAAM,EAAE,mBAAmB,SAAS,MAAM;EAC1C,SAAS,EAAE,gBAAgB,iBAAiB,EAAE,GAAG,MAAM;EACvD,MAAM,YAAY,KAAK,OAAO,OAAO,OAAO;EAC5C,OAAO,OAAO,UAAU,SAAS,IAAI,IAAI;CAC3C;CACA,OAAO,OAAO,QAAQ,KAAK,MAAM;CACjC,OAAO,OAAO,QAAS,WAAsC,2BAA2B;CACxF,OAAO,iBAAiB,SAAuB,MAAM;CACrD,OAAO,YAAY,UAAU,SAAS,OAAO,MAAM;CACnD,SACE;EACE,gBAAgB;EAChB,uBAAuB;EACvB,4BAA4B;EAC5B,0BAA0B;EAC1B,kCAAkC;EAClC,iBAAiB,MAAM,gBAAgB;CACzC,GACA,MACF;CACA,OAAO,SAAS,KAAK;CACrB,OAAO,UAAU,aACd,KAAK,OAAwE,OAC5E,QACA,QACF;CACF,OAAO,sBAAsB,YAAY,yBAAyB,OAAO;CACzE,OAAO,QAAQ,OAAO,YACnB,OAAO,OAAO,OAAO,CAAC,CAAkD,YAAY,EACnF,MAAM,KACR,CAAC;CACH,OAAO,OAAO,WAAW,MAAM;CAC/B,OAAO,eAAe,YAAY;EAChC,KAAK,OAAO,KAAK,WAAW,UAAU,KAAK,QAAQ,MAAM,IAAI,kBAAkB;CACjF;CACA,SAAS,WAAW,UAAkB;EACpC,MAAM,iCACJ,IAAI,MACF,uCAAuC,SAAS,YAAY,OAAO,SAAS,CAAC,CAAC,gBAChF;EACF,SACE;GAAE,0BAA0B;GAAU,kCAAkC;EAAS,GACjF,MACF;CACF;CACA,SAAS,gBAAgB;EAEvB,SACE;GAAE,uBAAuB;GAAM,4CAA4B,IAF3C,MAAM,gDAEyC;EAAE,GACjE,MACF;CACF;CACA,KAAK,KAAK,UAAU,QAAQ,cAAc,UAAU;CACpD,KAAK,KAAK,UAAU,QAAQ,iBAAiB,aAAa;CAC1D,OAAO,OAAO,cAAc;CAC5B,OAAO;AACT;;;;;AAMA,SAAS,WAAW,QAA4C;CAC9D,QAAQ,IAAI,UAAU,CAAC,MAAM;EAC3B,MAAM,EAAE,WAAW,IAAI,UAAU,KAAM,YAAY;EACnD,MAAM,YAAY,OAAO,MAAM,OAAO;EACtC,KAAK,KAAK,KAAK,WAAW,QAAQ,IAAI;EACtC,MAAM,QAAgB,IAAI,MAAM,WAAW,EACzC,IAAI,QAAQ,KAAK,UAAU;GACzB,MAAM,UAAU,QAAQ,IAAI,QAAQ,KAAK,QAAQ;GACjD,IAAI,OAAO,YAAY,YACrB,OAAO,mBAAmB,KAAK,YAAY,QAAQ;GACrD,QAAQ,GAAG,SAAoB;IAC7B,MAAM,2BAAW,IAAI,MAAM,aAAa;IACxC,QAAQ,YAAY;KAClB,MAAM,WAAW,KAAK,IAAI,IAAI;KAC9B,MAAM,SAAS,YAAY,QAAQ,OAAO;KAC1C,SACE,IAAI;MACF,KAAK,KAAK,KAAK,WAAW,UAAU,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC;MACxD,OAAO,MAAM,QAAQ,MAAM,SAAS,WAAW,IAAI;KACrD,SAAS,OAAO;MACd,IAAI,KAAK,IAAI,IAAI,YAAY,UAAU;OACrC,IAAI,iBAAiB,SAAS,SAAS,OACrC,MAAM,QAAQ,SAAS,MAAM,QAAQ,SAAS,SAAS,MAAM,OAAO;OACtE,MAAM;MACR;MACA,MAAM,IAAI,SAAS,YAAY,WAAW,SAAS,QAAQ,CAAC;KAC9D;IAEJ,EAAA,CAAG;GACL;EACF,EACF,CAAC;EACD,OAAO;CACT;AACF;;AAGA,SAAgB,mBAAmB,QAAsB,MAAqC;CAC5F,MAAM,CAAC,QAAQ,KAAK,QAAQ,UAAU,CAAC;CAEvC,IAAI,MAAM,OAAO,OAAO,uBAAO,IAAI,MAAM,GAAG,IAAI;CAChD,MAAM,QAAQ,OAAO,SAAS;CAC9B,IACE,MAAM,6BAA6B,QACnC,MAAM,6BAA6B,MAAM,gBAEzC,OAAQ,MAAM,iCAAiD;CACjE,IAAI,MAAM,yBAAyB,MAAM,mBAAmB,GAC1D,OAAO,MAAM;AAEjB;AAEA,MAAM,eAAe,aAAa;AAClC,OAAO,eAAe,YAAY,eAAe;CAC/C,OAAO;CACP,UAAU;CACV,cAAc;AAChB,CAAC;AAED,MAAM,gBAAgB,IAAI,kBAAgC;AAI1D,MAAM,iBAAiB,IAAI,kBAA6B;AAExD,MAAM,cAAc,UAClB,OAAO,UAAU,YACjB,UAAU,QACV,OAAQ,MAA2B,SAAS;;;;;AAM9C,SAAS,QAA0B,WAAc,SAAoB,MAAiB;CACpF,OAAO,IAAI,MAAM,WAAW,EAC1B,IAAI,QAAQ,UAAU;EACpB,MAAM,QAAQ,QAAQ,IAAI,QAAQ,QAAQ;EAC1C,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,QAAQ;EACvC,IAAI,OAAO,UAAU,YACnB,QAAQ,GAAG,SAAoB;GAC7B,MAAM,SAAS,QAAQ,MAAM,OAAO,QAAQ,IAAI;GAChD,IAAI,CAAC,WAAW,MAAM,GAAG,OAAO;GAChC,MAAM,QAAiB;IACrB,WAAW,GAAG,KAAK;IACnB,SAAS,QAAQ,QAAQ,MAAM,CAAC,CAAC,WACzB,KAAA,IACL,UAAmB,yBAAS,IAAI,MAAM,GAAG,KAAK,UAAU,CAC3D;IACA,UAAU;GACZ;GACA,QAAQ,KAAK,KAAK;GAClB,MAAM,WAAc,OAAmC;IACrD,MAAM,WAAW;IACjB,OAAO,GAAG,QAAQ,QAAQ,MAAM,CAAC;GACnC;GACA,OAAO;IAEL,OAAO,aAAqB,eAC1B,SAAS,MAAM,EAAE,KAAK,aAAa,UAAU,CAAC;IAChD,QAAQ,eAAuB,SAAS,MAAM,EAAE,MAAM,UAAU,CAAC;IACjE,UAAU,cAAsB,SAAS,MAAM,EAAE,QAAQ,SAAS,CAAC;KAClE,OAAO,cAAc;GACxB;EACF;EAEF,OAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,WAAW,KAAK,IACnE,QAAQ,OAAO,SAAS,IAAI,IAC5B;CACN,EACF,CAAC;AACH;AAEA,MAAM,SAAS,WAAoB,SAAiB;CAClD,MAAM,UAAU,eAAe,SAAS;CACxC,OAAO,WAAW,OAAO,cAAc,YAAY,cAAc,OAC7D,QAAQ,WAAW,SAAS,IAAI,IAChC;AACN;;AAGA,MAAM,SAAiC;CAAE,MAAM;CAAkB,MAAM;AAAiB;AAExF,MAAa,SAAuB,IAAI,MAAM,cAAc;CAC1D,QAAQ,QAAQ,SAAS,SACvB,MAAM,QAAQ,MAAM,cAAc,SAAS,KAAK,QAAQ,SAAS,IAAI,GAAG,WAAW;CACrF,MAAM,QAAQ,aAAa;EACzB,MAAM,QAAQ,cAAc,SAAS,KAAK;EAC1C,MAAM,QAAQ,QAAQ,IAAI,OAAO,QAAQ;EAEzC,IAAI,OAAO,UAAU,cAAc,aAAa,UAAU,OAAO;EACjE,MAAM,QAAQ,OAAO;EAErB,QAAQ,GAAG,SAAoB;GAC7B,MAAM,SAAS,QAAQ,MAAM,OAAO,OAAO,IAAI;GAC/C,OAAO,QAAQ,MAAM,QAAQ,KAAK,IAAI;EACxC;CACF;AACF,CAAC;;AAGD,SAAgB,kBACd,KACA,SACA,MACY;CACZ,OAAO,cAAc,IAAI,WAAW,eAAe,IAAI,SAAS,IAAI,CAAC;AACvE;;;;;AAMA,eAAsB,eAAe,SAAgD;CACnF,MAAM,SAAS,QAAQ,QAAQ,MAAM,CAAC,EAAE,QAAQ;CAChD,IAAI,OAAO,WAAW,GAAG,OAAO,KAAA;CAChC,KAAK,MAAM,KAAK,QAAQ,EAAE,WAAW;CACrC,MAAM,WAAW,MAAM,QAAQ,IAAI,OAAO,KAAK,MAAM,EAAE,OAAO,CAAC;CAC/D,MAAM,IAAI,KAAK,IACb,GACA,SAAS,WAAW,MAAM,MAAM,KAAA,CAAS,CAC3C;CACA,MAAM,OAAO,GAAG,OAAO,EAAE,EAAE,UAAU,qDAAqD,OAAO,EAAE,EAAE,UAAU;CAC/G,MAAM,UAAU,SAAS;CACzB,IAAI,mBAAmB,OAAO;EAC5B,QAAQ,UAAU,GAAG,QAAQ,QAAQ,MAAM;EAC3C,OAAO;CACT;CACA,MAAM,QAAQ,IAAI,MAAM,IAAI;CAC5B,MAAM,OAAO;CACb,OAAO;AACT;AAQA,SAAgB,aAAa,UAA0B;CACrD,aAAa,OAAO,QAAiD;AACvE;;;AChRA,IAAI;;AAGJ,SAAgB,UAAU,MAAoB;CAC5C,SAAS;AACX;AAEA,SAAS,SAAiB;CACxB,IAAI,CAAC,QACH,MAAM,IAAI,WACR,iGACF;CACF,OAAO;AACT;AAEA,MAAa,gBAAwC,MAAM,SAAS,SAClE,OAAO,CAAC,CAAC,aAAa,MAAM,SAAS,IAAI;AAE3C,MAAa,iBAA0C,MAAM,MAAM,SACjE,OAAO,CAAC,CAAC,cAAc,MAAM,MAAM,IAAI;AAEzC,MAAa,sBAAkC,OAAO,CAAC,CAAC,cAAc;AAEtE,MAAa,uBAA2C,OAAO,CAAC,CAAC,eAAe"}
|
package/dist/sandbox.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { A as DockerSandboxDescriptor, bt as SandboxDescriptor, q as FakeSandboxDescriptor } from "./types-_LOFdPUf.js";
|
|
2
2
|
//#region src/sandbox.d.ts
|
|
3
3
|
export type DockerOptions = Omit<DockerSandboxDescriptor, 'kind' | 'dockerfile'> & {
|
|
4
4
|
/** Relative to the file that calls `docker()`, or an absolute path or `URL`. */
|
|
@@ -3,12 +3,12 @@ import { a as parseDataPath, r as artifactType, t as RUN_ID } from "./paths-BmR-
|
|
|
3
3
|
import { s as recordFileName } from "./ids-Bp6ut7ZD.js";
|
|
4
4
|
import { l as parseDebugLog, u as parseNdjson } from "./debug-D24NOQH1.js";
|
|
5
5
|
import { i as parseAttemptKey, t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
|
|
6
|
-
import { o as sortAnalyses } from "./analysis-
|
|
7
|
-
import { v as indexFiles } from "./model-
|
|
8
|
-
import { o as readableDebugRequest } from "./record-
|
|
9
|
-
import { c as readableLiveProgress, i as isFinished, n as LIVE_FILE } from "./live-
|
|
10
|
-
import { l as recordFile, p as scratchDir } from "./files-
|
|
11
|
-
import { n as resolveFile, t as listHeaders } from "./source-
|
|
6
|
+
import { o as sortAnalyses } from "./analysis-BTNnKh1w.js";
|
|
7
|
+
import { v as indexFiles } from "./model-BndnmcLm.js";
|
|
8
|
+
import { o as readableDebugRequest } from "./record-BRP9vZrj.js";
|
|
9
|
+
import { c as readableLiveProgress, i as isFinished, n as LIVE_FILE } from "./live-S0qODb6o.js";
|
|
10
|
+
import { l as recordFile, p as scratchDir } from "./files-CMSUXAEg.js";
|
|
11
|
+
import { n as resolveFile, t as listHeaders } from "./source-BVOnL3sn.js";
|
|
12
12
|
import { createReadStream, existsSync } from "node:fs";
|
|
13
13
|
import { hostname } from "node:os";
|
|
14
14
|
import { extname, join, resolve, sep } from "node:path";
|
|
@@ -75,7 +75,7 @@ async function liveTranscript(source, runId, key) {
|
|
|
75
75
|
const [realBase, real] = await Promise.all([realpath(base), realpath(join(base, "evals", recordFileName(key)))]);
|
|
76
76
|
if (!real.startsWith(realBase + sep)) return null;
|
|
77
77
|
const record = JSON.parse(await readFile(real, "utf8"));
|
|
78
|
-
const { transcriptLines } = await import("./assemble-
|
|
78
|
+
const { transcriptLines } = await import("./assemble-CXiTVyHD.js");
|
|
79
79
|
return redactSecrets(transcriptLines(record));
|
|
80
80
|
} catch {
|
|
81
81
|
return null;
|
|
@@ -300,7 +300,7 @@ function createStudioApp(options) {
|
|
|
300
300
|
return c.json(body, 200, noStore);
|
|
301
301
|
}
|
|
302
302
|
if (target.kind === "analysis") {
|
|
303
|
-
const { listAnalyses, listNotes } = await import("./files-
|
|
303
|
+
const { listAnalyses, listNotes } = await import("./files-YCCVaTQC.js");
|
|
304
304
|
const entries = await listAnalyses(source.root, [source.historyDir], target.runId, true);
|
|
305
305
|
const notes = await listNotes(source.root, target.runId);
|
|
306
306
|
const index = {
|
|
@@ -349,4 +349,4 @@ async function serveApp(c, dir, path) {
|
|
|
349
349
|
//#endregion
|
|
350
350
|
export { watchLive as a, newToken as i, allowedHosts as n, createStudioApp as r, COOKIE as t };
|
|
351
351
|
|
|
352
|
-
//# sourceMappingURL=server-
|
|
352
|
+
//# sourceMappingURL=server-Da60oe4b.js.map
|