reeval 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/THIRD_PARTY_NOTICES.md +2160 -0
- package/assets/package.json +9 -0
- package/assets/pnpm-lock.yaml +1336 -0
- package/dist/agent-DVHkx8aT.js +532 -0
- package/dist/agent-DVHkx8aT.js.map +1 -0
- package/dist/analysis-CxSCt1YY.js +16 -0
- package/dist/analysis-CxSCt1YY.js.map +1 -0
- package/dist/assemble-B-nmz8oH.js +695 -0
- package/dist/assemble-B-nmz8oH.js.map +1 -0
- package/dist/assemble-CmRVDZ2Q.js +2 -0
- package/dist/attempt-BdRQ9jKX.js +176 -0
- package/dist/attempt-BdRQ9jKX.js.map +1 -0
- package/dist/bridge/index.mjs +1911 -0
- package/dist/bridge/package.json +13 -0
- package/dist/bridge/pnpm-lock.yaml +1041 -0
- package/dist/bridge/pnpm-workspace.yaml +2 -0
- package/dist/budget-DBUrH4jX.js +8 -0
- package/dist/budget-DBUrH4jX.js.map +1 -0
- package/dist/cli--CQ3Iy9q.js +1865 -0
- package/dist/cli--CQ3Iy9q.js.map +1 -0
- package/dist/cli-spec-DKAu09Gi.js +478 -0
- package/dist/cli-spec-DKAu09Gi.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +8 -0
- package/dist/cli.js.map +1 -0
- package/dist/colors-QV9dFVhN.js +34 -0
- package/dist/colors-QV9dFVhN.js.map +1 -0
- package/dist/command-BSJzHMtP.js +280 -0
- package/dist/command-BSJzHMtP.js.map +1 -0
- package/dist/command-CEm8PZ1N.js +1043 -0
- package/dist/command-CEm8PZ1N.js.map +1 -0
- package/dist/command-D-jnQWVG.js +673 -0
- package/dist/command-D-jnQWVG.js.map +1 -0
- package/dist/command-DQAvbG7d.js +432 -0
- package/dist/command-DQAvbG7d.js.map +1 -0
- package/dist/commands-A4udfyKZ.js +2 -0
- package/dist/commands-CF8mOfiu.js +207 -0
- package/dist/commands-CF8mOfiu.js.map +1 -0
- package/dist/config-Cuj0kVYj.js +415 -0
- package/dist/config-Cuj0kVYj.js.map +1 -0
- package/dist/context-loi9OCZv.js +106 -0
- package/dist/context-loi9OCZv.js.map +1 -0
- package/dist/core-tLN-z1Be.js +96 -0
- package/dist/core-tLN-z1Be.js.map +1 -0
- package/dist/db-Cjj2ZsfQ.js +2 -0
- package/dist/db-DxDzMhtB.js +827 -0
- package/dist/db-DxDzMhtB.js.map +1 -0
- package/dist/debug-D24NOQH1.js +89 -0
- package/dist/debug-D24NOQH1.js.map +1 -0
- package/dist/debug-SEmW4w8f.js +2 -0
- package/dist/dist-BZXYHN2X.js +1673 -0
- package/dist/dist-BZXYHN2X.js.map +1 -0
- package/dist/docker-DQDZWXF9.js +20 -0
- package/dist/docker-DQDZWXF9.js.map +1 -0
- package/dist/docker-DnWAVdEm.js +2 -0
- package/dist/docker-oSbCExHL.js +153 -0
- package/dist/docker-oSbCExHL.js.map +1 -0
- package/dist/doctor-CxlH1pDM.js +525 -0
- package/dist/doctor-CxlH1pDM.js.map +1 -0
- package/dist/egress-73SQfKdx.js +619 -0
- package/dist/egress-73SQfKdx.js.map +1 -0
- package/dist/env-BQqj93gu.js +21 -0
- package/dist/env-BQqj93gu.js.map +1 -0
- package/dist/environment-Mz7wvrtD.js +552 -0
- package/dist/environment-Mz7wvrtD.js.map +1 -0
- package/dist/errors-C9w_ud7g.js +160 -0
- package/dist/errors-C9w_ud7g.js.map +1 -0
- package/dist/errors-CSxfJ4qu.js +11 -0
- package/dist/errors-CSxfJ4qu.js.map +1 -0
- package/dist/files--6CEUXm7.js +2 -0
- package/dist/files-BLlrhU4-.js +62 -0
- package/dist/files-BLlrhU4-.js.map +1 -0
- package/dist/files-Bd25OAQH.js +2 -0
- package/dist/files-BvHCFWgD.js +2 -0
- package/dist/files-D2-qn_Dl.js +275 -0
- package/dist/files-D2-qn_Dl.js.map +1 -0
- package/dist/files-bF2fPuT9.js +99 -0
- package/dist/files-bF2fPuT9.js.map +1 -0
- package/dist/git-D6hMeieS.js +44 -0
- package/dist/git-D6hMeieS.js.map +1 -0
- package/dist/harness-ls5aX1UO.js +131 -0
- package/dist/harness-ls5aX1UO.js.map +1 -0
- package/dist/harness.d.ts +74 -0
- package/dist/harness.js +2 -0
- package/dist/header-DzFDoMCI.js +78 -0
- package/dist/header-DzFDoMCI.js.map +1 -0
- package/dist/ids-Bp6ut7ZD.js +49 -0
- package/dist/ids-Bp6ut7ZD.js.map +1 -0
- package/dist/images-Cc1-coFc.js +286 -0
- package/dist/images-Cc1-coFc.js.map +1 -0
- package/dist/index.d.ts +97 -0
- package/dist/index.js +1959 -0
- package/dist/index.js.map +1 -0
- package/dist/init-CFljS8BJ.js +344 -0
- package/dist/init-CFljS8BJ.js.map +1 -0
- package/dist/init-z2ZPn9Tj.js +2 -0
- package/dist/invocations-rXktgSAn.js +157 -0
- package/dist/invocations-rXktgSAn.js.map +1 -0
- package/dist/jiti-BOLXpKyG.js +46 -0
- package/dist/jiti-BOLXpKyG.js.map +1 -0
- package/dist/judge-D8Uf4dA9.js +146 -0
- package/dist/judge-D8Uf4dA9.js.map +1 -0
- package/dist/judge.d.ts +21 -0
- package/dist/judge.js +19 -0
- package/dist/judge.js.map +1 -0
- package/dist/live-Bs3LReMV.js +98 -0
- package/dist/live-Bs3LReMV.js.map +1 -0
- package/dist/matchers-DyLvm7EG.js +374 -0
- package/dist/matchers-DyLvm7EG.js.map +1 -0
- package/dist/model-BdO79gWz.js +44 -0
- package/dist/model-BdO79gWz.js.map +1 -0
- package/dist/model-CDL98xue.js +725 -0
- package/dist/model-CDL98xue.js.map +1 -0
- package/dist/model-v-0_Ik7S.js +2 -0
- package/dist/node-kG64XxdE.js +506 -0
- package/dist/node-kG64XxdE.js.map +1 -0
- package/dist/opencode-providers-DZuD_oYt.js +3149 -0
- package/dist/opencode-providers-DZuD_oYt.js.map +1 -0
- package/dist/options-CAQhu--C.js +452 -0
- package/dist/options-CAQhu--C.js.map +1 -0
- package/dist/patch-BkKje4cp.js +69 -0
- package/dist/patch-BkKje4cp.js.map +1 -0
- package/dist/paths-CBvm6pap.js +72 -0
- package/dist/paths-CBvm6pap.js.map +1 -0
- package/dist/plan-gwk6dJ8c.js +237 -0
- package/dist/plan-gwk6dJ8c.js.map +1 -0
- package/dist/preflight-BtubXKQ8.js +2 -0
- package/dist/preflight-BypZMNBG.js +185 -0
- package/dist/preflight-BypZMNBG.js.map +1 -0
- package/dist/prepare-Cwv0bHs_.js +3 -0
- package/dist/prepare-D6Uzcubb.js +213 -0
- package/dist/prepare-D6Uzcubb.js.map +1 -0
- package/dist/provenance-Dfb8YY5Y.js +129 -0
- package/dist/provenance-Dfb8YY5Y.js.map +1 -0
- package/dist/provider-BDbNOQhQ.js +2 -0
- package/dist/provider-CmAYTQQs.js +14 -0
- package/dist/provider-CmAYTQQs.js.map +1 -0
- package/dist/providers-2p3ylMHI.js +107 -0
- package/dist/providers-2p3ylMHI.js.map +1 -0
- package/dist/providers-BUOxitcT.js +2 -0
- package/dist/query-CFATHZmV.js +91 -0
- package/dist/query-CFATHZmV.js.map +1 -0
- package/dist/query-DR4m6CXf.js +2 -0
- package/dist/record-DQ8wzyco.js +99 -0
- package/dist/record-DQ8wzyco.js.map +1 -0
- package/dist/redact-C4fL_bZP.js +167 -0
- package/dist/redact-C4fL_bZP.js.map +1 -0
- package/dist/results-CeXb8Wde.js +37 -0
- package/dist/results-CeXb8Wde.js.map +1 -0
- package/dist/root-CFDjpWuc.js +110 -0
- package/dist/root-CFDjpWuc.js.map +1 -0
- package/dist/sandbox-By2gkdkC.js +133 -0
- package/dist/sandbox-By2gkdkC.js.map +1 -0
- package/dist/sandbox.d.ts +41 -0
- package/dist/sandbox.js +2 -0
- package/dist/server-Bi61uiY0.js +350 -0
- package/dist/server-Bi61uiY0.js.map +1 -0
- package/dist/server-CsSBL8NC.js +2 -0
- package/dist/source-BXzrmof5.js +2 -0
- package/dist/source-Bsfa3LZH.js +40 -0
- package/dist/source-Bsfa3LZH.js.map +1 -0
- package/dist/src-55j1zNHH.js +3 -0
- package/dist/src-AEwxd1UP.js +602 -0
- package/dist/src-AEwxd1UP.js.map +1 -0
- package/dist/src-BDwSuyIo.js +3 -0
- package/dist/src-BQyMiWri.js +490 -0
- package/dist/src-BQyMiWri.js.map +1 -0
- package/dist/src-C-LHyTSV.js +3 -0
- package/dist/src-Cmp-7YrD.js +1136 -0
- package/dist/src-Cmp-7YrD.js.map +1 -0
- package/dist/src-D0RdqpNb.js +252 -0
- package/dist/src-D0RdqpNb.js.map +1 -0
- package/dist/src-D8KYzxBX.js +793 -0
- package/dist/src-D8KYzxBX.js.map +1 -0
- package/dist/src-DEgff9Z7.js +4 -0
- package/dist/src-DXx_29pq.js +567 -0
- package/dist/src-DXx_29pq.js.map +1 -0
- package/dist/src-DwwiVLqv.js +2937 -0
- package/dist/src-DwwiVLqv.js.map +1 -0
- package/dist/src-N7E_mfcZ.js +2 -0
- package/dist/src-W9g-y5X5.js +3 -0
- package/dist/src-daS0wKH2.js +3 -0
- package/dist/state-Cyw3QTsN.js +2 -0
- package/dist/state-DZ7eqb1F.js +167 -0
- package/dist/state-DZ7eqb1F.js.map +1 -0
- package/dist/stats-BNAHlNMD.js +114 -0
- package/dist/stats-BNAHlNMD.js.map +1 -0
- package/dist/studio-app/assets/index-D6XyfFSH.js +58 -0
- package/dist/studio-app/assets/index-xwzWdcE_.css +2 -0
- package/dist/studio-app/favicon.svg +1 -0
- package/dist/studio-app/index.html +15 -0
- package/dist/summary-D7Ttky0z.js +166 -0
- package/dist/summary-D7Ttky0z.js.map +1 -0
- package/dist/text-gEcKjAyp.js +29 -0
- package/dist/text-gEcKjAyp.js.map +1 -0
- package/dist/types-C5vI3g1C.js +9 -0
- package/dist/types-C5vI3g1C.js.map +1 -0
- package/dist/types-DLec-6Cx.d.ts +1535 -0
- package/dist/usage-DsYUqIUM.js +24 -0
- package/dist/usage-DsYUqIUM.js.map +1 -0
- package/dist/view/index.d.ts +645 -0
- package/dist/view/index.js +8 -0
- package/dist/views-BrHOv6wg.js +220 -0
- package/dist/views-BrHOv6wg.js.map +1 -0
- package/dist/views-CwCWLWqo.js +3 -0
- package/dist/worker-entry-DnYkANPG.js +506 -0
- package/dist/worker-entry-DnYkANPG.js.map +1 -0
- package/dist/workspace-C2S0Z6gl.js +327 -0
- package/dist/workspace-C2S0Z6gl.js.map +1 -0
- package/dist/workspace-CyoXG6CY.js +2 -0
- package/docker/Dockerfile +19 -0
- package/package.json +79 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
//#region src/view/paths.ts
|
|
2
|
+
/**
|
|
3
|
+
* The studio's data API is a file layout: `reeval studio` serves these relative paths and
|
|
4
|
+
* `reeval studio build` writes them to disk, so the web client reads both the same way.
|
|
5
|
+
*
|
|
6
|
+
* studio.json mode, project and features
|
|
7
|
+
* data/index/manifest.json shard list and eval catalog
|
|
8
|
+
* data/index/<YYYY-MM>.json run headers started that UTC month, newest first
|
|
9
|
+
* data/runs/<runId>.json the record, byte for byte
|
|
10
|
+
* data/runs/<runId>/<artifact path> artifacts, raw
|
|
11
|
+
* data/analysis/<runId>.json the run's saved analyses (and, live, cached ones)
|
|
12
|
+
*/
|
|
13
|
+
const RUN_ID = /^\d{8}T\d{6}Z_[0-9a-z]+_[0-9a-z-]+_[0-9a-z]{4}$/;
|
|
14
|
+
const SHARD = /^\d{4}-\d{2}$/;
|
|
15
|
+
const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
16
|
+
const MAX_DEPTH = 8;
|
|
17
|
+
const studioPath = "studio.json";
|
|
18
|
+
const manifestPath = "data/index/manifest.json";
|
|
19
|
+
const shardPath = (id) => `data/index/${id}.json`;
|
|
20
|
+
const recordPath = (runId) => `data/runs/${runId}.json`;
|
|
21
|
+
const artifactPath = (runId, path) => `data/runs/${runId}/${path}`;
|
|
22
|
+
/** A shard id: the UTC month a run started in. */
|
|
23
|
+
const shardOf = (startedAt) => startedAt.slice(0, 7);
|
|
24
|
+
/**
|
|
25
|
+
* Maps a request path (relative, without a leading slash, already percent-decoded once) to what
|
|
26
|
+
* it names, or null. Every segment must match an allowlist, so `.`, `..`, empty segments,
|
|
27
|
+
* backslashes and NUL never get through.
|
|
28
|
+
*/
|
|
29
|
+
function parseDataPath(path) {
|
|
30
|
+
if (path === "studio.json") return { kind: "studio" };
|
|
31
|
+
if (path === "data/index/manifest.json") return { kind: "manifest" };
|
|
32
|
+
const parts = path.split("/");
|
|
33
|
+
if (parts[0] !== "data" || parts.length < 3) return null;
|
|
34
|
+
const [, area, first, ...rest] = parts;
|
|
35
|
+
if (area === "index" && rest.length === 0 && first.endsWith(".json")) {
|
|
36
|
+
const id = first.slice(0, -5);
|
|
37
|
+
return SHARD.test(id) ? {
|
|
38
|
+
kind: "shard",
|
|
39
|
+
id
|
|
40
|
+
} : null;
|
|
41
|
+
}
|
|
42
|
+
if (area === "analysis" && rest.length === 0 && first.endsWith(".json")) {
|
|
43
|
+
const runId = first.slice(0, -5);
|
|
44
|
+
return RUN_ID.test(runId) ? {
|
|
45
|
+
kind: "analysis",
|
|
46
|
+
runId
|
|
47
|
+
} : null;
|
|
48
|
+
}
|
|
49
|
+
if (area !== "runs") return null;
|
|
50
|
+
if (rest.length === 0) {
|
|
51
|
+
if (!first.endsWith(".json")) return null;
|
|
52
|
+
const runId = first.slice(0, -5);
|
|
53
|
+
return RUN_ID.test(runId) ? {
|
|
54
|
+
kind: "record",
|
|
55
|
+
runId
|
|
56
|
+
} : null;
|
|
57
|
+
}
|
|
58
|
+
if (!RUN_ID.test(first) || rest.length > MAX_DEPTH) return null;
|
|
59
|
+
if (!rest.every((s) => SEGMENT.test(s) && s !== "." && s !== "..")) return null;
|
|
60
|
+
return {
|
|
61
|
+
kind: "artifact",
|
|
62
|
+
runId: first,
|
|
63
|
+
segments: rest
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
const TYPES = [[/\.json$/, "application/json; charset=utf-8"], [/\.jsonl$/, "application/json; charset=utf-8"]];
|
|
67
|
+
/** Artifacts are untrusted agent output: JSON stays JSON, everything else is plain text. */
|
|
68
|
+
const artifactType = (name) => TYPES.find(([re]) => re.test(name))?.[1] ?? "text/plain; charset=utf-8";
|
|
69
|
+
//#endregion
|
|
70
|
+
export { parseDataPath as a, shardPath as c, manifestPath as i, studioPath as l, artifactPath as n, recordPath as o, artifactType as r, shardOf as s, RUN_ID as t };
|
|
71
|
+
|
|
72
|
+
//# sourceMappingURL=paths-CBvm6pap.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paths-CBvm6pap.js","names":[],"sources":["../src/view/paths.ts"],"mappings":";;;;;;;;;;;;AAYA,MAAa,SAAS;AACtB,MAAM,QAAQ;AACd,MAAM,UAAU;AAChB,MAAM,YAAY;AAElB,MAAa,aAAa;AAC1B,MAAa,eAAe;AAC5B,MAAa,aAAa,OAAe,cAAc,GAAG;AAC1D,MAAa,cAAc,UAAkB,aAAa,MAAM;AAChE,MAAa,gBAAgB,OAAe,SAAiB,aAAa,MAAM,GAAG;;AAGnF,MAAa,WAAW,cAAsB,UAAU,MAAM,GAAG,CAAC;;;;;;AAelE,SAAgB,cAAc,MAAiC;CAC7D,IAAI,SAAA,eAAqB,OAAO,EAAE,MAAM,SAAS;CACjD,IAAI,SAAA,4BAAuB,OAAO,EAAE,MAAM,WAAW;CACrD,MAAM,QAAQ,KAAK,MAAM,GAAG;CAC5B,IAAI,MAAM,OAAO,UAAU,MAAM,SAAS,GAAG,OAAO;CACpD,MAAM,GAAG,MAAM,OAAO,GAAG,QAAQ;CACjC,IAAI,SAAS,WAAW,KAAK,WAAW,KAAK,MAAM,SAAS,OAAO,GAAG;EACpE,MAAM,KAAK,MAAM,MAAM,GAAG,EAAE;EAC5B,OAAO,MAAM,KAAK,EAAE,IAAI;GAAE,MAAM;GAAS;EAAG,IAAI;CAClD;CACA,IAAI,SAAS,cAAc,KAAK,WAAW,KAAK,MAAM,SAAS,OAAO,GAAG;EACvE,MAAM,QAAQ,MAAM,MAAM,GAAG,EAAE;EAC/B,OAAO,OAAO,KAAK,KAAK,IAAI;GAAE,MAAM;GAAY;EAAM,IAAI;CAC5D;CACA,IAAI,SAAS,QAAQ,OAAO;CAC5B,IAAI,KAAK,WAAW,GAAG;EACrB,IAAI,CAAC,MAAM,SAAS,OAAO,GAAG,OAAO;EACrC,MAAM,QAAQ,MAAM,MAAM,GAAG,EAAE;EAC/B,OAAO,OAAO,KAAK,KAAK,IAAI;GAAE,MAAM;GAAU;EAAM,IAAI;CAC1D;CACA,IAAI,CAAC,OAAO,KAAK,KAAK,KAAK,KAAK,SAAS,WAAW,OAAO;CAC3D,IAAI,CAAC,KAAK,OAAO,MAAM,QAAQ,KAAK,CAAC,KAAK,MAAM,OAAO,MAAM,IAAI,GAAG,OAAO;CAC3E,OAAO;EAAE,MAAM;EAAY,OAAO;EAAO,UAAU;CAAK;AAC1D;AAEA,MAAM,QAAiC,CACrC,CAAC,WAAW,iCAAiC,GAC7C,CAAC,YAAY,iCAAiC,CAChD;;AAGA,MAAa,gBAAgB,SAC3B,MAAM,MAAM,CAAC,QAAQ,GAAG,KAAK,IAAI,CAAC,CAAC,GAAG,MAAM"}
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import { t as UsageError } from "./errors-CSxfJ4qu.js";
|
|
2
|
+
import { n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
|
|
3
|
+
import { f as scratchDir, s as recentRecords } from "./files-D2-qn_Dl.js";
|
|
4
|
+
//#region src/plan.ts
|
|
5
|
+
/** With no history for a harness, an attempt is assumed to take this long (the fake agent: 1s). */
|
|
6
|
+
const DEFAULT_ATTEMPT_MS = 18e4;
|
|
7
|
+
const DEFAULT_FAKE_MS = 1e3;
|
|
8
|
+
/** The most recent runs whose durations feed the estimate. */
|
|
9
|
+
const HISTORY_RUNS = 30;
|
|
10
|
+
const median = (values) => {
|
|
11
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
12
|
+
const mid = Math.floor(sorted.length / 2);
|
|
13
|
+
return sorted.length % 2 ? sorted[mid] : ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2;
|
|
14
|
+
};
|
|
15
|
+
/** Attempt durations from the latest recorded and scratch runs, by eval and harness. */
|
|
16
|
+
async function historyDurations(root, historyDir) {
|
|
17
|
+
const records = await recentRecords([historyDir, scratchDir(root)], HISTORY_RUNS);
|
|
18
|
+
const byEval = /* @__PURE__ */ new Map();
|
|
19
|
+
const byHarness = /* @__PURE__ */ new Map();
|
|
20
|
+
const push = (map, key, ms) => map.set(key, [...map.get(key) ?? [], ms]);
|
|
21
|
+
for (const record of records) for (const e of record.evals ?? []) for (const a of e.attempts ?? []) {
|
|
22
|
+
if (a.outcome !== "pass" && a.outcome !== "fail") continue;
|
|
23
|
+
const harness = e.cells?.find((c) => c.key === a.cell)?.harness;
|
|
24
|
+
if (!harness) continue;
|
|
25
|
+
push(byEval, `${e.id}\0${harness}`, a.durationMs);
|
|
26
|
+
push(byHarness, harness, a.durationMs);
|
|
27
|
+
}
|
|
28
|
+
return (id, harness, kind) => {
|
|
29
|
+
const own = byEval.get(`${id}\0${harness}`);
|
|
30
|
+
if (own?.length) return median(own);
|
|
31
|
+
const any = byHarness.get(harness);
|
|
32
|
+
if (any?.length) return median(any);
|
|
33
|
+
return kind === "fake" ? DEFAULT_FAKE_MS : DEFAULT_ATTEMPT_MS;
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
async function buildPlan(options) {
|
|
37
|
+
const { planned } = options;
|
|
38
|
+
const estimate = await historyDurations(options.root, options.historyDir);
|
|
39
|
+
const rows = /* @__PURE__ */ new Map();
|
|
40
|
+
const cells = /* @__PURE__ */ new Set();
|
|
41
|
+
const warnings = /* @__PURE__ */ new Set();
|
|
42
|
+
const sandboxWarnings = /* @__PURE__ */ new Map();
|
|
43
|
+
const sandboxes = /* @__PURE__ */ new Map();
|
|
44
|
+
const descriptors = /* @__PURE__ */ new Map();
|
|
45
|
+
const harnesses = /* @__PURE__ */ new Set();
|
|
46
|
+
const counted = /* @__PURE__ */ new Map();
|
|
47
|
+
let attempts = 0;
|
|
48
|
+
let estimateMs = 0;
|
|
49
|
+
for (const { id, attempt } of planned) {
|
|
50
|
+
if (!rows.has(id)) rows.set(id, {
|
|
51
|
+
id,
|
|
52
|
+
plan: attempt.plan
|
|
53
|
+
});
|
|
54
|
+
for (const w of attempt.plan.warnings) warnings.add(w);
|
|
55
|
+
if (attempt.plan.skip) continue;
|
|
56
|
+
for (const w of attempt.plan.sandboxWarnings) sandboxWarnings.set(`${w.file}\0${w.message}`, w);
|
|
57
|
+
attempts++;
|
|
58
|
+
cells.add(`${id}\0${attempt.cell.key}`);
|
|
59
|
+
estimateMs += estimate(id, attempt.cell.harness, attempt.harness.kind);
|
|
60
|
+
sandboxes.set(JSON.stringify(attempt.sandbox), attempt.sandbox);
|
|
61
|
+
descriptors.set(JSON.stringify(attempt.harness), attempt.harness);
|
|
62
|
+
harnesses.add(attempt.cell.harness);
|
|
63
|
+
const row = counted.get(id) ?? {
|
|
64
|
+
harnesses: /* @__PURE__ */ new Set(),
|
|
65
|
+
variants: /* @__PURE__ */ new Set(),
|
|
66
|
+
n: 0
|
|
67
|
+
};
|
|
68
|
+
row.harnesses.add(attempt.cell.harness);
|
|
69
|
+
if (attempt.cell.variant !== void 0) row.variants.add(attempt.cell.variant);
|
|
70
|
+
row.n++;
|
|
71
|
+
counted.set(id, row);
|
|
72
|
+
}
|
|
73
|
+
for (const row of rows.values()) {
|
|
74
|
+
const c = counted.get(row.id);
|
|
75
|
+
if (!c) continue;
|
|
76
|
+
const { variants } = row.plan;
|
|
77
|
+
row.plan = {
|
|
78
|
+
...row.plan,
|
|
79
|
+
harnesses: c.harnesses.size,
|
|
80
|
+
variants: variants && {
|
|
81
|
+
run: c.variants.size,
|
|
82
|
+
of: variants.of
|
|
83
|
+
},
|
|
84
|
+
attempts: c.n
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
profile: options.profile,
|
|
89
|
+
rows: [...rows.values()],
|
|
90
|
+
evals: [...rows.values()].filter((r) => !r.plan.skip).length,
|
|
91
|
+
cells: cells.size,
|
|
92
|
+
attempts,
|
|
93
|
+
estimateMs,
|
|
94
|
+
concurrency: options.concurrency,
|
|
95
|
+
budget: options.budget,
|
|
96
|
+
warnings: [...warnings],
|
|
97
|
+
sandboxWarnings: [...sandboxWarnings.values()].sort((a, b) => Number(a.kind !== "root") - Number(b.kind !== "root")),
|
|
98
|
+
hints: gateHint([...rows.values()]),
|
|
99
|
+
sandboxes: [...sandboxes.values()],
|
|
100
|
+
harnessDescriptors: [...descriptors.values()],
|
|
101
|
+
harnesses: [...harnesses]
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/** One line when evals run several times under the default gate, which every attempt must pass. */
|
|
105
|
+
function gateHint(rows) {
|
|
106
|
+
const n = rows.filter((r) => !r.plan.skip && r.plan.runs > 1 && r.plan.gate === void 0).length;
|
|
107
|
+
if (n === 0) return [];
|
|
108
|
+
return [`${plural(n, "eval")} ${n === 1 ? "runs" : "run"} more than once with the default gate, so every attempt must pass; set gate: { passRate: 0.8 } to gate on a pass rate instead`];
|
|
109
|
+
}
|
|
110
|
+
const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
|
|
111
|
+
/**
|
|
112
|
+
* `plan profile pr (auto: CI is set) · 3 evals · 4 cells · 12 attempts · ~9m serial · …`, led by
|
|
113
|
+
* `config ../reeval.config.ts` when `config` is given (the config isn't in the current directory).
|
|
114
|
+
*/
|
|
115
|
+
function planLine(plan, config) {
|
|
116
|
+
const parts = config ? [`config ${config}`] : [];
|
|
117
|
+
if (plan.profile) parts.push(`profile ${plan.profile.name} (${plan.profile.reason})`);
|
|
118
|
+
const skipped = plan.rows.length - plan.evals;
|
|
119
|
+
parts.push(plural(plan.evals, "eval") + (skipped ? ` (${skipped} skipped)` : ""));
|
|
120
|
+
parts.push(plural(plan.cells, "cell"), plural(plan.attempts, "attempt"));
|
|
121
|
+
const serial = `~${formatDuration(plan.estimateMs)} serial`;
|
|
122
|
+
const parallel = Math.min(plan.concurrency, plan.attempts);
|
|
123
|
+
parts.push(parallel > 1 ? `${serial} (~${formatDuration(plan.estimateMs / parallel)} at concurrency ${plan.concurrency})` : serial);
|
|
124
|
+
const { attempts, duration } = plan.budget;
|
|
125
|
+
const caps = [attempts !== void 0 ? `${plural(attempts, "attempt")} ${plan.attempts <= attempts ? "✓" : "✗"}` : "", duration !== void 0 ? `${duration}` : ""].filter(Boolean);
|
|
126
|
+
if (caps.length) parts.push(`budget ${caps.join(", ")}`);
|
|
127
|
+
return `plan ${parts.join(" · ")}`;
|
|
128
|
+
}
|
|
129
|
+
/** One row per eval: what it runs, and what the profile changed. */
|
|
130
|
+
/** The `--dry-run` table: one row per planned eval, then one per placeholder in `todos`. */
|
|
131
|
+
function planTable(plan, todos = []) {
|
|
132
|
+
const header = [
|
|
133
|
+
"eval",
|
|
134
|
+
"harnesses",
|
|
135
|
+
"variants",
|
|
136
|
+
"runs",
|
|
137
|
+
"attempts",
|
|
138
|
+
"notes"
|
|
139
|
+
];
|
|
140
|
+
const rows = plan.rows.map(({ id, plan: p }) => [
|
|
141
|
+
clip(id, 64),
|
|
142
|
+
String(p.harnesses),
|
|
143
|
+
p.variants === null ? "—" : p.variants.run === p.variants.of ? String(p.variants.run) : `${p.variants.run} of ${p.variants.of}`,
|
|
144
|
+
String(p.runs),
|
|
145
|
+
String(p.attempts),
|
|
146
|
+
[
|
|
147
|
+
p.skip ? `skipped: ${p.skip}` : "",
|
|
148
|
+
...p.notes,
|
|
149
|
+
p.brokered.length ? `brokered: ${p.brokered.join("; ")}` : "",
|
|
150
|
+
p.egress.length ? `egress: ${p.egress.join("; ")}` : "",
|
|
151
|
+
p.workspaces.length ? `workspace: ${p.workspaces.join("; ")}` : ""
|
|
152
|
+
].filter(Boolean).join(" · ")
|
|
153
|
+
]);
|
|
154
|
+
for (const id of todos) rows.push([
|
|
155
|
+
clip(id, 64),
|
|
156
|
+
"—",
|
|
157
|
+
"—",
|
|
158
|
+
"0",
|
|
159
|
+
"0",
|
|
160
|
+
"todo"
|
|
161
|
+
]);
|
|
162
|
+
const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => (r[i] ?? "").length)));
|
|
163
|
+
const line = (cols) => cols.map((c, i) => i === cols.length - 1 ? c : c.padEnd(widths[i] ?? 0)).join(" ").trimEnd();
|
|
164
|
+
return [line(header), ...rows.map(line)];
|
|
165
|
+
}
|
|
166
|
+
function planDocument(plan, planned, excluded, images) {
|
|
167
|
+
const evals = /* @__PURE__ */ new Map();
|
|
168
|
+
for (const { id, file, attempt } of planned) {
|
|
169
|
+
let e = evals.get(id);
|
|
170
|
+
if (!e) {
|
|
171
|
+
e = {
|
|
172
|
+
id,
|
|
173
|
+
file,
|
|
174
|
+
groups: attempt.groups,
|
|
175
|
+
name: attempt.name,
|
|
176
|
+
tags: attempt.tags,
|
|
177
|
+
skip: attempt.plan.skip ?? null,
|
|
178
|
+
notes: attempt.plan.notes,
|
|
179
|
+
cells: []
|
|
180
|
+
};
|
|
181
|
+
evals.set(id, e);
|
|
182
|
+
}
|
|
183
|
+
if (attempt.plan.skip) continue;
|
|
184
|
+
const cell = e.cells.find((c) => c.key === attempt.cell.key);
|
|
185
|
+
if (cell) cell.runs++;
|
|
186
|
+
else e.cells.push({
|
|
187
|
+
...attempt.cell,
|
|
188
|
+
gate: attempt.gate,
|
|
189
|
+
...attempt.minLift !== void 0 ? { minLift: attempt.minLift } : {},
|
|
190
|
+
runs: 1,
|
|
191
|
+
brokered: attempt.brokered,
|
|
192
|
+
egress: attempt.egress,
|
|
193
|
+
...attempt.workspace && { workspace: {
|
|
194
|
+
repo: attempt.workspace.repo,
|
|
195
|
+
ref: attempt.workspace.ref ?? null,
|
|
196
|
+
path: attempt.workspace.path ?? null
|
|
197
|
+
} }
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
const { attempts, duration } = plan.budget;
|
|
201
|
+
return {
|
|
202
|
+
profile: plan.profile && {
|
|
203
|
+
name: plan.profile.name,
|
|
204
|
+
reason: plan.profile.reason
|
|
205
|
+
},
|
|
206
|
+
totals: {
|
|
207
|
+
evals: plan.evals,
|
|
208
|
+
skipped: plan.rows.length - plan.evals,
|
|
209
|
+
cells: plan.cells,
|
|
210
|
+
attempts: plan.attempts
|
|
211
|
+
},
|
|
212
|
+
budget: {
|
|
213
|
+
attempts: attempts ?? null,
|
|
214
|
+
duration: duration ?? null,
|
|
215
|
+
fits: attempts === void 0 || plan.attempts <= attempts
|
|
216
|
+
},
|
|
217
|
+
concurrency: plan.concurrency,
|
|
218
|
+
estimateMs: plan.estimateMs,
|
|
219
|
+
warnings: plan.warnings,
|
|
220
|
+
sandboxWarnings: plan.sandboxWarnings,
|
|
221
|
+
hints: plan.hints,
|
|
222
|
+
evals: [...evals.values()],
|
|
223
|
+
excluded,
|
|
224
|
+
images
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
/** Exit 2 before anything starts when the plan is over `budget.attempts` (unless `--force`). */
|
|
228
|
+
function checkAttemptBudget(plan, force, configFile) {
|
|
229
|
+
const cap = plan.budget.attempts;
|
|
230
|
+
if (cap === void 0 || plan.attempts <= cap || force) return;
|
|
231
|
+
const where = plan.profile ? `profile ${plan.profile.name}` : configFile;
|
|
232
|
+
throw new UsageError(`this run plans ${plan.attempts} attempts, over budget.attempts ${cap} (${where}). Narrow it with --tag or --runs, or raise the budget. (--force runs it anyway)`);
|
|
233
|
+
}
|
|
234
|
+
//#endregion
|
|
235
|
+
export { buildPlan, checkAttemptBudget, planDocument, planLine, planTable };
|
|
236
|
+
|
|
237
|
+
//# sourceMappingURL=plan-gwk6dJ8c.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plan-gwk6dJ8c.js","names":[],"sources":["../src/plan.ts"],"mappings":";;;;;AAmBA,MAAM,qBAAqB;AAC3B,MAAM,kBAAkB;;AAExB,MAAM,eAAe;AA0BrB,MAAM,UAAU,WAAqB;CACnC,MAAM,SAAS,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;CAC/C,MAAM,MAAM,KAAK,MAAM,OAAO,SAAS,CAAC;CACxC,OAAO,OAAO,SAAS,IAClB,OAAO,SACN,OAAO,MAAM,MAAM,MAAM,OAAO,QAAQ,MAAM;AACtD;;AAGA,eAAsB,iBAAiB,MAAc,YAAoB;CACvE,MAAM,UAAU,MAAM,cAAc,CAAC,YAAY,WAAW,IAAI,CAAC,GAAG,YAAY;CAChF,MAAM,yBAAS,IAAI,IAAsB;CACzC,MAAM,4BAAY,IAAI,IAAsB;CAC5C,MAAM,QAAQ,KAA4B,KAAa,OACrD,IAAI,IAAI,KAAK,CAAC,GAAI,IAAI,IAAI,GAAG,KAAK,CAAC,GAAI,EAAE,CAAC;CAC5C,KAAK,MAAM,UAAU,SACnB,KAAK,MAAM,KAAK,OAAO,SAAS,CAAC,GAC/B,KAAK,MAAM,KAAK,EAAE,YAAY,CAAC,GAAG;EAChC,IAAI,EAAE,YAAY,UAAU,EAAE,YAAY,QAAQ;EAClD,MAAM,UAAU,EAAE,OAAO,MAAM,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,EAAE;EACxD,IAAI,CAAC,SAAS;EACd,KAAK,QAAQ,GAAG,EAAE,GAAG,IAAI,WAAW,EAAE,UAAU;EAChD,KAAK,WAAW,SAAS,EAAE,UAAU;CACvC;CAGJ,QAAQ,IAAY,SAAiB,SAAiB;EACpD,MAAM,MAAM,OAAO,IAAI,GAAG,GAAG,IAAI,SAAS;EAC1C,IAAI,KAAK,QAAQ,OAAO,OAAO,GAAG;EAClC,MAAM,MAAM,UAAU,IAAI,OAAO;EACjC,IAAI,KAAK,QAAQ,OAAO,OAAO,GAAG;EAClC,OAAO,SAAS,SAAS,kBAAkB;CAC7C;AACF;AAEA,eAAsB,UAAU,SAOX;CACnB,MAAM,EAAE,YAAY;CACpB,MAAM,WAAW,MAAM,iBAAiB,QAAQ,MAAM,QAAQ,UAAU;CACxE,MAAM,uBAAO,IAAI,IAAqB;CACtC,MAAM,wBAAQ,IAAI,IAAY;CAC9B,MAAM,2BAAW,IAAI,IAAY;CACjC,MAAM,kCAAkB,IAAI,IAA4B;CACxD,MAAM,4BAAY,IAAI,IAA+B;CACrD,MAAM,8BAAc,IAAI,IAA+B;CACvD,MAAM,4BAAY,IAAI,IAAY;CAElC,MAAM,0BAAU,IAAI,IAA0E;CAC9F,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,KAAK,MAAM,EAAE,IAAI,aAAa,SAAS;EACrC,IAAI,CAAC,KAAK,IAAI,EAAE,GAAG,KAAK,IAAI,IAAI;GAAE;GAAI,MAAM,QAAQ;EAAK,CAAC;EAC1D,KAAK,MAAM,KAAK,QAAQ,KAAK,UAAU,SAAS,IAAI,CAAC;EACrD,IAAI,QAAQ,KAAK,MAAM;EACvB,KAAK,MAAM,KAAK,QAAQ,KAAK,iBAC3B,gBAAgB,IAAI,GAAG,EAAE,KAAK,IAAI,EAAE,WAAW,CAAC;EAElD;EACA,MAAM,IAAI,GAAG,GAAG,IAAI,QAAQ,KAAK,KAAK;EACtC,cAAc,SAAS,IAAI,QAAQ,KAAK,SAAS,QAAQ,QAAQ,IAAI;EACrE,UAAU,IAAI,KAAK,UAAU,QAAQ,OAAO,GAAG,QAAQ,OAAO;EAC9D,YAAY,IAAI,KAAK,UAAU,QAAQ,OAAO,GAAG,QAAQ,OAAO;EAChE,UAAU,IAAI,QAAQ,KAAK,OAAO;EAClC,MAAM,MAAM,QAAQ,IAAI,EAAE,KAAK;GAAE,2BAAW,IAAI,IAAI;GAAG,0BAAU,IAAI,IAAI;GAAG,GAAG;EAAE;EACjF,IAAI,UAAU,IAAI,QAAQ,KAAK,OAAO;EACtC,IAAI,QAAQ,KAAK,YAAY,KAAA,GAAW,IAAI,SAAS,IAAI,QAAQ,KAAK,OAAO;EAC7E,IAAI;EACJ,QAAQ,IAAI,IAAI,GAAG;CACrB;CACA,KAAK,MAAM,OAAO,KAAK,OAAO,GAAG;EAC/B,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE;EAC5B,IAAI,CAAC,GAAG;EACR,MAAM,EAAE,aAAa,IAAI;EACzB,IAAI,OAAO;GACT,GAAG,IAAI;GACP,WAAW,EAAE,UAAU;GACvB,UAAU,YAAY;IAAE,KAAK,EAAE,SAAS;IAAM,IAAI,SAAS;GAAG;GAC9D,UAAU,EAAE;EACd;CACF;CACA,OAAO;EACL,SAAS,QAAQ;EACjB,MAAM,CAAC,GAAG,KAAK,OAAO,CAAC;EACvB,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;EACtD,OAAO,MAAM;EACb;EACA;EACA,aAAa,QAAQ;EACrB,QAAQ,QAAQ;EAChB,UAAU,CAAC,GAAG,QAAQ;EACtB,iBAAiB,CAAC,GAAG,gBAAgB,OAAO,CAAC,CAAC,CAAC,MAC5C,GAAG,MAAM,OAAO,EAAE,SAAS,MAAM,IAAI,OAAO,EAAE,SAAS,MAAM,CAChE;EACA,OAAO,SAAS,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC;EAClC,WAAW,CAAC,GAAG,UAAU,OAAO,CAAC;EACjC,oBAAoB,CAAC,GAAG,YAAY,OAAO,CAAC;EAC5C,WAAW,CAAC,GAAG,SAAS;CAC1B;AACF;;AAGA,SAAS,SAAS,MAA2B;CAC3C,MAAM,IAAI,KAAK,QAAQ,MAAM,CAAC,EAAE,KAAK,QAAQ,EAAE,KAAK,OAAO,KAAK,EAAE,KAAK,SAAS,KAAA,CAAS,CAAC,CAAC;CAC3F,IAAI,MAAM,GAAG,OAAO,CAAC;CACrB,OAAO,CACL,GAAG,OAAO,GAAG,MAAM,EAAE,GAAG,MAAM,IAAI,SAAS,MAAM,8HACnD;AACF;AAEA,MAAM,UAAU,GAAW,SAAiB,GAAG,EAAE,GAAG,OAAO,MAAM,IAAI,KAAK;;;;;AAM1E,SAAgB,SAAS,MAAe,QAAyB;CAC/D,MAAM,QAAkB,SAAS,CAAC,UAAU,QAAQ,IAAI,CAAC;CACzD,IAAI,KAAK,SAAS,MAAM,KAAK,WAAW,KAAK,QAAQ,KAAK,IAAI,KAAK,QAAQ,OAAO,EAAE;CACpF,MAAM,UAAU,KAAK,KAAK,SAAS,KAAK;CACxC,MAAM,KAAK,OAAO,KAAK,OAAO,MAAM,KAAK,UAAU,KAAK,QAAQ,aAAa,GAAG;CAChF,MAAM,KAAK,OAAO,KAAK,OAAO,MAAM,GAAG,OAAO,KAAK,UAAU,SAAS,CAAC;CACvE,MAAM,SAAS,IAAI,eAAe,KAAK,UAAU,EAAE;CACnD,MAAM,WAAW,KAAK,IAAI,KAAK,aAAa,KAAK,QAAQ;CACzD,MAAM,KACJ,WAAW,IACP,GAAG,OAAO,KAAK,eAAe,KAAK,aAAa,QAAQ,EAAE,kBAAkB,KAAK,YAAY,KAC7F,MACN;CACA,MAAM,EAAE,UAAU,aAAa,KAAK;CACpC,MAAM,OAAO,CACX,aAAa,KAAA,IACT,GAAG,OAAO,UAAU,SAAS,EAAE,GAAG,KAAK,YAAY,WAAW,MAAM,QACpE,IACJ,aAAa,KAAA,IAAY,GAAG,aAAa,EAC3C,CAAC,CAAC,OAAO,OAAO;CAChB,IAAI,KAAK,QAAQ,MAAM,KAAK,UAAU,KAAK,KAAK,IAAI,GAAG;CACvD,OAAO,SAAS,MAAM,KAAK,KAAK;AAClC;;;AAIA,SAAgB,UAAU,MAAe,QAA2B,CAAC,GAAa;CAChF,MAAM,SAAS;EAAC;EAAQ;EAAa;EAAY;EAAQ;EAAY;CAAO;CAC5E,MAAM,OAAO,KAAK,KAAK,KAAK,EAAE,IAAI,MAAM,QAAQ;EAC9C,KAAK,IAAI,EAAE;EACX,OAAO,EAAE,SAAS;EAClB,EAAE,aAAa,OACX,MACA,EAAE,SAAS,QAAQ,EAAE,SAAS,KAC5B,OAAO,EAAE,SAAS,GAAG,IACrB,GAAG,EAAE,SAAS,IAAI,MAAM,EAAE,SAAS;EACzC,OAAO,EAAE,IAAI;EACb,OAAO,EAAE,QAAQ;EACjB;GACE,EAAE,OAAO,YAAY,EAAE,SAAS;GAChC,GAAG,EAAE;GACL,EAAE,SAAS,SAAS,aAAa,EAAE,SAAS,KAAK,IAAI,MAAM;GAC3D,EAAE,OAAO,SAAS,WAAW,EAAE,OAAO,KAAK,IAAI,MAAM;GACrD,EAAE,WAAW,SAAS,cAAc,EAAE,WAAW,KAAK,IAAI,MAAM;EAClE,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,KAAK;CACf,CAAC;CACD,KAAK,MAAM,MAAM,OAAO,KAAK,KAAK;EAAC,KAAK,IAAI,EAAE;EAAG;EAAK;EAAK;EAAK;EAAK;CAAM,CAAC;CAC5E,MAAM,SAAS,OAAO,KAAK,GAAG,MAAM,KAAK,IAAI,EAAE,QAAQ,GAAG,KAAK,KAAK,OAAO,EAAE,MAAM,GAAA,CAAI,MAAM,CAAC,CAAC;CAC/F,MAAM,QAAQ,SACZ,KACG,KAAK,GAAG,MAAO,MAAM,KAAK,SAAS,IAAI,IAAI,EAAE,OAAO,OAAO,MAAM,CAAC,CAAE,CAAC,CACrE,KAAK,IAAI,CAAC,CACV,QAAQ;CACb,OAAO,CAAC,KAAK,MAAM,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC;AACzC;AAkDA,SAAgB,aACd,MACA,SACA,UACA,QACc;CACd,MAAM,wBAAQ,IAAI,IAAsB;CACxC,KAAK,MAAM,EAAE,IAAI,MAAM,aAAa,SAAS;EAC3C,IAAI,IAAI,MAAM,IAAI,EAAE;EACpB,IAAI,CAAC,GAAG;GACN,IAAI;IACF;IACA;IACA,QAAQ,QAAQ;IAChB,MAAM,QAAQ;IACd,MAAM,QAAQ;IACd,MAAM,QAAQ,KAAK,QAAQ;IAC3B,OAAO,QAAQ,KAAK;IACpB,OAAO,CAAC;GACV;GACA,MAAM,IAAI,IAAI,CAAC;EACjB;EACA,IAAI,QAAQ,KAAK,MAAM;EACvB,MAAM,OAAO,EAAE,MAAM,MAAM,MAAM,EAAE,QAAQ,QAAQ,KAAK,GAAG;EAC3D,IAAI,MAAM,KAAK;OAEb,EAAE,MAAM,KAAK;GACX,GAAG,QAAQ;GACX,MAAM,QAAQ;GACd,GAAI,QAAQ,YAAY,KAAA,IAAY,EAAE,SAAS,QAAQ,QAAQ,IAAI,CAAC;GACpE,MAAM;GACN,UAAU,QAAQ;GAClB,QAAQ,QAAQ;GAChB,GAAI,QAAQ,aAAa,EACvB,WAAW;IACT,MAAM,QAAQ,UAAU;IACxB,KAAK,QAAQ,UAAU,OAAO;IAC9B,MAAM,QAAQ,UAAU,QAAQ;GAClC,EACF;EACF,CAAC;CAEL;CACA,MAAM,EAAE,UAAU,aAAa,KAAK;CACpC,OAAO;EACL,SAAS,KAAK,WAAW;GAAE,MAAM,KAAK,QAAQ;GAAM,QAAQ,KAAK,QAAQ;EAAO;EAChF,QAAQ;GACN,OAAO,KAAK;GACZ,SAAS,KAAK,KAAK,SAAS,KAAK;GACjC,OAAO,KAAK;GACZ,UAAU,KAAK;EACjB;EACA,QAAQ;GACN,UAAU,YAAY;GACtB,UAAU,YAAY;GACtB,MAAM,aAAa,KAAA,KAAa,KAAK,YAAY;EACnD;EACA,aAAa,KAAK;EAClB,YAAY,KAAK;EACjB,UAAU,KAAK;EACf,iBAAiB,KAAK;EACtB,OAAO,KAAK;EACZ,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC;EACzB;EACA;CACF;AACF;;AAGA,SAAgB,mBAAmB,MAAe,OAAgB,YAA0B;CAC1F,MAAM,MAAM,KAAK,OAAO;CACxB,IAAI,QAAQ,KAAA,KAAa,KAAK,YAAY,OAAO,OAAO;CACxD,MAAM,QAAQ,KAAK,UAAU,WAAW,KAAK,QAAQ,SAAS;CAC9D,MAAM,IAAI,WACR,kBAAkB,KAAK,SAAS,kCAAkC,IAAI,IAAI,MAAM,iFAClF;AACF"}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { rt as RETRY_DELAYS } from "./src-DwwiVLqv.js";
|
|
2
|
+
import { c as levelOf } from "./options-CAQhu--C.js";
|
|
3
|
+
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
4
|
+
import { mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
const runWideLevel = (where) => {
|
|
7
|
+
const level = levelOf(where);
|
|
8
|
+
return level === "config" || level === "profile";
|
|
9
|
+
};
|
|
10
|
+
/** The distinct environments `uses` need, keyed by each provider. */
|
|
11
|
+
function prepareTargets(uses, providers, label) {
|
|
12
|
+
const targets = /* @__PURE__ */ new Map();
|
|
13
|
+
for (const use of uses) {
|
|
14
|
+
const provider = providers[use.sandbox.kind];
|
|
15
|
+
if (!provider) continue;
|
|
16
|
+
const key = `${use.sandbox.kind}\0${provider.key(use.sandbox, use.harness)}`;
|
|
17
|
+
const target = targets.get(key) ?? {
|
|
18
|
+
kind: use.sandbox.kind,
|
|
19
|
+
key: provider.key(use.sandbox, use.harness),
|
|
20
|
+
label: label(use),
|
|
21
|
+
sandbox: use.sandbox,
|
|
22
|
+
harness: use.harness,
|
|
23
|
+
setBy: [],
|
|
24
|
+
runWide: false
|
|
25
|
+
};
|
|
26
|
+
if (!target.setBy.includes(use.setBy.sandbox)) target.setBy.push(use.setBy.sandbox);
|
|
27
|
+
target.runWide ||= runWideLevel(use.setBy.sandbox) && runWideLevel(use.setBy.harness);
|
|
28
|
+
targets.set(key, target);
|
|
29
|
+
}
|
|
30
|
+
return [...targets.values()];
|
|
31
|
+
}
|
|
32
|
+
/** A config-level environment that can't be prepared: the run stops before any attempt. */
|
|
33
|
+
var RunStopError = class extends Error {
|
|
34
|
+
name = "RunStopError";
|
|
35
|
+
outcome;
|
|
36
|
+
constructor(outcome) {
|
|
37
|
+
super(`${outcome.failure.message}\nThe sandbox ${outcome.target.label} is set by ${outcome.target.setBy.join(", ")} for every eval, so nothing ran.`);
|
|
38
|
+
this.outcome = outcome;
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
const sleep = (ms, signal) => new Promise((resolve) => {
|
|
42
|
+
const timer = setTimeout(resolve, ms);
|
|
43
|
+
const done = () => {
|
|
44
|
+
clearTimeout(timer);
|
|
45
|
+
resolve();
|
|
46
|
+
};
|
|
47
|
+
signal.addEventListener("abort", done, { once: true });
|
|
48
|
+
});
|
|
49
|
+
/** Errors already retried as far as the policy allows, maybe by another target's step. */
|
|
50
|
+
const retried = /* @__PURE__ */ new WeakSet();
|
|
51
|
+
const failureOf = (provider, error) => error instanceof PrepareError ? error.failure : provider.classify(error);
|
|
52
|
+
/**
|
|
53
|
+
* Checks each provider, then prepares every target, `limit` at a time. A failed check stops
|
|
54
|
+
* everything (a {@link PrepareError}); so does a failed `runWide` target (a
|
|
55
|
+
* {@link RunStopError}), which also cancels the rest. Any other failure is its target's alone.
|
|
56
|
+
*/
|
|
57
|
+
async function prepareAll(targets, options) {
|
|
58
|
+
const { providers, root } = options;
|
|
59
|
+
const delays = options.delays ?? RETRY_DELAYS;
|
|
60
|
+
const stop = new AbortController();
|
|
61
|
+
const signal = options.signal ? AbortSignal.any([options.signal, stop.signal]) : stop.signal;
|
|
62
|
+
for (const kind of new Set(targets.map((t) => t.kind))) {
|
|
63
|
+
const provider = providers[kind];
|
|
64
|
+
try {
|
|
65
|
+
await provider.check();
|
|
66
|
+
} catch (error) {
|
|
67
|
+
throw error instanceof PrepareError ? error : new PrepareError(provider.classify(error));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (targets.length) options.onEvent?.({
|
|
71
|
+
state: "start",
|
|
72
|
+
targets
|
|
73
|
+
});
|
|
74
|
+
const one = async (target) => {
|
|
75
|
+
const provider = providers[target.kind];
|
|
76
|
+
const retry = async (step) => {
|
|
77
|
+
for (let i = 0;; i++) try {
|
|
78
|
+
return await step();
|
|
79
|
+
} catch (error) {
|
|
80
|
+
if (!(!(error instanceof Object && retried.has(error)) && failureOf(provider, error).class === "transient") || i >= delays.length || signal.aborted) {
|
|
81
|
+
if (error instanceof Object) retried.add(error);
|
|
82
|
+
throw error;
|
|
83
|
+
}
|
|
84
|
+
options.log?.(`retrying ${target.label}: ${String(error.message).split("\n")[0]}`);
|
|
85
|
+
await sleep(delays[i] ?? 0, signal);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
const start = performance.now();
|
|
89
|
+
try {
|
|
90
|
+
const ctx = {
|
|
91
|
+
root,
|
|
92
|
+
signal,
|
|
93
|
+
log: options.log ?? (() => {}),
|
|
94
|
+
retry
|
|
95
|
+
};
|
|
96
|
+
return {
|
|
97
|
+
target,
|
|
98
|
+
ok: true,
|
|
99
|
+
handle: await retry(() => provider.prepare(target.sandbox, target.harness, ctx)),
|
|
100
|
+
ms: performance.now() - start
|
|
101
|
+
};
|
|
102
|
+
} catch (error) {
|
|
103
|
+
return {
|
|
104
|
+
target,
|
|
105
|
+
ok: false,
|
|
106
|
+
failure: failureOf(provider, error),
|
|
107
|
+
ms: performance.now() - start
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
const outcomes = [];
|
|
112
|
+
let stopped = null;
|
|
113
|
+
let next = 0;
|
|
114
|
+
const worker = async () => {
|
|
115
|
+
while (next < targets.length && !signal.aborted) {
|
|
116
|
+
const target = targets[next++];
|
|
117
|
+
const outcome = await one(target);
|
|
118
|
+
if (stopped) return;
|
|
119
|
+
outcomes.push(outcome);
|
|
120
|
+
if (!outcome.ok && target.runWide && !options.signal?.aborted) {
|
|
121
|
+
stopped = outcome;
|
|
122
|
+
stop.abort();
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
options.onEvent?.({
|
|
126
|
+
state: outcome.ok ? "ready" : "failed",
|
|
127
|
+
outcome
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
const limit = Math.max(1, options.limit ?? 3);
|
|
132
|
+
await Promise.all(Array.from({ length: Math.min(limit, targets.length) }, worker));
|
|
133
|
+
if (stopped) throw new RunStopError(stopped);
|
|
134
|
+
options.signal?.throwIfAborted();
|
|
135
|
+
return targets.flatMap((t) => outcomes.filter((o) => o.target === t));
|
|
136
|
+
}
|
|
137
|
+
const preparedDir = (root) => join(root, ".reeval", "cache", "prepared");
|
|
138
|
+
const preparedFile = (root, runId) => join(preparedDir(root), `${runId}.json`);
|
|
139
|
+
/** Writes what the attempts start from, by provider and key, for the run's workers. */
|
|
140
|
+
function writePrepared(root, runId, outcomes) {
|
|
141
|
+
const doc = {};
|
|
142
|
+
for (const o of outcomes) {
|
|
143
|
+
const kind = doc[o.target.kind] ?? {};
|
|
144
|
+
doc[o.target.kind] = kind;
|
|
145
|
+
kind[o.target.key] = o.ok ? { handle: o.handle } : {
|
|
146
|
+
failure: o.failure,
|
|
147
|
+
label: o.target.label,
|
|
148
|
+
setBy: o.target.setBy
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
mkdirSync(preparedDir(root), { recursive: true });
|
|
152
|
+
writeFileSync(preparedFile(root, runId), JSON.stringify(doc));
|
|
153
|
+
for (const name of readdirSync(preparedDir(root))) {
|
|
154
|
+
const path = join(preparedDir(root), name);
|
|
155
|
+
if (Date.now() - statSync(path).mtimeMs > 864e5) rmSync(path, { force: true });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
const removePrepared = (root, runId) => rmSync(preparedFile(root, runId), { force: true });
|
|
159
|
+
const read = /* @__PURE__ */ new Map();
|
|
160
|
+
/**
|
|
161
|
+
* What an attempt starts from, as preflight prepared it: the handle, or the prepare failure its
|
|
162
|
+
* cells share, thrown as a {@link PrepareError}. Null when preflight didn't prepare this run.
|
|
163
|
+
*/
|
|
164
|
+
function preparedFor(root, runId, provider, sandbox, harness) {
|
|
165
|
+
const file = preparedFile(root, runId);
|
|
166
|
+
if (!read.has(file)) {
|
|
167
|
+
let doc = null;
|
|
168
|
+
try {
|
|
169
|
+
doc = JSON.parse(readFileSync(file, "utf8"));
|
|
170
|
+
} catch {}
|
|
171
|
+
read.set(file, doc);
|
|
172
|
+
}
|
|
173
|
+
const entry = read.get(file)?.[sandbox.kind]?.[provider.key(sandbox, harness)];
|
|
174
|
+
if (!entry) return void 0;
|
|
175
|
+
if ("failure" in entry) throw new PrepareError({
|
|
176
|
+
...entry.failure,
|
|
177
|
+
setBy: entry.setBy,
|
|
178
|
+
message: `${entry.failure.message}\n(the sandbox ${entry.label}, set by ${entry.setBy.join(", ")}, failed to prepare before the run started; every attempt that needs it fails with this)`
|
|
179
|
+
});
|
|
180
|
+
return entry.handle;
|
|
181
|
+
}
|
|
182
|
+
//#endregion
|
|
183
|
+
export { removePrepared as a, preparedFor as i, prepareAll as n, writePrepared as o, prepareTargets as r, RunStopError as t };
|
|
184
|
+
|
|
185
|
+
//# sourceMappingURL=preflight-BypZMNBG.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"preflight-BypZMNBG.js","names":[],"sources":["../src/sandbox/preflight.ts"],"mappings":";;;;;AAiDA,MAAM,gBAAgB,UAAkB;CACtC,MAAM,QAAQ,QAAQ,KAAK;CAC3B,OAAO,UAAU,YAAY,UAAU;AACzC;;AAGA,SAAgB,eACd,MACA,WACA,OACiB;CACjB,MAAM,0BAAU,IAAI,IAA2B;CAC/C,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,WAAW,UAAU,IAAI,QAAQ;EACvC,IAAI,CAAC,UAAU;EACf,MAAM,MAAM,GAAG,IAAI,QAAQ,KAAK,IAAI,SAAS,IAAI,IAAI,SAAS,IAAI,OAAO;EACzE,MAAM,SAAS,QAAQ,IAAI,GAAG,KAAK;GACjC,MAAM,IAAI,QAAQ;GAClB,KAAK,SAAS,IAAI,IAAI,SAAS,IAAI,OAAO;GAC1C,OAAO,MAAM,GAAG;GAChB,SAAS,IAAI;GACb,SAAS,IAAI;GACb,OAAO,CAAC;GACR,SAAS;EACX;EACA,IAAI,CAAC,OAAO,MAAM,SAAS,IAAI,MAAM,OAAO,GAAG,OAAO,MAAM,KAAK,IAAI,MAAM,OAAO;EAClF,OAAO,YAAY,aAAa,IAAI,MAAM,OAAO,KAAK,aAAa,IAAI,MAAM,OAAO;EACpF,QAAQ,IAAI,KAAK,MAAM;CACzB;CACA,OAAO,CAAC,GAAG,QAAQ,OAAO,CAAC;AAC7B;;AAGA,IAAa,eAAb,cAAkC,MAAM;CACtC,OAAgB;CAChB;CACA,YAAY,SAAiD;EAC3D,MACE,GAAG,QAAQ,QAAQ,QAAQ,gBAAgB,QAAQ,OAAO,MAAM,aAAa,QAAQ,OAAO,MAAM,KAAK,IAAI,EAAE,iCAC/G;EACA,KAAK,UAAU;CACjB;AACF;AAEA,MAAM,SAAS,IAAY,WACzB,IAAI,SAAe,YAAY;CAC7B,MAAM,QAAQ,WAAW,SAAS,EAAE;CACpC,MAAM,aAAa;EACjB,aAAa,KAAK;EAClB,QAAQ;CACV;CACA,OAAO,iBAAiB,SAAS,MAAM,EAAE,MAAM,KAAK,CAAC;AACvD,CAAC;;AAGH,MAAM,0BAAU,IAAI,QAAgB;AAEpC,MAAM,aAAa,UAA2B,UAC5C,iBAAiB,eAAe,MAAM,UAAU,SAAS,SAAS,KAAK;;;;;;AAOzE,eAAsB,WACpB,SACA,SAU2B;CAC3B,MAAM,EAAE,WAAW,SAAS;CAC5B,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,OAAO,IAAI,gBAAgB;CACjC,MAAM,SAAS,QAAQ,SAAS,YAAY,IAAI,CAAC,QAAQ,QAAQ,KAAK,MAAM,CAAC,IAAI,KAAK;CACtF,KAAK,MAAM,QAAQ,IAAI,IAAI,QAAQ,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG;EACtD,MAAM,WAAW,UAAU;EAC3B,IAAI;GACF,MAAM,SAAS,MAAM;EACvB,SAAS,OAAO;GACd,MAAM,iBAAiB,eAAe,QAAQ,IAAI,aAAa,SAAS,SAAS,KAAK,CAAC;EACzF;CACF;CACA,IAAI,QAAQ,QAAQ,QAAQ,UAAU;EAAE,OAAO;EAAS;CAAQ,CAAC;CAEjE,MAAM,MAAM,OAAO,WAAmD;EACpE,MAAM,WAAW,UAAU,OAAO;EAClC,MAAM,QAAQ,OAAU,SAAuC;GAC7D,KAAK,IAAI,IAAI,IAAK,KAChB,IAAI;IACF,OAAO,MAAM,KAAK;GACpB,SAAS,OAAO;IAGd,IAAI,EADc,EADL,iBAAiB,UAAU,QAAQ,IAAI,KAAK,MAC9B,UAAU,UAAU,KAAK,CAAC,CAAC,UAAU,gBAC9C,KAAK,OAAO,UAAU,OAAO,SAAS;KACtD,IAAI,iBAAiB,QAAQ,QAAQ,IAAI,KAAK;KAC9C,MAAM;IACR;IACA,QAAQ,MACN,YAAY,OAAO,MAAM,IAAI,OAAQ,MAAgB,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAC5E;IACA,MAAM,MAAM,OAAO,MAAM,GAAG,MAAM;GACpC;EAEJ;EACA,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI;GACF,MAAM,MAAM;IAAE;IAAM;IAAQ,KAAK,QAAQ,cAAc,CAAC;IAAI;GAAM;GAGlE,OAAO;IAAE;IAAQ,IAAI;IAAM,QAAA,MADN,YAAY,SAAS,QAAQ,OAAO,SAAS,OAAO,SAAS,GAAG,CAAC;IACnD,IAAI,YAAY,IAAI,IAAI;GAAM;EACnE,SAAS,OAAO;GAEd,OAAO;IAAE;IAAQ,IAAI;IAAO,SADZ,UAAU,UAAU,KACF;IAAG,IAAI,YAAY,IAAI,IAAI;GAAM;EACrE;CACF;CAEA,MAAM,WAA6B,CAAC;CACpC,IAAI,UAAyD;CAC7D,IAAI,OAAO;CACX,MAAM,SAAS,YAAY;EACzB,OAAO,OAAO,QAAQ,UAAU,CAAC,OAAO,SAAS;GAC/C,MAAM,SAAS,QAAQ;GACvB,MAAM,UAAU,MAAM,IAAI,MAAM;GAChC,IAAI,SAAS;GACb,SAAS,KAAK,OAAO;GACrB,IAAI,CAAC,QAAQ,MAAM,OAAO,WAAW,CAAC,QAAQ,QAAQ,SAAS;IAC7D,UAAU;IACV,KAAK,MAAM;IACX;GACF;GACA,QAAQ,UAAU;IAAE,OAAO,QAAQ,KAAK,UAAU;IAAU;GAAQ,CAAC;EACvE;CACF;CACA,MAAM,QAAQ,KAAK,IAAI,GAAG,QAAQ,SAAA,CAAsB;CACxD,MAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,OAAO,QAAQ,MAAM,EAAE,GAAG,MAAM,CAAC;CACjF,IAAI,SAAS,MAAM,IAAI,aAAa,OAAO;CAC3C,QAAQ,QAAQ,eAAe;CAC/B,OAAO,QAAQ,SAAS,MAAM,SAAS,QAAQ,MAAM,EAAE,WAAW,CAAC,CAAC;AACtE;AAMA,MAAM,eAAe,SAAiB,KAAK,MAAM,WAAW,SAAS,UAAU;AAC/E,MAAM,gBAAgB,MAAc,UAAkB,KAAK,YAAY,IAAI,GAAG,GAAG,MAAM,MAAM;;AAG7F,SAAgB,cAAc,MAAc,OAAe,UAAkC;CAC3F,MAAM,MAA6C,CAAC;CACpD,KAAK,MAAM,KAAK,UAAU;EACxB,MAAM,OAAO,IAAI,EAAE,OAAO,SAAS,CAAC;EACpC,IAAI,EAAE,OAAO,QAAQ;EACrB,KAAK,EAAE,OAAO,OAAO,EAAE,KACnB,EAAE,QAAQ,EAAE,OAAO,IACnB;GAAE,SAAS,EAAE;GAAS,OAAO,EAAE,OAAO;GAAO,OAAO,EAAE,OAAO;EAAM;CACzE;CACA,UAAU,YAAY,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;CAChD,cAAc,aAAa,MAAM,KAAK,GAAG,KAAK,UAAU,GAAG,CAAC;CAE5D,KAAK,MAAM,QAAQ,YAAY,YAAY,IAAI,CAAC,GAAG;EACjD,MAAM,OAAO,KAAK,YAAY,IAAI,GAAG,IAAI;EACzC,IAAI,KAAK,IAAI,IAAI,SAAS,IAAI,CAAC,CAAC,UAAU,OAAY,OAAO,MAAM,EAAE,OAAO,KAAK,CAAC;CACpF;AACF;AAEA,MAAa,kBAAkB,MAAc,UAC3C,OAAO,aAAa,MAAM,KAAK,GAAG,EAAE,OAAO,KAAK,CAAC;AAEnD,MAAM,uBAAO,IAAI,IAA0D;;;;;AAM3E,SAAgB,YACd,MACA,OACA,UACA,SACA,SAC4B;CAC5B,MAAM,OAAO,aAAa,MAAM,KAAK;CACrC,IAAI,CAAC,KAAK,IAAI,IAAI,GAAG;EACnB,IAAI,MAAoD;EACxD,IAAI;GACF,MAAM,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC;EAC7C,QAAQ,CAAC;EACT,KAAK,IAAI,MAAM,GAAG;CACpB;CACA,MAAM,QAAQ,KAAK,IAAI,IAAI,CAAC,GAAG,QAAQ,KAAK,GAAG,SAAS,IAAI,SAAS,OAAO;CAC5E,IAAI,CAAC,OAAO,OAAO,KAAA;CACnB,IAAI,aAAa,OACf,MAAM,IAAI,aAAa;EACrB,GAAG,MAAM;EACT,OAAO,MAAM;EACb,SAAS,GAAG,MAAM,QAAQ,QAAQ,iBAAiB,MAAM,MAAM,WAAW,MAAM,MAAM,KAAK,IAAI,EAAE;CACnG,CAAC;CAEH,OAAO,MAAM;AACf"}
|