reeval 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/THIRD_PARTY_NOTICES.md +2160 -0
- package/assets/package.json +9 -0
- package/assets/pnpm-lock.yaml +1336 -0
- package/dist/agent-DVHkx8aT.js +532 -0
- package/dist/agent-DVHkx8aT.js.map +1 -0
- package/dist/analysis-CxSCt1YY.js +16 -0
- package/dist/analysis-CxSCt1YY.js.map +1 -0
- package/dist/assemble-B-nmz8oH.js +695 -0
- package/dist/assemble-B-nmz8oH.js.map +1 -0
- package/dist/assemble-CmRVDZ2Q.js +2 -0
- package/dist/attempt-BdRQ9jKX.js +176 -0
- package/dist/attempt-BdRQ9jKX.js.map +1 -0
- package/dist/bridge/index.mjs +1911 -0
- package/dist/bridge/package.json +13 -0
- package/dist/bridge/pnpm-lock.yaml +1041 -0
- package/dist/bridge/pnpm-workspace.yaml +2 -0
- package/dist/budget-DBUrH4jX.js +8 -0
- package/dist/budget-DBUrH4jX.js.map +1 -0
- package/dist/cli--CQ3Iy9q.js +1865 -0
- package/dist/cli--CQ3Iy9q.js.map +1 -0
- package/dist/cli-spec-DKAu09Gi.js +478 -0
- package/dist/cli-spec-DKAu09Gi.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +8 -0
- package/dist/cli.js.map +1 -0
- package/dist/colors-QV9dFVhN.js +34 -0
- package/dist/colors-QV9dFVhN.js.map +1 -0
- package/dist/command-BSJzHMtP.js +280 -0
- package/dist/command-BSJzHMtP.js.map +1 -0
- package/dist/command-CEm8PZ1N.js +1043 -0
- package/dist/command-CEm8PZ1N.js.map +1 -0
- package/dist/command-D-jnQWVG.js +673 -0
- package/dist/command-D-jnQWVG.js.map +1 -0
- package/dist/command-DQAvbG7d.js +432 -0
- package/dist/command-DQAvbG7d.js.map +1 -0
- package/dist/commands-A4udfyKZ.js +2 -0
- package/dist/commands-CF8mOfiu.js +207 -0
- package/dist/commands-CF8mOfiu.js.map +1 -0
- package/dist/config-Cuj0kVYj.js +415 -0
- package/dist/config-Cuj0kVYj.js.map +1 -0
- package/dist/context-loi9OCZv.js +106 -0
- package/dist/context-loi9OCZv.js.map +1 -0
- package/dist/core-tLN-z1Be.js +96 -0
- package/dist/core-tLN-z1Be.js.map +1 -0
- package/dist/db-Cjj2ZsfQ.js +2 -0
- package/dist/db-DxDzMhtB.js +827 -0
- package/dist/db-DxDzMhtB.js.map +1 -0
- package/dist/debug-D24NOQH1.js +89 -0
- package/dist/debug-D24NOQH1.js.map +1 -0
- package/dist/debug-SEmW4w8f.js +2 -0
- package/dist/dist-BZXYHN2X.js +1673 -0
- package/dist/dist-BZXYHN2X.js.map +1 -0
- package/dist/docker-DQDZWXF9.js +20 -0
- package/dist/docker-DQDZWXF9.js.map +1 -0
- package/dist/docker-DnWAVdEm.js +2 -0
- package/dist/docker-oSbCExHL.js +153 -0
- package/dist/docker-oSbCExHL.js.map +1 -0
- package/dist/doctor-CxlH1pDM.js +525 -0
- package/dist/doctor-CxlH1pDM.js.map +1 -0
- package/dist/egress-73SQfKdx.js +619 -0
- package/dist/egress-73SQfKdx.js.map +1 -0
- package/dist/env-BQqj93gu.js +21 -0
- package/dist/env-BQqj93gu.js.map +1 -0
- package/dist/environment-Mz7wvrtD.js +552 -0
- package/dist/environment-Mz7wvrtD.js.map +1 -0
- package/dist/errors-C9w_ud7g.js +160 -0
- package/dist/errors-C9w_ud7g.js.map +1 -0
- package/dist/errors-CSxfJ4qu.js +11 -0
- package/dist/errors-CSxfJ4qu.js.map +1 -0
- package/dist/files--6CEUXm7.js +2 -0
- package/dist/files-BLlrhU4-.js +62 -0
- package/dist/files-BLlrhU4-.js.map +1 -0
- package/dist/files-Bd25OAQH.js +2 -0
- package/dist/files-BvHCFWgD.js +2 -0
- package/dist/files-D2-qn_Dl.js +275 -0
- package/dist/files-D2-qn_Dl.js.map +1 -0
- package/dist/files-bF2fPuT9.js +99 -0
- package/dist/files-bF2fPuT9.js.map +1 -0
- package/dist/git-D6hMeieS.js +44 -0
- package/dist/git-D6hMeieS.js.map +1 -0
- package/dist/harness-ls5aX1UO.js +131 -0
- package/dist/harness-ls5aX1UO.js.map +1 -0
- package/dist/harness.d.ts +74 -0
- package/dist/harness.js +2 -0
- package/dist/header-DzFDoMCI.js +78 -0
- package/dist/header-DzFDoMCI.js.map +1 -0
- package/dist/ids-Bp6ut7ZD.js +49 -0
- package/dist/ids-Bp6ut7ZD.js.map +1 -0
- package/dist/images-Cc1-coFc.js +286 -0
- package/dist/images-Cc1-coFc.js.map +1 -0
- package/dist/index.d.ts +97 -0
- package/dist/index.js +1959 -0
- package/dist/index.js.map +1 -0
- package/dist/init-CFljS8BJ.js +344 -0
- package/dist/init-CFljS8BJ.js.map +1 -0
- package/dist/init-z2ZPn9Tj.js +2 -0
- package/dist/invocations-rXktgSAn.js +157 -0
- package/dist/invocations-rXktgSAn.js.map +1 -0
- package/dist/jiti-BOLXpKyG.js +46 -0
- package/dist/jiti-BOLXpKyG.js.map +1 -0
- package/dist/judge-D8Uf4dA9.js +146 -0
- package/dist/judge-D8Uf4dA9.js.map +1 -0
- package/dist/judge.d.ts +21 -0
- package/dist/judge.js +19 -0
- package/dist/judge.js.map +1 -0
- package/dist/live-Bs3LReMV.js +98 -0
- package/dist/live-Bs3LReMV.js.map +1 -0
- package/dist/matchers-DyLvm7EG.js +374 -0
- package/dist/matchers-DyLvm7EG.js.map +1 -0
- package/dist/model-BdO79gWz.js +44 -0
- package/dist/model-BdO79gWz.js.map +1 -0
- package/dist/model-CDL98xue.js +725 -0
- package/dist/model-CDL98xue.js.map +1 -0
- package/dist/model-v-0_Ik7S.js +2 -0
- package/dist/node-kG64XxdE.js +506 -0
- package/dist/node-kG64XxdE.js.map +1 -0
- package/dist/opencode-providers-DZuD_oYt.js +3149 -0
- package/dist/opencode-providers-DZuD_oYt.js.map +1 -0
- package/dist/options-CAQhu--C.js +452 -0
- package/dist/options-CAQhu--C.js.map +1 -0
- package/dist/patch-BkKje4cp.js +69 -0
- package/dist/patch-BkKje4cp.js.map +1 -0
- package/dist/paths-CBvm6pap.js +72 -0
- package/dist/paths-CBvm6pap.js.map +1 -0
- package/dist/plan-gwk6dJ8c.js +237 -0
- package/dist/plan-gwk6dJ8c.js.map +1 -0
- package/dist/preflight-BtubXKQ8.js +2 -0
- package/dist/preflight-BypZMNBG.js +185 -0
- package/dist/preflight-BypZMNBG.js.map +1 -0
- package/dist/prepare-Cwv0bHs_.js +3 -0
- package/dist/prepare-D6Uzcubb.js +213 -0
- package/dist/prepare-D6Uzcubb.js.map +1 -0
- package/dist/provenance-Dfb8YY5Y.js +129 -0
- package/dist/provenance-Dfb8YY5Y.js.map +1 -0
- package/dist/provider-BDbNOQhQ.js +2 -0
- package/dist/provider-CmAYTQQs.js +14 -0
- package/dist/provider-CmAYTQQs.js.map +1 -0
- package/dist/providers-2p3ylMHI.js +107 -0
- package/dist/providers-2p3ylMHI.js.map +1 -0
- package/dist/providers-BUOxitcT.js +2 -0
- package/dist/query-CFATHZmV.js +91 -0
- package/dist/query-CFATHZmV.js.map +1 -0
- package/dist/query-DR4m6CXf.js +2 -0
- package/dist/record-DQ8wzyco.js +99 -0
- package/dist/record-DQ8wzyco.js.map +1 -0
- package/dist/redact-C4fL_bZP.js +167 -0
- package/dist/redact-C4fL_bZP.js.map +1 -0
- package/dist/results-CeXb8Wde.js +37 -0
- package/dist/results-CeXb8Wde.js.map +1 -0
- package/dist/root-CFDjpWuc.js +110 -0
- package/dist/root-CFDjpWuc.js.map +1 -0
- package/dist/sandbox-By2gkdkC.js +133 -0
- package/dist/sandbox-By2gkdkC.js.map +1 -0
- package/dist/sandbox.d.ts +41 -0
- package/dist/sandbox.js +2 -0
- package/dist/server-Bi61uiY0.js +350 -0
- package/dist/server-Bi61uiY0.js.map +1 -0
- package/dist/server-CsSBL8NC.js +2 -0
- package/dist/source-BXzrmof5.js +2 -0
- package/dist/source-Bsfa3LZH.js +40 -0
- package/dist/source-Bsfa3LZH.js.map +1 -0
- package/dist/src-55j1zNHH.js +3 -0
- package/dist/src-AEwxd1UP.js +602 -0
- package/dist/src-AEwxd1UP.js.map +1 -0
- package/dist/src-BDwSuyIo.js +3 -0
- package/dist/src-BQyMiWri.js +490 -0
- package/dist/src-BQyMiWri.js.map +1 -0
- package/dist/src-C-LHyTSV.js +3 -0
- package/dist/src-Cmp-7YrD.js +1136 -0
- package/dist/src-Cmp-7YrD.js.map +1 -0
- package/dist/src-D0RdqpNb.js +252 -0
- package/dist/src-D0RdqpNb.js.map +1 -0
- package/dist/src-D8KYzxBX.js +793 -0
- package/dist/src-D8KYzxBX.js.map +1 -0
- package/dist/src-DEgff9Z7.js +4 -0
- package/dist/src-DXx_29pq.js +567 -0
- package/dist/src-DXx_29pq.js.map +1 -0
- package/dist/src-DwwiVLqv.js +2937 -0
- package/dist/src-DwwiVLqv.js.map +1 -0
- package/dist/src-N7E_mfcZ.js +2 -0
- package/dist/src-W9g-y5X5.js +3 -0
- package/dist/src-daS0wKH2.js +3 -0
- package/dist/state-Cyw3QTsN.js +2 -0
- package/dist/state-DZ7eqb1F.js +167 -0
- package/dist/state-DZ7eqb1F.js.map +1 -0
- package/dist/stats-BNAHlNMD.js +114 -0
- package/dist/stats-BNAHlNMD.js.map +1 -0
- package/dist/studio-app/assets/index-D6XyfFSH.js +58 -0
- package/dist/studio-app/assets/index-xwzWdcE_.css +2 -0
- package/dist/studio-app/favicon.svg +1 -0
- package/dist/studio-app/index.html +15 -0
- package/dist/summary-D7Ttky0z.js +166 -0
- package/dist/summary-D7Ttky0z.js.map +1 -0
- package/dist/text-gEcKjAyp.js +29 -0
- package/dist/text-gEcKjAyp.js.map +1 -0
- package/dist/types-C5vI3g1C.js +9 -0
- package/dist/types-C5vI3g1C.js.map +1 -0
- package/dist/types-DLec-6Cx.d.ts +1535 -0
- package/dist/usage-DsYUqIUM.js +24 -0
- package/dist/usage-DsYUqIUM.js.map +1 -0
- package/dist/view/index.d.ts +645 -0
- package/dist/view/index.js +8 -0
- package/dist/views-BrHOv6wg.js +220 -0
- package/dist/views-BrHOv6wg.js.map +1 -0
- package/dist/views-CwCWLWqo.js +3 -0
- package/dist/worker-entry-DnYkANPG.js +506 -0
- package/dist/worker-entry-DnYkANPG.js.map +1 -0
- package/dist/workspace-C2S0Z6gl.js +327 -0
- package/dist/workspace-C2S0Z6gl.js.map +1 -0
- package/dist/workspace-CyoXG6CY.js +2 -0
- package/docker/Dockerfile +19 -0
- package/package.json +79 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { a as PACKAGE_NAMES, c as ownDir, n as COMPILED, u as ownRequire } from "./root-CFDjpWuc.js";
|
|
2
|
+
import { createRequire } from "node:module";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { createJiti } from "jiti";
|
|
5
|
+
//#region src/jiti.ts
|
|
6
|
+
let babel;
|
|
7
|
+
/**
|
|
8
|
+
* jiti's own Babel transform. jiti requires it lazily from beside its code, which a compiled es
|
|
9
|
+
* doesn't have, so reeval requires the same file: from the jiti package on disk, or, compiled, from
|
|
10
|
+
* the copy embedded in it (`vendor/jiti/babel.cjs`), copied out to disk.
|
|
11
|
+
*/
|
|
12
|
+
const jitiTransform = (options) => {
|
|
13
|
+
if (!babel) {
|
|
14
|
+
const file = COMPILED ? join(ownDir("vendor/jiti"), "babel.cjs") : join(dirname(ownRequire.resolve("jiti/package.json")), "dist/babel.cjs");
|
|
15
|
+
babel = createRequire(file)(file);
|
|
16
|
+
}
|
|
17
|
+
return babel(options);
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* A jiti that imports from `id`. Compiled, a file's imports of reeval itself get this es's own
|
|
21
|
+
* modules: a copy on disk would register evals, and hold state, the executable can't see.
|
|
22
|
+
*/
|
|
23
|
+
async function esJiti(id, options = {}) {
|
|
24
|
+
const virtualModules = COMPILED ? await ownModules() : void 0;
|
|
25
|
+
return createJiti(id, {
|
|
26
|
+
transform: jitiTransform,
|
|
27
|
+
...options,
|
|
28
|
+
...virtualModules && { virtualModules }
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
async function ownModules() {
|
|
32
|
+
const exports = {
|
|
33
|
+
"": import("./index.js"),
|
|
34
|
+
"/harness": import("./harness.js"),
|
|
35
|
+
"/sandbox": import("./sandbox.js"),
|
|
36
|
+
"/judge": import("./judge.js"),
|
|
37
|
+
"/view": import("./view/index.js")
|
|
38
|
+
};
|
|
39
|
+
const modules = {};
|
|
40
|
+
for (const [subpath, module] of Object.entries(exports)) for (const name of PACKAGE_NAMES) modules[name + subpath] = await module;
|
|
41
|
+
return modules;
|
|
42
|
+
}
|
|
43
|
+
//#endregion
|
|
44
|
+
export { esJiti as t };
|
|
45
|
+
|
|
46
|
+
//# sourceMappingURL=jiti-BOLXpKyG.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jiti-BOLXpKyG.js","names":[],"sources":["../src/jiti.ts"],"mappings":";;;;;AASA,IAAI;;;;;;AAOJ,MAAa,iBAA4B,YAAY;CACnD,IAAI,CAAC,OAAO;EACV,MAAM,OAAO,WACT,KAAK,OAAO,aAAa,GAAG,WAAW,IACvC,KAAK,QAAQ,WAAW,QAAQ,mBAAmB,CAAC,GAAG,gBAAgB;EAC3E,QAAQ,cAAc,IAAI,CAAC,CAAC,IAAI;CAClC;CACA,OAAO,MAAM,OAAO;AACtB;;;;;AAMA,eAAsB,OAAO,IAAY,UAAuB,CAAC,GAAkB;CACjF,MAAM,iBAAiB,WAAW,MAAM,WAAW,IAAI,KAAA;CACvD,OAAO,WAAW,IAAI;EACpB,WAAW;EACX,GAAG;EACH,GAAI,kBAAkB,EAAE,eAAe;CACzC,CAAC;AACH;AAEA,eAAe,aAA+C;CAC5D,MAAM,UAA4C;EAChD,IAAI,OAAO;EACX,YAAY,OAAO;EACnB,YAAY,OAAO;EACnB,UAAU,OAAO;EACjB,SAAS,OAAO;CAClB;CACA,MAAM,UAAmC,CAAC;CAC1C,KAAK,MAAM,CAAC,SAAS,WAAW,OAAO,QAAQ,OAAO,GACpD,KAAK,MAAM,QAAQ,eAAe,QAAQ,OAAO,WAAW,MAAM;CACpE,OAAO;AACT"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { setTimeout } from "node:timers/promises";
|
|
2
|
+
//#region src/judge/retry.ts
|
|
3
|
+
/** Up to six tries over about a minute: long enough to ride out a provider's "high demand". */
|
|
4
|
+
const JUDGE_RETRY = {
|
|
5
|
+
attempts: 6,
|
|
6
|
+
baseMs: 2e3,
|
|
7
|
+
maxMs: 3e4
|
|
8
|
+
};
|
|
9
|
+
const TRANSIENT_STATUS = /* @__PURE__ */ new Set([
|
|
10
|
+
408,
|
|
11
|
+
409,
|
|
12
|
+
425,
|
|
13
|
+
429
|
|
14
|
+
]);
|
|
15
|
+
const TRANSIENT_MESSAGE = /high demand|overloaded|unavailable|rate.?limit|resource.?exhausted|try again|timed? ?out|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|socket hang up|fetch failed/i;
|
|
16
|
+
/** The AI SDK wraps the provider's error once it has retried itself; look through that. */
|
|
17
|
+
const innermost = (error) => {
|
|
18
|
+
const e = error;
|
|
19
|
+
return e?.name === "AI_RetryError" && e.lastError ? innermost(e.lastError) : error;
|
|
20
|
+
};
|
|
21
|
+
/** Rate limits, 5xx, overload and network blips. Never a bad request, bad key or abort. */
|
|
22
|
+
function isTransient(error) {
|
|
23
|
+
const e = innermost(error);
|
|
24
|
+
if (!e || e.name === "AbortError") return false;
|
|
25
|
+
if (typeof e.statusCode === "number") return e.statusCode >= 500 || TRANSIENT_STATUS.has(e.statusCode) || e.isRetryable === true;
|
|
26
|
+
return e.isRetryable === true || TRANSIENT_MESSAGE.test(e.message ?? String(e));
|
|
27
|
+
}
|
|
28
|
+
function retryAfterMs(error) {
|
|
29
|
+
const header = innermost(error)?.responseHeaders?.["retry-after"];
|
|
30
|
+
const seconds = Number(header);
|
|
31
|
+
return Number.isFinite(seconds) && seconds > 0 ? seconds * 1e3 : 0;
|
|
32
|
+
}
|
|
33
|
+
/** Exponential backoff with equal jitter, at least any `retry-after`, never above `maxMs`. */
|
|
34
|
+
function backoffMs(policy, retry, error) {
|
|
35
|
+
const cap = Math.min(policy.maxMs, policy.baseMs * 2 ** retry);
|
|
36
|
+
const jittered = cap / 2 + Math.random() * (cap / 2);
|
|
37
|
+
return Math.round(Math.min(policy.maxMs, Math.max(jittered, retryAfterMs(error))));
|
|
38
|
+
}
|
|
39
|
+
/** Runs `call` until it succeeds, fails for good, or runs out of tries. */
|
|
40
|
+
async function withRetries(call, policy, signal) {
|
|
41
|
+
const retries = [];
|
|
42
|
+
for (;;) try {
|
|
43
|
+
return {
|
|
44
|
+
value: await call(),
|
|
45
|
+
retries
|
|
46
|
+
};
|
|
47
|
+
} catch (error) {
|
|
48
|
+
if (signal.aborted || retries.length + 1 >= policy.attempts || !isTransient(error)) {
|
|
49
|
+
const failed = error instanceof Error ? error : new Error(String(error));
|
|
50
|
+
failed.retries = retries;
|
|
51
|
+
throw failed;
|
|
52
|
+
}
|
|
53
|
+
const delayMs = backoffMs(policy, retries.length, error);
|
|
54
|
+
const e = innermost(error);
|
|
55
|
+
retries.push({
|
|
56
|
+
error: e?.message ?? String(e),
|
|
57
|
+
delayMs
|
|
58
|
+
});
|
|
59
|
+
await setTimeout(delayMs, void 0, { signal });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
//#endregion
|
|
63
|
+
//#region src/judge/index.ts
|
|
64
|
+
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
65
|
+
const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.13.0";
|
|
66
|
+
const DEFAULT_THRESHOLD = .8;
|
|
67
|
+
const GATEWAY_KEYS = ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"];
|
|
68
|
+
/** Providers reeval calls directly, through their own AI SDK package, when their key is set. */
|
|
69
|
+
const DIRECT_PROVIDERS = {
|
|
70
|
+
google: {
|
|
71
|
+
key: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
72
|
+
pkg: "@ai-sdk/google"
|
|
73
|
+
},
|
|
74
|
+
"typesafe-ai": {
|
|
75
|
+
key: "TYPESAFE_AI_API_KEY",
|
|
76
|
+
pkg: "@ai-sdk/typesafe-ai"
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
const isFakeJudge = (m) => typeof m === "object" && m !== null && m.kind === "fake-judge";
|
|
80
|
+
function judgeModelName(model) {
|
|
81
|
+
if (typeof model === "string") return model;
|
|
82
|
+
if (isFakeJudge(model)) return "fake";
|
|
83
|
+
const m = model;
|
|
84
|
+
return m.provider && m.modelId ? `${m.provider}/${m.modelId}` : "custom";
|
|
85
|
+
}
|
|
86
|
+
/** Checks a model string's shape; returns an error message, or undefined when it's valid. */
|
|
87
|
+
function checkModelString(model) {
|
|
88
|
+
if (model.includes("/")) return void 0;
|
|
89
|
+
return `Judge model "${model}" needs a provider prefix, e.g. "${RECOMMENDED_JUDGE_MODEL}" or "google/${model}"`;
|
|
90
|
+
}
|
|
91
|
+
const providerOf = (model) => model.slice(0, model.indexOf("/"));
|
|
92
|
+
/** Environment variables the judge model needs, as alternatives: any one of them is enough. */
|
|
93
|
+
function judgeCredentials(model) {
|
|
94
|
+
if (isFakeJudge(model)) return [];
|
|
95
|
+
const direct = DIRECT_PROVIDERS[providerOf(model)];
|
|
96
|
+
return direct ? [direct.key, ...GATEWAY_KEYS] : GATEWAY_KEYS;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* A known provider's own package when its key is set, else the Vercel AI Gateway when a
|
|
100
|
+
* Gateway credential is set. Throws, naming both, when neither is.
|
|
101
|
+
*/
|
|
102
|
+
function judgeRoute(model, env = process.env) {
|
|
103
|
+
const provider = providerOf(model);
|
|
104
|
+
const direct = DIRECT_PROVIDERS[provider];
|
|
105
|
+
if (direct && env[direct.key]) return {
|
|
106
|
+
kind: "direct",
|
|
107
|
+
provider,
|
|
108
|
+
pkg: direct.pkg
|
|
109
|
+
};
|
|
110
|
+
if (GATEWAY_KEYS.some((k) => env[k])) return { kind: "gateway" };
|
|
111
|
+
const options = direct ? `Set ${direct.key} to call ${direct.pkg} directly, or AI_GATEWAY_API_KEY (or VERCEL_OIDC_TOKEN) to go through the Vercel AI Gateway.` : "Set AI_GATEWAY_API_KEY (or VERCEL_OIDC_TOKEN) to go through the Vercel AI Gateway.";
|
|
112
|
+
throw new Error(`No credentials for judge model "${model}". ${options}`);
|
|
113
|
+
}
|
|
114
|
+
/** Transient provider errors (429, 5xx, "high demand") are retried with backoff and recorded. */
|
|
115
|
+
async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
|
|
116
|
+
const { model, threshold } = options;
|
|
117
|
+
let score;
|
|
118
|
+
let resolvedModel;
|
|
119
|
+
let retries = [];
|
|
120
|
+
if (isFakeJudge(model)) score = model.score;
|
|
121
|
+
else {
|
|
122
|
+
if (typeof model === "string") {
|
|
123
|
+
const problem = checkModelString(model);
|
|
124
|
+
if (problem) throw new Error(problem);
|
|
125
|
+
}
|
|
126
|
+
const { scoreWithModel } = await import("./model-BdO79gWz.js");
|
|
127
|
+
const scored = await withRetries(() => scoreWithModel(model, question, state, options.providerOptions, signal), retry, signal);
|
|
128
|
+
({score, resolvedModel} = scored.value);
|
|
129
|
+
retries = scored.retries;
|
|
130
|
+
}
|
|
131
|
+
return {
|
|
132
|
+
verdict: {
|
|
133
|
+
verdict: score >= threshold ? "pass" : "fail",
|
|
134
|
+
score,
|
|
135
|
+
threshold,
|
|
136
|
+
question,
|
|
137
|
+
model: judgeModelName(model),
|
|
138
|
+
...resolvedModel ? { resolvedModel } : {}
|
|
139
|
+
},
|
|
140
|
+
retries
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
//#endregion
|
|
144
|
+
export { isFakeJudge as a, judgeRoute as c, checkModelString as i, runJudge as l, DIRECT_PROVIDERS as n, judgeCredentials as o, RECOMMENDED_JUDGE_MODEL as r, judgeModelName as s, DEFAULT_THRESHOLD as t };
|
|
145
|
+
|
|
146
|
+
//# sourceMappingURL=judge-D8Uf4dA9.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-D8Uf4dA9.js","names":["sleep"],"sources":["../src/judge/retry.ts","../src/judge/index.ts"],"mappings":";;;AAOA,MAAa,cAA2B;CAAE,UAAU;CAAG,QAAQ;CAAO,OAAO;AAAO;AAEpF,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;AACrD,MAAM,oBACJ;;AAUF,MAAM,aAAa,UAA4B;CAC7C,MAAM,IAAI;CACV,OAAO,GAAG,SAAS,mBAAmB,EAAE,YAAY,UAAU,EAAE,SAAS,IAAI;AAC/E;;AAGA,SAAgB,YAAY,OAAyB;CACnD,MAAM,IAAI,UAAU,KAAK;CACzB,IAAI,CAAC,KAAK,EAAE,SAAS,cAAc,OAAO;CAC1C,IAAI,OAAO,EAAE,eAAe,UAC1B,OAAO,EAAE,cAAc,OAAO,iBAAiB,IAAI,EAAE,UAAU,KAAK,EAAE,gBAAgB;CAExF,OAAO,EAAE,gBAAgB,QAAQ,kBAAkB,KAAK,EAAE,WAAW,OAAO,CAAC,CAAC;AAChF;AAEA,SAAS,aAAa,OAAwB;CAC5C,MAAM,SAAU,UAAU,KAAK,CAAC,EAAoB,kBAAkB;CACtE,MAAM,UAAU,OAAO,MAAM;CAC7B,OAAO,OAAO,SAAS,OAAO,KAAK,UAAU,IAAI,UAAU,MAAO;AACpE;;AAGA,SAAgB,UAAU,QAAqB,OAAe,OAAwB;CACpF,MAAM,MAAM,KAAK,IAAI,OAAO,OAAO,OAAO,SAAS,KAAK,KAAK;CAC7D,MAAM,WAAW,MAAM,IAAI,KAAK,OAAO,KAAK,MAAM;CAClD,OAAO,KAAK,MAAM,KAAK,IAAI,OAAO,OAAO,KAAK,IAAI,UAAU,aAAa,KAAK,CAAC,CAAC,CAAC;AACnF;;AAKA,eAAsB,YACpB,MACA,QACA,QAC8C;CAC9C,MAAM,UAAwB,CAAC;CAC/B,SACE,IAAI;EACF,OAAO;GAAE,OAAO,MAAM,KAAK;GAAG;EAAQ;CACxC,SAAS,OAAO;EACd,IAAI,OAAO,WAAW,QAAQ,SAAS,KAAK,OAAO,YAAY,CAAC,YAAY,KAAK,GAAG;GAClF,MAAM,SAAU,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GACxE,OAAO,UAAU;GACjB,MAAM;EACR;EACA,MAAM,UAAU,UAAU,QAAQ,QAAQ,QAAQ,KAAK;EACvD,MAAM,IAAI,UAAU,KAAK;EACzB,QAAQ,KAAK;GAAE,OAAO,GAAG,WAAW,OAAO,CAAC;GAAG;EAAQ,CAAC;EACxD,MAAMA,WAAM,SAAS,KAAA,GAAW,EAAE,OAAO,CAAC;CAC5C;AAEJ;;;;ACpEA,MAAa,0BAA0B;AACvC,MAAa,oBAAoB;AAEjC,MAAM,eAAe,CAAC,sBAAsB,mBAAmB;;AAG/D,MAAa,mBAAiE;CAC5E,QAAQ;EAAE,KAAK;EAAgC,KAAK;CAAiB;CACrE,eAAe;EAAE,KAAK;EAAuB,KAAK;CAAsB;AAC1E;AAMA,MAAa,eAAe,MAC1B,OAAO,MAAM,YAAY,MAAM,QAAS,EAAqB,SAAS;AAExE,SAAgB,eAAe,OAA2B;CACxD,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI,YAAY,KAAK,GAAG,OAAO;CAC/B,MAAM,IAAI;CACV,OAAO,EAAE,YAAY,EAAE,UAAU,GAAG,EAAE,SAAS,GAAG,EAAE,YAAY;AAClE;;AAGA,SAAgB,iBAAiB,OAAmC;CAClE,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO,KAAA;CAChC,OAAO,gBAAgB,MAAM,mCAAmC,wBAAwB,eAAe,MAAM;AAC/G;AAEA,MAAM,cAAc,UAAkB,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;;AAGvE,SAAgB,iBAAiB,OAA0C;CACzE,IAAI,YAAY,KAAK,GAAG,OAAO,CAAC;CAChC,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SAAS,CAAC,OAAO,KAAK,GAAG,YAAY,IAAI;AAClD;;;;;AAMA,SAAgB,WAAW,OAAe,MAAyB,QAAQ,KAAiB;CAC1F,MAAM,WAAW,WAAW,KAAK;CACjC,MAAM,SAAS,iBAAiB;CAChC,IAAI,UAAU,IAAI,OAAO,MAAM,OAAO;EAAE,MAAM;EAAU;EAAU,KAAK,OAAO;CAAI;CAClF,IAAI,aAAa,MAAM,MAAM,IAAI,EAAE,GAAG,OAAO,EAAE,MAAM,UAAU;CAC/D,MAAM,UAAU,SACZ,OAAO,OAAO,IAAI,WAAW,OAAO,IAAI,gGACxC;CACJ,MAAM,IAAI,MAAM,mCAAmC,MAAM,KAAK,SAAS;AACzE;;AAMA,eAAsB,SACpB,UACA,OACA,SACA,QACA,QAAqB,aACC;CACtB,MAAM,EAAE,OAAO,cAAc;CAC7B,IAAI;CACJ,IAAI;CACJ,IAAI,UAAwB,CAAC;CAC7B,IAAI,YAAY,KAAK,GACnB,QAAQ,MAAM;MACT;EACL,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,UAAU,iBAAiB,KAAK;GACtC,IAAI,SAAS,MAAM,IAAI,MAAM,OAAO;EACtC;EACA,MAAM,EAAE,mBAAmB,MAAM,OAAO;EACxC,MAAM,SAAS,MAAM,kBACb,eAAe,OAAO,UAAU,OAAO,QAAQ,iBAAiB,MAAM,GAC5E,OACA,MACF;EACC,CAAC,CAAE,OAAO,iBAAkB,OAAO;EACpC,UAAU,OAAO;CACnB;CASA,OAAO;EAAE,SAAA;GAPP,SAAS,SAAS,YAAY,SAAS;GACvC;GACA;GACA;GACA,OAAO,eAAe,KAAK;GAC3B,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;EAE5B;EAAG;CAAQ;AAC5B"}
|
package/dist/judge.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { A as FakeJudgeModel, H as JudgeModel, O as FakeAnalysisModel, U as JudgeOptions, V as JudgeCallOptions, d as ConfigJudgeOptions, lt as Verdict, n as AnalysisConfig } from "./types-DLec-6Cx.js";
|
|
2
|
+
//#region src/judge/index.d.ts
|
|
3
|
+
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
4
|
+
export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.13.0";
|
|
5
|
+
export declare const DEFAULT_THRESHOLD = 0.8;
|
|
6
|
+
//#endregion
|
|
7
|
+
//#region src/judge.d.ts
|
|
8
|
+
/** A judge that always returns `score`, without calling a model. For tests and dry runs. */
|
|
9
|
+
export declare const fakeJudge: ({ score }?: {
|
|
10
|
+
score?: number;
|
|
11
|
+
}) => FakeJudgeModel;
|
|
12
|
+
/**
|
|
13
|
+
* An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
|
|
14
|
+
* model's structured answer, through the same parsing and validation as a real one.
|
|
15
|
+
*/
|
|
16
|
+
export declare const fakeAnalysis: ({ output }: {
|
|
17
|
+
output: FakeAnalysisModel['output'];
|
|
18
|
+
}) => FakeAnalysisModel;
|
|
19
|
+
//#endregion
|
|
20
|
+
export type { AnalysisConfig, ConfigJudgeOptions, FakeAnalysisModel, FakeJudgeModel, JudgeCallOptions, JudgeModel, JudgeOptions, Verdict };
|
|
21
|
+
//# sourceMappingURL=judge.d.ts.map
|
package/dist/judge.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { r as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-D8Uf4dA9.js";
|
|
2
|
+
//#region src/judge.ts
|
|
3
|
+
/** A judge that always returns `score`, without calling a model. For tests and dry runs. */
|
|
4
|
+
const fakeJudge = ({ score = 1 } = {}) => ({
|
|
5
|
+
kind: "fake-judge",
|
|
6
|
+
score
|
|
7
|
+
});
|
|
8
|
+
/**
|
|
9
|
+
* An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
|
|
10
|
+
* model's structured answer, through the same parsing and validation as a real one.
|
|
11
|
+
*/
|
|
12
|
+
const fakeAnalysis = ({ output }) => ({
|
|
13
|
+
kind: "fake-analysis",
|
|
14
|
+
output
|
|
15
|
+
});
|
|
16
|
+
//#endregion
|
|
17
|
+
export { DEFAULT_THRESHOLD, RECOMMENDED_JUDGE_MODEL, fakeAnalysis, fakeJudge };
|
|
18
|
+
|
|
19
|
+
//# sourceMappingURL=judge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;AAKA,MAAa,aAAa,EAAE,QAAQ,MAA0B,CAAC,OAAuB;CACpF,MAAM;CACN;AACF;;;;;AAMA,MAAa,gBAAgB,EAC3B,cAGwB;CACxB,MAAM;CACN;AACF"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { n as fillDefaults } from "./record-DQ8wzyco.js";
|
|
2
|
+
//#region src/view/live.ts
|
|
3
|
+
/** The progress file's name inside the run's staging directory. */
|
|
4
|
+
const LIVE_FILE = "live.json";
|
|
5
|
+
const LIVE_SCHEMA_VERSION = 1;
|
|
6
|
+
const liveApiPath = "api/live";
|
|
7
|
+
const liveTranscriptPath = (runId, attemptKey) => `api/live/${runId}/transcript?attempt=${encodeURIComponent(attemptKey)}`;
|
|
8
|
+
const FINISHED = [
|
|
9
|
+
"pass",
|
|
10
|
+
"fail",
|
|
11
|
+
"infra_error",
|
|
12
|
+
"skipped"
|
|
13
|
+
];
|
|
14
|
+
const isFinished = (status) => FINISHED.includes(status);
|
|
15
|
+
function liveCounts(attempts) {
|
|
16
|
+
const count = (...statuses) => attempts.filter((a) => statuses.includes(a.status)).length;
|
|
17
|
+
return {
|
|
18
|
+
done: count(...FINISHED),
|
|
19
|
+
passed: count("pass"),
|
|
20
|
+
failed: count("fail"),
|
|
21
|
+
errored: count("infra_error"),
|
|
22
|
+
skipped: count("skipped"),
|
|
23
|
+
running: count("running")
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
const STATUSES = [
|
|
27
|
+
"queued",
|
|
28
|
+
"running",
|
|
29
|
+
...FINISHED
|
|
30
|
+
];
|
|
31
|
+
const PHASES = [
|
|
32
|
+
"running",
|
|
33
|
+
"cancelling",
|
|
34
|
+
"finishing"
|
|
35
|
+
];
|
|
36
|
+
const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
37
|
+
const str = (v) => typeof v === "string";
|
|
38
|
+
const num = (v) => typeof v === "number" && Number.isFinite(v);
|
|
39
|
+
const orNull = (v, is) => v === null || v === void 0 || is(v);
|
|
40
|
+
function readableAttempt(v) {
|
|
41
|
+
if (!isObj(v)) return null;
|
|
42
|
+
const { key, evalId, name, file, cell, n, status, startedAt, durationMs } = v;
|
|
43
|
+
if (![
|
|
44
|
+
key,
|
|
45
|
+
evalId,
|
|
46
|
+
name,
|
|
47
|
+
file,
|
|
48
|
+
cell
|
|
49
|
+
].every(str) || !num(n)) return null;
|
|
50
|
+
if (!STATUSES.includes(status)) return null;
|
|
51
|
+
if (!orNull(startedAt, str) || !orNull(durationMs, num)) return null;
|
|
52
|
+
return {
|
|
53
|
+
key,
|
|
54
|
+
evalId,
|
|
55
|
+
name,
|
|
56
|
+
file,
|
|
57
|
+
cell,
|
|
58
|
+
n,
|
|
59
|
+
status,
|
|
60
|
+
startedAt: startedAt ?? null,
|
|
61
|
+
durationMs: durationMs ?? null
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* A progress file as the studio reads it, or null when it isn't one it can read: half-written,
|
|
66
|
+
* corrupt, or from a newer reeval. Counts are recomputed from the attempts, never trusted.
|
|
67
|
+
*/
|
|
68
|
+
function readableLiveProgress(value) {
|
|
69
|
+
if (!isObj(value) || value.schemaVersion !== 1) return null;
|
|
70
|
+
fillDefaults("live-progress", value);
|
|
71
|
+
const { runId, startedAt, pid, host, heartbeatAt, phase, attempts } = value;
|
|
72
|
+
if (![
|
|
73
|
+
runId,
|
|
74
|
+
startedAt,
|
|
75
|
+
host,
|
|
76
|
+
heartbeatAt
|
|
77
|
+
].every(str) || !num(pid)) return null;
|
|
78
|
+
if (!PHASES.includes(phase) || !Array.isArray(attempts)) return null;
|
|
79
|
+
const read = attempts.map(readableAttempt);
|
|
80
|
+
if (read.some((a) => a === null)) return null;
|
|
81
|
+
const list = read;
|
|
82
|
+
return {
|
|
83
|
+
schemaVersion: 1,
|
|
84
|
+
runId,
|
|
85
|
+
startedAt,
|
|
86
|
+
pid,
|
|
87
|
+
host,
|
|
88
|
+
heartbeatAt,
|
|
89
|
+
phase,
|
|
90
|
+
planned: list.length,
|
|
91
|
+
counts: liveCounts(list),
|
|
92
|
+
attempts: list
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
//#endregion
|
|
96
|
+
export { liveApiPath as a, readableLiveProgress as c, isFinished as i, LIVE_FILE as n, liveCounts as o, LIVE_SCHEMA_VERSION as r, liveTranscriptPath as s, FINISHED as t };
|
|
97
|
+
|
|
98
|
+
//# sourceMappingURL=live-Bs3LReMV.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"live-Bs3LReMV.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;AAOA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAsDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAA+B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEnG,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC;EAAC;EAAK;EAAQ;EAAM;EAAM;CAAI,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CACnE,IAAI,CAAC,SAAS,SAAS,MAAoB,GAAG,OAAO;CACrD,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACA;EACG;EACF;EACA;EACA;EACN;EACQ;EACR,WAAY,aAAoC;EAChD,YAAa,cAAqC;CACpD;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC;EAAC;EAAO;EAAW;EAAM;CAAW,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3E,IAAI,CAAC,OAAO,SAAS,KAAkB,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAC7E,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,KAAK,MAAM,MAAM,MAAM,IAAI,GAAG,OAAO;CACzC,MAAM,OAAO;CACb,OAAO;EACL,eAAA;EACO;EACI;EACX;EACM;EACO;EACN;EACP,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;CACZ;AACF"}
|