reeval 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/LICENSE +202 -0
  2. package/THIRD_PARTY_NOTICES.md +2160 -0
  3. package/assets/package.json +9 -0
  4. package/assets/pnpm-lock.yaml +1336 -0
  5. package/dist/agent-DVHkx8aT.js +532 -0
  6. package/dist/agent-DVHkx8aT.js.map +1 -0
  7. package/dist/analysis-CxSCt1YY.js +16 -0
  8. package/dist/analysis-CxSCt1YY.js.map +1 -0
  9. package/dist/assemble-B-nmz8oH.js +695 -0
  10. package/dist/assemble-B-nmz8oH.js.map +1 -0
  11. package/dist/assemble-CmRVDZ2Q.js +2 -0
  12. package/dist/attempt-BdRQ9jKX.js +176 -0
  13. package/dist/attempt-BdRQ9jKX.js.map +1 -0
  14. package/dist/bridge/index.mjs +1911 -0
  15. package/dist/bridge/package.json +13 -0
  16. package/dist/bridge/pnpm-lock.yaml +1041 -0
  17. package/dist/bridge/pnpm-workspace.yaml +2 -0
  18. package/dist/budget-DBUrH4jX.js +8 -0
  19. package/dist/budget-DBUrH4jX.js.map +1 -0
  20. package/dist/cli--CQ3Iy9q.js +1865 -0
  21. package/dist/cli--CQ3Iy9q.js.map +1 -0
  22. package/dist/cli-spec-DKAu09Gi.js +478 -0
  23. package/dist/cli-spec-DKAu09Gi.js.map +1 -0
  24. package/dist/cli.d.ts +1 -0
  25. package/dist/cli.js +8 -0
  26. package/dist/cli.js.map +1 -0
  27. package/dist/colors-QV9dFVhN.js +34 -0
  28. package/dist/colors-QV9dFVhN.js.map +1 -0
  29. package/dist/command-BSJzHMtP.js +280 -0
  30. package/dist/command-BSJzHMtP.js.map +1 -0
  31. package/dist/command-CEm8PZ1N.js +1043 -0
  32. package/dist/command-CEm8PZ1N.js.map +1 -0
  33. package/dist/command-D-jnQWVG.js +673 -0
  34. package/dist/command-D-jnQWVG.js.map +1 -0
  35. package/dist/command-DQAvbG7d.js +432 -0
  36. package/dist/command-DQAvbG7d.js.map +1 -0
  37. package/dist/commands-A4udfyKZ.js +2 -0
  38. package/dist/commands-CF8mOfiu.js +207 -0
  39. package/dist/commands-CF8mOfiu.js.map +1 -0
  40. package/dist/config-Cuj0kVYj.js +415 -0
  41. package/dist/config-Cuj0kVYj.js.map +1 -0
  42. package/dist/context-loi9OCZv.js +106 -0
  43. package/dist/context-loi9OCZv.js.map +1 -0
  44. package/dist/core-tLN-z1Be.js +96 -0
  45. package/dist/core-tLN-z1Be.js.map +1 -0
  46. package/dist/db-Cjj2ZsfQ.js +2 -0
  47. package/dist/db-DxDzMhtB.js +827 -0
  48. package/dist/db-DxDzMhtB.js.map +1 -0
  49. package/dist/debug-D24NOQH1.js +89 -0
  50. package/dist/debug-D24NOQH1.js.map +1 -0
  51. package/dist/debug-SEmW4w8f.js +2 -0
  52. package/dist/dist-BZXYHN2X.js +1673 -0
  53. package/dist/dist-BZXYHN2X.js.map +1 -0
  54. package/dist/docker-DQDZWXF9.js +20 -0
  55. package/dist/docker-DQDZWXF9.js.map +1 -0
  56. package/dist/docker-DnWAVdEm.js +2 -0
  57. package/dist/docker-oSbCExHL.js +153 -0
  58. package/dist/docker-oSbCExHL.js.map +1 -0
  59. package/dist/doctor-CxlH1pDM.js +525 -0
  60. package/dist/doctor-CxlH1pDM.js.map +1 -0
  61. package/dist/egress-73SQfKdx.js +619 -0
  62. package/dist/egress-73SQfKdx.js.map +1 -0
  63. package/dist/env-BQqj93gu.js +21 -0
  64. package/dist/env-BQqj93gu.js.map +1 -0
  65. package/dist/environment-Mz7wvrtD.js +552 -0
  66. package/dist/environment-Mz7wvrtD.js.map +1 -0
  67. package/dist/errors-C9w_ud7g.js +160 -0
  68. package/dist/errors-C9w_ud7g.js.map +1 -0
  69. package/dist/errors-CSxfJ4qu.js +11 -0
  70. package/dist/errors-CSxfJ4qu.js.map +1 -0
  71. package/dist/files--6CEUXm7.js +2 -0
  72. package/dist/files-BLlrhU4-.js +62 -0
  73. package/dist/files-BLlrhU4-.js.map +1 -0
  74. package/dist/files-Bd25OAQH.js +2 -0
  75. package/dist/files-BvHCFWgD.js +2 -0
  76. package/dist/files-D2-qn_Dl.js +275 -0
  77. package/dist/files-D2-qn_Dl.js.map +1 -0
  78. package/dist/files-bF2fPuT9.js +99 -0
  79. package/dist/files-bF2fPuT9.js.map +1 -0
  80. package/dist/git-D6hMeieS.js +44 -0
  81. package/dist/git-D6hMeieS.js.map +1 -0
  82. package/dist/harness-ls5aX1UO.js +131 -0
  83. package/dist/harness-ls5aX1UO.js.map +1 -0
  84. package/dist/harness.d.ts +74 -0
  85. package/dist/harness.js +2 -0
  86. package/dist/header-DzFDoMCI.js +78 -0
  87. package/dist/header-DzFDoMCI.js.map +1 -0
  88. package/dist/ids-Bp6ut7ZD.js +49 -0
  89. package/dist/ids-Bp6ut7ZD.js.map +1 -0
  90. package/dist/images-Cc1-coFc.js +286 -0
  91. package/dist/images-Cc1-coFc.js.map +1 -0
  92. package/dist/index.d.ts +97 -0
  93. package/dist/index.js +1959 -0
  94. package/dist/index.js.map +1 -0
  95. package/dist/init-CFljS8BJ.js +344 -0
  96. package/dist/init-CFljS8BJ.js.map +1 -0
  97. package/dist/init-z2ZPn9Tj.js +2 -0
  98. package/dist/invocations-rXktgSAn.js +157 -0
  99. package/dist/invocations-rXktgSAn.js.map +1 -0
  100. package/dist/jiti-BOLXpKyG.js +46 -0
  101. package/dist/jiti-BOLXpKyG.js.map +1 -0
  102. package/dist/judge-D8Uf4dA9.js +146 -0
  103. package/dist/judge-D8Uf4dA9.js.map +1 -0
  104. package/dist/judge.d.ts +21 -0
  105. package/dist/judge.js +19 -0
  106. package/dist/judge.js.map +1 -0
  107. package/dist/live-Bs3LReMV.js +98 -0
  108. package/dist/live-Bs3LReMV.js.map +1 -0
  109. package/dist/matchers-DyLvm7EG.js +374 -0
  110. package/dist/matchers-DyLvm7EG.js.map +1 -0
  111. package/dist/model-BdO79gWz.js +44 -0
  112. package/dist/model-BdO79gWz.js.map +1 -0
  113. package/dist/model-CDL98xue.js +725 -0
  114. package/dist/model-CDL98xue.js.map +1 -0
  115. package/dist/model-v-0_Ik7S.js +2 -0
  116. package/dist/node-kG64XxdE.js +506 -0
  117. package/dist/node-kG64XxdE.js.map +1 -0
  118. package/dist/opencode-providers-DZuD_oYt.js +3149 -0
  119. package/dist/opencode-providers-DZuD_oYt.js.map +1 -0
  120. package/dist/options-CAQhu--C.js +452 -0
  121. package/dist/options-CAQhu--C.js.map +1 -0
  122. package/dist/patch-BkKje4cp.js +69 -0
  123. package/dist/patch-BkKje4cp.js.map +1 -0
  124. package/dist/paths-CBvm6pap.js +72 -0
  125. package/dist/paths-CBvm6pap.js.map +1 -0
  126. package/dist/plan-gwk6dJ8c.js +237 -0
  127. package/dist/plan-gwk6dJ8c.js.map +1 -0
  128. package/dist/preflight-BtubXKQ8.js +2 -0
  129. package/dist/preflight-BypZMNBG.js +185 -0
  130. package/dist/preflight-BypZMNBG.js.map +1 -0
  131. package/dist/prepare-Cwv0bHs_.js +3 -0
  132. package/dist/prepare-D6Uzcubb.js +213 -0
  133. package/dist/prepare-D6Uzcubb.js.map +1 -0
  134. package/dist/provenance-Dfb8YY5Y.js +129 -0
  135. package/dist/provenance-Dfb8YY5Y.js.map +1 -0
  136. package/dist/provider-BDbNOQhQ.js +2 -0
  137. package/dist/provider-CmAYTQQs.js +14 -0
  138. package/dist/provider-CmAYTQQs.js.map +1 -0
  139. package/dist/providers-2p3ylMHI.js +107 -0
  140. package/dist/providers-2p3ylMHI.js.map +1 -0
  141. package/dist/providers-BUOxitcT.js +2 -0
  142. package/dist/query-CFATHZmV.js +91 -0
  143. package/dist/query-CFATHZmV.js.map +1 -0
  144. package/dist/query-DR4m6CXf.js +2 -0
  145. package/dist/record-DQ8wzyco.js +99 -0
  146. package/dist/record-DQ8wzyco.js.map +1 -0
  147. package/dist/redact-C4fL_bZP.js +167 -0
  148. package/dist/redact-C4fL_bZP.js.map +1 -0
  149. package/dist/results-CeXb8Wde.js +37 -0
  150. package/dist/results-CeXb8Wde.js.map +1 -0
  151. package/dist/root-CFDjpWuc.js +110 -0
  152. package/dist/root-CFDjpWuc.js.map +1 -0
  153. package/dist/sandbox-By2gkdkC.js +133 -0
  154. package/dist/sandbox-By2gkdkC.js.map +1 -0
  155. package/dist/sandbox.d.ts +41 -0
  156. package/dist/sandbox.js +2 -0
  157. package/dist/server-Bi61uiY0.js +350 -0
  158. package/dist/server-Bi61uiY0.js.map +1 -0
  159. package/dist/server-CsSBL8NC.js +2 -0
  160. package/dist/source-BXzrmof5.js +2 -0
  161. package/dist/source-Bsfa3LZH.js +40 -0
  162. package/dist/source-Bsfa3LZH.js.map +1 -0
  163. package/dist/src-55j1zNHH.js +3 -0
  164. package/dist/src-AEwxd1UP.js +602 -0
  165. package/dist/src-AEwxd1UP.js.map +1 -0
  166. package/dist/src-BDwSuyIo.js +3 -0
  167. package/dist/src-BQyMiWri.js +490 -0
  168. package/dist/src-BQyMiWri.js.map +1 -0
  169. package/dist/src-C-LHyTSV.js +3 -0
  170. package/dist/src-Cmp-7YrD.js +1136 -0
  171. package/dist/src-Cmp-7YrD.js.map +1 -0
  172. package/dist/src-D0RdqpNb.js +252 -0
  173. package/dist/src-D0RdqpNb.js.map +1 -0
  174. package/dist/src-D8KYzxBX.js +793 -0
  175. package/dist/src-D8KYzxBX.js.map +1 -0
  176. package/dist/src-DEgff9Z7.js +4 -0
  177. package/dist/src-DXx_29pq.js +567 -0
  178. package/dist/src-DXx_29pq.js.map +1 -0
  179. package/dist/src-DwwiVLqv.js +2937 -0
  180. package/dist/src-DwwiVLqv.js.map +1 -0
  181. package/dist/src-N7E_mfcZ.js +2 -0
  182. package/dist/src-W9g-y5X5.js +3 -0
  183. package/dist/src-daS0wKH2.js +3 -0
  184. package/dist/state-Cyw3QTsN.js +2 -0
  185. package/dist/state-DZ7eqb1F.js +167 -0
  186. package/dist/state-DZ7eqb1F.js.map +1 -0
  187. package/dist/stats-BNAHlNMD.js +114 -0
  188. package/dist/stats-BNAHlNMD.js.map +1 -0
  189. package/dist/studio-app/assets/index-D6XyfFSH.js +58 -0
  190. package/dist/studio-app/assets/index-xwzWdcE_.css +2 -0
  191. package/dist/studio-app/favicon.svg +1 -0
  192. package/dist/studio-app/index.html +15 -0
  193. package/dist/summary-D7Ttky0z.js +166 -0
  194. package/dist/summary-D7Ttky0z.js.map +1 -0
  195. package/dist/text-gEcKjAyp.js +29 -0
  196. package/dist/text-gEcKjAyp.js.map +1 -0
  197. package/dist/types-C5vI3g1C.js +9 -0
  198. package/dist/types-C5vI3g1C.js.map +1 -0
  199. package/dist/types-DLec-6Cx.d.ts +1535 -0
  200. package/dist/usage-DsYUqIUM.js +24 -0
  201. package/dist/usage-DsYUqIUM.js.map +1 -0
  202. package/dist/view/index.d.ts +645 -0
  203. package/dist/view/index.js +8 -0
  204. package/dist/views-BrHOv6wg.js +220 -0
  205. package/dist/views-BrHOv6wg.js.map +1 -0
  206. package/dist/views-CwCWLWqo.js +3 -0
  207. package/dist/worker-entry-DnYkANPG.js +506 -0
  208. package/dist/worker-entry-DnYkANPG.js.map +1 -0
  209. package/dist/workspace-C2S0Z6gl.js +327 -0
  210. package/dist/workspace-C2S0Z6gl.js.map +1 -0
  211. package/dist/workspace-CyoXG6CY.js +2 -0
  212. package/docker/Dockerfile +19 -0
  213. package/package.json +79 -0
@@ -0,0 +1,46 @@
1
+ import { a as PACKAGE_NAMES, c as ownDir, n as COMPILED, u as ownRequire } from "./root-CFDjpWuc.js";
2
+ import { createRequire } from "node:module";
3
+ import { dirname, join } from "node:path";
4
+ import { createJiti } from "jiti";
5
+ //#region src/jiti.ts
6
+ let babel;
7
+ /**
8
+ * jiti's own Babel transform. jiti requires it lazily from beside its code, which a compiled es
9
+ * doesn't have, so reeval requires the same file: from the jiti package on disk, or, compiled, from
10
+ * the copy embedded in it (`vendor/jiti/babel.cjs`), copied out to disk.
11
+ */
12
+ const jitiTransform = (options) => {
13
+ if (!babel) {
14
+ const file = COMPILED ? join(ownDir("vendor/jiti"), "babel.cjs") : join(dirname(ownRequire.resolve("jiti/package.json")), "dist/babel.cjs");
15
+ babel = createRequire(file)(file);
16
+ }
17
+ return babel(options);
18
+ };
19
+ /**
20
+ * A jiti that imports from `id`. Compiled, a file's imports of reeval itself get this es's own
21
+ * modules: a copy on disk would register evals, and hold state, the executable can't see.
22
+ */
23
+ async function esJiti(id, options = {}) {
24
+ const virtualModules = COMPILED ? await ownModules() : void 0;
25
+ return createJiti(id, {
26
+ transform: jitiTransform,
27
+ ...options,
28
+ ...virtualModules && { virtualModules }
29
+ });
30
+ }
31
+ async function ownModules() {
32
+ const exports = {
33
+ "": import("./index.js"),
34
+ "/harness": import("./harness.js"),
35
+ "/sandbox": import("./sandbox.js"),
36
+ "/judge": import("./judge.js"),
37
+ "/view": import("./view/index.js")
38
+ };
39
+ const modules = {};
40
+ for (const [subpath, module] of Object.entries(exports)) for (const name of PACKAGE_NAMES) modules[name + subpath] = await module;
41
+ return modules;
42
+ }
43
+ //#endregion
44
+ export { esJiti as t };
45
+
46
+ //# sourceMappingURL=jiti-BOLXpKyG.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"jiti-BOLXpKyG.js","names":[],"sources":["../src/jiti.ts"],"mappings":";;;;;AASA,IAAI;;;;;;AAOJ,MAAa,iBAA4B,YAAY;CACnD,IAAI,CAAC,OAAO;EACV,MAAM,OAAO,WACT,KAAK,OAAO,aAAa,GAAG,WAAW,IACvC,KAAK,QAAQ,WAAW,QAAQ,mBAAmB,CAAC,GAAG,gBAAgB;EAC3E,QAAQ,cAAc,IAAI,CAAC,CAAC,IAAI;CAClC;CACA,OAAO,MAAM,OAAO;AACtB;;;;;AAMA,eAAsB,OAAO,IAAY,UAAuB,CAAC,GAAkB;CACjF,MAAM,iBAAiB,WAAW,MAAM,WAAW,IAAI,KAAA;CACvD,OAAO,WAAW,IAAI;EACpB,WAAW;EACX,GAAG;EACH,GAAI,kBAAkB,EAAE,eAAe;CACzC,CAAC;AACH;AAEA,eAAe,aAA+C;CAC5D,MAAM,UAA4C;EAChD,IAAI,OAAO;EACX,YAAY,OAAO;EACnB,YAAY,OAAO;EACnB,UAAU,OAAO;EACjB,SAAS,OAAO;CAClB;CACA,MAAM,UAAmC,CAAC;CAC1C,KAAK,MAAM,CAAC,SAAS,WAAW,OAAO,QAAQ,OAAO,GACpD,KAAK,MAAM,QAAQ,eAAe,QAAQ,OAAO,WAAW,MAAM;CACpE,OAAO;AACT"}
@@ -0,0 +1,146 @@
1
+ import { setTimeout } from "node:timers/promises";
2
+ //#region src/judge/retry.ts
3
+ /** Up to six tries over about a minute: long enough to ride out a provider's "high demand". */
4
+ const JUDGE_RETRY = {
5
+ attempts: 6,
6
+ baseMs: 2e3,
7
+ maxMs: 3e4
8
+ };
9
+ const TRANSIENT_STATUS = /* @__PURE__ */ new Set([
10
+ 408,
11
+ 409,
12
+ 425,
13
+ 429
14
+ ]);
15
+ const TRANSIENT_MESSAGE = /high demand|overloaded|unavailable|rate.?limit|resource.?exhausted|try again|timed? ?out|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|socket hang up|fetch failed/i;
16
+ /** The AI SDK wraps the provider's error once it has retried itself; look through that. */
17
+ const innermost = (error) => {
18
+ const e = error;
19
+ return e?.name === "AI_RetryError" && e.lastError ? innermost(e.lastError) : error;
20
+ };
21
+ /** Rate limits, 5xx, overload and network blips. Never a bad request, bad key or abort. */
22
+ function isTransient(error) {
23
+ const e = innermost(error);
24
+ if (!e || e.name === "AbortError") return false;
25
+ if (typeof e.statusCode === "number") return e.statusCode >= 500 || TRANSIENT_STATUS.has(e.statusCode) || e.isRetryable === true;
26
+ return e.isRetryable === true || TRANSIENT_MESSAGE.test(e.message ?? String(e));
27
+ }
28
+ function retryAfterMs(error) {
29
+ const header = innermost(error)?.responseHeaders?.["retry-after"];
30
+ const seconds = Number(header);
31
+ return Number.isFinite(seconds) && seconds > 0 ? seconds * 1e3 : 0;
32
+ }
33
+ /** Exponential backoff with equal jitter, at least any `retry-after`, never above `maxMs`. */
34
+ function backoffMs(policy, retry, error) {
35
+ const cap = Math.min(policy.maxMs, policy.baseMs * 2 ** retry);
36
+ const jittered = cap / 2 + Math.random() * (cap / 2);
37
+ return Math.round(Math.min(policy.maxMs, Math.max(jittered, retryAfterMs(error))));
38
+ }
39
+ /** Runs `call` until it succeeds, fails for good, or runs out of tries. */
40
+ async function withRetries(call, policy, signal) {
41
+ const retries = [];
42
+ for (;;) try {
43
+ return {
44
+ value: await call(),
45
+ retries
46
+ };
47
+ } catch (error) {
48
+ if (signal.aborted || retries.length + 1 >= policy.attempts || !isTransient(error)) {
49
+ const failed = error instanceof Error ? error : new Error(String(error));
50
+ failed.retries = retries;
51
+ throw failed;
52
+ }
53
+ const delayMs = backoffMs(policy, retries.length, error);
54
+ const e = innermost(error);
55
+ retries.push({
56
+ error: e?.message ?? String(e),
57
+ delayMs
58
+ });
59
+ await setTimeout(delayMs, void 0, { signal });
60
+ }
61
+ }
62
+ //#endregion
63
+ //#region src/judge/index.ts
64
+ /** The judge the docs recommend, pinned so a model update never silently moves scores. */
65
+ const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.13.0";
66
+ const DEFAULT_THRESHOLD = .8;
67
+ const GATEWAY_KEYS = ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"];
68
+ /** Providers reeval calls directly, through their own AI SDK package, when their key is set. */
69
+ const DIRECT_PROVIDERS = {
70
+ google: {
71
+ key: "GOOGLE_GENERATIVE_AI_API_KEY",
72
+ pkg: "@ai-sdk/google"
73
+ },
74
+ "typesafe-ai": {
75
+ key: "TYPESAFE_AI_API_KEY",
76
+ pkg: "@ai-sdk/typesafe-ai"
77
+ }
78
+ };
79
+ const isFakeJudge = (m) => typeof m === "object" && m !== null && m.kind === "fake-judge";
80
+ function judgeModelName(model) {
81
+ if (typeof model === "string") return model;
82
+ if (isFakeJudge(model)) return "fake";
83
+ const m = model;
84
+ return m.provider && m.modelId ? `${m.provider}/${m.modelId}` : "custom";
85
+ }
86
+ /** Checks a model string's shape; returns an error message, or undefined when it's valid. */
87
+ function checkModelString(model) {
88
+ if (model.includes("/")) return void 0;
89
+ return `Judge model "${model}" needs a provider prefix, e.g. "${RECOMMENDED_JUDGE_MODEL}" or "google/${model}"`;
90
+ }
91
+ const providerOf = (model) => model.slice(0, model.indexOf("/"));
92
+ /** Environment variables the judge model needs, as alternatives: any one of them is enough. */
93
+ function judgeCredentials(model) {
94
+ if (isFakeJudge(model)) return [];
95
+ const direct = DIRECT_PROVIDERS[providerOf(model)];
96
+ return direct ? [direct.key, ...GATEWAY_KEYS] : GATEWAY_KEYS;
97
+ }
98
+ /**
99
+ * A known provider's own package when its key is set, else the Vercel AI Gateway when a
100
+ * Gateway credential is set. Throws, naming both, when neither is.
101
+ */
102
+ function judgeRoute(model, env = process.env) {
103
+ const provider = providerOf(model);
104
+ const direct = DIRECT_PROVIDERS[provider];
105
+ if (direct && env[direct.key]) return {
106
+ kind: "direct",
107
+ provider,
108
+ pkg: direct.pkg
109
+ };
110
+ if (GATEWAY_KEYS.some((k) => env[k])) return { kind: "gateway" };
111
+ const options = direct ? `Set ${direct.key} to call ${direct.pkg} directly, or AI_GATEWAY_API_KEY (or VERCEL_OIDC_TOKEN) to go through the Vercel AI Gateway.` : "Set AI_GATEWAY_API_KEY (or VERCEL_OIDC_TOKEN) to go through the Vercel AI Gateway.";
112
+ throw new Error(`No credentials for judge model "${model}". ${options}`);
113
+ }
114
+ /** Transient provider errors (429, 5xx, "high demand") are retried with backoff and recorded. */
115
+ async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
116
+ const { model, threshold } = options;
117
+ let score;
118
+ let resolvedModel;
119
+ let retries = [];
120
+ if (isFakeJudge(model)) score = model.score;
121
+ else {
122
+ if (typeof model === "string") {
123
+ const problem = checkModelString(model);
124
+ if (problem) throw new Error(problem);
125
+ }
126
+ const { scoreWithModel } = await import("./model-BdO79gWz.js");
127
+ const scored = await withRetries(() => scoreWithModel(model, question, state, options.providerOptions, signal), retry, signal);
128
+ ({score, resolvedModel} = scored.value);
129
+ retries = scored.retries;
130
+ }
131
+ return {
132
+ verdict: {
133
+ verdict: score >= threshold ? "pass" : "fail",
134
+ score,
135
+ threshold,
136
+ question,
137
+ model: judgeModelName(model),
138
+ ...resolvedModel ? { resolvedModel } : {}
139
+ },
140
+ retries
141
+ };
142
+ }
143
+ //#endregion
144
+ export { isFakeJudge as a, judgeRoute as c, checkModelString as i, runJudge as l, DIRECT_PROVIDERS as n, judgeCredentials as o, RECOMMENDED_JUDGE_MODEL as r, judgeModelName as s, DEFAULT_THRESHOLD as t };
145
+
146
+ //# sourceMappingURL=judge-D8Uf4dA9.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"judge-D8Uf4dA9.js","names":["sleep"],"sources":["../src/judge/retry.ts","../src/judge/index.ts"],"mappings":";;;AAOA,MAAa,cAA2B;CAAE,UAAU;CAAG,QAAQ;CAAO,OAAO;AAAO;AAEpF,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;AACrD,MAAM,oBACJ;;AAUF,MAAM,aAAa,UAA4B;CAC7C,MAAM,IAAI;CACV,OAAO,GAAG,SAAS,mBAAmB,EAAE,YAAY,UAAU,EAAE,SAAS,IAAI;AAC/E;;AAGA,SAAgB,YAAY,OAAyB;CACnD,MAAM,IAAI,UAAU,KAAK;CACzB,IAAI,CAAC,KAAK,EAAE,SAAS,cAAc,OAAO;CAC1C,IAAI,OAAO,EAAE,eAAe,UAC1B,OAAO,EAAE,cAAc,OAAO,iBAAiB,IAAI,EAAE,UAAU,KAAK,EAAE,gBAAgB;CAExF,OAAO,EAAE,gBAAgB,QAAQ,kBAAkB,KAAK,EAAE,WAAW,OAAO,CAAC,CAAC;AAChF;AAEA,SAAS,aAAa,OAAwB;CAC5C,MAAM,SAAU,UAAU,KAAK,CAAC,EAAoB,kBAAkB;CACtE,MAAM,UAAU,OAAO,MAAM;CAC7B,OAAO,OAAO,SAAS,OAAO,KAAK,UAAU,IAAI,UAAU,MAAO;AACpE;;AAGA,SAAgB,UAAU,QAAqB,OAAe,OAAwB;CACpF,MAAM,MAAM,KAAK,IAAI,OAAO,OAAO,OAAO,SAAS,KAAK,KAAK;CAC7D,MAAM,WAAW,MAAM,IAAI,KAAK,OAAO,KAAK,MAAM;CAClD,OAAO,KAAK,MAAM,KAAK,IAAI,OAAO,OAAO,KAAK,IAAI,UAAU,aAAa,KAAK,CAAC,CAAC,CAAC;AACnF;;AAKA,eAAsB,YACpB,MACA,QACA,QAC8C;CAC9C,MAAM,UAAwB,CAAC;CAC/B,SACE,IAAI;EACF,OAAO;GAAE,OAAO,MAAM,KAAK;GAAG;EAAQ;CACxC,SAAS,OAAO;EACd,IAAI,OAAO,WAAW,QAAQ,SAAS,KAAK,OAAO,YAAY,CAAC,YAAY,KAAK,GAAG;GAClF,MAAM,SAAU,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GACxE,OAAO,UAAU;GACjB,MAAM;EACR;EACA,MAAM,UAAU,UAAU,QAAQ,QAAQ,QAAQ,KAAK;EACvD,MAAM,IAAI,UAAU,KAAK;EACzB,QAAQ,KAAK;GAAE,OAAO,GAAG,WAAW,OAAO,CAAC;GAAG;EAAQ,CAAC;EACxD,MAAMA,WAAM,SAAS,KAAA,GAAW,EAAE,OAAO,CAAC;CAC5C;AAEJ;;;;ACpEA,MAAa,0BAA0B;AACvC,MAAa,oBAAoB;AAEjC,MAAM,eAAe,CAAC,sBAAsB,mBAAmB;;AAG/D,MAAa,mBAAiE;CAC5E,QAAQ;EAAE,KAAK;EAAgC,KAAK;CAAiB;CACrE,eAAe;EAAE,KAAK;EAAuB,KAAK;CAAsB;AAC1E;AAMA,MAAa,eAAe,MAC1B,OAAO,MAAM,YAAY,MAAM,QAAS,EAAqB,SAAS;AAExE,SAAgB,eAAe,OAA2B;CACxD,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI,YAAY,KAAK,GAAG,OAAO;CAC/B,MAAM,IAAI;CACV,OAAO,EAAE,YAAY,EAAE,UAAU,GAAG,EAAE,SAAS,GAAG,EAAE,YAAY;AAClE;;AAGA,SAAgB,iBAAiB,OAAmC;CAClE,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO,KAAA;CAChC,OAAO,gBAAgB,MAAM,mCAAmC,wBAAwB,eAAe,MAAM;AAC/G;AAEA,MAAM,cAAc,UAAkB,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;;AAGvE,SAAgB,iBAAiB,OAA0C;CACzE,IAAI,YAAY,KAAK,GAAG,OAAO,CAAC;CAChC,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SAAS,CAAC,OAAO,KAAK,GAAG,YAAY,IAAI;AAClD;;;;;AAMA,SAAgB,WAAW,OAAe,MAAyB,QAAQ,KAAiB;CAC1F,MAAM,WAAW,WAAW,KAAK;CACjC,MAAM,SAAS,iBAAiB;CAChC,IAAI,UAAU,IAAI,OAAO,MAAM,OAAO;EAAE,MAAM;EAAU;EAAU,KAAK,OAAO;CAAI;CAClF,IAAI,aAAa,MAAM,MAAM,IAAI,EAAE,GAAG,OAAO,EAAE,MAAM,UAAU;CAC/D,MAAM,UAAU,SACZ,OAAO,OAAO,IAAI,WAAW,OAAO,IAAI,gGACxC;CACJ,MAAM,IAAI,MAAM,mCAAmC,MAAM,KAAK,SAAS;AACzE;;AAMA,eAAsB,SACpB,UACA,OACA,SACA,QACA,QAAqB,aACC;CACtB,MAAM,EAAE,OAAO,cAAc;CAC7B,IAAI;CACJ,IAAI;CACJ,IAAI,UAAwB,CAAC;CAC7B,IAAI,YAAY,KAAK,GACnB,QAAQ,MAAM;MACT;EACL,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,UAAU,iBAAiB,KAAK;GACtC,IAAI,SAAS,MAAM,IAAI,MAAM,OAAO;EACtC;EACA,MAAM,EAAE,mBAAmB,MAAM,OAAO;EACxC,MAAM,SAAS,MAAM,kBACb,eAAe,OAAO,UAAU,OAAO,QAAQ,iBAAiB,MAAM,GAC5E,OACA,MACF;EACC,CAAC,CAAE,OAAO,iBAAkB,OAAO;EACpC,UAAU,OAAO;CACnB;CASA,OAAO;EAAE,SAAA;GAPP,SAAS,SAAS,YAAY,SAAS;GACvC;GACA;GACA;GACA,OAAO,eAAe,KAAK;GAC3B,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;EAE5B;EAAG;CAAQ;AAC5B"}
@@ -0,0 +1,21 @@
1
+ import { A as FakeJudgeModel, H as JudgeModel, O as FakeAnalysisModel, U as JudgeOptions, V as JudgeCallOptions, d as ConfigJudgeOptions, lt as Verdict, n as AnalysisConfig } from "./types-DLec-6Cx.js";
2
+ //#region src/judge/index.d.ts
3
+ /** The judge the docs recommend, pinned so a model update never silently moves scores. */
4
+ export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.13.0";
5
+ export declare const DEFAULT_THRESHOLD = 0.8;
6
+ //#endregion
7
+ //#region src/judge.d.ts
8
+ /** A judge that always returns `score`, without calling a model. For tests and dry runs. */
9
+ export declare const fakeJudge: ({ score }?: {
10
+ score?: number;
11
+ }) => FakeJudgeModel;
12
+ /**
13
+ * An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
14
+ * model's structured answer, through the same parsing and validation as a real one.
15
+ */
16
+ export declare const fakeAnalysis: ({ output }: {
17
+ output: FakeAnalysisModel['output'];
18
+ }) => FakeAnalysisModel;
19
+ //#endregion
20
+ export type { AnalysisConfig, ConfigJudgeOptions, FakeAnalysisModel, FakeJudgeModel, JudgeCallOptions, JudgeModel, JudgeOptions, Verdict };
21
+ //# sourceMappingURL=judge.d.ts.map
package/dist/judge.js ADDED
@@ -0,0 +1,19 @@
1
+ import { r as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-D8Uf4dA9.js";
2
+ //#region src/judge.ts
3
+ /** A judge that always returns `score`, without calling a model. For tests and dry runs. */
4
+ const fakeJudge = ({ score = 1 } = {}) => ({
5
+ kind: "fake-judge",
6
+ score
7
+ });
8
+ /**
9
+ * An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
10
+ * model's structured answer, through the same parsing and validation as a real one.
11
+ */
12
+ const fakeAnalysis = ({ output }) => ({
13
+ kind: "fake-analysis",
14
+ output
15
+ });
16
+ //#endregion
17
+ export { DEFAULT_THRESHOLD, RECOMMENDED_JUDGE_MODEL, fakeAnalysis, fakeJudge };
18
+
19
+ //# sourceMappingURL=judge.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;AAKA,MAAa,aAAa,EAAE,QAAQ,MAA0B,CAAC,OAAuB;CACpF,MAAM;CACN;AACF;;;;;AAMA,MAAa,gBAAgB,EAC3B,cAGwB;CACxB,MAAM;CACN;AACF"}
@@ -0,0 +1,98 @@
1
+ import { n as fillDefaults } from "./record-DQ8wzyco.js";
2
+ //#region src/view/live.ts
3
+ /** The progress file's name inside the run's staging directory. */
4
+ const LIVE_FILE = "live.json";
5
+ const LIVE_SCHEMA_VERSION = 1;
6
+ const liveApiPath = "api/live";
7
+ const liveTranscriptPath = (runId, attemptKey) => `api/live/${runId}/transcript?attempt=${encodeURIComponent(attemptKey)}`;
8
+ const FINISHED = [
9
+ "pass",
10
+ "fail",
11
+ "infra_error",
12
+ "skipped"
13
+ ];
14
+ const isFinished = (status) => FINISHED.includes(status);
15
+ function liveCounts(attempts) {
16
+ const count = (...statuses) => attempts.filter((a) => statuses.includes(a.status)).length;
17
+ return {
18
+ done: count(...FINISHED),
19
+ passed: count("pass"),
20
+ failed: count("fail"),
21
+ errored: count("infra_error"),
22
+ skipped: count("skipped"),
23
+ running: count("running")
24
+ };
25
+ }
26
+ const STATUSES = [
27
+ "queued",
28
+ "running",
29
+ ...FINISHED
30
+ ];
31
+ const PHASES = [
32
+ "running",
33
+ "cancelling",
34
+ "finishing"
35
+ ];
36
+ const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
37
+ const str = (v) => typeof v === "string";
38
+ const num = (v) => typeof v === "number" && Number.isFinite(v);
39
+ const orNull = (v, is) => v === null || v === void 0 || is(v);
40
+ function readableAttempt(v) {
41
+ if (!isObj(v)) return null;
42
+ const { key, evalId, name, file, cell, n, status, startedAt, durationMs } = v;
43
+ if (![
44
+ key,
45
+ evalId,
46
+ name,
47
+ file,
48
+ cell
49
+ ].every(str) || !num(n)) return null;
50
+ if (!STATUSES.includes(status)) return null;
51
+ if (!orNull(startedAt, str) || !orNull(durationMs, num)) return null;
52
+ return {
53
+ key,
54
+ evalId,
55
+ name,
56
+ file,
57
+ cell,
58
+ n,
59
+ status,
60
+ startedAt: startedAt ?? null,
61
+ durationMs: durationMs ?? null
62
+ };
63
+ }
64
+ /**
65
+ * A progress file as the studio reads it, or null when it isn't one it can read: half-written,
66
+ * corrupt, or from a newer reeval. Counts are recomputed from the attempts, never trusted.
67
+ */
68
+ function readableLiveProgress(value) {
69
+ if (!isObj(value) || value.schemaVersion !== 1) return null;
70
+ fillDefaults("live-progress", value);
71
+ const { runId, startedAt, pid, host, heartbeatAt, phase, attempts } = value;
72
+ if (![
73
+ runId,
74
+ startedAt,
75
+ host,
76
+ heartbeatAt
77
+ ].every(str) || !num(pid)) return null;
78
+ if (!PHASES.includes(phase) || !Array.isArray(attempts)) return null;
79
+ const read = attempts.map(readableAttempt);
80
+ if (read.some((a) => a === null)) return null;
81
+ const list = read;
82
+ return {
83
+ schemaVersion: 1,
84
+ runId,
85
+ startedAt,
86
+ pid,
87
+ host,
88
+ heartbeatAt,
89
+ phase,
90
+ planned: list.length,
91
+ counts: liveCounts(list),
92
+ attempts: list
93
+ };
94
+ }
95
+ //#endregion
96
+ export { liveApiPath as a, readableLiveProgress as c, isFinished as i, LIVE_FILE as n, liveCounts as o, LIVE_SCHEMA_VERSION as r, liveTranscriptPath as s, FINISHED as t };
97
+
98
+ //# sourceMappingURL=live-Bs3LReMV.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"live-Bs3LReMV.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;AAOA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAsDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAA+B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEnG,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC;EAAC;EAAK;EAAQ;EAAM;EAAM;CAAI,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CACnE,IAAI,CAAC,SAAS,SAAS,MAAoB,GAAG,OAAO;CACrD,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACA;EACG;EACF;EACA;EACA;EACN;EACQ;EACR,WAAY,aAAoC;EAChD,YAAa,cAAqC;CACpD;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC;EAAC;EAAO;EAAW;EAAM;CAAW,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3E,IAAI,CAAC,OAAO,SAAS,KAAkB,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAC7E,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,KAAK,MAAM,MAAM,MAAM,IAAI,GAAG,OAAO;CACzC,MAAM,OAAO;CACb,OAAO;EACL,eAAA;EACO;EACI;EACX;EACM;EACO;EACN;EACP,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;CACZ;AACF"}