reeval 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/THIRD_PARTY_NOTICES.md +2160 -0
- package/assets/package.json +9 -0
- package/assets/pnpm-lock.yaml +1336 -0
- package/dist/agent-DVHkx8aT.js +532 -0
- package/dist/agent-DVHkx8aT.js.map +1 -0
- package/dist/analysis-CxSCt1YY.js +16 -0
- package/dist/analysis-CxSCt1YY.js.map +1 -0
- package/dist/assemble-B-nmz8oH.js +695 -0
- package/dist/assemble-B-nmz8oH.js.map +1 -0
- package/dist/assemble-CmRVDZ2Q.js +2 -0
- package/dist/attempt-BdRQ9jKX.js +176 -0
- package/dist/attempt-BdRQ9jKX.js.map +1 -0
- package/dist/bridge/index.mjs +1911 -0
- package/dist/bridge/package.json +13 -0
- package/dist/bridge/pnpm-lock.yaml +1041 -0
- package/dist/bridge/pnpm-workspace.yaml +2 -0
- package/dist/budget-DBUrH4jX.js +8 -0
- package/dist/budget-DBUrH4jX.js.map +1 -0
- package/dist/cli--CQ3Iy9q.js +1865 -0
- package/dist/cli--CQ3Iy9q.js.map +1 -0
- package/dist/cli-spec-DKAu09Gi.js +478 -0
- package/dist/cli-spec-DKAu09Gi.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +8 -0
- package/dist/cli.js.map +1 -0
- package/dist/colors-QV9dFVhN.js +34 -0
- package/dist/colors-QV9dFVhN.js.map +1 -0
- package/dist/command-BSJzHMtP.js +280 -0
- package/dist/command-BSJzHMtP.js.map +1 -0
- package/dist/command-CEm8PZ1N.js +1043 -0
- package/dist/command-CEm8PZ1N.js.map +1 -0
- package/dist/command-D-jnQWVG.js +673 -0
- package/dist/command-D-jnQWVG.js.map +1 -0
- package/dist/command-DQAvbG7d.js +432 -0
- package/dist/command-DQAvbG7d.js.map +1 -0
- package/dist/commands-A4udfyKZ.js +2 -0
- package/dist/commands-CF8mOfiu.js +207 -0
- package/dist/commands-CF8mOfiu.js.map +1 -0
- package/dist/config-Cuj0kVYj.js +415 -0
- package/dist/config-Cuj0kVYj.js.map +1 -0
- package/dist/context-loi9OCZv.js +106 -0
- package/dist/context-loi9OCZv.js.map +1 -0
- package/dist/core-tLN-z1Be.js +96 -0
- package/dist/core-tLN-z1Be.js.map +1 -0
- package/dist/db-Cjj2ZsfQ.js +2 -0
- package/dist/db-DxDzMhtB.js +827 -0
- package/dist/db-DxDzMhtB.js.map +1 -0
- package/dist/debug-D24NOQH1.js +89 -0
- package/dist/debug-D24NOQH1.js.map +1 -0
- package/dist/debug-SEmW4w8f.js +2 -0
- package/dist/dist-BZXYHN2X.js +1673 -0
- package/dist/dist-BZXYHN2X.js.map +1 -0
- package/dist/docker-DQDZWXF9.js +20 -0
- package/dist/docker-DQDZWXF9.js.map +1 -0
- package/dist/docker-DnWAVdEm.js +2 -0
- package/dist/docker-oSbCExHL.js +153 -0
- package/dist/docker-oSbCExHL.js.map +1 -0
- package/dist/doctor-CxlH1pDM.js +525 -0
- package/dist/doctor-CxlH1pDM.js.map +1 -0
- package/dist/egress-73SQfKdx.js +619 -0
- package/dist/egress-73SQfKdx.js.map +1 -0
- package/dist/env-BQqj93gu.js +21 -0
- package/dist/env-BQqj93gu.js.map +1 -0
- package/dist/environment-Mz7wvrtD.js +552 -0
- package/dist/environment-Mz7wvrtD.js.map +1 -0
- package/dist/errors-C9w_ud7g.js +160 -0
- package/dist/errors-C9w_ud7g.js.map +1 -0
- package/dist/errors-CSxfJ4qu.js +11 -0
- package/dist/errors-CSxfJ4qu.js.map +1 -0
- package/dist/files--6CEUXm7.js +2 -0
- package/dist/files-BLlrhU4-.js +62 -0
- package/dist/files-BLlrhU4-.js.map +1 -0
- package/dist/files-Bd25OAQH.js +2 -0
- package/dist/files-BvHCFWgD.js +2 -0
- package/dist/files-D2-qn_Dl.js +275 -0
- package/dist/files-D2-qn_Dl.js.map +1 -0
- package/dist/files-bF2fPuT9.js +99 -0
- package/dist/files-bF2fPuT9.js.map +1 -0
- package/dist/git-D6hMeieS.js +44 -0
- package/dist/git-D6hMeieS.js.map +1 -0
- package/dist/harness-ls5aX1UO.js +131 -0
- package/dist/harness-ls5aX1UO.js.map +1 -0
- package/dist/harness.d.ts +74 -0
- package/dist/harness.js +2 -0
- package/dist/header-DzFDoMCI.js +78 -0
- package/dist/header-DzFDoMCI.js.map +1 -0
- package/dist/ids-Bp6ut7ZD.js +49 -0
- package/dist/ids-Bp6ut7ZD.js.map +1 -0
- package/dist/images-Cc1-coFc.js +286 -0
- package/dist/images-Cc1-coFc.js.map +1 -0
- package/dist/index.d.ts +97 -0
- package/dist/index.js +1959 -0
- package/dist/index.js.map +1 -0
- package/dist/init-CFljS8BJ.js +344 -0
- package/dist/init-CFljS8BJ.js.map +1 -0
- package/dist/init-z2ZPn9Tj.js +2 -0
- package/dist/invocations-rXktgSAn.js +157 -0
- package/dist/invocations-rXktgSAn.js.map +1 -0
- package/dist/jiti-BOLXpKyG.js +46 -0
- package/dist/jiti-BOLXpKyG.js.map +1 -0
- package/dist/judge-D8Uf4dA9.js +146 -0
- package/dist/judge-D8Uf4dA9.js.map +1 -0
- package/dist/judge.d.ts +21 -0
- package/dist/judge.js +19 -0
- package/dist/judge.js.map +1 -0
- package/dist/live-Bs3LReMV.js +98 -0
- package/dist/live-Bs3LReMV.js.map +1 -0
- package/dist/matchers-DyLvm7EG.js +374 -0
- package/dist/matchers-DyLvm7EG.js.map +1 -0
- package/dist/model-BdO79gWz.js +44 -0
- package/dist/model-BdO79gWz.js.map +1 -0
- package/dist/model-CDL98xue.js +725 -0
- package/dist/model-CDL98xue.js.map +1 -0
- package/dist/model-v-0_Ik7S.js +2 -0
- package/dist/node-kG64XxdE.js +506 -0
- package/dist/node-kG64XxdE.js.map +1 -0
- package/dist/opencode-providers-DZuD_oYt.js +3149 -0
- package/dist/opencode-providers-DZuD_oYt.js.map +1 -0
- package/dist/options-CAQhu--C.js +452 -0
- package/dist/options-CAQhu--C.js.map +1 -0
- package/dist/patch-BkKje4cp.js +69 -0
- package/dist/patch-BkKje4cp.js.map +1 -0
- package/dist/paths-CBvm6pap.js +72 -0
- package/dist/paths-CBvm6pap.js.map +1 -0
- package/dist/plan-gwk6dJ8c.js +237 -0
- package/dist/plan-gwk6dJ8c.js.map +1 -0
- package/dist/preflight-BtubXKQ8.js +2 -0
- package/dist/preflight-BypZMNBG.js +185 -0
- package/dist/preflight-BypZMNBG.js.map +1 -0
- package/dist/prepare-Cwv0bHs_.js +3 -0
- package/dist/prepare-D6Uzcubb.js +213 -0
- package/dist/prepare-D6Uzcubb.js.map +1 -0
- package/dist/provenance-Dfb8YY5Y.js +129 -0
- package/dist/provenance-Dfb8YY5Y.js.map +1 -0
- package/dist/provider-BDbNOQhQ.js +2 -0
- package/dist/provider-CmAYTQQs.js +14 -0
- package/dist/provider-CmAYTQQs.js.map +1 -0
- package/dist/providers-2p3ylMHI.js +107 -0
- package/dist/providers-2p3ylMHI.js.map +1 -0
- package/dist/providers-BUOxitcT.js +2 -0
- package/dist/query-CFATHZmV.js +91 -0
- package/dist/query-CFATHZmV.js.map +1 -0
- package/dist/query-DR4m6CXf.js +2 -0
- package/dist/record-DQ8wzyco.js +99 -0
- package/dist/record-DQ8wzyco.js.map +1 -0
- package/dist/redact-C4fL_bZP.js +167 -0
- package/dist/redact-C4fL_bZP.js.map +1 -0
- package/dist/results-CeXb8Wde.js +37 -0
- package/dist/results-CeXb8Wde.js.map +1 -0
- package/dist/root-CFDjpWuc.js +110 -0
- package/dist/root-CFDjpWuc.js.map +1 -0
- package/dist/sandbox-By2gkdkC.js +133 -0
- package/dist/sandbox-By2gkdkC.js.map +1 -0
- package/dist/sandbox.d.ts +41 -0
- package/dist/sandbox.js +2 -0
- package/dist/server-Bi61uiY0.js +350 -0
- package/dist/server-Bi61uiY0.js.map +1 -0
- package/dist/server-CsSBL8NC.js +2 -0
- package/dist/source-BXzrmof5.js +2 -0
- package/dist/source-Bsfa3LZH.js +40 -0
- package/dist/source-Bsfa3LZH.js.map +1 -0
- package/dist/src-55j1zNHH.js +3 -0
- package/dist/src-AEwxd1UP.js +602 -0
- package/dist/src-AEwxd1UP.js.map +1 -0
- package/dist/src-BDwSuyIo.js +3 -0
- package/dist/src-BQyMiWri.js +490 -0
- package/dist/src-BQyMiWri.js.map +1 -0
- package/dist/src-C-LHyTSV.js +3 -0
- package/dist/src-Cmp-7YrD.js +1136 -0
- package/dist/src-Cmp-7YrD.js.map +1 -0
- package/dist/src-D0RdqpNb.js +252 -0
- package/dist/src-D0RdqpNb.js.map +1 -0
- package/dist/src-D8KYzxBX.js +793 -0
- package/dist/src-D8KYzxBX.js.map +1 -0
- package/dist/src-DEgff9Z7.js +4 -0
- package/dist/src-DXx_29pq.js +567 -0
- package/dist/src-DXx_29pq.js.map +1 -0
- package/dist/src-DwwiVLqv.js +2937 -0
- package/dist/src-DwwiVLqv.js.map +1 -0
- package/dist/src-N7E_mfcZ.js +2 -0
- package/dist/src-W9g-y5X5.js +3 -0
- package/dist/src-daS0wKH2.js +3 -0
- package/dist/state-Cyw3QTsN.js +2 -0
- package/dist/state-DZ7eqb1F.js +167 -0
- package/dist/state-DZ7eqb1F.js.map +1 -0
- package/dist/stats-BNAHlNMD.js +114 -0
- package/dist/stats-BNAHlNMD.js.map +1 -0
- package/dist/studio-app/assets/index-D6XyfFSH.js +58 -0
- package/dist/studio-app/assets/index-xwzWdcE_.css +2 -0
- package/dist/studio-app/favicon.svg +1 -0
- package/dist/studio-app/index.html +15 -0
- package/dist/summary-D7Ttky0z.js +166 -0
- package/dist/summary-D7Ttky0z.js.map +1 -0
- package/dist/text-gEcKjAyp.js +29 -0
- package/dist/text-gEcKjAyp.js.map +1 -0
- package/dist/types-C5vI3g1C.js +9 -0
- package/dist/types-C5vI3g1C.js.map +1 -0
- package/dist/types-DLec-6Cx.d.ts +1535 -0
- package/dist/usage-DsYUqIUM.js +24 -0
- package/dist/usage-DsYUqIUM.js.map +1 -0
- package/dist/view/index.d.ts +645 -0
- package/dist/view/index.js +8 -0
- package/dist/views-BrHOv6wg.js +220 -0
- package/dist/views-BrHOv6wg.js.map +1 -0
- package/dist/views-CwCWLWqo.js +3 -0
- package/dist/worker-entry-DnYkANPG.js +506 -0
- package/dist/worker-entry-DnYkANPG.js.map +1 -0
- package/dist/workspace-C2S0Z6gl.js +327 -0
- package/dist/workspace-C2S0Z6gl.js.map +1 -0
- package/dist/workspace-CyoXG6CY.js +2 -0
- package/docker/Dockerfile +19 -0
- package/package.json +79 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
//#region src/usage.ts
|
|
2
|
+
/** Two counts as one; null only when neither reported any. */
|
|
3
|
+
function addUsage(a, b) {
|
|
4
|
+
if (!a || !b) return a ?? b;
|
|
5
|
+
return {
|
|
6
|
+
input: a.input + b.input,
|
|
7
|
+
output: a.output + b.output,
|
|
8
|
+
cacheRead: a.cacheRead + b.cacheRead,
|
|
9
|
+
cacheWrite: a.cacheWrite + b.cacheWrite,
|
|
10
|
+
reasoning: a.reasoning === null && b.reasoning === null ? null : (a.reasoning ?? 0) + (b.reasoning ?? 0)
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
/** The total of every count that was reported; null when none was. */
|
|
14
|
+
const sumUsage = (usages) => {
|
|
15
|
+
let total = null;
|
|
16
|
+
for (const u of usages) total = addUsage(total, u ?? null);
|
|
17
|
+
return total;
|
|
18
|
+
};
|
|
19
|
+
/** Every token the prompt side used: uncached input plus cache reads and writes. */
|
|
20
|
+
const promptTokens = (u) => u.input + u.cacheRead + u.cacheWrite;
|
|
21
|
+
//#endregion
|
|
22
|
+
export { promptTokens as n, sumUsage as r, addUsage as t };
|
|
23
|
+
|
|
24
|
+
//# sourceMappingURL=usage-DsYUqIUM.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"usage-DsYUqIUM.js","names":[],"sources":["../src/usage.ts"],"mappings":";;AAGA,SAAgB,SAAS,GAAsB,GAAyC;CACtF,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO,KAAK;CAC1B,OAAO;EACL,OAAO,EAAE,QAAQ,EAAE;EACnB,QAAQ,EAAE,SAAS,EAAE;EACrB,WAAW,EAAE,YAAY,EAAE;EAC3B,YAAY,EAAE,aAAa,EAAE;EAC7B,WACE,EAAE,cAAc,QAAQ,EAAE,cAAc,OAAO,QAAQ,EAAE,aAAa,MAAM,EAAE,aAAa;CAC/F;AACF;;AAGA,MAAa,YAAY,WAAuE;CAC9F,IAAI,QAA2B;CAC/B,KAAK,MAAM,KAAK,QAAQ,QAAQ,SAAS,OAAO,KAAK,IAAI;CACzD,OAAO;AACT;;AAGA,MAAa,gBAAgB,MAAkB,EAAE,QAAQ,EAAE,YAAY,EAAE"}
|
|
@@ -0,0 +1,645 @@
|
|
|
1
|
+
import { $ as SandboxWarning, At as contrastEffort, Ct as EffortSample, Dt as cellContrast, Et as VERDICT_TEXT, Ft as effortSummary, It as effortVerdict, K as Outcome, Lt as formatCount, Mt as effortLabel, Nt as effortOf, Ot as cellEffort, Pt as effortSample, Rt as formatRatio, St as EffortContrast, Tt as Spread, _t as JudgementRecord, at as TranscriptEvent, bt as CellEffort, ct as TurnStatus, dt as AttemptRecord, ft as CellRecord, gt as HistoryEntry, ht as HistoryCell, it as TokenUsage, jt as effortChangeLabel, kt as cellSamples, mt as FileRef, pt as EvalEntry, vt as RunRecord, wt as EffortVerdict, xt as EFFORT_ALPHA, yt as StackSummary, zt as formatSeconds } from "../types-DLec-6Cx.js";
|
|
2
|
+
//#region src/view/analysis.d.ts
|
|
3
|
+
/** Bumped when the system prompt or output schema changes: an older analysis is superseded. */
|
|
4
|
+
export declare const ANALYSIS_PROMPT_VERSION = "analysis-v2";
|
|
5
|
+
/** Bumped when the digest's shape or selection rules change. */
|
|
6
|
+
export declare const DIGEST_VERSION = 1;
|
|
7
|
+
export type Confidence = 'high' | 'medium' | 'low';
|
|
8
|
+
export type AnalysisClaim = {
|
|
9
|
+
text: string;
|
|
10
|
+
/** Fact (`F3`), evidence (`E2`, `E2:L39`), attempt (`r2/scaffold/reeval-good#1`) or check ids. */
|
|
11
|
+
cites: string[];
|
|
12
|
+
confidence: Confidence;
|
|
13
|
+
/** Exact spans from the cited evidence; a claim whose quote isn't there is dropped. */
|
|
14
|
+
quotes?: string[];
|
|
15
|
+
/** Numbers in the text that no cited fact or evidence contains. */
|
|
16
|
+
unverifiedNumbers?: string[];
|
|
17
|
+
};
|
|
18
|
+
export type AnomalyKind = 'es-bug' | 'eval-design' | 'judge' | 'noise' | 'other';
|
|
19
|
+
export type AnalysisAnomaly = {
|
|
20
|
+
text: string;
|
|
21
|
+
kind: AnomalyKind;
|
|
22
|
+
cites: string[];
|
|
23
|
+
};
|
|
24
|
+
export type ActionKind = 'skill' | 'docs' | 'eval' | 'judge' | 'es' | 'harness';
|
|
25
|
+
export type Level = 'high' | 'medium' | 'low';
|
|
26
|
+
export type AnalysisAction = {
|
|
27
|
+
title: string;
|
|
28
|
+
target: string;
|
|
29
|
+
kind: ActionKind;
|
|
30
|
+
change: string;
|
|
31
|
+
expectedEffect: string;
|
|
32
|
+
/** An `reeval` command that shows the effect; null when the model's didn't parse as reeval's CLI. */
|
|
33
|
+
verify: string | null;
|
|
34
|
+
/** Why the model's verify command was rejected. */
|
|
35
|
+
verifyProblem?: string;
|
|
36
|
+
impact: Level;
|
|
37
|
+
effort: Level;
|
|
38
|
+
cites: string[];
|
|
39
|
+
};
|
|
40
|
+
export type AnalysisBody = {
|
|
41
|
+
tldr: string;
|
|
42
|
+
whatHappened: AnalysisClaim[];
|
|
43
|
+
results: AnalysisClaim[];
|
|
44
|
+
why: AnalysisClaim[];
|
|
45
|
+
anomalies: AnalysisAnomaly[];
|
|
46
|
+
actions: AnalysisAction[];
|
|
47
|
+
confidence: {
|
|
48
|
+
overall: Confidence;
|
|
49
|
+
why: string;
|
|
50
|
+
};
|
|
51
|
+
};
|
|
52
|
+
export type AnalysisValidation = {
|
|
53
|
+
/** Citations checked, and those that named nothing in the digest. */
|
|
54
|
+
citations: number;
|
|
55
|
+
invalid: string[];
|
|
56
|
+
/** Claims, anomalies and actions dropped for having no valid citation or an invented quote. */
|
|
57
|
+
dropped: number;
|
|
58
|
+
commands: number;
|
|
59
|
+
invalidCommands: string[];
|
|
60
|
+
unverifiedNumbers: number;
|
|
61
|
+
};
|
|
62
|
+
export type AnalysisScope = {
|
|
63
|
+
kind: 'run';
|
|
64
|
+
head: string;
|
|
65
|
+
/** Resolved once, so a nearer ancestor recorded later doesn't change what this analysed. */
|
|
66
|
+
base: string | null;
|
|
67
|
+
/** An eval filter (substring of the eval id), or null for every eval. */
|
|
68
|
+
evals: string | null;
|
|
69
|
+
};
|
|
70
|
+
export type AnalysisDoc = {
|
|
71
|
+
kind: 'es-analysis';
|
|
72
|
+
schemaVersion: 1;
|
|
73
|
+
scope: AnalysisScope;
|
|
74
|
+
/** The run the analysis belongs to: the scope's head. */
|
|
75
|
+
anchor: string;
|
|
76
|
+
inputs: {
|
|
77
|
+
records: Array<{
|
|
78
|
+
runId: string;
|
|
79
|
+
sha256: string;
|
|
80
|
+
}>;
|
|
81
|
+
digestVersion: number;
|
|
82
|
+
digestHash: string;
|
|
83
|
+
};
|
|
84
|
+
promptVersion: string;
|
|
85
|
+
model: string;
|
|
86
|
+
resolvedModel: string | null;
|
|
87
|
+
route: 'direct' | 'gateway' | 'fake';
|
|
88
|
+
usage: {
|
|
89
|
+
inputTokens: number | null;
|
|
90
|
+
outputTokens: number | null;
|
|
91
|
+
};
|
|
92
|
+
latencyMs: number;
|
|
93
|
+
createdAt: string;
|
|
94
|
+
author: string;
|
|
95
|
+
validation: AnalysisValidation;
|
|
96
|
+
analysis: AnalysisBody;
|
|
97
|
+
/** Short attempt ids used in citations, and where each attempt is. */
|
|
98
|
+
attempts: Record<string, {
|
|
99
|
+
runId: string;
|
|
100
|
+
evalSlug: string;
|
|
101
|
+
attempt: string;
|
|
102
|
+
}>;
|
|
103
|
+
/** Each cited fact or evidence id, summarised in one line. */
|
|
104
|
+
cited: Record<string, string>;
|
|
105
|
+
};
|
|
106
|
+
/** One analysis file the studio can show: saved in the run's folder, or in the local cache. */
|
|
107
|
+
export type AnalysisEntry = {
|
|
108
|
+
file: string;
|
|
109
|
+
source: 'saved' | 'cached';
|
|
110
|
+
doc: AnalysisDoc;
|
|
111
|
+
};
|
|
112
|
+
export type AnalysisIndex = {
|
|
113
|
+
runId: string;
|
|
114
|
+
entries: AnalysisEntry[];
|
|
115
|
+
};
|
|
116
|
+
export declare const analysisDataPath: (runId: string) => string;
|
|
117
|
+
/** Newest first, saved before cached at the same time. */
|
|
118
|
+
export declare function sortAnalyses(entries: AnalysisEntry[]): AnalysisEntry[];
|
|
119
|
+
/** Superseded by a newer prompt or digest version: the studio says so. */
|
|
120
|
+
export declare const isSuperseded: (doc: AnalysisDoc) => boolean;
|
|
121
|
+
//#endregion
|
|
122
|
+
//#region src/view/compare.d.ts
|
|
123
|
+
export type CompareStatus = 'regressed' | 'fixed' | 'same' | 'drift' | 'added' | 'removed' | 'infra';
|
|
124
|
+
export type CompareRow = {
|
|
125
|
+
id: string;
|
|
126
|
+
status: CompareStatus;
|
|
127
|
+
base: Outcome | null;
|
|
128
|
+
head: Outcome | null;
|
|
129
|
+
/** For `drift`: what changed, e.g. `evals/helpers.ts changed` or `harness config changed`. */
|
|
130
|
+
changes?: string[];
|
|
131
|
+
/**
|
|
132
|
+
* When the runs ran different cells of the eval: the cells only head or only base has. Then
|
|
133
|
+
* `status`, `base` and `head` are over the cells both have.
|
|
134
|
+
*/
|
|
135
|
+
cells?: {
|
|
136
|
+
added: string[];
|
|
137
|
+
removed: string[];
|
|
138
|
+
};
|
|
139
|
+
/** Cells in both runs, with each side's lift (variants) and effort. */
|
|
140
|
+
perCell?: CompareCell[];
|
|
141
|
+
};
|
|
142
|
+
export type CompareCell = {
|
|
143
|
+
key: string;
|
|
144
|
+
base: Pick<HistoryCell, 'lift' | 'effort' | 'pass' | 'fail'>;
|
|
145
|
+
head: Pick<HistoryCell, 'lift' | 'effort' | 'pass' | 'fail'>;
|
|
146
|
+
};
|
|
147
|
+
export type StackChange = {
|
|
148
|
+
field: keyof StackSummary;
|
|
149
|
+
label: string;
|
|
150
|
+
base: string;
|
|
151
|
+
head: string;
|
|
152
|
+
/** Not covered by eval fingerprints, so outcome flips across it are drift, not regressions. */
|
|
153
|
+
outsideFingerprint: boolean;
|
|
154
|
+
};
|
|
155
|
+
type StackCell = Pick<HistoryCell, 'key' | 'harness' | 'environment' | 'environmentDigest' | 'fingerprint' | 'fingerprintParts'>;
|
|
156
|
+
/** A run's stack, and its cells when known: the per-cell fields then compare over shared cells. */
|
|
157
|
+
export type StackSource = Pick<HistoryEntry, 'stack'> & {
|
|
158
|
+
evals?: ReadonlyArray<{
|
|
159
|
+
id: string;
|
|
160
|
+
cells: readonly StackCell[];
|
|
161
|
+
}>;
|
|
162
|
+
};
|
|
163
|
+
/**
|
|
164
|
+
* What changed in the stack between two runs. With both runs' cells, images, agent CLIs,
|
|
165
|
+
* harnesses and environments compare over the cells both have, so a run over fewer cells isn't
|
|
166
|
+
* drift. Reported served-judge lists only compare when both have one.
|
|
167
|
+
*/
|
|
168
|
+
export declare function stackChanges(base: StackSource, head: StackSource): StackChange[];
|
|
169
|
+
/**
|
|
170
|
+
* Whether any of these cells changed fingerprint since `seen` last had it, then records them.
|
|
171
|
+
* A cell `seen` doesn't have yet is new coverage, not a change.
|
|
172
|
+
*/
|
|
173
|
+
export declare function cellsChanged(seen: Map<string, string>, evalId: string, cells: ReadonlyArray<Pick<HistoryCell, 'key' | 'fingerprint'>>): boolean;
|
|
174
|
+
type ComparableEval = Pick<HistoryEntry['evals'][number], 'id' | 'fingerprint' | 'outcome' | 'sources' | 'cells'>;
|
|
175
|
+
type Comparable = Pick<HistoryEntry, 'stack'> & {
|
|
176
|
+
evals: ComparableEval[];
|
|
177
|
+
};
|
|
178
|
+
/** Which source files were added, removed or changed. */
|
|
179
|
+
export declare function sourceChanges(before: ComparableEval['sources'], after: ComparableEval['sources']): string[];
|
|
180
|
+
/** Why two runs of an eval aren't comparable: the source files and fingerprint parts that differ. */
|
|
181
|
+
export declare function whyChanged(base: ComparableEval, head: ComparableEval): string[];
|
|
182
|
+
export declare function compare(base: Comparable, head: Comparable): CompareRow[];
|
|
183
|
+
//#endregion
|
|
184
|
+
//#region src/view/debug.d.ts
|
|
185
|
+
/**
|
|
186
|
+
* An attempt's local debug data, as `reeval run` writes it to `.reeval/debug/<runId>/<eval>/<cell>/<n>/`
|
|
187
|
+
* and `reeval studio` serves it. Browser-safe. Never part of a record or the history folder.
|
|
188
|
+
*
|
|
189
|
+
* debug.log the `-v` trace, one line per event, always written
|
|
190
|
+
* requests.ndjson one line per egress gateway request, always written
|
|
191
|
+
* trace.ndjson raw payloads (`-vv`), written when the attempt fails or with `--trace`
|
|
192
|
+
*/
|
|
193
|
+
export declare const DEBUG_LOG = "debug.log";
|
|
194
|
+
export declare const REQUESTS_FILE = "requests.ndjson";
|
|
195
|
+
export declare const TRACE_FILE = "trace.ndjson";
|
|
196
|
+
/** What part of the attempt a line is about, in the order an attempt goes through them. */
|
|
197
|
+
export declare const DEBUG_PHASES: readonly ['plan', 'image', 'container', 'env', 'broker', 'harness', 'check', 'judge', 'gateway', 'teardown', 'docker'];
|
|
198
|
+
export type DebugPhase = (typeof DEBUG_PHASES)[number];
|
|
199
|
+
/** One debug line: level 1 is `-v`, level 2 is `-vv` only. */
|
|
200
|
+
export type DebugLine = {
|
|
201
|
+
at: number;
|
|
202
|
+
level: 1 | 2;
|
|
203
|
+
phase: DebugPhase;
|
|
204
|
+
text: string;
|
|
205
|
+
/** More lines under it, e.g. a docker command's output or a raw payload. */
|
|
206
|
+
detail?: string;
|
|
207
|
+
};
|
|
208
|
+
/** One raw trace entry: `ms` is since the attempt started. */
|
|
209
|
+
export type TraceEntry = {
|
|
210
|
+
at: string;
|
|
211
|
+
ms: number;
|
|
212
|
+
phase: DebugPhase;
|
|
213
|
+
kind: string;
|
|
214
|
+
data: unknown;
|
|
215
|
+
};
|
|
216
|
+
/** One request through the egress gateway, or one tunnel it passed through without reading. */
|
|
217
|
+
export type DebugRequest = {
|
|
218
|
+
at: string;
|
|
219
|
+
method: string;
|
|
220
|
+
url: string;
|
|
221
|
+
host: string;
|
|
222
|
+
port: number;
|
|
223
|
+
via: 'tunnel' | 'decrypted' | 'http';
|
|
224
|
+
allowed: boolean;
|
|
225
|
+
status?: number;
|
|
226
|
+
reason?: string;
|
|
227
|
+
sent: number;
|
|
228
|
+
received: number;
|
|
229
|
+
/** Brokered secrets and the harness credential swapped in, by name. */
|
|
230
|
+
substituted: string[];
|
|
231
|
+
ms: number;
|
|
232
|
+
};
|
|
233
|
+
/** What the studio's Debug tab shows for one attempt. */
|
|
234
|
+
export type DebugView = {
|
|
235
|
+
lines: Array<{
|
|
236
|
+
at: string;
|
|
237
|
+
phase: string;
|
|
238
|
+
text: string;
|
|
239
|
+
}>;
|
|
240
|
+
requests: DebugRequest[];
|
|
241
|
+
/** Whether `trace.ndjson` was kept. */
|
|
242
|
+
trace: boolean;
|
|
243
|
+
};
|
|
244
|
+
/**
|
|
245
|
+
* The studio server's debug API (`reeval studio` only): an attempt's DebugView (null when it has no
|
|
246
|
+
* local data), and its raw trace.
|
|
247
|
+
*/
|
|
248
|
+
export declare const debugApiPath: (runId: string, attemptKey: string) => string;
|
|
249
|
+
export declare const traceApiPath: (runId: string, attemptKey: string) => string;
|
|
250
|
+
/** `12:50:01`, or `12:50:01.234` with milliseconds, in UTC like every reeval timestamp. */
|
|
251
|
+
export declare const debugTime: (at: number, ms: boolean) => string;
|
|
252
|
+
/** A line of debug.log: `2026-09-26T12:50:01.234Z plan text`. */
|
|
253
|
+
export declare const debugLogLine: (line: Pick<DebugLine, 'at' | 'phase' | 'text'>) => string;
|
|
254
|
+
/** debug.log's lines; a line that isn't one (a wrapped message) continues the one before. */
|
|
255
|
+
export declare function parseDebugLog(text: string): DebugView['lines'];
|
|
256
|
+
export declare function parseNdjson<T>(text: string): T[];
|
|
257
|
+
/** `512 B`, `1.2 KB`, `3.4 MB`. */
|
|
258
|
+
export declare function formatBytes(n: number): string;
|
|
259
|
+
/**
|
|
260
|
+
* The gateway line for a request: method, URL, status, allowed or blocked and why, bytes each way,
|
|
261
|
+
* time, and which secrets were swapped in (by name).
|
|
262
|
+
*/
|
|
263
|
+
export declare function requestLine(r: Omit<DebugRequest, 'at'>): string;
|
|
264
|
+
//#endregion
|
|
265
|
+
//#region src/view/header.d.ts
|
|
266
|
+
/** The derived header of one run: what listings, indexes and comparisons need without the record. */
|
|
267
|
+
export declare function runHeader(record: RunRecord, recorded: boolean): HistoryEntry;
|
|
268
|
+
//#endregion
|
|
269
|
+
//#region src/view/live.d.ts
|
|
270
|
+
/** The progress file's name inside the run's staging directory. */
|
|
271
|
+
export declare const LIVE_FILE = "live.json";
|
|
272
|
+
export declare const LIVE_SCHEMA_VERSION = 1;
|
|
273
|
+
/** Where an attempt is: waiting, running, or finished with its outcome. */
|
|
274
|
+
export type LiveStatus = 'queued' | 'running' | 'pass' | 'fail' | 'infra_error' | 'skipped';
|
|
275
|
+
export type LiveAttempt = {
|
|
276
|
+
/** The attempt key, `<eval id> # <cell key>#<n>`, as in the record. */
|
|
277
|
+
key: string;
|
|
278
|
+
evalId: string;
|
|
279
|
+
name: string;
|
|
280
|
+
file: string;
|
|
281
|
+
/** The cell key, e.g. `fake/default/fake`. */
|
|
282
|
+
cell: string;
|
|
283
|
+
n: number;
|
|
284
|
+
status: LiveStatus;
|
|
285
|
+
startedAt: string | null;
|
|
286
|
+
durationMs: number | null;
|
|
287
|
+
};
|
|
288
|
+
/** Attempts finished, and of those how many passed, failed or errored; and how many are running. */
|
|
289
|
+
export type LiveCounts = {
|
|
290
|
+
done: number;
|
|
291
|
+
passed: number;
|
|
292
|
+
failed: number;
|
|
293
|
+
errored: number;
|
|
294
|
+
skipped: number;
|
|
295
|
+
running: number;
|
|
296
|
+
};
|
|
297
|
+
/** What the run is doing: running attempts, cancelling them, or assembling its record. */
|
|
298
|
+
export type LivePhase = 'running' | 'cancelling' | 'finishing';
|
|
299
|
+
export type LiveProgress = {
|
|
300
|
+
schemaVersion: 1;
|
|
301
|
+
runId: string;
|
|
302
|
+
startedAt: string;
|
|
303
|
+
/** The `reeval run` process and the host it runs on, so a killed run can be told apart. */
|
|
304
|
+
pid: number;
|
|
305
|
+
host: string;
|
|
306
|
+
/** Rewritten every few seconds while the run is alive. */
|
|
307
|
+
heartbeatAt: string;
|
|
308
|
+
phase: LivePhase;
|
|
309
|
+
planned: number;
|
|
310
|
+
counts: LiveCounts;
|
|
311
|
+
attempts: LiveAttempt[];
|
|
312
|
+
};
|
|
313
|
+
/** A live run as the studio lists it: `stopped` when its process is gone or went quiet. */
|
|
314
|
+
export type LiveRun = Omit<LiveProgress, 'schemaVersion' | 'pid' | 'host' | 'heartbeatAt'> & {
|
|
315
|
+
state: LivePhase | 'stopped';
|
|
316
|
+
};
|
|
317
|
+
export type LiveIndex = {
|
|
318
|
+
runs: LiveRun[];
|
|
319
|
+
};
|
|
320
|
+
export declare const liveApiPath = "api/live";
|
|
321
|
+
export declare const liveTranscriptPath: (runId: string, attemptKey: string) => string;
|
|
322
|
+
export declare const FINISHED: readonly LiveStatus[];
|
|
323
|
+
export declare const isFinished: (status: LiveStatus) => boolean;
|
|
324
|
+
export declare function liveCounts(attempts: readonly Pick<LiveAttempt, 'status'>[]): LiveCounts;
|
|
325
|
+
/**
|
|
326
|
+
* A progress file as the studio reads it, or null when it isn't one it can read: half-written,
|
|
327
|
+
* corrupt, or from a newer reeval. Counts are recomputed from the attempts, never trusted.
|
|
328
|
+
*/
|
|
329
|
+
export declare function readableLiveProgress(value: unknown): LiveProgress | null;
|
|
330
|
+
//#endregion
|
|
331
|
+
//#region src/view/model.d.ts
|
|
332
|
+
export type EvalCatalogEntry = {
|
|
333
|
+
slug: string;
|
|
334
|
+
id: string;
|
|
335
|
+
file: string;
|
|
336
|
+
/** Group names, then the eval name. */
|
|
337
|
+
path: string[];
|
|
338
|
+
runs: number;
|
|
339
|
+
latest: {
|
|
340
|
+
runId: string;
|
|
341
|
+
outcome: Outcome;
|
|
342
|
+
};
|
|
343
|
+
};
|
|
344
|
+
export type IndexManifest = {
|
|
345
|
+
generatedAt: string;
|
|
346
|
+
/** Newest first. */
|
|
347
|
+
shards: Array<{
|
|
348
|
+
id: string;
|
|
349
|
+
runs: number;
|
|
350
|
+
}>;
|
|
351
|
+
latest: string | null;
|
|
352
|
+
evals: EvalCatalogEntry[];
|
|
353
|
+
};
|
|
354
|
+
export type StudioInfo = {
|
|
355
|
+
mode: 'live' | 'static';
|
|
356
|
+
es: string;
|
|
357
|
+
project: string;
|
|
358
|
+
generatedAt: string;
|
|
359
|
+
/** `debug`: local debug data (`.reeval/debug`) can be read; never in a static build. */
|
|
360
|
+
features: {
|
|
361
|
+
live: boolean;
|
|
362
|
+
scratch: boolean;
|
|
363
|
+
artifacts: boolean;
|
|
364
|
+
debug: boolean;
|
|
365
|
+
};
|
|
366
|
+
};
|
|
367
|
+
/** Splits `file › group › … › name` back into its parts. */
|
|
368
|
+
export declare function splitEvalId(id: string): {
|
|
369
|
+
file: string;
|
|
370
|
+
path: string[];
|
|
371
|
+
};
|
|
372
|
+
/** Month shards (newest run first) and the manifest over them, from every run's header. */
|
|
373
|
+
export declare function buildIndex(entries: HistoryEntry[], generatedAt?: string): {
|
|
374
|
+
manifest: IndexManifest;
|
|
375
|
+
shards: Map<string, HistoryEntry[]>;
|
|
376
|
+
};
|
|
377
|
+
/** Every file the index consists of, by relative path. */
|
|
378
|
+
export declare function indexFiles(entries: HistoryEntry[], generatedAt?: string): Map<string, unknown>;
|
|
379
|
+
export declare function formatDuration(ms: number | null | undefined): string;
|
|
380
|
+
export declare const formatPercent: (rate: number | null) => string;
|
|
381
|
+
/**
|
|
382
|
+
* A harness of the run's stack as `cursor (model composer-2.5, served composer-2.5[fast=true])`:
|
|
383
|
+
* the configured model, then the variant the harness says it resolved.
|
|
384
|
+
*/
|
|
385
|
+
export declare function harnessLabel(h: RunRecord['stack']['harnesses'][number]): string;
|
|
386
|
+
/** Harness-reported tokens as `127k in (98k cached) · 776 out`; null when none were reported. */
|
|
387
|
+
export declare function formatUsage(usage: TokenUsage | null | undefined): string | null;
|
|
388
|
+
export declare const shortSha: (sha: string | null | undefined) => string;
|
|
389
|
+
/**
|
|
390
|
+
* An attempt's effort, counted as the CLI's turn lines count it: tool calls, then file edits.
|
|
391
|
+
* Edit calls become one file edit per file they change, so they're counted apart, never summed.
|
|
392
|
+
*/
|
|
393
|
+
export declare function activityLabel(a: Pick<AttemptRecord['activity'], 'tools' | 'fileEdits'>): string;
|
|
394
|
+
/**
|
|
395
|
+
* A run's outcome, as `reeval run`'s exit code has it: a failed gated cell (or a budget stop) fails
|
|
396
|
+
* the run, while failures in ungated cells (controls, by default) don't.
|
|
397
|
+
*/
|
|
398
|
+
export declare function runOutcome(run: {
|
|
399
|
+
status: RunRecord['status'];
|
|
400
|
+
evals: ReadonlyArray<{
|
|
401
|
+
outcome: Outcome;
|
|
402
|
+
}>;
|
|
403
|
+
}): Outcome;
|
|
404
|
+
export type RunRow = {
|
|
405
|
+
entry: HistoryEntry;
|
|
406
|
+
passed: number;
|
|
407
|
+
/** Attempts that reached a verdict or errored; skips excluded. */
|
|
408
|
+
total: number;
|
|
409
|
+
/** Stack changes in the cells it shares with earlier runs, and in the judge since the previous complete run. */
|
|
410
|
+
drift: StackChange[];
|
|
411
|
+
/** Evals with a cell whose fingerprint changed since the last run that had that cell. */
|
|
412
|
+
changedEvals: number;
|
|
413
|
+
};
|
|
414
|
+
/**
|
|
415
|
+
* Newest first. Drift is measured per cell, against the last run that had it: a run over fewer
|
|
416
|
+
* cells (a filtered run) is no drift, and neither is the full run after it. The judge's settings
|
|
417
|
+
* compare with the next older complete run.
|
|
418
|
+
*/
|
|
419
|
+
export declare function runRows(entries: HistoryEntry[]): RunRow[];
|
|
420
|
+
/** Attempt counts; `missed` is ungated cells' failures, which `fail` leaves out. */
|
|
421
|
+
export type Tally = {
|
|
422
|
+
pass: number;
|
|
423
|
+
fail: number;
|
|
424
|
+
missed: number;
|
|
425
|
+
infra: number;
|
|
426
|
+
skipped: number;
|
|
427
|
+
};
|
|
428
|
+
export type TreeNode = {
|
|
429
|
+
kind: 'group';
|
|
430
|
+
key: string;
|
|
431
|
+
name: string;
|
|
432
|
+
tally: Tally;
|
|
433
|
+
children: TreeNode[];
|
|
434
|
+
} | {
|
|
435
|
+
kind: 'eval';
|
|
436
|
+
key: string;
|
|
437
|
+
name: string;
|
|
438
|
+
tally: Tally;
|
|
439
|
+
eval: EvalEntry;
|
|
440
|
+
};
|
|
441
|
+
/** Files, then groups, then evals, each with attempt counts; failing branches sort first. */
|
|
442
|
+
export declare function evalTree(record: Pick<RunRecord, 'evals'>): TreeNode[];
|
|
443
|
+
export type CellRow = {
|
|
444
|
+
key: string;
|
|
445
|
+
harness: string;
|
|
446
|
+
environment: string;
|
|
447
|
+
sandbox: string;
|
|
448
|
+
role: CellRecord['role'];
|
|
449
|
+
/** Whether the cell's failures fail the eval: its gate isn't `'none'`. */
|
|
450
|
+
gated: boolean;
|
|
451
|
+
gate: CellRecord['gate'];
|
|
452
|
+
overrides: string[];
|
|
453
|
+
runs: number;
|
|
454
|
+
pass: number;
|
|
455
|
+
fail: number;
|
|
456
|
+
infraError: number;
|
|
457
|
+
skipped: number;
|
|
458
|
+
passRate: number | null;
|
|
459
|
+
outcome: Outcome;
|
|
460
|
+
/** A variant's lift over its control, in whole points (from the record), else null. */
|
|
461
|
+
lift: number | null;
|
|
462
|
+
/** Its 95% interval in whole points, and whether that spans 0. */
|
|
463
|
+
liftInterval: {
|
|
464
|
+
lo: number;
|
|
465
|
+
hi: number;
|
|
466
|
+
withinNoise: boolean;
|
|
467
|
+
} | null;
|
|
468
|
+
/** The interval's lower bound is below the compare's `minLift`. */
|
|
469
|
+
belowMinLift: boolean;
|
|
470
|
+
/** Median tool calls and agent time over the cell's counted attempts. */
|
|
471
|
+
effort: CellEffort | null;
|
|
472
|
+
/** A variant's effort against its control, with the rank test's verdict. */
|
|
473
|
+
effortVsControl: EffortContrast | null;
|
|
474
|
+
attempts: AttemptRecord[];
|
|
475
|
+
};
|
|
476
|
+
/** One row per cell: controls and variants, with each variant's recorded lift. */
|
|
477
|
+
export declare function cellRows(e: EvalEntry): CellRow[];
|
|
478
|
+
/** Whether an eval needs the cells table: several cells, or several runs of one. */
|
|
479
|
+
export declare const isMatrix: (e: Pick<EvalEntry, 'cells'>) => boolean;
|
|
480
|
+
export type EvalHistoryPoint = {
|
|
481
|
+
run: HistoryEntry;
|
|
482
|
+
outcome: Outcome;
|
|
483
|
+
passRate: number | null;
|
|
484
|
+
cells: HistoryCell[];
|
|
485
|
+
/** A cell's fingerprint differs from the last run that had that cell. */
|
|
486
|
+
changed: boolean;
|
|
487
|
+
};
|
|
488
|
+
/** One eval across runs, oldest first. */
|
|
489
|
+
export declare function evalHistory(entries: HistoryEntry[], slug: string): EvalHistoryPoint[];
|
|
490
|
+
export type CompareViewRow = {
|
|
491
|
+
id: string;
|
|
492
|
+
slug: string | null;
|
|
493
|
+
status: CompareStatus;
|
|
494
|
+
base: Outcome | null;
|
|
495
|
+
head: Outcome | null;
|
|
496
|
+
basePassRate: number | null;
|
|
497
|
+
headPassRate: number | null;
|
|
498
|
+
/** See `CompareRow`: then the pass rates are over the cells both runs have too. */
|
|
499
|
+
cells?: CompareRow['cells'];
|
|
500
|
+
changes?: string[];
|
|
501
|
+
/** Mean judge score per run; null without judgements or without the record. */
|
|
502
|
+
baseScore: number | null;
|
|
503
|
+
headScore: number | null;
|
|
504
|
+
/** Cells in both runs, with lift and effort on each side. */
|
|
505
|
+
perCell: CompareCell[];
|
|
506
|
+
};
|
|
507
|
+
export declare const COMPARE_ORDER: CompareStatus[];
|
|
508
|
+
/**
|
|
509
|
+
* Whether some of an eval's cells aren't gated (`gate: 'none'`, e.g. controls under
|
|
510
|
+
* `compare.gate: 'variants'`). Its pass rate then covers fewer attempts than its pass count, so
|
|
511
|
+
* the eval line labels the rate `gated`: `2/4 · 100% gated`.
|
|
512
|
+
*/
|
|
513
|
+
export declare const hasUngatedCells: (cells: readonly Pick<HistoryCell, 'gate'>[]) => boolean;
|
|
514
|
+
/**
|
|
515
|
+
* An expected miss: a failed attempt in an ungated cell (a control, by default). It's reported
|
|
516
|
+
* apart from failures, since it fails neither its eval nor the run.
|
|
517
|
+
*/
|
|
518
|
+
export declare const isUngatedMiss: (e: {
|
|
519
|
+
cells: readonly Pick<HistoryCell, 'key' | 'gate'>[];
|
|
520
|
+
}, a: Pick<AttemptRecord, 'cell' | 'outcome' | 'error'>) => boolean;
|
|
521
|
+
/** `reeval compare`'s classification, plus pass rates and judge-score deltas from the records. */
|
|
522
|
+
export declare function compareView(base: HistoryEntry, head: HistoryEntry, records?: {
|
|
523
|
+
base?: RunRecord;
|
|
524
|
+
head?: RunRecord;
|
|
525
|
+
}): {
|
|
526
|
+
rows: CompareViewRow[];
|
|
527
|
+
counts: Record<CompareStatus, number>;
|
|
528
|
+
changes: StackChange[];
|
|
529
|
+
};
|
|
530
|
+
export type TranscriptLine = ({
|
|
531
|
+
turn: number;
|
|
532
|
+
} & TranscriptEvent) | {
|
|
533
|
+
turn: number;
|
|
534
|
+
ts: string;
|
|
535
|
+
atMs: number;
|
|
536
|
+
type: 'prompt';
|
|
537
|
+
text: string;
|
|
538
|
+
} | {
|
|
539
|
+
turn: number;
|
|
540
|
+
ts: string;
|
|
541
|
+
atMs: number;
|
|
542
|
+
type: 'turn-end';
|
|
543
|
+
status: TurnStatus;
|
|
544
|
+
durationMs: number;
|
|
545
|
+
};
|
|
546
|
+
export type TranscriptItem = {
|
|
547
|
+
kind: 'turn';
|
|
548
|
+
turn: number;
|
|
549
|
+
prompt: string;
|
|
550
|
+
ts: string;
|
|
551
|
+
} | {
|
|
552
|
+
kind: 'event';
|
|
553
|
+
turn: number;
|
|
554
|
+
event: TranscriptLine;
|
|
555
|
+
} | {
|
|
556
|
+
kind: 'turn-end';
|
|
557
|
+
turn: number;
|
|
558
|
+
status: TurnStatus;
|
|
559
|
+
durationMs: number;
|
|
560
|
+
};
|
|
561
|
+
/** Parses `transcript.jsonl`, skipping lines that aren't JSON (a truncated tail, say). */
|
|
562
|
+
export declare function parseTranscript(text: string): TranscriptLine[];
|
|
563
|
+
export declare function transcriptItems(lines: TranscriptLine[]): TranscriptItem[];
|
|
564
|
+
export type PatchFile = {
|
|
565
|
+
path: string;
|
|
566
|
+
status: 'added' | 'deleted' | 'modified' | 'renamed' | 'binary';
|
|
567
|
+
added: number;
|
|
568
|
+
removed: number;
|
|
569
|
+
text: string;
|
|
570
|
+
};
|
|
571
|
+
/** Splits a git patch into one entry per file, with line counts. */
|
|
572
|
+
export declare function splitPatch(patch: string): PatchFile[];
|
|
573
|
+
export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string) => EvalEntry | undefined;
|
|
574
|
+
export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
|
|
575
|
+
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
576
|
+
export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
|
|
577
|
+
//#endregion
|
|
578
|
+
//#region src/view/paths.d.ts
|
|
579
|
+
/**
|
|
580
|
+
* The studio's data API is a file layout: `reeval studio` serves these relative paths and
|
|
581
|
+
* `reeval studio build` writes them to disk, so the web client reads both the same way.
|
|
582
|
+
*
|
|
583
|
+
* studio.json mode, project and features
|
|
584
|
+
* data/index/manifest.json shard list and eval catalog
|
|
585
|
+
* data/index/<YYYY-MM>.json run headers started that UTC month, newest first
|
|
586
|
+
* data/runs/<runId>.json the record, byte for byte
|
|
587
|
+
* data/runs/<runId>/<artifact path> artifacts, raw
|
|
588
|
+
* data/analysis/<runId>.json the run's saved analyses (and, live, cached ones)
|
|
589
|
+
*/
|
|
590
|
+
export declare const RUN_ID: RegExp;
|
|
591
|
+
export declare const studioPath = "studio.json";
|
|
592
|
+
export declare const manifestPath = "data/index/manifest.json";
|
|
593
|
+
export declare const shardPath: (id: string) => string;
|
|
594
|
+
export declare const recordPath: (runId: string) => string;
|
|
595
|
+
export declare const artifactPath: (runId: string, path: string) => string;
|
|
596
|
+
/** A shard id: the UTC month a run started in. */
|
|
597
|
+
export declare const shardOf: (startedAt: string) => string;
|
|
598
|
+
export type DataTarget = {
|
|
599
|
+
kind: 'studio';
|
|
600
|
+
} | {
|
|
601
|
+
kind: 'manifest';
|
|
602
|
+
} | {
|
|
603
|
+
kind: 'shard';
|
|
604
|
+
id: string;
|
|
605
|
+
} | {
|
|
606
|
+
kind: 'record';
|
|
607
|
+
runId: string;
|
|
608
|
+
} | {
|
|
609
|
+
kind: 'analysis';
|
|
610
|
+
runId: string;
|
|
611
|
+
} | {
|
|
612
|
+
kind: 'artifact';
|
|
613
|
+
runId: string;
|
|
614
|
+
segments: string[];
|
|
615
|
+
};
|
|
616
|
+
/**
|
|
617
|
+
* Maps a request path (relative, without a leading slash, already percent-decoded once) to what
|
|
618
|
+
* it names, or null. Every segment must match an allowlist, so `.`, `..`, empty segments,
|
|
619
|
+
* backslashes and NUL never get through.
|
|
620
|
+
*/
|
|
621
|
+
export declare function parseDataPath(path: string): DataTarget | null;
|
|
622
|
+
/** Artifacts are untrusted agent output: JSON stays JSON, everything else is plain text. */
|
|
623
|
+
export declare const artifactType: (name: string) => string;
|
|
624
|
+
//#endregion
|
|
625
|
+
//#region src/view/record.d.ts
|
|
626
|
+
/** A file from a newer reeval, whose schemaVersion this reeval doesn't know. */
|
|
627
|
+
export declare class NewerSchemaError extends Error {
|
|
628
|
+
name: string;
|
|
629
|
+
/** The message without its subject: `written by a newer reeval (…). Upgrade reeval to read it.` */
|
|
630
|
+
readonly reason: string;
|
|
631
|
+
constructor(what: string, version: number, newest: number);
|
|
632
|
+
}
|
|
633
|
+
/**
|
|
634
|
+
* A record as every reader sees it: migrated to this reeval's schemaVersion, with each field added
|
|
635
|
+
* after it was written read as its default, so older runs keep loading in `reeval show`, `reeval query`,
|
|
636
|
+
* compare, history and the studio. A record from a newer reeval throws a `NewerSchemaError`.
|
|
637
|
+
*/
|
|
638
|
+
export declare function readableRecord(record: RunRecord, what?: string): RunRecord;
|
|
639
|
+
/** A transcript line as every reader sees it. */
|
|
640
|
+
export declare const readableTranscriptLine: (line: TranscriptLine) => TranscriptLine;
|
|
641
|
+
/** An analysis document as every reader sees it; one from a newer reeval throws. */
|
|
642
|
+
export declare const readableAnalysis: (doc: AnalysisDoc, what?: string) => AnalysisDoc;
|
|
643
|
+
//#endregion
|
|
644
|
+
export { type AttemptRecord, CellEffort, type CellRecord, EFFORT_ALPHA, EffortContrast, EffortSample, EffortVerdict, type EvalEntry, type FileRef, type HistoryCell, type HistoryEntry, type JudgementRecord, type Outcome, type RunRecord, type SandboxWarning, Spread, type StackSummary, type TranscriptEvent, VERDICT_TEXT, cellContrast, cellEffort, cellSamples, contrastEffort, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, formatCount, formatRatio, formatSeconds };
|
|
645
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { a as parseDataPath, c as shardPath, i as manifestPath, l as studioPath, n as artifactPath, o as recordPath, r as artifactType, s as shardOf, t as RUN_ID } from "../paths-CBvm6pap.js";
|
|
2
|
+
import { a as debugApiPath, c as formatBytes, d as requestLine, f as traceApiPath, i as TRACE_FILE, l as parseDebugLog, n as DEBUG_PHASES, o as debugLogLine, r as REQUESTS_FILE, s as debugTime, t as DEBUG_LOG, u as parseNdjson } from "../debug-D24NOQH1.js";
|
|
3
|
+
import { a as sortAnalyses, i as isSuperseded, n as DIGEST_VERSION, r as analysisDataPath, t as ANALYSIS_PROMPT_VERSION } from "../analysis-CxSCt1YY.js";
|
|
4
|
+
import { A as cellSamples, B as formatSeconds, C as splitEvalId, D as VERDICT_TEXT, E as EFFORT_ALPHA, F as effortSample, G as whyChanged, H as compare, I as effortSummary, L as effortVerdict, M as effortChangeLabel, N as effortLabel, O as cellContrast, P as effortOf, R as formatCount, S as shortSha, T as transcriptItems, U as sourceChanges, V as cellsChanged, W as stackChanges, _ as isMatrix, a as compareView, b as runOutcome, c as evalTree, d as formatDuration, f as formatPercent, g as indexFiles, h as hasUngatedCells, i as cellRows, j as contrastEffort, k as cellEffort, l as findAttempt, m as harnessLabel, n as activityLabel, o as defaultAttempt, p as formatUsage, r as buildIndex, s as evalHistory, t as COMPARE_ORDER, u as findEval, v as isUngatedMiss, w as splitPatch, x as runRows, y as parseTranscript, z as formatRatio } from "../model-CDL98xue.js";
|
|
5
|
+
import { t as runHeader } from "../header-DzFDoMCI.js";
|
|
6
|
+
import { o as readableRecord, r as readableAnalysis, s as readableTranscriptLine, t as NewerSchemaError } from "../record-DQ8wzyco.js";
|
|
7
|
+
import { a as liveApiPath, c as readableLiveProgress, i as isFinished, n as LIVE_FILE, o as liveCounts, r as LIVE_SCHEMA_VERSION, s as liveTranscriptPath, t as FINISHED } from "../live-Bs3LReMV.js";
|
|
8
|
+
export { ANALYSIS_PROMPT_VERSION, COMPARE_ORDER, DEBUG_LOG, DEBUG_PHASES, DIGEST_VERSION, EFFORT_ALPHA, FINISHED, LIVE_FILE, LIVE_SCHEMA_VERSION, NewerSchemaError, REQUESTS_FILE, RUN_ID, TRACE_FILE, VERDICT_TEXT, activityLabel, analysisDataPath, artifactPath, artifactType, buildIndex, cellContrast, cellEffort, cellRows, cellSamples, cellsChanged, compare, compareView, contrastEffort, debugApiPath, debugLogLine, debugTime, defaultAttempt, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, evalHistory, evalTree, findAttempt, findEval, formatBytes, formatCount, formatDuration, formatPercent, formatRatio, formatSeconds, formatUsage, harnessLabel, hasUngatedCells, indexFiles, isFinished, isMatrix, isSuperseded, isUngatedMiss, liveApiPath, liveCounts, liveTranscriptPath, manifestPath, parseDataPath, parseDebugLog, parseNdjson, parseTranscript, readableAnalysis, readableLiveProgress, readableRecord, readableTranscriptLine, recordPath, requestLine, runHeader, runOutcome, runRows, shardOf, shardPath, shortSha, sortAnalyses, sourceChanges, splitEvalId, splitPatch, stackChanges, studioPath, traceApiPath, transcriptItems, whyChanged };
|