reeval 0.6.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +265 -0
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +30 -307
- package/dist/{agent-DKuFnZPM.js → agent-B72lZ6w5.js} +6 -6
- package/dist/{agent-DKuFnZPM.js.map → agent-B72lZ6w5.js.map} +1 -1
- package/dist/{agent-oirN_8IS.js → agent-Cfj1wD4D.js} +5 -5
- package/dist/{agent-oirN_8IS.js.map → agent-Cfj1wD4D.js.map} +1 -1
- package/dist/{analysis-BV3TCznv.js → analysis-BTNnKh1w.js} +4 -4
- package/dist/{analysis-BV3TCznv.js.map → analysis-BTNnKh1w.js.map} +1 -1
- package/dist/{assemble-DYCDgbTq.js → assemble-BbFVjHeS.js} +125 -14
- package/dist/assemble-BbFVjHeS.js.map +1 -0
- package/dist/assemble-CXiTVyHD.js +2 -0
- package/dist/bridge/index.mjs +66 -7
- package/dist/bridge/package.json +3 -3
- package/dist/bridge/pnpm-lock.yaml +84 -89
- package/dist/capture-BXhBfmsM.js +1206 -0
- package/dist/capture-BXhBfmsM.js.map +1 -0
- package/dist/capture-N7r3yeak.js +2 -0
- package/dist/{cli-BIhzl_v_.js → cli-BMjTKY-D.js} +151 -193
- package/dist/cli-BMjTKY-D.js.map +1 -0
- package/dist/{cli-spec-DUeyuSIQ.js → cli-spec-DSvKsOK4.js} +25 -11
- package/dist/cli-spec-DSvKsOK4.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-CwzU9CiW.js +256 -0
- package/dist/client-CwzU9CiW.js.map +1 -0
- package/dist/{command-B8tcwUJk.js → command-BK4kOgkH.js} +15 -15
- package/dist/{command-B8tcwUJk.js.map → command-BK4kOgkH.js.map} +1 -1
- package/dist/{command-fJpZK205.js → command-BdstSEHu.js} +7 -7
- package/dist/{command-fJpZK205.js.map → command-BdstSEHu.js.map} +1 -1
- package/dist/{command-DuJBqcwu.js → command-BqJtFiPy.js} +153 -36
- package/dist/command-BqJtFiPy.js.map +1 -0
- package/dist/command-CTVGuH8y.js +2 -0
- package/dist/{command-KoOJ0ntD.js → command-JPGNzFsA.js} +9 -9
- package/dist/{command-KoOJ0ntD.js.map → command-JPGNzFsA.js.map} +1 -1
- package/dist/{commands-CHdTfPB6.js → commands-B618jg3C.js} +6 -6
- package/dist/{commands-CHdTfPB6.js.map → commands-B618jg3C.js.map} +1 -1
- package/dist/{commands-DE604ldB.js → commands-Duo2ur3r.js} +1 -1
- package/dist/{config-7-gXOgQf.js → config-B-E0xRkf.js} +4 -4
- package/dist/{config-7-gXOgQf.js.map → config-B-E0xRkf.js.map} +1 -1
- package/dist/db-CnoUy-FZ.js +3 -0
- package/dist/{db-DvkzBhM6.js → db-DFbtGIYy.js} +84 -37
- package/dist/db-DFbtGIYy.js.map +1 -0
- package/dist/{dist-DsZNsoD3.js → dist-CTzThfu_.js} +3 -3
- package/dist/{dist-DsZNsoD3.js.map → dist-CTzThfu_.js.map} +1 -1
- package/dist/{docker-BBOMThRi.js → docker-2qns7hJ2.js} +4 -4
- package/dist/{docker-BBOMThRi.js.map → docker-2qns7hJ2.js.map} +1 -1
- package/dist/{docker-CiNVc7di.js → docker-D6t23fZl.js} +1 -1
- package/dist/{docker-CzFiN2AZ.js → docker-IFRW8K5f.js} +2 -2
- package/dist/{docker-CzFiN2AZ.js.map → docker-IFRW8K5f.js.map} +1 -1
- package/dist/{doctor-C06kYs2r.js → doctor-B6oYTPoR.js} +11 -11
- package/dist/{doctor-C06kYs2r.js.map → doctor-B6oYTPoR.js.map} +1 -1
- package/dist/{egress-De64jpeQ.js → egress-IUKdeayc.js} +17 -10
- package/dist/egress-IUKdeayc.js.map +1 -0
- package/dist/fake-instructions-CUI33I_F.js +32 -0
- package/dist/fake-instructions-CUI33I_F.js.map +1 -0
- package/dist/{files-BqIF7iu4.js → files-B6aVVGrk.js} +3 -3
- package/dist/{files-BqIF7iu4.js.map → files-B6aVVGrk.js.map} +1 -1
- package/dist/files-BpYgF2z0.js +2 -0
- package/dist/{files-Cq99wif-.js → files-CMSUXAEg.js} +16 -5
- package/dist/{files-Cq99wif-.js.map → files-CMSUXAEg.js.map} +1 -1
- package/dist/files-YCCVaTQC.js +2 -0
- package/dist/{git-Dnl-kXgD.js → git-EmdFJ4DY.js} +2 -2
- package/dist/{git-Dnl-kXgD.js.map → git-EmdFJ4DY.js.map} +1 -1
- package/dist/{harness-3QUEWx8G.js → harness-iqWeJQMM.js} +35 -12
- package/dist/harness-iqWeJQMM.js.map +1 -0
- package/dist/harness.d.ts +22 -11
- package/dist/harness.js +1 -1
- package/dist/{header-IE-uIvX0.js → header-_F0EWob3.js} +2 -2
- package/dist/{header-IE-uIvX0.js.map → header-_F0EWob3.js.map} +1 -1
- package/dist/{images-B8Mp_VXZ.js → images-CscddrJU.js} +6 -6
- package/dist/{images-B8Mp_VXZ.js.map → images-CscddrJU.js.map} +1 -1
- package/dist/index.d.ts +13 -8
- package/dist/index.js +99 -24
- package/dist/index.js.map +1 -1
- package/dist/init-DBHyi92L.js +2 -0
- package/dist/{init-B8mw86E6.js → init-NQfIvi-0.js} +4 -4
- package/dist/{init-B8mw86E6.js.map → init-NQfIvi-0.js.map} +1 -1
- package/dist/{invocations-DHC946b-.js → invocations-nPvrlZPj.js} +3 -3
- package/dist/{invocations-DHC946b-.js.map → invocations-nPvrlZPj.js.map} +1 -1
- package/dist/{judge-7WjNdU9z.js → judge-CeYPN8P_.js} +2 -2
- package/dist/{judge-7WjNdU9z.js.map → judge-CeYPN8P_.js.map} +1 -1
- package/dist/judge.d.ts +5 -4
- package/dist/judge.js +4 -3
- package/dist/judge.js.map +1 -1
- package/dist/{live-DZkCtmNP.js → live-S0qODb6o.js} +2 -2
- package/dist/{live-DZkCtmNP.js.map → live-S0qODb6o.js.map} +1 -1
- package/dist/model-BbLGRtOs.js +2 -0
- package/dist/{model-CtrJdFi6.js → model-BndnmcLm.js} +241 -26
- package/dist/model-BndnmcLm.js.map +1 -0
- package/dist/{model-Cp6NoAT3.js → model-Cl507trk.js} +8 -7
- package/dist/model-Cl507trk.js.map +1 -0
- package/dist/{model-Bv8uUZ07.js → model-DEQo3FkZ.js} +6 -6
- package/dist/model-DEQo3FkZ.js.map +1 -0
- package/dist/{model-BeoRxCcJ.js → model-DMOraCcm.js} +1 -1
- package/dist/{opencode-providers-DjlA5fuC.js → opencode-providers-ByD85Q0l.js} +21 -7
- package/dist/{opencode-providers-DjlA5fuC.js.map → opencode-providers-ByD85Q0l.js.map} +1 -1
- package/dist/{options-CWYB613o.js → options-DPUlm4FI.js} +152 -29
- package/dist/options-DPUlm4FI.js.map +1 -0
- package/dist/pairs-DDsjgm5F.js +149 -0
- package/dist/pairs-DDsjgm5F.js.map +1 -0
- package/dist/{plan-CsPE24pp.js → plan-PPc4-6xd.js} +35 -18
- package/dist/plan-PPc4-6xd.js.map +1 -0
- package/dist/{preflight-r0DgibQV.js → preflight-CddycK92.js} +3 -3
- package/dist/{preflight-r0DgibQV.js.map → preflight-CddycK92.js.map} +1 -1
- package/dist/{preflight-DKZjb7ZH.js → preflight-Zumy0adT.js} +1 -1
- package/dist/{prepare-DOkvAa9D.js → prepare-CKdMoC7e.js} +2 -2
- package/dist/{prepare-CTLg3nuk.js → prepare-CsP-057p.js} +4 -4
- package/dist/{prepare-CTLg3nuk.js.map → prepare-CsP-057p.js.map} +1 -1
- package/dist/{provenance--cVNo80L.js → provenance-CsX6PHDR.js} +2 -2
- package/dist/{provenance--cVNo80L.js.map → provenance-CsX6PHDR.js.map} +1 -1
- package/dist/{providers-DNLQzVnc.js → providers-DkwiLKDS.js} +3 -3
- package/dist/{providers-DNLQzVnc.js.map → providers-DkwiLKDS.js.map} +1 -1
- package/dist/{providers-B2cSfewm.js → providers-yXZZcPfx.js} +1 -1
- package/dist/{query-DTtIZaua.js → query-BUcwCYsI.js} +2 -2
- package/dist/{query-DTtIZaua.js.map → query-BUcwCYsI.js.map} +1 -1
- package/dist/query-noGbekGF.js +2 -0
- package/dist/{questions-ZdT3JDyQ.js → questions-BEPj98fZ.js} +27 -118
- package/dist/questions-BEPj98fZ.js.map +1 -0
- package/dist/{record-CbuQH4l1.js → record-BRP9vZrj.js} +21 -1
- package/dist/record-BRP9vZrj.js.map +1 -0
- package/dist/results-BcSHWbgB.js +52 -0
- package/dist/results-BcSHWbgB.js.map +1 -0
- package/dist/resume-DZ9AmfPv.js +406 -0
- package/dist/resume-DZ9AmfPv.js.map +1 -0
- package/dist/{matchers-Bp-OH1vj.js → runtime-Dfb2r-cS.js} +2 -179
- package/dist/runtime-Dfb2r-cS.js.map +1 -0
- package/dist/sandbox.d.ts +1 -1
- package/dist/{server-BFlKK2j6.js → server-Da60oe4b.js} +9 -9
- package/dist/{server-BFlKK2j6.js.map → server-Da60oe4b.js.map} +1 -1
- package/dist/server-DiGR4JxK.js +2 -0
- package/dist/{source-B2ASbTdc.js → source-BVOnL3sn.js} +2 -2
- package/dist/{source-B2ASbTdc.js.map → source-BVOnL3sn.js.map} +1 -1
- package/dist/source-Dqu0jKwx.js +2 -0
- package/dist/src-BHNiP-5A.js +4 -0
- package/dist/{src-CDiGyoDt.js → src-BT4dB5CO.js} +1 -1
- package/dist/src-BT5pwco0.js +3 -0
- package/dist/src-BrMbt1B2.js +3 -0
- package/dist/{src-s-VffoYK.js → src-C1E9HDWO.js} +5 -3
- package/dist/{src-s-VffoYK.js.map → src-C1E9HDWO.js.map} +1 -1
- package/dist/{src-CqsERnCZ.js → src-CxTC5kNI.js} +10 -6
- package/dist/{src-CqsERnCZ.js.map → src-CxTC5kNI.js.map} +1 -1
- package/dist/src-D13vpMh0.js +3 -0
- package/dist/{src-DxFJrMUA.js → src-D95O-Smv.js} +12 -7
- package/dist/{src-DxFJrMUA.js.map → src-D95O-Smv.js.map} +1 -1
- package/dist/{src-CVoCJjEe.js → src-DDNXsD9H.js} +29 -5
- package/dist/{src-CVoCJjEe.js.map → src-DDNXsD9H.js.map} +1 -1
- package/dist/{src-DSE0FWaU.js → src-DVy58-zZ.js} +2 -2
- package/dist/{src-DSE0FWaU.js.map → src-DVy58-zZ.js.map} +1 -1
- package/dist/{src-BHFNhTTQ.js → src-D_cXqmnI.js} +59 -49
- package/dist/{src-BHFNhTTQ.js.map → src-D_cXqmnI.js.map} +1 -1
- package/dist/src-DdRlaiRc.js +3 -0
- package/dist/{src-zOeHvrzR.js → src-TdJuDYqB.js} +13 -13
- package/dist/{src-zOeHvrzR.js.map → src-TdJuDYqB.js.map} +1 -1
- package/dist/src-xczuQbE6.js +3 -0
- package/dist/{state-DBKcUJ2u.js → state-B72h_jEw.js} +14 -14
- package/dist/{state-DBKcUJ2u.js.map → state-B72h_jEw.js.map} +1 -1
- package/dist/state-Bf_Vq_pV.js +2 -0
- package/dist/studio-app/assets/{index-WQPq5K3E.js → index-Bgr-efZm.js} +9 -9
- package/dist/studio-app/index.html +1 -1
- package/dist/{summary-BBABbon9.js → summary-CJdxaRdM.js} +7 -7
- package/dist/{summary-BBABbon9.js.map → summary-CJdxaRdM.js.map} +1 -1
- package/dist/{summary-CahgSiVt.js → summary-k3mhWWXv.js} +8 -4
- package/dist/summary-k3mhWWXv.js.map +1 -0
- package/dist/testing.d.ts +14 -0
- package/dist/testing.js +2 -0
- package/dist/tool-target-BU1JnSE9.js +34 -0
- package/dist/tool-target-BU1JnSE9.js.map +1 -0
- package/dist/{tools-53YHomyk.js → tools-D0gF2QoV.js} +3 -3
- package/dist/{tools-53YHomyk.js.map → tools-D0gF2QoV.js.map} +1 -1
- package/dist/{types-CoAfC-_k.d.ts → types-_LOFdPUf.d.ts} +266 -36
- package/dist/{usage-DsYUqIUM.js → usage-DDF_6tkV.js} +4 -2
- package/dist/{usage-DsYUqIUM.js.map → usage-DDF_6tkV.js.map} +1 -1
- package/dist/view/index.d.ts +19 -6
- package/dist/view/index.js +6 -6
- package/dist/{views-Drl2OXuj.js → views-DZ1YDqey.js} +4 -4
- package/dist/{views-Drl2OXuj.js.map → views-DZ1YDqey.js.map} +1 -1
- package/dist/views-Ds0GXWtX.js +3 -0
- package/dist/{worker-entry-mCctDzig.js → worker-entry-DXeyWY-m.js} +73 -2
- package/dist/worker-entry-DXeyWY-m.js.map +1 -0
- package/dist/{workspace-BT4jd04k.js → workspace-DpXdrslo.js} +3 -3
- package/dist/{workspace-BT4jd04k.js.map → workspace-DpXdrslo.js.map} +1 -1
- package/dist/{workspace-U6M1gKzo.js → workspace-DzNL-3iO.js} +1 -1
- package/package.json +14 -8
- package/dist/assemble-DYCDgbTq.js.map +0 -1
- package/dist/assemble-DykFydSU.js +0 -2
- package/dist/capture-CisAmy-G.js +0 -587
- package/dist/capture-CisAmy-G.js.map +0 -1
- package/dist/capture-DquyqAHC.js +0 -2
- package/dist/cli-BIhzl_v_.js.map +0 -1
- package/dist/cli-spec-DUeyuSIQ.js.map +0 -1
- package/dist/command-CwuwuKJD.js +0 -2
- package/dist/command-DuJBqcwu.js.map +0 -1
- package/dist/db-BVE1kZ2w.js +0 -2
- package/dist/db-DvkzBhM6.js.map +0 -1
- package/dist/egress-De64jpeQ.js.map +0 -1
- package/dist/files-Cs8tbYQY.js +0 -2
- package/dist/files-wUtoO5aA.js +0 -2
- package/dist/harness-3QUEWx8G.js.map +0 -1
- package/dist/init-DDEpamaf.js +0 -2
- package/dist/matchers-Bp-OH1vj.js.map +0 -1
- package/dist/model-Bv8uUZ07.js.map +0 -1
- package/dist/model-C_B5JrhL.js +0 -2
- package/dist/model-Cp6NoAT3.js.map +0 -1
- package/dist/model-CtrJdFi6.js.map +0 -1
- package/dist/options-CWYB613o.js.map +0 -1
- package/dist/plan-CsPE24pp.js.map +0 -1
- package/dist/query-BWc_5Lhe.js +0 -2
- package/dist/questions-ZdT3JDyQ.js.map +0 -1
- package/dist/record-CbuQH4l1.js.map +0 -1
- package/dist/results-CrFs7_In.js +0 -37
- package/dist/results-CrFs7_In.js.map +0 -1
- package/dist/server-HRUHVcHc.js +0 -2
- package/dist/source-Co7h89WK.js +0 -2
- package/dist/src-BE19tji4.js +0 -3
- package/dist/src-Bo8GFUXg.js +0 -3
- package/dist/src-BoziVhKa.js +0 -3
- package/dist/src-CFxHPwQA.js +0 -3
- package/dist/src-CPkDaNxp.js +0 -3
- package/dist/src-D-go5-qz.js +0 -4
- package/dist/state-BaHjYXr1.js +0 -2
- package/dist/stats-BNAHlNMD.js +0 -114
- package/dist/stats-BNAHlNMD.js.map +0 -1
- package/dist/summary-CahgSiVt.js.map +0 -1
- package/dist/views-BhGinAz_.js +0 -3
- package/dist/worker-entry-mCctDzig.js.map +0 -1
package/dist/judge.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { E as ConfigJudgeOptions, G as FakeJudgeModel, K as FakeNoteInput, Mt as Verdict, U as FakeAnalysisModel, Y as FakeVote, at as JudgeCallOptions, ct as JudgeOptions, it as JudgeArtifactRef, lt as JudgeState, n as AnalysisConfig, o as BlindPair, ot as JudgeFailure, st as JudgeModel } from "./types-_LOFdPUf.js";
|
|
2
2
|
//#region src/judge/index.d.ts
|
|
3
3
|
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
4
4
|
export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.13.0";
|
|
@@ -15,11 +15,12 @@ export declare const fakeJudge: ({ score }?: {
|
|
|
15
15
|
/**
|
|
16
16
|
* An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
|
|
17
17
|
* model's structured answer, and each compare question's pair gets `vote` (a fixed `'A'`, `'B'` or
|
|
18
|
-
* `'tie'`, a function of the blinded pair, or `{ error }`), through the same
|
|
19
|
-
* validation as a real one. With `analysis.agent`, each note calls the same tools a real
|
|
18
|
+
* `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), through the same
|
|
19
|
+
* parsing and validation as a real one. With `analysis.agent`, each note calls the same tools a real
|
|
20
|
+
* model
|
|
20
21
|
* would, then answers with `note` (or `note(input)`), or throws `note.error`.
|
|
21
22
|
*/
|
|
22
23
|
export declare const fakeAnalysis: ({ output, note, vote }: Pick<FakeAnalysisModel, 'output' | 'note' | 'vote'>) => FakeAnalysisModel;
|
|
23
24
|
//#endregion
|
|
24
|
-
export type { AnalysisConfig, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeJudgeModel, FakeNoteInput, FakeVote, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, Verdict };
|
|
25
|
+
export type { AnalysisConfig, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeJudgeModel, FakeNoteInput, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
|
|
25
26
|
//# sourceMappingURL=judge.d.ts.map
|
package/dist/judge.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { r as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-
|
|
1
|
+
import { r as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-CeYPN8P_.js";
|
|
2
2
|
//#region src/judge.ts
|
|
3
3
|
/**
|
|
4
4
|
* A judge that returns `score` without calling a model, for tests and dry runs. `score` can be a
|
|
@@ -11,8 +11,9 @@ const fakeJudge = ({ score = 1 } = {}) => ({
|
|
|
11
11
|
/**
|
|
12
12
|
* An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
|
|
13
13
|
* model's structured answer, and each compare question's pair gets `vote` (a fixed `'A'`, `'B'` or
|
|
14
|
-
* `'tie'`, a function of the blinded pair, or `{ error }`), through the same
|
|
15
|
-
* validation as a real one. With `analysis.agent`, each note calls the same tools a real
|
|
14
|
+
* `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), through the same
|
|
15
|
+
* parsing and validation as a real one. With `analysis.agent`, each note calls the same tools a real
|
|
16
|
+
* model
|
|
16
17
|
* would, then answers with `note` (or `note(input)`), or throws `note.error`.
|
|
17
18
|
*/
|
|
18
19
|
const fakeAnalysis = ({ output, note, vote }) => ({
|
package/dist/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF
|
|
1
|
+
{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF;;;;;;;;;AAUA,MAAa,gBAAgB,EAC3B,QACA,MACA,YAC6E;CAC7E,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;AACvC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { n as fillDefaults } from "./record-
|
|
1
|
+
import { n as fillDefaults } from "./record-BRP9vZrj.js";
|
|
2
2
|
//#region src/view/live.ts
|
|
3
3
|
/** The progress file's name inside the run's staging directory. */
|
|
4
4
|
const LIVE_FILE = "live.json";
|
|
@@ -95,4 +95,4 @@ function readableLiveProgress(value) {
|
|
|
95
95
|
//#endregion
|
|
96
96
|
export { liveApiPath as a, readableLiveProgress as c, isFinished as i, LIVE_FILE as n, liveCounts as o, LIVE_SCHEMA_VERSION as r, liveTranscriptPath as s, FINISHED as t };
|
|
97
97
|
|
|
98
|
-
//# sourceMappingURL=live-
|
|
98
|
+
//# sourceMappingURL=live-S0qODb6o.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"live-
|
|
1
|
+
{"version":3,"file":"live-S0qODb6o.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;AAOA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAsDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAA+B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEnG,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC;EAAC;EAAK;EAAQ;EAAM;EAAM;CAAI,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CACnE,IAAI,CAAC,SAAS,SAAS,MAAoB,GAAG,OAAO;CACrD,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACA;EACG;EACF;EACA;EACA;EACN;EACQ;EACR,WAAY,aAAoC;EAChD,YAAa,cAAqC;CACpD;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC;EAAC;EAAO;EAAW;EAAM;CAAW,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3E,IAAI,CAAC,OAAO,SAAS,KAAkB,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAC7E,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,KAAK,MAAM,MAAM,MAAM,IAAI,GAAG,OAAO;CACzC,MAAM,OAAO;CACb,OAAO;EACL,eAAA;EACO;EACI;EACX;EACM;EACO;EACN;EACP,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;CACZ;AACF"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-BmR-7tx3.js";
|
|
2
2
|
import { n as clip } from "./text-gEcKjAyp.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { i as totalTokens, n as promptTokens, r as sumUsage } from "./usage-DDF_6tkV.js";
|
|
4
|
+
import { c as readableTranscriptLine } from "./record-BRP9vZrj.js";
|
|
5
|
+
import { a as sampleReasons } from "./pairs-DDsjgm5F.js";
|
|
6
6
|
//#region src/view/compare.ts
|
|
7
7
|
const STACK_FIELDS = [
|
|
8
8
|
{
|
|
@@ -243,13 +243,127 @@ function compare(base, head) {
|
|
|
243
243
|
});
|
|
244
244
|
}
|
|
245
245
|
//#endregion
|
|
246
|
+
//#region src/stats.ts
|
|
247
|
+
/** Two-sided 95%. */
|
|
248
|
+
const Z = 1.959964;
|
|
249
|
+
/**
|
|
250
|
+
* The Wilson score interval for `pass` successes out of `n`. Unlike the normal approximation it
|
|
251
|
+
* stays inside [0, 1] and is sensible at 0/n and n/n. With no attempts it's [0, 1].
|
|
252
|
+
*/
|
|
253
|
+
function wilson(pass, n) {
|
|
254
|
+
if (n === 0) return {
|
|
255
|
+
low: 0,
|
|
256
|
+
high: 1
|
|
257
|
+
};
|
|
258
|
+
const p = pass / n;
|
|
259
|
+
const z2 = Z * Z;
|
|
260
|
+
const denominator = 1 + z2 / n;
|
|
261
|
+
const center = (p + z2 / (2 * n)) / denominator;
|
|
262
|
+
const half = Z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n)) / denominator;
|
|
263
|
+
return {
|
|
264
|
+
low: Math.max(0, center - half),
|
|
265
|
+
high: Math.min(1, center + half)
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* The difference of two pass rates (variant minus control), with Newcombe's hybrid score
|
|
270
|
+
* interval: each rate's Wilson interval combined, so it inherits Wilson's behaviour at 0 and 1.
|
|
271
|
+
*/
|
|
272
|
+
function liftInterval(variant, control) {
|
|
273
|
+
const p1 = variant.n ? variant.pass / variant.n : 0;
|
|
274
|
+
const p2 = control.n ? control.pass / control.n : 0;
|
|
275
|
+
const a = wilson(variant.pass, variant.n);
|
|
276
|
+
const b = wilson(control.pass, control.n);
|
|
277
|
+
const points = p1 - p2;
|
|
278
|
+
return {
|
|
279
|
+
points,
|
|
280
|
+
low: points - Math.sqrt((p1 - a.low) ** 2 + (b.high - p2) ** 2),
|
|
281
|
+
high: points + Math.sqrt((a.high - p1) ** 2 + (p2 - b.low) ** 2)
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
/** Rounded to three places, as the record stores rates and points. */
|
|
285
|
+
const round3 = (x) => Math.round(x * 1e3) / 1e3;
|
|
286
|
+
/** Linear-interpolated quantile (`q` from 0 to 1) of unsorted numbers; null for none. */
|
|
287
|
+
function quantile(values, q) {
|
|
288
|
+
if (values.length === 0) return null;
|
|
289
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
290
|
+
const at = (sorted.length - 1) * q;
|
|
291
|
+
const lo = Math.floor(at);
|
|
292
|
+
const hi = Math.ceil(at);
|
|
293
|
+
const a = sorted[lo];
|
|
294
|
+
return a + (sorted[hi] - a) * (at - lo);
|
|
295
|
+
}
|
|
296
|
+
const median = (values) => quantile(values, .5);
|
|
297
|
+
/** Above this many ways to split the pooled values, the normal approximation stands in. */
|
|
298
|
+
const EXACT_LIMIT = 2e5;
|
|
299
|
+
const choose = (n, k) => {
|
|
300
|
+
let r = 1;
|
|
301
|
+
for (let i = 1; i <= k; i++) r = r * (n - k + i) / i;
|
|
302
|
+
return Math.round(r);
|
|
303
|
+
};
|
|
304
|
+
/**
|
|
305
|
+
* Two-sided Mann–Whitney (Wilcoxon rank-sum) p-value for `xs` against `ys`: exact, by every split
|
|
306
|
+
* of the pooled values, with mid-ranks for ties; the normal approximation with a tie correction
|
|
307
|
+
* above `EXACT_LIMIT` splits. Null when either side is empty.
|
|
308
|
+
*/
|
|
309
|
+
function mannWhitney(xs, ys) {
|
|
310
|
+
const n1 = xs.length;
|
|
311
|
+
const n2 = ys.length;
|
|
312
|
+
if (n1 === 0 || n2 === 0) return null;
|
|
313
|
+
const pooled = [...xs, ...ys].map((v, i) => ({
|
|
314
|
+
v,
|
|
315
|
+
i
|
|
316
|
+
})).sort((a, b) => a.v - b.v);
|
|
317
|
+
const ranks = new Array(pooled.length);
|
|
318
|
+
for (let i = 0; i < pooled.length;) {
|
|
319
|
+
let j = i;
|
|
320
|
+
while (j + 1 < pooled.length && pooled[j + 1]?.v === pooled[i]?.v) j++;
|
|
321
|
+
for (let k = i; k <= j; k++) ranks[pooled[k]?.i] = (i + j) / 2 + 1;
|
|
322
|
+
i = j + 1;
|
|
323
|
+
}
|
|
324
|
+
const n = n1 + n2;
|
|
325
|
+
const expected = n1 * (n + 1) / 2;
|
|
326
|
+
const observed = Math.abs(ranks.slice(0, n1).reduce((a, b) => a + b, 0) - expected);
|
|
327
|
+
if (choose(n, n1) > EXACT_LIMIT) {
|
|
328
|
+
const counts = /* @__PURE__ */ new Map();
|
|
329
|
+
for (const { v } of pooled) counts.set(v, (counts.get(v) ?? 0) + 1);
|
|
330
|
+
const ties = [...counts.values()].reduce((a, t) => a + t ** 3 - t, 0);
|
|
331
|
+
const variance = n1 * n2 / 12 * (n + 1 - ties / (n * (n - 1)));
|
|
332
|
+
if (variance <= 0) return 1;
|
|
333
|
+
const z = Math.max(0, observed - .5) / Math.sqrt(variance);
|
|
334
|
+
return Math.min(1, 2 * (1 - normalCdf(z)));
|
|
335
|
+
}
|
|
336
|
+
let extreme = 0;
|
|
337
|
+
let total = 0;
|
|
338
|
+
const pick = (start, left, sum) => {
|
|
339
|
+
if (left === 0) {
|
|
340
|
+
total++;
|
|
341
|
+
if (Math.abs(sum - expected) >= observed - 1e-9) extreme++;
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
for (let i = start; i <= n - left; i++) pick(i + 1, left - 1, sum + ranks[i]);
|
|
345
|
+
};
|
|
346
|
+
pick(0, n1, 0);
|
|
347
|
+
return extreme / total;
|
|
348
|
+
}
|
|
349
|
+
/** The smallest two-sided p an exact rank test can reach with these group sizes. */
|
|
350
|
+
const minimumP = (n1, n2) => n1 === 0 || n2 === 0 ? 1 : Math.min(1, 2 / choose(n1 + n2, n1));
|
|
351
|
+
function normalCdf(z) {
|
|
352
|
+
const t = 1 / (1 + .3275911 * Math.abs(z) / Math.SQRT2);
|
|
353
|
+
const erf = 1 - t * (.254829592 + t * (-.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))) * Math.exp(-(z * z) / 2);
|
|
354
|
+
return z >= 0 ? (1 + erf) / 2 : (1 - erf) / 2;
|
|
355
|
+
}
|
|
356
|
+
//#endregion
|
|
246
357
|
//#region src/view/effort.ts
|
|
247
358
|
/** Below this two-sided p, an effort difference is an effect. */
|
|
248
359
|
const EFFORT_ALPHA = .05;
|
|
360
|
+
/** Some counted attempts reported no usage. Not a verdict; never printed as one. */
|
|
361
|
+
const TOKENS_UNREPORTED = "some cells did not report usage";
|
|
249
362
|
const effortSample = (a) => ({
|
|
250
363
|
tools: a.activity.tools,
|
|
251
364
|
agentMs: a.timings.agentMs,
|
|
252
|
-
fileEdits: a.activity.fileEdits
|
|
365
|
+
fileEdits: a.activity.fileEdits,
|
|
366
|
+
...a.usage ? { tokens: totalTokens(a.usage) } : {}
|
|
253
367
|
});
|
|
254
368
|
const counted$1 = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
255
369
|
const spread = (values) => ({
|
|
@@ -257,13 +371,17 @@ const spread = (values) => ({
|
|
|
257
371
|
q1: quantile(values, .25) ?? 0,
|
|
258
372
|
q3: quantile(values, .75) ?? 0
|
|
259
373
|
});
|
|
374
|
+
/** Token totals from the attempts that reported usage; empty when none did. */
|
|
375
|
+
const tokenValues = (samples) => samples.flatMap((s) => s.tokens === void 0 ? [] : [s.tokens]);
|
|
260
376
|
function effortOf(samples) {
|
|
261
377
|
if (samples.length === 0) return null;
|
|
378
|
+
const tokens = tokenValues(samples);
|
|
262
379
|
return {
|
|
263
380
|
n: samples.length,
|
|
264
381
|
tools: spread(samples.map((s) => s.tools)),
|
|
265
382
|
agentMs: spread(samples.map((s) => s.agentMs)),
|
|
266
|
-
fileEdits: median(samples.map((s) => s.fileEdits)) ?? 0
|
|
383
|
+
fileEdits: median(samples.map((s) => s.fileEdits)) ?? 0,
|
|
384
|
+
...tokens.length ? { tokens: spread(tokens) } : {}
|
|
267
385
|
};
|
|
268
386
|
}
|
|
269
387
|
/** The verdict of an exact rank test of `xs` (variant) against `ys` (control). */
|
|
@@ -289,6 +407,11 @@ function contrastEffort(variant, control) {
|
|
|
289
407
|
if (!v || !c) return null;
|
|
290
408
|
const tools = effortVerdict(variant.map((s) => s.tools), control.map((s) => s.tools));
|
|
291
409
|
const time = effortVerdict(variant.map((s) => s.agentMs), control.map((s) => s.agentMs));
|
|
410
|
+
const vTok = tokenValues(variant);
|
|
411
|
+
const cTok = tokenValues(control);
|
|
412
|
+
const tokens = vTok.length && cTok.length ? effortVerdict(vTok, cTok) : null;
|
|
413
|
+
const reported = vTok.length + cTok.length;
|
|
414
|
+
const unreported = reported > 0 && reported < variant.length + control.length;
|
|
292
415
|
const vt = variant.map((s) => s.tools);
|
|
293
416
|
const ct = control.map((s) => s.tools);
|
|
294
417
|
return {
|
|
@@ -301,7 +424,11 @@ function contrastEffort(variant, control) {
|
|
|
301
424
|
agentMsP: time.p,
|
|
302
425
|
separated: Math.max(...vt) < Math.min(...ct) || Math.min(...vt) > Math.max(...ct),
|
|
303
426
|
verdict: tools.verdict,
|
|
304
|
-
timeVerdict: time.verdict
|
|
427
|
+
timeVerdict: time.verdict,
|
|
428
|
+
tokensRatio: tokens && v.tokens && c.tokens ? ratio(v.tokens.median, c.tokens.median) : null,
|
|
429
|
+
tokensP: tokens?.p ?? null,
|
|
430
|
+
tokensVerdict: tokens?.verdict ?? null,
|
|
431
|
+
...unreported ? { tokensNote: TOKENS_UNREPORTED } : {}
|
|
305
432
|
};
|
|
306
433
|
}
|
|
307
434
|
/** The counted attempts of one cell of an eval, as effort samples. */
|
|
@@ -329,20 +456,35 @@ const VERDICT_TEXT = {
|
|
|
329
456
|
noise: () => "within noise",
|
|
330
457
|
insufficient: () => "too few attempts to test"
|
|
331
458
|
};
|
|
459
|
+
const toolsVerdictText = (c) => c.verdict === "fewer" || c.verdict === "more" ? `${VERDICT_TEXT[c.verdict]("tool calls")}${formatP(c.toolsP)}` : VERDICT_TEXT[c.verdict]("");
|
|
460
|
+
/** Token verdict after the tools one; empty when omitted or already covered as insufficient. */
|
|
461
|
+
const tokensVerdictText = (c) => {
|
|
462
|
+
if (c.tokensVerdict === null) return "";
|
|
463
|
+
if (c.verdict === "insufficient" && c.tokensVerdict === "insufficient") return "";
|
|
464
|
+
if (c.tokensVerdict === "fewer" || c.tokensVerdict === "more") return `; ${VERDICT_TEXT[c.tokensVerdict]("tokens")}${formatP(c.tokensP)}`;
|
|
465
|
+
if (c.tokensVerdict === "noise") return "; tokens within noise";
|
|
466
|
+
return `; ${VERDICT_TEXT[c.tokensVerdict]("")}`;
|
|
467
|
+
};
|
|
332
468
|
/**
|
|
333
469
|
* `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
|
|
334
|
-
* against its control; with enough attempts, `fewer tool calls (p = 0.029)`.
|
|
470
|
+
* against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
|
|
471
|
+
* when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
|
|
335
472
|
*/
|
|
336
473
|
function effortLabel(c) {
|
|
337
|
-
|
|
474
|
+
const parts = [`tools ${formatCount(c.variant.tools.median)} vs ${formatCount(c.control.tools.median)} (${formatRatio(c.toolsRatio)})`, `agent ${formatSeconds(c.variant.agentMs.median)} vs ${formatSeconds(c.control.agentMs.median)} (${formatRatio(c.agentMsRatio)})`];
|
|
475
|
+
if (c.tokensVerdict !== null && c.variant.tokens && c.control.tokens) parts.push(`tokens ${formatCount(c.variant.tokens.median)} vs ${formatCount(c.control.tokens.median)} (${formatRatio(c.tokensRatio)})`);
|
|
476
|
+
return `${parts.join(", ")}: ${toolsVerdictText(c)}${tokensVerdictText(c)}`;
|
|
338
477
|
}
|
|
339
|
-
/** One cell's effort on its own: `tools 43 · agent 46s
|
|
340
|
-
const effortSummary = (e) => `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)}`;
|
|
478
|
+
/** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
|
|
479
|
+
const effortSummary = (e) => e.tokens ? `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)} · tokens ${formatCount(e.tokens.median)}` : `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)}`;
|
|
341
480
|
/** The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`. */
|
|
342
481
|
function effortChangeLabel(base, head) {
|
|
343
482
|
const t = ratio(head.tools.median, base.tools.median);
|
|
344
483
|
const a = ratio(head.agentMs.median, base.agentMs.median);
|
|
345
|
-
|
|
484
|
+
const core = `tools ${formatCount(base.tools.median)} → ${formatCount(head.tools.median)} (${formatRatio(t)}) · agent ${formatSeconds(base.agentMs.median)} → ${formatSeconds(head.agentMs.median)} (${formatRatio(a)})`;
|
|
485
|
+
if (!base.tokens || !head.tokens) return core;
|
|
486
|
+
const tok = ratio(head.tokens.median, base.tokens.median);
|
|
487
|
+
return `${core} · tokens ${formatCount(base.tokens.median)} → ${formatCount(head.tokens.median)} (${formatRatio(tok)})`;
|
|
346
488
|
}
|
|
347
489
|
//#endregion
|
|
348
490
|
//#region src/view/judgements.ts
|
|
@@ -354,10 +496,69 @@ const LIFT_MIN_COUNTED = 2;
|
|
|
354
496
|
/** Show each key until this many; then the minimum score plus a count. */
|
|
355
497
|
const COLLAPSE_AFTER = 2;
|
|
356
498
|
const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
499
|
+
/** Short names stay at this many characters; a collision takes the distinctive tail instead. */
|
|
500
|
+
const LABEL_MAX = 20;
|
|
501
|
+
const judgeText = (question) => question.replace(/\?+$/, "").trim();
|
|
502
|
+
/** Last `max` characters, cut at a word boundary where there is one, marked with a leading `…`. */
|
|
503
|
+
function clipTail(text, max) {
|
|
504
|
+
if (text.length <= max) return text;
|
|
505
|
+
const cut = text.slice(-(max - 1));
|
|
506
|
+
const space = cut.search(/\s/);
|
|
507
|
+
return `…${(space >= 0 && space < max / 2 ? cut.slice(space + 1) : cut).trimStart()}`;
|
|
508
|
+
}
|
|
357
509
|
/** Short name for a judge question, e.g. `login` from `login?`. */
|
|
358
510
|
function judgeLabel(question) {
|
|
359
|
-
|
|
360
|
-
|
|
511
|
+
return clip(judgeText(question), LABEL_MAX);
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* One short label per key. Shared prefixes take the distinctive tail so
|
|
515
|
+
* `Does this change…` does not collapse several checks; a tail that still
|
|
516
|
+
* collides falls back to the key.
|
|
517
|
+
*/
|
|
518
|
+
function uniqueJudgeLabels(items) {
|
|
519
|
+
const first = /* @__PURE__ */ new Map();
|
|
520
|
+
for (const item of items) if (!first.has(item.key)) first.set(item.key, item);
|
|
521
|
+
const labels = /* @__PURE__ */ new Map();
|
|
522
|
+
const used = /* @__PURE__ */ new Map();
|
|
523
|
+
const setLabel = (key, label) => {
|
|
524
|
+
const prev = labels.get(key);
|
|
525
|
+
if (prev !== void 0) {
|
|
526
|
+
const group = used.get(prev);
|
|
527
|
+
if (group) {
|
|
528
|
+
const next = group.filter((k) => k !== key);
|
|
529
|
+
if (next.length) used.set(prev, next);
|
|
530
|
+
else used.delete(prev);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
labels.set(key, label);
|
|
534
|
+
const group = used.get(label) ?? [];
|
|
535
|
+
group.push(key);
|
|
536
|
+
used.set(label, group);
|
|
537
|
+
};
|
|
538
|
+
for (const row of first.values()) setLabel(row.key, judgeLabel(row.question));
|
|
539
|
+
for (const [label, keys] of [...used]) {
|
|
540
|
+
if (keys.length < 2) continue;
|
|
541
|
+
const tails = /* @__PURE__ */ new Map();
|
|
542
|
+
for (const key of keys) {
|
|
543
|
+
const row = first.get(key);
|
|
544
|
+
if (!row) continue;
|
|
545
|
+
const tail = clipTail(judgeText(row.question), LABEL_MAX);
|
|
546
|
+
const group = tails.get(tail) ?? [];
|
|
547
|
+
group.push(key);
|
|
548
|
+
tails.set(tail, group);
|
|
549
|
+
}
|
|
550
|
+
for (const [tail, tkeys] of tails) {
|
|
551
|
+
const [only] = tkeys;
|
|
552
|
+
const taken = (used.get(tail) ?? []).some((k) => !tkeys.includes(k));
|
|
553
|
+
if (only && tkeys.length === 1 && tail !== label && !taken) setLabel(only, tail);
|
|
554
|
+
else for (const key of tkeys) setLabel(key, key);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
for (const [, keys] of [...used]) {
|
|
558
|
+
if (keys.length < 2) continue;
|
|
559
|
+
for (const key of keys) setLabel(key, key);
|
|
560
|
+
}
|
|
561
|
+
return labels;
|
|
361
562
|
}
|
|
362
563
|
function keyTallies(e, cellKey) {
|
|
363
564
|
const byKey = /* @__PURE__ */ new Map();
|
|
@@ -365,7 +566,7 @@ function keyTallies(e, cellKey) {
|
|
|
365
566
|
if (a.cell !== cellKey || !counted(a)) continue;
|
|
366
567
|
for (const j of a.judgements) {
|
|
367
568
|
const row = byKey.get(j.key) ?? {
|
|
368
|
-
label:
|
|
569
|
+
label: "",
|
|
369
570
|
question: j.question,
|
|
370
571
|
pass: 0,
|
|
371
572
|
counted: 0
|
|
@@ -375,6 +576,11 @@ function keyTallies(e, cellKey) {
|
|
|
375
576
|
byKey.set(j.key, row);
|
|
376
577
|
}
|
|
377
578
|
}
|
|
579
|
+
const labels = uniqueJudgeLabels([...byKey].map(([key, row]) => ({
|
|
580
|
+
key,
|
|
581
|
+
question: row.question
|
|
582
|
+
})));
|
|
583
|
+
for (const [key, row] of byKey) row.label = labels.get(key) ?? judgeLabel(row.question);
|
|
378
584
|
return byKey;
|
|
379
585
|
}
|
|
380
586
|
function pairedControl(e, cellKey) {
|
|
@@ -425,11 +631,12 @@ function judgeRatesLabel(rates) {
|
|
|
425
631
|
}
|
|
426
632
|
/** Each key, or the lowest score plus how many judges there were. */
|
|
427
633
|
function collapsedJudges(judgements) {
|
|
634
|
+
const labels = uniqueJudgeLabels(judgements);
|
|
428
635
|
const items = judgements.map((j) => ({
|
|
429
636
|
verdict: j.verdict,
|
|
430
637
|
score: j.score,
|
|
431
638
|
threshold: j.threshold,
|
|
432
|
-
label: judgeLabel(j.question),
|
|
639
|
+
label: labels.get(j.key) ?? judgeLabel(j.question),
|
|
433
640
|
key: j.key
|
|
434
641
|
}));
|
|
435
642
|
if (items.length <= COLLAPSE_AFTER) return {
|
|
@@ -526,7 +733,12 @@ const formatPercent = (rate) => rate === null ? "–" : `${Math.round(rate * 100
|
|
|
526
733
|
* the configured model, then the variant the harness says it resolved.
|
|
527
734
|
*/
|
|
528
735
|
function harnessLabel(h) {
|
|
529
|
-
const parts = [
|
|
736
|
+
const parts = [
|
|
737
|
+
...h.model ? [`model ${h.model}`] : [],
|
|
738
|
+
...h.effort ? [`effort ${h.effort}`] : [],
|
|
739
|
+
...h.resolvedModels.length ? [`served ${h.resolvedModels.join(", ")}`] : [],
|
|
740
|
+
...h.resolvedEffort.length ? [`served effort ${h.resolvedEffort.join(", ")}`] : []
|
|
741
|
+
];
|
|
530
742
|
return parts.length ? `${h.id} (${parts.join(", ")})` : h.id;
|
|
531
743
|
}
|
|
532
744
|
/**
|
|
@@ -581,18 +793,20 @@ function andList(items) {
|
|
|
581
793
|
/** Distinct judge labels that failed on a counted attempt, in first-seen order. */
|
|
582
794
|
function failedJudgeLabels(record) {
|
|
583
795
|
const seen = /* @__PURE__ */ new Set();
|
|
584
|
-
const
|
|
796
|
+
const items = [];
|
|
585
797
|
for (const e of record.evals) for (const a of e.attempts) {
|
|
586
798
|
if (a.outcome !== "pass" && a.outcome !== "fail") continue;
|
|
587
799
|
for (const j of a.judgements) {
|
|
588
|
-
if (j.verdict !== "fail") continue;
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
800
|
+
if (j.verdict !== "fail" || seen.has(j.key)) continue;
|
|
801
|
+
seen.add(j.key);
|
|
802
|
+
items.push({
|
|
803
|
+
key: j.key,
|
|
804
|
+
question: j.question
|
|
805
|
+
});
|
|
593
806
|
}
|
|
594
807
|
}
|
|
595
|
-
|
|
808
|
+
const labels = uniqueJudgeLabels(items);
|
|
809
|
+
return items.map((item) => labels.get(item.key) ?? item.key);
|
|
596
810
|
}
|
|
597
811
|
/**
|
|
598
812
|
* One sentence for the studio run header: counted pass/fail, which judges failed, and infra.
|
|
@@ -767,7 +981,8 @@ function answerRows(e) {
|
|
|
767
981
|
within: rates("within"),
|
|
768
982
|
across: rates("across"),
|
|
769
983
|
error: a.error ?? null,
|
|
770
|
-
noPairs: a.votes.length === 0 && a.error === void 0
|
|
984
|
+
noPairs: a.votes.length === 0 && a.error === void 0,
|
|
985
|
+
reasons: sampleReasons(a.votes)
|
|
771
986
|
};
|
|
772
987
|
});
|
|
773
988
|
}
|
|
@@ -908,6 +1123,6 @@ const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId)
|
|
|
908
1123
|
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
909
1124
|
const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
|
|
910
1125
|
//#endregion
|
|
911
|
-
export {
|
|
1126
|
+
export { formatRatio as $, transcriptItems as A, TOKENS_UNREPORTED as B, runLead as C, shortSha as D, runtimeLabel as E, collapsedJudges as F, contrastEffort as G, cellContrast as H, judgeLabel as I, effortOf as J, effortChangeLabel as K, judgeRatesLabel as L, cellJudgeRates as M, cellUsage as N, splitEvalId as O, collapsedJudgeLabel as P, formatCount as Q, uniqueJudgeLabels as R, parseTranscript as S, runRows as T, cellEffort as U, VERDICT_TEXT as V, cellSamples as W, effortSummary as X, effortSample as Y, effortVerdict as Z, hasUngatedCells as _, cellRows as a, cellsChanged as at, isUngatedMiss as b, evalHistory as c, stackChanges as ct, findAttempt as d, formatSeconds as et, findEval as f, harnessLabel as g, formatUsage as h, buildIndex as i, wilson as it, LIFT_MIN_COUNTED as j, splitPatch as k, evalTree as l, whyChanged as lt, formatPercent as m, activityLabel as n, liftInterval as nt, compareView as o, compare as ot, formatDuration as p, effortLabel as q, answerRows as r, round3 as rt, defaultAttempt as s, sourceChanges as st, COMPARE_ORDER as t, tokenValues as tt, failedJudgeLabels as u, indexFiles as v, runOutcome as w, languageLabel as x, isMatrix as y, EFFORT_ALPHA as z };
|
|
912
1127
|
|
|
913
|
-
//# sourceMappingURL=model-
|
|
1128
|
+
//# sourceMappingURL=model-BndnmcLm.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-BndnmcLm.js","names":["counted"],"sources":["../src/view/compare.ts","../src/stats.ts","../src/view/effort.ts","../src/view/judgements.ts","../src/view/model.ts"],"mappings":";;;;;;AAoCA,MAAM,eAAsF;CAC1F;EAAE,OAAO;EAAY,OAAO;EAAa,SAAS;CAAM;CACxD;EAAE,OAAO;EAAa,OAAO;EAAa,SAAS;CAAM;CACzD;EAAE,OAAO;EAAa,OAAO;EAAa,SAAS;CAAM;CACzD;EAAE,OAAO;EAAgB,OAAO;EAAgB,SAAS;CAAM;CAC/D;EAAE,OAAO;EAAc,OAAO;EAAe,SAAS;CAAM;CAC5D;EAAE,OAAO;EAAa,OAAO;EAAmB,SAAS;CAAM;CAC/D;EAAE,OAAO;EAAe,OAAO;EAAsB,SAAS;CAAK;CACnE;EAAE,OAAO;EAAgB,OAAO;EAAiB,SAAS;CAAM;CAChE;EAAE,OAAO;EAAoB,OAAO;EAAqB,SAAS;CAAK;AACzE;AAEA,MAAM,QAAQ,UACZ,MAAM,QAAQ,KAAK,IACf,MAAM,KAAK,IAAI,KAAK,SACpB,UAAU,OACR,SACA,OAAO,KAAK,CAAC,CACV,QAAQ,YAAY,EAAE,CAAC,CACvB,MAAM,GAAG,EAAE;;AAatB,MAAM,WAAiF;CACrF,WAAW,MAAM,EAAE,iBAAiB;CACpC,YAAY,MAAM,EAAE,iBAAiB;CACrC,YAAY,MAAM,EAAE;CACpB,eAAe,MACb,GAAG,EAAE,YAAY,IAAI,EAAE,qBAAqB,GAAA,CAAI,QAAQ,YAAY,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;AACvF;;AAGA,SAAS,YAAY,MAAmB,MAAyD;CAC/F,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,OAAO,MAAM,EAAE,OAAO,OAAO,MAAM,MAAM,QAAQ,EAAE,KAAK,CAAC,CAAC,GAAG,OAAO;CACtF,IAAI,CAAC,KAAK,SAAS,CAAC,KAAK,OAAO,OAAO;CACvC,MAAM,SAAS,IAAI,IAAI,KAAK,MAAM,KAAK,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CAC7F,OAAO,KAAK,MAAM,SAAS,MACzB,EAAE,MAAM,SAAS,MAAqC;EACpD,MAAM,IAAI,OAAO,IAAI,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG;EACrC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;CACzB,CAAC,CACH;AACF;;;;;;AAOA,SAAgB,aAAa,MAAmB,MAAkC;CAChF,MAAM,SAAS,YAAY,MAAM,IAAI;CACrC,MAAM,UAAyB,CAAC;CAChC,KAAK,MAAM,EAAE,OAAO,OAAO,aAAa,cAAc;EACpD,MAAM,OAAO,SAAS;EACtB,MAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,CAAC,GAAG,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC;EAClF,MAAM,QAAQ,SACZ,CAAC,GAAG,IAAI,IAAI,OAAO,SAAS,SAAS,OAAO,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;EACxE,MAAM,IAAI,QAAQ,SAAS,KAAK,CAAC,IAAI,KAAK,MAAM;EAChD,MAAM,IAAI,QAAQ,SAAS,KAAK,CAAC,IAAI,KAAK,MAAM;EAChD,IAAI,UAAU,kBAAkB,CAAE,EAAe,UAAU,CAAE,EAAe,SAAS;EACrF,IAAI,KAAK,UAAU,CAAC,MAAM,KAAK,UAAU,CAAC,GAAG;EAC7C,QAAQ,KAAK;GAAE;GAAO;GAAO,MAAM,KAAK,CAAC;GAAG,MAAM,KAAK,CAAC;GAAG,oBAAoB;EAAQ,CAAC;CAC1F;CACA,OAAO;AACT;;;;;AAMA,SAAgB,aACd,MACA,QACA,OACS;CACT,IAAI,UAAU;CACd,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,GAAG,OAAO,IAAI,EAAE;EAC5B,MAAM,MAAM,KAAK,IAAI,GAAG;EACxB,IAAI,QAAQ,KAAA,KAAa,QAAQ,EAAE,aAAa,UAAU;EAC1D,KAAK,IAAI,KAAK,EAAE,WAAW;CAC7B;CACA,OAAO;AACT;;AAUA,SAAgB,cACd,QACA,OACU;CACV,MAAM,IAAI,IAAI,IAAI,OAAO,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;CACvD,MAAM,IAAI,IAAI,IAAI,MAAM,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;CACtD,OAAO,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,EAAE,KAAK,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,SAAS;EACvE,IAAI,CAAC,EAAE,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,KAAK,OAAO;EACzC,IAAI,CAAC,EAAE,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS;EAC3C,OAAO,EAAE,IAAI,IAAI,MAAM,EAAE,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,SAAS;CAC9D,CAAC;AACH;AAEA,MAAM,cAA0E;CAC9E,SAAS;CACT,aAAa;CACb,OAAO;CACP,UAAU;CACV,OAAO;CACP,SAAS;AACX;;AAGA,SAAgB,WAAW,MAAsB,MAAgC;CAC/E,MAAM,MAAM,cAAc,KAAK,SAAS,KAAK,OAAO;CACpD,MAAM,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,KAAK,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,KAAK,MAAM,EAAE,GAAG,CAAC,CAAC;CAC1E,MAAM,wBAAQ,IAAI,IAAY;CAC9B,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,IAAI,KAAK,MAAM,MAAM,MAAM,EAAE,QAAQ,GAAG;EAC9C,MAAM,IAAI,KAAK,MAAM,MAAM,MAAM,EAAE,QAAQ,GAAG;EAC9C,IAAI,CAAC,KAAK,CAAC,GAAG;GACZ,IAAI,KAAK,QAAQ,IAAI,GAAG,IAAI,YAAY,SAAS;GACjD;EACF;EACA,KAAK,MAAM,QAAQ,OAAO,KAAK,WAAW,GACxC,IAAI,EAAE,iBAAiB,UAAU,EAAE,iBAAiB,OAAO,MAAM,IAAI,YAAY,KAAK;CAE1F;CACA,OAAO,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,MAAM,GAAG,EAAE,SAAS,CAAC;AAC1D;;AAGA,SAAS,UAAU,OAAsE;CACvF,MAAM,WAAW,MAAM,KAAK,MAC1B,EAAE,SAAS,UAAU,EAAE,YAAY,SAAS,SAAS,EAAE,OACzD;CACA,IAAI,SAAS,SAAS,MAAM,GAAG,OAAO;CACtC,IAAI,SAAS,SAAS,aAAa,GAAG,OAAO;CAC7C,OAAO,SAAS,OAAO,MAAM,MAAM,SAAS,IAAI,YAAY;AAC9D;;;;;;AAOA,SAAS,YAAY,GAAmB,GAAmB;CACzD,MAAM,SAAS,IAAI,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;CACrD,MAAM,SAAS,EAAE,MAAM,QAAQ,MAAM,OAAO,IAAI,EAAE,GAAG,CAAC;CACtD,IAAI,CAAC,OAAO,UAAU,OAAO,MAAM,MAAM,OAAO,IAAI,EAAE,GAAG,CAAC,EAAE,gBAAgB,EAAE,WAAW,GACvF,OAAO;CAET,MAAM,OAAO,IAAI,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,GAAG,CAAC;CAC9C,OAAO;EACL,MAAM,UAAU,OAAO,KAAK,MAAM,OAAO,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC;EACzD,MAAM,UAAU,MAAM;EACtB,OAAO;GACL,OAAO,EAAE,MAAM,QAAQ,MAAM,CAAC,OAAO,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,GAAG;GACjE,SAAS,EAAE,MAAM,QAAQ,MAAM,CAAC,KAAK,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,GAAG;EACnE;CACF;AACF;;AAGA,SAAS,aAAa,MAAsB,MAAqC;CAC/E,MAAM,QAAQ,OAAwC;EACpD,GAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC;EACjC,QAAQ,EAAE,UAAU;EACpB,MAAM,EAAE;EACR,MAAM,EAAE;CACV;CACA,QAAQ,KAAK,SAAS,CAAC,EAAA,CAAG,SAAS,MAAM;EACvC,MAAM,IAAI,KAAK,OAAO,MAAM,MAAM,EAAE,QAAQ,EAAE,GAAG;EACjD,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,QAAS,EAAE,UAAU,EAAE,SAAU,OAAO,CAAC;EACjE,OAAO,CAAC;GAAE,KAAK,EAAE;GAAK,MAAM,KAAK,CAAC;GAAG,MAAM,KAAK,CAAC;EAAE,CAAC;CACtD,CAAC;AACH;AAEA,SAAgB,QAAQ,MAAkB,MAAgC;CACxE,MAAM,gBAAgB,aAAa,MAAM,IAAI,CAAC,CAAC,MAAM,MAAM,EAAE,kBAAkB;CAE/E,OAAO,CADM,mBAAG,IAAI,IAAI,CAAC,GAAG,KAAK,MAAM,KAAK,MAAM,EAAE,EAAE,GAAG,GAAG,KAAK,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,CAAC,CAC/E,CAAC,CAAC,KAAK,OAAO;EACrB,MAAM,IAAI,KAAK,MAAM,MAAM,MAAM,EAAE,OAAO,EAAE;EAC5C,MAAM,IAAI,KAAK,MAAM,MAAM,MAAM,EAAE,OAAO,EAAE;EAC5C,IAAI,CAAC,GAAG,OAAO;GAAE;GAAI,MAAM;GAAM,MAAM,GAAG,WAAW;GAAM,QAAQ;EAAQ;EAC3E,IAAI,CAAC,GAAG,OAAO;GAAE;GAAI,MAAM,EAAE;GAAS,MAAM;GAAM,QAAQ;EAAU;EACpE,MAAM,SAAS,EAAE,gBAAgB,EAAE,cAAc,OAAO,YAAY,GAAG,CAAC;EACxE,MAAM,MAAkC,SACpC;GAAE;GAAI,GAAG;EAAO,IAChB;GAAE;GAAI,MAAM,EAAE;GAAS,MAAM,EAAE;EAAQ;EAC3C,MAAM,UAAU,aAAa,GAAG,CAAC;EACjC,IAAI,QAAQ,QAAQ,IAAI,UAAU;EAClC,IAAI,IAAI,SAAS,iBAAiB,IAAI,SAAS,eAAe,OAAO;GAAE,GAAG;GAAK,QAAQ;EAAQ;EAC/F,IAAI,EAAE,gBAAgB,EAAE,eAAe,CAAC,QACtC,OAAO;GAAE,GAAG;GAAK,QAAQ;GAAS,SAAS,WAAW,GAAG,CAAC;EAAE;EAE9D,IAAI,iBAAiB,IAAI,SAAS,IAAI,MAAM,OAAO;GAAE,GAAG;GAAK,QAAQ;EAAQ;EAC7E,IAAI,IAAI,SAAS,UAAU,IAAI,SAAS,QAAQ,OAAO;GAAE,GAAG;GAAK,QAAQ;EAAY;EACrF,IAAI,IAAI,SAAS,UAAU,IAAI,SAAS,QAAQ,OAAO;GAAE,GAAG;GAAK,QAAQ;EAAQ;EACjF,OAAO;GAAE,GAAG;GAAK,QAAQ;EAAO;CAClC,CAAC;AACH;;;;ACrPA,MAAM,IAAI;;;;;AAQV,SAAgB,OAAO,MAAc,GAAqB;CACxD,IAAI,MAAM,GAAG,OAAO;EAAE,KAAK;EAAG,MAAM;CAAE;CACtC,MAAM,IAAI,OAAO;CACjB,MAAM,KAAK,IAAI;CACf,MAAM,cAAc,IAAI,KAAK;CAC7B,MAAM,UAAU,IAAI,MAAM,IAAI,MAAM;CACpC,MAAM,OAAQ,IAAI,KAAK,KAAM,KAAK,IAAI,KAAM,IAAI,MAAM,IAAI,IAAI,EAAE,IAAK;CACrE,OAAO;EAAE,KAAK,KAAK,IAAI,GAAG,SAAS,IAAI;EAAG,MAAM,KAAK,IAAI,GAAG,SAAS,IAAI;CAAE;AAC7E;;;;;AAMA,SAAgB,aACd,SACA,SAC+B;CAC/B,MAAM,KAAK,QAAQ,IAAI,QAAQ,OAAO,QAAQ,IAAI;CAClD,MAAM,KAAK,QAAQ,IAAI,QAAQ,OAAO,QAAQ,IAAI;CAClD,MAAM,IAAI,OAAO,QAAQ,MAAM,QAAQ,CAAC;CACxC,MAAM,IAAI,OAAO,QAAQ,MAAM,QAAQ,CAAC;CACxC,MAAM,SAAS,KAAK;CACpB,OAAO;EACL;EACA,KAAK,SAAS,KAAK,MAAM,KAAK,EAAE,QAAQ,KAAK,EAAE,OAAO,OAAO,CAAC;EAC9D,MAAM,SAAS,KAAK,MAAM,EAAE,OAAO,OAAO,KAAK,KAAK,EAAE,QAAQ,CAAC;CACjE;AACF;;AAGA,MAAa,UAAU,MAAc,KAAK,MAAM,IAAI,GAAI,IAAI;;AAG5D,SAAgB,SAAS,QAA2B,GAA0B;CAC5E,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,MAAM,SAAS,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;CAC/C,MAAM,MAAM,OAAO,SAAS,KAAK;CACjC,MAAM,KAAK,KAAK,MAAM,EAAE;CACxB,MAAM,KAAK,KAAK,KAAK,EAAE;CACvB,MAAM,IAAI,OAAO;CAEjB,OAAO,KADG,OAAO,MACD,MAAM,KAAK;AAC7B;AAEA,MAAa,UAAU,WAA8B,SAAS,QAAQ,EAAG;;AAGzE,MAAM,cAAc;AAEpB,MAAM,UAAU,GAAW,MAAc;CACvC,IAAI,IAAI;CACR,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG,KAAK,IAAK,KAAK,IAAI,IAAI,KAAM;CACrD,OAAO,KAAK,MAAM,CAAC;AACrB;;;;;;AAOA,SAAgB,YAAY,IAAuB,IAAsC;CACvF,MAAM,KAAK,GAAG;CACd,MAAM,KAAK,GAAG;CACd,IAAI,OAAO,KAAK,OAAO,GAAG,OAAO;CACjC,MAAM,SAAS,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,KAAK,GAAG,OAAO;EAAE;EAAG;CAAE,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC;CAChF,MAAM,QAAQ,IAAI,MAAc,OAAO,MAAM;CAC7C,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,SAAU;EACnC,IAAI,IAAI;EACR,OAAO,IAAI,IAAI,OAAO,UAAU,OAAO,IAAI,EAAE,EAAE,MAAM,OAAO,EAAE,EAAE,GAAG;EACnE,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG,KAAK,MAAM,OAAO,EAAE,EAAE,MAAgB,IAAI,KAAK,IAAI;EAC3E,IAAI,IAAI;CACV;CACA,MAAM,IAAI,KAAK;CACf,MAAM,WAAY,MAAM,IAAI,KAAM;CAClC,MAAM,WAAW,KAAK,IAAI,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;CAClF,IAAI,OAAO,GAAG,EAAE,IAAI,aAAa;EAC/B,MAAM,yBAAS,IAAI,IAAoB;EACvC,KAAK,MAAM,EAAE,OAAO,QAAQ,OAAO,IAAI,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,CAAC;EAClE,MAAM,OAAO,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC,CAAC,QAAQ,GAAG,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC;EACpE,MAAM,WAAa,KAAK,KAAM,MAAO,IAAI,IAAI,QAAQ,KAAK,IAAI;EAC9D,IAAI,YAAY,GAAG,OAAO;EAC1B,MAAM,IAAI,KAAK,IAAI,GAAG,WAAW,EAAG,IAAI,KAAK,KAAK,QAAQ;EAC1D,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,UAAU,CAAC,EAAE;CAC3C;CACA,IAAI,UAAU;CACd,IAAI,QAAQ;CACZ,MAAM,QAAQ,OAAe,MAAc,QAAgB;EACzD,IAAI,SAAS,GAAG;GACd;GACA,IAAI,KAAK,IAAI,MAAM,QAAQ,KAAK,WAAW,MAAM;GACjD;EACF;EACA,KAAK,IAAI,IAAI,OAAO,KAAK,IAAI,MAAM,KAAK,KAAK,IAAI,GAAG,OAAO,GAAG,MAAO,MAAM,EAAa;CAC1F;CACA,KAAK,GAAG,IAAI,CAAC;CACb,OAAO,UAAU;AACnB;;AAGA,MAAa,YAAY,IAAY,OACnC,OAAO,KAAK,OAAO,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;AAEhE,SAAS,UAAU,GAAmB;CAEpC,MAAM,IAAI,KAAK,IAAK,WAAY,KAAK,IAAI,CAAC,IAAK,KAAK;CAIpD,MAAM,MAAM,IAFV,KACC,aAAc,KAAK,cAAe,KAAK,cAAc,KAAK,eAAe,IAAI,kBACzD,KAAK,IAAI,EAAE,IAAI,KAAK,CAAC;CAC5C,OAAO,KAAK,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO;AAC9C;;;;AClHA,MAAa,eAAe;;AAG5B,MAAa,oBAAoB;AAwDjC,MAAa,gBACX,OACkB;CAClB,OAAO,EAAE,SAAS;CAClB,SAAS,EAAE,QAAQ;CACnB,WAAW,EAAE,SAAS;CACtB,GAAI,EAAE,QAAQ,EAAE,QAAQ,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC;AACpD;AAEA,MAAMA,aAAW,MAAsC,EAAE,YAAY,UAAU,EAAE,YAAY;AAE7F,MAAM,UAAU,YAA8B;CAC5C,QAAQ,OAAO,MAAM,KAAK;CAC1B,IAAI,SAAS,QAAQ,GAAI,KAAK;CAC9B,IAAI,SAAS,QAAQ,GAAI,KAAK;AAChC;;AAGA,MAAa,eAAe,YAC1B,QAAQ,SAAS,MAAO,EAAE,WAAW,KAAA,IAAY,CAAC,IAAI,CAAC,EAAE,MAAM,CAAE;AAEnE,SAAgB,SAAS,SAA4C;CACnE,IAAI,QAAQ,WAAW,GAAG,OAAO;CACjC,MAAM,SAAS,YAAY,OAAO;CAClC,OAAO;EACL,GAAG,QAAQ;EACX,OAAO,OAAO,QAAQ,KAAK,MAAM,EAAE,KAAK,CAAC;EACzC,SAAS,OAAO,QAAQ,KAAK,MAAM,EAAE,OAAO,CAAC;EAC7C,WAAW,OAAO,QAAQ,KAAK,MAAM,EAAE,SAAS,CAAC,KAAK;EACtD,GAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,MAAM,EAAE,IAAI,CAAC;CACpD;AACF;;AAGA,SAAgB,cACd,IACA,IAC8C;CAC9C,MAAM,IAAI,YAAY,IAAI,EAAE;CAC5B,IAAI,SAAS,GAAG,QAAQ,GAAG,MAAM,KAAA,KAAmB,OAAO;EAAE;EAAG,SAAS;CAAe;CACxF,IAAI,MAAM,QAAQ,KAAA,KAAmB,OAAO;EAAE;EAAG,SAAS;CAAQ;CAClE,OAAO;EAAE;EAAG,UAAU,OAAO,EAAE,KAAK,MAAM,OAAO,EAAE,KAAK,KAAK,UAAU;CAAO;AAChF;AAEA,MAAM,SAAS,GAAW,MAAe,MAAM,IAAI,OAAO,IAAI;AAE9D,SAAgB,eACd,SACA,SACuB;CACvB,MAAM,IAAI,SAAS,OAAO;CAC1B,MAAM,IAAI,SAAS,OAAO;CAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO;CACrB,MAAM,QAAQ,cACZ,QAAQ,KAAK,MAAM,EAAE,KAAK,GAC1B,QAAQ,KAAK,MAAM,EAAE,KAAK,CAC5B;CACA,MAAM,OAAO,cACX,QAAQ,KAAK,MAAM,EAAE,OAAO,GAC5B,QAAQ,KAAK,MAAM,EAAE,OAAO,CAC9B;CACA,MAAM,OAAO,YAAY,OAAO;CAChC,MAAM,OAAO,YAAY,OAAO;CAEhC,MAAM,SAAS,KAAK,UAAU,KAAK,SAAS,cAAc,MAAM,IAAI,IAAI;CACxE,MAAM,WAAW,KAAK,SAAS,KAAK;CACpC,MAAM,aAAa,WAAW,KAAK,WAAW,QAAQ,SAAS,QAAQ;CACvE,MAAM,KAAK,QAAQ,KAAK,MAAM,EAAE,KAAK;CACrC,MAAM,KAAK,QAAQ,KAAK,MAAM,EAAE,KAAK;CACrC,OAAO;EACL,SAAS;EACT,SAAS;EACT,YAAY,MAAM,EAAE,MAAM,QAAQ,EAAE,MAAM,MAAM;EAChD,cAAc,MAAM,EAAE,QAAQ,QAAQ,EAAE,QAAQ,MAAM;EACtD,cAAc,EAAE,QAAQ,SAAS,EAAE,QAAQ;EAC3C,QAAQ,MAAM;EACd,UAAU,KAAK;EACf,WAAW,KAAK,IAAI,GAAG,EAAE,IAAI,KAAK,IAAI,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,EAAE,IAAI,KAAK,IAAI,GAAG,EAAE;EAChF,SAAS,MAAM;EACf,aAAa,KAAK;EAClB,aAAa,UAAU,EAAE,UAAU,EAAE,SAAS,MAAM,EAAE,OAAO,QAAQ,EAAE,OAAO,MAAM,IAAI;EACxF,SAAS,QAAQ,KAAK;EACtB,eAAe,QAAQ,WAAW;EAClC,GAAI,aAAa,EAAE,YAAY,kBAAkB,IAAI,CAAC;CACxD;AACF;;AAGA,MAAa,eAAe,GAAgC,YAC1D,EAAE,SAAS,QAAQ,MAAM,EAAE,SAAS,WAAWA,UAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,YAAY;AAE7E,MAAa,cAAc,GAAgC,YACzD,SAAS,YAAY,GAAG,OAAO,CAAC;;AAGlC,SAAgB,aACd,GACA,SACuB;CACvB,MAAM,OAAO,EAAE,MAAM,MAAM,MAAM,EAAE,QAAQ,OAAO;CAClD,IAAI,CAAC,MAAM,MAAM,OAAO;CACxB,OAAO,eAAe,YAAY,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,KAAK,EAAE,CAAC;AAC7E;;AAKA,MAAa,eAAe,MAAe,OAAO,UAAU,CAAC,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;AAExF,MAAa,eAAe,MAAsB,MAAM,OAAO,OAAO,IAAI,EAAE,QAAQ,CAAC;;AAGrF,SAAgB,cAAc,IAAoB;CAEhD,IAAI,KAAK,MAAM,OAAO,KAAK,KAAK,OAAO,IAAI,KAAK,IAAA,CAAM,QAAQ,CAAC,EAAE;CACjE,MAAM,IAAI,KAAK,MAAM,KAAK,GAAI;CAC9B,OAAO,IAAI,KAAK,GAAG,EAAE,KAAK,GAAG,KAAK,MAAM,IAAI,EAAE,EAAE,IAAI,OAAO,IAAI,EAAE,CAAC,CAAC,SAAS,GAAG,GAAG,EAAE;AACtF;AAEA,MAAM,WAAW,MACf,MAAM,OAAO,KAAK,SAAS,IAAI,OAAQ,WAAW,EAAE,QAAQ,CAAC,EAAE;AAEjE,MAAa,eAAgE;CAC3E,QAAQ,SAAS,SAAS;CAC1B,OAAO,SAAS,QAAQ;CACxB,aAAa;CACb,oBAAoB;AACtB;AAEA,MAAM,oBAAoB,MACxB,EAAE,YAAY,WAAW,EAAE,YAAY,SACnC,GAAG,aAAa,EAAE,QAAQ,CAAC,YAAY,IAAI,QAAQ,EAAE,MAAM,MAC3D,aAAa,EAAE,QAAQ,CAAC,EAAE;;AAGhC,MAAM,qBAAqB,MAA8B;CACvD,IAAI,EAAE,kBAAkB,MAAM,OAAO;CACrC,IAAI,EAAE,YAAY,kBAAkB,EAAE,kBAAkB,gBAAgB,OAAO;CAC/E,IAAI,EAAE,kBAAkB,WAAW,EAAE,kBAAkB,QACrD,OAAO,KAAK,aAAa,EAAE,cAAc,CAAC,QAAQ,IAAI,QAAQ,EAAE,OAAO;CAEzE,IAAI,EAAE,kBAAkB,SAAS,OAAO;CACxC,OAAO,KAAK,aAAa,EAAE,cAAc,CAAC,EAAE;AAC9C;;;;;;AAOA,SAAgB,YAAY,GAA2B;CAGrD,MAAM,QAAQ,CAAC,SAFQ,YAAY,EAAE,QAAQ,MAAM,MAAM,EAAE,MAAM,YAAY,EAAE,QAAQ,MAAM,MAAM,EAAE,IAAI,YAAY,EAAE,UAAU,EAAE,IAE7G,SADA,cAAc,EAAE,QAAQ,QAAQ,MAAM,EAAE,MAAM,cAAc,EAAE,QAAQ,QAAQ,MAAM,EAAE,IAAI,YAAY,EAAE,YAAY,EAAE,EAClH;CAC1B,IAAI,EAAE,kBAAkB,QAAQ,EAAE,QAAQ,UAAU,EAAE,QAAQ,QAC5D,MAAM,KACJ,UAAU,YAAY,EAAE,QAAQ,OAAO,MAAM,EAAE,MAAM,YAAY,EAAE,QAAQ,OAAO,MAAM,EAAE,IAAI,YAAY,EAAE,WAAW,EAAE,EAC3H;CAEF,OAAO,GAAG,MAAM,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,IAAI,kBAAkB,CAAC;AAC1E;;AAGA,MAAa,iBAAiB,MAC5B,EAAE,SACE,SAAS,YAAY,EAAE,MAAM,MAAM,EAAE,WAAW,cAAc,EAAE,QAAQ,MAAM,EAAE,YAAY,YAAY,EAAE,OAAO,MAAM,MACvH,SAAS,YAAY,EAAE,MAAM,MAAM,EAAE,WAAW,cAAc,EAAE,QAAQ,MAAM;;AAGpF,SAAgB,kBACd,MACA,MACQ;CACR,MAAM,IAAI,MAAM,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM;CACpD,MAAM,IAAI,MAAM,KAAK,QAAQ,QAAQ,KAAK,QAAQ,MAAM;CACxD,MAAM,OAAO,SAAS,YAAY,KAAK,MAAM,MAAM,EAAE,KAAK,YAAY,KAAK,MAAM,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE,YAAY,cAAc,KAAK,QAAQ,MAAM,EAAE,KAAK,cAAc,KAAK,QAAQ,MAAM,EAAE,IAAI,YAAY,CAAC,EAAE;CACtN,IAAI,CAAC,KAAK,UAAU,CAAC,KAAK,QAAQ,OAAO;CACzC,MAAM,MAAM,MAAM,KAAK,OAAO,QAAQ,KAAK,OAAO,MAAM;CACxD,OAAO,GAAG,KAAK,YAAY,YAAY,KAAK,OAAO,MAAM,EAAE,KAAK,YAAY,KAAK,OAAO,MAAM,EAAE,IAAI,YAAY,GAAG,EAAE;AACvH;;;;;;;ACrOA,MAAa,mBAAmB;;AAGhC,MAAM,iBAAiB;AAEvB,MAAM,WAAW,MAAsC,EAAE,YAAY,UAAU,EAAE,YAAY;;AAG7F,MAAM,YAAY;AAElB,MAAM,aAAa,aAAqB,SAAS,QAAQ,QAAQ,EAAE,CAAC,CAAC,KAAK;;AAG1E,SAAS,SAAS,MAAc,KAAqB;CACnD,IAAI,KAAK,UAAU,KAAK,OAAO;CAC/B,MAAM,MAAM,KAAK,MAAM,EAAE,MAAM,EAAE;CACjC,MAAM,QAAQ,IAAI,OAAO,IAAI;CAE7B,OAAO,KADM,SAAS,KAAK,QAAQ,MAAM,IAAI,IAAI,MAAM,QAAQ,CAAC,IAAI,IAAA,CACpD,UAAU;AAC5B;;AAGA,SAAgB,WAAW,UAA0B;CACnD,OAAO,KAAK,UAAU,QAAQ,GAAG,SAAS;AAC5C;;;;;;AAOA,SAAgB,kBACd,OACqB;CACrB,MAAM,wBAAQ,IAAI,IAA+C;CACjE,KAAK,MAAM,QAAQ,OACjB,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,GAAG,MAAM,IAAI,KAAK,KAAK,IAAI;CAEpD,MAAM,yBAAS,IAAI,IAAoB;CACvC,MAAM,uBAAO,IAAI,IAAsB;CAEvC,MAAM,YAAY,KAAa,UAAkB;EAC/C,MAAM,OAAO,OAAO,IAAI,GAAG;EAC3B,IAAI,SAAS,KAAA,GAAW;GACtB,MAAM,QAAQ,KAAK,IAAI,IAAI;GAC3B,IAAI,OAAO;IACT,MAAM,OAAO,MAAM,QAAQ,MAAM,MAAM,GAAG;IAC1C,IAAI,KAAK,QAAQ,KAAK,IAAI,MAAM,IAAI;SAC/B,KAAK,OAAO,IAAI;GACvB;EACF;EACA,OAAO,IAAI,KAAK,KAAK;EACrB,MAAM,QAAQ,KAAK,IAAI,KAAK,KAAK,CAAC;EAClC,MAAM,KAAK,GAAG;EACd,KAAK,IAAI,OAAO,KAAK;CACvB;CAEA,KAAK,MAAM,OAAO,MAAM,OAAO,GAAG,SAAS,IAAI,KAAK,WAAW,IAAI,QAAQ,CAAC;CAE5E,KAAK,MAAM,CAAC,OAAO,SAAS,CAAC,GAAG,IAAI,GAAG;EACrC,IAAI,KAAK,SAAS,GAAG;EACrB,MAAM,wBAAQ,IAAI,IAAsB;EACxC,KAAK,MAAM,OAAO,MAAM;GACtB,MAAM,MAAM,MAAM,IAAI,GAAG;GACzB,IAAI,CAAC,KAAK;GACV,MAAM,OAAO,SAAS,UAAU,IAAI,QAAQ,GAAG,SAAS;GACxD,MAAM,QAAQ,MAAM,IAAI,IAAI,KAAK,CAAC;GAClC,MAAM,KAAK,GAAG;GACd,MAAM,IAAI,MAAM,KAAK;EACvB;EACA,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO;GACjC,MAAM,CAAC,QAAQ;GACf,MAAM,SAAS,KAAK,IAAI,IAAI,KAAK,CAAC,EAAA,CAAG,MAAM,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;GACnE,IAAI,QAAQ,MAAM,WAAW,KAAK,SAAS,SAAS,CAAC,OAAO,SAAS,MAAM,IAAI;QAC1E,KAAK,MAAM,OAAO,OAAO,SAAS,KAAK,GAAG;EACjD;CACF;CAEA,KAAK,MAAM,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG;EAChC,IAAI,KAAK,SAAS,GAAG;EACrB,KAAK,MAAM,OAAO,MAAM,SAAS,KAAK,GAAG;CAC3C;CAEA,OAAO;AACT;AAcA,SAAS,WAAW,GAAgC,SAAwC;CAC1F,MAAM,wBAAQ,IAAI,IAAsB;CACxC,KAAK,MAAM,KAAK,EAAE,UAAU;EAC1B,IAAI,EAAE,SAAS,WAAW,CAAC,QAAQ,CAAC,GAAG;EACvC,KAAK,MAAM,KAAK,EAAE,YAAY;GAC5B,MAAM,MAAM,MAAM,IAAI,EAAE,GAAG,KAAK;IAC9B,OAAO;IACP,UAAU,EAAE;IACZ,MAAM;IACN,SAAS;GACX;GACA,IAAI;GACJ,IAAI,EAAE,YAAY,QAAQ,IAAI;GAC9B,MAAM,IAAI,EAAE,KAAK,GAAG;EACtB;CACF;CACA,MAAM,SAAS,kBACb,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,UAAU;EAAE;EAAK,UAAU,IAAI;CAAS,EAAE,CAClE;CACA,KAAK,MAAM,CAAC,KAAK,QAAQ,OAAO,IAAI,QAAQ,OAAO,IAAI,GAAG,KAAK,WAAW,IAAI,QAAQ;CACtF,OAAO;AACT;AAEA,SAAS,cAAc,GAA6B,SAAyC;CAC3F,MAAM,OAAO,EAAE,MAAM,MAAM,MAAM,EAAE,QAAQ,OAAO;CAClD,IAAI,MAAM,SAAS,WAAW,OAAO,KAAA;CACrC,OAAO,EAAE,MAAM,MACZ,MAAM,EAAE,SAAS,aAAa,EAAE,YAAY,KAAK,WAAW,EAAE,YAAY,KAAK,OAClF;AACF;AAEA,SAAS,WACP,SACA,SACsB;CACtB,IAAI,CAAC,WAAW,QAAQ,UAAA,KAA8B,QAAQ,UAAA,GAC5D,OAAO;CAET,MAAM,EAAE,QAAQ,KAAK,SAAS,aAC5B;EAAE,MAAM,QAAQ;EAAM,GAAG,QAAQ;CAAQ,GACzC;EAAE,MAAM,QAAQ;EAAM,GAAG,QAAQ;CAAQ,CAC3C;CACA,OAAO;EACL,QAAQ,OAAO,MAAM;EACrB,SAAS;GAAE,MAAM,QAAQ;GAAM,SAAS,QAAQ;EAAQ;EACxD,SAAS;GAAE,MAAM,QAAQ;GAAM,SAAS,QAAQ;EAAQ;EACxD,aAAa,MAAM,KAAK,OAAO;CACjC;AACF;;AAGA,SAAgB,eACd,GACA,SACgB;CAChB,MAAM,UAAU,WAAW,GAAG,OAAO;CACrC,MAAM,UAAU,cAAc,GAAG,OAAO;CACxC,MAAM,iBAAiB,UAAU,WAAW,GAAG,QAAQ,GAAG,IAAI,KAAA;CAC9D,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,UAAU;EACvC;EACA,GAAG;EACH,MAAM,WAAW,KAAK,gBAAgB,IAAI,GAAG,CAAC;CAChD,EAAE;AACJ;AAEA,MAAM,gBAAgB,MAAc,GAAG,KAAK,IAAI,MAAM,MAAM,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,GAAG;;AAGxF,SAAgB,gBAAgB,OAAwC;CACtE,OAAO,MACJ,KAAK,MAAM;EACV,MAAM,IAAI,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK,GAAG,EAAE;EACpC,OAAO,EAAE,OAAO,GAAG,EAAE,IAAI,aAAa,EAAE,KAAK,MAAM,EAAE,SAAS;CAChE,CAAC,CAAC,CACD,KAAK,KAAK;AACf;;AAWA,SAAgB,gBAAgB,YAG9B;CACA,MAAM,SAAS,kBAAkB,UAAU;CAC3C,MAAM,QAAQ,WAAW,KAAK,OAAO;EACnC,SAAS,EAAE;EACX,OAAO,EAAE;EACT,WAAW,EAAE;EACb,OAAO,OAAO,IAAI,EAAE,GAAG,KAAK,WAAW,EAAE,QAAQ;EACjD,KAAK,EAAE;CACT,EAAE;CACF,IAAI,MAAM,UAAU,gBAAgB,OAAO;EAAE,OAAO;EAAO,OAAO;CAAE;CAEpE,OAAO;EAAE,OAAO,CADJ,MAAM,QAAQ,GAAG,MAAO,EAAE,SAAS,EAAE,QAAQ,IAAI,CAC1C,CAAC;EAAG,OAAO,MAAM;CAAO;AAC7C;;AAGA,SAAgB,oBAAoB,YAAuD;CACzF,IAAI,CAAC,WAAW,QAAQ,OAAO;CAC/B,MAAM,EAAE,OAAO,UAAU,gBAAgB,UAAU;CACnD,MAAM,QAAQ,MAAM,KAAK,MAAM,GAAG,EAAE,QAAQ,GAAG,EAAE,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,UAAU,QAAQ,CAAC,GAAG;CAC7F,IAAI,OAAO,MAAM,KAAK,GAAG,MAAM,QAAQ;CACvC,OAAO,MAAM,KAAK,KAAK;AACzB;;AAGA,SAAgB,UAAU,GAAgC,SAAoC;CAC5F,OAAO,SAAS,EAAE,SAAS,QAAQ,MAAM,EAAE,SAAS,WAAW,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,KAAK,CAAC;AAChG;;;;AChLA,SAAgB,YAAY,IAA8C;CACxE,MAAM,CAAC,OAAO,IAAI,GAAG,QAAQ,GAAG,MAAM,KAAK;CAC3C,OAAO;EAAE;EAAM,MAAM,KAAK,SAAS,OAAO,CAAC,IAAI;CAAE;AACnD;;AAGA,SAAgB,WAAW,SAAyB,+BAAc,IAAI,KAAK,EAAA,CAAE,YAAY,GAAG;CAC1F,MAAM,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,GAAG,MAAO,EAAE,QAAQ,EAAE,QAAQ,IAAI,EAAG;CACvE,MAAM,yBAAS,IAAI,IAA4B;CAC/C,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,KAAK,QAAQ,MAAM,SAAS;EAClC,MAAM,OAAO,OAAO,IAAI,EAAE,KAAK,CAAC;EAChC,KAAK,KAAK,KAAK;EACf,OAAO,IAAI,IAAI,IAAI;CACrB;CACA,MAAM,wBAAQ,IAAI,IAA8B;CAChD,KAAK,MAAM,OAAO,QAChB,KAAK,MAAM,KAAK,IAAI,OAAO;EACzB,MAAM,QAAQ,MAAM,IAAI,EAAE,IAAI;EAC9B,IAAI,OAAO;GACT,MAAM;GACN;EACF;EACA,MAAM,IAAI,EAAE,MAAM;GAChB,MAAM,EAAE;GACR,IAAI,EAAE;GACN,GAAG,YAAY,EAAE,EAAE;GACnB,MAAM;GACN,QAAQ;IAAE,OAAO,IAAI;IAAO,SAAS,EAAE;GAAQ;EACjD,CAAC;CACH;CAQF,OAAO;EAAE,UAAA;GALP;GACA,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,WAAW;IAAE;IAAI,MAAM,KAAK;GAAO,EAAE;GACnE,QAAQ,OAAO,EAAE,EAAE,SAAS;GAC5B,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAO,EAAE,KAAK,EAAE,KAAK,KAAK,CAAE;EAElD;EAAG;CAAO;AAC5B;;AAGA,SAAgB,WAAW,SAAyB,aAA4C;CAC9F,MAAM,EAAE,UAAU,WAAW,WAAW,SAAS,WAAW;CAC5D,MAAM,wBAAQ,IAAI,IAAqB,CAAC,CAAC,cAAc,QAAQ,CAAC,CAAC;CACjE,KAAK,MAAM,CAAC,IAAI,SAAS,QAAQ,MAAM,IAAI,UAAU,EAAE,GAAG,IAAI;CAC9D,OAAO;AACT;AAIA,SAAgB,eAAe,IAAuC;CACpE,IAAI,OAAO,QAAQ,OAAO,KAAA,GAAW,OAAO;CAC5C,IAAI,KAAK,KAAM,OAAO,GAAG,KAAK,MAAM,EAAE,EAAE;CACxC,IAAI,KAAK,KAAQ,OAAO,IAAI,KAAK,IAAA,CAAM,QAAQ,KAAK,MAAS,IAAI,CAAC,EAAE;CACpE,MAAM,IAAI,KAAK,MAAM,KAAK,GAAM;CAChC,MAAM,IAAI,KAAK,MAAO,KAAK,MAAU,GAAI;CACzC,IAAI,IAAI,IAAI,OAAO,IAAI,GAAG,EAAE,IAAI,EAAE,KAAK,GAAG,EAAE;CAC5C,OAAO,GAAG,KAAK,MAAM,IAAI,EAAE,EAAE,IAAI,IAAI,GAAG;AAC1C;AAEA,MAAa,iBAAiB,SAC5B,SAAS,OAAO,MAAM,GAAG,KAAK,MAAM,OAAO,GAAG,EAAE;;;;;AAMlD,SAAgB,aAAa,GAAoD;CAC/E,MAAM,QAAQ;EACZ,GAAI,EAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,IAAI,CAAC;EACtC,GAAI,EAAE,SAAS,CAAC,UAAU,EAAE,QAAQ,IAAI,CAAC;EACzC,GAAI,EAAE,eAAe,SAAS,CAAC,UAAU,EAAE,eAAe,KAAK,IAAI,GAAG,IAAI,CAAC;EAC3E,GAAI,EAAE,eAAe,SAAS,CAAC,iBAAiB,EAAE,eAAe,KAAK,IAAI,GAAG,IAAI,CAAC;CACpF;CACA,OAAO,MAAM,SAAS,GAAG,EAAE,GAAG,IAAI,MAAM,KAAK,IAAI,EAAE,KAAK,EAAE;AAC5D;;;;;AAMA,SAAgB,aAAa,OAA6D;CACxF,MAAM,UAAU,MAAM,WAAW;EAAE,MAAM;EAAQ,SAAS,MAAM;CAAK;CACrE,OAAO,QAAQ,SAAS,SACpB,QAAQ,QAAQ,YAChB,GAAG,QAAQ,KAAK,GAAG,QAAQ,QAAQ,SAAS,MAAM,KAAK;AAC7D;;AAGA,SAAgB,cAAc,OAA4D;CACxF,MAAM,WAAW,MAAM;CACvB,IAAI,CAAC,UAAU,OAAO;CACtB,OAAO,GAAG,SAAS,UAAU,GAAG,SAAS,KAAK,GAAG,SAAS,YAAY,SAAS,KAAK;AACtF;;AAGA,SAAgB,YAAY,OAAqD;CAC/E,IAAI,CAAC,OAAO,OAAO;CACnB,MAAM,KAAK,MAAe,KAAK,MAAS,GAAG,KAAK,MAAM,IAAI,GAAI,EAAE,KAAK,OAAO,CAAC;CAC7E,MAAM,SAAS,MAAM,YAAY,KAAK,EAAE,MAAM,SAAS,EAAE,YAAY;CACrE,OAAO,GAAG,EAAE,aAAa,KAAK,CAAC,EAAE,KAAK,OAAO,KAAK,EAAE,MAAM,MAAM,EAAE;AACpE;AAEA,MAAa,YAAY,QAAoC,MAAM,IAAI,MAAM,GAAG,CAAC,IAAI;;;;;AAMrF,SAAgB,cAAc,GAAmE;CAC/F,MAAM,UAAU,GAAW,SAAiB,GAAG,EAAE,GAAG,OAAO,MAAM,IAAI,KAAK;CAC1E,OAAO,GAAG,OAAO,EAAE,OAAO,WAAW,EAAE,KAAK,OAAO,EAAE,WAAW,WAAW;AAC7E;;;;;AAQA,SAAgB,WAAW,KAGf;CACV,IAAI,IAAI,WAAW,eAAe,OAAO;CACzC,MAAM,WAAW,IAAI,MAAM,KAAK,MAAM,EAAE,OAAO;CAC/C,IAAI,SAAS,SAAS,MAAM,KAAK,IAAI,WAAW,UAAU,OAAO;CACjE,IAAI,SAAS,SAAS,aAAa,GAAG,OAAO;CAC7C,OAAO;AACT;AAEA,SAAS,QAAQ,OAAkC;CACjD,IAAI,MAAM,UAAU,GAAG,OAAO,MAAM,MAAM;CAC1C,IAAI,MAAM,WAAW,GAAG,OAAO,GAAG,MAAM,GAAG,OAAO,MAAM;CACxD,OAAO,GAAG,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,OAAO,MAAM,MAAM,SAAS;AACtE;;AAGA,SAAgB,kBAAkB,QAA4C;CAC5E,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,QAA6C,CAAC;CACpD,KAAK,MAAM,KAAK,OAAO,OACrB,KAAK,MAAM,KAAK,EAAE,UAAU;EAC1B,IAAI,EAAE,YAAY,UAAU,EAAE,YAAY,QAAQ;EAClD,KAAK,MAAM,KAAK,EAAE,YAAY;GAC5B,IAAI,EAAE,YAAY,UAAU,KAAK,IAAI,EAAE,GAAG,GAAG;GAC7C,KAAK,IAAI,EAAE,GAAG;GACd,MAAM,KAAK;IAAE,KAAK,EAAE;IAAK,UAAU,EAAE;GAAS,CAAC;EACjD;CACF;CAEF,MAAM,SAAS,kBAAkB,KAAK;CACtC,OAAO,MAAM,KAAK,SAAS,OAAO,IAAI,KAAK,GAAG,KAAK,KAAK,GAAG;AAC7D;;;;;AAMA,SAAgB,QAAQ,QAAsD;CAC5E,MAAM,EAAE,MAAM,MAAM,eAAe,OAAO;CAC1C,MAAM,UAAU,OAAO;CACvB,MAAM,SAAS,kBAAkB,MAAM;CACvC,MAAM,QAAkB,CAAC;CACzB,IAAI,YAAY,GAAG,MAAM,KAAK,qBAAqB;MAC9C;EACH,MAAM,KAAK,GAAG,KAAK,MAAM,QAAQ,yBAAyB;EAC1D,IAAI,OAAO,GACT,MAAM,KACJ,OAAO,SACH,GAAG,KAAK,cAAc,QAAQ,MAAM,EAAE,QAAQ,OAAO,WAAW,IAAI,KAAK,QACzE,GAAG,KAAK,QACd;CAEJ;CACA,IAAI,aAAa,GACf,MAAM,KAAK,GAAG,WAAW,cAAc,eAAe,IAAI,KAAK,KAAK;CAEtE,OAAO,GAAG,MAAM,KAAK,IAAI,EAAE;AAC7B;;;;;;AAkBA,SAAgB,QAAQ,SAAmC;CACzD,MAAM,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,GAAG,MAAO,EAAE,QAAQ,EAAE,QAAQ,KAAK,CAAE;CACvE,MAAM,uBAAO,IAAI,IAAoB;CACrC,MAAM,4BAAY,IAAI,IAAsC;CAC5D,IAAI,WAAgC;CACpC,MAAM,OAAiB,CAAC;CACxB,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,EAAE,MAAM,MAAM,eAAe,MAAM;EACzC,MAAM,OAAO,YAAY;GACvB,OAAO,SAAS;GAChB,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,YAAY;IAAE;IAAI,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC;GAAE,EAAE;EACjF;EACA,IAAI,eAAe;EACnB,KAAK,MAAM,KAAK,MAAM,OAAO;GAC3B,IAAI,aAAa,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG;GACvC,MAAM,QAAQ,UAAU,IAAI,EAAE,EAAE,qBAAK,IAAI,IAAyB;GAClE,KAAK,MAAM,KAAK,EAAE,OAAO,MAAM,IAAI,EAAE,KAAK,CAAC;GAC3C,UAAU,IAAI,EAAE,IAAI,KAAK;EAC3B;EACA,KAAK,KAAK;GACR;GACA,QAAQ;GACR,OAAO,OAAO,OAAO;GACrB,OAAO,OAAO,aAAa,MAAM,KAAK,IAAI,CAAC;GAC3C;EACF,CAAC;EACD,IAAI,MAAM,WAAW,YAAY,WAAW;CAC9C;CACA,OAAO,KAAK,QAAQ;AACtB;AAWA,MAAM,WAAW,OAAyB;CACxC,MAAM,EAAE,SAAS,QAAQ,MAAM,EAAE,YAAY,MAAM,CAAC,CAAC;CACrD,MAAM,EAAE,SAAS,QAAQ,MAAM,EAAE,YAAY,UAAU,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC;CAC7E,QAAQ,EAAE,SAAS,QAAQ,MAAM,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC;CACtD,OAAO,EAAE,SAAS,QAAQ,MAAM,EAAE,YAAY,aAAa,CAAC,CAAC;CAC7D,SAAS,EAAE,SAAS,QAAQ,MAAM,EAAE,YAAY,SAAS,CAAC,CAAC;AAC7D;AAEA,MAAM,YAAY,GAAU,OAAqB;CAC/C,MAAM,EAAE,OAAO,EAAE;CACjB,MAAM,EAAE,OAAO,EAAE;CACjB,QAAQ,EAAE,SAAS,EAAE;CACrB,OAAO,EAAE,QAAQ,EAAE;CACnB,SAAS,EAAE,UAAU,EAAE;AACzB;AAEA,MAAM,OAAc;CAAE,MAAM;CAAG,MAAM;CAAG,QAAQ;CAAG,OAAO;CAAG,SAAS;AAAE;;AAGxE,SAAgB,SAAS,QAA8C;CACrE,MAAM,OAA6C;EACjD,MAAM;EACN,KAAK;EACL,MAAM;EACN,OAAO;EACP,UAAU,CAAC;CACb;CACA,KAAK,MAAM,KAAK,OAAO,OAAO;EAC5B,MAAM,QAAQ,CAAC,EAAE,MAAM,GAAG,EAAE,KAAK;EACjC,IAAI,OAAO;EACX,KAAK,MAAM,CAAC,GAAG,SAAS,MAAM,QAAQ,GAAG;GACvC,MAAM,MAAM,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK;GAC5C,IAAI,QAAQ,KAAK,SAAS,MACvB,MAAiD,EAAE,SAAS,WAAW,EAAE,QAAQ,GACpF;GACA,IAAI,CAAC,OAAO;IACV,QAAQ;KAAE,MAAM;KAAS;KAAK;KAAM,OAAO;KAAM,UAAU,CAAC;IAAE;IAC9D,KAAK,SAAS,KAAK,KAAK;GAC1B;GACA,OAAO;EACT;EACA,KAAK,SAAS,KAAK;GAAE,MAAM;GAAQ,KAAK,EAAE;GAAI,MAAM,EAAE;GAAM,OAAO,QAAQ,CAAC;GAAG,MAAM;EAAE,CAAC;CAC1F;CACA,MAAM,UAAU,MAA0B;EACxC,IAAI,EAAE,SAAS,QAAQ,OAAO;EAC9B,MAAM,WAAW,EAAE,SAAS,IAAI,MAAM;EACtC,SAAS,MAAM,GAAG,MAAM,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,KAAK,CAAC;EACrD,OAAO;GAAE,GAAG;GAAG;GAAU,OAAO,SAAS,QAAQ,GAAG,MAAM,SAAS,GAAG,EAAE,KAAK,GAAG,IAAI;EAAE;CACxF;CACA,OAAQ,OAAO,IAAI,CAAC,CAA0C;AAChE;AAEA,MAAM,QAAQ,MAAc,EAAE,OAAO,IAAI,EAAE,QAAQ,IAAI,EAAE,OAAO,IAAI;;AAqCpE,SAAgB,SAAS,GAAyB;CAChD,OAAO,EAAE,MAAM,KAAK,UAAU;EAC5B,KAAK,KAAK;EACV,SAAS,KAAK;EACd,aAAa,KAAK;EAClB,SAAS,KAAK;EACd,MAAM,KAAK;EACX,OAAO,KAAK,SAAS;EACrB,MAAM,KAAK;EACX,WAAW,KAAK;EAChB,MAAM,KAAK;EACX,MAAM,KAAK;EACX,MAAM,KAAK;EACX,YAAY,KAAK;EACjB,SAAS,KAAK;EACd,UAAU,KAAK;EACf,SAAS,KAAK;EACd,MAAM,KAAK,OAAO,KAAK,MAAM,KAAK,KAAK,SAAS,GAAG,IAAI;EACvD,cAAc,KAAK,OACf;GACE,IAAI,KAAK,MAAM,KAAK,KAAK,SAAS,KAAK,GAAG;GAC1C,IAAI,KAAK,MAAM,KAAK,KAAK,SAAS,KAAK,GAAG;GAC1C,aAAa,KAAK,KAAK;EACzB,IACA;EACJ,cAAc,KAAK,MAAM,YAAY,KAAA,KAAa,KAAK,KAAK,SAAS,KAAK,KAAK,KAAK;EACpF,QAAQ,WAAW,GAAG,KAAK,GAAG;EAC9B,iBAAiB,aAAa,GAAG,KAAK,GAAG;EACzC,YAAY,eAAe,GAAG,KAAK,GAAG;EACtC,OAAO,UAAU,GAAG,KAAK,GAAG;EAC5B,UAAU,EAAE,SAAS,QAAQ,MAAM,EAAE,SAAS,KAAK,GAAG;CACxD,EAAE;AACJ;;AAuBA,SAAgB,WAAW,GAA4C;CACrE,QAAQ,EAAE,WAAW,CAAC,EAAA,CAAG,KAAK,MAAM;EAClC,MAAM,SAAS,QACb,EAAE,MACC,QAAQ,MAAM,EAAE,QAAQ,GAAG,CAAC,CAC5B,KAAK,OAAO;GAAE,MAAM,EAAE;GAAM,MAAM,EAAE;GAAM,GAAG,EAAE;GAAG,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI;EAAK,EAAE;EACzF,OAAO;GACL,UAAU,EAAE;GACZ,OAAO,EAAE,iBAAiB,EAAE;GAC5B,OAAO,EAAE,MAAM;GACf,MAAM,EAAE,MAAM,QAAQ,MAAM,EAAE,WAAW,KAAK,CAAC,CAAC;GAChD,QAAQ,MAAM,QAAQ;GACtB,QAAQ,MAAM,QAAQ;GACtB,OAAO,EAAE,SAAS;GAClB,SAAS,EAAE,MAAM,WAAW,KAAK,EAAE,UAAU,KAAA;GAC7C,SAAS,cAAc,EAAE,KAAK;EAChC;CACF,CAAC;AACH;;AAGA,MAAa,YAAY,MACvB,EAAE,MAAM,SAAS,KAAK,EAAE,MAAM,MAAM,MAAM,EAAE,OAAO,CAAC;;AActD,SAAgB,YAAY,SAAyB,MAAkC;CACrF,MAAM,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,GAAG,MAAO,EAAE,QAAQ,EAAE,QAAQ,KAAK,CAAE;CACvE,MAAM,SAA6B,CAAC;CACpC,MAAM,uBAAO,IAAI,IAAoB;CACrC,KAAK,MAAM,OAAO,QAAQ;EACxB,MAAM,IAAI,IAAI,MAAM,MAAM,MAAM,EAAE,SAAS,IAAI;EAC/C,IAAI,CAAC,GAAG;EACR,OAAO,KAAK;GACV;GACA,SAAS,EAAE;GACX,UAAU,EAAE;GACZ,OAAO,EAAE;GACT,SAAS,aAAa,MAAM,EAAE,IAAI,EAAE,KAAK;EAC3C,CAAC;CACH;CACA,OAAO;AACT;AAsBA,MAAM,aAAa,MAA6B;CAC9C,MAAM,SAAS,GAAG,SAAS,SAAS,MAAM,EAAE,WAAW,KAAK,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC;CAChF,OAAO,OAAO,SAAS,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO,SAAS;AAC7E;AAEA,MAAa,gBAAiC;CAC5C;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;AAOA,MAAa,mBAAmB,UAC9B,MAAM,MAAM,MAAM,EAAE,SAAS,MAAM;;;;;AAMrC,MAAa,iBACX,GACA,MAEA,EAAE,YAAY,UACd,EAAE,OAAO,SAAS,aAClB,EAAE,MAAM,MAAM,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,EAAE,SAAS;;AAGlD,SAAS,cAAc,OAAqC;CAC1D,MAAM,QAAQ,MAAM,QAAQ,MAAM,EAAE,SAAS,MAAM;CACnD,MAAM,OAAO,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,MAAM,CAAC;CACjD,MAAM,UAAU,OAAO,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,MAAM,CAAC;CAC3D,OAAO,UAAU,OAAO,UAAU;AACpC;;AAGA,SAAgB,YACd,MACA,MACA,UAAkD,CAAC,GACnD;CACA,MAAM,OAAyB,QAAQ,MAAM,IAAI,CAAC,CAAC,KAAK,QAAQ;EAC9D,MAAM,IAAI,KAAK,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,EAAE;EAChD,MAAM,IAAI,KAAK,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,EAAE;EAChD,MAAM,WAAW,IAAI,IAAI,GAAG,MAAM,KAAK,MAAM,EAAE,GAAG,CAAC;EACnD,MAAM,SACJ,IAAI,SAAS,IAAI,IAAI,GAAG,MAAM,QAAQ,MAAM,SAAS,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,GAAG,CAAC;EACpF,MAAM,YAAY,MAAgB;GAChC,IAAI,CAAC,GAAG,OAAO;GACf,OAAO,SAAS,cAAc,EAAE,MAAM,QAAQ,MAAM,OAAO,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE;EAC9E;EACA,OAAO;GACL,GAAG;GACH,MAAM,GAAG,QAAQ,GAAG,QAAQ;GAC5B,cAAc,SAAS,CAAC;GACxB,cAAc,SAAS,CAAC;GACxB,WAAW,UAAU,QAAQ,MAAM,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC;GACrE,WAAW,UAAU,QAAQ,MAAM,MAAM,MAAM,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC;GACrE,SAAS,IAAI,WAAW,CAAC;EAC3B;CACF,CAAC;CACD,KAAK,MAAM,GAAG,MAAM,cAAc,QAAQ,EAAE,MAAM,IAAI,cAAc,QAAQ,EAAE,MAAM,CAAC;CAIrF,OAAO;EAAE;EAAM,QAHA,OAAO,YACpB,cAAc,KAAK,MAAM,CAAC,GAAG,KAAK,QAAQ,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAErD;EAAG,SAAS,aAAa,MAAM,IAAI;CAAE;AAC3D;;AAsBA,SAAgB,gBAAgB,MAAgC;CAC9D,MAAM,QAA0B,CAAC;CACjC,KAAK,MAAM,QAAQ,KAAK,MAAM,IAAI,GAAG;EACnC,IAAI,CAAC,KAAK,KAAK,GAAG;EAClB,IAAI;GACF,MAAM,QAAQ,KAAK,MAAM,IAAI;GAC7B,IAAI,SAAS,OAAO,MAAM,SAAS,UAAU,MAAM,KAAK,uBAAuB,KAAK,CAAC;EACvF,QAAQ,CAAC;CACX;CACA,OAAO;AACT;AAEA,SAAgB,gBAAgB,OAA2C;CACzE,OAAO,MAAM,KAAK,SAAS;EACzB,IAAI,KAAK,SAAS,UAChB,OAAO;GAAE,MAAM;GAAQ,MAAM,KAAK;GAAM,QAAQ,KAAK;GAAM,IAAI,KAAK;EAAG;EAEzE,IAAI,KAAK,SAAS,YAChB,OAAO;GAAE,MAAM;GAAY,MAAM,KAAK;GAAM,QAAQ,KAAK;GAAQ,YAAY,KAAK;EAAW;EAE/F,OAAO;GAAE,MAAM;GAAS,MAAM,KAAK;GAAM,OAAO;EAAK;CACvD,CAAC;AACH;AAYA,MAAM,SAAS;;AAGf,SAAgB,WAAW,OAA4B;CACrD,IAAI,CAAC,MAAM,KAAK,GAAG,OAAO,CAAC;CAC3B,OAAO,MACJ,MAAM,mBAAmB,CAAC,CAC1B,QAAQ,SAAS,KAAK,WAAW,aAAa,CAAC,CAAC,CAChD,KAAK,SAAS;EACb,MAAM,SAAS,OAAO,KAAK,IAAI;EAC/B,MAAM,OAAO,SAAS,MAAM,SAAS,MAAM;EAC3C,IAAI,QAAQ;EACZ,IAAI,UAAU;EACd,KAAK,MAAM,QAAQ,KAAK,MAAM,IAAI,GAChC,IAAI,KAAK,WAAW,GAAG,KAAK,CAAC,KAAK,WAAW,MAAM,GAAG;OACjD,IAAI,KAAK,WAAW,GAAG,KAAK,CAAC,KAAK,WAAW,MAAM,GAAG;EAW7D,OAAO;GAAE;GAAM,QATqB,kBAAkB,KAAK,IAAI,IAC3D,UACA,sBAAsB,KAAK,IAAI,IAC7B,YACA,gBAAgB,KAAK,IAAI,IACvB,YACA,oCAAoC,KAAK,IAAI,IAC3C,WACA;GACa;GAAO;GAAS;EAAK;CAC9C,CAAC;AACL;AAIA,MAAa,YAAY,QAAkC,SACzD,OAAO,MAAM,MAAM,MAAM,EAAE,SAAS,IAAI;AAE1C,MAAa,eAAe,GAAc,cACxC,EAAE,SAAS,MAAM,MAAM,EAAE,OAAO,SAAS;;AAG3C,MAAa,kBAAkB,MAC7B,EAAE,SAAS,MAAM,MAAM,EAAE,YAAY,UAAU,EAAE,YAAY,aAAa,KAAK,EAAE,SAAS"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-yDAEwmZo.js";
|
|
2
|
-
import { l as judgeRoute, o as isFakeJudge } from "./judge-
|
|
3
|
-
import { z } from "zod";
|
|
2
|
+
import { l as judgeRoute, o as isFakeJudge } from "./judge-CeYPN8P_.js";
|
|
4
3
|
import { Output, generateText } from "ai";
|
|
4
|
+
import { z } from "zod";
|
|
5
5
|
//#region src/analysis/schema.ts
|
|
6
6
|
const confidence = z.enum([
|
|
7
7
|
"high",
|
|
@@ -44,7 +44,7 @@ const analysisSchema = z.object({
|
|
|
44
44
|
]),
|
|
45
45
|
change: z.string(),
|
|
46
46
|
expectedEffect: z.string().describe("What should measurably change"),
|
|
47
|
-
verify: z.string().describe("One reeval command from the CLI list that would show the effect"),
|
|
47
|
+
verify: z.string().describe("One reeval command from the CLI list that would show the effect. --tag only if that tag is on setup.evals; otherwise reeval run <id>"),
|
|
48
48
|
impact: z.enum([
|
|
49
49
|
"high",
|
|
50
50
|
"medium",
|
|
@@ -63,18 +63,19 @@ const analysisSchema = z.object({
|
|
|
63
63
|
})
|
|
64
64
|
});
|
|
65
65
|
const SYSTEM_PROMPT = `You analyse the results of a reeval run for the engineer who wrote the evals.
|
|
66
|
-
reeval runs coding agents in sandboxes. A compare eval runs a control cell and variant cells (with a skill, docs or MCP, say) and reports pass-rate lift with a 95% interval, and effort (tool calls, agent time) next to it.
|
|
66
|
+
reeval runs coding agents in sandboxes. A compare eval runs a control cell and variant cells (with a skill, docs or MCP, say) and reports pass-rate lift with a 95% interval, and effort (tool calls, agent time, and tokens when both sides reported usage) next to it.
|
|
67
67
|
|
|
68
68
|
You get a DIGEST: facts computed deterministically by reeval (ids F1...), evidence excerpts (ids E1...; transcript lines start with their line number L<n>), an attempts table (ids like r2/fix-config/control#1), the RULES reeval used for every verdict, and the CLI.
|
|
69
69
|
|
|
70
70
|
Rules:
|
|
71
71
|
- Every claim cites at least one id from the digest: F<n>, E<n>, E<n>:L<line> for a line the evidence shows, an attempt id, or <attempt id>:check<n>. Never cite anything that isn't in the digest.
|
|
72
72
|
- Take numbers from the facts. Don't compute new statistics beyond simple ratios of cited numbers.
|
|
73
|
-
-
|
|
73
|
+
- A cited judge score must be the recorded score on that attempt. Do not invent a different number.
|
|
74
|
+
- The verdicts in the facts are final: if a lift or effort verdict says within noise, insufficient or not comparable, say so; never call it an effect. insufficient is not measured: never put it on a results line. insufficient on effort is statistical only (too few attempts); a missing token metric is omitted, not insufficient — do not treat unreportedUsage as a result. When sources or the eval fingerprint differ across the compared runs, the first line says the eval changed; that pass-rate difference is not an effect, and mixed counts such as 6/10 vs 8/9 are not a results line. A lift is an effect only when sources match. Name each factor claim's tier (controlled, pooled, observed). Compare-question facts (kind compare-question) are annotation: win rates, ties, errors and vote reasons never change pass, fail or the exit code. Do not put them on a results line as a gate. A vote's reason is the model's wording: do not treat it as a check.
|
|
74
75
|
- Say what can't be attributed (factors that are not_measured).
|
|
75
76
|
- Separate causes: the thing under test (skill, docs), the eval itself (task, checks, judge question), and reeval or infrastructure (bugs, fingerprints, harness).
|
|
76
77
|
- Explain a failed judge verdict by comparing the cited output with the judge question, and say it's your reading: judges record no reason.
|
|
77
|
-
- Actions: at most 5, each with a concrete target, the change, the expected measurable effect, and a verify command that is exactly one of the reeval commands in the CLI list, with only the flags listed for it. "No actions" is a valid answer when nothing is worth doing. No generic advice ("add more tests") without a concrete target.
|
|
78
|
+
- Actions: at most 5, each with a concrete target, the change, the expected measurable effect, and a verify command that is exactly one of the reeval commands in the CLI list, with only the flags listed for it. A \`--tag\` value must appear on some eval's \`tags\` in the digest. If none does, select that eval with \`reeval run <id>\` using its \`id\` from setup.evals; do not invent a tag from the eval name. "No actions" is a valid answer when nothing is worth doing. No generic advice ("add more tests") without a concrete target.
|
|
78
79
|
- Quotes, if any, must be copied exactly from the cited evidence.
|
|
79
80
|
- TL;DR: one sentence: what happened, how sure, why, next. No fact ids or intervals.
|
|
80
81
|
- Do not rate your own citation or command checks; reeval caps confidence after it checks them.
|
|
@@ -227,4 +228,4 @@ async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
|
|
|
227
228
|
//#endregion
|
|
228
229
|
export { providerModel as a, resolveAnalysisModel as c, isFake as i, SYSTEM_PROMPT as l, callAnalysisModel as n, questionModel as o, cannedModel as r, questionModelLabel as s, analysisModelName as t };
|
|
229
230
|
|
|
230
|
-
//# sourceMappingURL=model-
|
|
231
|
+
//# sourceMappingURL=model-Cl507trk.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-Cl507trk.js","names":[],"sources":["../src/analysis/schema.ts","../src/analysis/model.ts"],"mappings":";;;;;AAGA,MAAM,aAAa,EAAE,KAAK;CAAC;CAAQ;CAAU;AAAK,CAAC;AACnD,MAAM,QAAQ,EACX,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SACC,wFACF;AAEF,MAAM,QAAQ,EAAE,OAAO;CACrB,MAAM,EAAE,OAAO;CACf;CACA;CACA,QAAQ,EACL,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,CAAC,CACV,SAAS,qEAAqE;AACnF,CAAC;AAED,MAAa,iBAAiB,EAAE,OAAO;CACrC,MAAM,EACH,OAAO,CAAC,CACR,SAAS,4EAA4E;CACxF,cAAc,EAAE,MAAM,KAAK;CAC3B,SAAS,EACN,MAAM,KAAK,CAAC,CACZ,SACC,0HACF;CACF,KAAK,EACF,MAAM,KAAK,CAAC,CACZ,SAAS,yEAAyE;CACrF,WAAW,EAAE,MACX,EAAE,OAAO;EACP,MAAM,EAAE,OAAO;EACf,MAAM,EAAE,KAAK;GAAC;GAAU;GAAe;GAAS;GAAS;EAAO,CAAC;EACjE;CACF,CAAC,CACH;CACA,SAAS,EACN,MACC,EAAE,OAAO;EACP,OAAO,EAAE,OAAO;EAChB,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,mDAAmD;EAC/E,MAAM,EAAE,KAAK;GAAC;GAAS;GAAQ;GAAQ;GAAS;GAAM;EAAS,CAAC;EAChE,QAAQ,EAAE,OAAO;EACjB,gBAAgB,EAAE,OAAO,CAAC,CAAC,SAAS,+BAA+B;EACnE,QAAQ,EACL,OAAO,CAAC,CACR,SACC,sIACF;EACF,QAAQ,EAAE,KAAK;GAAC;GAAQ;GAAU;EAAK,CAAC;EACxC,QAAQ,EAAE,KAAK;GAAC;GAAO;GAAU;EAAM,CAAC;EACxC;CACF,CAAC,CACH,CAAC,CACA,IAAI,CAAC;CACR,YAAY,EAAE,OAAO;EAAE,SAAS;EAAY,KAAK,EAAE,OAAO;CAAE,CAAC;AAC/D,CAAC;AAID,MAAa,gBAAgB;;;;;;;;;;;;;;;;;;;;ACjD7B,MAAa,UAAU,MACrB,OAAO,MAAM,YAAY,MAAM,QAAS,EAAwB,SAAS;AAE3E,MAAM,qBAAqB,UAAkB,MAAM,WAAW,cAAc;;AAG5E,SAAS,aAAa,QAAgE;CACpF,MAAM,QAAQ,OAAO,SAAS;CAC9B,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,QAAQ,OAAO,MAAM;CAC3B,IAAI,OAAO,UAAU,YAAY,CAAC,kBAAkB,KAAK,GAAG,OAAO;AAErE;;AAGA,SAAS,WAAW,QAAwB,MAAsB;CAChE,MAAM,QAAQ,OAAO,MAAM;CAC3B,IAAI,OAAO,UAAU,YAAY,CAAC,YAAY,KAAK,GAAG,OAAO;CAC7D,OAAO,gBAAgB,YAAY,KAAK,IAAI,iBAAiB,MAAM,iEAAiE,KAAK;AAC3I;;AAMA,SAAgB,kBACd,QACA,MAAmB,WACS;CAC5B,MAAM,QAAQ,aAAa,MAAM;CACjC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,IAAI,WACR,GAAG,QAAQ,UAAU,0BAA0B,iBAAiB,iFAAiF,OAAO,OAAO,WAAW,QAAQ,mBAAmB,IAAI,QAAQ,UAAU,wBAAwB,gFAAgF,EACrU;AACF;;AAGA,SAAgB,mBAAmB,QAAuC;CACxE,MAAM,QAAQ,aAAa,MAAM;CACjC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,KAAK,IAAI,SAAS;AAClC;;;;;AAMA,SAAgB,cACd,QACA,UACA,OAC4B;CAC5B,IAAI,SAAS,UAAU,KAAA,GAAW,OAAO,SAAS;CAClD,MAAM,QAAQ,aAAa,MAAM;CACjC,MAAM,MAAM;CACZ,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,WACR,sBAAsB,SAAS,SAAS,KAAK,MAAM,+FAA+F,OAAO,OAAO,WAAW,QAAQ,2BAA2B,EAAE,OAAO,IAAI,EAC7N;CAEF,IAAI,OAAO,UAAU,YAAY,kBAAkB,KAAK,GACtD,MAAM,IAAI,WACR,sBAAsB,SAAS,SAAS,KAAK,MAAM,gDAAgD,MAAM,mJAAmJ,IAAI,EAClQ;CAEF,OAAO;AACT;;AAGA,SAAgB,qBACd,QACA,MAAmB,WACI;CACvB,MAAM,QAAQ,kBAAkB,QAAQ,GAAG;CAC3C,IAAI,OAAO,KAAK,GAAG,OAAO;EAAE,MAAM;EAAQ,OAAO;EAAQ,MAAM;CAAM;CACrE,IAAI,kBAAkB,KAAK,GACzB,MAAM,IAAI,WACR,0BAA0B,MAAM,wIAClC;CAEF,IAAI;EACF,OAAO;GAAE,MAAM;GAAO,OAAO,WAAW,KAAK,CAAC,CAAC;EAAK;CACtD,SAAS,OAAO;EACd,MAAM,UAAW,MAAgB,QAAQ,QAAQ,eAAe,gBAAgB;EAChF,MAAM,IAAI,WACR,QAAQ,UAAU,UAAU,GAAG,QAAQ,sDACzC;CACF;AACF;;AAGA,SAAgB,YAAY,QAA6D;CACvF,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,YAAY;GACtB,IAAI,WAAW,QAAQ,MAAM,IAAI,MAAM,OAAO,KAAK;GACnD,OAAO;IACL,SAAS,CAAC;KAAE,MAAM;KAAQ,MAAM,OAAO;IAAK,CAAC;IAC7C,cAAc;KAAE,SAAS;KAAQ,KAAK;IAAO;IAC7C,OAAO;KACL,aAAa;MAAE,OAAO;MAAG,SAAS;MAAG,WAAW;MAAG,YAAY;KAAE;KACjE,cAAc;MAAE,OAAO;MAAG,MAAM;MAAG,WAAW;KAAE;IAClD;IACA,UAAU,CAAC;IACX,UAAU,EAAE,SAAS,gBAAgB;GACvC;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF;;AAGA,SAAS,UAAU,MAAyB,QAA+B;CACzE,IAAI,KAAK,WAAW,KAAA,GAClB,MAAM,IAAI,MAAM,+DAA+D;CAEjF,MAAM,SAAS,OAAO,KAAK,WAAW,aAAa,KAAK,OAAO,MAAM,IAAI,KAAK;CAC9E,OAAO,YAAY,EAAE,MAAM,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM,EAAE,CAAC;AAC3F;AAEA,eAAe,cAAc,OAA8B,QAAwC;CACjG,IAAI,MAAM,MAAM,OAAO,UAAU,MAAM,MAAM,MAAM;CACnD,OAAO,cAAc,KAAK;AAC5B;;AAGA,eAAsB,cAAc,OAAsD;CACxF,IAAI,MAAM,UAAU,WAAW,OAAO,MAAM;CAC5C,MAAM,WAAW,MAAM,KAAK,MAAM,GAAG,MAAM,KAAK,QAAQ,GAAG,CAAC;CAC5D,MAAM,KAAK,MAAM,KAAK,MAAM,SAAS,SAAS,CAAC;CAC/C,IAAI,aAAa,UAAU;EACzB,MAAM,EAAE,WAAW,MAAM,OAAO;EAChC,OAAO,OAAO,EAAE;CAClB;CACA,OAAO,MAAM;AACf;AASA,eAAsB,kBACpB,OACA,QACA,QACA,iBACsB;CACtB,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,cAAc,OAAO,MAAM;EACxC,QAAQ;EACR;EACA,QAAQ,OAAO,OAAO;GAAE,QAAQ;GAAgB,MAAM;EAAW,CAAC;EAClE,aAAa;EACb;EACA,YAAY;CACd,CAAC;CACD,OAAO;EACL,QAAQ,OAAO;EACf,eAAe,OAAO,UAAU,WAAW;EAC3C,OAAO;GACL,aAAa,OAAO,OAAO,eAAe;GAC1C,cAAc,OAAO,OAAO,gBAAgB;EAC9C;EACA,WAAW,KAAK,IAAI,IAAI;CAC1B;AACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { l as judgeRoute } from "./judge-
|
|
1
|
+
import { l as judgeRoute } from "./judge-CeYPN8P_.js";
|
|
2
2
|
import { i as debugTrace } from "./attempt-CGPV3BZY.js";
|
|
3
|
-
import {
|
|
3
|
+
import { experimental_decide } from "ai";
|
|
4
4
|
import { google } from "@ai-sdk/google";
|
|
5
5
|
//#region src/judge/model.ts
|
|
6
6
|
/** See `judgeRoute`: a known provider's own package when its key is set, else the AI Gateway. */
|
|
@@ -9,12 +9,12 @@ async function resolveModel(model) {
|
|
|
9
9
|
const route = judgeRoute(model);
|
|
10
10
|
if (route.kind === "gateway") return model;
|
|
11
11
|
const id = model.slice(route.provider.length + 1);
|
|
12
|
-
if (route.provider === "google") return google.
|
|
12
|
+
if (route.provider === "google") return google.decisionModel(id);
|
|
13
13
|
const { typeSafeAi } = await import("@ai-sdk/typesafe-ai");
|
|
14
|
-
return typeSafeAi.
|
|
14
|
+
return typeSafeAi.decisionModel(id);
|
|
15
15
|
}
|
|
16
16
|
async function scoreWithModel(model, question, state, providerOptions, signal) {
|
|
17
|
-
const result = await
|
|
17
|
+
const result = await experimental_decide({
|
|
18
18
|
model: await resolveModel(model),
|
|
19
19
|
state,
|
|
20
20
|
questions: { answer: {
|
|
@@ -41,4 +41,4 @@ async function scoreWithModel(model, question, state, providerOptions, signal) {
|
|
|
41
41
|
//#endregion
|
|
42
42
|
export { scoreWithModel };
|
|
43
43
|
|
|
44
|
-
//# sourceMappingURL=model-
|
|
44
|
+
//# sourceMappingURL=model-DEQo3FkZ.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-DEQo3FkZ.js","names":["decide"],"sources":["../src/judge/model.ts"],"mappings":";;;;;;AASA,eAAe,aACb,OACwB;CACxB,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,MAAM,QAAQ,WAAW,KAAK;CAC9B,IAAI,MAAM,SAAS,WAAW,OAAO;CACrC,MAAM,KAAK,MAAM,MAAM,MAAM,SAAS,SAAS,CAAC;CAChD,IAAI,MAAM,aAAa,UAAU,OAAO,OAAO,cAAc,EAAE;CAC/D,MAAM,EAAE,eAAe,MAAM,OAAO;CACpC,OAAO,WAAW,cAAc,EAAE;AACpC;AAEA,eAAsB,eACpB,OACA,UACA,OACA,iBACA,QAC+D;CAC/D,MAAM,SAAS,MAAMA,oBAAO;EAC1B,OAAO,MAAM,aAAa,KAAK;EAC/B;EACA,WAAW,EAAE,QAAQ;GAAE,MAAM;GAAW,cAAc;EAAS,EAAE;EACjE;EACA,aAAa;EAEb,YAAY;CACd,CAAC;CACD,MAAM,EAAE,SAAS,OAAO,UAAU,UAAU,qBAAqB;CACjE,WACE,SACA,qBACA;EAAE;EAAS;EAAO;EAAU;EAAU;CAAiB,GACvD,yBACF;CACA,OAAO;EAAE,OAAO,OAAO,QAAQ,OAAO;EAAa,eAAe,OAAO,UAAU;CAAQ;AAC7F"}
|