vskill 0.5.21 → 0.5.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/eval/sweep.d.ts +1 -0
- package/dist/commands/eval/sweep.js +78 -18
- package/dist/commands/eval/sweep.js.map +1 -1
- package/dist/commands/eval.d.ts +1 -0
- package/dist/commands/eval.js +1 -0
- package/dist/commands/eval.js.map +1 -1
- package/dist/eval/activation-history.d.ts +21 -0
- package/dist/eval/activation-history.js +41 -0
- package/dist/eval/activation-history.js.map +1 -0
- package/dist/eval-server/api-routes.js +104 -1
- package/dist/eval-server/api-routes.js.map +1 -1
- package/dist/eval-server/skill-create-routes.d.ts +11 -0
- package/dist/eval-server/skill-create-routes.js +131 -4
- package/dist/eval-server/skill-create-routes.js.map +1 -1
- package/dist/eval-server/sweep-routes.js +1 -0
- package/dist/eval-server/sweep-routes.js.map +1 -1
- package/dist/eval-server/sweep-runner.d.ts +33 -0
- package/dist/eval-server/sweep-runner.js +233 -84
- package/dist/eval-server/sweep-runner.js.map +1 -1
- package/dist/eval-ui/assets/index-BM579QSw.js +73 -0
- package/dist/eval-ui/assets/index-DcbzllbY.css +1 -0
- package/dist/eval-ui/index.html +2 -2
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/eval-ui/assets/index-C9_Pey9T.css +0 -1
- package/dist/eval-ui/assets/index-KfkLPyh3.js +0 -74
|
@@ -8,6 +8,8 @@ export interface ModelSpec {
|
|
|
8
8
|
export interface ModelStats {
|
|
9
9
|
mean: number;
|
|
10
10
|
stddev: number;
|
|
11
|
+
median?: number;
|
|
12
|
+
ci95?: [number, number];
|
|
11
13
|
}
|
|
12
14
|
export interface ModelResult {
|
|
13
15
|
provider: string;
|
|
@@ -22,6 +24,10 @@ export interface ModelResult {
|
|
|
22
24
|
status: "complete" | "error";
|
|
23
25
|
errorMessage: string | null;
|
|
24
26
|
caseResults: BenchmarkCase[][];
|
|
27
|
+
baselinePassRate?: ModelStats;
|
|
28
|
+
skillDelta?: ModelStats;
|
|
29
|
+
amplificationPct?: number;
|
|
30
|
+
compositeScore?: number;
|
|
25
31
|
}
|
|
26
32
|
export interface SweepResult {
|
|
27
33
|
sweepId: string;
|
|
@@ -29,6 +35,10 @@ export interface SweepResult {
|
|
|
29
35
|
judge: string;
|
|
30
36
|
runs: number;
|
|
31
37
|
models: ModelResult[];
|
|
38
|
+
baselineEnabled?: boolean;
|
|
39
|
+
skillQualityScore?: number;
|
|
40
|
+
skillQualityRating?: "excellent" | "good" | "marginal" | "minimal" | "harmful";
|
|
41
|
+
judgeBiasWarning?: string;
|
|
32
42
|
}
|
|
33
43
|
export interface SweepOpts {
|
|
34
44
|
skillDir: string;
|
|
@@ -39,6 +49,8 @@ export interface SweepOpts {
|
|
|
39
49
|
judge: string;
|
|
40
50
|
runs: number;
|
|
41
51
|
concurrency: number;
|
|
52
|
+
baseline?: boolean;
|
|
53
|
+
baselinePrompt?: string;
|
|
42
54
|
}
|
|
43
55
|
export type SweepSSEEvent = {
|
|
44
56
|
type: "sweep_start";
|
|
@@ -46,6 +58,7 @@ export type SweepSSEEvent = {
|
|
|
46
58
|
totalModels: number;
|
|
47
59
|
runs: number;
|
|
48
60
|
judge: string;
|
|
61
|
+
baseline: boolean;
|
|
49
62
|
};
|
|
50
63
|
} | {
|
|
51
64
|
type: "sweep_model_start";
|
|
@@ -64,6 +77,7 @@ export type SweepSSEEvent = {
|
|
|
64
77
|
run: number;
|
|
65
78
|
totalRuns: number;
|
|
66
79
|
percentComplete: number;
|
|
80
|
+
phase?: "skill" | "baseline";
|
|
67
81
|
};
|
|
68
82
|
} | {
|
|
69
83
|
type: "sweep_model_complete";
|
|
@@ -72,8 +86,18 @@ export type SweepSSEEvent = {
|
|
|
72
86
|
provider: string;
|
|
73
87
|
status: "complete" | "error";
|
|
74
88
|
passRate?: ModelStats;
|
|
89
|
+
baselinePassRate?: ModelStats;
|
|
90
|
+
skillDelta?: ModelStats;
|
|
91
|
+
amplificationPct?: number;
|
|
75
92
|
errorMessage?: string;
|
|
76
93
|
};
|
|
94
|
+
} | {
|
|
95
|
+
type: "sweep_judge_bias_warning";
|
|
96
|
+
data: {
|
|
97
|
+
judge: string;
|
|
98
|
+
matchedModel: string;
|
|
99
|
+
warning: string;
|
|
100
|
+
};
|
|
77
101
|
} | {
|
|
78
102
|
type: "sweep_complete";
|
|
79
103
|
data: SweepResult;
|
|
@@ -85,7 +109,16 @@ export type SweepSSEEvent = {
|
|
|
85
109
|
export declare function parseModelSpec(spec: string): ModelSpec;
|
|
86
110
|
export declare function computeMean(values: number[]): number;
|
|
87
111
|
export declare function computeStddev(values: number[], mean: number): number;
|
|
112
|
+
export declare function computeMedian(values: number[]): number;
|
|
113
|
+
export declare function computeCI95(values: number[]): [number, number] | undefined;
|
|
88
114
|
export declare function computeStats(values: number[]): ModelStats;
|
|
115
|
+
export declare function detectJudgeBias(judgeSpec: string, modelSpecs: string[]): string | undefined;
|
|
116
|
+
export declare function computeCompositeScore(passRate: ModelStats, runs: number): number;
|
|
117
|
+
export type SkillQualityRating = "excellent" | "good" | "marginal" | "minimal" | "harmful";
|
|
118
|
+
export declare function computeSkillQualityScore(amplifications: number[]): {
|
|
119
|
+
score: number;
|
|
120
|
+
rating: SkillQualityRating;
|
|
121
|
+
};
|
|
89
122
|
export declare function aggregateRuns(results: BenchmarkResult[], provider: string, model: string): Omit<ModelResult, "status" | "errorMessage">;
|
|
90
123
|
export declare function runSweep(opts: SweepOpts): AsyncGenerator<SweepSSEEvent>;
|
|
91
124
|
export declare function writeLeaderboard(skillDir: string, result: SweepResult): Promise<string>;
|
|
@@ -44,9 +44,84 @@ export function computeStddev(values, mean) {
|
|
|
44
44
|
const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
|
|
45
45
|
return Math.sqrt(variance);
|
|
46
46
|
}
|
|
47
|
+
export function computeMedian(values) {
|
|
48
|
+
if (values.length === 0)
|
|
49
|
+
return 0;
|
|
50
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
51
|
+
const mid = Math.floor(sorted.length / 2);
|
|
52
|
+
return sorted.length % 2 !== 0 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
|
|
53
|
+
}
|
|
54
|
+
export function computeCI95(values) {
|
|
55
|
+
if (values.length < 2)
|
|
56
|
+
return undefined;
|
|
57
|
+
const mean = computeMean(values);
|
|
58
|
+
const stddev = computeStddev(values, mean);
|
|
59
|
+
// t-distribution critical value for 95% CI (approximation for small samples)
|
|
60
|
+
const tCritical = values.length <= 5 ? 2.776 : values.length <= 10 ? 2.262 : 1.96;
|
|
61
|
+
const margin = tCritical * (stddev / Math.sqrt(values.length));
|
|
62
|
+
return [Math.max(0, mean - margin), Math.min(1, mean + margin)];
|
|
63
|
+
}
|
|
47
64
|
export function computeStats(values) {
|
|
48
65
|
const mean = computeMean(values);
|
|
49
|
-
return {
|
|
66
|
+
return {
|
|
67
|
+
mean,
|
|
68
|
+
stddev: computeStddev(values, mean),
|
|
69
|
+
median: computeMedian(values),
|
|
70
|
+
ci95: computeCI95(values),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
// ---------------------------------------------------------------------------
|
|
74
|
+
// Judge bias detection
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
export function detectJudgeBias(judgeSpec, modelSpecs) {
|
|
77
|
+
const judgeParts = judgeSpec.split("/");
|
|
78
|
+
const judgeModel = judgeParts.slice(1).join("/").toLowerCase();
|
|
79
|
+
const judgeProvider = judgeParts[0].toLowerCase();
|
|
80
|
+
for (const spec of modelSpecs) {
|
|
81
|
+
const parts = spec.split("/");
|
|
82
|
+
const modelName = parts.slice(1).join("/").toLowerCase();
|
|
83
|
+
const modelProvider = parts[0].toLowerCase();
|
|
84
|
+
// Exact match
|
|
85
|
+
if (judgeModel === modelName && judgeProvider === modelProvider) {
|
|
86
|
+
return `Judge model "${judgeSpec}" is identical to competitor model "${spec}". Self-preference bias is likely. Consider using a different judge model.`;
|
|
87
|
+
}
|
|
88
|
+
// Same provider family (e.g., anthropic/claude-sonnet judging anthropic/claude-opus)
|
|
89
|
+
if (judgeProvider === modelProvider && judgeModel !== modelName) {
|
|
90
|
+
// Extract family prefix: "claude-sonnet-4" → "claude", "meta-llama/llama-3.1-70b" → "meta-llama/llama"
|
|
91
|
+
const extractFamily = (m) => m.split(/[-_]/)[0];
|
|
92
|
+
const judgeFamily = extractFamily(judgeModel);
|
|
93
|
+
const modelFamily = extractFamily(modelName);
|
|
94
|
+
if (judgeFamily === modelFamily && judgeFamily.length > 0) {
|
|
95
|
+
return `Judge "${judgeSpec}" is from the same model family as "${spec}". Family-preference bias is possible. Consider a judge from a different provider.`;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
// ---------------------------------------------------------------------------
|
|
102
|
+
// Composite ranking score
|
|
103
|
+
// ---------------------------------------------------------------------------
|
|
104
|
+
export function computeCompositeScore(passRate, runs) {
|
|
105
|
+
const confidence = Math.min(1, runs / 5);
|
|
106
|
+
const stability = 1 - Math.min(1, passRate.stddev * 5); // lower stddev = higher stability
|
|
107
|
+
return passRate.mean * 0.7 + stability * 0.15 + confidence * 0.15;
|
|
108
|
+
}
|
|
109
|
+
export function computeSkillQualityScore(amplifications) {
|
|
110
|
+
if (amplifications.length === 0)
|
|
111
|
+
return { score: 0, rating: "minimal" };
|
|
112
|
+
const score = computeMedian(amplifications);
|
|
113
|
+
let rating;
|
|
114
|
+
if (score >= 20)
|
|
115
|
+
rating = "excellent";
|
|
116
|
+
else if (score >= 10)
|
|
117
|
+
rating = "good";
|
|
118
|
+
else if (score >= 5)
|
|
119
|
+
rating = "marginal";
|
|
120
|
+
else if (score >= 0)
|
|
121
|
+
rating = "minimal";
|
|
122
|
+
else
|
|
123
|
+
rating = "harmful";
|
|
124
|
+
return { score, rating };
|
|
50
125
|
}
|
|
51
126
|
// ---------------------------------------------------------------------------
|
|
52
127
|
// Aggregate multiple benchmark runs into ModelResult stats
|
|
@@ -73,13 +148,112 @@ export function aggregateRuns(results, provider, model) {
|
|
|
73
148
|
// ---------------------------------------------------------------------------
|
|
74
149
|
// Sweep runner — yields SSE events as an async generator
|
|
75
150
|
// ---------------------------------------------------------------------------
|
|
151
|
+
/**
|
|
152
|
+
* Run a single model through all eval cases with a given system prompt.
|
|
153
|
+
* Returns aggregated BenchmarkResult[] (one per run).
|
|
154
|
+
*/
|
|
155
|
+
async function* runModelCases(client, judgeSpec, systemPrompt, evalCases, skillName, spec, runs, modelIndex, totalModels, totalPhases, phaseOffset, phase) {
|
|
156
|
+
const runResults = [];
|
|
157
|
+
const totalSteps = totalModels * totalPhases * runs * evalCases.length;
|
|
158
|
+
for (let run = 0; run < runs; run++) {
|
|
159
|
+
const cases = [];
|
|
160
|
+
for (let ci = 0; ci < evalCases.length; ci++) {
|
|
161
|
+
const evalCase = evalCases[ci];
|
|
162
|
+
const step = modelIndex * totalPhases * runs * evalCases.length
|
|
163
|
+
+ phaseOffset * runs * evalCases.length
|
|
164
|
+
+ run * evalCases.length + ci + 1;
|
|
165
|
+
const percentComplete = Math.round((step / totalSteps) * 100);
|
|
166
|
+
yield {
|
|
167
|
+
type: "sweep_model_progress",
|
|
168
|
+
data: {
|
|
169
|
+
model: spec.model,
|
|
170
|
+
currentCase: ci + 1,
|
|
171
|
+
totalCases: evalCases.length,
|
|
172
|
+
run: run + 1,
|
|
173
|
+
totalRuns: runs,
|
|
174
|
+
percentComplete,
|
|
175
|
+
phase,
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
try {
|
|
179
|
+
const genResult = await client.generate(systemPrompt, evalCase.prompt);
|
|
180
|
+
const benchCase = {
|
|
181
|
+
eval_id: evalCase.id,
|
|
182
|
+
eval_name: evalCase.name,
|
|
183
|
+
status: "pass",
|
|
184
|
+
error_message: null,
|
|
185
|
+
pass_rate: 1,
|
|
186
|
+
durationMs: genResult.durationMs,
|
|
187
|
+
tokens: (genResult.inputTokens ?? 0) + (genResult.outputTokens ?? 0) || null,
|
|
188
|
+
inputTokens: genResult.inputTokens,
|
|
189
|
+
outputTokens: genResult.outputTokens,
|
|
190
|
+
output: genResult.text,
|
|
191
|
+
assertions: [],
|
|
192
|
+
};
|
|
193
|
+
benchCase.cost = genResult.cost;
|
|
194
|
+
if (evalCase.assertions.length > 0) {
|
|
195
|
+
const judgeClient = createLlmClient({
|
|
196
|
+
provider: judgeSpec.provider,
|
|
197
|
+
model: judgeSpec.model,
|
|
198
|
+
});
|
|
199
|
+
const { judgeAssertion } = await import("../eval/judge.js");
|
|
200
|
+
const assertionResults = await Promise.all(evalCase.assertions.map((a) => judgeAssertion(genResult.text, a, judgeClient)));
|
|
201
|
+
benchCase.assertions = assertionResults;
|
|
202
|
+
benchCase.pass_rate =
|
|
203
|
+
assertionResults.length > 0
|
|
204
|
+
? assertionResults.filter((a) => a.pass).length / assertionResults.length
|
|
205
|
+
: 0;
|
|
206
|
+
benchCase.status = assertionResults.every((a) => a.pass) ? "pass" : "fail";
|
|
207
|
+
}
|
|
208
|
+
cases.push(benchCase);
|
|
209
|
+
}
|
|
210
|
+
catch (err) {
|
|
211
|
+
cases.push({
|
|
212
|
+
eval_id: evalCase.id,
|
|
213
|
+
eval_name: evalCase.name,
|
|
214
|
+
status: "error",
|
|
215
|
+
error_message: err instanceof Error ? err.message : String(err),
|
|
216
|
+
pass_rate: 0,
|
|
217
|
+
assertions: [],
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
const totalAssertions = cases.reduce((s, c) => s + c.assertions.length, 0);
|
|
222
|
+
const passedAssertions = cases.reduce((s, c) => s + c.assertions.filter((a) => a.pass).length, 0);
|
|
223
|
+
runResults.push({
|
|
224
|
+
timestamp: new Date().toISOString(),
|
|
225
|
+
model: spec.model,
|
|
226
|
+
skill_name: skillName,
|
|
227
|
+
cases,
|
|
228
|
+
overall_pass_rate: totalAssertions > 0 ? passedAssertions / totalAssertions : 0,
|
|
229
|
+
type: "benchmark",
|
|
230
|
+
provider: spec.provider,
|
|
231
|
+
totalDurationMs: cases.reduce((s, c) => s + (c.durationMs ?? 0), 0),
|
|
232
|
+
totalInputTokens: cases.reduce((s, c) => s + (c.inputTokens ?? 0), 0),
|
|
233
|
+
totalOutputTokens: cases.reduce((s, c) => s + (c.outputTokens ?? 0), 0),
|
|
234
|
+
scope: "bulk",
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
return runResults;
|
|
238
|
+
}
|
|
76
239
|
export async function* runSweep(opts) {
|
|
77
240
|
const { skillDir, skillName, systemPrompt, evalCases, models, judge, runs, concurrency } = opts;
|
|
241
|
+
const baseline = opts.baseline ?? false;
|
|
242
|
+
const baselinePrompt = opts.baselinePrompt ?? "You are a helpful AI assistant.";
|
|
78
243
|
const modelSpecs = models.map(parseModelSpec);
|
|
79
244
|
const judgeSpec = parseModelSpec(judge);
|
|
245
|
+
const totalPhases = baseline ? 2 : 1;
|
|
246
|
+
// Judge bias detection
|
|
247
|
+
const judgeBiasWarning = detectJudgeBias(judge, models);
|
|
248
|
+
if (judgeBiasWarning) {
|
|
249
|
+
yield {
|
|
250
|
+
type: "sweep_judge_bias_warning",
|
|
251
|
+
data: { judge, matchedModel: models.find((m) => judgeBiasWarning.includes(m)) ?? models[0], warning: judgeBiasWarning },
|
|
252
|
+
};
|
|
253
|
+
}
|
|
80
254
|
yield {
|
|
81
255
|
type: "sweep_start",
|
|
82
|
-
data: { totalModels: modelSpecs.length, runs, judge },
|
|
256
|
+
data: { totalModels: modelSpecs.length, runs, judge, baseline },
|
|
83
257
|
};
|
|
84
258
|
const sweepResults = [];
|
|
85
259
|
for (let mi = 0; mi < modelSpecs.length; mi++) {
|
|
@@ -95,92 +269,47 @@ export async function* runSweep(opts) {
|
|
|
95
269
|
};
|
|
96
270
|
try {
|
|
97
271
|
const client = createLlmClient({ provider: spec.provider, model: spec.model });
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
yield {
|
|
106
|
-
type: "sweep_model_progress",
|
|
107
|
-
data: {
|
|
108
|
-
model: spec.model,
|
|
109
|
-
currentCase: ci + 1,
|
|
110
|
-
totalCases: evalCases.length,
|
|
111
|
-
run: run + 1,
|
|
112
|
-
totalRuns: runs,
|
|
113
|
-
percentComplete,
|
|
114
|
-
},
|
|
115
|
-
};
|
|
116
|
-
try {
|
|
117
|
-
const genResult = await client.generate(systemPrompt, evalCase.prompt);
|
|
118
|
-
// Create a basic benchmark case from the generation result
|
|
119
|
-
const benchCase = {
|
|
120
|
-
eval_id: evalCase.id,
|
|
121
|
-
eval_name: evalCase.name,
|
|
122
|
-
status: "pass",
|
|
123
|
-
error_message: null,
|
|
124
|
-
pass_rate: 1,
|
|
125
|
-
durationMs: genResult.durationMs,
|
|
126
|
-
tokens: (genResult.inputTokens ?? 0) + (genResult.outputTokens ?? 0) || null,
|
|
127
|
-
inputTokens: genResult.inputTokens,
|
|
128
|
-
outputTokens: genResult.outputTokens,
|
|
129
|
-
output: genResult.text,
|
|
130
|
-
assertions: [],
|
|
131
|
-
};
|
|
132
|
-
// Attach cost if available
|
|
133
|
-
benchCase.cost = genResult.cost;
|
|
134
|
-
// Judge assertions if a judge is configured
|
|
135
|
-
if (evalCase.assertions.length > 0) {
|
|
136
|
-
const judgeClient = createLlmClient({
|
|
137
|
-
provider: judgeSpec.provider,
|
|
138
|
-
model: judgeSpec.model,
|
|
139
|
-
});
|
|
140
|
-
const { judgeAssertion } = await import("../eval/judge.js");
|
|
141
|
-
const assertionResults = await Promise.all(evalCase.assertions.map((a) => judgeAssertion(genResult.text, a, judgeClient)));
|
|
142
|
-
benchCase.assertions = assertionResults;
|
|
143
|
-
benchCase.pass_rate =
|
|
144
|
-
assertionResults.length > 0
|
|
145
|
-
? assertionResults.filter((a) => a.pass).length / assertionResults.length
|
|
146
|
-
: 0;
|
|
147
|
-
benchCase.status = assertionResults.every((a) => a.pass) ? "pass" : "fail";
|
|
148
|
-
}
|
|
149
|
-
cases.push(benchCase);
|
|
150
|
-
}
|
|
151
|
-
catch (err) {
|
|
152
|
-
cases.push({
|
|
153
|
-
eval_id: evalCase.id,
|
|
154
|
-
eval_name: evalCase.name,
|
|
155
|
-
status: "error",
|
|
156
|
-
error_message: err instanceof Error ? err.message : String(err),
|
|
157
|
-
pass_rate: 0,
|
|
158
|
-
assertions: [],
|
|
159
|
-
});
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
const totalAssertions = cases.reduce((s, c) => s + c.assertions.length, 0);
|
|
163
|
-
const passedAssertions = cases.reduce((s, c) => s + c.assertions.filter((a) => a.pass).length, 0);
|
|
164
|
-
runResults.push({
|
|
165
|
-
timestamp: new Date().toISOString(),
|
|
166
|
-
model: spec.model,
|
|
167
|
-
skill_name: skillName,
|
|
168
|
-
cases,
|
|
169
|
-
overall_pass_rate: totalAssertions > 0 ? passedAssertions / totalAssertions : 0,
|
|
170
|
-
type: "benchmark",
|
|
171
|
-
provider: spec.provider,
|
|
172
|
-
totalDurationMs: cases.reduce((s, c) => s + (c.durationMs ?? 0), 0),
|
|
173
|
-
totalInputTokens: cases.reduce((s, c) => s + (c.inputTokens ?? 0), 0),
|
|
174
|
-
totalOutputTokens: cases.reduce((s, c) => s + (c.outputTokens ?? 0), 0),
|
|
175
|
-
scope: "bulk",
|
|
176
|
-
});
|
|
272
|
+
// Phase 1: Run WITH skill prompt
|
|
273
|
+
const skillGen = runModelCases(client, judgeSpec, systemPrompt, evalCases, skillName, spec, runs, mi, modelSpecs.length, totalPhases, 0, "skill");
|
|
274
|
+
let skillRunResults = [];
|
|
275
|
+
let next = await skillGen.next();
|
|
276
|
+
while (!next.done) {
|
|
277
|
+
yield next.value;
|
|
278
|
+
next = await skillGen.next();
|
|
177
279
|
}
|
|
178
|
-
|
|
179
|
-
|
|
280
|
+
skillRunResults = next.value;
|
|
281
|
+
const aggregated = aggregateRuns(skillRunResults, spec.provider, spec.model);
|
|
282
|
+
const modelResult = {
|
|
180
283
|
...aggregated,
|
|
284
|
+
compositeScore: computeCompositeScore(aggregated.passRate, runs),
|
|
181
285
|
status: "complete",
|
|
182
286
|
errorMessage: null,
|
|
183
|
-
}
|
|
287
|
+
};
|
|
288
|
+
// Phase 2: Run WITHOUT skill prompt (baseline)
|
|
289
|
+
if (baseline) {
|
|
290
|
+
const baselineGen = runModelCases(client, judgeSpec, baselinePrompt, evalCases, skillName, spec, runs, mi, modelSpecs.length, totalPhases, 1, "baseline");
|
|
291
|
+
let baselineRunResults = [];
|
|
292
|
+
let bNext = await baselineGen.next();
|
|
293
|
+
while (!bNext.done) {
|
|
294
|
+
yield bNext.value;
|
|
295
|
+
bNext = await baselineGen.next();
|
|
296
|
+
}
|
|
297
|
+
baselineRunResults = bNext.value;
|
|
298
|
+
const baselineAgg = aggregateRuns(baselineRunResults, spec.provider, spec.model);
|
|
299
|
+
modelResult.baselinePassRate = baselineAgg.passRate;
|
|
300
|
+
// Compute skill delta
|
|
301
|
+
const deltaValues = skillRunResults.map((sr, i) => {
|
|
302
|
+
const br = baselineRunResults[i];
|
|
303
|
+
return (sr.overall_pass_rate ?? 0) - (br?.overall_pass_rate ?? 0);
|
|
304
|
+
});
|
|
305
|
+
modelResult.skillDelta = computeStats(deltaValues);
|
|
306
|
+
// Compute amplification percentage
|
|
307
|
+
const baselineMean = baselineAgg.passRate.mean;
|
|
308
|
+
modelResult.amplificationPct = baselineMean > 0
|
|
309
|
+
? ((aggregated.passRate.mean - baselineMean) / baselineMean) * 100
|
|
310
|
+
: aggregated.passRate.mean > 0 ? Infinity : 0;
|
|
311
|
+
}
|
|
312
|
+
sweepResults.push(modelResult);
|
|
184
313
|
yield {
|
|
185
314
|
type: "sweep_model_complete",
|
|
186
315
|
data: {
|
|
@@ -188,6 +317,9 @@ export async function* runSweep(opts) {
|
|
|
188
317
|
provider: spec.provider,
|
|
189
318
|
status: "complete",
|
|
190
319
|
passRate: aggregated.passRate,
|
|
320
|
+
baselinePassRate: modelResult.baselinePassRate,
|
|
321
|
+
skillDelta: modelResult.skillDelta,
|
|
322
|
+
amplificationPct: modelResult.amplificationPct,
|
|
191
323
|
},
|
|
192
324
|
};
|
|
193
325
|
}
|
|
@@ -215,6 +347,19 @@ export async function* runSweep(opts) {
|
|
|
215
347
|
};
|
|
216
348
|
}
|
|
217
349
|
}
|
|
350
|
+
// Compute skill quality score if baseline was enabled
|
|
351
|
+
let skillQualityScore;
|
|
352
|
+
let skillQualityRating;
|
|
353
|
+
if (baseline) {
|
|
354
|
+
const amplifications = sweepResults
|
|
355
|
+
.filter((m) => m.status === "complete" && m.amplificationPct != null && isFinite(m.amplificationPct))
|
|
356
|
+
.map((m) => m.amplificationPct);
|
|
357
|
+
if (amplifications.length > 0) {
|
|
358
|
+
const quality = computeSkillQualityScore(amplifications);
|
|
359
|
+
skillQualityScore = quality.score;
|
|
360
|
+
skillQualityRating = quality.rating;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
218
363
|
// Write leaderboard file
|
|
219
364
|
const sweepResult = {
|
|
220
365
|
sweepId: randomUUID(),
|
|
@@ -222,6 +367,10 @@ export async function* runSweep(opts) {
|
|
|
222
367
|
judge,
|
|
223
368
|
runs,
|
|
224
369
|
models: sweepResults,
|
|
370
|
+
baselineEnabled: baseline || undefined,
|
|
371
|
+
skillQualityScore,
|
|
372
|
+
skillQualityRating,
|
|
373
|
+
judgeBiasWarning: judgeBiasWarning || undefined,
|
|
225
374
|
};
|
|
226
375
|
await writeLeaderboard(skillDir, sweepResult);
|
|
227
376
|
yield { type: "sweep_complete", data: sweepResult };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sweep-runner.js","sourceRoot":"","sources":["../../src/eval-server/sweep-runner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAyDjD,8EAA8E;AAC9E,iDAAiD;AACjD,8EAA8E;AAE9E,MAAM,eAAe,GAAgB,IAAI,GAAG,CAAC;IAC3C,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY;CAC7E,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,qCAAqC,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;IAEvC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,wBAAwB,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,oBAAoB,IAAI,uBAAuB,CAAC,GAAG,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC9G,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,QAAwB,EAAE,KAAK,EAAE,CAAC;AACvD,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,UAAU,WAAW,CAAC,MAAgB;IAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAgB,EAAE,IAAY;IAC1D,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACrF,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAgB;IAC3C,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACjC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;AACvD,CAAC;AAED,8EAA8E;AAC9E,2DAA2D;AAC3D,8EAA8E;AAE9E,MAAM,UAAU,aAAa,CAC3B,OAA0B,EAC1B,QAAgB,EAChB,KAAa;IAEb,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9B,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAE,CAAS,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAC5D,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEnE,OAAO;QACL,QAAQ;QACR,KAAK;QACL,QAAQ,EAAE,YAAY,CAAC,SAAS,CAAC;QACjC,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,YAAY,CAAC,SAAS,CAAC;QACjC,IAAI,EAAE;YACJ,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;SACrD;QACD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;KACzC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,yDAAyD;AACzD,8EAA8E;AAE9E,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAe;IAC7C,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IAEhG,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAExC,MAAM;QACJ,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,EAAE,WAAW,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE;KACtD,CAAC;IAEF,MAAM,YAAY,GAAkB,EAAE,CAAC;IAEvC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;QAE5B,MAAM;YACJ,IAAI,EAAE,mBAAmB;YACzB,IAAI,EAAE;gBACJ,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,EAAE;gBACd,WAAW,EAAE,UAAU,CAAC,MAAM;aAC/B;SACF,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/E,MAAM,UAAU,GAAsB,EAAE,CAAC;YAEzC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAoB,EAAE,CAAC;gBAElC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;oBAC7C,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;oBAC/B,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAChC,CAAC,CAAC,EAAE,GAAG,IAAI,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG,GAAG,SAAS,CAAC,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC;wBAC/D,CAAC,UAAU,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,GAAG,CACvD,CAAC;oBAEF,MAAM;wBACJ,IAAI,EAAE,sBAAsB;wBAC5B,IAAI,EAAE;4BACJ,KAAK,EAAE,IAAI,CAAC,KAAK;4BACjB,WAAW,EAAE,EAAE,GAAG,CAAC;4BACnB,UAAU,EAAE,SAAS,CAAC,MAAM;4BAC5B,GAAG,EAAE,GAAG,GAAG,CAAC;4BACZ,SAAS,EAAE,IAAI;4BACf,eAAe;yBAChB;qBACF,CAAC;oBAEF,IAAI,CAAC;wBACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;wBACvE,2DAA2D;wBAC3D,MAAM,SAAS,GAAkB;4BAC/B,OAAO,EAAE,QAAQ,CAAC,EAAE;4BACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;4BACxB,MAAM,EAAE,MAAM;4BACd,aAAa,EAAE,IAAI;4BACnB,SAAS,EAAE,CAAC;4BACZ,UAAU,EAAE,SAAS,CAAC,UAAU;4BAChC,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,IAAI;4BAC5E,WAAW,EAAE,SAAS,CAAC,WAAW;4BAClC,YAAY,EAAE,SAAS,CAAC,YAAY;4BACpC,MAAM,EAAE,SAAS,CAAC,IAAI;4BACtB,UAAU,EAAE,EAAE;yBACf,CAAC;wBACF,2BAA2B;wBAC1B,SAAiB,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;wBAEzC,4CAA4C;wBAC5C,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACnC,MAAM,WAAW,GAAG,eAAe,CAAC;gCAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ;gCAC5B,KAAK,EAAE,SAAS,CAAC,KAAK;6BACvB,CAAC,CAAC;4BACH,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;4BAE5D,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CACxC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5B,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,WAAW,CAAC,CAC/C,CACF,CAAC;4BACF,SAAS,CAAC,UAAU,GAAG,gBAAgB,CAAC;4BACxC,SAAS,CAAC,SAAS;gCACjB,gBAAgB,CAAC,MAAM,GAAG,CAAC;oCACzB,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM;oCACzE,CAAC,CAAC,CAAC,CAAC;4BACR,SAAS,CAAC,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;wBAC7E,CAAC;wBAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxB,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,KAAK,CAAC,IAAI,CAAC;4BACT,OAAO,EAAE,QAAQ,CAAC,EAAE;4BACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;4BACxB,MAAM,EAAE,OAAO;4BACf,aAAa,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;4BAC/D,SAAS,EAAE,CAAC;4BACZ,UAAU,EAAE,EAAE;yBACf,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAC3E,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CACnC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAC3D,CAAC;gBAEF,UAAU,CAAC,IAAI,CAAC;oBACd,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,SAAS;oBACrB,KAAK;oBACL,iBAAiB,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;oBAC/E,IAAI,EAAE,WAAW;oBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;oBACnE,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;oBACrE,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;oBACvE,KAAK,EAAE,MAAM;iBACd,CAAC,CAAC;YACL,CAAC;YAED,MAAM,UAAU,GAAG,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YACxE,YAAY,CAAC,IAAI,CAAC;gBAChB,GAAG,UAAU;gBACb,MAAM,EAAE,UAAU;gBAClB,YAAY,EAAE,IAAI;aACnB,CAAC,CAAC;YAEH,MAAM;gBACJ,IAAI,EAAE,sBAAsB;gBAC5B,IAAI,EAAE;oBACJ,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,MAAM,EAAE,UAAU;oBAClB,QAAQ,EAAE,UAAU,CAAC,QAAQ;iBAC9B;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,YAAY,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACtE,YAAY,CAAC,IAAI,CAAC;gBAChB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;gBAChC,WAAW,EAAE,IAAI;gBACjB,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;gBAChC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC9B,MAAM,EAAE,OAAO;gBACf,YAAY;gBACZ,WAAW,EAAE,EAAE;aAChB,CAAC,CAAC;YAEH,MAAM;gBACJ,IAAI,EAAE,sBAAsB;gBAC5B,IAAI,EAAE;oBACJ,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,MAAM,EAAE,OAAO;oBACf,YAAY;iBACb;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,WAAW,GAAgB;QAC/B,OAAO,EAAE,UAAU,EAAE;QACrB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK;QACL,IAAI;QACJ,MAAM,EAAE,YAAY;KACrB,CAAC;IAEF,MAAM,gBAAgB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IAE9C,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;AACtD,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,MAAmB;IAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9D,MAAM,KAAK,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEjD,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC;IAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3D,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB,EAAE,KAAK,GAAG,EAAE;IAChE,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QAC5E,MAAM,OAAO,GAAkB,EAAE,CAAC;QAElC,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;gBACpE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7E,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;QACrE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"sweep-runner.js","sourceRoot":"","sources":["../../src/eval-server/sweep-runner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2EAA2E;AAC3E,8EAA8E;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAuEjD,8EAA8E;AAC9E,iDAAiD;AACjD,8EAA8E;AAE9E,MAAM,eAAe,GAAgB,IAAI,GAAG,CAAC;IAC3C,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY;CAC7E,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,qCAAqC,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;IAEvC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,wBAAwB,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,oBAAoB,IAAI,uBAAuB,CAAC,GAAG,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC9G,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,QAAwB,EAAE,KAAK,EAAE,CAAC;AACvD,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,UAAU,WAAW,CAAC,MAAgB;IAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAgB,EAAE,IAAY;IAC1D,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACrF,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAgB;IAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAAgB;IAC1C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3C,6EAA6E;IAC7E,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAClF,MAAM,MAAM,GAAG,SAAS,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAgB;IAC3C,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACjC,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC;QACnC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC;QAC7B,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC;KAC1B,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E,MAAM,UAAU,eAAe,CAAC,SAAiB,EAAE,UAAoB;IACrE,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/D,MAAM,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAE7C,cAAc;QACd,IAAI,UAAU,KAAK,SAAS,IAAI,aAAa,KAAK,aAAa,EAAE,CAAC;YAChE,OAAO,gBAAgB,SAAS,uCAAuC,IAAI,4EAA4E,CAAC;QAC1J,CAAC;QACD,qFAAqF;QACrF,IAAI,aAAa,KAAK,aAAa,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAChE,uGAAuG;YACvG,MAAM,aAAa,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACxD,MAAM,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YAC9C,MAAM,WAAW,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;YAC7C,IAAI,WAAW,KAAK,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1D,OAAO,UAAU,SAAS,uCAAuC,IAAI,oFAAoF,CAAC;YAC5J,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAE9E,MAAM,UAAU,qBAAqB,CAAC,QAAoB,EAAE,IAAY;IACtE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,kCAAkC;IAC1F,OAAO,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,SAAS,GAAG,IAAI,GAAG,UAAU,GAAG,IAAI,CAAC;AACpE,CAAC;AAQD,MAAM,UAAU,wBAAwB,CAAC,cAAwB;IAC/D,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;IAC5C,IAAI,MAA0B,CAAC;IAC/B,IAAI,KAAK,IAAI,EAAE;QAAE,MAAM,GAAG,WAAW,CAAC;SACjC,IAAI,KAAK,IAAI,EAAE;QAAE,MAAM,GAAG,MAAM,CAAC;SACjC,IAAI,KAAK,IAAI,CAAC;QAAE,MAAM,GAAG,UAAU,CAAC;SACpC,IAAI,KAAK,IAAI,CAAC;QAAE,MAAM,GAAG,SAAS,CAAC;;QACnC,MAAM,GAAG,SAAS,CAAC;IACxB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC;AAED,8EAA8E;AAC9E,2DAA2D;AAC3D,8EAA8E;AAE9E,MAAM,UAAU,aAAa,CAC3B,OAA0B,EAC1B,QAAgB,EAChB,KAAa;IAEb,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9B,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAE,CAAS,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAC5D,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEnE,OAAO;QACL,QAAQ;QACR,KAAK;QACL,QAAQ,EAAE,YAAY,CAAC,SAAS,CAAC;QACjC,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,YAAY,CAAC,SAAS,CAAC;QACjC,IAAI,EAAE;YACJ,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;SACrD;QACD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;KACzC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,yDAAyD;AACzD,8EAA8E;AAE9E;;;GAGG;AACH,KAAK,SAAS,CAAC,CAAC,aAAa,CAC3B,MAA0C,EAC1C,SAAoB,EACpB,YAAoB,EACpB,SAAqB,EACrB,SAAiB,EACjB,IAAe,EACf,IAAY,EACZ,UAAkB,EAClB,WAAmB,EACnB,WAAmB,EACnB,WAAmB,EACnB,KAA2B;IAE3B,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,UAAU,GAAG,WAAW,GAAG,WAAW,GAAG,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC;IAEvE,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAoB,EAAE,CAAC;QAElC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;YAC/B,MAAM,IAAI,GAAG,UAAU,GAAG,WAAW,GAAG,IAAI,GAAG,SAAS,CAAC,MAAM;kBAC3D,WAAW,GAAG,IAAI,GAAG,SAAS,CAAC,MAAM;kBACrC,GAAG,GAAG,SAAS,CAAC,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC;YACpC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;YAE9D,MAAM;gBACJ,IAAI,EAAE,sBAAsB;gBAC5B,IAAI,EAAE;oBACJ,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,WAAW,EAAE,EAAE,GAAG,CAAC;oBACnB,UAAU,EAAE,SAAS,CAAC,MAAM;oBAC5B,GAAG,EAAE,GAAG,GAAG,CAAC;oBACZ,SAAS,EAAE,IAAI;oBACf,eAAe;oBACf,KAAK;iBACN;aACF,CAAC;YAEF,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACvE,MAAM,SAAS,GAAkB;oBAC/B,OAAO,EAAE,QAAQ,CAAC,EAAE;oBACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;oBACxB,MAAM,EAAE,MAAM;oBACd,aAAa,EAAE,IAAI;oBACnB,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,SAAS,CAAC,UAAU;oBAChC,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,IAAI,CAAC,CAAC,IAAI,IAAI;oBAC5E,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,YAAY,EAAE,SAAS,CAAC,YAAY;oBACpC,MAAM,EAAE,SAAS,CAAC,IAAI;oBACtB,UAAU,EAAE,EAAE;iBACf,CAAC;gBACD,SAAiB,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;gBAEzC,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,MAAM,WAAW,GAAG,eAAe,CAAC;wBAClC,QAAQ,EAAE,SAAS,CAAC,QAAQ;wBAC5B,KAAK,EAAE,SAAS,CAAC,KAAK;qBACvB,CAAC,CAAC;oBACH,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;oBAE5D,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CACxC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5B,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,WAAW,CAAC,CAC/C,CACF,CAAC;oBACF,SAAS,CAAC,UAAU,GAAG,gBAAgB,CAAC;oBACxC,SAAS,CAAC,SAAS;wBACjB,gBAAgB,CAAC,MAAM,GAAG,CAAC;4BACzB,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM;4BACzE,CAAC,CAAC,CAAC,CAAC;oBACR,SAAS,CAAC,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC7E,CAAC;gBAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,CAAC,IAAI,CAAC;oBACT,OAAO,EAAE,QAAQ,CAAC,EAAE;oBACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;oBACxB,MAAM,EAAE,OAAO;oBACf,aAAa,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;oBAC/D,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,EAAE;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3E,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CACnC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAC3D,CAAC;QAEF,UAAU,CAAC,IAAI,CAAC;YACd,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,SAAS;YACrB,KAAK;YACL,iBAAiB,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YAC/E,IAAI,EAAE,WAAW;YACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACnE,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACrE,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;YACvE,KAAK,EAAE,MAAM;SACd,CAAC,CAAC;IACL,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAe;IAC7C,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IACxC,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,iCAAiC,CAAC;IAEhF,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErC,uBAAuB;IACvB,MAAM,gBAAgB,GAAG,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACxD,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM;YACJ,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE;SACxH,CAAC;IACJ,CAAC;IAED,MAAM;QACJ,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,EAAE,WAAW,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE;KAChE,CAAC;IAEF,MAAM,YAAY,GAAkB,EAAE,CAAC;IAEvC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;QAE5B,MAAM;YACJ,IAAI,EAAE,mBAAmB;YACzB,IAAI,EAAE;gBACJ,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,EAAE;gBACd,WAAW,EAAE,UAAU,CAAC,MAAM;aAC/B;SACF,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAE/E,iCAAiC;YACjC,MAAM,QAAQ,GAAG,aAAa,CAC5B,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EACrD,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,CAC3D,CAAC;YACF,IAAI,eAAe,GAAsB,EAAE,CAAC;YAC5C,IAAI,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,KAAK,CAAC;gBACjB,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC/B,CAAC;YACD,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC;YAE7B,MAAM,UAAU,GAAG,aAAa,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7E,MAAM,WAAW,GAAgB;gBAC/B,GAAG,UAAU;gBACb,cAAc,EAAE,qBAAqB,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC;gBAChE,MAAM,EAAE,UAAU;gBAClB,YAAY,EAAE,IAAI;aACnB,CAAC;YAEF,+CAA+C;YAC/C,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,WAAW,GAAG,aAAa,CAC/B,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,EACvD,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,CAC9D,CAAC;gBACF,IAAI,kBAAkB,GAAsB,EAAE,CAAC;gBAC/C,IAAI,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;gBACrC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;oBACnB,MAAM,KAAK,CAAC,KAAK,CAAC;oBAClB,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;gBACnC,CAAC;gBACD,kBAAkB,GAAG,KAAK,CAAC,KAAK,CAAC;gBAEjC,MAAM,WAAW,GAAG,aAAa,CAAC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;gBACjF,WAAW,CAAC,gBAAgB,GAAG,WAAW,CAAC,QAAQ,CAAC;gBAEpD,sBAAsB;gBACtB,MAAM,WAAW,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;oBAChD,MAAM,EAAE,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;oBACjC,OAAO,CAAC,EAAE,CAAC,iBAAiB,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,iBAAiB,IAAI,CAAC,CAAC,CAAC;gBACpE,CAAC,CAAC,CAAC;gBACH,WAAW,CAAC,UAAU,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;gBAEnD,mCAAmC;gBACnC,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC/C,WAAW,CAAC,gBAAgB,GAAG,YAAY,GAAG,CAAC;oBAC7C,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,GAAG,YAAY,CAAC,GAAG,YAAY,CAAC,GAAG,GAAG;oBAClE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAE/B,MAAM;gBACJ,IAAI,EAAE,sBAAsB;gBAC5B,IAAI,EAAE;oBACJ,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,MAAM,EAAE,UAAU;oBAClB,QAAQ,EAAE,UAAU,CAAC,QAAQ;oBAC7B,gBAAgB,EAAE,WAAW,CAAC,gBAAgB;oBAC9C,UAAU,EAAE,WAAW,CAAC,UAAU;oBAClC,gBAAgB,EAAE,WAAW,CAAC,gBAAgB;iBAC/C;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,YAAY,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACtE,YAAY,CAAC,IAAI,CAAC;gBAChB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;gBAChC,WAAW,EAAE,IAAI;gBACjB,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;gBAChC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC9B,MAAM,EAAE,OAAO;gBACf,YAAY;gBACZ,WAAW,EAAE,EAAE;aAChB,CAAC,CAAC;YAEH,MAAM;gBACJ,IAAI,EAAE,sBAAsB;gBAC5B,IAAI,EAAE;oBACJ,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,MAAM,EAAE,OAAO;oBACf,YAAY;iBACb;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,sDAAsD;IACtD,IAAI,iBAAqC,CAAC;IAC1C,IAAI,kBAAkD,CAAC;IACvD,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,cAAc,GAAG,YAAY;aAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,gBAAgB,IAAI,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,gBAAiB,CAAC,CAAC;aACrG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAiB,CAAC,CAAC;QACnC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,wBAAwB,CAAC,cAAc,CAAC,CAAC;YACzD,iBAAiB,GAAG,OAAO,CAAC,KAAK,CAAC;YAClC,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC;QACtC,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,WAAW,GAAgB;QAC/B,OAAO,EAAE,UAAU,EAAE;QACrB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK;QACL,IAAI;QACJ,MAAM,EAAE,YAAY;QACpB,eAAe,EAAE,QAAQ,IAAI,SAAS;QACtC,iBAAiB;QACjB,kBAAkB;QAClB,gBAAgB,EAAE,gBAAgB,IAAI,SAAS;KAChD,CAAC;IAEF,MAAM,gBAAgB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IAE9C,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;AACtD,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,MAAmB;IAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9D,MAAM,KAAK,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEjD,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC;IAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3D,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB,EAAE,KAAK,GAAG,EAAE;IAChE,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QAC5E,MAAM,OAAO,GAAkB,EAAE,CAAC;QAElC,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;gBACpE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAgB,EAChB,SAAiB;IAEjB,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7E,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;QACrE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|