veryfront 0.1.884 → 0.1.886
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command-help.d.ts.map +1 -1
- package/esm/cli/commands/eval/command-help.js +5 -0
- package/esm/cli/commands/eval/command.d.ts.map +1 -1
- package/esm/cli/commands/eval/command.js +24 -0
- package/esm/cli/commands/eval/handler.d.ts +2 -0
- package/esm/cli/commands/eval/handler.d.ts.map +1 -1
- package/esm/cli/commands/eval/handler.js +2 -0
- package/esm/deno.js +2 -1
- package/esm/src/eval/index.d.ts +1 -1
- package/esm/src/eval/index.d.ts.map +1 -1
- package/esm/src/eval/runner.d.ts.map +1 -1
- package/esm/src/eval/runner.js +79 -1
- package/esm/src/eval/types.d.ts +9 -0
- package/esm/src/eval/types.d.ts.map +1 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,WA8CtB,CAAC"}
|
|
@@ -20,6 +20,10 @@ export const evalHelp = {
|
|
|
20
20
|
flag: "--junit <path>",
|
|
21
21
|
description: "Write a JUnit XML report to a file",
|
|
22
22
|
},
|
|
23
|
+
{
|
|
24
|
+
flag: "--export <ids>",
|
|
25
|
+
description: "Export the report through registered eval exporters",
|
|
26
|
+
},
|
|
23
27
|
{
|
|
24
28
|
flag: "--model <provider/model>",
|
|
25
29
|
description: "Override the target agent model",
|
|
@@ -37,6 +41,7 @@ export const evalHelp = {
|
|
|
37
41
|
"veryfront eval --list",
|
|
38
42
|
"veryfront eval deep-research",
|
|
39
43
|
"veryfront eval eval:deep-research --report .veryfront/evals/deep-research.json --junit .veryfront/evals/deep-research.xml",
|
|
44
|
+
"veryfront eval deep-research --export braintrust,langfuse --json",
|
|
40
45
|
"veryfront eval deep-research --json",
|
|
41
46
|
],
|
|
42
47
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAKV,UAAU,
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAKV,UAAU,EAGX,MAAM,4BAA4B,CAAC;AAWpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAiDF,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AA+ID,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA8FrE"}
|
|
@@ -165,6 +165,29 @@ function printReport(report) {
|
|
|
165
165
|
for (const metric of report.summary.metrics) {
|
|
166
166
|
cliLogger.info(` ${metric.name}: ${metric.passed}/${metric.passed + metric.failed} passed (${Math.round(metric.passRate * 100)}%)`);
|
|
167
167
|
}
|
|
168
|
+
for (const result of report.exports ?? []) {
|
|
169
|
+
if (result.ok) {
|
|
170
|
+
cliLogger.info(`Export ${result.exporterId}: ok`);
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
cliLogger.warn(`Export ${result.exporterId}: failed: ${result.error}`);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
function createEvalCliExportConfig(evalItem, options, projectDir) {
|
|
178
|
+
if (options.exporters.length === 0)
|
|
179
|
+
return undefined;
|
|
180
|
+
return {
|
|
181
|
+
exporterIds: options.exporters,
|
|
182
|
+
context: {
|
|
183
|
+
evalId: evalItem.definition.id,
|
|
184
|
+
sourcePath: displaySourcePath(evalItem.filePath, projectDir),
|
|
185
|
+
reportPath: options.report,
|
|
186
|
+
tags: evalItem.definition.tags,
|
|
187
|
+
metadata: evalItem.definition.metadata,
|
|
188
|
+
redaction: {},
|
|
189
|
+
},
|
|
190
|
+
};
|
|
168
191
|
}
|
|
169
192
|
async function outputEvalNotFound(id, evals) {
|
|
170
193
|
if (isJsonMode()) {
|
|
@@ -264,6 +287,7 @@ export async function evalCommand(options) {
|
|
|
264
287
|
adapters: {
|
|
265
288
|
agent: createAgentAdapter(agent, options),
|
|
266
289
|
},
|
|
290
|
+
export: createEvalCliExportConfig(evalItem, options, projectDir),
|
|
267
291
|
});
|
|
268
292
|
if (options.report) {
|
|
269
293
|
await writeTextFileEnsuringDir(options.report, JSON.stringify(report, null, 2));
|
|
@@ -6,6 +6,7 @@ declare const getEvalArgsSchema: () => import("../../../src/internal-agents/sche
|
|
|
6
6
|
datasetBase: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
|
|
7
7
|
report: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
|
|
8
8
|
junit: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
|
|
9
|
+
exporters: import("../../../src/internal-agents/schema.js").Schema<string[]>;
|
|
9
10
|
debug: import("../../../src/internal-agents/schema.js").Schema<boolean>;
|
|
10
11
|
model: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
|
|
11
12
|
maxOutputTokens: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
|
|
@@ -17,6 +18,7 @@ export declare const parseEvalArgs: (args: ParsedArgs) => import("../../shared/a
|
|
|
17
18
|
datasetBase: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
|
|
18
19
|
report: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
|
|
19
20
|
junit: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
|
|
21
|
+
exporters: import("../../../src/internal-agents/schema.js").Schema<string[]>;
|
|
20
22
|
debug: import("../../../src/internal-agents/schema.js").Schema<boolean>;
|
|
21
23
|
model: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
|
|
22
24
|
maxOutputTokens: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/handler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AAE3E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAA,MAAM,iBAAiB
|
|
1
|
+
{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/handler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AAE3E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAA,MAAM,iBAAiB;;;;;;;;;;GAYtB,CAAC;AAIF,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AAEzE,eAAO,MAAM,aAAa;;;;;;;;;;GAUxB,CAAC;AAEH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAIvE"}
|
|
@@ -6,6 +6,7 @@ const getEvalArgsSchema = defineSchema((v) => v.object({
|
|
|
6
6
|
datasetBase: v.string().optional(),
|
|
7
7
|
report: v.string().optional(),
|
|
8
8
|
junit: v.string().optional(),
|
|
9
|
+
exporters: v.array(v.string()).default([]),
|
|
9
10
|
debug: v.boolean().default(false),
|
|
10
11
|
model: v.string().optional(),
|
|
11
12
|
maxOutputTokens: v.number().int().positive().optional(),
|
|
@@ -17,6 +18,7 @@ export const parseEvalArgs = createArgParser(EvalArgsSchema, {
|
|
|
17
18
|
datasetBase: { keys: ["dataset-base"], type: "string" },
|
|
18
19
|
report: { keys: ["report"], type: "string" },
|
|
19
20
|
junit: { keys: ["junit"], type: "string" },
|
|
21
|
+
exporters: { keys: ["export"], type: "array" },
|
|
20
22
|
debug: { keys: ["debug"], type: "boolean" },
|
|
21
23
|
model: { keys: ["model"], type: "string" },
|
|
22
24
|
maxOutputTokens: { keys: ["max-output-tokens"], type: "number" },
|
package/esm/deno.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export default {
|
|
2
2
|
"name": "veryfront",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.886",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"nodeModulesDir": "auto",
|
|
6
6
|
"minimumDependencyAge": {
|
|
@@ -50,6 +50,7 @@ export default {
|
|
|
50
50
|
"./extensions/ext-parser-babel",
|
|
51
51
|
"./extensions/ext-sandbox-shell-tools",
|
|
52
52
|
"./extensions/ext-observability-opentelemetry",
|
|
53
|
+
"./extensions/ext-eval-report-http",
|
|
53
54
|
"./extensions/ext-content-mdx",
|
|
54
55
|
"./extensions/ext-schema-zod"
|
|
55
56
|
],
|
package/esm/src/eval/index.d.ts
CHANGED
|
@@ -50,6 +50,6 @@ export { runEval } from "./runner.js";
|
|
|
50
50
|
export { deriveEvalId, discoverEvals, findEvalById } from "./discovery.js";
|
|
51
51
|
export { createEvalSourceDocument, getEvalEditableFieldSchema, getEvalRunSchema, getEvalSourceDocumentSchema, getEvalSourcePatchSchema, getEvalSourceReferenceSchema, getEvalStudioCapabilitySchema, } from "./studio.js";
|
|
52
52
|
export type { DiscoveredEval, EvalDiscoveryOptions, EvalDiscoveryResult } from "./discovery.js";
|
|
53
|
-
export type { EvalAgentAdapter, EvalAgentAdapterContext, EvalAgentAdapterResult, EvalAgentInput, EvalCheckContext, EvalDataset, EvalDatasetLoadContext, EvalDefinition, EvalExample, EvalExampleInput, EvalExpect, EvalExpectation, EvalMetric, EvalMetricContext, EvalMetricFamily, EvalMetricResult, EvalMetricSummary, EvalMetricThreshold, EvalRecord, EvalReport, EvalReportSummary, EvalSeverity, EvalSource, EvalTargetKind, EvalToolCall, EvalTrace, EvalUsage, RunEvalOptions, } from "./types.js";
|
|
53
|
+
export type { EvalAgentAdapter, EvalAgentAdapterContext, EvalAgentAdapterResult, EvalAgentInput, EvalCheckContext, EvalDataset, EvalDatasetLoadContext, EvalDefinition, EvalExample, EvalExampleInput, EvalExpect, EvalExpectation, EvalMetric, EvalMetricContext, EvalMetricFamily, EvalMetricResult, EvalMetricSummary, EvalMetricThreshold, EvalRecord, EvalReport, EvalReportExportConfig, EvalReportSummary, EvalSeverity, EvalSource, EvalTargetKind, EvalToolCall, EvalTrace, EvalUsage, RunEvalOptions, } from "./types.js";
|
|
54
54
|
export type { CreateEvalSourceDocumentOptions, EvalEditableField, EvalRun, EvalSourceDocument, EvalSourcePatch, EvalSourceReference, EvalStudioCapability, } from "./studio.js";
|
|
55
55
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,UAAU,EACV,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,SAAS,EACT,SAAS,EACT,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,gBAAgB,EAChB,2BAA2B,EAC3B,wBAAwB,EACxB,4BAA4B,EAC5B,6BAA6B,GAC9B,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAChG,YAAY,EACV,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,UAAU,EACV,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,sBAAsB,EACtB,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,cAAc,EACd,YAAY,EACZ,SAAS,EACT,SAAS,EACT,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,+BAA+B,EAC/B,iBAAiB,EACjB,OAAO,EACP,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/src/eval/runner.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAEV,cAAc,EAMd,cAAc,EACf,MAAM,YAAY,CAAC;AA+LpB,6DAA6D;AAC7D,wBAAsB,OAAO,CAC3B,UAAU,EAAE,cAAc,EAC1B,OAAO,EAAE,cAAc,4CAuBxB"}
|
package/esm/src/eval/runner.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import * as dntShim from "../../_dnt.shims.js";
|
|
2
2
|
import { createEvalCheckContext } from "./expect.js";
|
|
3
3
|
import { createEvalReport } from "./report.js";
|
|
4
|
+
import { createEvalReportExporterRegistry, EvalReportExporterRegistryName, } from "../extensions/eval/index.js";
|
|
5
|
+
import { trace } from "../observability/tracing/api-shim.js";
|
|
6
|
+
import { tryResolve } from "../extensions/contracts.js";
|
|
4
7
|
function createRunId(now) {
|
|
5
8
|
return `evalrun_${now.toISOString().replace(/[-:.]/g, "").replace("T", "_").replace("Z", "")}`;
|
|
6
9
|
}
|
|
@@ -29,6 +32,79 @@ function isBlockingFailure(record) {
|
|
|
29
32
|
return [...(record.metrics ?? []), ...(record.checks ?? [])].some((result) => !result.skipped && result.pass === false &&
|
|
30
33
|
(result.severity === "gate" || result.severity === "budget"));
|
|
31
34
|
}
|
|
35
|
+
function createMissingRegistryResults(exporterIds) {
|
|
36
|
+
return exporterIds.map((exporterId) => ({
|
|
37
|
+
exporterId,
|
|
38
|
+
ok: false,
|
|
39
|
+
error: "No EvalReportExporter registry resolved.",
|
|
40
|
+
}));
|
|
41
|
+
}
|
|
42
|
+
function createMissingExporterResult(exporterId) {
|
|
43
|
+
return {
|
|
44
|
+
exporterId,
|
|
45
|
+
ok: false,
|
|
46
|
+
error: `No EvalReportExporter registered for "${exporterId}".`,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
function resolveExporterRegistry(config) {
|
|
50
|
+
return config.registry ??
|
|
51
|
+
tryResolve(EvalReportExporterRegistryName);
|
|
52
|
+
}
|
|
53
|
+
function isEmptyTraceId(value) {
|
|
54
|
+
return value === undefined || /^0+$/.test(value);
|
|
55
|
+
}
|
|
56
|
+
function getActiveEvalExportTraceContext() {
|
|
57
|
+
const spanContext = trace.getActiveSpan()?.spanContext();
|
|
58
|
+
if (!spanContext)
|
|
59
|
+
return undefined;
|
|
60
|
+
if (isEmptyTraceId(spanContext.traceId) || isEmptyTraceId(spanContext.spanId)) {
|
|
61
|
+
return undefined;
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
traceId: spanContext.traceId,
|
|
65
|
+
spanId: spanContext.spanId,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
function withActiveTraceContext(context) {
|
|
69
|
+
if (context?.trace)
|
|
70
|
+
return context;
|
|
71
|
+
const activeTrace = getActiveEvalExportTraceContext();
|
|
72
|
+
if (!activeTrace)
|
|
73
|
+
return context;
|
|
74
|
+
return {
|
|
75
|
+
...(context ?? {}),
|
|
76
|
+
trace: activeTrace,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
async function exportWithSelectedExporter(registry, report, config, exporterId) {
|
|
80
|
+
const exporter = registry.get(exporterId);
|
|
81
|
+
if (!exporter)
|
|
82
|
+
return createMissingExporterResult(exporterId);
|
|
83
|
+
const selectedRegistry = createEvalReportExporterRegistry();
|
|
84
|
+
selectedRegistry.register(exporter);
|
|
85
|
+
const [result] = await selectedRegistry.export(report, withActiveTraceContext(config.context));
|
|
86
|
+
return result ?? {
|
|
87
|
+
exporterId,
|
|
88
|
+
ok: false,
|
|
89
|
+
error: `EvalReportExporter "${exporterId}" did not return an export result.`,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
async function exportEvalReport(report, config) {
|
|
93
|
+
if (!config)
|
|
94
|
+
return undefined;
|
|
95
|
+
const exporterIds = config.exporterIds?.filter((id) => id.length > 0) ?? [];
|
|
96
|
+
const registry = resolveExporterRegistry(config);
|
|
97
|
+
if (!registry)
|
|
98
|
+
return createMissingRegistryResults(exporterIds);
|
|
99
|
+
if (exporterIds.length === 0) {
|
|
100
|
+
return registry.export(report, withActiveTraceContext(config.context));
|
|
101
|
+
}
|
|
102
|
+
const results = [];
|
|
103
|
+
for (const exporterId of exporterIds) {
|
|
104
|
+
results.push(await exportWithSelectedExporter(registry, report, config, exporterId));
|
|
105
|
+
}
|
|
106
|
+
return results;
|
|
107
|
+
}
|
|
32
108
|
async function runRecord(definition, options, example, repetition) {
|
|
33
109
|
const started = Date.now();
|
|
34
110
|
let result;
|
|
@@ -91,11 +167,13 @@ export async function runEval(definition, options) {
|
|
|
91
167
|
}
|
|
92
168
|
}
|
|
93
169
|
const endedAt = options.now?.() ?? new Date();
|
|
94
|
-
|
|
170
|
+
const report = createEvalReport({
|
|
95
171
|
definition,
|
|
96
172
|
records,
|
|
97
173
|
runId: options.runId ?? createRunId(startedAt),
|
|
98
174
|
startedAt,
|
|
99
175
|
endedAt,
|
|
100
176
|
});
|
|
177
|
+
const exports = await exportEvalReport(report, options.export);
|
|
178
|
+
return exports === undefined ? report : { ...report, exports };
|
|
101
179
|
}
|
package/esm/src/eval/types.d.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* @module eval
|
|
5
5
|
*/
|
|
6
|
+
import type { EvalReportExportContext, EvalReportExporterRegistry, EvalReportExportResult } from "../extensions/eval/index.js";
|
|
6
7
|
/** Primitive kind an eval can execute. V1 supports agent targets. */
|
|
7
8
|
export type EvalTargetKind = "agent";
|
|
8
9
|
/** How a metric result affects the final eval result. */
|
|
@@ -187,6 +188,13 @@ export interface RunEvalOptions {
|
|
|
187
188
|
baseDir?: string;
|
|
188
189
|
runId?: string;
|
|
189
190
|
now?: () => Date;
|
|
191
|
+
export?: EvalReportExportConfig;
|
|
192
|
+
}
|
|
193
|
+
/** Export configuration for a completed eval report. */
|
|
194
|
+
export interface EvalReportExportConfig {
|
|
195
|
+
registry?: EvalReportExporterRegistry;
|
|
196
|
+
exporterIds?: string[];
|
|
197
|
+
context?: EvalReportExportContext;
|
|
190
198
|
}
|
|
191
199
|
/** Aggregate pass/fail summary for one metric. */
|
|
192
200
|
export interface EvalMetricSummary {
|
|
@@ -217,5 +225,6 @@ export interface EvalReport {
|
|
|
217
225
|
endedAt: string;
|
|
218
226
|
summary: EvalReportSummary;
|
|
219
227
|
records: EvalRecord[];
|
|
228
|
+
exports?: EvalReportExportResult[];
|
|
220
229
|
}
|
|
221
230
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC;AAErC,yDAAyD;AACzD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEtD,wDAAwD;AACxD,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;AAE9E,yDAAyD;AACzD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,gEAAgE;AAChE,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAEjD,mEAAmE;AACnE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,0DAA0D;AAC1D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,yCAAyC;AACzC,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,IAAI,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAC/D;AAED,yDAAyD;AACzD,MAAM,WAAW,SAAS;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,0DAA0D;AAC1D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,IAAI,GAAG,OAAO,GAAG,SAAS,CAAC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,mDAAmD;AACnD,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;CAC3B;AAED,iEAAiE;AACjE,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED,qDAAqD;AACrD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,4DAA4D;AAC5D,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,gDAAgD;AAChD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;IAC9F,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;CACrD;AAED,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;CAC3D;AAED,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,SAAS,IAAI,eAAe,CAAC;IAC7B,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC;IAC9C,aAAa,IAAI,eAAe,CAAC;CAClC;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;CACpB;AAED,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,CAAC;IACrB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,GAAG,gBAAgB,EAAE,CAAC;IAC1C,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,6EAA6E;AAC7E,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,6DAA6D;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAC7B,OAAO,EAAE,uBAAuB,KAC7B,gBAAgB,CAAC,MAAM,GAAG,sBAAsB,CAAC,CAAC;AAEvD,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE;QACR,KAAK,EAAE,gBAAgB,CAAC;KACzB,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,uBAAuB,EACvB,0BAA0B,EAC1B,sBAAsB,EACvB,MAAM,6BAA6B,CAAC;AAErC,qEAAqE;AACrE,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC;AAErC,yDAAyD;AACzD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEtD,wDAAwD;AACxD,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;AAE9E,yDAAyD;AACzD,MAAM,MAAM,mBAAmB,GAAG;IAChC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,gEAAgE;AAChE,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAEjD,mEAAmE;AACnE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,0DAA0D;AAC1D,MAAM,MAAM,gBAAgB,GAAG;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,yCAAyC;AACzC,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,IAAI,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAC/D;AAED,yDAAyD;AACzD,MAAM,WAAW,SAAS;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,0DAA0D;AAC1D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,IAAI,GAAG,OAAO,GAAG,SAAS,CAAC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,mDAAmD;AACnD,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;CAC3B;AAED,iEAAiE;AACjE,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC7B,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED,qDAAqD;AACrD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,4DAA4D;AAC5D,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,gDAAgD;AAChD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;IAC9F,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;IAClD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,UAAU,CAAC;CACrD;AAED,wDAAwD;AACxD,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,IAAI,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;IACxD,MAAM,CAAC,SAAS,CAAC,EAAE,mBAAmB,GAAG,gBAAgB,CAAC;CAC3D;AAED,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,SAAS,IAAI,eAAe,CAAC;IAC7B,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAAC;IAC9C,aAAa,IAAI,eAAe,CAAC;CAClC;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;CACpB;AAED,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,CAAC;IACrB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC7B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,GAAG,gBAAgB,EAAE,CAAC;IAC1C,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,gBAAgB,CAAC,IAAI,CAAC,CAAC;CAC/D;AAED,6EAA6E;AAC7E,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,6DAA6D;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAC7B,OAAO,EAAE,uBAAuB,KAC7B,gBAAgB,CAAC,MAAM,GAAG,sBAAsB,CAAC,CAAC;AAEvD,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE;QACR,KAAK,EAAE,gBAAgB,CAAC;KACzB,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,MAAM,CAAC,EAAE,sBAAsB,CAAC;CACjC;AAED,wDAAwD;AACxD,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,EAAE,0BAA0B,CAAC;IACtC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,uBAAuB,CAAC;CACnC;AAED,kDAAkD;AAClD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,uDAAuD;AACvD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,iBAAiB,EAAE,CAAC;CAC9B;AAED,sDAAsD;AACtD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,cAAc,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,OAAO,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACpC"}
|