veryfront 0.1.1089 → 0.1.1090
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command-help.d.ts.map +1 -1
- package/esm/cli/commands/eval/command-help.js +2 -1
- package/esm/cli/commands/eval/command.d.ts +20 -0
- package/esm/cli/commands/eval/command.d.ts.map +1 -1
- package/esm/cli/commands/eval/command.js +35 -1
- package/esm/deno.js +1 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +5 -5
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,WAuGtB,CAAC"}
|
|
@@ -22,7 +22,7 @@ export const evalHelp = {
|
|
|
22
22
|
},
|
|
23
23
|
{
|
|
24
24
|
flag: "--junit <path>",
|
|
25
|
-
description: "Write
|
|
25
|
+
description: "Write an eval or suite JUnit XML report to a file",
|
|
26
26
|
},
|
|
27
27
|
{
|
|
28
28
|
flag: "--baseline <path>",
|
|
@@ -88,6 +88,7 @@ export const evalHelp = {
|
|
|
88
88
|
examples: [
|
|
89
89
|
"veryfront eval --list",
|
|
90
90
|
"veryfront eval",
|
|
91
|
+
"veryfront eval --report-dir .veryfront/evals/suite --junit .veryfront/evals/suite/junit.xml",
|
|
91
92
|
"veryfront eval deep-research",
|
|
92
93
|
"veryfront eval eval:deep-research --report-dir .veryfront/evals/deep-research",
|
|
93
94
|
"veryfront eval eval:deep-research --report .veryfront/evals/deep-research/report.json --junit .veryfront/evals/deep-research/junit.xml",
|
|
@@ -27,6 +27,25 @@ type EvalArtifactPaths = {
|
|
|
27
27
|
results: string;
|
|
28
28
|
reportMarkdown: string;
|
|
29
29
|
};
|
|
30
|
+
type EvalSuiteResult = {
|
|
31
|
+
id: string;
|
|
32
|
+
name: string;
|
|
33
|
+
target: string;
|
|
34
|
+
status: "passed" | "failed" | "error";
|
|
35
|
+
artifacts?: EvalArtifactPaths;
|
|
36
|
+
summary?: CliEvalSummary;
|
|
37
|
+
error?: string;
|
|
38
|
+
};
|
|
39
|
+
type EvalSuiteSummary = {
|
|
40
|
+
kind: "eval-suite-summary";
|
|
41
|
+
runId: string;
|
|
42
|
+
startedAt: string;
|
|
43
|
+
endedAt: string;
|
|
44
|
+
total: number;
|
|
45
|
+
passed: number;
|
|
46
|
+
failed: number;
|
|
47
|
+
results: EvalSuiteResult[];
|
|
48
|
+
};
|
|
30
49
|
type EvalModelArtifactPaths = EvalArtifactPaths & {
|
|
31
50
|
junit: string;
|
|
32
51
|
};
|
|
@@ -79,6 +98,7 @@ export declare function createResultsJsonl(report: EvalReport): string;
|
|
|
79
98
|
/** Render a human-reviewable markdown report for a single eval run. */
|
|
80
99
|
export declare function createEvalMarkdownReport(report: EvalReport, baseline?: EvalReportComparison): string;
|
|
81
100
|
export declare function createJunitXml(report: EvalReport): string;
|
|
101
|
+
export declare function createEvalSuiteJunitXml(summary: EvalSuiteSummary): string;
|
|
82
102
|
export declare function writeEvalArtifacts(report: EvalReport, paths: EvalArtifactPaths, baseline?: EvalReportComparison): Promise<void>;
|
|
83
103
|
export declare function createEvalExitCode(report: EvalReport, baseline?: EvalReportComparison, exportRequired?: boolean): 0 | 1;
|
|
84
104
|
export declare function resolveToolTargetId(target: string): string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAgC,KAAK,IAAI,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAChH,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAEL,KAAK,0BAA0B,EAE/B,KAAK,yBAAyB,EAC/B,MAAM,uCAAuC,CAAC;AAmB/C,OAAO,EACL,2BAA2B,EAG5B,MAAM,6CAA6C,CAAC;AASrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,uBAAuB;IAC/B,2BAA2B,CAAC,EAAE,OAAO,2BAA2B,CAAC;CAClE;AAED,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAgC,KAAK,IAAI,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAChH,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAEL,KAAK,0BAA0B,EAE/B,KAAK,yBAAyB,EAC/B,MAAM,uCAAuC,CAAC;AAmB/C,OAAO,EACL,2BAA2B,EAG5B,MAAM,6CAA6C,CAAC;AASrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,uBAAuB;IAC/B,2BAA2B,CAAC,EAAE,OAAO,2BAA2B,CAAC;CAClE;AAED,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AASF,KAAK,eAAe,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;IACtC,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,KAAK,gBAAgB,GAAG;IACtB,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,eAAe,EAAE,CAAC;CAC5B,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAgEnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAgCD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAgBrB;AA8GD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,cAAc,UAAQ,GACrB,CAAC,GAAG,CAAC,CAIP;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAOD,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,gBAAgB,GAAG,MAAM,CA0BzE;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,EAC/B,cAAc,UAAQ,GACrB,CAAC,GAAG,CAAC,CAIP;AAMD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAaD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA6CD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,GAAE,oBAAyB,IACpE,SAAS,sBAAsB;;;;;;GAiB9C;AAiJD,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,iBAAiB,EAChC,QAAQ,EAAE,0BAA0B,EACpC,MAAM,CAAC,EAAE,qBAAqB,GAAG,IAAI,GACpC,sBAAsB,GAAG,SAAS,CAmBpC;AAED,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AAuBF,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,MAAM,EAAE,CAUxF;AAED,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,GAC1C,OAAO,CAGT;AA2BD,wBAAgB,iCAAiC,IAAI,yBAAyB,CAuB7E;AA4VD,wBAAsB,cAAc,CAClC,OAAO,EAAE,WAAW,EACpB,YAAY,GAAE,uBAA4B,GACzC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAwP7B;AAED,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAKrE"}
|
|
@@ -84,6 +84,7 @@ function createEvalSuiteArtifactPaths(reportDir) {
|
|
|
84
84
|
return {
|
|
85
85
|
directory: reportDir,
|
|
86
86
|
summary: join(reportDir, "summary.json"),
|
|
87
|
+
results: join(reportDir, "results.jsonl"),
|
|
87
88
|
reportMarkdown: join(reportDir, "report.md"),
|
|
88
89
|
};
|
|
89
90
|
}
|
|
@@ -634,6 +635,34 @@ export function createJunitXml(report) {
|
|
|
634
635
|
lines.push("</testsuite>");
|
|
635
636
|
return `${lines.join("\n")}\n`;
|
|
636
637
|
}
|
|
638
|
+
function createEvalSuiteResultsJsonl(summary) {
|
|
639
|
+
return summary.results.map((result) => JSON.stringify(result)).join("\n") +
|
|
640
|
+
(summary.results.length > 0 ? "\n" : "");
|
|
641
|
+
}
|
|
642
|
+
export function createEvalSuiteJunitXml(summary) {
|
|
643
|
+
const lines = [
|
|
644
|
+
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
645
|
+
`<testsuites tests="${summary.total}" failures="${summary.failed}" skipped="0">`,
|
|
646
|
+
` <testsuite name="veryfront eval suite" tests="${summary.total}" failures="${summary.failed}" skipped="0">`,
|
|
647
|
+
];
|
|
648
|
+
for (const result of summary.results) {
|
|
649
|
+
const attrs = `classname="eval" name="${xmlEscape(result.id)}"`;
|
|
650
|
+
if (result.status === "passed") {
|
|
651
|
+
lines.push(` <testcase ${attrs} />`);
|
|
652
|
+
continue;
|
|
653
|
+
}
|
|
654
|
+
const message = result.error ??
|
|
655
|
+
(result.summary?.failed
|
|
656
|
+
? `${result.summary.failed} record(s) failed`
|
|
657
|
+
: "A required eval export failed.");
|
|
658
|
+
lines.push(` <testcase ${attrs}>`);
|
|
659
|
+
lines.push(` <failure message="${xmlEscape(message)}">${xmlEscape(message)}</failure>`);
|
|
660
|
+
lines.push(" </testcase>");
|
|
661
|
+
}
|
|
662
|
+
lines.push(" </testsuite>");
|
|
663
|
+
lines.push("</testsuites>");
|
|
664
|
+
return `${lines.join("\n")}\n`;
|
|
665
|
+
}
|
|
637
666
|
async function writeTextFileEnsuringDir(path, content) {
|
|
638
667
|
const dir = dirname(path);
|
|
639
668
|
if (dir && dir !== ".")
|
|
@@ -824,7 +853,6 @@ function listEvals(evals, projectDir) {
|
|
|
824
853
|
function unsupportedEvalSuiteOption(options) {
|
|
825
854
|
const flags = [
|
|
826
855
|
options.report ? "--report" : undefined,
|
|
827
|
-
options.junit ? "--junit" : undefined,
|
|
828
856
|
options.baseline ? "--baseline" : undefined,
|
|
829
857
|
options.writeBaseline ? "--write-baseline" : undefined,
|
|
830
858
|
options.baselinePassRateDropThreshold !== undefined
|
|
@@ -881,6 +909,7 @@ function createEvalSuiteMarkdown(summary) {
|
|
|
881
909
|
}
|
|
882
910
|
async function writeEvalSuiteArtifacts(summary, artifacts) {
|
|
883
911
|
await writeTextFileEnsuringDir(artifacts.summary, JSON.stringify(summary, null, 2));
|
|
912
|
+
await writeTextFileEnsuringDir(artifacts.results, createEvalSuiteResultsJsonl(summary));
|
|
884
913
|
await writeTextFileEnsuringDir(artifacts.reportMarkdown, createEvalSuiteMarkdown(summary));
|
|
885
914
|
}
|
|
886
915
|
function getDiscoveredEvals(runtime) {
|
|
@@ -1250,6 +1279,9 @@ async function runEvalSuite(input) {
|
|
|
1250
1279
|
}
|
|
1251
1280
|
const summary = createEvalSuiteSummary(runId, startedAt, results);
|
|
1252
1281
|
await writeEvalSuiteArtifacts(summary, artifacts);
|
|
1282
|
+
if (input.options.junit) {
|
|
1283
|
+
await writeTextFileEnsuringDir(input.options.junit, createEvalSuiteJunitXml(summary));
|
|
1284
|
+
}
|
|
1253
1285
|
if (isJsonMode()) {
|
|
1254
1286
|
await outputJson(createSuccessEnvelope("eval", { suite: summary, artifacts }));
|
|
1255
1287
|
}
|
|
@@ -1257,6 +1289,8 @@ async function runEvalSuite(input) {
|
|
|
1257
1289
|
cliLogger.info(`Eval suite: ${summary.passed}/${summary.total} passed`);
|
|
1258
1290
|
cliLogger.info(`Report directory: ${artifacts.directory}`);
|
|
1259
1291
|
cliLogger.info(`Suite report: ${artifacts.reportMarkdown}`);
|
|
1292
|
+
if (input.options.junit)
|
|
1293
|
+
cliLogger.info(`JUnit: ${input.options.junit}`);
|
|
1260
1294
|
}
|
|
1261
1295
|
return summary.failed === 0 ? 0 : 1;
|
|
1262
1296
|
}
|
package/esm/deno.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "veryfront",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1090",
|
|
4
4
|
"description": "The simplest way to build AI-powered apps",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"react",
|
|
@@ -328,10 +328,10 @@
|
|
|
328
328
|
"@types/react": "19.2.14",
|
|
329
329
|
"@types/react-dom": "19.2.3",
|
|
330
330
|
"ws": "8.21.0",
|
|
331
|
-
"@veryfront/ext-bundler-esbuild": "0.1.
|
|
332
|
-
"@veryfront/ext-content-mdx": "0.1.
|
|
333
|
-
"@veryfront/ext-css-tailwind": "0.1.
|
|
334
|
-
"@veryfront/ext-parser-babel": "0.1.
|
|
331
|
+
"@veryfront/ext-bundler-esbuild": "0.1.1090",
|
|
332
|
+
"@veryfront/ext-content-mdx": "0.1.1090",
|
|
333
|
+
"@veryfront/ext-css-tailwind": "0.1.1090",
|
|
334
|
+
"@veryfront/ext-parser-babel": "0.1.1090"
|
|
335
335
|
},
|
|
336
336
|
"devDependencies": {
|
|
337
337
|
"@types/node": "20.9.0"
|