veryfront 0.1.1084 → 0.1.1086
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command-help.d.ts.map +1 -1
- package/esm/cli/commands/eval/command-help.js +4 -2
- package/esm/cli/commands/eval/command.d.ts.map +1 -1
- package/esm/cli/commands/eval/command.js +196 -11
- package/esm/deno.js +1 -1
- package/esm/src/internal-agents/run-stream.d.ts.map +1 -1
- package/esm/src/internal-agents/run-stream.js +6 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +5 -5
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,WAiGtB,CAAC"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
export const evalHelp = {
|
|
2
2
|
name: "eval",
|
|
3
3
|
category: "ai",
|
|
4
|
-
description: "
|
|
4
|
+
description: "List, run, and export discovered eval definitions",
|
|
5
5
|
usage: "veryfront eval [eval-id] [options]",
|
|
6
6
|
options: [
|
|
7
7
|
{
|
|
8
8
|
flag: "-l, --list",
|
|
9
|
-
description: "List discovered evals",
|
|
9
|
+
description: "List discovered evals without running them",
|
|
10
10
|
},
|
|
11
11
|
{
|
|
12
12
|
flag: "--dataset-base <path>",
|
|
@@ -83,6 +83,7 @@ export const evalHelp = {
|
|
|
83
83
|
],
|
|
84
84
|
examples: [
|
|
85
85
|
"veryfront eval --list",
|
|
86
|
+
"veryfront eval",
|
|
86
87
|
"veryfront eval deep-research",
|
|
87
88
|
"veryfront eval eval:deep-research --report-dir .veryfront/evals/deep-research",
|
|
88
89
|
"veryfront eval eval:deep-research --report .veryfront/evals/deep-research/report.json --junit .veryfront/evals/deep-research/junit.xml",
|
|
@@ -90,6 +91,7 @@ export const evalHelp = {
|
|
|
90
91
|
"veryfront eval deep-research --baseline-model anthropic/claude-sonnet-4-6 --candidate-model moonshotai/kimi-k2.6",
|
|
91
92
|
"veryfront eval deep-research --baseline-model anthropic/claude-sonnet-4-6 --candidate-model moonshotai/kimi-k2.6 --comparison-policy evals/model-comparison.policy.json",
|
|
92
93
|
"veryfront eval deep-research --export braintrust,langfuse --json",
|
|
94
|
+
"MLFLOW_TRACKING_URI=https://mlflow.example.com veryfront eval",
|
|
93
95
|
"veryfront eval deep-research --json",
|
|
94
96
|
],
|
|
95
97
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAgC,KAAK,IAAI,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAChH,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAEL,KAAK,0BAA0B,EAE/B,KAAK,yBAAyB,EAC/B,MAAM,uCAAuC,CAAC;AAmB/C,OAAO,EACL,2BAA2B,EAG5B,MAAM,6CAA6C,CAAC;AASrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,uBAAuB;IAC/B,2BAA2B,CAAC,EAAE,OAAO,2BAA2B,CAAC;CAClE;AAED,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAgC,KAAK,IAAI,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAChH,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAEL,KAAK,0BAA0B,EAE/B,KAAK,yBAAyB,EAC/B,MAAM,uCAAuC,CAAC;AAmB/C,OAAO,EACL,2BAA2B,EAG5B,MAAM,6CAA6C,CAAC;AASrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,uBAAuB;IAC/B,2BAA2B,CAAC,EAAE,OAAO,2BAA2B,CAAC;CAClE;AAED,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AA6BF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AA+DnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AA+BD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAgBrB;AA8GD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAaD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA6CD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,GAAE,oBAAyB,IACpE,SAAS,sBAAsB;;;;;;GAiB9C;AAiJD,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,iBAAiB,EAChC,QAAQ,EAAE,0BAA0B,EACpC,MAAM,CAAC,EAAE,qBAAqB,GAAG,IAAI,GACpC,sBAAsB,GAAG,SAAS,CAkBpC;AAED,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AAuBF,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,MAAM,EAAE,CAUxF;AA2BD,wBAAgB,iCAAiC,IAAI,yBAAyB,CAuB7E;AAmVD,wBAAsB,cAAc,CAClC,OAAO,EAAE,WAAW,EACpB,YAAY,GAAE,uBAA4B,GACzC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAkP7B;AAED,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAKrE"}
|
|
@@ -18,6 +18,7 @@ import { createEvalCliBuiltinExtensions } from "../../../src/extensions/builtin-
|
|
|
18
18
|
const GATEWAY_BILLING_GROUP_USAGE_NOT_READY_CODE = "gateway_billing_group_usage_not_ready";
|
|
19
19
|
const ENV_EVAL_EXPORTERS = "VERYFRONT_EVAL_EXPORTERS";
|
|
20
20
|
const ENV_EVAL_EXPORT = "VERYFRONT_EVAL_EXPORT";
|
|
21
|
+
const ENV_MLFLOW_TRACKING_URI = "MLFLOW_TRACKING_URI";
|
|
21
22
|
const ENV_EVAL_EXPORT_INCLUDE_INPUTS = "VERYFRONT_EVAL_EXPORT_INCLUDE_INPUTS";
|
|
22
23
|
const ENV_EVAL_EXPORT_INCLUDE_OUTPUTS = "VERYFRONT_EVAL_EXPORT_INCLUDE_OUTPUTS";
|
|
23
24
|
const ENV_EVAL_EXPORT_INCLUDE_REFERENCES = "VERYFRONT_EVAL_EXPORT_INCLUDE_REFERENCES";
|
|
@@ -78,6 +79,19 @@ export function createEvalArtifactPaths(reportDir) {
|
|
|
78
79
|
reportMarkdown: join(reportDir, "report.md"),
|
|
79
80
|
};
|
|
80
81
|
}
|
|
82
|
+
function createEvalSuiteArtifactPaths(reportDir) {
|
|
83
|
+
return {
|
|
84
|
+
directory: reportDir,
|
|
85
|
+
summary: join(reportDir, "summary.json"),
|
|
86
|
+
reportMarkdown: join(reportDir, "report.md"),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
function sortEvals(evals) {
|
|
90
|
+
return [...evals].sort((left, right) => left.id.localeCompare(right.id) || left.filePath.localeCompare(right.filePath));
|
|
91
|
+
}
|
|
92
|
+
function createEvalSuiteChildDirectory(suiteDirectory, index, evalId) {
|
|
93
|
+
return join(suiteDirectory, `${String(index + 1).padStart(3, "0")}-${sanitizeEvalReportDirLabel(evalId)}`);
|
|
94
|
+
}
|
|
81
95
|
function sanitizeModelIdForPath(model) {
|
|
82
96
|
return model.trim().replace(/[^A-Za-z0-9._-]+/g, "__").replace(/^_+|_+$/g, "") || "model";
|
|
83
97
|
}
|
|
@@ -744,10 +758,18 @@ function createEvalToolCallId(toolId, context) {
|
|
|
744
758
|
return `eval-${toolId}-${context.example.id}-${context.repetition}-${dntShim.crypto.randomUUID()}`;
|
|
745
759
|
}
|
|
746
760
|
function createAgentAdapter(agent, options) {
|
|
747
|
-
return async ({ example }) => {
|
|
761
|
+
return async ({ definition, example, repetition }) => {
|
|
748
762
|
const started = Date.now();
|
|
749
763
|
const response = await agent.generate({
|
|
750
764
|
input: normalizeEvalInputForAgent(example.input),
|
|
765
|
+
context: {
|
|
766
|
+
eval: {
|
|
767
|
+
definitionId: definition.id,
|
|
768
|
+
exampleId: example.id,
|
|
769
|
+
repetition,
|
|
770
|
+
metadata: example.metadata ?? {},
|
|
771
|
+
},
|
|
772
|
+
},
|
|
751
773
|
...(options.model ? { model: options.model } : {}),
|
|
752
774
|
...(options.maxOutputTokens ? { maxOutputTokens: options.maxOutputTokens } : {}),
|
|
753
775
|
});
|
|
@@ -784,7 +806,7 @@ export function createToolAdapter(tool, baseContext = {}) {
|
|
|
784
806
|
};
|
|
785
807
|
}
|
|
786
808
|
function listEvals(evals, projectDir) {
|
|
787
|
-
return evals.map((item) => ({
|
|
809
|
+
return sortEvals(evals).map((item) => ({
|
|
788
810
|
id: item.id,
|
|
789
811
|
name: item.name,
|
|
790
812
|
target: item.definition.target,
|
|
@@ -794,6 +816,68 @@ function listEvals(evals, projectDir) {
|
|
|
794
816
|
},
|
|
795
817
|
}));
|
|
796
818
|
}
|
|
819
|
+
function unsupportedEvalSuiteOption(options) {
|
|
820
|
+
const flags = [
|
|
821
|
+
options.report ? "--report" : undefined,
|
|
822
|
+
options.junit ? "--junit" : undefined,
|
|
823
|
+
options.baseline ? "--baseline" : undefined,
|
|
824
|
+
options.writeBaseline ? "--write-baseline" : undefined,
|
|
825
|
+
options.baselinePassRateDropThreshold !== undefined
|
|
826
|
+
? "--baseline-pass-rate-drop-threshold"
|
|
827
|
+
: undefined,
|
|
828
|
+
options.baselineMetricPassRateDropThreshold !== undefined
|
|
829
|
+
? "--baseline-metric-pass-rate-drop-threshold"
|
|
830
|
+
: undefined,
|
|
831
|
+
options.baselineFailedDeltaThreshold !== undefined
|
|
832
|
+
? "--baseline-failed-delta-threshold"
|
|
833
|
+
: undefined,
|
|
834
|
+
options.baselineUsageIncreaseThreshold !== undefined
|
|
835
|
+
? "--baseline-usage-increase-threshold"
|
|
836
|
+
: undefined,
|
|
837
|
+
options.baselineLatencyIncreaseThreshold !== undefined
|
|
838
|
+
? "--baseline-latency-increase-threshold"
|
|
839
|
+
: undefined,
|
|
840
|
+
options.model ? "--model" : undefined,
|
|
841
|
+
options.maxOutputTokens ? "--max-output-tokens" : undefined,
|
|
842
|
+
options.baselineModel ? "--baseline-model" : undefined,
|
|
843
|
+
options.candidateModels.length > 0 ? "--candidate-model" : undefined,
|
|
844
|
+
options.comparisonPolicy ? "--comparison-policy" : undefined,
|
|
845
|
+
].filter((flag) => Boolean(flag));
|
|
846
|
+
if (flags.length === 0)
|
|
847
|
+
return undefined;
|
|
848
|
+
return `${flags.join(", ")} require a named eval. Run \`veryfront eval <eval-id>\`.`;
|
|
849
|
+
}
|
|
850
|
+
function createEvalSuiteSummary(runId, startedAt, results) {
|
|
851
|
+
const passed = results.filter((result) => result.status === "passed").length;
|
|
852
|
+
return {
|
|
853
|
+
kind: "eval-suite-summary",
|
|
854
|
+
runId,
|
|
855
|
+
startedAt: startedAt.toISOString(),
|
|
856
|
+
endedAt: new Date().toISOString(),
|
|
857
|
+
total: results.length,
|
|
858
|
+
passed,
|
|
859
|
+
failed: results.length - passed,
|
|
860
|
+
results,
|
|
861
|
+
};
|
|
862
|
+
}
|
|
863
|
+
function createEvalSuiteMarkdown(summary) {
|
|
864
|
+
const rows = summary.results.map((result) => `| ${markdownCell(result.id)} | ${result.status} | ${result.summary ? `${result.summary.passed}/${result.summary.records}` : "n/a"} | ${result.error ? markdownCell(result.error) : ""} |`);
|
|
865
|
+
return [
|
|
866
|
+
"# Eval suite report",
|
|
867
|
+
"",
|
|
868
|
+
`Run: \`${markdownCell(summary.runId)}\``,
|
|
869
|
+
`Result: \`${summary.passed}/${summary.total} passed\``,
|
|
870
|
+
"",
|
|
871
|
+
"| Eval | Status | Records | Error |",
|
|
872
|
+
"| --- | --- | --- | --- |",
|
|
873
|
+
...rows,
|
|
874
|
+
"",
|
|
875
|
+
].join("\n");
|
|
876
|
+
}
|
|
877
|
+
async function writeEvalSuiteArtifacts(summary, artifacts) {
|
|
878
|
+
await writeTextFileEnsuringDir(artifacts.summary, JSON.stringify(summary, null, 2));
|
|
879
|
+
await writeTextFileEnsuringDir(artifacts.reportMarkdown, createEvalSuiteMarkdown(summary));
|
|
880
|
+
}
|
|
797
881
|
function getDiscoveredEvals(runtime) {
|
|
798
882
|
return [...runtime.evals.entries()].map(([id, definition]) => {
|
|
799
883
|
if (!definition.source) {
|
|
@@ -876,7 +960,10 @@ export function resolveEvalExporterIds(options) {
|
|
|
876
960
|
const envExporters = parseEvalExporterList(readEvalCliEnv(ENV_EVAL_EXPORTERS));
|
|
877
961
|
if (envExporters.length > 0)
|
|
878
962
|
return uniqueValues(envExporters);
|
|
879
|
-
|
|
963
|
+
const legacyExporters = parseEvalExporterList(readEvalCliEnv(ENV_EVAL_EXPORT));
|
|
964
|
+
if (legacyExporters.length > 0)
|
|
965
|
+
return uniqueValues(legacyExporters);
|
|
966
|
+
return readEvalCliEnv(ENV_MLFLOW_TRACKING_URI) ? ["mlflow"] : [];
|
|
880
967
|
}
|
|
881
968
|
function parseEvalExportBooleanEnv(name) {
|
|
882
969
|
const value = readEvalCliEnv(name)?.toLowerCase();
|
|
@@ -1085,6 +1172,82 @@ async function outputEvalUsageError(message) {
|
|
|
1085
1172
|
}
|
|
1086
1173
|
return 2;
|
|
1087
1174
|
}
|
|
1175
|
+
async function runEvalSuite(input) {
|
|
1176
|
+
const runId = createEvalRunId();
|
|
1177
|
+
const startedAt = new Date();
|
|
1178
|
+
const artifacts = createEvalSuiteArtifactPaths(input.options.reportDir ?? createDefaultEvalReportDir(runId));
|
|
1179
|
+
const provenance = await resolveEvalRunProvenance({
|
|
1180
|
+
projectDir: input.projectDir,
|
|
1181
|
+
frameworkVersion: VERSION,
|
|
1182
|
+
});
|
|
1183
|
+
const results = [];
|
|
1184
|
+
for (const [index, evalItem] of sortEvals(input.evals).entries()) {
|
|
1185
|
+
const evalArtifacts = createEvalArtifactPaths(createEvalSuiteChildDirectory(artifacts.directory, index, evalItem.id));
|
|
1186
|
+
try {
|
|
1187
|
+
const agentId = evalItem.definition.targetKind === "agent"
|
|
1188
|
+
? resolveAgentTargetId(evalItem.definition.target)
|
|
1189
|
+
: undefined;
|
|
1190
|
+
const toolId = evalItem.definition.targetKind === "tool"
|
|
1191
|
+
? resolveToolTargetId(evalItem.definition.target)
|
|
1192
|
+
: undefined;
|
|
1193
|
+
const agent = agentId ? input.projectRuntime.agents.get(agentId) : undefined;
|
|
1194
|
+
const tool = toolId ? input.projectRuntime.tools.get(toolId) : undefined;
|
|
1195
|
+
if (agentId && !agent)
|
|
1196
|
+
throw new Error(`Agent "${agentId}" not found for eval target.`);
|
|
1197
|
+
if (toolId && !tool)
|
|
1198
|
+
throw new Error(`Tool "${toolId}" not found for eval target.`);
|
|
1199
|
+
const evalRunId = `${runId}_${String(index + 1).padStart(3, "0")}`;
|
|
1200
|
+
const exportConfig = createEvalCliExportConfig(evalItem, input.options, input.projectDir, evalArtifacts, input.exporterRegistry, input.config);
|
|
1201
|
+
const finalizedReport = await runEvalWithGatewayBillingGroup(evalRunId, () => runEval(evalItem.definition, {
|
|
1202
|
+
baseDir: input.options.datasetBase ?? input.projectDir,
|
|
1203
|
+
runId: evalRunId,
|
|
1204
|
+
adapters: evalItem.definition.targetKind === "tool"
|
|
1205
|
+
? { tool: createToolAdapter(tool, createEvalToolExecutionContext(input.config)) }
|
|
1206
|
+
: { agent: createAgentAdapter(agent, input.options) },
|
|
1207
|
+
metadata: { provenance },
|
|
1208
|
+
}));
|
|
1209
|
+
const report = await exportEvalReportForCli(finalizedReport, exportConfig);
|
|
1210
|
+
await writeEvalArtifacts(report, evalArtifacts);
|
|
1211
|
+
const status = createEvalExitCode(report) === 0 ? "passed" : "failed";
|
|
1212
|
+
results.push({
|
|
1213
|
+
id: evalItem.id,
|
|
1214
|
+
name: evalItem.name,
|
|
1215
|
+
target: evalItem.definition.target,
|
|
1216
|
+
status,
|
|
1217
|
+
artifacts: evalArtifacts,
|
|
1218
|
+
summary: summarizeReportForCli(report),
|
|
1219
|
+
});
|
|
1220
|
+
if (!isJsonMode()) {
|
|
1221
|
+
printReport(report);
|
|
1222
|
+
cliLogger.info(`Report directory: ${evalArtifacts.directory}`);
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
catch (error) {
|
|
1226
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1227
|
+
results.push({
|
|
1228
|
+
id: evalItem.id,
|
|
1229
|
+
name: evalItem.name,
|
|
1230
|
+
target: evalItem.definition.target,
|
|
1231
|
+
status: "error",
|
|
1232
|
+
artifacts: evalArtifacts,
|
|
1233
|
+
error: message,
|
|
1234
|
+
});
|
|
1235
|
+
if (!isJsonMode())
|
|
1236
|
+
cliLogger.error(`Eval ${evalItem.id}: ${message}`);
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
const summary = createEvalSuiteSummary(runId, startedAt, results);
|
|
1240
|
+
await writeEvalSuiteArtifacts(summary, artifacts);
|
|
1241
|
+
if (isJsonMode()) {
|
|
1242
|
+
await outputJson(createSuccessEnvelope("eval", { suite: summary, artifacts }));
|
|
1243
|
+
}
|
|
1244
|
+
else {
|
|
1245
|
+
cliLogger.info(`Eval suite: ${summary.passed}/${summary.total} passed`);
|
|
1246
|
+
cliLogger.info(`Report directory: ${artifacts.directory}`);
|
|
1247
|
+
cliLogger.info(`Suite report: ${artifacts.reportMarkdown}`);
|
|
1248
|
+
}
|
|
1249
|
+
return summary.failed === 0 ? 0 : 1;
|
|
1250
|
+
}
|
|
1088
1251
|
export async function runEvalCommand(options, dependencies = {}) {
|
|
1089
1252
|
const projectDir = options.projectDir ?? dntShim.Deno.cwd();
|
|
1090
1253
|
const discoverRuntime = dependencies.discoverProjectAgentRuntime ?? discoverProjectAgentRuntime;
|
|
@@ -1128,17 +1291,39 @@ export async function runEvalCommand(options, dependencies = {}) {
|
|
|
1128
1291
|
return undefined;
|
|
1129
1292
|
}
|
|
1130
1293
|
if (!options.id) {
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1294
|
+
const invalidOption = unsupportedEvalSuiteOption(options);
|
|
1295
|
+
if (invalidOption)
|
|
1296
|
+
return await outputEvalUsageError(invalidOption);
|
|
1297
|
+
if (evals.length === 0) {
|
|
1298
|
+
if (isJsonMode()) {
|
|
1299
|
+
await outputJson(createSuccessEnvelope("eval", {
|
|
1300
|
+
suite: null,
|
|
1301
|
+
errors: projectRuntime.errors.map((error) => ({
|
|
1302
|
+
filePath: error.file,
|
|
1303
|
+
error: error.error.message,
|
|
1304
|
+
})),
|
|
1305
|
+
}));
|
|
1306
|
+
}
|
|
1307
|
+
else {
|
|
1308
|
+
cliLogger.info("No evals found.");
|
|
1309
|
+
}
|
|
1310
|
+
return 0;
|
|
1311
|
+
}
|
|
1312
|
+
const selectedExporterIds = resolveEvalExporterIds(options);
|
|
1313
|
+
const extensionSetup = await setupEvalCliExtensions(projectDir, config, selectedExporterIds);
|
|
1314
|
+
try {
|
|
1315
|
+
return await runWithProjectAgentRuntime(projectRuntime, () => runEvalSuite({
|
|
1316
|
+
evals,
|
|
1317
|
+
options,
|
|
1318
|
+
projectDir,
|
|
1319
|
+
projectRuntime,
|
|
1320
|
+
config,
|
|
1321
|
+
exporterRegistry: extensionSetup.exporterRegistry,
|
|
1136
1322
|
}));
|
|
1137
1323
|
}
|
|
1138
|
-
|
|
1139
|
-
|
|
1324
|
+
finally {
|
|
1325
|
+
await extensionSetup.loader.teardownAll();
|
|
1140
1326
|
}
|
|
1141
|
-
return 2;
|
|
1142
1327
|
}
|
|
1143
1328
|
const evalId = normalizeEvalCliId(options.id);
|
|
1144
1329
|
const evalItem = findEvalForCliId(evals, options.id);
|
package/esm/deno.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-stream.d.ts","sourceRoot":"","sources":["../../../src/src/internal-agents/run-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,KAAK,EACV,KAAK,YAAY,IAAI,OAAO,EAC5B,KAAK,aAAa,EAEnB,MAAM,mBAAmB,CAAC;AAY3B,OAAO,KAAK,EACV,+BAA+B,EAC/B,8BAA8B,EAC/B,MAAM,qBAAqB,CAAC;AAU7B,OAAO,EAEL,KAAK,IAAI,EAGV,MAAM,kBAAkB,CAAC;AAe1B,OAAO,EAA0B,KAAK,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC3F,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAsCxD,MAAM,WAAW,+BAA+B;IAC9C,cAAc,EAAE,sBAAsB,CAAC;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC;IAC5C,mBAAmB,CAAC,EAAE;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,cAAc,CAAC,EAAE,+BAA+B,CAAC,gBAAgB,CAAC,CAAC;IACnE,8BAA8B,CAAC,EAAE,CAC/B,KAAK,EAAE,+BAA+B,KACnC,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAC7C,aAAa,CAAC,EAAE,CACd,KAAK,EAAE,KAAK,EACZ,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,KAClC;QACH,MAAM,EAAE,CACN,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,SAAS,CAAC,EAAE;YACV,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;SAC9C,EACD,aAAa,CAAC,EAAE,MAAM,EACtB,uBAAuB,CAAC,EAAE,MAAM,EAChC,WAAW,CAAC,EAAE,WAAW,KACtB,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;KAC1C,CAAC;CACH;AAoCD,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,KAAK,EACZ,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,sBAAsB,EACtC,2BAA2B,CAAC,EAAE,MAAM,EAAE,EACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;;cAsFrD;
|
|
1
|
+
{"version":3,"file":"run-stream.d.ts","sourceRoot":"","sources":["../../../src/src/internal-agents/run-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,KAAK,EACV,KAAK,YAAY,IAAI,OAAO,EAC5B,KAAK,aAAa,EAEnB,MAAM,mBAAmB,CAAC;AAY3B,OAAO,KAAK,EACV,+BAA+B,EAC/B,8BAA8B,EAC/B,MAAM,qBAAqB,CAAC;AAU7B,OAAO,EAEL,KAAK,IAAI,EAGV,MAAM,kBAAkB,CAAC;AAe1B,OAAO,EAA0B,KAAK,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC3F,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAsCxD,MAAM,WAAW,+BAA+B;IAC9C,cAAc,EAAE,sBAAsB,CAAC;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC;IAC5C,mBAAmB,CAAC,EAAE;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,cAAc,CAAC,EAAE,+BAA+B,CAAC,gBAAgB,CAAC,CAAC;IACnE,8BAA8B,CAAC,EAAE,CAC/B,KAAK,EAAE,+BAA+B,KACnC,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAC7C,aAAa,CAAC,EAAE,CACd,KAAK,EAAE,KAAK,EACZ,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,KAClC;QACH,MAAM,EAAE,CACN,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,SAAS,CAAC,EAAE;YACV,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;SAC9C,EACD,aAAa,CAAC,EAAE,MAAM,EACtB,uBAAuB,CAAC,EAAE,MAAM,EAChC,WAAW,CAAC,EAAE,WAAW,KACtB,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;KAC1C,CAAC;CACH;AAoCD,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,KAAK,EACZ,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,sBAAsB,EACtC,2BAA2B,CAAC,EAAE,MAAM,EAAE,EACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;;cAsFrD;AAyVD,wBAAsB,gCAAgC,CACpD,KAAK,EAAE,oBAAoB,EAC3B,KAAK,EAAE,KAAK,EACZ,IAAI,EAAE,+BAA+B,GACpC,OAAO,CAAC,QAAQ,CAAC,CA2anB"}
|
|
@@ -238,6 +238,10 @@ function getAllowedRemoteToolNames(forwardedProps) {
|
|
|
238
238
|
}
|
|
239
239
|
return allowedTools.every((toolName) => typeof toolName === "string") ? allowedTools : [];
|
|
240
240
|
}
|
|
241
|
+
function getForwardedMaxOutputTokens(forwardedProps) {
|
|
242
|
+
const value = forwardedProps?.maxOutputTokens;
|
|
243
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined;
|
|
244
|
+
}
|
|
241
245
|
/**
|
|
242
246
|
* Reads the restrictive `runtimeOverrides.toolAllowlist` override.
|
|
243
247
|
*
|
|
@@ -442,6 +446,7 @@ export async function createRuntimeAgentStreamResponse(input, agent, deps) {
|
|
|
442
446
|
new AgentRuntime(runtimeAgent.id, runtimeAgent.config);
|
|
443
447
|
let completedResponse = null;
|
|
444
448
|
const runtimeMessages = compactRuntimeMessagesForStream(normalizeAgUiRuntimeMessages(input.messages), mergedTools);
|
|
449
|
+
const maxOutputTokens = getForwardedMaxOutputTokens(input.forwardedProps);
|
|
445
450
|
let runtimeStream;
|
|
446
451
|
let clientAttached = true;
|
|
447
452
|
try {
|
|
@@ -459,7 +464,7 @@ export async function createRuntimeAgentStreamResponse(input, agent, deps) {
|
|
|
459
464
|
onFinish: (response) => {
|
|
460
465
|
completedResponse = response;
|
|
461
466
|
},
|
|
462
|
-
}, undefined,
|
|
467
|
+
}, undefined, maxOutputTokens, abortSignal);
|
|
463
468
|
logger.info("Internal agent runtime stream attached", {
|
|
464
469
|
runId: input.runId,
|
|
465
470
|
threadId: input.threadId,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "veryfront",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1086",
|
|
4
4
|
"description": "The simplest way to build AI-powered apps",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"react",
|
|
@@ -328,10 +328,10 @@
|
|
|
328
328
|
"@types/react": "19.2.14",
|
|
329
329
|
"@types/react-dom": "19.2.3",
|
|
330
330
|
"ws": "8.21.0",
|
|
331
|
-
"@veryfront/ext-bundler-esbuild": "0.1.
|
|
332
|
-
"@veryfront/ext-content-mdx": "0.1.
|
|
333
|
-
"@veryfront/ext-css-tailwind": "0.1.
|
|
334
|
-
"@veryfront/ext-parser-babel": "0.1.
|
|
331
|
+
"@veryfront/ext-bundler-esbuild": "0.1.1086",
|
|
332
|
+
"@veryfront/ext-content-mdx": "0.1.1086",
|
|
333
|
+
"@veryfront/ext-css-tailwind": "0.1.1086",
|
|
334
|
+
"@veryfront/ext-parser-babel": "0.1.1086"
|
|
335
335
|
},
|
|
336
336
|
"devDependencies": {
|
|
337
337
|
"@types/node": "20.9.0"
|