veryfront 0.1.1085 → 0.1.1087
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command-help.d.ts.map +1 -1
- package/esm/cli/commands/eval/command-help.js +4 -2
- package/esm/cli/commands/eval/command.d.ts.map +1 -1
- package/esm/cli/commands/eval/command.js +196 -11
- package/esm/deno.js +1 -1
- package/esm/extensions/ext-eval-report-mlflow/src/index.d.ts +5 -0
- package/esm/extensions/ext-eval-report-mlflow/src/index.d.ts.map +1 -1
- package/esm/extensions/ext-eval-report-mlflow/src/index.js +109 -21
- package/esm/src/extensions/builtin-extensions.d.ts.map +1 -1
- package/esm/src/extensions/builtin-extensions.js +5 -0
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +5 -5
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,WAiGtB,CAAC"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
export const evalHelp = {
|
|
2
2
|
name: "eval",
|
|
3
3
|
category: "ai",
|
|
4
|
-
description: "
|
|
4
|
+
description: "List, run, and export discovered eval definitions",
|
|
5
5
|
usage: "veryfront eval [eval-id] [options]",
|
|
6
6
|
options: [
|
|
7
7
|
{
|
|
8
8
|
flag: "-l, --list",
|
|
9
|
-
description: "List discovered evals",
|
|
9
|
+
description: "List discovered evals without running them",
|
|
10
10
|
},
|
|
11
11
|
{
|
|
12
12
|
flag: "--dataset-base <path>",
|
|
@@ -83,6 +83,7 @@ export const evalHelp = {
|
|
|
83
83
|
],
|
|
84
84
|
examples: [
|
|
85
85
|
"veryfront eval --list",
|
|
86
|
+
"veryfront eval",
|
|
86
87
|
"veryfront eval deep-research",
|
|
87
88
|
"veryfront eval eval:deep-research --report-dir .veryfront/evals/deep-research",
|
|
88
89
|
"veryfront eval eval:deep-research --report .veryfront/evals/deep-research/report.json --junit .veryfront/evals/deep-research/junit.xml",
|
|
@@ -90,6 +91,7 @@ export const evalHelp = {
|
|
|
90
91
|
"veryfront eval deep-research --baseline-model anthropic/claude-sonnet-4-6 --candidate-model moonshotai/kimi-k2.6",
|
|
91
92
|
"veryfront eval deep-research --baseline-model anthropic/claude-sonnet-4-6 --candidate-model moonshotai/kimi-k2.6 --comparison-policy evals/model-comparison.policy.json",
|
|
92
93
|
"veryfront eval deep-research --export braintrust,langfuse --json",
|
|
94
|
+
"MLFLOW_TRACKING_URI=https://mlflow.example.com veryfront eval",
|
|
93
95
|
"veryfront eval deep-research --json",
|
|
94
96
|
],
|
|
95
97
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAgC,KAAK,IAAI,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAChH,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAEL,KAAK,0BAA0B,EAE/B,KAAK,yBAAyB,EAC/B,MAAM,uCAAuC,CAAC;AAmB/C,OAAO,EACL,2BAA2B,EAG5B,MAAM,6CAA6C,CAAC;AASrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,uBAAuB;IAC/B,2BAA2B,CAAC,EAAE,OAAO,2BAA2B,CAAC;CAClE;AAED,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAgC,KAAK,IAAI,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAChH,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAEL,KAAK,0BAA0B,EAE/B,KAAK,yBAAyB,EAC/B,MAAM,uCAAuC,CAAC;AAmB/C,OAAO,EACL,2BAA2B,EAG5B,MAAM,6CAA6C,CAAC;AASrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,uBAAuB;IAC/B,2BAA2B,CAAC,EAAE,OAAO,2BAA2B,CAAC;CAClE;AAED,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AA6BF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AA+DnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AA+BD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAgBrB;AA8GD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAaD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA6CD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,GAAE,oBAAyB,IACpE,SAAS,sBAAsB;;;;;;GAiB9C;AAiJD,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,iBAAiB,EAChC,QAAQ,EAAE,0BAA0B,EACpC,MAAM,CAAC,EAAE,qBAAqB,GAAG,IAAI,GACpC,sBAAsB,GAAG,SAAS,CAkBpC;AAED,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AAuBF,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,MAAM,EAAE,CAUxF;AA2BD,wBAAgB,iCAAiC,IAAI,yBAAyB,CAuB7E;AAmVD,wBAAsB,cAAc,CAClC,OAAO,EAAE,WAAW,EACpB,YAAY,GAAE,uBAA4B,GACzC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAkP7B;AAED,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAKrE"}
|
|
@@ -18,6 +18,7 @@ import { createEvalCliBuiltinExtensions } from "../../../src/extensions/builtin-
|
|
|
18
18
|
const GATEWAY_BILLING_GROUP_USAGE_NOT_READY_CODE = "gateway_billing_group_usage_not_ready";
|
|
19
19
|
const ENV_EVAL_EXPORTERS = "VERYFRONT_EVAL_EXPORTERS";
|
|
20
20
|
const ENV_EVAL_EXPORT = "VERYFRONT_EVAL_EXPORT";
|
|
21
|
+
const ENV_MLFLOW_TRACKING_URI = "MLFLOW_TRACKING_URI";
|
|
21
22
|
const ENV_EVAL_EXPORT_INCLUDE_INPUTS = "VERYFRONT_EVAL_EXPORT_INCLUDE_INPUTS";
|
|
22
23
|
const ENV_EVAL_EXPORT_INCLUDE_OUTPUTS = "VERYFRONT_EVAL_EXPORT_INCLUDE_OUTPUTS";
|
|
23
24
|
const ENV_EVAL_EXPORT_INCLUDE_REFERENCES = "VERYFRONT_EVAL_EXPORT_INCLUDE_REFERENCES";
|
|
@@ -78,6 +79,19 @@ export function createEvalArtifactPaths(reportDir) {
|
|
|
78
79
|
reportMarkdown: join(reportDir, "report.md"),
|
|
79
80
|
};
|
|
80
81
|
}
|
|
82
|
+
function createEvalSuiteArtifactPaths(reportDir) {
|
|
83
|
+
return {
|
|
84
|
+
directory: reportDir,
|
|
85
|
+
summary: join(reportDir, "summary.json"),
|
|
86
|
+
reportMarkdown: join(reportDir, "report.md"),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
function sortEvals(evals) {
|
|
90
|
+
return [...evals].sort((left, right) => left.id.localeCompare(right.id) || left.filePath.localeCompare(right.filePath));
|
|
91
|
+
}
|
|
92
|
+
function createEvalSuiteChildDirectory(suiteDirectory, index, evalId) {
|
|
93
|
+
return join(suiteDirectory, `${String(index + 1).padStart(3, "0")}-${sanitizeEvalReportDirLabel(evalId)}`);
|
|
94
|
+
}
|
|
81
95
|
function sanitizeModelIdForPath(model) {
|
|
82
96
|
return model.trim().replace(/[^A-Za-z0-9._-]+/g, "__").replace(/^_+|_+$/g, "") || "model";
|
|
83
97
|
}
|
|
@@ -744,10 +758,18 @@ function createEvalToolCallId(toolId, context) {
|
|
|
744
758
|
return `eval-${toolId}-${context.example.id}-${context.repetition}-${dntShim.crypto.randomUUID()}`;
|
|
745
759
|
}
|
|
746
760
|
function createAgentAdapter(agent, options) {
|
|
747
|
-
return async ({ example }) => {
|
|
761
|
+
return async ({ definition, example, repetition }) => {
|
|
748
762
|
const started = Date.now();
|
|
749
763
|
const response = await agent.generate({
|
|
750
764
|
input: normalizeEvalInputForAgent(example.input),
|
|
765
|
+
context: {
|
|
766
|
+
eval: {
|
|
767
|
+
definitionId: definition.id,
|
|
768
|
+
exampleId: example.id,
|
|
769
|
+
repetition,
|
|
770
|
+
metadata: example.metadata ?? {},
|
|
771
|
+
},
|
|
772
|
+
},
|
|
751
773
|
...(options.model ? { model: options.model } : {}),
|
|
752
774
|
...(options.maxOutputTokens ? { maxOutputTokens: options.maxOutputTokens } : {}),
|
|
753
775
|
});
|
|
@@ -784,7 +806,7 @@ export function createToolAdapter(tool, baseContext = {}) {
|
|
|
784
806
|
};
|
|
785
807
|
}
|
|
786
808
|
function listEvals(evals, projectDir) {
|
|
787
|
-
return evals.map((item) => ({
|
|
809
|
+
return sortEvals(evals).map((item) => ({
|
|
788
810
|
id: item.id,
|
|
789
811
|
name: item.name,
|
|
790
812
|
target: item.definition.target,
|
|
@@ -794,6 +816,68 @@ function listEvals(evals, projectDir) {
|
|
|
794
816
|
},
|
|
795
817
|
}));
|
|
796
818
|
}
|
|
819
|
+
function unsupportedEvalSuiteOption(options) {
|
|
820
|
+
const flags = [
|
|
821
|
+
options.report ? "--report" : undefined,
|
|
822
|
+
options.junit ? "--junit" : undefined,
|
|
823
|
+
options.baseline ? "--baseline" : undefined,
|
|
824
|
+
options.writeBaseline ? "--write-baseline" : undefined,
|
|
825
|
+
options.baselinePassRateDropThreshold !== undefined
|
|
826
|
+
? "--baseline-pass-rate-drop-threshold"
|
|
827
|
+
: undefined,
|
|
828
|
+
options.baselineMetricPassRateDropThreshold !== undefined
|
|
829
|
+
? "--baseline-metric-pass-rate-drop-threshold"
|
|
830
|
+
: undefined,
|
|
831
|
+
options.baselineFailedDeltaThreshold !== undefined
|
|
832
|
+
? "--baseline-failed-delta-threshold"
|
|
833
|
+
: undefined,
|
|
834
|
+
options.baselineUsageIncreaseThreshold !== undefined
|
|
835
|
+
? "--baseline-usage-increase-threshold"
|
|
836
|
+
: undefined,
|
|
837
|
+
options.baselineLatencyIncreaseThreshold !== undefined
|
|
838
|
+
? "--baseline-latency-increase-threshold"
|
|
839
|
+
: undefined,
|
|
840
|
+
options.model ? "--model" : undefined,
|
|
841
|
+
options.maxOutputTokens ? "--max-output-tokens" : undefined,
|
|
842
|
+
options.baselineModel ? "--baseline-model" : undefined,
|
|
843
|
+
options.candidateModels.length > 0 ? "--candidate-model" : undefined,
|
|
844
|
+
options.comparisonPolicy ? "--comparison-policy" : undefined,
|
|
845
|
+
].filter((flag) => Boolean(flag));
|
|
846
|
+
if (flags.length === 0)
|
|
847
|
+
return undefined;
|
|
848
|
+
return `${flags.join(", ")} require a named eval. Run \`veryfront eval <eval-id>\`.`;
|
|
849
|
+
}
|
|
850
|
+
function createEvalSuiteSummary(runId, startedAt, results) {
|
|
851
|
+
const passed = results.filter((result) => result.status === "passed").length;
|
|
852
|
+
return {
|
|
853
|
+
kind: "eval-suite-summary",
|
|
854
|
+
runId,
|
|
855
|
+
startedAt: startedAt.toISOString(),
|
|
856
|
+
endedAt: new Date().toISOString(),
|
|
857
|
+
total: results.length,
|
|
858
|
+
passed,
|
|
859
|
+
failed: results.length - passed,
|
|
860
|
+
results,
|
|
861
|
+
};
|
|
862
|
+
}
|
|
863
|
+
function createEvalSuiteMarkdown(summary) {
|
|
864
|
+
const rows = summary.results.map((result) => `| ${markdownCell(result.id)} | ${result.status} | ${result.summary ? `${result.summary.passed}/${result.summary.records}` : "n/a"} | ${result.error ? markdownCell(result.error) : ""} |`);
|
|
865
|
+
return [
|
|
866
|
+
"# Eval suite report",
|
|
867
|
+
"",
|
|
868
|
+
`Run: \`${markdownCell(summary.runId)}\``,
|
|
869
|
+
`Result: \`${summary.passed}/${summary.total} passed\``,
|
|
870
|
+
"",
|
|
871
|
+
"| Eval | Status | Records | Error |",
|
|
872
|
+
"| --- | --- | --- | --- |",
|
|
873
|
+
...rows,
|
|
874
|
+
"",
|
|
875
|
+
].join("\n");
|
|
876
|
+
}
|
|
877
|
+
async function writeEvalSuiteArtifacts(summary, artifacts) {
|
|
878
|
+
await writeTextFileEnsuringDir(artifacts.summary, JSON.stringify(summary, null, 2));
|
|
879
|
+
await writeTextFileEnsuringDir(artifacts.reportMarkdown, createEvalSuiteMarkdown(summary));
|
|
880
|
+
}
|
|
797
881
|
function getDiscoveredEvals(runtime) {
|
|
798
882
|
return [...runtime.evals.entries()].map(([id, definition]) => {
|
|
799
883
|
if (!definition.source) {
|
|
@@ -876,7 +960,10 @@ export function resolveEvalExporterIds(options) {
|
|
|
876
960
|
const envExporters = parseEvalExporterList(readEvalCliEnv(ENV_EVAL_EXPORTERS));
|
|
877
961
|
if (envExporters.length > 0)
|
|
878
962
|
return uniqueValues(envExporters);
|
|
879
|
-
|
|
963
|
+
const legacyExporters = parseEvalExporterList(readEvalCliEnv(ENV_EVAL_EXPORT));
|
|
964
|
+
if (legacyExporters.length > 0)
|
|
965
|
+
return uniqueValues(legacyExporters);
|
|
966
|
+
return readEvalCliEnv(ENV_MLFLOW_TRACKING_URI) ? ["mlflow"] : [];
|
|
880
967
|
}
|
|
881
968
|
function parseEvalExportBooleanEnv(name) {
|
|
882
969
|
const value = readEvalCliEnv(name)?.toLowerCase();
|
|
@@ -1085,6 +1172,82 @@ async function outputEvalUsageError(message) {
|
|
|
1085
1172
|
}
|
|
1086
1173
|
return 2;
|
|
1087
1174
|
}
|
|
1175
|
+
async function runEvalSuite(input) {
|
|
1176
|
+
const runId = createEvalRunId();
|
|
1177
|
+
const startedAt = new Date();
|
|
1178
|
+
const artifacts = createEvalSuiteArtifactPaths(input.options.reportDir ?? createDefaultEvalReportDir(runId));
|
|
1179
|
+
const provenance = await resolveEvalRunProvenance({
|
|
1180
|
+
projectDir: input.projectDir,
|
|
1181
|
+
frameworkVersion: VERSION,
|
|
1182
|
+
});
|
|
1183
|
+
const results = [];
|
|
1184
|
+
for (const [index, evalItem] of sortEvals(input.evals).entries()) {
|
|
1185
|
+
const evalArtifacts = createEvalArtifactPaths(createEvalSuiteChildDirectory(artifacts.directory, index, evalItem.id));
|
|
1186
|
+
try {
|
|
1187
|
+
const agentId = evalItem.definition.targetKind === "agent"
|
|
1188
|
+
? resolveAgentTargetId(evalItem.definition.target)
|
|
1189
|
+
: undefined;
|
|
1190
|
+
const toolId = evalItem.definition.targetKind === "tool"
|
|
1191
|
+
? resolveToolTargetId(evalItem.definition.target)
|
|
1192
|
+
: undefined;
|
|
1193
|
+
const agent = agentId ? input.projectRuntime.agents.get(agentId) : undefined;
|
|
1194
|
+
const tool = toolId ? input.projectRuntime.tools.get(toolId) : undefined;
|
|
1195
|
+
if (agentId && !agent)
|
|
1196
|
+
throw new Error(`Agent "${agentId}" not found for eval target.`);
|
|
1197
|
+
if (toolId && !tool)
|
|
1198
|
+
throw new Error(`Tool "${toolId}" not found for eval target.`);
|
|
1199
|
+
const evalRunId = `${runId}_${String(index + 1).padStart(3, "0")}`;
|
|
1200
|
+
const exportConfig = createEvalCliExportConfig(evalItem, input.options, input.projectDir, evalArtifacts, input.exporterRegistry, input.config);
|
|
1201
|
+
const finalizedReport = await runEvalWithGatewayBillingGroup(evalRunId, () => runEval(evalItem.definition, {
|
|
1202
|
+
baseDir: input.options.datasetBase ?? input.projectDir,
|
|
1203
|
+
runId: evalRunId,
|
|
1204
|
+
adapters: evalItem.definition.targetKind === "tool"
|
|
1205
|
+
? { tool: createToolAdapter(tool, createEvalToolExecutionContext(input.config)) }
|
|
1206
|
+
: { agent: createAgentAdapter(agent, input.options) },
|
|
1207
|
+
metadata: { provenance },
|
|
1208
|
+
}));
|
|
1209
|
+
const report = await exportEvalReportForCli(finalizedReport, exportConfig);
|
|
1210
|
+
await writeEvalArtifacts(report, evalArtifacts);
|
|
1211
|
+
const status = createEvalExitCode(report) === 0 ? "passed" : "failed";
|
|
1212
|
+
results.push({
|
|
1213
|
+
id: evalItem.id,
|
|
1214
|
+
name: evalItem.name,
|
|
1215
|
+
target: evalItem.definition.target,
|
|
1216
|
+
status,
|
|
1217
|
+
artifacts: evalArtifacts,
|
|
1218
|
+
summary: summarizeReportForCli(report),
|
|
1219
|
+
});
|
|
1220
|
+
if (!isJsonMode()) {
|
|
1221
|
+
printReport(report);
|
|
1222
|
+
cliLogger.info(`Report directory: ${evalArtifacts.directory}`);
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
catch (error) {
|
|
1226
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1227
|
+
results.push({
|
|
1228
|
+
id: evalItem.id,
|
|
1229
|
+
name: evalItem.name,
|
|
1230
|
+
target: evalItem.definition.target,
|
|
1231
|
+
status: "error",
|
|
1232
|
+
artifacts: evalArtifacts,
|
|
1233
|
+
error: message,
|
|
1234
|
+
});
|
|
1235
|
+
if (!isJsonMode())
|
|
1236
|
+
cliLogger.error(`Eval ${evalItem.id}: ${message}`);
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
const summary = createEvalSuiteSummary(runId, startedAt, results);
|
|
1240
|
+
await writeEvalSuiteArtifacts(summary, artifacts);
|
|
1241
|
+
if (isJsonMode()) {
|
|
1242
|
+
await outputJson(createSuccessEnvelope("eval", { suite: summary, artifacts }));
|
|
1243
|
+
}
|
|
1244
|
+
else {
|
|
1245
|
+
cliLogger.info(`Eval suite: ${summary.passed}/${summary.total} passed`);
|
|
1246
|
+
cliLogger.info(`Report directory: ${artifacts.directory}`);
|
|
1247
|
+
cliLogger.info(`Suite report: ${artifacts.reportMarkdown}`);
|
|
1248
|
+
}
|
|
1249
|
+
return summary.failed === 0 ? 0 : 1;
|
|
1250
|
+
}
|
|
1088
1251
|
export async function runEvalCommand(options, dependencies = {}) {
|
|
1089
1252
|
const projectDir = options.projectDir ?? dntShim.Deno.cwd();
|
|
1090
1253
|
const discoverRuntime = dependencies.discoverProjectAgentRuntime ?? discoverProjectAgentRuntime;
|
|
@@ -1128,17 +1291,39 @@ export async function runEvalCommand(options, dependencies = {}) {
|
|
|
1128
1291
|
return undefined;
|
|
1129
1292
|
}
|
|
1130
1293
|
if (!options.id) {
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1294
|
+
const invalidOption = unsupportedEvalSuiteOption(options);
|
|
1295
|
+
if (invalidOption)
|
|
1296
|
+
return await outputEvalUsageError(invalidOption);
|
|
1297
|
+
if (evals.length === 0) {
|
|
1298
|
+
if (isJsonMode()) {
|
|
1299
|
+
await outputJson(createSuccessEnvelope("eval", {
|
|
1300
|
+
suite: null,
|
|
1301
|
+
errors: projectRuntime.errors.map((error) => ({
|
|
1302
|
+
filePath: error.file,
|
|
1303
|
+
error: error.error.message,
|
|
1304
|
+
})),
|
|
1305
|
+
}));
|
|
1306
|
+
}
|
|
1307
|
+
else {
|
|
1308
|
+
cliLogger.info("No evals found.");
|
|
1309
|
+
}
|
|
1310
|
+
return 0;
|
|
1311
|
+
}
|
|
1312
|
+
const selectedExporterIds = resolveEvalExporterIds(options);
|
|
1313
|
+
const extensionSetup = await setupEvalCliExtensions(projectDir, config, selectedExporterIds);
|
|
1314
|
+
try {
|
|
1315
|
+
return await runWithProjectAgentRuntime(projectRuntime, () => runEvalSuite({
|
|
1316
|
+
evals,
|
|
1317
|
+
options,
|
|
1318
|
+
projectDir,
|
|
1319
|
+
projectRuntime,
|
|
1320
|
+
config,
|
|
1321
|
+
exporterRegistry: extensionSetup.exporterRegistry,
|
|
1136
1322
|
}));
|
|
1137
1323
|
}
|
|
1138
|
-
|
|
1139
|
-
|
|
1324
|
+
finally {
|
|
1325
|
+
await extensionSetup.loader.teardownAll();
|
|
1140
1326
|
}
|
|
1141
|
-
return 2;
|
|
1142
1327
|
}
|
|
1143
1328
|
const evalId = normalizeEvalCliId(options.id);
|
|
1144
1329
|
const evalItem = findEvalForCliId(evals, options.id);
|
package/esm/deno.js
CHANGED
|
@@ -11,6 +11,11 @@ export interface EvalReportMlflowExtensionConfig {
|
|
|
11
11
|
trackingToken?: string;
|
|
12
12
|
trackingUsername?: string;
|
|
13
13
|
trackingPassword?: string;
|
|
14
|
+
oauthTokenUrl?: string;
|
|
15
|
+
oauthClientId?: string;
|
|
16
|
+
oauthClientSecret?: string;
|
|
17
|
+
oauthScope?: string;
|
|
18
|
+
exportArtifacts?: boolean;
|
|
14
19
|
fetch?: EvalReportMlflowFetch;
|
|
15
20
|
}
|
|
16
21
|
export interface MlflowArtifactUploadResult {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-eval-report-mlflow/src/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EACL,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EAGvB,KAAK,uBAAuB,EAC7B,MAAM,uCAAuC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-eval-report-mlflow/src/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EACL,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EAGvB,KAAK,uBAAuB,EAC7B,MAAM,uCAAuC,CAAC;AA4C/C,KAAK,qBAAqB,GAAG,CAC3B,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAC7B,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AA8CvB,MAAM,WAAW,+BAA+B;IAC9C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,KAAK,CAAC,EAAE,qBAAqB,CAAC;CAC/B;AASD,MAAM,WAAW,0BAA0B;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAkpCD,qBAAa,wBAAyB,YAAW,kBAAkB;IACjE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAIrB;IACF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAwB;gBAGhD,MAAM,EAAE,+BAA+B,GAAG;QACxC,EAAE,EAAE,MAAM,CAAC;QACX,WAAW,EAAE,MAAM,CAAC;KACrB,EACD,SAAS,GAAE,qBAA6B;IAgBpC,MAAM,CACV,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE,uBAAuB,GAC/B,OAAO,CAAC,uBAAuB,CAAC;CA6GpC;AAED,wBAAgB,8BAA8B,CAC5C,MAAM,EAAE,+BAA+B,GAAG;IACxC,WAAW,EAAE,MAAM,CAAC;CACrB,EACD,SAAS,CAAC,EAAE,qBAAqB,GAChC,wBAAwB,CAK1B;AAED,QAAA,MAAM,mBAAmB,EAAE,gBA0C1B,CAAC;AAEF,eAAe,mBAAmB,CAAC"}
|
|
@@ -18,6 +18,11 @@ const ENV_ARTIFACTS_URI = "MLFLOW_ARTIFACTS_URI";
|
|
|
18
18
|
const ENV_TRACKING_TOKEN = "MLFLOW_TRACKING_TOKEN";
|
|
19
19
|
const ENV_TRACKING_USERNAME = "MLFLOW_TRACKING_USERNAME";
|
|
20
20
|
const ENV_TRACKING_PASSWORD = "MLFLOW_TRACKING_PASSWORD";
|
|
21
|
+
const ENV_OAUTH_TOKEN_URL = "MLFLOW_OAUTH_TOKEN_URL";
|
|
22
|
+
const ENV_OAUTH_CLIENT_ID = "MLFLOW_OAUTH_CLIENT_ID";
|
|
23
|
+
const ENV_OAUTH_CLIENT_SECRET = "MLFLOW_OAUTH_CLIENT_SECRET";
|
|
24
|
+
const ENV_OAUTH_SCOPE = "MLFLOW_OAUTH_SCOPE";
|
|
25
|
+
const ENV_EXPORT_ARTIFACTS = "MLFLOW_EXPORT_ARTIFACTS";
|
|
21
26
|
const EXTENSION_METADATA = {
|
|
22
27
|
contracts: {
|
|
23
28
|
requires: ["EvalReportExporterRegistry"],
|
|
@@ -35,6 +40,11 @@ const EXTENSION_METADATA = {
|
|
|
35
40
|
"MLFLOW_TRACKING_TOKEN",
|
|
36
41
|
"MLFLOW_TRACKING_URI",
|
|
37
42
|
"MLFLOW_TRACKING_USERNAME",
|
|
43
|
+
"MLFLOW_OAUTH_TOKEN_URL",
|
|
44
|
+
"MLFLOW_OAUTH_CLIENT_ID",
|
|
45
|
+
"MLFLOW_OAUTH_CLIENT_SECRET",
|
|
46
|
+
"MLFLOW_OAUTH_SCOPE",
|
|
47
|
+
"MLFLOW_EXPORT_ARTIFACTS",
|
|
38
48
|
],
|
|
39
49
|
},
|
|
40
50
|
],
|
|
@@ -82,6 +92,15 @@ function normalizeConfig(config) {
|
|
|
82
92
|
...(typeof config.trackingPassword === "string"
|
|
83
93
|
? { trackingPassword: config.trackingPassword }
|
|
84
94
|
: {}),
|
|
95
|
+
...(typeof config.oauthTokenUrl === "string" ? { oauthTokenUrl: config.oauthTokenUrl } : {}),
|
|
96
|
+
...(typeof config.oauthClientId === "string" ? { oauthClientId: config.oauthClientId } : {}),
|
|
97
|
+
...(typeof config.oauthClientSecret === "string"
|
|
98
|
+
? { oauthClientSecret: config.oauthClientSecret }
|
|
99
|
+
: {}),
|
|
100
|
+
...(typeof config.oauthScope === "string" ? { oauthScope: config.oauthScope } : {}),
|
|
101
|
+
...(typeof config.exportArtifacts === "boolean"
|
|
102
|
+
? { exportArtifacts: config.exportArtifacts }
|
|
103
|
+
: {}),
|
|
85
104
|
...(typeof config.fetch === "function" ? { fetch: config.fetch } : {}),
|
|
86
105
|
};
|
|
87
106
|
}
|
|
@@ -163,6 +182,56 @@ function createTrackingAuthHeaders(config) {
|
|
|
163
182
|
}
|
|
164
183
|
return {};
|
|
165
184
|
}
|
|
185
|
+
function readBooleanEnv(name) {
|
|
186
|
+
const value = readEnv(name)?.toLowerCase();
|
|
187
|
+
if (value === undefined)
|
|
188
|
+
return undefined;
|
|
189
|
+
if (["1", "true", "yes", "on"].includes(value))
|
|
190
|
+
return true;
|
|
191
|
+
if (["0", "false", "no", "off"].includes(value))
|
|
192
|
+
return false;
|
|
193
|
+
throw new Error(`MLflow ${name} must be true or false.`);
|
|
194
|
+
}
|
|
195
|
+
function resolveOAuthClientCredentials(config) {
|
|
196
|
+
const configured = [
|
|
197
|
+
config.oauthTokenUrl,
|
|
198
|
+
config.oauthClientId,
|
|
199
|
+
config.oauthClientSecret,
|
|
200
|
+
].some((value) => value !== undefined && value.length > 0);
|
|
201
|
+
if (!configured)
|
|
202
|
+
return undefined;
|
|
203
|
+
if (!config.oauthTokenUrl || !config.oauthClientId || !config.oauthClientSecret) {
|
|
204
|
+
throw new Error("MLflow OAuth requires MLFLOW_OAUTH_TOKEN_URL, MLFLOW_OAUTH_CLIENT_ID, and MLFLOW_OAUTH_CLIENT_SECRET together.");
|
|
205
|
+
}
|
|
206
|
+
return {
|
|
207
|
+
tokenUrl: normalizeHttpUri(config.oauthTokenUrl, "OAuth token URL"),
|
|
208
|
+
clientId: config.oauthClientId,
|
|
209
|
+
clientSecret: config.oauthClientSecret,
|
|
210
|
+
...(config.oauthScope ? { scope: config.oauthScope } : {}),
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
async function createOAuthTrackingAuthHeaders(fetchImpl, config) {
|
|
214
|
+
const body = new URLSearchParams({ grant_type: "client_credentials" });
|
|
215
|
+
if (config.scope)
|
|
216
|
+
body.set("scope", config.scope);
|
|
217
|
+
const response = await fetchImpl(config.tokenUrl, {
|
|
218
|
+
method: "POST",
|
|
219
|
+
headers: {
|
|
220
|
+
accept: "application/json",
|
|
221
|
+
authorization: `Basic ${btoa(`${config.clientId}:${config.clientSecret}`)}`,
|
|
222
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
223
|
+
},
|
|
224
|
+
body: body.toString(),
|
|
225
|
+
});
|
|
226
|
+
const payload = await response.json().catch(() => undefined);
|
|
227
|
+
if (!response.ok) {
|
|
228
|
+
throw new Error(`MLflow OAuth token request failed (${response.status}).`);
|
|
229
|
+
}
|
|
230
|
+
if (!isRecord(payload) || typeof payload.access_token !== "string" || !payload.access_token) {
|
|
231
|
+
throw new Error("MLflow OAuth token response did not include an access_token.");
|
|
232
|
+
}
|
|
233
|
+
return { authorization: `Bearer ${payload.access_token}` };
|
|
234
|
+
}
|
|
166
235
|
function resolveExporterConfig(config) {
|
|
167
236
|
return {
|
|
168
237
|
id: DEFAULT_EXPORTER_ID,
|
|
@@ -174,6 +243,11 @@ function resolveExporterConfig(config) {
|
|
|
174
243
|
trackingToken: config.trackingToken ?? readEnv(ENV_TRACKING_TOKEN),
|
|
175
244
|
trackingUsername: config.trackingUsername ?? readEnv(ENV_TRACKING_USERNAME),
|
|
176
245
|
trackingPassword: config.trackingPassword ?? readEnv(ENV_TRACKING_PASSWORD),
|
|
246
|
+
oauthTokenUrl: config.oauthTokenUrl ?? readEnv(ENV_OAUTH_TOKEN_URL),
|
|
247
|
+
oauthClientId: config.oauthClientId ?? readEnv(ENV_OAUTH_CLIENT_ID),
|
|
248
|
+
oauthClientSecret: config.oauthClientSecret ?? readEnv(ENV_OAUTH_CLIENT_SECRET),
|
|
249
|
+
oauthScope: config.oauthScope ?? readEnv(ENV_OAUTH_SCOPE),
|
|
250
|
+
exportArtifacts: config.exportArtifacts ?? readBooleanEnv(ENV_EXPORT_ARTIFACTS),
|
|
177
251
|
...(config.fetch ? { fetch: config.fetch } : {}),
|
|
178
252
|
};
|
|
179
253
|
}
|
|
@@ -775,16 +849,21 @@ export class EvalReportMlflowExporter {
|
|
|
775
849
|
this.id = config.id;
|
|
776
850
|
const trackingUri = normalizeHttpUri(config.trackingUri, "trackingUri");
|
|
777
851
|
const artifactsUri = resolveArtifactsUri(config, trackingUri);
|
|
852
|
+
const oauth = resolveOAuthClientCredentials(config);
|
|
778
853
|
this.config = {
|
|
779
854
|
...config,
|
|
780
855
|
trackingUri,
|
|
856
|
+
exportArtifacts: config.exportArtifacts ?? true,
|
|
781
857
|
...(artifactsUri ? { artifactsUri: normalizeHttpUri(artifactsUri, "artifactsUri") } : {}),
|
|
858
|
+
...(oauth ? { oauth } : {}),
|
|
782
859
|
};
|
|
783
860
|
this.fetchImpl = fetchImpl;
|
|
784
861
|
}
|
|
785
862
|
async export(report, context) {
|
|
786
863
|
const trackingUri = this.config.trackingUri;
|
|
787
|
-
const authHeaders =
|
|
864
|
+
const authHeaders = this.config.oauth
|
|
865
|
+
? await createOAuthTrackingAuthHeaders(this.fetchImpl, this.config.oauth)
|
|
866
|
+
: createTrackingAuthHeaders(this.config);
|
|
788
867
|
const selectedExperimentName = experimentName(this.config, report, context);
|
|
789
868
|
const experimentId = await getOrCreateMlflowExperiment({
|
|
790
869
|
fetchImpl: this.fetchImpl,
|
|
@@ -815,30 +894,39 @@ export class EvalReportMlflowExporter {
|
|
|
815
894
|
tag("veryfront.summary.passed", report.summary.passed),
|
|
816
895
|
]),
|
|
817
896
|
});
|
|
818
|
-
const artifacts =
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
897
|
+
const artifacts = this.config.exportArtifacts
|
|
898
|
+
? await uploadReportArtifacts({
|
|
899
|
+
fetchImpl: this.fetchImpl,
|
|
900
|
+
artifactsUri: this.config.artifactsUri ??
|
|
901
|
+
(runArtifactUri.startsWith("mlflow-artifacts:/") ? trackingUri : undefined),
|
|
902
|
+
runArtifactUri,
|
|
903
|
+
report,
|
|
904
|
+
})
|
|
905
|
+
: [];
|
|
906
|
+
const artifactVerification = this.config.exportArtifacts
|
|
907
|
+
? await verifyUploadedArtifacts({
|
|
908
|
+
fetchImpl: this.fetchImpl,
|
|
909
|
+
trackingUri,
|
|
910
|
+
authHeaders,
|
|
911
|
+
runId,
|
|
912
|
+
artifacts,
|
|
913
|
+
})
|
|
914
|
+
: undefined;
|
|
831
915
|
await logMlflowBatch(this.fetchImpl, trackingUri, {
|
|
832
916
|
runId,
|
|
833
917
|
authHeaders,
|
|
834
918
|
tags: [
|
|
835
|
-
{ key: "artifacts.logged", value:
|
|
919
|
+
{ key: "artifacts.logged", value: String(this.config.exportArtifacts) },
|
|
836
920
|
{ key: "artifacts.count", value: String(artifacts.length) },
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
921
|
+
...(artifactVerification
|
|
922
|
+
? [
|
|
923
|
+
{ key: "artifacts.verified", value: String(artifactVerification.verified) },
|
|
924
|
+
{
|
|
925
|
+
key: "artifacts.verified_count",
|
|
926
|
+
value: String(artifactVerification.artifacts.length - artifactVerification.missing.length),
|
|
927
|
+
},
|
|
928
|
+
]
|
|
929
|
+
: []),
|
|
842
930
|
],
|
|
843
931
|
});
|
|
844
932
|
await mlflowPost(this.fetchImpl, trackingUri, authHeaders, "runs/update", {
|
|
@@ -853,7 +941,7 @@ export class EvalReportMlflowExporter {
|
|
|
853
941
|
experimentId,
|
|
854
942
|
experimentName: selectedExperimentName,
|
|
855
943
|
artifacts,
|
|
856
|
-
artifactVerification,
|
|
944
|
+
...(artifactVerification ? { artifactVerification } : {}),
|
|
857
945
|
},
|
|
858
946
|
};
|
|
859
947
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"builtin-extensions.d.ts","sourceRoot":"","sources":["../../../src/src/extensions/builtin-extensions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAGV,yBAAyB,EACzB,gBAAgB,EAChB,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAEpB,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAMlE,OAAO,KAAK,EAAe,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAevE,MAAM,MAAM,kCAAkC,GAAG;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B,CAAC;AAoBF,eAAO,MAAM,2BAA2B,EAAE,kCAAkC,
|
|
1
|
+
{"version":3,"file":"builtin-extensions.d.ts","sourceRoot":"","sources":["../../../src/src/extensions/builtin-extensions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EAGV,yBAAyB,EACzB,gBAAgB,EAChB,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAEpB,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAMlE,OAAO,KAAK,EAAe,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAevE,MAAM,MAAM,kCAAkC,GAAG;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,yBAAyB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B,CAAC;AAoBF,eAAO,MAAM,2BAA2B,EAAE,kCAAkC,EAsI3E,CAAC;AAWF,wBAAgB,uCAAuC,IAAI,0BAA0B,CASpF;AAWD,wBAAgB,yBAAyB,IAAI,mBAAmB,CAM/D;AAED,wBAAgB,4BAA4B,IAAI,IAAI,CAInD;AA0CD,wBAAgB,8BAA8B,CAC5C,UAAU,EAAE,kCAAkC,GAC7C,iBAAiB,CA6BnB;AAwDD,wBAAgB,uBAAuB,IAAI,iBAAiB,EAAE,CAa7D;AAED,wBAAgB,8BAA8B,CAC5C,mBAAmB,GAAE,MAAM,EAAO,GACjC,iBAAiB,EAAE,CAiBrB"}
|
|
@@ -150,6 +150,11 @@ export const OPTIONAL_BUILTIN_EXTENSIONS = [
|
|
|
150
150
|
"MLFLOW_TRACKING_TOKEN",
|
|
151
151
|
"MLFLOW_TRACKING_URI",
|
|
152
152
|
"MLFLOW_TRACKING_USERNAME",
|
|
153
|
+
"MLFLOW_OAUTH_TOKEN_URL",
|
|
154
|
+
"MLFLOW_OAUTH_CLIENT_ID",
|
|
155
|
+
"MLFLOW_OAUTH_CLIENT_SECRET",
|
|
156
|
+
"MLFLOW_OAUTH_SCOPE",
|
|
157
|
+
"MLFLOW_EXPORT_ARTIFACTS",
|
|
153
158
|
],
|
|
154
159
|
},
|
|
155
160
|
],
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "veryfront",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1087",
|
|
4
4
|
"description": "The simplest way to build AI-powered apps",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"react",
|
|
@@ -328,10 +328,10 @@
|
|
|
328
328
|
"@types/react": "19.2.14",
|
|
329
329
|
"@types/react-dom": "19.2.3",
|
|
330
330
|
"ws": "8.21.0",
|
|
331
|
-
"@veryfront/ext-bundler-esbuild": "0.1.
|
|
332
|
-
"@veryfront/ext-content-mdx": "0.1.
|
|
333
|
-
"@veryfront/ext-css-tailwind": "0.1.
|
|
334
|
-
"@veryfront/ext-parser-babel": "0.1.
|
|
331
|
+
"@veryfront/ext-bundler-esbuild": "0.1.1087",
|
|
332
|
+
"@veryfront/ext-content-mdx": "0.1.1087",
|
|
333
|
+
"@veryfront/ext-css-tailwind": "0.1.1087",
|
|
334
|
+
"@veryfront/ext-parser-babel": "0.1.1087"
|
|
335
335
|
},
|
|
336
336
|
"devDependencies": {
|
|
337
337
|
"@types/node": "20.9.0"
|