veryfront 0.1.1055 → 0.1.1059

package/esm/cli/commands/eval/command-help.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,WA2EtB,CAAC"}
+{"version":3,"file":"command-help.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command-help.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,eAAO,MAAM,QAAQ,EAAE,WA+FtB,CAAC"}

package/esm/cli/commands/eval/command-help.js

@@ -32,6 +32,26 @@ export const evalHelp = {
             flag: "--write-baseline <path>",
             description: "Write the eval report JSON as the next baseline",
         },
+        {
+            flag: "--baseline-pass-rate-drop-threshold <fraction>",
+            description: "Allow this aggregate pass-rate drop before a baseline regression fails",
+        },
+        {
+            flag: "--baseline-metric-pass-rate-drop-threshold <fraction>",
+            description: "Allow this per-metric pass-rate drop before a baseline regression fails",
+        },
+        {
+            flag: "--baseline-failed-delta-threshold <count>",
+            description: "Allow this failed-result count increase before a baseline regression fails",
+        },
+        {
+            flag: "--baseline-usage-increase-threshold <fraction>",
+            description: "Fail when reported usage or cost increases beyond this baseline fraction",
+        },
+        {
+            flag: "--baseline-latency-increase-threshold <fraction>",
+            description: "Fail when p95 latency increases beyond this baseline fraction",
+        },
         {
             flag: "--export <ids>",
             description: "Export the report through registered eval exporters",

package/esm/cli/commands/eval/command.d.ts

@@ -25,10 +25,12 @@ type EvalModelArtifactPaths = EvalArtifactPaths & {
 };
 type EvalSummaryArtifact = {
     kind: "eval-summary";
+    schemaVersion: number;
     runId: string;
     definitionId: string;
     targetKind: EvalReport["targetKind"];
     target: string;
+    dataset?: EvalReport["dataset"];
     startedAt: string;
     endedAt: string;
     summary: EvalReport["summary"];

package/esm/cli/commands/eval/command.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA0BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAoDnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAcrB;AA0FD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA8LF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAsJrE"}
+{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/command.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAS,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,mBAAmB,EAEnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EAEpB,sBAAsB,EACtB,YAAY,EAEb,MAAM,4BAA4B,CAAC;AA2BpC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,WAAW,WAAY,SAAQ,QAAQ;CAAG;AAEhD,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AASF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAC/B,QAAQ,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC;IAChC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,CAAC;AAEF,KAAK,+BAA+B,GAAG;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAOF,KAAK,6BAA6B,GAAG;IACnC,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,yBAAyB,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAoDnF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAO5E;AAMD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CASxB;AAiDD,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAErD;AAQD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAKhG;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CASjE;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,cAAc,CAWxE;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,mBAAmB,CAgBrB;AA8GD,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,+BAA+B,GAC5C,UAAU,CAiBZ;AAWD,wBAAsB,2BAA2B,CAC/C,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE,6BAAkC,GAC1C,OAAO,CAAC,+BAA+B,GAAG,SAAS,CAAC,CAgDtD;AAED,wBAAsB,8BAA8B,CAClD,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAerB;AAED,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAGrB;AA8FD,wBAAsB,6BAA6B,CACjD,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,yBAAyB,CAAC,CA0CpC;AAED,wBAAsB,uCAAuC,CAC3D,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,iCAAiC,GAAG,SAAS,CAAC,CAKxD;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,UAAU,EAAE,EACrB,aAAa,EAAE,MAAM,EACrB,MAAM,GAAE,yBAA8B,GACrC,mBAAmB,CAErB;AAED,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,CAE9E;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAG7D;AAkED,uEAAuE;AACvE,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,MAAM,CAkFR;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA0CzD;AAQD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,iBAAiB,EACxB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAQf;AAMD,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,oBAAoB,GAC9B,CAAC,GAAG,CAAC,CAEP;AAMD,KAAK,qBAAqB,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAQF,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,qBAAqB,GAAG,IAAI,GAAG,SAAS,sEAMjD;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,aAAa,MA0DrD;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,GAAG,YAAY,EAAE,CAY1E;AA+FD,KAAK,yBAAyB,GAAG;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CAAC;AAEF,KAAK,iCAAiC,GAAG;IACvC,MAAM,EAAE,yBAAyB,CAAC;IAClC,MAAM,EAAE,yBAAyB,CAAC;CACnC,CAAC;AA8LF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA0JrE"}

package/esm/cli/commands/eval/command.js

@@ -4,7 +4,7 @@
 import * as dntShim from "../../../_dnt.shims.js";
 import { dirname, isAbsolute, join, relative, resolve } from "../../../deps/jsr.io/@std/path/1.1.4/mod.js";
 import { createProjectDiscoveryConfig, discoverAll } from "../../../src/discovery/index.js";
-import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, createEvalRunId, discoverEvals, exportEvalReport, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
+import { compareEvalModelReports, compareEvalReports, createEvalModelComparisonMarkdown, createEvalRunId, discoverEvals, EVAL_REPORT_SCHEMA_VERSION, exportEvalReport, resolveEvalRunProvenance, runEval, } from "../../../src/eval/index.js";
 import { getCurrentVeryfrontCloudContext, getVeryfrontCloudBootstrap, runWithVeryfrontCloudContextAsync, } from "../../../src/provider/index.js";
 import { applyRuntimeAuthContext } from "../../shared/runtime-auth.js";
 import { cliLogger, exitProcess, VERSION } from "../../utils/index.js";
@@ -153,10 +153,12 @@ export function summarizeReportForCli(report) {
 export function createSummaryArtifact(report, baseline) {
     return {
         kind: "eval-summary",
+        schemaVersion: report.schemaVersion ?? EVAL_REPORT_SCHEMA_VERSION,
         runId: report.runId,
         definitionId: report.definitionId,
         targetKind: report.targetKind,
         target: report.target,
+        ...(report.dataset ? { dataset: report.dataset } : {}),
         startedAt: report.startedAt,
         endedAt: report.endedAt,
         summary: report.summary,
@@ -165,6 +167,25 @@ export function createSummaryArtifact(report, baseline) {
         ...(baseline ? { baseline } : {}),
     };
 }
+function createEvalBaselineComparisonPolicy(options) {
+    return {
+        ...(options.baselinePassRateDropThreshold !== undefined
+            ? { passRateDropThreshold: options.baselinePassRateDropThreshold }
+            : {}),
+        ...(options.baselineMetricPassRateDropThreshold !== undefined
+            ? { metricPassRateDropThreshold: options.baselineMetricPassRateDropThreshold }
+            : {}),
+        ...(options.baselineFailedDeltaThreshold !== undefined
+            ? { failedDeltaThreshold: options.baselineFailedDeltaThreshold }
+            : {}),
+        ...(options.baselineUsageIncreaseThreshold !== undefined
+            ? { usageIncreaseThreshold: options.baselineUsageIncreaseThreshold }
+            : {}),
+        ...(options.baselineLatencyIncreaseThreshold !== undefined
+            ? { latencyIncreaseThreshold: options.baselineLatencyIncreaseThreshold }
+            : {}),
+    };
+}
 function isRecord(value) {
     return value !== null && typeof value === "object" && !Array.isArray(value);
 }
@@ -1003,7 +1024,7 @@ export async function evalCommand(options) {
         }));
         const report = await exportEvalReportForCli(finalizedReport, exportConfig);
         const baseline = options.baseline
-            ? compareEvalReports(report, await readEvalReport(options.baseline))
+            ? compareEvalReports(report, await readEvalReport(options.baseline), createEvalBaselineComparisonPolicy(options))
             : undefined;
         await writeEvalArtifacts(report, artifactPaths, baseline);
         if (options.report) {

package/esm/cli/commands/eval/handler.d.ts

@@ -9,6 +9,11 @@ declare const getEvalArgsSchema: () => import("../../../src/internal-agents/sche
     junit: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     baseline: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     writeBaseline: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    baselinePassRateDropThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+    baselineMetricPassRateDropThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+    baselineFailedDeltaThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+    baselineUsageIncreaseThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+    baselineLatencyIncreaseThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
     exporters: import("../../../src/internal-agents/schema.js").Schema<string[]>;
     debug: import("../../../src/internal-agents/schema.js").Schema<boolean>;
     model: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
@@ -27,6 +32,11 @@ export declare const parseEvalArgs: (args: ParsedArgs) => import("../../shared/a
     junit: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     baseline: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
     writeBaseline: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;
+    baselinePassRateDropThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+    baselineMetricPassRateDropThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+    baselineFailedDeltaThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+    baselineUsageIncreaseThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
+    baselineLatencyIncreaseThreshold: import("../../../src/internal-agents/schema.js").Schema<number | undefined>;
     exporters: import("../../../src/internal-agents/schema.js").Schema<string[]>;
     debug: import("../../../src/internal-agents/schema.js").Schema<boolean>;
     model: import("../../../src/internal-agents/schema.js").Schema<string | undefined>;

package/esm/cli/commands/eval/handler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/handler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AAE3E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAA,MAAM,iBAAiB;;;;;;;;;;;;;;;;GAkBtB,CAAC;AAIF,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AAEzE,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;GAgBxB,CAAC;AAEH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAIvE"}
+{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval/handler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yCAAyC,CAAC;AAE3E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAExD,QAAA,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;GAuBtB,CAAC;AAIF,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AAEzE,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;GA2BxB,CAAC;AAEH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAIvE"}

package/esm/cli/commands/eval/handler.js

@@ -9,6 +9,11 @@ const getEvalArgsSchema = defineSchema((v) => v.object({
     junit: v.string().optional(),
     baseline: v.string().optional(),
     writeBaseline: v.string().optional(),
+    baselinePassRateDropThreshold: v.number().min(0).optional(),
+    baselineMetricPassRateDropThreshold: v.number().min(0).optional(),
+    baselineFailedDeltaThreshold: v.number().min(0).optional(),
+    baselineUsageIncreaseThreshold: v.number().min(0).optional(),
+    baselineLatencyIncreaseThreshold: v.number().min(0).optional(),
     exporters: v.array(v.string()).default([]),
     debug: v.boolean().default(false),
     model: v.string().optional(),
@@ -27,6 +32,17 @@ export const parseEvalArgs = createArgParser(EvalArgsSchema, {
     junit: { keys: ["junit"], type: "string" },
     baseline: { keys: ["baseline"], type: "string" },
     writeBaseline: { keys: ["write-baseline"], type: "string" },
+    baselinePassRateDropThreshold: { keys: ["baseline-pass-rate-drop-threshold"], type: "number" },
+    baselineMetricPassRateDropThreshold: {
+        keys: ["baseline-metric-pass-rate-drop-threshold"],
+        type: "number",
+    },
+    baselineFailedDeltaThreshold: { keys: ["baseline-failed-delta-threshold"], type: "number" },
+    baselineUsageIncreaseThreshold: { keys: ["baseline-usage-increase-threshold"], type: "number" },
+    baselineLatencyIncreaseThreshold: {
+        keys: ["baseline-latency-increase-threshold"],
+        type: "number",
+    },
     exporters: { keys: ["export"], type: "array" },
     debug: { keys: ["debug"], type: "boolean" },
     model: { keys: ["model"], type: "string" },

package/esm/deno.js

@@ -1,6 +1,6 @@
 export default {
     "name": "veryfront",
-    "version": "0.1.1055",
+    "version": "0.1.1059",
     "license": "Apache-2.0",
     "nodeModulesDir": "auto",
     "minimumDependencyAge": {

package/esm/extensions/ext-bundler-esbuild/src/binary.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Idempotent one-shot setup — copies the esbuild binary out of the VFS (if
+ * running in `deno compile`) and sets ESBUILD_BINARY_PATH. No-op on normal
+ * Deno or when the env var is already set.
+ */
+export declare function ensureEsbuildBinary(): Promise<void>;
+//# sourceMappingURL=binary.d.ts.map

package/esm/extensions/ext-bundler-esbuild/src/binary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"binary.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-bundler-esbuild/src/binary.ts"],"names":[],"mappings":"AAmHA;;;;GAIG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAiCzD"}

package/esm/extensions/ext-bundler-esbuild/src/binary.js

@@ -0,0 +1,143 @@
+/**
+ * Binary extraction for `deno compile` VFS.
+ *
+ * Ported from src/platform/compat/esbuild.ts + esbuild-shared.ts. When the
+ * framework runs as a compiled Deno binary, the esbuild native binary lives
+ * inside the VFS and must be copied to a writable path before esbuild's
+ * child-process spawn can find it.
+ *
+ * On normal Deno (not `deno compile`) or when ESBUILD_BINARY_PATH is already
+ * set, `ensureEsbuildBinary()` is a no-op.
+ *
+ * @module extensions/ext-bundler-esbuild/binary
+ */
+import * as dntShim from "../../../_dnt.shims.js";
+import { isDenoCompiled } from "./runtime.js";
+const ESBUILD_VERSION = "0.28.1";
+function getTempDir() {
+    try {
+        return (dntShim.Deno.env.get("TMPDIR") ?? dntShim.Deno.env.get("TEMP") ?? dntShim.Deno.env.get("TMP") ?? "/tmp");
+    }
+    catch {
+        return "/tmp";
+    }
+}
+function getEsbuildBinaryName() {
+    const archMap = { x86_64: "x64", aarch64: "arm64" };
+    const arch = archMap[dntShim.Deno.build.arch] ?? dntShim.Deno.build.arch;
+    return `@esbuild/${dntShim.Deno.build.os}-${arch}`;
+}
+function getVFSBasePath(filePath, tempDir) {
+    const denoCompileMatch = filePath.match(/^(.*\/deno-compile-[^/]+)\//);
+    if (denoCompileMatch?.[1])
+        return denoCompileMatch[1];
+    const parts = filePath.split("/");
+    const srcIndex = parts.lastIndexOf("src");
+    if (srcIndex > 0)
+        return parts.slice(0, srcIndex).join("/");
+    return `${tempDir}/deno-compile-veryfront`;
+}
+async function isFile(path) {
+    try {
+        const stat = await dntShim.Deno.stat(path);
+        return stat.isFile;
+    }
+    catch {
+        return false;
+    }
+}
+async function findEsbuildInVFS() {
+    const binaryName = getEsbuildBinaryName();
+    const vfsBase = getVFSBasePath(new URL(globalThis[Symbol.for("import-meta-ponyfill-esmodule")](import.meta).url).pathname, getTempDir());
+    const possiblePaths = [
+        `${vfsBase}/node_modules/${binaryName}/bin/esbuild`,
+        `${vfsBase}/node_modules/.deno/${binaryName}@${ESBUILD_VERSION}/node_modules/${binaryName}/bin/esbuild`,
+        `${vfsBase}/node_modules/.deno/esbuild@${ESBUILD_VERSION}/node_modules/${binaryName}/bin/esbuild`,
+        `${vfsBase}/node_modules/esbuild/bin/esbuild`,
+        `${vfsBase}/node_modules/.package/esbuild@${ESBUILD_VERSION}/bin/esbuild`,
+        `${vfsBase}/node_modules/.package/${binaryName}@${ESBUILD_VERSION}/bin/esbuild`,
+    ];
+    for (const p of possiblePaths) {
+        if (await isFile(p))
+            return p;
+    }
+    try {
+        const nodeModulesPath = `${vfsBase}/node_modules`;
+        for await (const entry of dntShim.Deno.readDir(nodeModulesPath)) {
+            if (entry.name !== binaryName && !entry.name.startsWith("@esbuild"))
+                continue;
+            const binPath = `${nodeModulesPath}/${entry.name}/bin/esbuild`;
+            if (await isFile(binPath))
+                return binPath;
+        }
+    }
+    catch {
+        /* node_modules not readable in VFS */
+    }
+    return null;
+}
+async function extractEsbuildBinary() {
+    const cacheDir = `${getTempDir()}/veryfront-esbuild`;
+    const targetPath = `${cacheDir}/esbuild-${ESBUILD_VERSION}`;
+    try {
+        const stat = await dntShim.Deno.stat(targetPath);
+        if (stat.isFile && stat.mode && (stat.mode & 0o111))
+            return targetPath;
+    }
+    catch {
+        /* cache miss */
+    }
+    const vfsPath = await findEsbuildInVFS();
+    if (!vfsPath) {
+        throw new Error(`[ext-bundler-esbuild] Could not find esbuild binary in deno compile VFS. ` +
+            `Platform: ${getEsbuildBinaryName()}. ` +
+            `Ensure esbuild is in dependencies and deno compile includes node_modules.`);
+    }
+    await dntShim.Deno.mkdir(cacheDir, { recursive: true });
+    const binary = await dntShim.Deno.readFile(vfsPath);
+    await dntShim.Deno.writeFile(targetPath, binary, { mode: 0o755 });
+    return targetPath;
+}
+let setupComplete = false;
+let setupPromise = null;
+/**
+ * Idempotent one-shot setup — copies the esbuild binary out of the VFS (if
+ * running in `deno compile`) and sets ESBUILD_BINARY_PATH. No-op on normal
+ * Deno or when the env var is already set.
+ */
+export async function ensureEsbuildBinary() {
+    if (setupComplete)
+        return;
+    if (setupPromise) {
+        await setupPromise;
+        return;
+    }
+    setupPromise = (async () => {
+        try {
+            if (dntShim.Deno.env.get("ESBUILD_BINARY_PATH") || !isDenoCompiled) {
+                setupComplete = true;
+                return;
+            }
+            try {
+                const binaryPath = await extractEsbuildBinary();
+                dntShim.Deno.env.set("ESBUILD_BINARY_PATH", binaryPath);
+                // esbuild reads process.env (not Deno.env) on some code paths.
+                const proc = dntShim.dntGlobalThis.process;
+                if (proc?.env)
+                    proc.env.ESBUILD_BINARY_PATH = binaryPath;
+            }
+            catch (err) {
+                console.error("[ext-bundler-esbuild] Binary extraction failed:", err);
+            }
+        }
+        finally {
+            setupComplete = true;
+        }
+    })();
+    try {
+        await setupPromise;
+    }
+    finally {
+        setupPromise = null;
+    }
+}

package/esm/extensions/ext-bundler-esbuild/src/es-module-lexer.d.ts

@@ -0,0 +1,17 @@
+/**
+ * es-module-lexer-backed implementation of the {@link ModuleLexer} contract.
+ *
+ * es-module-lexer@2 exports `init` as a Promise in its ESM build but older
+ * typings expect a function. We tolerate both shapes (as the in-tree
+ * src/transforms/esm/lexer.ts did).
+ *
+ * @module extensions/ext-bundler-esbuild/es-module-lexer
+ */
+import type { ImportSpecifier, ModuleLexer } from "../../../src/extensions/bundler/index.js";
+/** es-module-lexer-backed {@link ModuleLexer} implementation. */
+export declare class EsModuleLexer implements ModuleLexer {
+    private initPromise;
+    init(): Promise<void>;
+    parse(code: string): readonly ImportSpecifier[];
+}
+//# sourceMappingURL=es-module-lexer.d.ts.map

package/esm/extensions/ext-bundler-esbuild/src/es-module-lexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"es-module-lexer.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-bundler-esbuild/src/es-module-lexer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AAG7F,iEAAiE;AACjE,qBAAa,aAAc,YAAW,WAAW;IAC/C,OAAO,CAAC,WAAW,CAA8B;IAE3C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAY3B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,eAAe,EAAE;CAIhD"}

package/esm/extensions/ext-bundler-esbuild/src/es-module-lexer.js

@@ -0,0 +1,29 @@
+/**
+ * es-module-lexer-backed implementation of the {@link ModuleLexer} contract.
+ *
+ * es-module-lexer@2 exports `init` as a Promise in its ESM build but older
+ * typings expect a function. We tolerate both shapes (as the in-tree
+ * src/transforms/esm/lexer.ts did).
+ *
+ * @module extensions/ext-bundler-esbuild/es-module-lexer
+ */
+import { init, parse } from "es-module-lexer";
+/** es-module-lexer-backed {@link ModuleLexer} implementation. */
+export class EsModuleLexer {
+    initPromise = null;
+    async init() {
+        if (this.initPromise) {
+            await this.initPromise;
+            return;
+        }
+        const anyInit = init;
+        this.initPromise = typeof anyInit === "function"
+            ? anyInit()
+            : anyInit;
+        await this.initPromise;
+    }
+    parse(code) {
+        const [imports] = parse(code);
+        return imports;
+    }
+}

package/esm/extensions/ext-bundler-esbuild/src/esbuild-bundler.d.ts

@@ -0,0 +1,25 @@
+/**
+ * esbuild-backed implementation of the {@link Bundler} contract.
+ *
+ * Lazy-initializes the esbuild binary (including `deno compile` VFS
+ * extraction) on first use. All options pass through to esbuild unchanged
+ * because the {@link BundleOptions} shape was designed to be esbuild-compatible;
+ * the only translation is converting {@link BundlerPlugin}s into esbuild
+ * plugins via {@link toEsbuildPlugin}.
+ *
+ * @module extensions/ext-bundler-esbuild/esbuild-bundler
+ */
+import type { BuildContext, BundleOptions, Bundler, BundleResult, TransformOptions, TransformResult } from "../../../src/extensions/bundler/index.js";
+/**
+ * esbuild-backed {@link Bundler} implementation.
+ *
+ * Every instance coordinates through one module-wide service lifecycle. Raw
+ * asynchronous esbuild calls must not share the same module in this process.
+ */
+export declare class EsbuildBundler implements Bundler {
+    bundle(options: BundleOptions): Promise<BundleResult>;
+    transform(options: TransformOptions): Promise<TransformResult>;
+    context(options: BundleOptions): Promise<BuildContext>;
+    stop(): Promise<void>;
+}
+//# sourceMappingURL=esbuild-bundler.d.ts.map

package/esm/extensions/ext-bundler-esbuild/src/esbuild-bundler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"esbuild-bundler.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-bundler-esbuild/src/esbuild-bundler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,aAAa,EAEb,OAAO,EACP,YAAY,EAGZ,gBAAgB,EAChB,eAAe,EAChB,MAAM,0CAA0C,CAAC;AAwTlD;;;;;GAKG;AACH,qBAAa,cAAe,YAAW,OAAO;IACtC,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAkBrD,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;IAa9D,OAAO,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAiCtD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAiF5B"}