veryfront 0.1.1127 → 0.1.1132

package/esm/src/agent/mcp-tool-policy.js

@@ -0,0 +1,66 @@
+import { PERMISSION_DENIED } from "../errors/index.js";
+function isPolicyEmpty(policy) {
+    return policy?.allow === undefined && policy?.deny === undefined;
+}
+function defaultDeniedDetail(toolName) {
+    return `Tool "${toolName}" is not allowed for this run`;
+}
+export function createMcpToolPolicyGate(policy, options) {
+    const deniedDetail = options?.deniedDetail ?? defaultDeniedDetail;
+    const allows = (toolName) => {
+        const deny = policy?.deny;
+        if (deny?.includes(toolName))
+            return false;
+        const allow = policy?.allow;
+        if (allow !== undefined)
+            return allow.includes(toolName);
+        return true;
+    };
+    const filterDefinitions = (definitions) => definitions.filter((definition) => allows(definition.name));
+    const assertAllowed = (toolName) => {
+        if (allows(toolName))
+            return;
+        throw PERMISSION_DENIED.create({ detail: deniedDetail(toolName) });
+    };
+    return { allows, filterDefinitions, assertAllowed };
+}
+export function wrapRemoteToolSourceWithMcpPolicy(source, policy, options) {
+    if (isPolicyEmpty(policy))
+        return source;
+    const gate = createMcpToolPolicyGate(policy, {
+        deniedDetail: (toolName) => options?.deniedDetail?.(toolName, source.id) ??
+            defaultDeniedDetail(toolName),
+    });
+    return {
+        ...source,
+        id: source.id,
+        listTools: async (context) => gate.filterDefinitions(await source.listTools(context)),
+        executeTool: (toolName, args, context) => {
+            gate.assertAllowed(toolName);
+            return source.executeTool(toolName, args, context);
+        },
+    };
+}
+export function wrapHostToolSetWithMcpPolicy(tools, policy, options) {
+    if (isPolicyEmpty(policy))
+        return tools;
+    const gate = createMcpToolPolicyGate(policy, options);
+    const wrapped = {};
+    for (const [toolName, definition] of Object.entries(tools)) {
+        if (!gate.allows(toolName))
+            continue;
+        if (definition.execute === undefined) {
+            wrapped[toolName] = { ...definition };
+            continue;
+        }
+        const execute = definition.execute;
+        wrapped[toolName] = {
+            ...definition,
+            execute: (toolInput, execOptions) => {
+                gate.assertAllowed(toolName);
+                return execute(toolInput, execOptions);
+            },
+        };
+    }
+    return wrapped;
+}

package/esm/src/agent/runtime/mcp-server-tool-sources.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mcp-server-tool-sources.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/mcp-server-tool-sources.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,yBAAyB,EAC9B,KAAK,gBAAgB,EAEtB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EACV,WAAW,EAKZ,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAkB,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,qCAAqC,CAAC;AAC3F,OAAO,EAEL,KAAK,uBAAuB,EAC7B,MAAM,kCAAkC,CAAC;AAI1C,MAAM,MAAM,uBAAuB,GAAG;IACpC,qBAAqB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC3C,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC,2BAA2B,CAAC,EAAE,+BAA+B,CAAC;CAC/D,CAAC;AAEF,4DAA4D;AAC5D,eAAO,MAAM,2BAA2B,2BAA2B,CAAC;AACpE,+DAA+D;AAC/D,eAAO,MAAM,8BAA8B,eAAe,CAAC;AAgB3D,kFAAkF;AAClF,wBAAgB,sCAAsC,CAAC,KAAK,EAAE;IAC5D,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACxC,GAAG,MAAM,EAAE,CAeX;AAkGD,mEAAmE;AACnE,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,gBAAgB,EAAE,GAAG,SAAS,EACvC,gBAAgB,EAAE,MAAM,EAAE,GAAG,SAAS,GACrC,gBAAgB,EAAE,GAAG,SAAS,CAQhC;AAkBD;;;;;;;;GAQG;AACH,wBAAgB,6CAA6C,CAC3D,OAAO,EAAE,gBAAgB,EAAE,GAAG,SAAS,EACvC,OAAO,EAAE,oBAAoB,GAC5B,gBAAgB,EAAE,GAAG,SAAS,CAoBhC;AAED,MAAM,MAAM,sCAAsC,GAAG;IACnD,sBAAsB,CAAC,EAAE,CAAC,MAAM,EAAE,yBAAyB,KAAK,gBAAgB,CAAC;IACjF,qBAAqB,CAAC,EAAE,MAAM,uBAAuB,CAAC;CACvD,CAAC;AAoBF,yEAAyE;AACzE,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,gBAAgB,EACxB,SAAS,EAAE,MAAM,GAChB,gBAAgB,CAsBlB;AAmED,kEAAkE;AAClE,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,WAAW,EACnB,YAAY,GAAE,sCAA2C,EACzD,OAAO,qBAAY,GAClB,gBAAgB,EAAE,GAAG,SAAS,CAiEhC"}
+{"version":3,"file":"mcp-server-tool-sources.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/mcp-server-tool-sources.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,yBAAyB,EAC9B,KAAK,gBAAgB,EAEtB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EACV,WAAW,EAKZ,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,qCAAqC,CAAC;AAC3F,OAAO,EAEL,KAAK,uBAAuB,EAC7B,MAAM,kCAAkC,CAAC;AAK1C,MAAM,MAAM,uBAAuB,GAAG;IACpC,qBAAqB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC3C,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC,2BAA2B,CAAC,EAAE,+BAA+B,CAAC;CAC/D,CAAC;AAEF,4DAA4D;AAC5D,eAAO,MAAM,2BAA2B,2BAA2B,CAAC;AACpE,+DAA+D;AAC/D,eAAO,MAAM,8BAA8B,eAAe,CAAC;AAgB3D,kFAAkF;AAClF,wBAAgB,sCAAsC,CAAC,KAAK,EAAE;IAC5D,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACxC,GAAG,MAAM,EAAE,CAeX;AAqDD,mEAAmE;AACnE,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,gBAAgB,EAAE,GAAG,SAAS,EACvC,gBAAgB,EAAE,MAAM,EAAE,GAAG,SAAS,GACrC,gBAAgB,EAAE,GAAG,SAAS,CAQhC;AAkBD;;;;;;;;GAQG;AACH,wBAAgB,6CAA6C,CAC3D,OAAO,EAAE,gBAAgB,EAAE,GAAG,SAAS,EACvC,OAAO,EAAE,oBAAoB,GAC5B,gBAAgB,EAAE,GAAG,SAAS,CAoBhC;AAED,MAAM,MAAM,sCAAsC,GAAG;IACnD,sBAAsB,CAAC,EAAE,CAAC,MAAM,EAAE,yBAAyB,KAAK,gBAAgB,CAAC;IACjF,qBAAqB,CAAC,EAAE,MAAM,uBAAuB,CAAC;CACvD,CAAC;AAoBF,yEAAyE;AACzE,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,gBAAgB,EACxB,SAAS,EAAE,MAAM,GAChB,gBAAgB,CAsBlB;AAmED,kEAAkE;AAClE,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,WAAW,EACnB,YAAY,GAAE,sCAA2C,EACzD,OAAO,qBAAY,GAClB,gBAAgB,EAAE,GAAG,SAAS,CAiEhC"}

package/esm/src/agent/runtime/mcp-server-tool-sources.js

@@ -1,7 +1,8 @@
 import { createProjectScopedRemoteToolCatalog, createRemoteMCPToolSource, isToolVisibleTo, toolRegistry, } from "../../tool/index.js";
-import { CONFIG_INVALID, PERMISSION_DENIED } from "../../errors/index.js";
+import { CONFIG_INVALID } from "../../errors/index.js";
 import { getVeryfrontCloudHostBootstrap, } from "../../platform/cloud/resolver.js";
 import { createAgentServiceRemoteMcpConfig } from "../service/mcp-server-config.js";
+import { wrapRemoteToolSourceWithMcpPolicy } from "../mcp-tool-policy.js";
 import { getActiveRuntimeRemoteToolSources } from "./remote-tool-source-context.js";
 /** Canonical source id for the Veryfront API MCP server. */
 export const VERYFRONT_API_MCP_SOURCE_ID = "veryfront-platform-mcp";
@@ -47,18 +48,6 @@ async function resolveHeaders(auth, context) {
     }
     return await resolveValue(auth.headers, context);
 }
-function isToolAllowed(toolName, policy) {
-    if (policy?.allow && !policy.allow.includes(toolName)) {
-        return false;
-    }
-    if (policy?.deny?.includes(toolName)) {
-        return false;
-    }
-    return true;
-}
-function filterToolDefinitions(definitions, policy) {
-    return definitions.filter((definition) => isToolAllowed(definition.name, policy));
-}
 function isHttpMcpServerConfig(server) {
     return "transport" in server;
 }
@@ -69,39 +58,14 @@ function createMcpServerToolSource(server) {
         headers: (context) => resolveHeaders(server.auth, context),
         ...(server.fetch ? { fetch: server.fetch } : {}),
     });
-    return {
-        id: source.id,
-        async listTools(context) {
-            return filterToolDefinitions(await source.listTools(context), server.toolPolicy);
-        },
-        executeTool(toolName, args, context) {
-            if (!isToolAllowed(toolName, server.toolPolicy)) {
-                throw PERMISSION_DENIED.create({
-                    detail: `Tool "${toolName}" is not allowed for MCP server "${server.id}"`,
-                });
-            }
-            return source.executeTool(toolName, args, context);
-        },
-    };
+    return wrapRemoteToolSourceWithMcpPolicy(source, server.toolPolicy, {
+        deniedDetail: (toolName) => `Tool "${toolName}" is not allowed for MCP server "${server.id}"`,
+    });
 }
 function createMcpToolPolicySource(source, policy) {
-    if (!policy?.allow && !policy?.deny) {
-        return source;
-    }
-    return {
-        id: source.id,
-        async listTools(context) {
-            return filterToolDefinitions(await source.listTools(context), policy);
-        },
-        executeTool(toolName, args, context) {
-            if (!isToolAllowed(toolName, policy)) {
-                throw PERMISSION_DENIED.create({
-                    detail: `Tool "${toolName}" is not allowed for MCP server "${source.id}"`,
-                });
-            }
-            return source.executeTool(toolName, args, context);
-        },
-    };
+    return wrapRemoteToolSourceWithMcpPolicy(source, policy, {
+        deniedDetail: (toolName, sourceId) => `Tool "${toolName}" is not allowed for MCP server "${sourceId}"`,
+    });
 }
 /** Carry an explicit remote-tool ceiling into nested execution. */
 export function constrainRuntimeRemoteToolSources(sources, allowedToolNames) {

package/esm/src/discovery/import-rewriter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"import-rewriter.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/import-rewriter.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAI5D,eAAO,MAAM,kCAAkC,oSAerC,CAAC;AAEX,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;CAClD;AAwHD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,kBAAuB,GAC/B,MAAM,CA2BR;AAqFD;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAClB,EAAE,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EACvC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAyRjB"}
+{"version":3,"file":"import-rewriter.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/import-rewriter.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAS5D,eAAO,MAAM,kCAAkC,oSAerC,CAAC;AAEX,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;CAClD;AAwHD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,kBAAuB,GAC/B,MAAM,CA2BR;AAeD;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAClB,EAAE,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EACvC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAyQjB"}

package/esm/src/discovery/import-rewriter.js

@@ -8,6 +8,7 @@ import * as dntShim from "../../_dnt.shims.js";
 import { isDenoCompiled } from "../platform/compat/runtime.js";
 import * as pathHelper from "../platform/compat/path/index.js";
 import { LRUCache } from "../utils/lru-wrapper.js";
+import { resolveContainedPackagePath, resolvePackageExportPath, splitPackageSubpath, } from "../transforms/import-rewriter/package-resolution.js";
 export const DISCOVERY_GLOBAL_VERYFRONT_MODULES = [
     "veryfront/agent",
     "veryfront/tool",
@@ -156,76 +157,6 @@ const RESOLVED_SPECIFIER_CACHE_MAX_ENTRIES = 1000;
 const resolvedSpecifierCache = new LRUCache({
     maxEntries: RESOLVED_SPECIFIER_CACHE_MAX_ENTRIES,
 });
-// Split `react/jsx-runtime` → { name: "react", subpath: "./jsx-runtime" } and
-// `@scope/pkg/sub/path` → { name: "@scope/pkg", subpath: "./sub/path" }.
-function splitPackageSubpath(specifier) {
-    const parts = specifier.split("/");
-    const segments = specifier.startsWith("@") ? parts.slice(0, 2) : parts.slice(0, 1);
-    const name = segments.join("/");
-    const rest = parts.slice(segments.length).join("/");
-    return { name, subpath: rest ? `./${rest}` : "." };
-}
-// Pick the relative file path from a `package.json#exports` entry, which can
-// be a string, a conditional object (`{ import, default, ... }`), or an
-// array of those.
-function pickExportEntry(entry) {
-    if (typeof entry === "string")
-        return entry;
-    if (Array.isArray(entry)) {
-        for (const e of entry) {
-            const v = pickExportEntry(e);
-            if (v)
-                return v;
-        }
-        return null;
-    }
-    if (entry && typeof entry === "object") {
-        const obj = entry;
-        const candidate = obj.import ?? obj.node ?? obj.default;
-        return candidate ? pickExportEntry(candidate) : null;
-    }
-    return null;
-}
-// Resolve a subpath (`.` or `./foo/bar`) against a `package.json#exports`
-// map. Honors literal keys first, then matches `./*`-style glob patterns
-// where the trailing `*` is substituted with the captured remainder.
-// Returns the resolved relative path (e.g. `./debounce.js`) or null when
-// no entry matches.
-function resolveExportPath(exports, subpath) {
-    if (!exports || typeof exports !== "object")
-        return null;
-    const map = exports;
-    // Literal key (covers "." and exact subpaths like "./jsx-runtime").
-    if (subpath in map)
-        return pickExportEntry(map[subpath]);
-    // Glob keys like "./*", "./feature/*", "./lib/*.js". Pick the longest
-    // matching prefix so more specific patterns win over `./*`.
-    let bestKey = null;
-    let bestPrefixLen = -1;
-    for (const key of Object.keys(map)) {
-        const star = key.indexOf("*");
-        if (star === -1)
-            continue;
-        const prefix = key.slice(0, star);
-        const suffix = key.slice(star + 1);
-        if (!subpath.startsWith(prefix) || !subpath.endsWith(suffix))
-            continue;
-        if (subpath.length < prefix.length + suffix.length)
-            continue;
-        if (prefix.length > bestPrefixLen) {
-            bestKey = key;
-            bestPrefixLen = prefix.length;
-        }
-    }
-    if (!bestKey)
-        return null;
-    const star = bestKey.indexOf("*");
-    const captured = subpath.slice(bestKey.slice(0, star).length, subpath.length - bestKey.slice(star + 1).length);
-    const template = pickExportEntry(map[bestKey]);
-    if (!template)
-        return null;
-    return template.replaceAll("*", captured);
-}
 /**
  * Rewrite imports for Node.js runtime
  * - Resolves relative imports to file:// URLs
@@ -257,26 +188,16 @@ export async function rewriteDiscoveryImports(code, projectDir, fs, fileDir) {
                 const packageJsonPath = pathHelper.join(packagePath, "package.json");
                 try {
                     const pkgJson = JSON.parse(await fs.readTextFile(packageJsonPath));
-                    const exportPath = resolveExportPath(pkgJson.exports, subpath);
+                    const exportPath = resolvePackageExportPath(pkgJson.exports, subpath);
                     const entryPoint = exportPath ??
                         (subpath === "."
                             ? (pkgJson.module ?? pkgJson.main ?? "index.js")
                             // No exports entry matched: fall back to joining the subpath
                             // onto the package dir (e.g. `dotenv/config.js`).
                             : subpath.replace(/^\.\//, ""));
-                    // Defense in depth: refuse resolved paths that escape the package
-                    // directory. A malicious package shipping `exports: { ".": "../foo" }`
-                    // would otherwise yield a `file://` URL outside `node_modules/<pkg>`
-                    // that the discovery loader would still `import()`. `path.resolve`
-                    // (unlike `path.join`) normalizes `..` segments, so the prefix
-                    // check correctly catches escape attempts.
-                    const normalized = pathHelper.resolve(packagePath, entryPoint);
-                    const packagePathPrefix = packagePath.endsWith(pathHelper.SEPARATOR)
-                        ? packagePath
-                        : packagePath + pathHelper.SEPARATOR;
-                    if (normalized !== packagePath && !normalized.startsWith(packagePathPrefix)) {
+                    const normalized = resolveContainedPackagePath(packagePath, entryPoint);
+                    if (!normalized)
                         return null;
-                    }
                     const resolved = pathToFileURL(normalized).href;
                     resolvedSpecifierCache.set(cacheKey, resolved);
                     return resolved;
@@ -429,17 +350,13 @@ export async function rewriteDiscoveryImports(code, projectDir, fs, fileDir) {
             if (veryfrontExportSource.kind === "absent")
                 return { kind: "rejected" };
             const subpath = specifier === "veryfront" ? "." : "./" + specifier.replace("veryfront/", "");
-            const exportPath = resolveExportPath(veryfrontExportSource.exportsMap, subpath);
+            const exportPath = resolvePackageExportPath(veryfrontExportSource.exportsMap, subpath);
             if (!exportPath)
                 return { kind: "rejected" };
             const packagePath = veryfrontExportSource.packagePath;
-            const resolvedPath = pathHelper.resolve(packagePath, exportPath);
-            const packagePathPrefix = packagePath.endsWith(pathHelper.SEPARATOR)
-                ? packagePath
-                : packagePath + pathHelper.SEPARATOR;
-            if (resolvedPath !== packagePath && !resolvedPath.startsWith(packagePathPrefix)) {
+            const resolvedPath = resolveContainedPackagePath(packagePath, exportPath);
+            if (!resolvedPath)
                 return { kind: "rejected" };
-            }
             return { kind: "resolved", url: pathToFileURL(resolvedPath).href };
         };
         for (const specifier of veryfrontSpecifiers) {

package/esm/src/eval/run-report.d.ts

@@ -0,0 +1,189 @@
+import type { DiscoveredEval } from "./discovery.js";
+import type { EvalModelComparison, EvalModelComparisonOptions, EvalReport, EvalReportComparison, EvalReportComparisonPolicy, EvalReportExportConfig, EvalRunProvenance } from "./types.js";
+type CliEvalSummary = {
+    runId: string;
+    evalId: string;
+    target: string;
+    records: number;
+    passed: number;
+    failed: number;
+    passRate: number;
+    metrics: EvalReport["summary"]["metrics"];
+};
+type EvalArtifactPaths = {
+    directory: string;
+    summary: string;
+    results: string;
+    reportMarkdown: string;
+};
+type EvalModelArtifactPaths = EvalArtifactPaths & {
+    junit: string;
+};
+type EvalModelComparisonArtifactPaths = {
+    directory: string;
+    comparisonJson: string;
+    comparisonMarkdown: string;
+    models: Record<string, EvalModelArtifactPaths>;
+};
+type EvalRunReportOutputHints = {
+    reportDirectory: string;
+    reportMarkdown: string;
+    report?: string;
+    junit?: string;
+    baselineWritten?: string;
+    children?: EvalRunReportChildOutputHint[];
+    models?: EvalRunReportModelOutputHint[];
+    comparisonJson?: string;
+    comparisonMarkdown?: string;
+    recommendation?: EvalModelComparison["recommendation"];
+};
+type EvalRunReportChildOutputHint = {
+    kind: "report";
+    evalId: string;
+    reportDirectory: string;
+    report: EvalReport;
+} | {
+    kind: "error";
+    evalId: string;
+    error: string;
+};
+type EvalRunReportModelOutputHint = {
+    model: string;
+    report: EvalReport;
+};
+type EvalRunReportSingleInput = {
+    kind: "single";
+    projectDir: string;
+    frameworkVersion: string;
+    datasetBase?: string;
+    reportDir?: string;
+    report?: string;
+    junit?: string;
+    baseline?: string;
+    writeBaseline?: string;
+    baselinePolicy?: EvalReportComparisonPolicy;
+    exportConfig?: EvalReportExportConfig;
+    provenance?: EvalRunProvenance;
+    evalItem: DiscoveredEval;
+    targetKind: EvalReport["targetKind"];
+    target: string;
+    targetAdapter: unknown;
+    selectedModel?: string;
+    maxOutputTokens?: number;
+};
+type EvalRunReportSuiteInput = {
+    kind: "suite";
+    projectDir: string;
+    frameworkVersion: string;
+    datasetBase?: string;
+    reportDir?: string;
+    junit?: string;
+    exportConfig?: EvalReportExportConfig;
+    provenance?: EvalRunProvenance;
+    evalItems: DiscoveredEval[];
+};
+type EvalRunReportModelComparisonPolicy = Omit<EvalModelComparisonOptions, "baselineModel">;
+type EvalRunReportModelComparisonInput = {
+    kind: "model-comparison";
+    projectDir: string;
+    frameworkVersion: string;
+    datasetBase?: string;
+    reportDir?: string;
+    report?: string;
+    exportConfig?: EvalReportExportConfig;
+    provenance?: EvalRunProvenance;
+    evalItem: DiscoveredEval;
+    target: string;
+    baselineModel: string;
+    candidateModels: string[];
+    comparisonPolicy?: EvalRunReportModelComparisonPolicy;
+    maxOutputTokens?: number;
+};
+type EvalRunReportInput = EvalRunReportSingleInput | EvalRunReportSuiteInput | EvalRunReportModelComparisonInput;
+type EvalRunTargetOptions = {
+    baseDir: string;
+    runId: string;
+    frameworkVersion: string;
+    targetKind: EvalReport["targetKind"];
+    target: string;
+    targetAdapter: unknown;
+    selectedModel?: string;
+    maxOutputTokens?: number;
+    metadata: EvalReport["metadata"];
+};
+type EvalRunReportTargetAdapters = {
+    runEval(evalItem: DiscoveredEval, options: EvalRunTargetOptions): Promise<EvalReport>;
+    createModelTargetAdapter?(model: string): unknown | Promise<unknown>;
+    resolveTarget?(evalItem: DiscoveredEval): Promise<Pick<EvalRunTargetOptions, "targetKind" | "target" | "targetAdapter">> | Pick<EvalRunTargetOptions, "targetKind" | "target" | "targetAdapter">;
+};
+type EvalRunReportArtifactAdapters = {
+    readTextFile(path: string): Promise<string>;
+    writeTextFileEnsuringDir(path: string, content: string): Promise<void>;
+};
+type EvalRunReportBillingAdapters = {
+    runWithGatewayBillingGroup(billingGroupId: string, operation: () => Promise<EvalReport>): Promise<EvalReport>;
+};
+type EvalRunReportExporterAdapters = {
+    exportReport(report: EvalReport, config?: EvalReportExportConfig): Promise<EvalReport>;
+};
+type EvalRunReportClock = {
+    now?: () => Date;
+    createSuffix?: () => string;
+};
+type EvalRunReportAdapters = {
+    targets: EvalRunReportTargetAdapters;
+    artifacts: EvalRunReportArtifactAdapters;
+    billing: EvalRunReportBillingAdapters;
+    exporters: EvalRunReportExporterAdapters;
+    clock?: EvalRunReportClock;
+};
+type EvalRunReportOutcome = {
+    kind: "single";
+    report: EvalReport;
+    summary: CliEvalSummary;
+    baseline?: EvalReportComparison;
+    artifacts: EvalArtifactPaths;
+    exitCode: 0 | 1;
+    outputHints: EvalRunReportOutputHints;
+} | {
+    kind: "suite";
+    suite: EvalSuiteSummary;
+    artifacts: EvalSuiteArtifactPaths;
+    exitCode: 0 | 1;
+    outputHints: EvalRunReportOutputHints;
+} | {
+    kind: "model-comparison";
+    reports: EvalReport[];
+    comparison: EvalModelComparison;
+    artifacts: EvalModelComparisonArtifactPaths;
+    exitCode: 0 | 1;
+    outputHints: EvalRunReportOutputHints;
+};
+type EvalSuiteArtifactPaths = {
+    directory: string;
+    summary: string;
+    results: string;
+    reportMarkdown: string;
+};
+type EvalSuiteResult = {
+    id: string;
+    name: string;
+    target: string;
+    status: "passed" | "failed" | "error";
+    artifacts?: EvalArtifactPaths;
+    summary?: CliEvalSummary;
+    error?: string;
+};
+type EvalSuiteSummary = {
+    kind: "eval-suite-summary";
+    runId: string;
+    startedAt: string;
+    endedAt: string;
+    total: number;
+    passed: number;
+    failed: number;
+    results: EvalSuiteResult[];
+};
+export declare function runEvalReport(input: EvalRunReportInput, adapters: EvalRunReportAdapters): Promise<EvalRunReportOutcome>;
+export {};
+//# sourceMappingURL=run-report.d.ts.map

package/esm/src/eval/run-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-report.d.ts","sourceRoot":"","sources":["../../../src/src/eval/run-report.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAIrD,OAAO,KAAK,EAEV,mBAAmB,EACnB,0BAA0B,EAE1B,UAAU,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,sBAAsB,EACtB,iBAAiB,EAElB,MAAM,YAAY,CAAC;AAEpB,KAAK,cAAc,GAAG;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,CAAC;CAC3C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,sBAAsB,GAAG,iBAAiB,GAAG;IAChD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,KAAK,gCAAgC,GAAG;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;CAChD,CAAC;AAkBF,KAAK,wBAAwB,GAAG;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,4BAA4B,EAAE,CAAC;IAC1C,MAAM,CAAC,EAAE,4BAA4B,EAAE,CAAC;IACxC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,cAAc,CAAC,EAAE,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;CACxD,CAAC;AAEF,KAAK,4BAA4B,GAC7B;IACA,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,UAAU,CAAC;CACpB,GACC;IACA,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEJ,KAAK,4BAA4B,GAAG;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,UAAU,CAAC;CACpB,CAAC;AAEF,KAAK,wBAAwB,GAAG;IAC9B,IAAI,EAAE,QAAQ,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,0BAA0B,CAAC;IAC5C,YAAY,CAAC,EAAE,sBAAsB,CAAC;IACtC,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,QAAQ,EAAE,cAAc,CAAC;IACzB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,KAAK,uBAAuB,GAAG;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,sBAAsB,CAAC;IACtC,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,SAAS,EAAE,cAAc,EAAE,CAAC;CAC7B,CAAC;AAEF,KAAK,kCAAkC,GAAG,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AAE5F,KAAK,iCAAiC,GAAG;IACvC,IAAI,EAAE,kBAAkB,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,sBAAsB,CAAC;IACtC,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,kCAAkC,CAAC;IACtD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,KAAK,kBAAkB,GACnB,wBAAwB,GACxB,uBAAuB,GACvB,iCAAiC,CAAC;AAEtC,KAAK,oBAAoB,GAAG;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;CAClC,CAAC;AAEF,KAAK,2BAA2B,GAAG;IACjC,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACtF,wBAAwB,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACrE,aAAa,CAAC,CACZ,QAAQ,EAAE,cAAc,GAEtB,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,YAAY,GAAG,QAAQ,GAAG,eAAe,CAAC,CAAC,GAC9E,IAAI,CAAC,oBAAoB,EAAE,YAAY,GAAG,QAAQ,GAAG,eAAe,CAAC,CAAC;CAC3E,CAAC;AAEF,KAAK,6BAA6B,GAAG;IACnC,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxE,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAClC,0BAA0B,CACxB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,GACnC,OAAO,CAAC,UAAU,CAAC,CAAC;CACxB,CAAC;AAEF,KAAK,6BAA6B,GAAG;IACnC,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACxF,CAAC;AAEF,KAAK,kBAAkB,GAAG;IACxB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,MAAM,CAAC;CAC7B,CAAC;AAEF,KAAK,qBAAqB,GAAG;IAC3B,OAAO,EAAE,2BAA2B,CAAC;IACrC,SAAS,EAAE,6BAA6B,CAAC;IACzC,OAAO,EAAE,4BAA4B,CAAC;IACtC,SAAS,EAAE,6BAA6B,CAAC;IACzC,KAAK,CAAC,EAAE,kBAAkB,CAAC;CAC5B,CAAC;AAEF,KAAK,oBAAoB,GAAG;IAC1B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,cAAc,CAAC;IACxB,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,SAAS,EAAE,iBAAiB,CAAC;IAC7B,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;IAChB,WAAW,EAAE,wBAAwB,CAAC;CACvC,GAAG;IACF,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,gBAAgB,CAAC;IACxB,SAAS,EAAE,sBAAsB,CAAC;IAClC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;IAChB,WAAW,EAAE,wBAAwB,CAAC;CACvC,GAAG;IACF,IAAI,EAAE,kBAAkB,CAAC;IACzB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,EAAE,mBAAmB,CAAC;IAChC,SAAS,EAAE,gCAAgC,CAAC;IAC5C,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;IAChB,WAAW,EAAE,wBAAwB,CAAC;CACvC,CAAC;AAEF,KAAK,sBAAsB,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;IACtC,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,KAAK,gBAAgB,GAAG;IACtB,IAAI,EAAE,oBAAoB,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,eAAe,EAAE,CAAC;CAC5B,CAAC;AAswBF,wBAAsB,aAAa,CACjC,KAAK,EAAE,kBAAkB,EACzB,QAAQ,EAAE,qBAAqB,GAC9B,OAAO,CAAC,oBAAoB,CAAC,CAiE/B"}