veryfront 0.1.871 → 0.1.873

package/esm/src/agent/testing/live-evals/index.js

@@ -1,11 +0,0 @@
-export { runLiveEvalCli, } from "./cli-runner.js";
-export { DEFAULT_LIVE_EVAL_ENDPOINT, resolveLiveEvalEnvironment, } from "./environment.js";
-export { cancelLiveEvalInputRequest, createLiveEvalApiClient, createLiveEvalConversation, createLiveEvalProjectUploadFixture, createLiveEvalRelease, deleteLiveEvalConversation, deleteLiveEvalProjectFile, getLiveEvalProjectFile, listOpenLiveEvalInputRequests, submitLiveEvalInputResponse, waitForOpenLiveEvalInputRequest, } from "./api-client.js";
-export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
-export { evaluateRuntimeConfidenceEnv, printRuntimeConfidencePreflight, } from "./preflight.js";
-export { buildRuntimePerformanceSummary, } from "./performance.js";
-export { buildLiveEvalCaseMetadata, DEFAULT_LIVE_EVAL_AREA_TAG_RULES, DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES, withLiveEvalMetadata, } from "./metadata.js";
-export { buildLiveEvalRequestBody, } from "./request.js";
-export { buildLiveEvalCaseTagSummary, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, hasEveryLiveEvalTag, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./report.js";
-export { createFailedEvalResult, createPassedEvalResult, createSkippedEvalResult, } from "./result.js";
-export { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, liveEvalRunnerInternals, } from "./runner.js";

package/esm/src/agent/testing/live-evals/metadata.d.ts

@@ -1,34 +0,0 @@
-import type { LiveEvalCaseMetadata } from "./report.js";
-import type { LiveEvalCase } from "./runner.js";
-/** Public API contract for live eval case surface. */
-export type LiveEvalCaseSurface = "read-only" | "write" | "experimental";
-/** Public API contract for live eval case tag rule. */
-export interface LiveEvalCaseTagRule {
-    tag: string;
-    equals?: string | readonly string[];
-    startsWith?: string | readonly string[];
-    includes?: string | readonly string[];
-}
-/** Options accepted by live eval case metadata. */
-export interface LiveEvalCaseMetadataOptions {
-    releaseGateCaseIds?: readonly string[] | ReadonlySet<string>;
-    optionalJudgeCasePrefixes?: readonly string[];
-    areaTagRules?: readonly LiveEvalCaseTagRule[];
-}
-/** Input payload for build live eval case metadata. */
-export interface BuildLiveEvalCaseMetadataInput extends LiveEvalCaseMetadataOptions {
-    caseId: string;
-    surface: LiveEvalCaseSurface;
-    requireProject: boolean;
-}
-/** Default value for live eval optional judge case prefixes. */
-export declare const DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES: readonly string[];
-/** Default value for live eval area tag rules. */
-export declare const DEFAULT_LIVE_EVAL_AREA_TAG_RULES: readonly LiveEvalCaseTagRule[];
-/** Builds live eval case metadata. */
-export declare function buildLiveEvalCaseMetadata(input: BuildLiveEvalCaseMetadataInput): LiveEvalCaseMetadata;
-/** Applies live eval metadata. */
-export declare function withLiveEvalMetadata<TCase extends LiveEvalCase>(cases: readonly TCase[], surface: LiveEvalCaseSurface, options?: LiveEvalCaseMetadataOptions): Array<TCase & {
-    metadata: LiveEvalCaseMetadata;
-}>;
-//# sourceMappingURL=metadata.d.ts.map

package/esm/src/agent/testing/live-evals/metadata.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/metadata.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,sDAAsD;AACtD,MAAM,MAAM,mBAAmB,GAAG,WAAW,GAAG,OAAO,GAAG,cAAc,CAAC;AAEzE,uDAAuD;AACvD,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;CACvC;AAED,mDAAmD;AACnD,MAAM,WAAW,2BAA2B;IAC1C,kBAAkB,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAC7D,yBAAyB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C,YAAY,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;CAC/C;AAED,uDAAuD;AACvD,MAAM,WAAW,8BAA+B,SAAQ,2BAA2B;IACjF,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,mBAAmB,CAAC;IAC7B,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,gEAAgE;AAChE,eAAO,MAAM,8CAA8C,EAAE,SAAS,MAAM,EAI3E,CAAC;AAEF,kDAAkD;AAClD,eAAO,MAAM,gCAAgC,EAAE,SAAS,mBAAmB,EAqB1E,CAAC;AA+DF,sCAAsC;AACtC,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,8BAA8B,GACpC,oBAAoB,CAkCtB;AAED,kCAAkC;AAClC,wBAAgB,oBAAoB,CAAC,KAAK,SAAS,YAAY,EAC7D,KAAK,EAAE,SAAS,KAAK,EAAE,EACvB,OAAO,EAAE,mBAAmB,EAC5B,OAAO,GAAE,2BAAgC,GACxC,KAAK,CAAC,KAAK,GAAG;IAAE,QAAQ,EAAE,oBAAoB,CAAA;CAAE,CAAC,CAUnD"}

package/esm/src/agent/testing/live-evals/metadata.js

@@ -1,116 +0,0 @@
-/** Default value for live eval optional judge case prefixes. */
-export const DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES = [
-    "knowledge-",
-    "grounded-",
-    "judged-",
-];
-/** Default value for live eval area tag rules. */
-export const DEFAULT_LIVE_EVAL_AREA_TAG_RULES = [
-    { startsWith: "starter-", tag: "area:starter-routing" },
-    { startsWith: "starter-task-", tag: "area:starter-artifact-flow" },
-    { startsWith: "starter-", tag: "behavior:conversation-first" },
-    { startsWith: "workflow-", tag: "area:workflow" },
-    { startsWith: "platform-", tag: "area:platform" },
-    { startsWith: "security-", tag: "area:security" },
-    { startsWith: ["knowledge-", "grounded-", "judged-"], tag: "area:knowledge" },
-    { startsWith: "tool-truthfulness", tag: "area:tool-truthfulness" },
-    { startsWith: "degraded-", tag: "area:resilience" },
-    { equals: "error-recovery-missing-file", tag: "area:resilience" },
-    { includes: "deploy", tag: "area:deployment" },
-    { includes: "sandbox", tag: "area:sandbox" },
-    { includes: "debug", tag: "area:debugging" },
-    { includes: "operate", tag: "area:operations" },
-    { includes: "research", tag: "area:research" },
-    { includes: "knowledge", tag: "area:knowledge-lifecycle" },
-    { includes: "agent", tag: "area:agent-authoring" },
-    { includes: "form-input", tag: "area:interactive-input" },
-    { includes: ["invoke-agent", "delegation"], tag: "area:delegation" },
-    { includes: ["create-page", "create-api-route", "create-skill"], tag: "area:file-generation" },
-];
-function toStringArray(value) {
-    if (!value) {
-        return [];
-    }
-    return typeof value === "string" ? [value] : value;
-}
-function caseIdCollectionHas(collection, caseId) {
-    if (!collection) {
-        return false;
-    }
-    if ("has" in collection) {
-        return collection.has(caseId);
-    }
-    return collection.includes(caseId);
-}
-function matchesTagRule(caseId, rule) {
-    for (const value of toStringArray(rule.equals)) {
-        if (caseId === value) {
-            return true;
-        }
-    }
-    for (const value of toStringArray(rule.startsWith)) {
-        if (caseId.startsWith(value)) {
-            return true;
-        }
-    }
-    for (const value of toStringArray(rule.includes)) {
-        if (caseId.includes(value)) {
-            return true;
-        }
-    }
-    return false;
-}
-function isOptionalJudgeCase(caseId, prefixes) {
-    return prefixes.some((prefix) => caseId.startsWith(prefix));
-}
-function buildAreaTags(caseId, rules) {
-    const tags = new Set();
-    for (const rule of rules) {
-        if (matchesTagRule(caseId, rule)) {
-            tags.add(rule.tag);
-        }
-    }
-    return [...tags];
-}
-/** Builds live eval case metadata. */
-export function buildLiveEvalCaseMetadata(input) {
-    const optionalJudgeCasePrefixes = input.optionalJudgeCasePrefixes ??
-        DEFAULT_LIVE_EVAL_OPTIONAL_JUDGE_CASE_PREFIXES;
-    const areaTagRules = input.areaTagRules ?? DEFAULT_LIVE_EVAL_AREA_TAG_RULES;
-    const gradingTag = isOptionalJudgeCase(input.caseId, optionalJudgeCasePrefixes)
-        ? "grading:deterministic-plus-optional-llm"
-        : "grading:deterministic-only";
-    const tags = new Set([
-        `surface:${input.surface}`,
-        input.requireProject ? "project:required" : "project:optional",
-        input.surface === "experimental" ? "stability:experimental" : "stability:stable",
-        gradingTag,
-    ]);
-    if (input.surface !== "experimental") {
-        tags.add("gate:nightly");
-    }
-    if (gradingTag === "grading:deterministic-only" && input.surface !== "experimental") {
-        tags.add("gate:ci");
-    }
-    if (caseIdCollectionHas(input.releaseGateCaseIds, input.caseId)) {
-        tags.add("gate:release");
-    }
-    for (const tag of buildAreaTags(input.caseId, areaTagRules)) {
-        tags.add(tag);
-    }
-    return {
-        tags: [...tags].sort(),
-    };
-}
-/** Applies live eval metadata. */
-export function withLiveEvalMetadata(cases, surface, options = {}) {
-    return cases.map((testCase) => ({
-        ...testCase,
-        metadata: buildLiveEvalCaseMetadata({
-            ...options,
-            caseId: testCase.id,
-            surface,
-            requireProject: testCase.requireProject === true,
-        }),
-    }));
-}

package/esm/src/agent/testing/live-evals/performance.d.ts

@@ -1,19 +0,0 @@
-/** Public API contract for live eval runtime. */
-export type LiveEvalRuntime = "framework";
-/** Public API contract for live eval result for performance. */
-export interface LiveEvalResultForPerformance {
-    runtime: LiveEvalRuntime;
-    durationMs: number;
-}
-/** Public API contract for runtime performance summary. */
-export interface RuntimePerformanceSummary {
-    count: number;
-    avgDurationMs: number;
-    p50DurationMs: number;
-    p95DurationMs: number;
-    minDurationMs: number;
-    maxDurationMs: number;
-}
-/** Builds runtime performance summary. */
-export declare function buildRuntimePerformanceSummary(results: LiveEvalResultForPerformance[]): Record<LiveEvalRuntime, RuntimePerformanceSummary>;
-//# sourceMappingURL=performance.d.ts.map

package/esm/src/agent/testing/live-evals/performance.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"performance.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/performance.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC;AAE1C,gEAAgE;AAChE,MAAM,WAAW,4BAA4B;IAC3C,OAAO,EAAE,eAAe,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,2DAA2D;AAC3D,MAAM,WAAW,yBAAyB;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB;AAkBD,0CAA0C;AAC1C,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,4BAA4B,EAAE,GACtC,MAAM,CAAC,eAAe,EAAE,yBAAyB,CAAC,CAiBpD"}

package/esm/src/agent/testing/live-evals/performance.js

@@ -1,22 +0,0 @@
-function calculateDurationPercentile(sortedDurations, percentile) {
-    if (sortedDurations.length === 0) {
-        return 0;
-    }
-    const index = Math.min(sortedDurations.length - 1, Math.max(0, Math.ceil((percentile / 100) * sortedDurations.length) - 1));
-    return sortedDurations[index] ?? 0;
-}
-/** Builds runtime performance summary. */
-export function buildRuntimePerformanceSummary(results) {
-    const durations = results.map((result) => result.durationMs).sort((left, right) => left - right);
-    const totalDuration = durations.reduce((sum, value) => sum + value, 0);
-    return {
-        framework: {
-            count: durations.length,
-            avgDurationMs: durations.length > 0 ? Math.round(totalDuration / durations.length) : 0,
-            p50DurationMs: calculateDurationPercentile(durations, 50),
-            p95DurationMs: calculateDurationPercentile(durations, 95),
-            minDurationMs: durations[0] ?? 0,
-            maxDurationMs: durations.at(-1) ?? 0,
-        },
-    };
-}

package/esm/src/agent/testing/live-evals/preflight.d.ts

@@ -1,12 +0,0 @@
-import { type AgentServiceConfigInput } from "../../service/config.js";
-/** Result returned from runtime confidence preflight. */
-export interface RuntimeConfidencePreflightResult {
-    ok: boolean;
-    resolvedApiUrl: string;
-    messages: string[];
-}
-/** Evaluate runtime confidence env helper. */
-export declare function evaluateRuntimeConfidenceEnv(env?: AgentServiceConfigInput, resolvedApiUrl?: string): RuntimeConfidencePreflightResult;
-/** Print runtime confidence preflight helper. */
-export declare function printRuntimeConfidencePreflight(result: RuntimeConfidencePreflightResult, output?: Pick<Console, "error" | "log">): void;
-//# sourceMappingURL=preflight.d.ts.map

package/esm/src/agent/testing/live-evals/preflight.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"preflight.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/preflight.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,uBAAuB,EAA2B,MAAM,yBAAyB,CAAC;AAEhG,yDAAyD;AACzD,MAAM,WAAW,gCAAgC;IAC/C,EAAE,EAAE,OAAO,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,8CAA8C;AAC9C,wBAAgB,4BAA4B,CAC1C,GAAG,GAAE,uBAA4B,EACjC,cAAc,GAAE,MAAuD,GACtE,gCAAgC,CAoBlC;AAED,iDAAiD;AACjD,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,gCAAgC,EACxC,MAAM,GAAE,IAAI,CAAC,OAAO,EAAE,OAAO,GAAG,KAAK,CAAW,GAC/C,IAAI,CAKN"}

package/esm/src/agent/testing/live-evals/preflight.js

@@ -1,27 +0,0 @@
-import { parseAgentServiceConfig } from "../../service/config.js";
-/** Evaluate runtime confidence env helper. */
-export function evaluateRuntimeConfidenceEnv(env = {}, resolvedApiUrl = parseAgentServiceConfig(env).VERYFRONT_API_URL) {
-    const messages = [`Resolved VERYFRONT_API_URL: ${resolvedApiUrl}`];
-    let hasBlockers = false;
-    if (typeof env.VERYFRONT_TOKEN !== "string" || env.VERYFRONT_TOKEN.length === 0) {
-        hasBlockers = true;
-        messages.push("BLOCKER: VERYFRONT_TOKEN is missing");
-    }
-    if (typeof env.AG_UI_EVAL_PROJECT_ID !== "string" || env.AG_UI_EVAL_PROJECT_ID.length === 0) {
-        hasBlockers = true;
-        messages.push("BLOCKER: AG_UI_EVAL_PROJECT_ID is missing");
-    }
-    if (!hasBlockers) {
-        messages.push("Runtime-confidence preflight: PASS");
-        return { ok: true, resolvedApiUrl, messages };
-    }
-    messages.push("Runtime-confidence preflight: FAIL");
-    return { ok: false, resolvedApiUrl, messages };
-}
-/** Print runtime confidence preflight helper. */
-export function printRuntimeConfidencePreflight(result, output = console) {
-    for (const message of result.messages) {
-        const writer = result.ok ? output.log : output.error;
-        writer(message);
-    }
-}

package/esm/src/agent/testing/live-evals/report.d.ts

@@ -1,55 +0,0 @@
-import type { LiveEvalRuntime } from "./performance.js";
-/** Public API contract for live eval case metadata. */
-export interface LiveEvalCaseMetadata {
-    tags: readonly string[];
-}
-/** Public API contract for live eval result for report. */
-export interface LiveEvalResultForReport {
-    runtime: LiveEvalRuntime;
-    status: "pass" | "fail" | "skip";
-}
-/** Input payload for live eval case selection. */
-export interface LiveEvalCaseSelectionInput<TCase extends {
-    id: string;
-}> {
-    allCases: TCase[];
-    readOnlyCases: TCase[];
-    writeCases: TCase[];
-    experimentalWriteCases: TCase[];
-    requestedCaseIds: Set<string>;
-    requestedCaseTags?: Set<string>;
-    runWriteEvals: boolean;
-    runExperimentalWriteEvals: boolean;
-}
-/** Check whether every live eval tag is present. */
-export declare function hasEveryLiveEvalTag(tags: readonly string[], requestedTags: Set<string>): boolean;
-/** Builds live eval case tag summary. */
-export declare function buildLiveEvalCaseTagSummary(cases: readonly {
-    metadata?: LiveEvalCaseMetadata;
-}[]): Record<string, number>;
-/** Select live eval cases helper. */
-export declare function selectLiveEvalCases<TCase extends {
-    id: string;
-    metadata?: LiveEvalCaseMetadata;
-}>(input: LiveEvalCaseSelectionInput<TCase>): TCase[];
-/** Resolves live eval requested case IDs. */
-export declare function resolveLiveEvalRequestedCaseIds(input: {
-    caseSets: Record<string, readonly string[]>;
-    requestedCaseIds: Set<string>;
-    requestedCaseSetId?: string | null;
-}): Set<string>;
-interface LiveEvalRuntimeCounts {
-    passed: number;
-    failed: number;
-    skipped: number;
-}
-/** Builds live eval runtime summary. */
-export declare function buildLiveEvalRuntimeSummary(runtimes: readonly LiveEvalRuntime[], results: LiveEvalResultForReport[]): Record<LiveEvalRuntime, LiveEvalRuntimeCounts>;
-/** Builds live eval status summary. */
-export declare function buildLiveEvalStatusSummary(results: LiveEvalResultForReport[]): {
-    passed: number;
-    failed: number;
-    skipped: number;
-};
-export {};
-//# sourceMappingURL=report.d.ts.map

package/esm/src/agent/testing/live-evals/report.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,uDAAuD;AACvD,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED,2DAA2D;AAC3D,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;CAClC;AAED,kDAAkD;AAClD,MAAM,WAAW,0BAA0B,CAAC,KAAK,SAAS;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE;IACtE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAClB,aAAa,EAAE,KAAK,EAAE,CAAC;IACvB,UAAU,EAAE,KAAK,EAAE,CAAC;IACpB,sBAAsB,EAAE,KAAK,EAAE,CAAC;IAChC,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAChC,aAAa,EAAE,OAAO,CAAC;IACvB,yBAAyB,EAAE,OAAO,CAAC;CACpC;AAED,oDAAoD;AACpD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,GACzB,OAAO,CAQT;AAED,yCAAyC;AACzC,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,SAAS;IACd,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,EAAE,GACF,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAaxB;AAED,qCAAqC;AACrC,wBAAgB,mBAAmB,CACjC,KAAK,SAAS;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,oBAAoB,CAAA;CAAE,EAE7D,KAAK,EAAE,0BAA0B,CAAC,KAAK,CAAC,GACvC,KAAK,EAAE,CAoBT;AAED,6CAA6C;AAC7C,wBAAgB,+BAA+B,CAAC,KAAK,EAAE;IACrD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAC5C,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC,GAAG,GAAG,CAAC,MAAM,CAAC,CAsBd;AAED,UAAU,qBAAqB;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAgBD,wCAAwC;AACxC,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,SAAS,eAAe,EAAE,EACpC,OAAO,EAAE,uBAAuB,EAAE,GACjC,MAAM,CAAC,eAAe,EAAE,qBAAqB,CAAC,CAShD;AAED,uCAAuC;AACvC,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,uBAAuB,EAAE,GACjC;IACD,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB,CAMA"}

package/esm/src/agent/testing/live-evals/report.js

@@ -1,79 +0,0 @@
-/** Check whether every live eval tag is present. */
-export function hasEveryLiveEvalTag(tags, requestedTags) {
-    for (const requestedTag of requestedTags) {
-        if (!tags.includes(requestedTag)) {
-            return false;
-        }
-    }
-    return true;
-}
-/** Builds live eval case tag summary. */
-export function buildLiveEvalCaseTagSummary(cases) {
-    const counts = new Map();
-    for (const testCase of cases) {
-        const caseTags = testCase.metadata?.tags ?? [];
-        for (const tag of caseTags) {
-            counts.set(tag, (counts.get(tag) ?? 0) + 1);
-        }
-    }
-    return Object.fromEntries([...counts.entries()].sort(([left], [right]) => left.localeCompare(right)));
-}
-/** Select live eval cases helper. */
-export function selectLiveEvalCases(input) {
-    const configuredCases = input.runWriteEvals
-        ? [
-            ...input.readOnlyCases,
-            ...input.writeCases,
-            ...(input.runExperimentalWriteEvals ? input.experimentalWriteCases : []),
-        ]
-        : input.readOnlyCases;
-    const selectedCases = input.requestedCaseIds.size > 0
-        ? input.allCases.filter((testCase) => input.requestedCaseIds.has(testCase.id))
-        : configuredCases;
-    const requestedCaseTags = input.requestedCaseTags ?? new Set();
-    return requestedCaseTags.size > 0
-        ? selectedCases.filter((testCase) => hasEveryLiveEvalTag(testCase.metadata?.tags ?? [], requestedCaseTags))
-        : selectedCases;
-}
-/** Resolves live eval requested case IDs. */
-export function resolveLiveEvalRequestedCaseIds(input) {
-    const resolved = new Set(input.requestedCaseIds);
-    const requestedCaseSetId = input.requestedCaseSetId?.trim();
-    if (!requestedCaseSetId) {
-        return resolved;
-    }
-    const caseIds = input.caseSets[requestedCaseSetId];
-    if (!caseIds) {
-        throw new Error(`Unknown AG_UI_EVAL_CASE_SET "${requestedCaseSetId}". Known sets: ${Object.keys(input.caseSets).join(", ")}`);
-    }
-    for (const caseId of caseIds) {
-        resolved.add(caseId);
-    }
-    return resolved;
-}
-function buildRuntimeCounts(results, runtime) {
-    return {
-        passed: results.filter((result) => result.runtime === runtime && result.status === "pass").length,
-        failed: results.filter((result) => result.runtime === runtime && result.status === "fail").length,
-        skipped: results.filter((result) => result.runtime === runtime && result.status === "skip").length,
-    };
-}
-/** Builds live eval runtime summary. */
-export function buildLiveEvalRuntimeSummary(runtimes, results) {
-    const empty = { passed: 0, failed: 0, skipped: 0 };
-    const summary = {
-        framework: empty,
-    };
-    for (const runtime of runtimes) {
-        summary[runtime] = buildRuntimeCounts(results, runtime);
-    }
-    return summary;
-}
-/** Builds live eval status summary. */
-export function buildLiveEvalStatusSummary(results) {
-    return {
-        passed: results.filter((result) => result.status === "pass").length,
-        failed: results.filter((result) => result.status === "fail").length,
-        skipped: results.filter((result) => result.status === "skip").length,
-    };
-}

package/esm/src/agent/testing/live-evals/request.d.ts

@@ -1,31 +0,0 @@
-export interface LiveEvalRequestBody {
-    threadId: string;
-    runId: string;
-    state: Record<string, string>;
-    tools: unknown[];
-    context: unknown[];
-    forwardedProps?: {
-        veryfront: Record<string, unknown>;
-    };
-    messages: Array<{
-        id: string;
-        role: "user";
-        content: string;
-    }>;
-}
-/** Input payload for build live eval request body. */
-export interface BuildLiveEvalRequestBodyInput {
-    testCaseId: string;
-    prompt: string;
-    metadata?: Record<string, string>;
-    projectId: string | null;
-    branchId?: string;
-    model?: string;
-    conversationId?: string | null;
-    allowedTools?: string[];
-    forceRuntimeOverrides?: boolean;
-    maxSteps?: number;
-}
-/** Builds live eval request body. */
-export declare function buildLiveEvalRequestBody(input: BuildLiveEvalRequestBodyInput): LiveEvalRequestBody;
-//# sourceMappingURL=request.d.ts.map

package/esm/src/agent/testing/live-evals/request.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/request.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE;QACf,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC;IACF,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AAED,sDAAsD;AACtD,MAAM,WAAW,6BAA6B;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qCAAqC;AACrC,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,6BAA6B,GACnC,mBAAmB,CA6CrB"}

package/esm/src/agent/testing/live-evals/request.js

@@ -1,48 +0,0 @@
-/** Public API contract for live eval request body. */
-import * as dntShim from "../../../../_dnt.shims.js";
-/** Builds live eval request body. */
-export function buildLiveEvalRequestBody(input) {
-    const veryfront = {};
-    if (input.projectId) {
-        veryfront.projectId = input.projectId;
-    }
-    if (input.conversationId) {
-        veryfront.conversationId = input.conversationId;
-    }
-    if (input.branchId) {
-        veryfront.branchId = input.branchId;
-    }
-    if (input.model) {
-        veryfront.model = input.model;
-    }
-    if (input.allowedTools || input.forceRuntimeOverrides) {
-        veryfront.runtimeOverrides = {
-            allowedTools: input.allowedTools ?? [],
-            ...(input.maxSteps ? { maxSteps: input.maxSteps } : {}),
-        };
-    }
-    return {
-        threadId: dntShim.crypto.randomUUID(),
-        runId: `eval-run-${dntShim.crypto.randomUUID()}`,
-        state: {
-            evalCase: input.testCaseId,
-            ...(input.metadata ?? {}),
-        },
-        tools: [],
-        context: [],
-        ...(Object.keys(veryfront).length > 0
-            ? {
-                forwardedProps: {
-                    veryfront,
-                },
-            }
-            : {}),
-        messages: [
-            {
-                id: dntShim.crypto.randomUUID(),
-                role: "user",
-                content: input.prompt,
-            },
-        ],
-    };
-}

package/esm/src/agent/testing/live-evals/result.d.ts

@@ -1,56 +0,0 @@
-import type { LiveEvalRuntime } from "./performance.js";
-/** Record shape for live eval result. */
-export interface LiveEvalResultRecord {
-    id: string;
-    label: string;
-    runtime: LiveEvalRuntime;
-    status: "pass" | "fail" | "skip";
-    details: string;
-    durationMs: number;
-    conversationId?: string;
-    runId?: string;
-    artifactPaths?: string[];
-    traceSignature?: string;
-    toolStarts?: string[];
-    toolArgsPreview?: string;
-    textPreview?: string;
-}
-/** Result returned from create skipped eval. */
-export declare function createSkippedEvalResult(input: {
-    id: string;
-    label: string;
-    runtime: LiveEvalRuntime;
-    details: string;
-    startedAt: number;
-}): LiveEvalResultRecord;
-/** Result returned from create failed eval. */
-export declare function createFailedEvalResult(input: {
-    id: string;
-    label: string;
-    runtime: LiveEvalRuntime;
-    details: string;
-    startedAt: number;
-    conversationId?: string;
-    runId?: string;
-    artifactPaths?: string[];
-    traceSignature?: string;
-    toolStarts?: string[];
-    toolArgsPreview?: string;
-    textPreview?: string;
-}): LiveEvalResultRecord;
-/** Result returned from create passed eval. */
-export declare function createPassedEvalResult(input: {
-    id: string;
-    label: string;
-    runtime: LiveEvalRuntime;
-    details: string;
-    startedAt: number;
-    conversationId?: string;
-    runId?: string;
-    artifactPaths?: string[];
-    traceSignature?: string;
-    toolStarts?: string[];
-    toolArgsPreview?: string;
-    textPreview?: string;
-}): LiveEvalResultRecord;
-//# sourceMappingURL=result.d.ts.map

package/esm/src/agent/testing/live-evals/result.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD,yCAAyC;AACzC,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AA2CD,gDAAgD;AAChD,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,oBAAoB,CASvB;AAED,+CAA+C;AAC/C,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,oBAAoB,CAEvB;AAED,+CAA+C;AAC/C,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,eAAe,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,oBAAoB,CAEvB"}

package/esm/src/agent/testing/live-evals/result.js

@@ -1,36 +0,0 @@
-function createEvalResult(status, input) {
-    return {
-        id: input.id,
-        label: input.label,
-        runtime: input.runtime,
-        status,
-        details: input.details,
-        durationMs: Date.now() - input.startedAt,
-        ...(input.conversationId ? { conversationId: input.conversationId } : {}),
-        ...(input.runId ? { runId: input.runId } : {}),
-        ...(input.artifactPaths?.length ? { artifactPaths: input.artifactPaths } : {}),
-        ...(input.traceSignature ? { traceSignature: input.traceSignature } : {}),
-        ...(input.toolStarts ? { toolStarts: input.toolStarts } : {}),
-        ...(input.toolArgsPreview ? { toolArgsPreview: input.toolArgsPreview } : {}),
-        ...(input.textPreview ? { textPreview: input.textPreview } : {}),
-    };
-}
-/** Result returned from create skipped eval. */
-export function createSkippedEvalResult(input) {
-    return {
-        id: input.id,
-        label: input.label,
-        runtime: input.runtime,
-        status: "skip",
-        details: input.details,
-        durationMs: Date.now() - input.startedAt,
-    };
-}
-/** Result returned from create failed eval. */
-export function createFailedEvalResult(input) {
-    return createEvalResult("fail", input);
-}
-/** Result returned from create passed eval. */
-export function createPassedEvalResult(input) {
-    return createEvalResult("pass", input);
-}