veryfront 0.1.877 → 0.1.878

package/esm/deno.js

@@ -1,6 +1,6 @@
 export default {
     "name": "veryfront",
-    "version": "0.1.877",
+    "version": "0.1.878",
     "license": "Apache-2.0",
     "nodeModulesDir": "auto",
     "minimumDependencyAge": {

package/esm/src/eval/agent-service/durable-run-canaries/cli-runner.d.ts

@@ -0,0 +1,21 @@
+import { type LiveEvalApiContext } from "../live-evals/api-client.js";
+import { createDurableRunCanaryRunner, type DurableRunCanaryCase, type DurableRunCanaryRunnerConfig } from "./runner.js";
+type EnvRecord = Record<string, string | undefined>;
+/** Input payload for durable run canary cli case factory. */
+export interface DurableRunCanaryCliCaseFactoryInput {
+    context: LiveEvalApiContext;
+    requestTimeoutMs: number;
+}
+/** Input payload for run durable run canary cli. */
+export interface RunDurableRunCanaryCliInput {
+    env: EnvRecord;
+    agentId: string;
+    createCases: (input: DurableRunCanaryCliCaseFactoryInput) => DurableRunCanaryCase[];
+    cwd?: string;
+    log?: (message: string) => void;
+    createRunner?: (config: DurableRunCanaryRunnerConfig) => ReturnType<typeof createDurableRunCanaryRunner>;
+}
+/** Run durable run canary cli. */
+export declare function runDurableRunCanaryCli(input: RunDurableRunCanaryCliInput): Promise<number>;
+export {};
+//# sourceMappingURL=cli-runner.d.ts.map

package/esm/src/eval/agent-service/durable-run-canaries/cli-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/durable-run-canaries/cli-runner.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AAEtE,OAAO,EACL,4BAA4B,EAC5B,KAAK,oBAAoB,EAEzB,KAAK,4BAA4B,EAClC,MAAM,aAAa,CAAC;AAErB,KAAK,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAEpD,6DAA6D;AAC7D,MAAM,WAAW,mCAAmC;IAClD,OAAO,EAAE,kBAAkB,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,oDAAoD;AACpD,MAAM,WAAW,2BAA2B;IAC1C,GAAG,EAAE,SAAS,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,CAAC,KAAK,EAAE,mCAAmC,KAAK,oBAAoB,EAAE,CAAC;IACpF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,YAAY,CAAC,EAAE,CACb,MAAM,EAAE,4BAA4B,KACjC,UAAU,CAAC,OAAO,4BAA4B,CAAC,CAAC;CACtD;AAcD,kCAAkC;AAClC,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,2BAA2B,GACjC,OAAO,CAAC,MAAM,CAAC,CAwEjB"}

package/esm/src/eval/agent-service/durable-run-canaries/cli-runner.js

@@ -0,0 +1,66 @@
+import { mkdir, writeTextFile } from "../../../platform/compat/fs.js";
+import { dirname, resolve } from "../../../platform/compat/path/index.js";
+import { cwd as getProcessCwd } from "../../../platform/compat/process.js";
+import { resolveDurableRunCanaryEnvironment } from "./environment.js";
+import { createDurableRunCanaryRunner, } from "./runner.js";
+function createTimestampedReportPath(input) {
+    return resolve(input.cwd, ".omx/logs", input.directory, `${new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-")}.json`);
+}
+/** Run durable run canary cli. */
+export async function runDurableRunCanaryCli(input) {
+    const log = input.log ?? console.log;
+    const cwd = input.cwd ?? getProcessCwd();
+    const { apiUrl, authToken, projectId, requestTimeoutMs, keepSuccessfulEvidence } = resolveDurableRunCanaryEnvironment(input.env);
+    const reportPath = input.env.DURABLE_CANARY_REPORT_PATH ??
+        createTimestampedReportPath({ cwd, directory: "durable-run-staging-canaries" });
+    if (!authToken) {
+        throw new Error("Missing VERYFRONT_TOKEN");
+    }
+    if (!projectId) {
+        throw new Error("Missing AG_UI_EVAL_PROJECT_ID");
+    }
+    const context = {
+        apiUrl,
+        authToken,
+        projectId,
+    };
+    const createRunner = input.createRunner ?? createDurableRunCanaryRunner;
+    const { runCase } = createRunner({
+        apiUrl,
+        authToken,
+        agentId: input.agentId,
+        projectId,
+        requestTimeoutMs,
+        keepSuccessfulEvidence,
+    });
+    const testCases = input.createCases({
+        context,
+        requestTimeoutMs,
+    });
+    log(`Durable run canaries -> ${apiUrl}`);
+    log(`Project scope -> ${projectId}`);
+    const results = [];
+    for (const testCase of testCases) {
+        log(`\n[run] ${testCase.label}`);
+        const result = await runCase(testCase);
+        results.push(result);
+        log(`[${result.status}] ${result.id}: ${result.details}`);
+    }
+    const summary = {
+        passed: results.filter((result) => result.status === "pass").length,
+        failed: results.filter((result) => result.status === "fail").length,
+    };
+    await mkdir(dirname(reportPath), { recursive: true });
+    await writeTextFile(reportPath, JSON.stringify({
+        generatedAt: new Date().toISOString(),
+        apiUrl,
+        projectId,
+        results,
+        summary,
+    }, null, 2));
+    log("\nSummary");
+    log(`passed: ${summary.passed}`);
+    log(`failed: ${summary.failed}`);
+    log(`report: ${reportPath}`);
+    return summary.failed > 0 ? 1 : 0;
+}

package/esm/src/eval/agent-service/durable-run-canaries/environment.d.ts

@@ -0,0 +1,14 @@
+import { type AgentServiceConfigInput } from "../../../agent/service/config.js";
+/** Public API contract for durable run canary environment. */
+export interface DurableRunCanaryEnvironment {
+    apiUrl: string;
+    authToken: string;
+    projectId: string;
+    requestTimeoutMs: number;
+    keepSuccessfulEvidence: boolean;
+}
+/** Default value for durable run canary timeout ms. */
+export declare const DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS = 240000;
+/** Resolves durable run canary environment. */
+export declare function resolveDurableRunCanaryEnvironment(env?: AgentServiceConfigInput): DurableRunCanaryEnvironment;
+//# sourceMappingURL=environment.d.ts.map

package/esm/src/eval/agent-service/durable-run-canaries/environment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"environment.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/durable-run-canaries/environment.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,uBAAuB,EAE7B,MAAM,kCAAkC,CAAC;AAE1C,8DAA8D;AAC9D,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,OAAO,CAAC;CACjC;AAED,uDAAuD;AACvD,eAAO,MAAM,qCAAqC,SAAU,CAAC;AAE7D,+CAA+C;AAC/C,wBAAgB,kCAAkC,CAChD,GAAG,GAAE,uBAA4B,GAChC,2BAA2B,CAY7B"}

package/esm/src/eval/agent-service/durable-run-canaries/environment.js

@@ -0,0 +1,15 @@
+import { parseAgentServiceConfig, } from "../../../agent/service/config.js";
+/** Default value for durable run canary timeout ms. */
+export const DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS = 240_000;
+/** Resolves durable run canary environment. */
+export function resolveDurableRunCanaryEnvironment(env = {}) {
+    return {
+        apiUrl: typeof env.VERYFRONT_API_URL === "string"
+            ? env.VERYFRONT_API_URL
+            : parseAgentServiceConfig(env).VERYFRONT_API_URL,
+        authToken: typeof env.VERYFRONT_TOKEN === "string" ? env.VERYFRONT_TOKEN : "",
+        projectId: typeof env.AG_UI_EVAL_PROJECT_ID === "string" ? env.AG_UI_EVAL_PROJECT_ID : "",
+        requestTimeoutMs: Number(env.DURABLE_CANARY_TIMEOUT_MS ?? DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS),
+        keepSuccessfulEvidence: env.DURABLE_CANARY_KEEP_SUCCESS === "1",
+    };
+}

package/esm/src/eval/agent-service/durable-run-canaries/index.d.ts

@@ -0,0 +1,5 @@
+export { type DurableRunCanaryCliCaseFactoryInput, runDurableRunCanaryCli, type RunDurableRunCanaryCliInput, } from "./cli-runner.js";
+export { DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS, type DurableRunCanaryEnvironment, resolveDurableRunCanaryEnvironment, } from "./environment.js";
+export { createDurableRunCanaryApiClient, createDurableRunCanaryRunner, type DurableRunCanaryApiClient, type DurableRunCanaryApiConfig, type DurableRunCanaryCase, type DurableRunCanaryCreateRootRunInput, type DurableRunCanaryMessage, type DurableRunCanaryPreparedCase, type DurableRunCanaryResult, type DurableRunCanaryRunnerConfig, durableRunCanaryRunnerInternals, type DurableRunCanaryRunSummary, type DurableRunCanarySendUserMessageInput, type DurableRunCanaryStartRunInput, getDurableRunCanaryMessageSchema, parseDurableRunCanaryRunSummary, } from "./runner.js";
+export { assertCompleted, assertNoMalformedCreateFileToolCalls, collectAssistantText, findAssistantMessage, stringifyUnknown, } from "./validation.js";
+//# sourceMappingURL=index.d.ts.map

package/esm/src/eval/agent-service/durable-run-canaries/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/durable-run-canaries/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,mCAAmC,EACxC,sBAAsB,EACtB,KAAK,2BAA2B,GACjC,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,qCAAqC,EACrC,KAAK,2BAA2B,EAChC,kCAAkC,GACnC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,EAC5B,KAAK,yBAAyB,EAC9B,KAAK,yBAAyB,EAC9B,KAAK,oBAAoB,EACzB,KAAK,kCAAkC,EACvC,KAAK,uBAAuB,EAC5B,KAAK,4BAA4B,EACjC,KAAK,sBAAsB,EAC3B,KAAK,4BAA4B,EACjC,+BAA+B,EAC/B,KAAK,0BAA0B,EAC/B,KAAK,oCAAoC,EACzC,KAAK,6BAA6B,EAClC,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,aAAa,CAAC;AAErB,OAAO,EACL,eAAe,EACf,oCAAoC,EACpC,oBAAoB,EACpB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC"}

package/esm/src/eval/agent-service/durable-run-canaries/index.js

@@ -0,0 +1,4 @@
+export { runDurableRunCanaryCli, } from "./cli-runner.js";
+export { DEFAULT_DURABLE_RUN_CANARY_TIMEOUT_MS, resolveDurableRunCanaryEnvironment, } from "./environment.js";
+export { createDurableRunCanaryApiClient, createDurableRunCanaryRunner, durableRunCanaryRunnerInternals, getDurableRunCanaryMessageSchema, parseDurableRunCanaryRunSummary, } from "./runner.js";
+export { assertCompleted, assertNoMalformedCreateFileToolCalls, collectAssistantText, findAssistantMessage, stringifyUnknown, } from "./validation.js";

package/esm/src/eval/agent-service/durable-run-canaries/runner.d.ts

@@ -0,0 +1,118 @@
+import type { InferSchema } from "../../../extensions/schema/index.js";
+/** Configuration used by durable run canary API. */
+export interface DurableRunCanaryApiConfig {
+    apiUrl: string;
+    authToken: string;
+    agentId: string;
+    projectId: string | null;
+    branchId?: string | null;
+    requestTimeoutMs: number;
+    fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+}
+/** Input payload for durable run canary create root run. */
+export interface DurableRunCanaryCreateRootRunInput {
+    conversationId: string;
+    runId: string;
+}
+/** Input payload for durable run canary send user message. */
+export interface DurableRunCanarySendUserMessageInput {
+    conversationId: string;
+    prompt: string;
+}
+/** Input payload for durable run canary start run. */
+export interface DurableRunCanaryStartRunInput extends DurableRunCanaryCreateRootRunInput {
+    messageId: string;
+    prompt: string;
+    userMessageId: string;
+}
+/** Zod schema for get durable run canary message. */
+export declare const getDurableRunCanaryMessageSchema: () => import("../../../internal-agents/schema.js").Schema<{
+    id: string;
+    role: "tool" | "user" | "assistant" | "system";
+    parts: ({
+        type: string;
+    } & {} & Record<string, unknown>)[];
+} & {
+    status?: string | undefined;
+} & Record<string, unknown>>;
+/** Message shape for durable run canary. */
+export type DurableRunCanaryMessage = InferSchema<ReturnType<typeof getDurableRunCanaryMessageSchema>>;
+/** Public API contract for durable run canary run summary. */
+export interface DurableRunCanaryRunSummary {
+    runId: string;
+    conversationId: string;
+    messageId: string;
+    agentId: string;
+    status: string;
+    latestEventId: number;
+    latestExternalEventSequence: number | null;
+    waitingToolCallId: string | null;
+    waitingToolName: string | null;
+    terminalErrorCode: string | null;
+    terminalErrorMessage: string | null;
+    startedAt: string | null;
+    finishedAt: string | null;
+}
+/** Parses durable run canary run summary. */
+export declare function parseDurableRunCanaryRunSummary(value: unknown): DurableRunCanaryRunSummary;
+/** Public API contract for durable run canary API client. */
+export interface DurableRunCanaryApiClient {
+    createDurableRootRun: (input: DurableRunCanaryCreateRootRunInput) => Promise<void>;
+    getRunSummary: (input: DurableRunCanaryCreateRootRunInput) => Promise<DurableRunCanaryRunSummary>;
+    listMessagesForCanary: (input: {
+        conversationId: string;
+    }) => Promise<DurableRunCanaryMessage[]>;
+    sendUserMessageForCanary: (input: DurableRunCanarySendUserMessageInput) => Promise<DurableRunCanaryMessage>;
+    startDurableRun: (input: DurableRunCanaryStartRunInput) => Promise<void>;
+}
+/** Create durable run canary API client. */
+export declare function createDurableRunCanaryApiClient(config: DurableRunCanaryApiConfig): DurableRunCanaryApiClient;
+/** Result returned from durable run canary. */
+export interface DurableRunCanaryResult {
+    id: string;
+    label: string;
+    status: "pass" | "fail";
+    details: string;
+    durationMs: number;
+    conversationId: string;
+    runId: string;
+    artifactPaths?: string[];
+}
+/** Public API contract for durable run canary prepared case. */
+export interface DurableRunCanaryPreparedCase {
+    artifactPaths?: string[] | ((runId: string) => string[]);
+    cleanup: (input?: {
+        runId: string;
+    }) => Promise<void>;
+    conversationId: string;
+    prompt: string;
+    startSidecar?: () => Promise<(() => Promise<void>) | void>;
+    title: string;
+    validate: (input: {
+        messages: DurableRunCanaryMessage[];
+        run: DurableRunCanaryRunSummary;
+    }) => Promise<void> | void;
+}
+/** Public API contract for durable run canary case. */
+export interface DurableRunCanaryCase {
+    id: string;
+    label: string;
+    prepare: () => Promise<DurableRunCanaryPreparedCase>;
+}
+/** Configuration used by durable run canary runner. */
+export interface DurableRunCanaryRunnerConfig extends DurableRunCanaryApiConfig {
+    keepSuccessfulEvidence: boolean;
+}
+declare function collectReferencedChildConversationIds(messages: DurableRunCanaryMessage[]): string[];
+declare function isTerminalRunStatus(status: string): boolean;
+/** Create durable run canary runner. */
+export declare function createDurableRunCanaryRunner(config: DurableRunCanaryRunnerConfig, apiClient?: DurableRunCanaryApiClient): {
+    runCase: (testCase: DurableRunCanaryCase) => Promise<DurableRunCanaryResult>;
+};
+/** White-box helpers used by durable run canary tests. */
+export declare const durableRunCanaryRunnerInternals: {
+    collectReferencedChildConversationIds: typeof collectReferencedChildConversationIds;
+    isTerminalRunStatus: typeof isTerminalRunStatus;
+};
+export {};
+//# sourceMappingURL=runner.d.ts.map

package/esm/src/eval/agent-service/durable-run-canaries/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/durable-run-canaries/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAIvE,oDAAoD;AACpD,MAAM,WAAW,yBAAyB;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CAClF;AAED,4DAA4D;AAC5D,MAAM,WAAW,kCAAkC;IACjD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,8DAA8D;AAC9D,MAAM,WAAW,oCAAoC;IACnD,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,sDAAsD;AACtD,MAAM,WAAW,6BAA8B,SAAQ,kCAAkC;IACvF,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,qDAAqD;AACrD,eAAO,MAAM,gCAAgC;;;;;;;;4BAO5C,CAAC;AAEF,4CAA4C;AAC5C,MAAM,MAAM,uBAAuB,GAAG,WAAW,CAC/C,UAAU,CAAC,OAAO,gCAAgC,CAAC,CACpD,CAAC;AAEF,8DAA8D;AAC9D,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AA4CD,6CAA6C;AAC7C,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,OAAO,GAAG,0BAA0B,CAoC1F;AAkGD,6DAA6D;AAC7D,MAAM,WAAW,yBAAyB;IACxC,oBAAoB,EAAE,CAAC,KAAK,EAAE,kCAAkC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACnF,aAAa,EAAE,CAAC,KAAK,EAAE,kCAAkC,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC;IAClG,qBAAqB,EAAE,CAAC,KAAK,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACjG,wBAAwB,EAAE,CACxB,KAAK,EAAE,oCAAoC,KACxC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACtC,eAAe,EAAE,CAAC,KAAK,EAAE,6BAA6B,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED,4CAA4C;AAC5C,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,yBAAyB,GAChC,yBAAyB,CAgF3B;AAED,+CAA+C;AAC/C,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,gEAAgE;AAChE,MAAM,WAAW,4BAA4B;IAC3C,aAAa,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IACzD,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC3D,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,CAAC,KAAK,EAAE;QAChB,QAAQ,EAAE,uBAAuB,EAAE,CAAC;QACpC,GAAG,EAAE,0BAA0B,CAAC;KACjC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5B;AAED,uDAAuD;AACvD,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,OAAO,CAAC,4BAA4B,CAAC,CAAC;CACtD;AAED,uDAAuD;AACvD,MAAM,WAAW,4BAA6B,SAAQ,yBAAyB;IAC7E,sBAAsB,EAAE,OAAO,CAAC;CACjC;AA0DD,iBAAS,qCAAqC,CAAC,QAAQ,EAAE,uBAAuB,EAAE,GAAG,MAAM,EAAE,CAc5F;AAQD,iBAAS,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEpD;AA2CD,wCAAwC;AACxC,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,4BAA4B,EACpC,SAAS,GAAE,yBAAmE;wBAkB7C,oBAAoB,KAAG,OAAO,CAAC,sBAAsB,CAAC;EAqFxF;AAED,0DAA0D;AAC1D,eAAO,MAAM,+BAA+B;;;CAG3C,CAAC"}

package/esm/src/eval/agent-service/durable-run-canaries/runner.js

@@ -0,0 +1,391 @@
+import * as dntShim from "../../../../_dnt.shims.js";
+import { defineSchema } from "../../../schemas/index.js";
+import { ensureBuiltinSchemaValidator } from "../../../extensions/builtin-extensions.js";
+ensureBuiltinSchemaValidator();
+/** Zod schema for get durable run canary message. */
+export const getDurableRunCanaryMessageSchema = defineSchema((v) => v.object({
+    id: v.string(),
+    role: v.enum(["user", "assistant", "system", "tool"]),
+    status: v.string().optional(),
+    parts: v.array(v.object({ type: v.string() }).passthrough()).default([]),
+}).passthrough());
+const getSnakeRunSummarySchema = defineSchema((v) => v.object({
+    run_id: v.string(),
+    conversation_id: v.string().uuid(),
+    message_id: v.string().uuid(),
+    agent_id: v.string(),
+    status: v.string(),
+    latest_event_id: v.number().int().nonnegative(),
+    latest_external_event_sequence: v.number().int().nonnegative().optional(),
+    waiting_tool_call_id: v.string().nullable().optional(),
+    waiting_tool_name: v.string().nullable().optional(),
+    terminal_error_code: v.string().nullable().optional(),
+    terminal_error_message: v.string().nullable().optional(),
+    started_at: v.string().nullable().optional(),
+    finished_at: v.string().nullable().optional(),
+}).passthrough());
+const getCamelRunSummarySchema = defineSchema((v) => v.object({
+    runId: v.string(),
+    conversationId: v.string().uuid(),
+    messageId: v.string().uuid(),
+    agentId: v.string(),
+    status: v.string(),
+    latestEventId: v.number().int().nonnegative(),
+    latestExternalEventSequence: v.number().int().nonnegative().optional(),
+    waitingToolCallId: v.string().nullable().optional(),
+    waitingToolName: v.string().nullable().optional(),
+    terminalErrorCode: v.string().nullable().optional(),
+    terminalErrorMessage: v.string().nullable().optional(),
+    startedAt: v.string().nullable().optional(),
+    finishedAt: v.string().nullable().optional(),
+}).passthrough());
+const getDurableRunCanaryMessageListSchema = defineSchema((v) => v.object({
+    data: v.array(getDurableRunCanaryMessageSchema()),
+}));
+/** Parses durable run canary run summary. */
+export function parseDurableRunCanaryRunSummary(value) {
+    const snake = getSnakeRunSummarySchema().safeParse(value);
+    if (snake.success) {
+        return {
+            runId: snake.data.run_id,
+            conversationId: snake.data.conversation_id,
+            messageId: snake.data.message_id,
+            agentId: snake.data.agent_id,
+            status: snake.data.status,
+            latestEventId: snake.data.latest_event_id,
+            latestExternalEventSequence: snake.data.latest_external_event_sequence ?? null,
+            waitingToolCallId: snake.data.waiting_tool_call_id ?? null,
+            waitingToolName: snake.data.waiting_tool_name ?? null,
+            terminalErrorCode: snake.data.terminal_error_code ?? null,
+            terminalErrorMessage: snake.data.terminal_error_message ?? null,
+            startedAt: snake.data.started_at ?? null,
+            finishedAt: snake.data.finished_at ?? null,
+        };
+    }
+    const camel = getCamelRunSummarySchema().parse(value);
+    return {
+        runId: camel.runId,
+        conversationId: camel.conversationId,
+        messageId: camel.messageId,
+        agentId: camel.agentId,
+        status: camel.status,
+        latestEventId: camel.latestEventId,
+        latestExternalEventSequence: camel.latestExternalEventSequence ?? null,
+        waitingToolCallId: camel.waitingToolCallId ?? null,
+        waitingToolName: camel.waitingToolName ?? null,
+        terminalErrorCode: camel.terminalErrorCode ?? null,
+        terminalErrorMessage: camel.terminalErrorMessage ?? null,
+        startedAt: camel.startedAt ?? null,
+        finishedAt: camel.finishedAt ?? null,
+    };
+}
+function createJsonHeaders(config, headers) {
+    const result = new Headers(headers);
+    if (!result.has("Content-Type")) {
+        result.set("Content-Type", "application/json");
+    }
+    result.set("Authorization", `Bearer ${config.authToken}`);
+    return result;
+}
+function createFetch(config) {
+    return config.fetch ?? fetch;
+}
+function createApiUrl(config, path) {
+    const baseHref = config.apiUrl.endsWith("/") ? config.apiUrl : `${config.apiUrl}/`;
+    const relativePath = path.startsWith("/") ? path.slice(1) : path;
+    return new URL(relativePath, baseHref);
+}
+function buildCreateRootRunTargetFields(config) {
+    if (!config.projectId) {
+        return {};
+    }
+    if (config.branchId) {
+        return {
+            source_target_kind: "preview_branch",
+            runtime_target_kind: "preview_branch",
+            source_target_branch_id: config.branchId,
+            runtime_target_branch_id: config.branchId,
+        };
+    }
+    return {
+        source_target_kind: "project",
+        runtime_target_kind: "main_branch",
+        runtime_target_branch_id: null,
+    };
+}
+function buildCreateRootRunBody(config, input) {
+    return {
+        kind: "agent",
+        owner: {
+            kind: "conversation",
+            id: input.conversationId,
+        },
+        public_id: input.runId,
+        request: {
+            mode: "agent",
+            agent_id: config.agentId,
+            initial_status: "pending",
+            ...buildCreateRootRunTargetFields(config),
+        },
+    };
+}
+function buildStartRunBody(config, input) {
+    return {
+        kind: "agent",
+        owner: {
+            kind: "conversation",
+            id: input.conversationId,
+        },
+        public_id: input.runId,
+        request: {
+            mode: "agent",
+            agent_id: config.agentId,
+            input: {
+                messages: [
+                    {
+                        id: input.userMessageId,
+                        role: "user",
+                        parts: [{ type: "text", text: input.prompt }],
+                    },
+                ],
+                context: {
+                    conversation_id: input.conversationId,
+                    project_id: config.projectId,
+                    branch_id: config.branchId ?? null,
+                },
+                durable_root_run: {
+                    run_id: input.runId,
+                    message_id: input.messageId,
+                },
+            },
+        },
+    };
+}
+/** Create durable run canary API client. */
+export function createDurableRunCanaryApiClient(config) {
+    const request = createFetch(config);
+    async function apiFetch(path, init, parse) {
+        const response = await request(createApiUrl(config, path), {
+            ...init,
+            headers: createJsonHeaders(config, init?.headers),
+            signal: AbortSignal.timeout(config.requestTimeoutMs),
+        });
+        if (!response.ok) {
+            throw new Error(`API ${init?.method ?? "GET"} ${path} failed: ${response.status} ${await response.text()}`);
+        }
+        const payload = await response.json();
+        return parse ? parse(payload) : payload;
+    }
+    async function sendUserMessageForCanary(input) {
+        return apiFetch(`/conversations/${input.conversationId}/messages`, {
+            method: "POST",
+            body: JSON.stringify({
+                role: "user",
+                parts: [{ type: "text", text: input.prompt }],
+            }),
+        }, (value) => getDurableRunCanaryMessageSchema().parse(value));
+    }
+    async function createDurableRootRun(input) {
+        await apiFetch("/runs", {
+            method: "POST",
+            body: JSON.stringify(buildCreateRootRunBody(config, input)),
+        });
+    }
+    async function startDurableRun(input) {
+        await apiFetch("/runs", {
+            method: "POST",
+            body: JSON.stringify(buildStartRunBody(config, input)),
+        });
+    }
+    async function getRunSummary(input) {
+        const response = await apiFetch(`/conversations/${input.conversationId}/runs/${input.runId}`);
+        return parseDurableRunCanaryRunSummary(response);
+    }
+    async function listMessagesForCanary(input) {
+        const payload = await apiFetch(`/conversations/${input.conversationId}/messages?limit=100`, undefined, (value) => getDurableRunCanaryMessageListSchema().parse(value));
+        return payload.data;
+    }
+    return {
+        createDurableRootRun,
+        getRunSummary,
+        listMessagesForCanary,
+        sendUserMessageForCanary,
+        startDurableRun,
+    };
+}
+const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+function isRecord(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function collectChildConversationIdsFromValue(value, childConversationIds, depth = 0) {
+    if (depth > 8) {
+        return;
+    }
+    if (typeof value === "string") {
+        try {
+            collectChildConversationIdsFromValue(JSON.parse(value), childConversationIds, depth + 1);
+        }
+        catch {
+            return;
+        }
+        return;
+    }
+    if (Array.isArray(value)) {
+        for (const entry of value) {
+            collectChildConversationIdsFromValue(entry, childConversationIds, depth + 1);
+        }
+        return;
+    }
+    if (!isRecord(value)) {
+        return;
+    }
+    for (const key of ["childConversationId", "child_conversation_id"]) {
+        const childConversationId = value[key];
+        if (typeof childConversationId === "string" && UUID_PATTERN.test(childConversationId)) {
+            childConversationIds.add(childConversationId);
+        }
+    }
+    for (const nestedValue of Object.values(value)) {
+        collectChildConversationIdsFromValue(nestedValue, childConversationIds, depth + 1);
+    }
+}
+function collectReferencedChildConversationIds(messages) {
+    const childConversationIds = new Set();
+    for (const message of messages) {
+        for (const part of message.parts) {
+            if (!isRecord(part) || (part.type !== "tool_result" && part.type !== "tool-result")) {
+                continue;
+            }
+            collectChildConversationIdsFromValue(part.output, childConversationIds);
+        }
+    }
+    return [...childConversationIds];
+}
+function sleep(ms) {
+    return new Promise((resolve) => {
+        dntShim.setTimeout(resolve, ms);
+    });
+}
+function isTerminalRunStatus(status) {
+    return status === "completed" || status === "failed" || status === "cancelled";
+}
+function createDurableRunCanaryRunId() {
+    return `run_${dntShim.crypto.randomUUID()}`;
+}
+async function waitForRunSummaryVisibility(input) {
+    const deadline = Date.now() + 30_000;
+    while (Date.now() < deadline) {
+        try {
+            return await input.getRunSummary(input);
+        }
+        catch (error) {
+            if (!(error instanceof Error) || !error.message.includes(" 404 ")) {
+                throw error;
+            }
+        }
+        await sleep(500);
+    }
+    throw new Error(`Run ${input.runId} did not become visible in time`);
+}
+async function waitForTerminalRun(input) {
+    const deadline = Date.now() + input.requestTimeoutMs;
+    while (Date.now() < deadline) {
+        const run = await input.getRunSummary(input);
+        if (isTerminalRunStatus(run.status)) {
+            return run;
+        }
+        await sleep(1_500);
+    }
+    throw new Error(`Timed out waiting for run ${input.runId} to reach a terminal state`);
+}
+/** Create durable run canary runner. */
+export function createDurableRunCanaryRunner(config, apiClient = createDurableRunCanaryApiClient(config)) {
+    const getRunSummary = apiClient.getRunSummary;
+    async function listMessagesWithReferencedChildren(conversationId) {
+        const messages = await apiClient.listMessagesForCanary({ conversationId });
+        const childConversationIds = collectReferencedChildConversationIds(messages);
+        const childMessages = await Promise.all(childConversationIds.map((childConversationId) => apiClient.listMessagesForCanary({ conversationId: childConversationId })));
+        return [...messages, ...childMessages.flat()];
+    }
+    async function runCase(testCase) {
+        const startedAt = Date.now();
+        const prepared = await testCase.prepare();
+        let runId = "unknown";
+        const stopSidecar = await prepared.startSidecar?.();
+        const resolveArtifactPaths = (currentRunId) => typeof prepared.artifactPaths === "function"
+            ? prepared.artifactPaths(currentRunId)
+            : prepared.artifactPaths;
+        try {
+            const userMessage = await apiClient.sendUserMessageForCanary({
+                conversationId: prepared.conversationId,
+                prompt: prepared.prompt,
+            });
+            runId = createDurableRunCanaryRunId();
+            await apiClient.createDurableRootRun({
+                conversationId: prepared.conversationId,
+                runId,
+            });
+            const visibleRun = await waitForRunSummaryVisibility({
+                conversationId: prepared.conversationId,
+                getRunSummary,
+                runId,
+            });
+            await apiClient.startDurableRun({
+                conversationId: prepared.conversationId,
+                messageId: visibleRun.messageId,
+                prompt: prepared.prompt,
+                runId,
+                userMessageId: userMessage.id,
+            });
+            const terminalRun = await waitForTerminalRun({
+                conversationId: prepared.conversationId,
+                getRunSummary,
+                requestTimeoutMs: config.requestTimeoutMs,
+                runId,
+            });
+            const messages = await listMessagesWithReferencedChildren(prepared.conversationId);
+            await prepared.validate({
+                messages,
+                run: terminalRun,
+            });
+            const artifactPaths = resolveArtifactPaths(runId);
+            if (!config.keepSuccessfulEvidence) {
+                await prepared.cleanup({ runId });
+            }
+            return {
+                id: testCase.id,
+                label: testCase.label,
+                status: "pass",
+                details: "OK",
+                durationMs: Date.now() - startedAt,
+                conversationId: prepared.conversationId,
+                runId,
+                ...(artifactPaths?.length ? { artifactPaths } : {}),
+            };
+        }
+        catch (error) {
+            const artifactPaths = resolveArtifactPaths(runId);
+            return {
+                id: testCase.id,
+                label: testCase.label,
+                status: "fail",
+                details: error instanceof Error ? error.message : String(error),
+                durationMs: Date.now() - startedAt,
+                conversationId: prepared.conversationId,
+                runId,
+                ...(artifactPaths?.length ? { artifactPaths } : {}),
+            };
+        }
+        finally {
+            await stopSidecar?.();
+        }
+    }
+    return {
+        runCase,
+    };
+}
+/** White-box helpers used by durable run canary tests. */
+export const durableRunCanaryRunnerInternals = {
+    collectReferencedChildConversationIds,
+    isTerminalRunStatus,
+};