veryfront 0.1.522 → 0.1.524

package/src/src/agent/testing/durable-run-canaries/runner.ts

@@ -0,0 +1,582 @@
+import * as dntShim from "../../../../_dnt.shims.js";
+import { defineSchema } from "../../../schemas/index.js";
+import type { InferSchema } from "../../../extensions/schema/index.js";
+
+export interface DurableRunCanaryApiConfig {
+  apiUrl: string;
+  authToken: string;
+  agentId: string;
+  projectId: string | null;
+  branchId?: string | null;
+  requestTimeoutMs: number;
+  fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+}
+
+export interface DurableRunCanaryCreateRootRunInput {
+  conversationId: string;
+  runId: string;
+}
+
+export interface DurableRunCanarySendUserMessageInput {
+  conversationId: string;
+  prompt: string;
+}
+
+export interface DurableRunCanaryStartRunInput extends DurableRunCanaryCreateRootRunInput {
+  messageId: string;
+  prompt: string;
+  userMessageId: string;
+}
+
+export const getDurableRunCanaryMessageSchema = defineSchema((v) =>
+  v.object({
+    id: v.string(),
+    role: v.enum(["user", "assistant", "system", "tool"] as const),
+    status: v.string().optional(),
+    parts: v.array(v.object({ type: v.string() }).passthrough()).default([]),
+  }).passthrough()
+);
+
+export type DurableRunCanaryMessage = InferSchema<
+  ReturnType<typeof getDurableRunCanaryMessageSchema>
+>;
+
+export interface DurableRunCanaryRunSummary {
+  runId: string;
+  conversationId: string;
+  messageId: string;
+  agentId: string;
+  status: string;
+  latestEventId: number;
+  latestExternalEventSequence: number | null;
+  waitingToolCallId: string | null;
+  waitingToolName: string | null;
+  terminalErrorCode: string | null;
+  terminalErrorMessage: string | null;
+  startedAt: string | null;
+  finishedAt: string | null;
+}
+
+const getSnakeRunSummarySchema = defineSchema((v) =>
+  v.object({
+    run_id: v.string(),
+    conversation_id: v.string().uuid(),
+    message_id: v.string().uuid(),
+    agent_id: v.string(),
+    status: v.string(),
+    latest_event_id: v.number().int().nonnegative(),
+    latest_external_event_sequence: v.number().int().nonnegative().optional(),
+    waiting_tool_call_id: v.string().nullable().optional(),
+    waiting_tool_name: v.string().nullable().optional(),
+    terminal_error_code: v.string().nullable().optional(),
+    terminal_error_message: v.string().nullable().optional(),
+    started_at: v.string().nullable().optional(),
+    finished_at: v.string().nullable().optional(),
+  }).passthrough()
+);
+
+const getCamelRunSummarySchema = defineSchema((v) =>
+  v.object({
+    runId: v.string(),
+    conversationId: v.string().uuid(),
+    messageId: v.string().uuid(),
+    agentId: v.string(),
+    status: v.string(),
+    latestEventId: v.number().int().nonnegative(),
+    latestExternalEventSequence: v.number().int().nonnegative().optional(),
+    waitingToolCallId: v.string().nullable().optional(),
+    waitingToolName: v.string().nullable().optional(),
+    terminalErrorCode: v.string().nullable().optional(),
+    terminalErrorMessage: v.string().nullable().optional(),
+    startedAt: v.string().nullable().optional(),
+    finishedAt: v.string().nullable().optional(),
+  }).passthrough()
+);
+
+const getDurableRunCanaryMessageListSchema = defineSchema((v) =>
+  v.object({
+    data: v.array(getDurableRunCanaryMessageSchema()),
+  })
+);
+
+export function parseDurableRunCanaryRunSummary(value: unknown): DurableRunCanaryRunSummary {
+  const snake = getSnakeRunSummarySchema().safeParse(value);
+  if (snake.success) {
+    return {
+      runId: snake.data.run_id,
+      conversationId: snake.data.conversation_id,
+      messageId: snake.data.message_id,
+      agentId: snake.data.agent_id,
+      status: snake.data.status,
+      latestEventId: snake.data.latest_event_id,
+      latestExternalEventSequence: snake.data.latest_external_event_sequence ?? null,
+      waitingToolCallId: snake.data.waiting_tool_call_id ?? null,
+      waitingToolName: snake.data.waiting_tool_name ?? null,
+      terminalErrorCode: snake.data.terminal_error_code ?? null,
+      terminalErrorMessage: snake.data.terminal_error_message ?? null,
+      startedAt: snake.data.started_at ?? null,
+      finishedAt: snake.data.finished_at ?? null,
+    };
+  }
+
+  const camel = getCamelRunSummarySchema().parse(value);
+  return {
+    runId: camel.runId,
+    conversationId: camel.conversationId,
+    messageId: camel.messageId,
+    agentId: camel.agentId,
+    status: camel.status,
+    latestEventId: camel.latestEventId,
+    latestExternalEventSequence: camel.latestExternalEventSequence ?? null,
+    waitingToolCallId: camel.waitingToolCallId ?? null,
+    waitingToolName: camel.waitingToolName ?? null,
+    terminalErrorCode: camel.terminalErrorCode ?? null,
+    terminalErrorMessage: camel.terminalErrorMessage ?? null,
+    startedAt: camel.startedAt ?? null,
+    finishedAt: camel.finishedAt ?? null,
+  };
+}
+
+function createJsonHeaders(config: DurableRunCanaryApiConfig, headers?: HeadersInit): Headers {
+  const result = new Headers(headers);
+  if (!result.has("Content-Type")) {
+    result.set("Content-Type", "application/json");
+  }
+  result.set("Authorization", `Bearer ${config.authToken}`);
+  return result;
+}
+
+function createFetch(config: DurableRunCanaryApiConfig) {
+  return config.fetch ?? fetch;
+}
+
+function createApiUrl(config: DurableRunCanaryApiConfig, path: string): URL {
+  const baseHref = config.apiUrl.endsWith("/") ? config.apiUrl : `${config.apiUrl}/`;
+  const relativePath = path.startsWith("/") ? path.slice(1) : path;
+  return new URL(relativePath, baseHref);
+}
+
+function buildCreateRootRunBody(
+  config: DurableRunCanaryApiConfig,
+  input: DurableRunCanaryCreateRootRunInput,
+) {
+  return {
+    kind: "agent",
+    owner: {
+      kind: "conversation",
+      id: input.conversationId,
+    },
+    public_id: input.runId,
+    request: {
+      mode: "default_chat",
+      agent_id: config.agentId,
+      initial_status: "pending",
+      ...(config.projectId
+        ? {
+          source_target_kind: "project",
+          runtime_target_kind: "production",
+          runtime_target_branch_id: config.branchId ?? null,
+        }
+        : {}),
+    },
+  };
+}
+
+function buildStartRunBody(
+  config: DurableRunCanaryApiConfig,
+  input: DurableRunCanaryStartRunInput,
+) {
+  return {
+    kind: "agent",
+    owner: {
+      kind: "conversation",
+      id: input.conversationId,
+    },
+    public_id: input.runId,
+    request: {
+      mode: "default_chat",
+      agent_id: config.agentId,
+      input: {
+        messages: [
+          {
+            id: input.userMessageId,
+            role: "user",
+            parts: [{ type: "text", text: input.prompt }],
+          },
+        ],
+        context: {
+          conversation_id: input.conversationId,
+          project_id: config.projectId,
+          branch_id: config.branchId ?? null,
+        },
+        durable_root_run: {
+          run_id: input.runId,
+          message_id: input.messageId,
+        },
+      },
+    },
+  };
+}
+
+export interface DurableRunCanaryApiClient {
+  createDurableRootRun: (input: DurableRunCanaryCreateRootRunInput) => Promise<void>;
+  getRunSummary: (input: DurableRunCanaryCreateRootRunInput) => Promise<DurableRunCanaryRunSummary>;
+  listMessagesForCanary: (input: { conversationId: string }) => Promise<DurableRunCanaryMessage[]>;
+  sendUserMessageForCanary: (
+    input: DurableRunCanarySendUserMessageInput,
+  ) => Promise<DurableRunCanaryMessage>;
+  startDurableRun: (input: DurableRunCanaryStartRunInput) => Promise<void>;
+}
+
+export function createDurableRunCanaryApiClient(
+  config: DurableRunCanaryApiConfig,
+): DurableRunCanaryApiClient {
+  const request = createFetch(config);
+
+  async function apiFetch<T>(
+    path: string,
+    init: RequestInit | undefined,
+    parse: (value: unknown) => T,
+  ): Promise<T>;
+  async function apiFetch(path: string, init?: RequestInit): Promise<unknown>;
+  async function apiFetch<T>(
+    path: string,
+    init?: RequestInit,
+    parse?: (value: unknown) => T,
+  ): Promise<T | unknown> {
+    const response = await request(createApiUrl(config, path), {
+      ...init,
+      headers: createJsonHeaders(config, init?.headers),
+      signal: AbortSignal.timeout(config.requestTimeoutMs),
+    });
+
+    if (!response.ok) {
+      throw new Error(
+        `API ${init?.method ?? "GET"} ${path} failed: ${response.status} ${await response.text()}`,
+      );
+    }
+
+    const payload: unknown = await response.json();
+    return parse ? parse(payload) : payload;
+  }
+
+  async function sendUserMessageForCanary(input: DurableRunCanarySendUserMessageInput) {
+    return apiFetch(
+      `/conversations/${input.conversationId}/messages`,
+      {
+        method: "POST",
+        body: JSON.stringify({
+          role: "user",
+          parts: [{ type: "text", text: input.prompt }],
+        }),
+      },
+      (value) => getDurableRunCanaryMessageSchema().parse(value),
+    );
+  }
+
+  async function createDurableRootRun(input: DurableRunCanaryCreateRootRunInput): Promise<void> {
+    await apiFetch("/runs", {
+      method: "POST",
+      body: JSON.stringify(buildCreateRootRunBody(config, input)),
+    });
+  }
+
+  async function startDurableRun(input: DurableRunCanaryStartRunInput): Promise<void> {
+    await apiFetch("/runs", {
+      method: "POST",
+      body: JSON.stringify(buildStartRunBody(config, input)),
+    });
+  }
+
+  async function getRunSummary(input: DurableRunCanaryCreateRootRunInput) {
+    const response = await apiFetch(`/conversations/${input.conversationId}/runs/${input.runId}`);
+    return parseDurableRunCanaryRunSummary(response);
+  }
+
+  async function listMessagesForCanary(input: { conversationId: string }) {
+    const payload = await apiFetch(
+      `/conversations/${input.conversationId}/messages?limit=100`,
+      undefined,
+      (value) => getDurableRunCanaryMessageListSchema().parse(value),
+    );
+
+    return payload.data;
+  }
+
+  return {
+    createDurableRootRun,
+    getRunSummary,
+    listMessagesForCanary,
+    sendUserMessageForCanary,
+    startDurableRun,
+  };
+}
+
+export interface DurableRunCanaryResult {
+  id: string;
+  label: string;
+  status: "pass" | "fail";
+  details: string;
+  durationMs: number;
+  conversationId: string;
+  runId: string;
+  artifactPaths?: string[];
+}
+
+export interface DurableRunCanaryPreparedCase {
+  artifactPaths?: string[] | ((runId: string) => string[]);
+  cleanup: (input?: { runId: string }) => Promise<void>;
+  conversationId: string;
+  prompt: string;
+  startSidecar?: () => Promise<(() => Promise<void>) | void>;
+  title: string;
+  validate: (input: {
+    messages: DurableRunCanaryMessage[];
+    run: DurableRunCanaryRunSummary;
+  }) => Promise<void> | void;
+}
+
+export interface DurableRunCanaryCase {
+  id: string;
+  label: string;
+  prepare: () => Promise<DurableRunCanaryPreparedCase>;
+}
+
+export interface DurableRunCanaryRunnerConfig extends DurableRunCanaryApiConfig {
+  keepSuccessfulEvidence: boolean;
+}
+
+interface RunSummaryLocator {
+  conversationId: string;
+  runId: string;
+}
+
+interface WaitForRunInput extends RunSummaryLocator {
+  getRunSummary: (input: RunSummaryLocator) => Promise<DurableRunCanaryRunSummary>;
+}
+
+const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function collectChildConversationIdsFromValue(
+  value: unknown,
+  childConversationIds: Set<string>,
+  depth = 0,
+): void {
+  if (depth > 8) {
+    return;
+  }
+
+  if (typeof value === "string") {
+    try {
+      collectChildConversationIdsFromValue(JSON.parse(value), childConversationIds, depth + 1);
+    } catch {
+      return;
+    }
+    return;
+  }
+
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      collectChildConversationIdsFromValue(entry, childConversationIds, depth + 1);
+    }
+    return;
+  }
+
+  if (!isRecord(value)) {
+    return;
+  }
+
+  for (const key of ["childConversationId", "child_conversation_id"]) {
+    const childConversationId = value[key];
+    if (typeof childConversationId === "string" && UUID_PATTERN.test(childConversationId)) {
+      childConversationIds.add(childConversationId);
+    }
+  }
+
+  for (const nestedValue of Object.values(value)) {
+    collectChildConversationIdsFromValue(nestedValue, childConversationIds, depth + 1);
+  }
+}
+
+function collectReferencedChildConversationIds(messages: DurableRunCanaryMessage[]): string[] {
+  const childConversationIds = new Set<string>();
+
+  for (const message of messages) {
+    for (const part of message.parts) {
+      if (!isRecord(part) || (part.type !== "tool_result" && part.type !== "tool-result")) {
+        continue;
+      }
+
+      collectChildConversationIdsFromValue(part.output, childConversationIds);
+    }
+  }
+
+  return [...childConversationIds];
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    dntShim.setTimeout(resolve, ms);
+  });
+}
+
+function isTerminalRunStatus(status: string): boolean {
+  return status === "completed" || status === "failed" || status === "cancelled";
+}
+
+function createDurableRunCanaryRunId(): string {
+  return `run_${dntShim.crypto.randomUUID()}`;
+}
+
+async function waitForRunSummaryVisibility(
+  input: WaitForRunInput,
+): Promise<DurableRunCanaryRunSummary> {
+  const deadline = Date.now() + 30_000;
+
+  while (Date.now() < deadline) {
+    try {
+      return await input.getRunSummary(input);
+    } catch (error) {
+      if (!(error instanceof Error) || !error.message.includes(" 404 ")) {
+        throw error;
+      }
+    }
+
+    await sleep(500);
+  }
+
+  throw new Error(`Run ${input.runId} did not become visible in time`);
+}
+
+async function waitForTerminalRun(
+  input: WaitForRunInput & { requestTimeoutMs: number },
+): Promise<DurableRunCanaryRunSummary> {
+  const deadline = Date.now() + input.requestTimeoutMs;
+
+  while (Date.now() < deadline) {
+    const run = await input.getRunSummary(input);
+    if (isTerminalRunStatus(run.status)) {
+      return run;
+    }
+
+    await sleep(1_500);
+  }
+
+  throw new Error(`Timed out waiting for run ${input.runId} to reach a terminal state`);
+}
+
+export function createDurableRunCanaryRunner(
+  config: DurableRunCanaryRunnerConfig,
+  apiClient: DurableRunCanaryApiClient = createDurableRunCanaryApiClient(config),
+) {
+  const getRunSummary = apiClient.getRunSummary;
+
+  async function listMessagesWithReferencedChildren(
+    conversationId: string,
+  ): Promise<DurableRunCanaryMessage[]> {
+    const messages = await apiClient.listMessagesForCanary({ conversationId });
+    const childConversationIds = collectReferencedChildConversationIds(messages);
+    const childMessages = await Promise.all(
+      childConversationIds.map((childConversationId) =>
+        apiClient.listMessagesForCanary({ conversationId: childConversationId })
+      ),
+    );
+
+    return [...messages, ...childMessages.flat()];
+  }
+
+  async function runCase(testCase: DurableRunCanaryCase): Promise<DurableRunCanaryResult> {
+    const startedAt = Date.now();
+    const prepared = await testCase.prepare();
+    let runId = "unknown";
+    const stopSidecar = await prepared.startSidecar?.();
+    const resolveArtifactPaths = (currentRunId: string): string[] | undefined =>
+      typeof prepared.artifactPaths === "function"
+        ? prepared.artifactPaths(currentRunId)
+        : prepared.artifactPaths;
+
+    try {
+      const userMessage = await apiClient.sendUserMessageForCanary({
+        conversationId: prepared.conversationId,
+        prompt: prepared.prompt,
+      });
+      runId = createDurableRunCanaryRunId();
+
+      await apiClient.createDurableRootRun({
+        conversationId: prepared.conversationId,
+        runId,
+      });
+      const visibleRun = await waitForRunSummaryVisibility({
+        conversationId: prepared.conversationId,
+        getRunSummary,
+        runId,
+      });
+
+      await apiClient.startDurableRun({
+        conversationId: prepared.conversationId,
+        messageId: visibleRun.messageId,
+        prompt: prepared.prompt,
+        runId,
+        userMessageId: userMessage.id,
+      });
+
+      const terminalRun = await waitForTerminalRun({
+        conversationId: prepared.conversationId,
+        getRunSummary,
+        requestTimeoutMs: config.requestTimeoutMs,
+        runId,
+      });
+      const messages = await listMessagesWithReferencedChildren(prepared.conversationId);
+
+      await prepared.validate({
+        messages,
+        run: terminalRun,
+      });
+
+      const artifactPaths = resolveArtifactPaths(runId);
+
+      if (!config.keepSuccessfulEvidence) {
+        await prepared.cleanup({ runId });
+      }
+
+      return {
+        id: testCase.id,
+        label: testCase.label,
+        status: "pass",
+        details: "OK",
+        durationMs: Date.now() - startedAt,
+        conversationId: prepared.conversationId,
+        runId,
+        ...(artifactPaths?.length ? { artifactPaths } : {}),
+      };
+    } catch (error) {
+      const artifactPaths = resolveArtifactPaths(runId);
+
+      return {
+        id: testCase.id,
+        label: testCase.label,
+        status: "fail",
+        details: error instanceof Error ? error.message : String(error),
+        durationMs: Date.now() - startedAt,
+        conversationId: prepared.conversationId,
+        runId,
+        ...(artifactPaths?.length ? { artifactPaths } : {}),
+      };
+    } finally {
+      await stopSidecar?.();
+    }
+  }
+
+  return {
+    runCase,
+  };
+}
+
+export const durableRunCanaryRunnerInternals = {
+  collectReferencedChildConversationIds,
+  isTerminalRunStatus,
+};

package/src/src/agent/testing/index.ts

@@ -17,6 +17,25 @@ export {
   type TestSuite,
 } from "./agent-tester.js";
 
+export {
+  createDurableRunCanaryApiClient,
+  createDurableRunCanaryRunner,
+  type DurableRunCanaryApiClient,
+  type DurableRunCanaryApiConfig,
+  type DurableRunCanaryCase,
+  type DurableRunCanaryCreateRootRunInput,
+  type DurableRunCanaryMessage,
+  type DurableRunCanaryPreparedCase,
+  type DurableRunCanaryResult,
+  type DurableRunCanaryRunnerConfig,
+  durableRunCanaryRunnerInternals,
+  type DurableRunCanaryRunSummary,
+  type DurableRunCanarySendUserMessageInput,
+  type DurableRunCanaryStartRunInput,
+  getDurableRunCanaryMessageSchema,
+  parseDurableRunCanaryRunSummary,
+} from "./durable-run-canaries/index.js";
+
 export {
   buildFailureSuffix,
   buildLiveEvalCaseTagSummary,
@@ -27,17 +46,29 @@ export {
   buildProgressLine,
   buildRuntimePerformanceSummary,
   containsOrderedSubsequence,
+  containsSkillLoad,
+  countStepStartedEvents,
   createFailedEvalResult,
+  createLiveEvalCaseSupport,
   createPassedEvalResult,
   createPlainTextPdf,
   createSkippedEvalResult,
   hasEveryLiveEvalTag,
+  hasFinished,
+  type LiveEvalCase,
   type LiveEvalCaseMetadata,
   type LiveEvalCaseSelectionInput,
+  type LiveEvalContext,
+  type LiveEvalProjectFile,
+  type LiveEvalProjectFileReaderInput,
+  type LiveEvalRequestBody,
   type LiveEvalResultForPerformance,
   type LiveEvalResultForReport,
   type LiveEvalResultRecord,
+  type LiveEvalRunnerConfig,
+  liveEvalRunnerInternals,
   type LiveEvalRuntime,
+  type PreparedLiveEvalInput,
   resolveLiveEvalRequestedCaseIds,
   type RuntimePerformanceSummary,
   selectLiveEvalCases,

package/src/src/agent/testing/live-evals/index.ts

@@ -10,7 +10,11 @@ export {
   type LiveEvalRuntime,
   type RuntimePerformanceSummary,
 } from "./performance.js";
-export { buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput } from "./request.js";
+export {
+  buildLiveEvalRequestBody,
+  type BuildLiveEvalRequestBodyInput,
+  type LiveEvalRequestBody,
+} from "./request.js";
 export {
   buildLiveEvalCaseTagSummary,
   buildLiveEvalRuntimeSummary,
@@ -28,3 +32,16 @@ export {
   createSkippedEvalResult,
   type LiveEvalResultRecord,
 } from "./result.js";
+export {
+  containsSkillLoad,
+  countStepStartedEvents,
+  createLiveEvalCaseSupport,
+  hasFinished,
+  type LiveEvalCase,
+  type LiveEvalContext,
+  type LiveEvalProjectFile,
+  type LiveEvalProjectFileReaderInput,
+  type LiveEvalRunnerConfig,
+  liveEvalRunnerInternals,
+  type PreparedLiveEvalInput,
+} from "./runner.js";

package/src/src/agent/testing/live-evals/request.ts

@@ -1,4 +1,20 @@
 import * as dntShim from "../../../../_dnt.shims.js";
+export interface LiveEvalRequestBody {
+  threadId: string;
+  runId: string;
+  state: Record<string, string>;
+  tools: unknown[];
+  context: unknown[];
+  forwardedProps?: {
+    veryfront: Record<string, unknown>;
+  };
+  messages: Array<{
+    id: string;
+    role: "user";
+    content: string;
+  }>;
+}
+
 export interface BuildLiveEvalRequestBodyInput {
   testCaseId: string;
   prompt: string;
@@ -12,7 +28,9 @@ export interface BuildLiveEvalRequestBodyInput {
   maxSteps?: number;
 }
 
-export function buildLiveEvalRequestBody(input: BuildLiveEvalRequestBodyInput) {
+export function buildLiveEvalRequestBody(
+  input: BuildLiveEvalRequestBodyInput,
+): LiveEvalRequestBody {
   const veryfront: Record<string, unknown> = {};
   if (input.projectId) {
     veryfront.projectId = input.projectId;