veryfront 0.1.529 → 0.1.531

package/esm/deno.js

@@ -1,6 +1,6 @@
 export default {
     "name": "veryfront",
-    "version": "0.1.529",
+    "version": "0.1.531",
     "license": "Apache-2.0",
     "nodeModulesDir": "auto",
     "workspace": [

package/esm/src/agent/testing/index.d.ts

@@ -6,5 +6,5 @@
 import "../../../_dnt.polyfills.js";
 export { assertCompleted, assertContains, assertToolCalled, printTestResults, testAgent, type TestCase, type TestResult, type TestSuite, } from "./agent-tester.js";
 export { createDurableRunCanaryApiClient, createDurableRunCanaryRunner, type DurableRunCanaryApiClient, type DurableRunCanaryApiConfig, type DurableRunCanaryCase, type DurableRunCanaryCreateRootRunInput, type DurableRunCanaryMessage, type DurableRunCanaryPreparedCase, type DurableRunCanaryResult, type DurableRunCanaryRunnerConfig, durableRunCanaryRunnerInternals, type DurableRunCanaryRunSummary, type DurableRunCanarySendUserMessageInput, type DurableRunCanaryStartRunInput, getDurableRunCanaryMessageSchema, parseDurableRunCanaryRunSummary, } from "./durable-run-canaries/index.js";
-export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, containsSkillLoad, countStepStartedEvents, createFailedEvalResult, createLiveEvalCaseSupport, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, hasFinished, type LiveEvalCase, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalContext, type LiveEvalProjectFile, type LiveEvalProjectFileReaderInput, type LiveEvalRequestBody, type LiveEvalResultForPerformance, type LiveEvalResultForReport, type LiveEvalResultRecord, type LiveEvalRunnerConfig, liveEvalRunnerInternals, type LiveEvalRuntime, type PreparedLiveEvalInput, resolveLiveEvalRequestedCaseIds, type RuntimePerformanceSummary, selectLiveEvalCases, } from "./live-evals/index.js";
+export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, cancelLiveEvalInputRequest, containsOrderedSubsequence, containsSkillLoad, countStepStartedEvents, createFailedEvalResult, createLiveEvalApiClient, createLiveEvalCaseSupport, createLiveEvalConversation, createLiveEvalProjectUploadFixture, createLiveEvalRelease, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, deleteLiveEvalConversation, deleteLiveEvalProjectFile, getLiveEvalProjectFile, hasEveryLiveEvalTag, hasFinished, listOpenLiveEvalInputRequests, type LiveEvalApiClient, type LiveEvalApiContext, type LiveEvalCase, type LiveEvalCaseMetadata, type LiveEvalCaseSelectionInput, type LiveEvalContext, type LiveEvalConversationInput, type LiveEvalCreateConversationInput, type LiveEvalCreateReleaseInput, type LiveEvalInputRequestInput, type LiveEvalInputRequestRecord, type LiveEvalInputResponseValues, type LiveEvalProjectFile, type LiveEvalProjectFileInput, type LiveEvalProjectFileReaderInput, type LiveEvalProjectUploadFixtureInput, type LiveEvalRequestBody, type LiveEvalRequestTimeoutInput, type LiveEvalResultForPerformance, type LiveEvalResultForReport, type LiveEvalResultRecord, type LiveEvalRunnerConfig, liveEvalRunnerInternals, type LiveEvalRuntime, type LiveEvalSubmitInputResponseInput, type LiveEvalWaitForOpenInputRequestInput, type PreparedLiveEvalInput, resolveLiveEvalRequestedCaseIds, type RuntimePerformanceSummary, selectLiveEvalCases, submitLiveEvalInputResponse, waitForOpenLiveEvalInputRequest, } from "./live-evals/index.js";
 //# sourceMappingURL=index.d.ts.map

package/esm/src/agent/testing/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/testing/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,4BAA4B,CAAC;AAGpC,OAAO,EACL,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,SAAS,GACf,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,EAC5B,KAAK,yBAAyB,EAC9B,KAAK,yBAAyB,EAC9B,KAAK,oBAAoB,EACzB,KAAK,kCAAkC,EACvC,KAAK,uBAAuB,EAC5B,KAAK,4BAA4B,EACjC,KAAK,sBAAsB,EAC3B,KAAK,4BAA4B,EACjC,+BAA+B,EAC/B,KAAK,0BAA0B,EAC/B,KAAK,oCAAoC,EACzC,KAAK,6BAA6B,EAClC,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,2BAA2B,EAC3B,0BAA0B,EAC1B,iBAAiB,EACjB,8BAA8B,EAC9B,0BAA0B,EAC1B,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,eAAe,EACpB,KAAK,qBAAqB,EAC1B,+BAA+B,EAC/B,KAAK,yBAAyB,EAC9B,mBAAmB,GACpB,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/testing/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,4BAA4B,CAAC;AAGpC,OAAO,EACL,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,SAAS,GACf,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,EAC5B,KAAK,yBAAyB,EAC9B,KAAK,yBAAyB,EAC9B,KAAK,oBAAoB,EACzB,KAAK,kCAAkC,EACvC,KAAK,uBAAuB,EAC5B,KAAK,4BAA4B,EACjC,KAAK,sBAAsB,EAC3B,KAAK,4BAA4B,EACjC,+BAA+B,EAC/B,KAAK,0BAA0B,EAC/B,KAAK,oCAAoC,EACzC,KAAK,6BAA6B,EAClC,gCAAgC,EAChC,+BAA+B,GAChC,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,2BAA2B,EAC3B,0BAA0B,EAC1B,iBAAiB,EACjB,8BAA8B,EAC9B,0BAA0B,EAC1B,0BAA0B,EAC1B,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,EACzB,0BAA0B,EAC1B,kCAAkC,EAClC,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAClB,uBAAuB,EACvB,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACtB,mBAAmB,EACnB,WAAW,EACX,6BAA6B,EAC7B,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,YAAY,EACjB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,eAAe,EACpB,KAAK,yBAAyB,EAC9B,KAAK,+BAA+B,EACpC,KAAK,0BAA0B,EAC/B,KAAK,yBAAyB,EAC9B,KAAK,0BAA0B,EAC/B,KAAK,2BAA2B,EAChC,KAAK,mBAAmB,EACxB,KAAK,wBAAwB,EAC7B,KAAK,8BAA8B,EACnC,KAAK,iCAAiC,EACtC,KAAK,mBAAmB,EACxB,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,eAAe,EACpB,KAAK,gCAAgC,EACrC,KAAK,oCAAoC,EACzC,KAAK,qBAAqB,EAC1B,+BAA+B,EAC/B,KAAK,yBAAyB,EAC9B,mBAAmB,EACnB,2BAA2B,EAC3B,+BAA+B,GAChC,MAAM,uBAAuB,CAAC"}

package/esm/src/agent/testing/index.js

@@ -6,4 +6,4 @@
 import "../../../_dnt.polyfills.js";
 export { assertCompleted, assertContains, assertToolCalled, printTestResults, testAgent, } from "./agent-tester.js";
 export { createDurableRunCanaryApiClient, createDurableRunCanaryRunner, durableRunCanaryRunnerInternals, getDurableRunCanaryMessageSchema, parseDurableRunCanaryRunSummary, } from "./durable-run-canaries/index.js";
-export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, containsOrderedSubsequence, containsSkillLoad, countStepStartedEvents, createFailedEvalResult, createLiveEvalCaseSupport, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, hasEveryLiveEvalTag, hasFinished, liveEvalRunnerInternals, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, } from "./live-evals/index.js";
+export { buildFailureSuffix, buildLiveEvalCaseTagSummary, buildLiveEvalRequestBody, buildLiveEvalRuntimeSummary, buildLiveEvalStatusSummary, buildProgressLine, buildRuntimePerformanceSummary, cancelLiveEvalInputRequest, containsOrderedSubsequence, containsSkillLoad, countStepStartedEvents, createFailedEvalResult, createLiveEvalApiClient, createLiveEvalCaseSupport, createLiveEvalConversation, createLiveEvalProjectUploadFixture, createLiveEvalRelease, createPassedEvalResult, createPlainTextPdf, createSkippedEvalResult, deleteLiveEvalConversation, deleteLiveEvalProjectFile, getLiveEvalProjectFile, hasEveryLiveEvalTag, hasFinished, listOpenLiveEvalInputRequests, liveEvalRunnerInternals, resolveLiveEvalRequestedCaseIds, selectLiveEvalCases, submitLiveEvalInputResponse, waitForOpenLiveEvalInputRequest, } from "./live-evals/index.js";

package/esm/src/agent/testing/live-evals/api-client.d.ts

@@ -0,0 +1,78 @@
+import type { InferSchema } from "../../../extensions/schema/index.js";
+import type { LiveEvalProjectFile } from "./runner.js";
+export interface LiveEvalApiContext {
+    apiUrl: string;
+    authToken: string;
+    projectId: string | null;
+    fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+}
+export interface LiveEvalRequestTimeoutInput {
+    requestTimeoutMs: number;
+}
+export interface LiveEvalCreateConversationInput extends LiveEvalRequestTimeoutInput {
+    title: string;
+}
+export interface LiveEvalConversationInput extends LiveEvalRequestTimeoutInput {
+    conversationId: string;
+}
+export interface LiveEvalProjectUploadFixtureInput extends LiveEvalRequestTimeoutInput {
+    filePath: string;
+    contentType: string;
+    body: BodyInit | Uint8Array;
+    size?: number;
+    pollIntervalMs?: number;
+    maxAttempts?: number;
+}
+export interface LiveEvalProjectFileInput extends LiveEvalRequestTimeoutInput {
+    filePath: string;
+}
+export interface LiveEvalCreateReleaseInput extends LiveEvalRequestTimeoutInput {
+    description?: string;
+}
+export interface LiveEvalWaitForOpenInputRequestInput extends LiveEvalConversationInput {
+    abortSignal: AbortSignal;
+    pollIntervalMs?: number;
+    timeoutMs?: number;
+}
+export interface LiveEvalInputResponseValues {
+    [key: string]: string | boolean | number | null;
+}
+export interface LiveEvalSubmitInputResponseInput extends LiveEvalRequestTimeoutInput {
+    conversationId: string;
+    inputRequestId: string;
+    values: LiveEvalInputResponseValues;
+}
+export interface LiveEvalInputRequestInput extends LiveEvalRequestTimeoutInput {
+    conversationId: string;
+    inputRequestId: string;
+}
+declare const getInputRequestRecordSchema: () => import("../../../internal-agents/schema.js").Schema<import("../../../extensions/schema/schema-validator.js").InferShape<{
+    id: import("../../../internal-agents/schema.js").Schema<string>;
+    status: import("../../../internal-agents/schema.js").Schema<string>;
+}>>;
+export type LiveEvalInputRequestRecord = InferSchema<ReturnType<typeof getInputRequestRecordSchema>>;
+export interface LiveEvalApiClient {
+    createConversation(input: LiveEvalCreateConversationInput): Promise<string>;
+    deleteConversation(input: LiveEvalConversationInput): Promise<void>;
+    createProjectUploadFixture(input: LiveEvalProjectUploadFixtureInput): Promise<string>;
+    getProjectFile(input: LiveEvalProjectFileInput): Promise<LiveEvalProjectFile | null>;
+    createRelease(input: LiveEvalCreateReleaseInput): Promise<string>;
+    deleteProjectFile(input: LiveEvalProjectFileInput): Promise<void>;
+    listOpenInputRequests(input: LiveEvalConversationInput): Promise<LiveEvalInputRequestRecord[]>;
+    waitForOpenInputRequest(input: LiveEvalWaitForOpenInputRequestInput): Promise<string>;
+    submitInputResponse(input: LiveEvalSubmitInputResponseInput): Promise<void>;
+    cancelInputRequest(input: LiveEvalInputRequestInput): Promise<void>;
+}
+export declare function createLiveEvalApiClient(context: LiveEvalApiContext): LiveEvalApiClient;
+export declare function createLiveEvalConversation(context: LiveEvalApiContext, input: LiveEvalCreateConversationInput): Promise<string>;
+export declare function deleteLiveEvalConversation(context: LiveEvalApiContext, input: LiveEvalConversationInput): Promise<void>;
+export declare function createLiveEvalProjectUploadFixture(context: LiveEvalApiContext, input: LiveEvalProjectUploadFixtureInput): Promise<string>;
+export declare function getLiveEvalProjectFile(context: LiveEvalApiContext, input: LiveEvalProjectFileInput): Promise<LiveEvalProjectFile | null>;
+export declare function createLiveEvalRelease(context: LiveEvalApiContext, input: LiveEvalCreateReleaseInput): Promise<string>;
+export declare function deleteLiveEvalProjectFile(context: LiveEvalApiContext, input: LiveEvalProjectFileInput): Promise<void>;
+export declare function listOpenLiveEvalInputRequests(context: LiveEvalApiContext, input: LiveEvalConversationInput): Promise<LiveEvalInputRequestRecord[]>;
+export declare function waitForOpenLiveEvalInputRequest(context: LiveEvalApiContext, input: LiveEvalWaitForOpenInputRequestInput): Promise<string>;
+export declare function submitLiveEvalInputResponse(context: LiveEvalApiContext, input: LiveEvalSubmitInputResponseInput): Promise<void>;
+export declare function cancelLiveEvalInputRequest(context: LiveEvalApiContext, input: LiveEvalInputRequestInput): Promise<void>;
+export {};
+//# sourceMappingURL=api-client.d.ts.map

package/esm/src/agent/testing/live-evals/api-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-client.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/api-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAEvE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAIvD,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CAClF;AAED,MAAM,WAAW,2BAA2B;IAC1C,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,+BAAgC,SAAQ,2BAA2B;IAClF,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,yBAA0B,SAAQ,2BAA2B;IAC5E,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iCAAkC,SAAQ,2BAA2B;IACpF,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,QAAQ,GAAG,UAAU,CAAC;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,wBAAyB,SAAQ,2BAA2B;IAC3E,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,0BAA2B,SAAQ,2BAA2B;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,oCAAqC,SAAQ,yBAAyB;IACrF,WAAW,EAAE,WAAW,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,2BAA2B;IAC1C,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,IAAI,CAAC;CACjD;AAED,MAAM,WAAW,gCAAiC,SAAQ,2BAA2B;IACnF,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,2BAA2B,CAAC;CACrC;AAED,MAAM,WAAW,yBAA0B,SAAQ,2BAA2B;IAC5E,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AA4BD,QAAA,MAAM,2BAA2B;;;GAKhC,CAAC;AAQF,MAAM,MAAM,0BAA0B,GAAG,WAAW,CAClD,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAC/C,CAAC;AAEF,MAAM,WAAW,iBAAiB;IAChC,kBAAkB,CAAC,KAAK,EAAE,+BAA+B,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,kBAAkB,CAAC,KAAK,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpE,0BAA0B,CAAC,KAAK,EAAE,iCAAiC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACtF,cAAc,CAAC,KAAK,EAAE,wBAAwB,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;IACrF,aAAa,CAAC,KAAK,EAAE,0BAA0B,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClE,iBAAiB,CAAC,KAAK,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,qBAAqB,CAAC,KAAK,EAAE,yBAAyB,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAAC;IAC/F,uBAAuB,CAAC,KAAK,EAAE,oCAAoC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACtF,mBAAmB,CAAC,KAAK,EAAE,gCAAgC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,kBAAkB,CAAC,KAAK,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrE;AAsID,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,kBAAkB,GAAG,iBAAiB,CAatF;AAED,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,+BAA+B,GACrC,OAAO,CAAC,MAAM,CAAC,CAuBjB;AAED,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,yBAAyB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAiBf;AAED,wBAAsB,kCAAkC,CACtD,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,MAAM,CAAC,CAuDjB;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CA4BrC;AAED,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,0BAA0B,GAChC,OAAO,CAAC,MAAM,CAAC,CA6BjB;AAED,wBAAsB,yBAAyB,CAC7C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAoBf;AAED,wBAAsB,6BAA6B,CACjD,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,yBAAyB,GAC/B,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAoBvC;AAED,wBAAsB,+BAA+B,CACnD,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,oCAAoC,GAC1C,OAAO,CAAC,MAAM,CAAC,CAsBjB;AAED,wBAAsB,2BAA2B,CAC/C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,gCAAgC,GACtC,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,yBAAyB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAkBf"}

package/esm/src/agent/testing/live-evals/api-client.js

@@ -0,0 +1,302 @@
+import * as dntShim from "../../../../_dnt.shims.js";
+import { ensureBuiltinSchemaValidator } from "../../../extensions/builtin-extensions.js";
+import { defineSchema } from "../../../schemas/index.js";
+ensureBuiltinSchemaValidator();
+const getLiveEvalIdResponseSchema = defineSchema((v) => v.object({
+    id: v.string().optional(),
+}));
+const getProjectUploadResponseSchema = defineSchema((v) => v.object({
+    file_upload_url: v.string().optional(),
+    required_headers: v.record(v.string(), v.string()).optional(),
+}));
+const getProjectUploadListResponseSchema = defineSchema((v) => v.object({
+    data: v.array(v.object({ path: v.string().optional() }).passthrough()).optional(),
+}));
+const getProjectFileResponseSchema = defineSchema((v) => v.object({
+    path: v.string().optional(),
+    content: v.string().optional(),
+}));
+const getInputRequestRecordSchema = defineSchema((v) => v.object({
+    id: v.string(),
+    status: v.string(),
+}));
+const getInputRequestListResponseSchema = defineSchema((v) => v.object({
+    data: v.array(v.unknown()).optional(),
+}));
+function createLiveEvalAuthHeaders(context) {
+    const headers = new Headers();
+    headers.set("Authorization", `Bearer ${context.authToken}`);
+    return headers;
+}
+function createLiveEvalJsonHeaders(context) {
+    const headers = createLiveEvalAuthHeaders(context);
+    headers.set("Content-Type", "application/json");
+    return headers;
+}
+function requireLiveEvalProjectId(projectId, errorMessage) {
+    if (!projectId) {
+        throw new Error(errorMessage);
+    }
+    return projectId;
+}
+function createFetch(context) {
+    return context.fetch ?? fetch;
+}
+function createApiUrl(context, path) {
+    const baseHref = context.apiUrl.endsWith("/") ? context.apiUrl : `${context.apiUrl}/`;
+    const relativePath = path.startsWith("/") ? path.slice(1) : path;
+    return new URL(relativePath, baseHref);
+}
+function createProjectUploadHeaders(requiredHeaders, contentType) {
+    const uploadHeaders = new Headers(requiredHeaders);
+    if (!uploadHeaders.has("Content-Type")) {
+        uploadHeaders.set("Content-Type", contentType);
+    }
+    return uploadHeaders;
+}
+function getProjectUploadBodySize(body, explicitSize) {
+    if (typeof explicitSize === "number") {
+        return explicitSize;
+    }
+    if (typeof body === "string") {
+        return new TextEncoder().encode(body).byteLength;
+    }
+    if (body instanceof Blob) {
+        return body.size;
+    }
+    if (body instanceof URLSearchParams) {
+        return new TextEncoder().encode(body.toString()).byteLength;
+    }
+    if (body instanceof ArrayBuffer) {
+        return body.byteLength;
+    }
+    if (ArrayBuffer.isView(body)) {
+        return body.byteLength;
+    }
+    throw new Error("Project upload fixtures require size when body length cannot be inferred");
+}
+function createProjectUploadBody(body, contentType) {
+    if (body instanceof Blob) {
+        return body;
+    }
+    if (typeof body === "string") {
+        return new Blob([body], { type: contentType });
+    }
+    if (body instanceof Uint8Array) {
+        return new Blob([body.slice()], { type: contentType });
+    }
+    return body;
+}
+function getResponseText(response) {
+    return response.text();
+}
+async function wait(input) {
+    await new Promise((resolve) => {
+        dntShim.setTimeout(resolve, input.ms);
+    });
+}
+async function waitForProjectUploadFixture(context, input) {
+    const listUrl = createApiUrl(context, `/projects/${input.projectId}/uploads`);
+    const requestFetch = createFetch(context);
+    const maxAttempts = input.maxAttempts ?? 12;
+    const pollIntervalMs = input.pollIntervalMs ?? 2_000;
+    for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+        const listResponse = await requestFetch(listUrl, {
+            headers: createLiveEvalAuthHeaders(context),
+            signal: AbortSignal.timeout(input.requestTimeoutMs),
+        });
+        if (!listResponse.ok) {
+            throw new Error(`Failed to confirm project upload fixture: ${listResponse.status} ${await getResponseText(listResponse)}`);
+        }
+        const payload = getProjectUploadListResponseSchema().parse(await listResponse.json());
+        if (payload.data?.some((upload) => upload.path === input.filePath)) {
+            return input.filePath;
+        }
+        if (attempt + 1 < maxAttempts) {
+            await wait({ ms: pollIntervalMs });
+        }
+    }
+    throw new Error(`Project upload fixture did not appear in time: ${input.filePath}`);
+}
+export function createLiveEvalApiClient(context) {
+    return {
+        createConversation: (input) => createLiveEvalConversation(context, input),
+        deleteConversation: (input) => deleteLiveEvalConversation(context, input),
+        createProjectUploadFixture: (input) => createLiveEvalProjectUploadFixture(context, input),
+        getProjectFile: (input) => getLiveEvalProjectFile(context, input),
+        createRelease: (input) => createLiveEvalRelease(context, input),
+        deleteProjectFile: (input) => deleteLiveEvalProjectFile(context, input),
+        listOpenInputRequests: (input) => listOpenLiveEvalInputRequests(context, input),
+        waitForOpenInputRequest: (input) => waitForOpenLiveEvalInputRequest(context, input),
+        submitInputResponse: (input) => submitLiveEvalInputResponse(context, input),
+        cancelInputRequest: (input) => cancelLiveEvalInputRequest(context, input),
+    };
+}
+export async function createLiveEvalConversation(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, "/conversations"), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({
+            ...(context.projectId ? { project_id: context.projectId } : {}),
+            title: input.title,
+        }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to create eval conversation: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getLiveEvalIdResponseSchema().parse(await response.json());
+    if (!payload.id) {
+        throw new Error("Conversation creation response did not include id");
+    }
+    return payload.id;
+}
+export async function deleteLiveEvalConversation(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}`), {
+        method: "DELETE",
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok && response.status !== 404) {
+        throw new Error(`Failed to delete eval conversation ${input.conversationId}: ${response.status} ${await getResponseText(response)}`);
+    }
+}
+export async function createLiveEvalProjectUploadFixture(context, input) {
+    const projectId = requireLiveEvalProjectId(context.projectId, "Project upload fixtures require a live-eval project id");
+    const createResponse = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/uploads`), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({
+            file_path: input.filePath,
+            content_type: input.contentType,
+            size: getProjectUploadBodySize(input.body, input.size),
+        }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!createResponse.ok) {
+        throw new Error(`Failed to create project upload URL: ${createResponse.status} ${await getResponseText(createResponse)}`);
+    }
+    const createPayload = getProjectUploadResponseSchema().parse(await createResponse.json());
+    if (!createPayload.file_upload_url) {
+        throw new Error("Project upload response did not include file_upload_url");
+    }
+    const uploadResponse = await createFetch(context)(createPayload.file_upload_url, {
+        method: "PUT",
+        headers: createProjectUploadHeaders(createPayload.required_headers, input.contentType),
+        body: createProjectUploadBody(input.body, input.contentType),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!uploadResponse.ok) {
+        throw new Error(`Failed to upload project fixture: ${uploadResponse.status} ${await getResponseText(uploadResponse)}`);
+    }
+    return waitForProjectUploadFixture(context, {
+        projectId,
+        filePath: input.filePath,
+        requestTimeoutMs: input.requestTimeoutMs,
+        pollIntervalMs: input.pollIntervalMs,
+        maxAttempts: input.maxAttempts,
+    });
+}
+export async function getLiveEvalProjectFile(context, input) {
+    const projectId = requireLiveEvalProjectId(context.projectId, "getLiveEvalProjectFile requires a live-eval project id");
+    const response = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/files/${encodeURIComponent(input.filePath)}`), {
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (response.status === 404) {
+        return null;
+    }
+    if (!response.ok) {
+        throw new Error(`Failed to read project file: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getProjectFileResponseSchema().parse(await response.json());
+    return {
+        path: payload.path ?? input.filePath,
+        content: payload.content ?? "",
+    };
+}
+export async function createLiveEvalRelease(context, input) {
+    const projectId = requireLiveEvalProjectId(context.projectId, "createLiveEvalRelease requires a live-eval project id");
+    const response = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/releases`), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({
+            description: input.description ?? "eval platform capability release",
+        }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to create release: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getLiveEvalIdResponseSchema().parse(await response.json());
+    if (!payload.id) {
+        throw new Error("Release creation response did not include id");
+    }
+    return payload.id;
+}
+export async function deleteLiveEvalProjectFile(context, input) {
+    const projectId = context.projectId;
+    if (!projectId) {
+        return;
+    }
+    const response = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/files/${encodeURIComponent(input.filePath)}`), {
+        method: "DELETE",
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok && response.status !== 404) {
+        throw new Error(`Failed to delete project file: ${response.status} ${await getResponseText(response)}`);
+    }
+}
+export async function listOpenLiveEvalInputRequests(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}/input-requests?status=open`), {
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to list eval input requests: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getInputRequestListResponseSchema().parse(await response.json());
+    return (payload.data ?? []).flatMap((item) => {
+        const parsed = getInputRequestRecordSchema().safeParse(item);
+        return parsed.success ? [parsed.data] : [];
+    });
+}
+export async function waitForOpenLiveEvalInputRequest(context, input) {
+    const timeoutMs = input.timeoutMs ?? 30_000;
+    const pollIntervalMs = input.pollIntervalMs ?? 500;
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (input.abortSignal.aborted) {
+            throw new Error("Eval sidecar aborted before an input request appeared");
+        }
+        const requests = await listOpenLiveEvalInputRequests(context, input);
+        const request = requests[0];
+        if (request) {
+            return request.id;
+        }
+        await wait({ ms: pollIntervalMs });
+    }
+    throw new Error(`Timed out while waiting for an open input request in conversation ${input.conversationId}`);
+}
+export async function submitLiveEvalInputResponse(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}/input-requests/${input.inputRequestId}/responses`), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({ values: input.values }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to submit eval input response: ${response.status} ${await getResponseText(response)}`);
+    }
+}
+export async function cancelLiveEvalInputRequest(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}/input-requests/${input.inputRequestId}/cancel`), {
+        method: "POST",
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to cancel eval input request: ${response.status} ${await getResponseText(response)}`);
+    }
+}

package/esm/src/agent/testing/live-evals/index.d.ts

@@ -1,3 +1,4 @@
+export { cancelLiveEvalInputRequest, createLiveEvalApiClient, createLiveEvalConversation, createLiveEvalProjectUploadFixture, createLiveEvalRelease, deleteLiveEvalConversation, deleteLiveEvalProjectFile, getLiveEvalProjectFile, listOpenLiveEvalInputRequests, type LiveEvalApiClient, type LiveEvalApiContext, type LiveEvalConversationInput, type LiveEvalCreateConversationInput, type LiveEvalCreateReleaseInput, type LiveEvalInputRequestInput, type LiveEvalInputRequestRecord, type LiveEvalInputResponseValues, type LiveEvalProjectFileInput, type LiveEvalProjectUploadFixtureInput, type LiveEvalRequestTimeoutInput, type LiveEvalSubmitInputResponseInput, type LiveEvalWaitForOpenInputRequestInput, submitLiveEvalInputResponse, waitForOpenLiveEvalInputRequest, } from "./api-client.js";
 export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
 export { buildRuntimePerformanceSummary, type LiveEvalResultForPerformance, type LiveEvalRuntime, type RuntimePerformanceSummary, } from "./performance.js";
 export { buildLiveEvalRequestBody, type BuildLiveEvalRequestBodyInput, type LiveEvalRequestBody, } from "./request.js";

package/esm/src/agent/testing/live-evals/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,8BAA8B,EAC9B,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,KAAK,mBAAmB,GACzB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,2BAA2B,EAC3B,2BAA2B,EAC3B,0BAA0B,EAC1B,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAC5B,+BAA+B,EAC/B,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,qBAAqB,GAC3B,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/agent/testing/live-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,0BAA0B,EAC1B,uBAAuB,EACvB,0BAA0B,EAC1B,kCAAkC,EAClC,qBAAqB,EACrB,0BAA0B,EAC1B,yBAAyB,EACzB,sBAAsB,EACtB,6BAA6B,EAC7B,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,+BAA+B,EACpC,KAAK,0BAA0B,EAC/B,KAAK,yBAAyB,EAC9B,KAAK,0BAA0B,EAC/B,KAAK,2BAA2B,EAChC,KAAK,wBAAwB,EAC7B,KAAK,iCAAiC,EACtC,KAAK,2BAA2B,EAChC,KAAK,gCAAgC,EACrC,KAAK,oCAAoC,EACzC,2BAA2B,EAC3B,+BAA+B,GAChC,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,GACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,8BAA8B,EAC9B,KAAK,4BAA4B,EACjC,KAAK,eAAe,EACpB,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,wBAAwB,EACxB,KAAK,6BAA6B,EAClC,KAAK,mBAAmB,GACzB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,2BAA2B,EAC3B,2BAA2B,EAC3B,0BAA0B,EAC1B,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,0BAA0B,EAC/B,KAAK,uBAAuB,EAC5B,+BAA+B,EAC/B,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACnC,KAAK,oBAAoB,EACzB,uBAAuB,EACvB,KAAK,qBAAqB,GAC3B,MAAM,aAAa,CAAC"}

package/esm/src/agent/testing/live-evals/index.js

@@ -1,3 +1,4 @@
+export { cancelLiveEvalInputRequest, createLiveEvalApiClient, createLiveEvalConversation, createLiveEvalProjectUploadFixture, createLiveEvalRelease, deleteLiveEvalConversation, deleteLiveEvalProjectFile, getLiveEvalProjectFile, listOpenLiveEvalInputRequests, submitLiveEvalInputResponse, waitForOpenLiveEvalInputRequest, } from "./api-client.js";
 export { buildFailureSuffix, buildProgressLine, containsOrderedSubsequence, createPlainTextPdf, } from "./formatting.js";
 export { buildRuntimePerformanceSummary, } from "./performance.js";
 export { buildLiveEvalRequestBody, } from "./request.js";

package/esm/src/utils/version-constant.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "0.1.529";
+export declare const VERSION = "0.1.531";
 //# sourceMappingURL=version-constant.d.ts.map

package/esm/src/utils/version-constant.js

@@ -1,3 +1,3 @@
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
-export const VERSION = "0.1.529";
+export const VERSION = "0.1.531";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "veryfront",
-  "version": "0.1.529",
+  "version": "0.1.531",
   "description": "The simplest way to build AI-powered apps",
   "keywords": [
     "react",

package/src/deno.js

@@ -1,6 +1,6 @@
 export default {
   "name": "veryfront",
-  "version": "0.1.529",
+  "version": "0.1.531",
   "license": "Apache-2.0",
   "nodeModulesDir": "auto",
   "workspace": [

package/src/src/agent/testing/index.ts

@@ -45,31 +45,55 @@ export {
   buildLiveEvalStatusSummary,
   buildProgressLine,
   buildRuntimePerformanceSummary,
+  cancelLiveEvalInputRequest,
   containsOrderedSubsequence,
   containsSkillLoad,
   countStepStartedEvents,
   createFailedEvalResult,
+  createLiveEvalApiClient,
   createLiveEvalCaseSupport,
+  createLiveEvalConversation,
+  createLiveEvalProjectUploadFixture,
+  createLiveEvalRelease,
   createPassedEvalResult,
   createPlainTextPdf,
   createSkippedEvalResult,
+  deleteLiveEvalConversation,
+  deleteLiveEvalProjectFile,
+  getLiveEvalProjectFile,
   hasEveryLiveEvalTag,
   hasFinished,
+  listOpenLiveEvalInputRequests,
+  type LiveEvalApiClient,
+  type LiveEvalApiContext,
   type LiveEvalCase,
   type LiveEvalCaseMetadata,
   type LiveEvalCaseSelectionInput,
   type LiveEvalContext,
+  type LiveEvalConversationInput,
+  type LiveEvalCreateConversationInput,
+  type LiveEvalCreateReleaseInput,
+  type LiveEvalInputRequestInput,
+  type LiveEvalInputRequestRecord,
+  type LiveEvalInputResponseValues,
   type LiveEvalProjectFile,
+  type LiveEvalProjectFileInput,
   type LiveEvalProjectFileReaderInput,
+  type LiveEvalProjectUploadFixtureInput,
   type LiveEvalRequestBody,
+  type LiveEvalRequestTimeoutInput,
   type LiveEvalResultForPerformance,
   type LiveEvalResultForReport,
   type LiveEvalResultRecord,
   type LiveEvalRunnerConfig,
   liveEvalRunnerInternals,
   type LiveEvalRuntime,
+  type LiveEvalSubmitInputResponseInput,
+  type LiveEvalWaitForOpenInputRequestInput,
   type PreparedLiveEvalInput,
   resolveLiveEvalRequestedCaseIds,
   type RuntimePerformanceSummary,
   selectLiveEvalCases,
+  submitLiveEvalInputResponse,
+  waitForOpenLiveEvalInputRequest,
 } from "./live-evals/index.js";

package/src/src/agent/testing/live-evals/api-client.ts

@@ -0,0 +1,568 @@
+import * as dntShim from "../../../../_dnt.shims.js";
+import { ensureBuiltinSchemaValidator } from "../../../extensions/builtin-extensions.js";
+import type { InferSchema } from "../../../extensions/schema/index.js";
+import { defineSchema } from "../../../schemas/index.js";
+import type { LiveEvalProjectFile } from "./runner.js";
+
+ensureBuiltinSchemaValidator();
+
+export interface LiveEvalApiContext {
+  apiUrl: string;
+  authToken: string;
+  projectId: string | null;
+  fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+}
+
+export interface LiveEvalRequestTimeoutInput {
+  requestTimeoutMs: number;
+}
+
+export interface LiveEvalCreateConversationInput extends LiveEvalRequestTimeoutInput {
+  title: string;
+}
+
+export interface LiveEvalConversationInput extends LiveEvalRequestTimeoutInput {
+  conversationId: string;
+}
+
+export interface LiveEvalProjectUploadFixtureInput extends LiveEvalRequestTimeoutInput {
+  filePath: string;
+  contentType: string;
+  body: BodyInit | Uint8Array;
+  size?: number;
+  pollIntervalMs?: number;
+  maxAttempts?: number;
+}
+
+export interface LiveEvalProjectFileInput extends LiveEvalRequestTimeoutInput {
+  filePath: string;
+}
+
+export interface LiveEvalCreateReleaseInput extends LiveEvalRequestTimeoutInput {
+  description?: string;
+}
+
+export interface LiveEvalWaitForOpenInputRequestInput extends LiveEvalConversationInput {
+  abortSignal: AbortSignal;
+  pollIntervalMs?: number;
+  timeoutMs?: number;
+}
+
+export interface LiveEvalInputResponseValues {
+  [key: string]: string | boolean | number | null;
+}
+
+export interface LiveEvalSubmitInputResponseInput extends LiveEvalRequestTimeoutInput {
+  conversationId: string;
+  inputRequestId: string;
+  values: LiveEvalInputResponseValues;
+}
+
+export interface LiveEvalInputRequestInput extends LiveEvalRequestTimeoutInput {
+  conversationId: string;
+  inputRequestId: string;
+}
+
+const getLiveEvalIdResponseSchema = defineSchema((v) =>
+  v.object({
+    id: v.string().optional(),
+  })
+);
+
+const getProjectUploadResponseSchema = defineSchema((v) =>
+  v.object({
+    file_upload_url: v.string().optional(),
+    required_headers: v.record(v.string(), v.string()).optional(),
+  })
+);
+
+const getProjectUploadListResponseSchema = defineSchema((v) =>
+  v.object({
+    data: v.array(v.object({ path: v.string().optional() }).passthrough()).optional(),
+  })
+);
+
+const getProjectFileResponseSchema = defineSchema((v) =>
+  v.object({
+    path: v.string().optional(),
+    content: v.string().optional(),
+  })
+);
+
+const getInputRequestRecordSchema = defineSchema((v) =>
+  v.object({
+    id: v.string(),
+    status: v.string(),
+  })
+);
+
+const getInputRequestListResponseSchema = defineSchema((v) =>
+  v.object({
+    data: v.array(v.unknown()).optional(),
+  })
+);
+
+export type LiveEvalInputRequestRecord = InferSchema<
+  ReturnType<typeof getInputRequestRecordSchema>
+>;
+
+export interface LiveEvalApiClient {
+  createConversation(input: LiveEvalCreateConversationInput): Promise<string>;
+  deleteConversation(input: LiveEvalConversationInput): Promise<void>;
+  createProjectUploadFixture(input: LiveEvalProjectUploadFixtureInput): Promise<string>;
+  getProjectFile(input: LiveEvalProjectFileInput): Promise<LiveEvalProjectFile | null>;
+  createRelease(input: LiveEvalCreateReleaseInput): Promise<string>;
+  deleteProjectFile(input: LiveEvalProjectFileInput): Promise<void>;
+  listOpenInputRequests(input: LiveEvalConversationInput): Promise<LiveEvalInputRequestRecord[]>;
+  waitForOpenInputRequest(input: LiveEvalWaitForOpenInputRequestInput): Promise<string>;
+  submitInputResponse(input: LiveEvalSubmitInputResponseInput): Promise<void>;
+  cancelInputRequest(input: LiveEvalInputRequestInput): Promise<void>;
+}
+
+function createLiveEvalAuthHeaders(context: LiveEvalApiContext): Headers {
+  const headers = new Headers();
+  headers.set("Authorization", `Bearer ${context.authToken}`);
+  return headers;
+}
+
+function createLiveEvalJsonHeaders(context: LiveEvalApiContext): Headers {
+  const headers = createLiveEvalAuthHeaders(context);
+  headers.set("Content-Type", "application/json");
+  return headers;
+}
+
+function requireLiveEvalProjectId(projectId: string | null, errorMessage: string): string {
+  if (!projectId) {
+    throw new Error(errorMessage);
+  }
+
+  return projectId;
+}
+
+function createFetch(context: LiveEvalApiContext) {
+  return context.fetch ?? fetch;
+}
+
+function createApiUrl(context: LiveEvalApiContext, path: string): URL {
+  const baseHref = context.apiUrl.endsWith("/") ? context.apiUrl : `${context.apiUrl}/`;
+  const relativePath = path.startsWith("/") ? path.slice(1) : path;
+  return new URL(relativePath, baseHref);
+}
+
+function createProjectUploadHeaders(
+  requiredHeaders: Record<string, string> | undefined,
+  contentType: string,
+): Headers {
+  const uploadHeaders = new Headers(requiredHeaders);
+  if (!uploadHeaders.has("Content-Type")) {
+    uploadHeaders.set("Content-Type", contentType);
+  }
+
+  return uploadHeaders;
+}
+
+function getProjectUploadBodySize(
+  body: BodyInit | Uint8Array,
+  explicitSize: number | undefined,
+): number {
+  if (typeof explicitSize === "number") {
+    return explicitSize;
+  }
+  if (typeof body === "string") {
+    return new TextEncoder().encode(body).byteLength;
+  }
+  if (body instanceof Blob) {
+    return body.size;
+  }
+  if (body instanceof URLSearchParams) {
+    return new TextEncoder().encode(body.toString()).byteLength;
+  }
+  if (body instanceof ArrayBuffer) {
+    return body.byteLength;
+  }
+  if (ArrayBuffer.isView(body)) {
+    return body.byteLength;
+  }
+  throw new Error("Project upload fixtures require size when body length cannot be inferred");
+}
+
+function createProjectUploadBody(body: BodyInit | Uint8Array, contentType: string): BodyInit {
+  if (body instanceof Blob) {
+    return body;
+  }
+  if (typeof body === "string") {
+    return new Blob([body], { type: contentType });
+  }
+  if (body instanceof Uint8Array) {
+    return new Blob([body.slice()], { type: contentType });
+  }
+  return body;
+}
+
+function getResponseText(response: Response): Promise<string> {
+  return response.text();
+}
+
+async function wait(input: { ms: number }): Promise<void> {
+  await new Promise((resolve) => {
+    dntShim.setTimeout(resolve, input.ms);
+  });
+}
+
+async function waitForProjectUploadFixture(
+  context: LiveEvalApiContext,
+  input: {
+    projectId: string;
+    filePath: string;
+    requestTimeoutMs: number;
+    pollIntervalMs?: number;
+    maxAttempts?: number;
+  },
+): Promise<string> {
+  const listUrl = createApiUrl(context, `/projects/${input.projectId}/uploads`);
+  const requestFetch = createFetch(context);
+  const maxAttempts = input.maxAttempts ?? 12;
+  const pollIntervalMs = input.pollIntervalMs ?? 2_000;
+
+  for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+    const listResponse = await requestFetch(listUrl, {
+      headers: createLiveEvalAuthHeaders(context),
+      signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+
+    if (!listResponse.ok) {
+      throw new Error(
+        `Failed to confirm project upload fixture: ${listResponse.status} ${await getResponseText(
+          listResponse,
+        )}`,
+      );
+    }
+
+    const payload = getProjectUploadListResponseSchema().parse(await listResponse.json());
+    if (payload.data?.some((upload) => upload.path === input.filePath)) {
+      return input.filePath;
+    }
+
+    if (attempt + 1 < maxAttempts) {
+      await wait({ ms: pollIntervalMs });
+    }
+  }
+
+  throw new Error(`Project upload fixture did not appear in time: ${input.filePath}`);
+}
+
+export function createLiveEvalApiClient(context: LiveEvalApiContext): LiveEvalApiClient {
+  return {
+    createConversation: (input) => createLiveEvalConversation(context, input),
+    deleteConversation: (input) => deleteLiveEvalConversation(context, input),
+    createProjectUploadFixture: (input) => createLiveEvalProjectUploadFixture(context, input),
+    getProjectFile: (input) => getLiveEvalProjectFile(context, input),
+    createRelease: (input) => createLiveEvalRelease(context, input),
+    deleteProjectFile: (input) => deleteLiveEvalProjectFile(context, input),
+    listOpenInputRequests: (input) => listOpenLiveEvalInputRequests(context, input),
+    waitForOpenInputRequest: (input) => waitForOpenLiveEvalInputRequest(context, input),
+    submitInputResponse: (input) => submitLiveEvalInputResponse(context, input),
+    cancelInputRequest: (input) => cancelLiveEvalInputRequest(context, input),
+  } satisfies LiveEvalApiClient;
+}
+
+export async function createLiveEvalConversation(
+  context: LiveEvalApiContext,
+  input: LiveEvalCreateConversationInput,
+): Promise<string> {
+  const response = await createFetch(context)(createApiUrl(context, "/conversations"), {
+    method: "POST",
+    headers: createLiveEvalJsonHeaders(context),
+    body: JSON.stringify({
+      ...(context.projectId ? { project_id: context.projectId } : {}),
+      title: input.title,
+    }),
+    signal: AbortSignal.timeout(input.requestTimeoutMs),
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to create eval conversation: ${response.status} ${await getResponseText(response)}`,
+    );
+  }
+
+  const payload = getLiveEvalIdResponseSchema().parse(await response.json());
+  if (!payload.id) {
+    throw new Error("Conversation creation response did not include id");
+  }
+
+  return payload.id;
+}
+
+export async function deleteLiveEvalConversation(
+  context: LiveEvalApiContext,
+  input: LiveEvalConversationInput,
+): Promise<void> {
+  const response = await createFetch(context)(
+    createApiUrl(context, `/conversations/${input.conversationId}`),
+    {
+      method: "DELETE",
+      headers: createLiveEvalAuthHeaders(context),
+      signal: AbortSignal.timeout(input.requestTimeoutMs),
+    },
+  );
+
+  if (!response.ok && response.status !== 404) {
+    throw new Error(
+      `Failed to delete eval conversation ${input.conversationId}: ${response.status} ${await getResponseText(
+        response,
+      )}`,
+    );
+  }
+}
+
+export async function createLiveEvalProjectUploadFixture(
+  context: LiveEvalApiContext,
+  input: LiveEvalProjectUploadFixtureInput,
+): Promise<string> {
+  const projectId = requireLiveEvalProjectId(
+    context.projectId,
+    "Project upload fixtures require a live-eval project id",
+  );
+
+  const createResponse = await createFetch(context)(
+    createApiUrl(context, `/projects/${projectId}/uploads`),
+    {
+      method: "POST",
+      headers: createLiveEvalJsonHeaders(context),
+      body: JSON.stringify({
+        file_path: input.filePath,
+        content_type: input.contentType,
+        size: getProjectUploadBodySize(input.body, input.size),
+      }),
+      signal: AbortSignal.timeout(input.requestTimeoutMs),
+    },
+  );
+
+  if (!createResponse.ok) {
+    throw new Error(
+      `Failed to create project upload URL: ${createResponse.status} ${await getResponseText(
+        createResponse,
+      )}`,
+    );
+  }
+
+  const createPayload = getProjectUploadResponseSchema().parse(await createResponse.json());
+  if (!createPayload.file_upload_url) {
+    throw new Error("Project upload response did not include file_upload_url");
+  }
+
+  const uploadResponse = await createFetch(context)(createPayload.file_upload_url, {
+    method: "PUT",
+    headers: createProjectUploadHeaders(createPayload.required_headers, input.contentType),
+    body: createProjectUploadBody(input.body, input.contentType),
+    signal: AbortSignal.timeout(input.requestTimeoutMs),
+  });
+
+  if (!uploadResponse.ok) {
+    throw new Error(
+      `Failed to upload project fixture: ${uploadResponse.status} ${await getResponseText(
+        uploadResponse,
+      )}`,
+    );
+  }
+
+  return waitForProjectUploadFixture(context, {
+    projectId,
+    filePath: input.filePath,
+    requestTimeoutMs: input.requestTimeoutMs,
+    pollIntervalMs: input.pollIntervalMs,
+    maxAttempts: input.maxAttempts,
+  });
+}
+
+export async function getLiveEvalProjectFile(
+  context: LiveEvalApiContext,
+  input: LiveEvalProjectFileInput,
+): Promise<LiveEvalProjectFile | null> {
+  const projectId = requireLiveEvalProjectId(
+    context.projectId,
+    "getLiveEvalProjectFile requires a live-eval project id",
+  );
+  const response = await createFetch(context)(
+    createApiUrl(context, `/projects/${projectId}/files/${encodeURIComponent(input.filePath)}`),
+    {
+      headers: createLiveEvalAuthHeaders(context),
+      signal: AbortSignal.timeout(input.requestTimeoutMs),
+    },
+  );
+
+  if (response.status === 404) {
+    return null;
+  }
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to read project file: ${response.status} ${await getResponseText(response)}`,
+    );
+  }
+
+  const payload = getProjectFileResponseSchema().parse(await response.json());
+  return {
+    path: payload.path ?? input.filePath,
+    content: payload.content ?? "",
+  };
+}
+
+export async function createLiveEvalRelease(
+  context: LiveEvalApiContext,
+  input: LiveEvalCreateReleaseInput,
+): Promise<string> {
+  const projectId = requireLiveEvalProjectId(
+    context.projectId,
+    "createLiveEvalRelease requires a live-eval project id",
+  );
+  const response = await createFetch(context)(
+    createApiUrl(context, `/projects/${projectId}/releases`),
+    {
+      method: "POST",
+      headers: createLiveEvalJsonHeaders(context),
+      body: JSON.stringify({
+        description: input.description ?? "eval platform capability release",
+      }),
+      signal: AbortSignal.timeout(input.requestTimeoutMs),
+    },
+  );
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to create release: ${response.status} ${await getResponseText(response)}`,
+    );
+  }
+
+  const payload = getLiveEvalIdResponseSchema().parse(await response.json());
+  if (!payload.id) {
+    throw new Error("Release creation response did not include id");
+  }
+
+  return payload.id;
+}
+
+export async function deleteLiveEvalProjectFile(
+  context: LiveEvalApiContext,
+  input: LiveEvalProjectFileInput,
+): Promise<void> {
+  const projectId = context.projectId;
+  if (!projectId) {
+    return;
+  }
+
+  const response = await createFetch(context)(
+    createApiUrl(context, `/projects/${projectId}/files/${encodeURIComponent(input.filePath)}`),
+    {
+      method: "DELETE",
+      headers: createLiveEvalAuthHeaders(context),
+      signal: AbortSignal.timeout(input.requestTimeoutMs),
+    },
+  );
+
+  if (!response.ok && response.status !== 404) {
+    throw new Error(
+      `Failed to delete project file: ${response.status} ${await getResponseText(response)}`,
+    );
+  }
+}
+
+export async function listOpenLiveEvalInputRequests(
+  context: LiveEvalApiContext,
+  input: LiveEvalConversationInput,
+): Promise<LiveEvalInputRequestRecord[]> {
+  const response = await createFetch(context)(
+    createApiUrl(context, `/conversations/${input.conversationId}/input-requests?status=open`),
+    {
+      headers: createLiveEvalAuthHeaders(context),
+      signal: AbortSignal.timeout(input.requestTimeoutMs),
+    },
+  );
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to list eval input requests: ${response.status} ${await getResponseText(response)}`,
+    );
+  }
+
+  const payload = getInputRequestListResponseSchema().parse(await response.json());
+  return (payload.data ?? []).flatMap((item) => {
+    const parsed = getInputRequestRecordSchema().safeParse(item);
+    return parsed.success ? [parsed.data] : [];
+  });
+}
+
+export async function waitForOpenLiveEvalInputRequest(
+  context: LiveEvalApiContext,
+  input: LiveEvalWaitForOpenInputRequestInput,
+): Promise<string> {
+  const timeoutMs = input.timeoutMs ?? 30_000;
+  const pollIntervalMs = input.pollIntervalMs ?? 500;
+  const deadline = Date.now() + timeoutMs;
+
+  while (Date.now() < deadline) {
+    if (input.abortSignal.aborted) {
+      throw new Error("Eval sidecar aborted before an input request appeared");
+    }
+
+    const requests = await listOpenLiveEvalInputRequests(context, input);
+    const request = requests[0];
+    if (request) {
+      return request.id;
+    }
+
+    await wait({ ms: pollIntervalMs });
+  }
+
+  throw new Error(
+    `Timed out while waiting for an open input request in conversation ${input.conversationId}`,
+  );
+}
+
+export async function submitLiveEvalInputResponse(
+  context: LiveEvalApiContext,
+  input: LiveEvalSubmitInputResponseInput,
+): Promise<void> {
+  const response = await createFetch(context)(
+    createApiUrl(
+      context,
+      `/conversations/${input.conversationId}/input-requests/${input.inputRequestId}/responses`,
+    ),
+    {
+      method: "POST",
+      headers: createLiveEvalJsonHeaders(context),
+      body: JSON.stringify({ values: input.values }),
+      signal: AbortSignal.timeout(input.requestTimeoutMs),
+    },
+  );
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to submit eval input response: ${response.status} ${await getResponseText(response)}`,
+    );
+  }
+}
+
+export async function cancelLiveEvalInputRequest(
+  context: LiveEvalApiContext,
+  input: LiveEvalInputRequestInput,
+): Promise<void> {
+  const response = await createFetch(context)(
+    createApiUrl(
+      context,
+      `/conversations/${input.conversationId}/input-requests/${input.inputRequestId}/cancel`,
+    ),
+    {
+      method: "POST",
+      headers: createLiveEvalAuthHeaders(context),
+      signal: AbortSignal.timeout(input.requestTimeoutMs),
+    },
+  );
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to cancel eval input request: ${response.status} ${await getResponseText(response)}`,
+    );
+  }
+}

package/src/src/agent/testing/live-evals/index.ts

@@ -1,3 +1,29 @@
+export {
+  cancelLiveEvalInputRequest,
+  createLiveEvalApiClient,
+  createLiveEvalConversation,
+  createLiveEvalProjectUploadFixture,
+  createLiveEvalRelease,
+  deleteLiveEvalConversation,
+  deleteLiveEvalProjectFile,
+  getLiveEvalProjectFile,
+  listOpenLiveEvalInputRequests,
+  type LiveEvalApiClient,
+  type LiveEvalApiContext,
+  type LiveEvalConversationInput,
+  type LiveEvalCreateConversationInput,
+  type LiveEvalCreateReleaseInput,
+  type LiveEvalInputRequestInput,
+  type LiveEvalInputRequestRecord,
+  type LiveEvalInputResponseValues,
+  type LiveEvalProjectFileInput,
+  type LiveEvalProjectUploadFixtureInput,
+  type LiveEvalRequestTimeoutInput,
+  type LiveEvalSubmitInputResponseInput,
+  type LiveEvalWaitForOpenInputRequestInput,
+  submitLiveEvalInputResponse,
+  waitForOpenLiveEvalInputRequest,
+} from "./api-client.js";
 export {
   buildFailureSuffix,
   buildProgressLine,

package/src/src/utils/version-constant.ts

@@ -1,3 +1,3 @@
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
-export const VERSION = "0.1.529";
+export const VERSION = "0.1.531";