veryfront 0.1.877 → 0.1.878

package/esm/src/eval/agent-service/durable-run-canaries/validation.d.ts

@@ -0,0 +1,12 @@
+import type { DurableRunCanaryMessage, DurableRunCanaryRunSummary } from "./runner.js";
+/** Assert that a durable run canary completed successfully. */
+export declare function assertCompleted(run: DurableRunCanaryRunSummary): void;
+/** Message shape for find assistant. */
+export declare function findAssistantMessage(messages: DurableRunCanaryMessage[], messageId: string): DurableRunCanaryMessage;
+/** Stringify unknown helper. */
+export declare function stringifyUnknown(value: unknown): string;
+/** Collect assistant text helper. */
+export declare function collectAssistantText(message: DurableRunCanaryMessage): string;
+/** Assert no malformed create file tool calls helper. */
+export declare function assertNoMalformedCreateFileToolCalls(messages: DurableRunCanaryMessage[]): void;
+//# sourceMappingURL=validation.d.ts.map

package/esm/src/eval/agent-service/durable-run-canaries/validation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/durable-run-canaries/validation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AA8BvF,+DAA+D;AAC/D,wBAAgB,eAAe,CAAC,GAAG,EAAE,0BAA0B,GAAG,IAAI,CAQrE;AAED,wCAAwC;AACxC,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,uBAAuB,EAAE,EACnC,SAAS,EAAE,MAAM,GAChB,uBAAuB,CASzB;AAED,gCAAgC;AAChC,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAUvD;AAED,qCAAqC;AACrC,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,uBAAuB,GAAG,MAAM,CAK7E;AAED,yDAAyD;AACzD,wBAAgB,oCAAoC,CAAC,QAAQ,EAAE,uBAAuB,EAAE,GAAG,IAAI,CAY9F"}

package/esm/src/eval/agent-service/durable-run-canaries/validation.js

@@ -0,0 +1,71 @@
+function isRecord(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function getStringProperty(record, key) {
+    const value = record[key];
+    return typeof value === "string" ? value : null;
+}
+function getToolCallName(part) {
+    if (part.type === "tool_call") {
+        return getStringProperty(part, "name");
+    }
+    if (part.type === "tool-call") {
+        return getStringProperty(part, "toolName");
+    }
+    return null;
+}
+function hasCreateFileInput(value) {
+    if (!isRecord(value)) {
+        return false;
+    }
+    return getStringProperty(value, "path") !== null && getStringProperty(value, "content") !== null;
+}
+/** Assert that a durable run canary completed successfully. */
+export function assertCompleted(run) {
+    if (run.status !== "completed") {
+        throw new Error(`Expected completed run, got ${run.status} (${run.terminalErrorCode ?? "no-code"}: ${run.terminalErrorMessage ?? "no message"})`);
+    }
+}
+/** Message shape for find assistant. */
+export function findAssistantMessage(messages, messageId) {
+    const message = messages.find((candidate) => candidate.id === messageId);
+    if (!message) {
+        throw new Error(`Assistant message ${messageId} was not persisted`);
+    }
+    if (message.role !== "assistant") {
+        throw new Error(`Expected assistant message ${messageId}, got role ${message.role}`);
+    }
+    return message;
+}
+/** Stringify unknown helper. */
+export function stringifyUnknown(value) {
+    if (typeof value === "string") {
+        return value;
+    }
+    try {
+        return JSON.stringify(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+/** Collect assistant text helper. */
+export function collectAssistantText(message) {
+    return message.parts
+        .filter((part) => part.type === "text" && typeof part.text === "string")
+        .map((part) => part.text)
+        .join("\n");
+}
+/** Assert no malformed create file tool calls helper. */
+export function assertNoMalformedCreateFileToolCalls(messages) {
+    for (const message of messages) {
+        for (const part of message.parts) {
+            if (!isRecord(part) || getToolCallName(part) !== "create_file") {
+                continue;
+            }
+            if (!hasCreateFileInput(part.input)) {
+                throw new Error("Expected create_file tool_call input to include a path and content");
+            }
+        }
+    }
+}

package/esm/src/eval/agent-service/live-evals/api-client.d.ts

@@ -0,0 +1,102 @@
+import type { InferSchema } from "../../../extensions/schema/index.js";
+import type { LiveEvalProjectFile } from "./runner.js";
+/** Context for live eval API. */
+export interface LiveEvalApiContext {
+    apiUrl: string;
+    authToken: string;
+    projectId: string | null;
+    fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+}
+/** Input payload for live eval request timeout. */
+export interface LiveEvalRequestTimeoutInput {
+    requestTimeoutMs: number;
+}
+/** Input payload for live eval create conversation. */
+export interface LiveEvalCreateConversationInput extends LiveEvalRequestTimeoutInput {
+    title: string;
+}
+/** Input payload for live eval conversation. */
+export interface LiveEvalConversationInput extends LiveEvalRequestTimeoutInput {
+    conversationId: string;
+}
+/** Input payload for live eval project upload fixture. */
+export interface LiveEvalProjectUploadFixtureInput extends LiveEvalRequestTimeoutInput {
+    filePath: string;
+    contentType: string;
+    body: BodyInit | Uint8Array;
+    size?: number;
+    pollIntervalMs?: number;
+    maxAttempts?: number;
+}
+/** Input payload for live eval project file. */
+export interface LiveEvalProjectFileInput extends LiveEvalRequestTimeoutInput {
+    filePath: string;
+}
+/** Input payload for live eval create release. */
+export interface LiveEvalCreateReleaseInput extends LiveEvalRequestTimeoutInput {
+    description?: string;
+}
+/** Input payload for live eval wait for open input request. */
+export interface LiveEvalWaitForOpenInputRequestInput extends LiveEvalConversationInput {
+    abortSignal: AbortSignal;
+    pollIntervalMs?: number;
+    timeoutMs?: number;
+}
+/** Public API contract for live eval input response values. */
+export interface LiveEvalInputResponseValues {
+    [key: string]: string | boolean | number | null;
+}
+/** Input payload for live eval submit input response. */
+export interface LiveEvalSubmitInputResponseInput extends LiveEvalRequestTimeoutInput {
+    conversationId: string;
+    inputRequestId: string;
+    values: LiveEvalInputResponseValues;
+}
+/** Input payload for live eval input request. */
+export interface LiveEvalInputRequestInput extends LiveEvalRequestTimeoutInput {
+    conversationId: string;
+    inputRequestId: string;
+}
+declare const getInputRequestRecordSchema: () => import("../../../internal-agents/schema.js").Schema<import("../../../extensions/schema/schema-validator.js").InferShape<{
+    id: import("../../../internal-agents/schema.js").Schema<string>;
+    status: import("../../../internal-agents/schema.js").Schema<string>;
+}>>;
+/** Record shape for live eval input request. */
+export type LiveEvalInputRequestRecord = InferSchema<ReturnType<typeof getInputRequestRecordSchema>>;
+/** Public API contract for live eval API client. */
+export interface LiveEvalApiClient {
+    createConversation(input: LiveEvalCreateConversationInput): Promise<string>;
+    deleteConversation(input: LiveEvalConversationInput): Promise<void>;
+    createProjectUploadFixture(input: LiveEvalProjectUploadFixtureInput): Promise<string>;
+    getProjectFile(input: LiveEvalProjectFileInput): Promise<LiveEvalProjectFile | null>;
+    createRelease(input: LiveEvalCreateReleaseInput): Promise<string>;
+    deleteProjectFile(input: LiveEvalProjectFileInput): Promise<void>;
+    listOpenInputRequests(input: LiveEvalConversationInput): Promise<LiveEvalInputRequestRecord[]>;
+    waitForOpenInputRequest(input: LiveEvalWaitForOpenInputRequestInput): Promise<string>;
+    submitInputResponse(input: LiveEvalSubmitInputResponseInput): Promise<void>;
+    cancelInputRequest(input: LiveEvalInputRequestInput): Promise<void>;
+}
+/** Create live eval API client. */
+export declare function createLiveEvalApiClient(context: LiveEvalApiContext): LiveEvalApiClient;
+/** Create live eval conversation. */
+export declare function createLiveEvalConversation(context: LiveEvalApiContext, input: LiveEvalCreateConversationInput): Promise<string>;
+/** Delete live eval conversation helper. */
+export declare function deleteLiveEvalConversation(context: LiveEvalApiContext, input: LiveEvalConversationInput): Promise<void>;
+/** Create live eval project upload fixture. */
+export declare function createLiveEvalProjectUploadFixture(context: LiveEvalApiContext, input: LiveEvalProjectUploadFixtureInput): Promise<string>;
+/** Return live eval project file. */
+export declare function getLiveEvalProjectFile(context: LiveEvalApiContext, input: LiveEvalProjectFileInput): Promise<LiveEvalProjectFile | null>;
+/** Create live eval release. */
+export declare function createLiveEvalRelease(context: LiveEvalApiContext, input: LiveEvalCreateReleaseInput): Promise<string>;
+/** Delete live eval project file helper. */
+export declare function deleteLiveEvalProjectFile(context: LiveEvalApiContext, input: LiveEvalProjectFileInput): Promise<void>;
+/** List open live eval input requests. */
+export declare function listOpenLiveEvalInputRequests(context: LiveEvalApiContext, input: LiveEvalConversationInput): Promise<LiveEvalInputRequestRecord[]>;
+/** Request payload for wait for open live eval input. */
+export declare function waitForOpenLiveEvalInputRequest(context: LiveEvalApiContext, input: LiveEvalWaitForOpenInputRequestInput): Promise<string>;
+/** Response payload for submit live eval input. */
+export declare function submitLiveEvalInputResponse(context: LiveEvalApiContext, input: LiveEvalSubmitInputResponseInput): Promise<void>;
+/** Request payload for cancel live eval input. */
+export declare function cancelLiveEvalInputRequest(context: LiveEvalApiContext, input: LiveEvalInputRequestInput): Promise<void>;
+export {};
+//# sourceMappingURL=api-client.d.ts.map

package/esm/src/eval/agent-service/live-evals/api-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-client.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/api-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AAEvE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAIvD,iCAAiC;AACjC,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;CAClF;AAED,mDAAmD;AACnD,MAAM,WAAW,2BAA2B;IAC1C,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,uDAAuD;AACvD,MAAM,WAAW,+BAAgC,SAAQ,2BAA2B;IAClF,KAAK,EAAE,MAAM,CAAC;CACf;AAED,gDAAgD;AAChD,MAAM,WAAW,yBAA0B,SAAQ,2BAA2B;IAC5E,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,0DAA0D;AAC1D,MAAM,WAAW,iCAAkC,SAAQ,2BAA2B;IACpF,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,QAAQ,GAAG,UAAU,CAAC;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,gDAAgD;AAChD,MAAM,WAAW,wBAAyB,SAAQ,2BAA2B;IAC3E,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,kDAAkD;AAClD,MAAM,WAAW,0BAA2B,SAAQ,2BAA2B;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+DAA+D;AAC/D,MAAM,WAAW,oCAAqC,SAAQ,yBAAyB;IACrF,WAAW,EAAE,WAAW,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,+DAA+D;AAC/D,MAAM,WAAW,2BAA2B;IAC1C,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,IAAI,CAAC;CACjD;AAED,yDAAyD;AACzD,MAAM,WAAW,gCAAiC,SAAQ,2BAA2B;IACnF,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,2BAA2B,CAAC;CACrC;AAED,iDAAiD;AACjD,MAAM,WAAW,yBAA0B,SAAQ,2BAA2B;IAC5E,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;CACxB;AA4BD,QAAA,MAAM,2BAA2B;;;GAKhC,CAAC;AAQF,gDAAgD;AAChD,MAAM,MAAM,0BAA0B,GAAG,WAAW,CAClD,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAC/C,CAAC;AAEF,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC,kBAAkB,CAAC,KAAK,EAAE,+BAA+B,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,kBAAkB,CAAC,KAAK,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpE,0BAA0B,CAAC,KAAK,EAAE,iCAAiC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACtF,cAAc,CAAC,KAAK,EAAE,wBAAwB,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;IACrF,aAAa,CAAC,KAAK,EAAE,0BAA0B,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClE,iBAAiB,CAAC,KAAK,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,qBAAqB,CAAC,KAAK,EAAE,yBAAyB,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAAC;IAC/F,uBAAuB,CAAC,KAAK,EAAE,oCAAoC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACtF,mBAAmB,CAAC,KAAK,EAAE,gCAAgC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,kBAAkB,CAAC,KAAK,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrE;AAsID,mCAAmC;AACnC,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,kBAAkB,GAAG,iBAAiB,CAatF;AAED,qCAAqC;AACrC,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,+BAA+B,GACrC,OAAO,CAAC,MAAM,CAAC,CAuBjB;AAED,4CAA4C;AAC5C,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,yBAAyB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAiBf;AAED,+CAA+C;AAC/C,wBAAsB,kCAAkC,CACtD,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,MAAM,CAAC,CAuDjB;AAED,qCAAqC;AACrC,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CA4BrC;AAED,gCAAgC;AAChC,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,0BAA0B,GAChC,OAAO,CAAC,MAAM,CAAC,CA6BjB;AAED,4CAA4C;AAC5C,wBAAsB,yBAAyB,CAC7C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAoBf;AAED,0CAA0C;AAC1C,wBAAsB,6BAA6B,CACjD,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,yBAAyB,GAC/B,OAAO,CAAC,0BAA0B,EAAE,CAAC,CAoBvC;AAED,yDAAyD;AACzD,wBAAsB,+BAA+B,CACnD,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,oCAAoC,GAC1C,OAAO,CAAC,MAAM,CAAC,CAsBjB;AAED,mDAAmD;AACnD,wBAAsB,2BAA2B,CAC/C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,gCAAgC,GACtC,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED,kDAAkD;AAClD,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,kBAAkB,EAC3B,KAAK,EAAE,yBAAyB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAkBf"}

package/esm/src/eval/agent-service/live-evals/api-client.js

@@ -0,0 +1,313 @@
+import * as dntShim from "../../../../_dnt.shims.js";
+import { ensureBuiltinSchemaValidator } from "../../../extensions/builtin-extensions.js";
+import { defineSchema } from "../../../schemas/index.js";
+ensureBuiltinSchemaValidator();
+const getLiveEvalIdResponseSchema = defineSchema((v) => v.object({
+    id: v.string().optional(),
+}));
+const getProjectUploadResponseSchema = defineSchema((v) => v.object({
+    file_upload_url: v.string().optional(),
+    required_headers: v.record(v.string(), v.string()).optional(),
+}));
+const getProjectUploadListResponseSchema = defineSchema((v) => v.object({
+    data: v.array(v.object({ path: v.string().optional() }).passthrough()).optional(),
+}));
+const getProjectFileResponseSchema = defineSchema((v) => v.object({
+    path: v.string().optional(),
+    content: v.string().optional(),
+}));
+const getInputRequestRecordSchema = defineSchema((v) => v.object({
+    id: v.string(),
+    status: v.string(),
+}));
+const getInputRequestListResponseSchema = defineSchema((v) => v.object({
+    data: v.array(v.unknown()).optional(),
+}));
+function createLiveEvalAuthHeaders(context) {
+    const headers = new Headers();
+    headers.set("Authorization", `Bearer ${context.authToken}`);
+    return headers;
+}
+function createLiveEvalJsonHeaders(context) {
+    const headers = createLiveEvalAuthHeaders(context);
+    headers.set("Content-Type", "application/json");
+    return headers;
+}
+function requireLiveEvalProjectId(projectId, errorMessage) {
+    if (!projectId) {
+        throw new Error(errorMessage);
+    }
+    return projectId;
+}
+function createFetch(context) {
+    return context.fetch ?? fetch;
+}
+function createApiUrl(context, path) {
+    const baseHref = context.apiUrl.endsWith("/") ? context.apiUrl : `${context.apiUrl}/`;
+    const relativePath = path.startsWith("/") ? path.slice(1) : path;
+    return new URL(relativePath, baseHref);
+}
+function createProjectUploadHeaders(requiredHeaders, contentType) {
+    const uploadHeaders = new Headers(requiredHeaders);
+    if (!uploadHeaders.has("Content-Type")) {
+        uploadHeaders.set("Content-Type", contentType);
+    }
+    return uploadHeaders;
+}
+function getProjectUploadBodySize(body, explicitSize) {
+    if (typeof explicitSize === "number") {
+        return explicitSize;
+    }
+    if (typeof body === "string") {
+        return new TextEncoder().encode(body).byteLength;
+    }
+    if (body instanceof Blob) {
+        return body.size;
+    }
+    if (body instanceof URLSearchParams) {
+        return new TextEncoder().encode(body.toString()).byteLength;
+    }
+    if (body instanceof ArrayBuffer) {
+        return body.byteLength;
+    }
+    if (ArrayBuffer.isView(body)) {
+        return body.byteLength;
+    }
+    throw new Error("Project upload fixtures require size when body length cannot be inferred");
+}
+function createProjectUploadBody(body, contentType) {
+    if (body instanceof Blob) {
+        return body;
+    }
+    if (typeof body === "string") {
+        return new Blob([body], { type: contentType });
+    }
+    if (body instanceof Uint8Array) {
+        return new Blob([body.slice()], { type: contentType });
+    }
+    return body;
+}
+function getResponseText(response) {
+    return response.text();
+}
+async function wait(input) {
+    await new Promise((resolve) => {
+        dntShim.setTimeout(resolve, input.ms);
+    });
+}
+async function waitForProjectUploadFixture(context, input) {
+    const listUrl = createApiUrl(context, `/projects/${input.projectId}/uploads`);
+    const requestFetch = createFetch(context);
+    const maxAttempts = input.maxAttempts ?? 12;
+    const pollIntervalMs = input.pollIntervalMs ?? 2_000;
+    for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+        const listResponse = await requestFetch(listUrl, {
+            headers: createLiveEvalAuthHeaders(context),
+            signal: AbortSignal.timeout(input.requestTimeoutMs),
+        });
+        if (!listResponse.ok) {
+            throw new Error(`Failed to confirm project upload fixture: ${listResponse.status} ${await getResponseText(listResponse)}`);
+        }
+        const payload = getProjectUploadListResponseSchema().parse(await listResponse.json());
+        if (payload.data?.some((upload) => upload.path === input.filePath)) {
+            return input.filePath;
+        }
+        if (attempt + 1 < maxAttempts) {
+            await wait({ ms: pollIntervalMs });
+        }
+    }
+    throw new Error(`Project upload fixture did not appear in time: ${input.filePath}`);
+}
+/** Create live eval API client. */
+export function createLiveEvalApiClient(context) {
+    return {
+        createConversation: (input) => createLiveEvalConversation(context, input),
+        deleteConversation: (input) => deleteLiveEvalConversation(context, input),
+        createProjectUploadFixture: (input) => createLiveEvalProjectUploadFixture(context, input),
+        getProjectFile: (input) => getLiveEvalProjectFile(context, input),
+        createRelease: (input) => createLiveEvalRelease(context, input),
+        deleteProjectFile: (input) => deleteLiveEvalProjectFile(context, input),
+        listOpenInputRequests: (input) => listOpenLiveEvalInputRequests(context, input),
+        waitForOpenInputRequest: (input) => waitForOpenLiveEvalInputRequest(context, input),
+        submitInputResponse: (input) => submitLiveEvalInputResponse(context, input),
+        cancelInputRequest: (input) => cancelLiveEvalInputRequest(context, input),
+    };
+}
+/** Create live eval conversation. */
+export async function createLiveEvalConversation(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, "/conversations"), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({
+            ...(context.projectId ? { project_id: context.projectId } : {}),
+            title: input.title,
+        }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to create eval conversation: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getLiveEvalIdResponseSchema().parse(await response.json());
+    if (!payload.id) {
+        throw new Error("Conversation creation response did not include id");
+    }
+    return payload.id;
+}
+/** Delete live eval conversation helper. */
+export async function deleteLiveEvalConversation(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}`), {
+        method: "DELETE",
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok && response.status !== 404) {
+        throw new Error(`Failed to delete eval conversation ${input.conversationId}: ${response.status} ${await getResponseText(response)}`);
+    }
+}
+/** Create live eval project upload fixture. */
+export async function createLiveEvalProjectUploadFixture(context, input) {
+    const projectId = requireLiveEvalProjectId(context.projectId, "Project upload fixtures require a live-eval project id");
+    const createResponse = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/uploads`), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({
+            file_path: input.filePath,
+            content_type: input.contentType,
+            size: getProjectUploadBodySize(input.body, input.size),
+        }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!createResponse.ok) {
+        throw new Error(`Failed to create project upload URL: ${createResponse.status} ${await getResponseText(createResponse)}`);
+    }
+    const createPayload = getProjectUploadResponseSchema().parse(await createResponse.json());
+    if (!createPayload.file_upload_url) {
+        throw new Error("Project upload response did not include file_upload_url");
+    }
+    const uploadResponse = await createFetch(context)(createPayload.file_upload_url, {
+        method: "PUT",
+        headers: createProjectUploadHeaders(createPayload.required_headers, input.contentType),
+        body: createProjectUploadBody(input.body, input.contentType),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!uploadResponse.ok) {
+        throw new Error(`Failed to upload project fixture: ${uploadResponse.status} ${await getResponseText(uploadResponse)}`);
+    }
+    return waitForProjectUploadFixture(context, {
+        projectId,
+        filePath: input.filePath,
+        requestTimeoutMs: input.requestTimeoutMs,
+        pollIntervalMs: input.pollIntervalMs,
+        maxAttempts: input.maxAttempts,
+    });
+}
+/** Return live eval project file. */
+export async function getLiveEvalProjectFile(context, input) {
+    const projectId = requireLiveEvalProjectId(context.projectId, "getLiveEvalProjectFile requires a live-eval project id");
+    const response = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/files/${encodeURIComponent(input.filePath)}`), {
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (response.status === 404) {
+        return null;
+    }
+    if (!response.ok) {
+        throw new Error(`Failed to read project file: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getProjectFileResponseSchema().parse(await response.json());
+    return {
+        path: payload.path ?? input.filePath,
+        content: payload.content ?? "",
+    };
+}
+/** Create live eval release. */
+export async function createLiveEvalRelease(context, input) {
+    const projectId = requireLiveEvalProjectId(context.projectId, "createLiveEvalRelease requires a live-eval project id");
+    const response = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/releases`), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({
+            description: input.description ?? "eval platform capability release",
+        }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to create release: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getLiveEvalIdResponseSchema().parse(await response.json());
+    if (!payload.id) {
+        throw new Error("Release creation response did not include id");
+    }
+    return payload.id;
+}
+/** Delete live eval project file helper. */
+export async function deleteLiveEvalProjectFile(context, input) {
+    const projectId = context.projectId;
+    if (!projectId) {
+        return;
+    }
+    const response = await createFetch(context)(createApiUrl(context, `/projects/${projectId}/files/${encodeURIComponent(input.filePath)}`), {
+        method: "DELETE",
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok && response.status !== 404) {
+        throw new Error(`Failed to delete project file: ${response.status} ${await getResponseText(response)}`);
+    }
+}
+/** List open live eval input requests. */
+export async function listOpenLiveEvalInputRequests(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}/input-requests?status=open`), {
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to list eval input requests: ${response.status} ${await getResponseText(response)}`);
+    }
+    const payload = getInputRequestListResponseSchema().parse(await response.json());
+    return (payload.data ?? []).flatMap((item) => {
+        const parsed = getInputRequestRecordSchema().safeParse(item);
+        return parsed.success ? [parsed.data] : [];
+    });
+}
+/** Request payload for wait for open live eval input. */
+export async function waitForOpenLiveEvalInputRequest(context, input) {
+    const timeoutMs = input.timeoutMs ?? 30_000;
+    const pollIntervalMs = input.pollIntervalMs ?? 500;
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (input.abortSignal.aborted) {
+            throw new Error("Eval sidecar aborted before an input request appeared");
+        }
+        const requests = await listOpenLiveEvalInputRequests(context, input);
+        const request = requests[0];
+        if (request) {
+            return request.id;
+        }
+        await wait({ ms: pollIntervalMs });
+    }
+    throw new Error(`Timed out while waiting for an open input request in conversation ${input.conversationId}`);
+}
+/** Response payload for submit live eval input. */
+export async function submitLiveEvalInputResponse(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}/input-requests/${input.inputRequestId}/responses`), {
+        method: "POST",
+        headers: createLiveEvalJsonHeaders(context),
+        body: JSON.stringify({ values: input.values }),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to submit eval input response: ${response.status} ${await getResponseText(response)}`);
+    }
+}
+/** Request payload for cancel live eval input. */
+export async function cancelLiveEvalInputRequest(context, input) {
+    const response = await createFetch(context)(createApiUrl(context, `/conversations/${input.conversationId}/input-requests/${input.inputRequestId}/cancel`), {
+        method: "POST",
+        headers: createLiveEvalAuthHeaders(context),
+        signal: AbortSignal.timeout(input.requestTimeoutMs),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to cancel eval input request: ${response.status} ${await getResponseText(response)}`);
+    }
+}

package/esm/src/eval/agent-service/live-evals/cli-runner.d.ts

@@ -0,0 +1,40 @@
+import { type LiveEvalRuntime } from "./performance.js";
+import { containsSkillLoad, countStepStartedEvents, createLiveEvalCaseSupport, hasFinished, type LiveEvalCase, type LiveEvalRunnerConfig } from "./runner.js";
+type EnvRecord = Record<string, string | undefined>;
+/** Public API contract for live eval cli case groups. */
+export interface LiveEvalCliCaseGroups {
+    readOnlyCases: LiveEvalCase[];
+    writeCases: LiveEvalCase[];
+    experimentalWriteCases: LiveEvalCase[];
+}
+/** Input payload for live eval cli case factory. */
+export interface LiveEvalCliCaseFactoryInput {
+    authToken: string;
+    endpoint: string;
+    projectId: string | null;
+    branchId: string | null;
+    model: string | null;
+    requestTimeoutMs: number;
+    enableLlmJudge: boolean;
+    hasFinished: typeof hasFinished;
+    containsSkillLoad: typeof containsSkillLoad;
+    countStepStartedEvents: typeof countStepStartedEvents;
+    verifyFileExists: ReturnType<typeof createLiveEvalCaseSupport>["verifyFileExists"];
+    withJudge: ReturnType<typeof createLiveEvalCaseSupport>["withJudge"];
+    judgeLlm: ReturnType<typeof createLiveEvalCaseSupport>["judgeLlm"];
+}
+/** Input payload for run live eval cli. */
+export interface RunLiveEvalCliInput {
+    env: EnvRecord;
+    caseSets: Record<string, readonly string[]>;
+    createCases: (input: LiveEvalCliCaseFactoryInput) => LiveEvalCliCaseGroups;
+    runtimes?: readonly LiveEvalRuntime[];
+    cwd?: string;
+    log?: (message: string) => void;
+    error?: (message: string) => void;
+    createCaseSupport?: (config: LiveEvalRunnerConfig) => ReturnType<typeof createLiveEvalCaseSupport>;
+}
+/** Run live eval cli. */
+export declare function runLiveEvalCli(input: RunLiveEvalCliInput): Promise<number>;
+export {};
+//# sourceMappingURL=cli-runner.d.ts.map

package/esm/src/eval/agent-service/live-evals/cli-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-runner.d.ts","sourceRoot":"","sources":["../../../../../src/src/eval/agent-service/live-evals/cli-runner.ts"],"names":[],"mappings":"AAGA,OAAO,EAAkC,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAQxF,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,yBAAyB,EACzB,WAAW,EACX,KAAK,YAAY,EACjB,KAAK,oBAAoB,EAC1B,MAAM,aAAa,CAAC;AAKrB,KAAK,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;AAEpD,yDAAyD;AACzD,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,sBAAsB,EAAE,YAAY,EAAE,CAAC;CACxC;AAED,oDAAoD;AACpD,MAAM,WAAW,2BAA2B;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,OAAO,CAAC;IACxB,WAAW,EAAE,OAAO,WAAW,CAAC;IAChC,iBAAiB,EAAE,OAAO,iBAAiB,CAAC;IAC5C,sBAAsB,EAAE,OAAO,sBAAsB,CAAC;IACtD,gBAAgB,EAAE,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC,kBAAkB,CAAC,CAAC;IACnF,SAAS,EAAE,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC,WAAW,CAAC,CAAC;IACrE,QAAQ,EAAE,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC,UAAU,CAAC,CAAC;CACpE;AAED,2CAA2C;AAC3C,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,SAAS,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAC5C,WAAW,EAAE,CAAC,KAAK,EAAE,2BAA2B,KAAK,qBAAqB,CAAC;IAC3E,QAAQ,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IACtC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,iBAAiB,CAAC,EAAE,CAClB,MAAM,EAAE,oBAAoB,KACzB,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC;CACnD;AAuBD,yBAAyB;AACzB,wBAAsB,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAwJhF"}