veryfront 0.1.438 → 0.1.440

package/src/src/agent/hosted-durable-chat-run-start.ts

@@ -0,0 +1,246 @@
+import { parseProviderError } from "../chat/provider-errors.js";
+import {
+  AgUiDetachedStartAcceptedSchema,
+  buildDetachedAgUiStartRequest,
+  executeAgUiDetachedStart,
+} from "./ag-ui-detached-start.js";
+import type { AgUiResumeValue } from "./ag-ui-tool-shared.js";
+import type { DetachedRunTracker } from "./detached-run-tracker.js";
+import type { ParsedHostedChatRequest } from "./hosted-chat-request-parser.js";
+
+export type HostedDurableRunSetupErrorStatusCode = 400 | 402 | 413 | 429 | 500 | 503;
+
+export type HostedDurableRunAccepted = {
+  accepted: boolean;
+  duplicate: boolean;
+};
+
+export type HostedDurableRunAuthErrorResponse = {
+  errorCode: string;
+  statusCode: number;
+  metadata?: Record<string, unknown>;
+};
+
+export type HostedDurableRunLogger = {
+  error(message: string, metadata?: Record<string, unknown>): void;
+};
+
+export type HostedDurableRunStartExecutionInput<TExecution> = {
+  execution: TExecution;
+  abortSignal: AbortSignal;
+};
+
+export type HostedDurableRunStartCleanupInput<TExecution> = {
+  execution: TExecution;
+  runId: string;
+  conversationId: string;
+};
+
+export type ExecuteHostedDurableChatRunInput<TExecution> = {
+  req: ParsedHostedChatRequest;
+  rawRequest: Request;
+  requestOrCtx?: unknown;
+  tracker: DetachedRunTracker<AgUiResumeValue>;
+  prepareExecution: (req: ParsedHostedChatRequest) => Promise<TExecution>;
+  startDetachedExecution: (
+    input: HostedDurableRunStartExecutionInput<TExecution>,
+  ) => Promise<void>;
+  cleanupExecution?: (input: HostedDurableRunStartCleanupInput<TExecution>) => Promise<void>;
+  resolveAuthError?: (error: unknown) => HostedDurableRunAuthErrorResponse | null | undefined;
+  logger?: HostedDurableRunLogger;
+};
+
+function readBooleanProperty(input: object | null, propertyName: string): boolean {
+  if (!input) {
+    return false;
+  }
+
+  return Object.getOwnPropertyDescriptor(input, propertyName)?.value === true;
+}
+
+function isDurableRunSetupErrorStatusCode(
+  status: number | undefined,
+): status is HostedDurableRunSetupErrorStatusCode {
+  return status === 400 || status === 402 || status === 413 || status === 429 ||
+    status === 500 || status === 503;
+}
+
+function fallbackDurableRunSetupErrorStatusCode(
+  code: string,
+): HostedDurableRunSetupErrorStatusCode {
+  if (code === "OVERLOADED_ERROR") return 503;
+  if (code === "CONTEXT_LENGTH_EXCEEDED") return 413;
+
+  return 500;
+}
+
+export function resolveHostedDurableRunSetupErrorResponse(input: {
+  code: string;
+  status?: number;
+  originalError: unknown;
+}): {
+  errorCode: string;
+  statusCode: HostedDurableRunSetupErrorStatusCode;
+} {
+  if (
+    input.originalError instanceof Error &&
+    input.originalError.message === "DURABLE_CHAT_ROOT_REQUIRES_CONVERSATION"
+  ) {
+    return {
+      errorCode: "DURABLE_CHAT_ROOT_REQUIRES_CONVERSATION",
+      statusCode: 400,
+    };
+  }
+
+  return {
+    errorCode: input.code,
+    statusCode: isDurableRunSetupErrorStatusCode(input.status)
+      ? input.status
+      : fallbackDurableRunSetupErrorStatusCode(input.code),
+  };
+}
+
+async function parseAcceptedDetachedStartResponse(
+  response: Response,
+): Promise<HostedDurableRunAccepted> {
+  if (response.status !== 202) {
+    return { accepted: false, duplicate: false };
+  }
+
+  const payload = await response.json().catch((): null => null);
+  const parsed = AgUiDetachedStartAcceptedSchema.safeParse(payload);
+  if (parsed.success) {
+    return {
+      accepted: parsed.data.accepted,
+      duplicate: parsed.data.duplicate,
+    };
+  }
+
+  const payloadObject = typeof payload === "object" ? payload : null;
+  return {
+    accepted: readBooleanProperty(payloadObject, "accepted"),
+    duplicate: readBooleanProperty(payloadObject, "duplicate"),
+  };
+}
+
+async function executeHostedDurableChatRunStart<TExecution>(
+  input: ExecuteHostedDurableChatRunInput<TExecution>,
+): Promise<Response | HostedDurableRunAccepted> {
+  const { durableRootRun, conversationId } = input.req;
+  if (!durableRootRun || !conversationId) {
+    throw new Error("DURABLE_CHAT_ROOT_REQUIRES_CONVERSATION");
+  }
+
+  const execution = await input.prepareExecution(input.req);
+  const detachedStartRequest = buildDetachedAgUiStartRequest({
+    runId: durableRootRun.runId,
+    threadId: conversationId,
+    messages: input.req.messages,
+    model: input.req.model,
+    forwardedProps: input.req.forwardedProps,
+  });
+  const detachedStartResponse = await executeAgUiDetachedStart(
+    {
+      sessionManager: input.tracker.sessionManager,
+      startDetachedExecution: async ({ abortSignal }) => {
+        const detachedExecution = input.startDetachedExecution({
+          execution,
+          abortSignal,
+        });
+        input.tracker.registerExecution(durableRootRun.runId, detachedExecution);
+        await detachedExecution;
+      },
+      onDuplicate: async () => {
+        await input.cleanupExecution?.({
+          execution,
+          runId: durableRootRun.runId,
+          conversationId,
+        });
+      },
+      onAccepted: async () => {
+        input.tracker.trackRun(durableRootRun.runId);
+      },
+      onError: async ({ error }) => {
+        input.tracker.untrackRun(durableRootRun.runId);
+        input.logger?.error("Detached durable run execution failed", {
+          runId: durableRootRun.runId,
+          conversationId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      },
+    },
+    {
+      request: detachedStartRequest,
+      rawRequest: input.rawRequest,
+      requestOrCtx: input.requestOrCtx ?? input.rawRequest,
+    },
+  );
+
+  if (detachedStartResponse.status !== 202) {
+    return detachedStartResponse;
+  }
+
+  return await parseAcceptedDetachedStartResponse(detachedStartResponse);
+}
+
+export async function executeHostedDurableChatRun<TExecution>(
+  input: ExecuteHostedDurableChatRunInput<TExecution>,
+): Promise<Response> {
+  const { durableRootRun, conversationId, projectId, userId } = input.req;
+  if (!durableRootRun || !conversationId) {
+    return Response.json(
+      { errorCode: "DURABLE_CHAT_ROOT_REQUIRES_CONVERSATION" },
+      { status: 400 },
+    );
+  }
+
+  const existingRunStatus = input.tracker.sessionManager.getRunStatus(durableRootRun.runId);
+  if (existingRunStatus === "running" || existingRunStatus === "waiting") {
+    return Response.json({ accepted: true, duplicate: true }, { status: 202 });
+  }
+
+  try {
+    const startResult = await executeHostedDurableChatRunStart(input);
+
+    if (startResult instanceof Response) {
+      return startResult;
+    }
+
+    return Response.json(startResult, { status: 202 });
+  } catch (error) {
+    const authError = input.resolveAuthError?.(error);
+    if (authError) {
+      input.logger?.error("Durable chat auth error from API", {
+        errorCode: authError.errorCode,
+        statusCode: authError.statusCode,
+        projectId,
+        userId,
+        runId: durableRootRun.runId,
+        ...authError.metadata,
+      });
+      return Response.json(
+        { errorCode: authError.errorCode },
+        { status: authError.statusCode },
+      );
+    }
+
+    const { code, status } = parseProviderError(error);
+    const response = resolveHostedDurableRunSetupErrorResponse({
+      code,
+      status,
+      originalError: error,
+    });
+    input.logger?.error("Durable chat execute failed during setup", {
+      errorCode: code,
+      originalError: error instanceof Error ? error.message : String(error),
+      projectId,
+      userId,
+      runId: durableRootRun.runId,
+    });
+
+    return Response.json(
+      { errorCode: response.errorCode },
+      { status: response.statusCode },
+    );
+  }
+}

package/src/src/agent/index.ts

@@ -282,6 +282,17 @@ export type {
   HostedChatRuntimeToUiMessageStreamOptions,
 } from "./hosted-chat-runtime-contract.js";
 
+export {
+  executeHostedDurableChatRun,
+  type ExecuteHostedDurableChatRunInput,
+  type HostedDurableRunAccepted,
+  type HostedDurableRunAuthErrorResponse,
+  type HostedDurableRunLogger,
+  type HostedDurableRunSetupErrorStatusCode,
+  type HostedDurableRunStartCleanupInput,
+  type HostedDurableRunStartExecutionInput,
+  resolveHostedDurableRunSetupErrorResponse,
+} from "./hosted-durable-chat-run-start.js";
 export {
   buildParsedHostedChatRequest,
   type HostedChatProjectAccessError,
@@ -1082,6 +1093,7 @@ export {
   executeAgUiDetachedStart,
   type ExecuteAgUiDetachedStartInput,
 } from "./ag-ui-detached-start.js";
+export type { AgUiResumeValue } from "./ag-ui-tool-shared.js";
 export {
   createDetachedRunTracker,
   type DetachedRunDrainResult,

package/src/src/agent/runtime/text-generation-runtime-message-converter.ts

@@ -9,6 +9,7 @@
 
 import type {
   TextGenerationRuntimeAssistantMessage,
+  TextGenerationRuntimeFilePart,
   TextGenerationRuntimeMessage,
   TextGenerationRuntimeTextPart,
   TextGenerationRuntimeToolCallPart,
@@ -75,6 +76,26 @@ function getUserTextWithAttachmentContext(parts: Message["parts"]): string {
     : appendReadableAttachmentContext(text, buildAttachmentContextFromParts(parts));
 }
 
+function getUserFileParts(parts: Message["parts"]): TextGenerationRuntimeFilePart[] {
+  return parts.flatMap((part) => {
+    const type = getStringPartField(part, "type");
+    if (type !== "file" && type !== "image") return [];
+
+    const mediaType = getStringPartField(part, "mediaType");
+    const url = getStringPartField(part, "url");
+    if (!mediaType || !url || url.startsWith("data:")) return [];
+
+    return [{
+      type,
+      mediaType,
+      url,
+      ...(getStringPartField(part, "filename")
+        ? { filename: getStringPartField(part, "filename") }
+        : {}),
+    }];
+  });
+}
+
 /**
  * Convert a veryfront Message to the current text-generation runtime message format.
  */
@@ -86,8 +107,26 @@ export function convertToTextGenerationRuntimeMessage(msg: Message): TextGenerat
     }
 
     case "user": {
-      const text = getUserTextWithAttachmentContext(msg.parts);
-      return { role: "user", content: text };
+      const fileParts = getUserFileParts(msg.parts);
+      if (fileParts.length === 0) {
+        const text = getUserTextWithAttachmentContext(msg.parts);
+        return { role: "user", content: text };
+      }
+
+      const text = getTextFromParts(msg.parts);
+      const attachmentContext = text.includes("<uploaded_files>")
+        ? ""
+        : buildAttachmentContextFromParts(msg.parts);
+      return {
+        role: "user",
+        content: [
+          ...(text.length > 0 ? [{ type: "text" as const, text }] : []),
+          ...fileParts,
+          ...(attachmentContext.length > 0
+            ? [{ type: "text" as const, text: attachmentContext.trimStart() }]
+            : []),
+        ],
+      };
     }
 
     case "assistant": {

package/src/src/agent/runtime/text-generation-runtime-message-types.ts

@@ -11,6 +11,13 @@ export interface TextGenerationRuntimeTextPart {
   text: string;
 }
 
+export interface TextGenerationRuntimeFilePart {
+  type: "file" | "image";
+  mediaType: string;
+  url: string;
+  filename?: string;
+}
+
 export interface TextGenerationRuntimeToolCallPart {
   type: "tool-call";
   toolCallId: string;
@@ -35,7 +42,7 @@ export interface TextGenerationRuntimeSystemMessage {
 
 export interface TextGenerationRuntimeUserMessage {
   role: "user";
-  content: string;
+  content: string | Array<TextGenerationRuntimeTextPart | TextGenerationRuntimeFilePart>;
 }
 
 export interface TextGenerationRuntimeAssistantMessage {

package/src/src/agent/runtime-message-file-url-refresh.ts

@@ -1,4 +1,4 @@
-import type { ChatFileUiPart, ChatUiMessage, FileUIPartWithUpload } from "../chat/types.js";
+import type { ChatUiMessage, FileUIPartWithUpload } from "../chat/types.js";
 
 export type RuntimeFileUrlResolverInput = {
   uploadId: string;
@@ -18,25 +18,30 @@ export async function resolveRuntimeMessageFileUrls(
 
   return Promise.all(
     messages.map(async (message) => {
-      if (!message.parts.some((part) => part.type === "file" && part.uploadId)) {
+      if (!message.parts.some((part) => getUploadId(part))) {
         return message;
       }
 
       const parts = await Promise.all(
         message.parts.map(async (part) => {
-          if (!isFilePartWithUpload(part)) return part;
+          const uploadId = getUploadId(part);
+          if (!uploadId) return part;
 
-          let urlPromise = urlByUploadId.get(part.uploadId);
+          let urlPromise = urlByUploadId.get(uploadId);
           if (!urlPromise) {
-            urlPromise = resolveFileUrl({ uploadId: part.uploadId, part, message });
-            urlByUploadId.set(part.uploadId, urlPromise);
+            urlPromise = resolveFileUrl({
+              uploadId,
+              part: toResolverPart(part, uploadId),
+              message,
+            });
+            urlByUploadId.set(uploadId, urlPromise);
           }
 
           const signedUrl = await urlPromise;
-          if (!signedUrl || signedUrl === part.url) return part;
+          if (!signedUrl) return normalizeUploadedFilePart(part, uploadId);
 
           return {
-            ...part,
+            ...normalizeUploadedFilePart(part, uploadId),
             url: signedUrl,
           };
         }),
@@ -47,8 +52,69 @@ export async function resolveRuntimeMessageFileUrls(
   );
 }
 
-function isFilePartWithUpload(part: ChatUiMessage["parts"][number]): part is ChatFileUiPart & {
-  uploadId: string;
-} {
-  return part.type === "file" && typeof part.uploadId === "string" && part.uploadId.length > 0;
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function getStringField(part: unknown, key: string): string | undefined {
+  if (!isRecord(part)) return undefined;
+
+  const value = part[key];
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function getUploadId(part: unknown): string | undefined {
+  if (!isRecord(part) || (part.type !== "file" && part.type !== "image")) {
+    return undefined;
+  }
+
+  return getStringField(part, "uploadId") ?? getStringField(part, "upload_id");
+}
+
+function getMediaType(part: unknown): string | undefined {
+  return getStringField(part, "mediaType") ?? getStringField(part, "media_type");
+}
+
+function normalizeUploadedFilePart(
+  part: ChatUiMessage["parts"][number],
+  uploadId: string,
+): ChatUiMessage["parts"][number] {
+  if (!isRecord(part)) return part;
+
+  const partRecord: Record<string, unknown> = part;
+  const partType = partRecord.type;
+  if (partType !== "file" && partType !== "image") return part;
+
+  const mediaType = getMediaType(part);
+  const url = getStringField(part, "url");
+  if (!mediaType || !url) return part;
+
+  const filename = getStringField(part, "filename");
+  const uploadPath = getStringField(part, "uploadPath") ?? getStringField(part, "upload_path");
+
+  return {
+    type: partType === "image" ? "image" : "file",
+    mediaType,
+    url,
+    ...(filename ? { filename } : {}),
+    uploadId,
+    ...(uploadPath ? { uploadPath } : {}),
+  } as ChatUiMessage["parts"][number];
+}
+
+function toResolverPart(
+  part: ChatUiMessage["parts"][number],
+  uploadId: string,
+): FileUIPartWithUpload {
+  const normalized = normalizeUploadedFilePart(part, uploadId);
+  if (isRecord(normalized) && normalized.type === "file") {
+    return normalized as FileUIPartWithUpload;
+  }
+
+  return {
+    type: "file",
+    mediaType: getMediaType(part) ?? "application/octet-stream",
+    url: getStringField(part, "url") ?? "",
+    uploadId,
+  };
 }

package/src/src/chat/conversation.ts

@@ -515,7 +515,7 @@ function toJsonValue(value: unknown): JsonValue {
 }
 
 function getFilePart(part: unknown): {
-  type: "file";
+  type: "file" | "image";
   mediaType: string;
   data: string;
   url: string;
@@ -523,22 +523,25 @@ function getFilePart(part: unknown): {
   uploadId?: string;
   uploadPath?: string;
 } | null {
-  if (!isRecord(part) || part.type !== "file") {
+  if (!isRecord(part) || (part.type !== "file" && part.type !== "image")) {
     return null;
   }
 
-  const mediaType = getNonEmptyStringField(part, "mediaType");
+  const mediaType = getNonEmptyStringField(part, "mediaType") ??
+    getNonEmptyStringField(part, "media_type");
   const data = getNonEmptyStringField(part, "url");
   if (!mediaType || !data) {
     return null;
   }
 
   const filename = getNonEmptyStringField(part, "filename");
-  const uploadId = getNonEmptyStringField(part, "uploadId");
-  const uploadPath = getNonEmptyStringField(part, "uploadPath");
+  const uploadId = getNonEmptyStringField(part, "uploadId") ??
+    getNonEmptyStringField(part, "upload_id");
+  const uploadPath = getNonEmptyStringField(part, "uploadPath") ??
+    getNonEmptyStringField(part, "upload_path");
 
   return {
-    type: "file",
+    type: part.type === "image" ? "image" : "file",
     mediaType,
     data,
     url: data,
@@ -721,10 +724,13 @@ function convertSystemMessage(message: ChatUiMessage): ProviderModelMessage[] {
 function convertUserMessage(message: ChatUiMessage): ProviderModelMessage[] {
   const content: Array<
     { type: "text"; text: string } | {
-      type: "file";
+      type: "file" | "image";
       mediaType: string;
       data: string;
+      url: string;
       filename?: string;
+      uploadId?: string;
+      uploadPath?: string;
     }
   > = [];
 
@@ -757,7 +763,7 @@ function convertAssistantMessage(message: ChatUiMessage): ProviderModelMessage[]
   const assistantContent: Array<
     | { type: "text"; text: string }
     | { type: "reasoning"; text: string }
-    | { type: "file"; mediaType: string; data: string; filename?: string }
+    | { type: "file" | "image"; mediaType: string; data: string; filename?: string }
     | { type: "tool-call"; toolCallId: string; toolName: string; input: Record<string, unknown> }
   > = [];
   const deferredAssistantContent: typeof assistantContent = [];
@@ -806,7 +812,7 @@ function convertAssistantMessage(message: ChatUiMessage): ProviderModelMessage[]
     part:
       | { type: "text"; text: string }
       | { type: "reasoning"; text: string }
-      | { type: "file"; mediaType: string; data: string; filename?: string }
+      | { type: "file" | "image"; mediaType: string; data: string; filename?: string }
       | { type: "tool-call"; toolCallId: string; toolName: string; input: Record<string, unknown> },
   ) => {
     if (toolResults.length > 0) {

package/src/src/provider/runtime-loader.ts

@@ -34,7 +34,13 @@ export type { RuntimeUsage };
 
 export type RuntimePromptMessage =
   | { role: "system"; content: string }
-  | { role: "user"; content: Array<{ type: "text"; text: string }> }
+  | {
+    role: "user";
+    content: Array<
+      | { type: "text"; text: string }
+      | { type: "image" | "file"; mediaType: string; url: string; filename?: string }
+    >;
+  }
   | {
     role: "assistant";
     content: Array<
@@ -261,7 +267,15 @@ type OpenAICompatibleLanguageOptions = {
 };
 export type OpenAICompatibleChatMessage =
   | { role: "system"; content: string }
-  | { role: "user"; content: string }
+  | {
+    role: "user";
+    content:
+      | string
+      | Array<
+        | { type: "text"; text: string }
+        | { type: "image_url"; image_url: { url: string } }
+      >;
+  }
   | {
     role: "assistant";
     content: string | null;
@@ -279,6 +293,11 @@ export type OpenAICompatibleChatMessage =
     tool_call_id: string;
     content: string;
   };
+type RuntimePromptUserContent = Extract<RuntimePromptMessage, { role: "user" }>["content"];
+type OpenAICompatibleUserContent = Extract<
+  OpenAICompatibleChatMessage,
+  { role: "user" }
+>["content"];
 export type OpenAICompatibleChatRequest = {
   model: string;
   messages: OpenAICompatibleChatMessage[];
@@ -359,6 +378,30 @@ export function readTextParts(parts: Array<{ type: string; text?: string }>): st
   return text;
 }
 
+function toOpenAICompatibleUserContent(
+  parts: RuntimePromptUserContent,
+): OpenAICompatibleUserContent {
+  if (!parts.some((part) => part.type !== "text" && part.mediaType.startsWith("image/"))) {
+    return readTextParts(parts);
+  }
+
+  const content: Exclude<OpenAICompatibleUserContent, string> = [];
+
+  for (const part of parts) {
+    if (part.type === "text") {
+      if (part.text.length > 0) {
+        content.push({ type: "text", text: part.text });
+      }
+      continue;
+    }
+    if (part.type === "image" || part.mediaType.startsWith("image/")) {
+      content.push({ type: "image_url", image_url: { url: part.url } });
+    }
+  }
+
+  return content.length > 0 ? content : readTextParts(parts);
+}
+
 export function toOpenAICompatibleMessages(
   prompt: RuntimePromptMessage[],
 ): OpenAICompatibleChatMessage[] {
@@ -370,7 +413,7 @@ export function toOpenAICompatibleMessages(
         messages.push({ role: "system", content: message.content });
         break;
       case "user":
-        messages.push({ role: "user", content: readTextParts(message.content) });
+        messages.push({ role: "user", content: toOpenAICompatibleUserContent(message.content) });
         break;
       case "assistant": {
         let text = "";

package/src/src/runtime/runtime-bridge.ts

@@ -73,7 +73,13 @@ type EmbedManyOptions = {
 
 type RuntimePromptMessage =
   | { role: "system"; content: string }
-  | { role: "user"; content: Array<{ type: "text"; text: string }> }
+  | {
+    role: "user";
+    content: Array<
+      | { type: "text"; text: string }
+      | { type: "image" | "file"; mediaType: string; url: string; filename?: string }
+    >;
+  }
   | {
     role: "assistant";
     content: Array<
@@ -181,7 +187,9 @@ function toRuntimePrompt(
       case "user":
         prompt.push({
           role: "user",
-          content: [{ type: "text", text: message.content }],
+          content: typeof message.content === "string"
+            ? [{ type: "text", text: message.content }]
+            : message.content,
         });
         break;
       case "assistant":

package/src/src/utils/version-constant.ts

@@ -1,3 +1,3 @@
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
-export const VERSION = "0.1.438";
+export const VERSION = "0.1.440";