plugin-ai-api 1.0.15 → 1.0.21

package/src/server/routes/chat-completions.ts

@@ -19,10 +19,17 @@ import {
   OpenAIToolCallChunk,
 } from '../utils/openai-format';
 import { resolveModelString } from '../utils/resolve-service';
-import { createRequestAbortController, isStreamingRequested, writeResponse } from '../utils/streaming';
+import {
+  createRequestAbortController,
+  isClientDisconnected,
+  isStreamingRequested,
+  writeResponse,
+} from '../utils/streaming';
 import { checkEmployeeAccess } from '../middleware/role-permission';
 import { extractProviderRequestId, normalizeUsage, setAiApiUsageResult, type Usage } from '../usage';
 import type PluginAiApiServer from '../plugin';
+import { AiApiQuotaError, markLlmProviderAttempted, prepareLlmBilling } from '../billing';
+import { markAiApiFirstProviderOutput } from '../utils/app-observability';
 
 /**
  * POST /api/ai-llm/v1/chat/completions
@@ -46,6 +53,30 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
     return;
   }
 
+  const messageProblem = findMessageProblem(body.messages);
+  if (messageProblem) {
+    ctx.status = 400;
+    ctx.body = toOpenAIError(
+      400,
+      `Invalid messages[${messageProblem.index}]: ${messageProblem.reason}.`,
+      'invalid_request_error',
+      'invalid_message',
+    );
+    return;
+  }
+
+  const blockProblem = findContentBlockProblem(body.messages);
+  if (blockProblem) {
+    ctx.status = 400;
+    ctx.body = toOpenAIError(
+      400,
+      `Invalid content block in messages[${blockProblem.index}]: ${blockProblem.reason}.`,
+      'invalid_request_error',
+      'invalid_content_block',
+    );
+    return;
+  }
+
   // ─── Reject unsupported n parameter ───
   if (body.n !== undefined && body.n !== null && body.n !== 1) {
     ctx.status = 400;
@@ -121,6 +152,8 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
       return;
     }
 
+    await prepareLlmBilling(ctx, resolved);
+
     const providerRequestParameters = getProviderRequestParameters(body);
     const modelOptions: Record<string, unknown> = {
       model: modelId,
@@ -176,9 +209,14 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
     // ─── Build message tuples for LangChain model ───
     // LangChain chat models accept [role, content] tuples or BaseMessage objects.
     // We use tuples to avoid importing @langchain/core directly.
+    //
+    // `content` may be a string OR an array of content blocks (OpenAI vision
+    // format: [{type:'text'}, {type:'image_url', image_url:{url:'data:...'}}]).
+    // Arrays must be forwarded structurally — stringifying them would turn an
+    // image into literal JSON text and the model would never see the picture.
     const langchainMessages = messages.map((msg: any) => {
       const role = msg.role === 'assistant' ? 'ai' : msg.role;
-      const content = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content);
+      const content = normalizeMessageContent(msg.content);
       if (msg.role === 'assistant' && msg.tool_calls) {
         return {
           role,
@@ -190,13 +228,14 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
       if (msg.role === 'tool') {
         return { role: 'tool', content, tool_call_id: msg.tool_call_id, name: msg.name };
       }
-      return [role, content] as [string, string];
+      return [role, content] as [string, MessageContent];
     });
 
     const completionId = generateCompletionId();
     const baseModel = provider.createModel();
     applyProviderRequestParameters(baseModel, providerRequestParameters);
     const chatModel = bindRequestTools(baseModel, body.tools, body.tool_choice, providerRequestParameters);
+    markLlmProviderAttempted(ctx);
 
     if (stream) {
       // ─── Streaming mode ───
@@ -222,8 +261,15 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
   } catch (err) {
     ctx.log.error('AI API chat completions error:', err);
     if (!ctx.res.headersSent) {
-      ctx.status = 500;
-      ctx.body = toOpenAIError(500, getErrorMessage(err, 'Internal server error'), 'server_error');
+      const isQuotaError = err instanceof AiApiQuotaError;
+      ctx.status = isQuotaError ? 429 : 500;
+      if (isQuotaError) ctx.set('X-RateLimit-Reason', err.code);
+      ctx.body = toOpenAIError(
+        ctx.status,
+        getErrorMessage(err, 'Internal server error'),
+        isQuotaError ? 'quota_error' : 'server_error',
+        isQuotaError ? err.code : undefined,
+      );
     }
   }
 }
@@ -315,6 +361,7 @@ async function handleStreamingCompletion(
       }
 
       if (content) {
+        markAiApiFirstProviderOutput(ctx);
         await writeResponse(
           ctx,
           formatSSE(
@@ -329,6 +376,7 @@ async function handleStreamingCompletion(
 
       const toolCallChunks = normalizeToolCallChunks(chunk.tool_call_chunks);
       if (toolCallChunks.length) {
+        markAiApiFirstProviderOutput(ctx);
         finishReason = 'tool_calls';
         await writeResponse(
           ctx,
@@ -359,6 +407,7 @@ async function handleStreamingCompletion(
     setAiApiUsageResult(ctx, usage, { gatewayResponseId: completionId, providerRequestId });
     ctx.state.aiApiStreamResult = { succeeded: true, id: completionId };
   } catch (err) {
+    const cancelled = isClientDisconnected(ctx, err);
     ctx.log.error('AI API streaming error:', err);
     // Send error as SSE event before closing
     if (!ctx.res.destroyed && !ctx.res.writableEnded) {
@@ -373,7 +422,11 @@ async function handleStreamingCompletion(
       );
     }
     setAiApiUsageResult(ctx, usage, { gatewayResponseId: completionId, providerRequestId });
-    ctx.state.aiApiStreamResult = { succeeded: false, id: completionId, errorCode: 'stream_error' };
+    ctx.state.aiApiStreamResult = {
+      succeeded: false,
+      id: completionId,
+      errorCode: cancelled ? 'client_disconnected' : 'stream_error',
+    };
   } finally {
     requestAbort.dispose();
     if (!ctx.res.writableEnded && !ctx.res.destroyed) ctx.res.end();
@@ -384,6 +437,222 @@ function getErrorMessage(error: unknown, fallback: string) {
   return error instanceof Error && error.message ? error.message : fallback;
 }
 
+/**
+ * Content block types every provider adapter in `@nocobase/plugin-ai` maps to a
+ * native equivalent.
+ *
+ * Anything else is rejected rather than forwarded: the LangChain block
+ * converters are if/else-if chains with no fallback branch, so an unrecognized
+ * block (OpenAI's `{type:'file'}` on Anthropic, for example) yields nothing and
+ * the model answers as if the attachment was never sent. A 400 is far easier to
+ * debug than a confidently wrong completion.
+ */
+const SUPPORTED_CONTENT_BLOCK_TYPES = new Set(['text', 'image_url']);
+
+/**
+ * Deliberately mirrors the exact grammar `@langchain/core`'s `parseBase64DataUrl`
+ * accepts (`\w+/\w+`, standard base64). A looser pattern here would admit URLs
+ * the provider adapter then fails to parse — it falls through to `new URL()`,
+ * sees the `data:` protocol and throws, which reaches the client as a 500.
+ * Compound subtypes such as `image/svg+xml` are rejected for that reason.
+ */
+const BASE64_DATA_URL_PATTERN = /^data:(\w+\/\w+);base64,([A-Za-z0-9+/]+=*)$/;
+
+/**
+ * The regex above is LangChain's, and LangChain's is lenient: `A===`, `A=`,
+ * `AAAAA` and `AAAA=` all match it but are not decodable base64. LangChain then
+ * calls `atob` on the payload, which throws a DOMException for each of them, and
+ * that escapes as an HTTP 500. Re-encoding is the cheapest exact check — the
+ * canonical form of a valid payload is the payload itself.
+ */
+function isDecodableBase64(payload: string): boolean {
+  try {
+    return Buffer.from(payload, 'base64').toString('base64') === payload;
+  } catch {
+    return false;
+  }
+}
+
+export interface ContentBlockProblem {
+  index: number;
+  reason: string;
+}
+
+/**
+ * Roles `_constructMessageFromParams` can turn into a message. Anything else
+ * reaches its final `else` and throws MESSAGE_COERCION_FAILURE.
+ *
+ * `function` is deliberately absent: OpenAI deprecated it, and LangChain has no
+ * branch for it despite mapping the class name internally.
+ */
+const SUPPORTED_MESSAGE_ROLES = new Set(['system', 'developer', 'user', 'human', 'assistant', 'ai', 'tool']);
+
+/**
+ * Validate the shape of each message before any provider work happens.
+ *
+ * Requiring only "non-empty array" lets `messages: [null]` through to
+ * `messages.some((m) => m.role === 'system')`, which throws a TypeError and is
+ * reported as a 500. An unsupported role travels further still and dies inside
+ * LangChain with MESSAGE_COERCION_FAILURE. Both are caller errors, so both
+ * should be a 400 that names the offending index.
+ */
+export function findMessageProblem(messages: unknown[]): ContentBlockProblem | undefined {
+  for (const [index, message] of messages.entries()) {
+    if (!isRecord(message)) return { index, reason: 'each message must be an object' };
+
+    const role = typeof message.role === 'string' ? message.role : undefined;
+    if (!role) return { index, reason: "each message requires a string 'role' field" };
+    if (!SUPPORTED_MESSAGE_ROLES.has(role)) {
+      return {
+        index,
+        reason: `role '${role}' is not supported — use one of ` + `${[...SUPPORTED_MESSAGE_ROLES].join(', ')}`,
+      };
+    }
+
+    if (role === 'tool' && typeof message.tool_call_id !== 'string') {
+      return { index, reason: "a 'tool' message requires a string 'tool_call_id' field" };
+    }
+
+    const { content } = message;
+    const hasToolCalls = Array.isArray(message.tool_calls) && message.tool_calls.length > 0;
+    if (content === undefined || content === null) {
+      // An assistant turn that only calls tools legitimately carries no content.
+      if ((role === 'assistant' || role === 'ai') && hasToolCalls) continue;
+      return { index, reason: "each message requires a 'content' field" };
+    }
+    if (typeof content !== 'string' && !Array.isArray(content)) {
+      return { index, reason: "'content' must be a string or an array of content blocks" };
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Validate the multimodal content blocks of a chat request.
+ *
+ * Checking `type` alone is not enough. The provider adapters either drop or
+ * throw on malformed blocks, and both outcomes surface badly:
+ *
+ * - A block whose payload fails every branch of the converter yields nothing,
+ *   so the model answers as if the attachment was never sent.
+ * - `_formatImage` throws on a malformed or non-http(s) URL, and that escapes as
+ *   a generic HTTP 500 instead of telling the caller what was wrong.
+ * - `parseBase64DataUrl` matches any `data:<type>/<subtype>;base64,` URL without
+ *   checking for `image/*`, so a PDF becomes an `image` block with
+ *   `media_type: application/pdf` that the model cannot read.
+ *
+ * Validating up front turns all of those into an actionable 400.
+ */
+export function findContentBlockProblem(messages: unknown[]): ContentBlockProblem | undefined {
+  for (const [index, message] of messages.entries()) {
+    const content = isRecord(message) ? message.content : undefined;
+    if (!Array.isArray(content)) continue;
+    for (const block of content) {
+      if (typeof block === 'string') continue;
+      const reason = describeContentBlockProblem(block);
+      if (reason) return { index, reason };
+    }
+  }
+  return undefined;
+}
+
+function describeContentBlockProblem(block: unknown): string | undefined {
+  if (!isRecord(block)) return 'each content block must be an object';
+
+  const type = typeof block.type === 'string' ? block.type : undefined;
+  if (!type) return "each content block requires a 'type' field";
+  if (!SUPPORTED_CONTENT_BLOCK_TYPES.has(type)) {
+    return (
+      `content block type '${type}' is not supported — this gateway forwards 'text' and 'image_url' only. ` +
+      `Send documents as text, or inline them as an 'image_url' data URL if the model reads images`
+    );
+  }
+
+  if (type === 'text') {
+    return typeof block.text === 'string' ? undefined : "a 'text' block requires a string 'text' field";
+  }
+
+  return describeImageUrlProblem(block.image_url);
+}
+
+function describeImageUrlProblem(imageUrl: unknown): string | undefined {
+  const url = typeof imageUrl === 'string' ? imageUrl : isRecord(imageUrl) ? imageUrl.url : undefined;
+  if (typeof url !== 'string' || url === '') {
+    return "an 'image_url' block requires a non-empty 'image_url.url' string";
+  }
+
+  if (url.startsWith('data:')) {
+    const match = BASE64_DATA_URL_PATTERN.exec(url);
+    if (!match) {
+      return (
+        `malformed base64 data URL. Expected 'data:<mime-type>;base64,<base64>' ` +
+        `with standard base64 (no whitespace or URL-safe characters)`
+      );
+    }
+    const mimeType = match[1].toLowerCase();
+    if (!mimeType.startsWith('image/')) {
+      return (
+        `data URL MIME type '${mimeType}' is not an image. Only 'image/*' data URLs are forwarded, ` +
+        `because providers reject or ignore other types on an 'image_url' block`
+      );
+    }
+    if (!isDecodableBase64(match[2])) {
+      return (
+        `base64 payload is not decodable. Check the padding and length — ` +
+        `the data must be a multiple of 4 characters with at most two trailing '='`
+      );
+    }
+    return undefined;
+  }
+
+  let protocol: string;
+  try {
+    protocol = new URL(url).protocol;
+  } catch {
+    return `'${url}' is not a valid URL. Use an http(s) URL or a base64 data URL`;
+  }
+  if (protocol !== 'http:' && protocol !== 'https:') {
+    return `URL protocol '${protocol}' is not supported. Use an http(s) URL or a base64 data URL`;
+  }
+  return undefined;
+}
+
+/**
+ * A LangChain message content value: plain text, or an array of content blocks
+ * (`{type:'text'}`, `{type:'image_url'}`, ...) for multimodal requests.
+ */
+type MessageContent = string | Record<string, unknown>[];
+
+/**
+ * Normalize an OpenAI `message.content` into something LangChain accepts.
+ *
+ * Content blocks are passed through unchanged so vision requests reach the
+ * provider intact — `@langchain/core` coerces `image_url` blocks into the
+ * provider's native format. Only genuinely unusable shapes (numbers, objects)
+ * are stringified as a last resort.
+ *
+ * The one rewrite is `image_url: '<url>'` → `image_url: { url: '<url>' }`.
+ * `isOpenAIDataBlock` gates on `_isObject(block.image_url)`, so the string form
+ * is never recognised as a data block: core forwards it untouched and only
+ * Anthropic's adapter happens to accept it. Widening it here keeps the lenient
+ * request working on every provider instead of just one.
+ */
+export function normalizeMessageContent(content: unknown): MessageContent {
+  if (typeof content === 'string') return content;
+  if (Array.isArray(content)) {
+    return content.map((block) => {
+      if (typeof block === 'string') return { type: 'text', text: block };
+      const record = block as Record<string, unknown>;
+      if (record?.type === 'image_url' && typeof record.image_url === 'string') {
+        return { ...record, image_url: { url: record.image_url } };
+      }
+      return record;
+    });
+  }
+  if (content === null || content === undefined) return '';
+  return JSON.stringify(content);
+}
+
 const GATEWAY_MANAGED_PARAMETERS = new Set(['model', 'messages', 'tools', 'tool_choice', 'stream', 'n']);
 
 export function getProviderRequestParameters(body: Record<string, unknown>): Record<string, unknown> {

package/src/server/routes/completions.ts

@@ -10,9 +10,16 @@
 import { Context } from '@nocobase/actions';
 import { generateCompletionId, toOpenAIError, formatSSE, formatSSEDone } from '../utils/openai-format';
 import { resolveModelString } from '../utils/resolve-service';
-import { createRequestAbortController, isStreamingRequested, writeResponse } from '../utils/streaming';
+import {
+  createRequestAbortController,
+  isClientDisconnected,
+  isStreamingRequested,
+  writeResponse,
+} from '../utils/streaming';
 import { extractProviderRequestId, normalizeUsage, setAiApiUsageResult, type Usage } from '../usage';
 import type PluginAiApiServer from '../plugin';
+import { AiApiQuotaError, markLlmProviderAttempted, prepareLlmBilling } from '../billing';
+import { markAiApiFirstProviderOutput } from '../utils/app-observability';
 
 /**
  * POST /api/ai-llm/v1/completions
@@ -112,6 +119,8 @@ export async function handleCompletions(ctx: Context, plugin: PluginAiApiServer)
       return;
     }
 
+    await prepareLlmBilling(ctx, resolved);
+
     const modelOptions: Record<string, any> = {
       model: modelId,
       llmService: service.name,
@@ -154,6 +163,7 @@ export async function handleCompletions(ctx: Context, plugin: PluginAiApiServer)
 
     const completionId = generateCompletionId().replace('chatcmpl-', 'cmpl-');
     const chatModel = provider.createModel();
+    markLlmProviderAttempted(ctx);
 
     if (stream) {
       await handleStreamingTextCompletion(ctx, chatModel, langchainMessages, completionId, body.model);
@@ -163,8 +173,15 @@ export async function handleCompletions(ctx: Context, plugin: PluginAiApiServer)
   } catch (err) {
     ctx.log.error('AI API completions error:', err);
     if (!ctx.res.headersSent) {
-      ctx.status = 500;
-      ctx.body = toOpenAIError(500, getErrorMessage(err, 'Internal server error'), 'server_error');
+      const isQuotaError = err instanceof AiApiQuotaError;
+      ctx.status = isQuotaError ? 429 : 500;
+      if (isQuotaError) ctx.set('X-RateLimit-Reason', err.code);
+      ctx.body = toOpenAIError(
+        ctx.status,
+        getErrorMessage(err, 'Internal server error'),
+        isQuotaError ? 'quota_error' : 'server_error',
+        isQuotaError ? err.code : undefined,
+      );
     }
   }
 }
@@ -246,6 +263,7 @@ async function handleStreamingTextCompletion(
       }
 
       if (text) {
+        markAiApiFirstProviderOutput(ctx);
         await writeResponse(
           ctx,
           formatSSE({
@@ -295,6 +313,7 @@ async function handleStreamingTextCompletion(
     setAiApiUsageResult(ctx, usage, { gatewayResponseId: completionId, providerRequestId });
     ctx.state.aiApiStreamResult = { succeeded: true, id: completionId };
   } catch (err) {
+    const cancelled = isClientDisconnected(ctx, err);
     ctx.log.error('AI API completions streaming error:', err);
     if (!ctx.res.destroyed && !ctx.res.writableEnded) {
       await writeResponse(
@@ -308,7 +327,11 @@ async function handleStreamingTextCompletion(
       );
     }
     setAiApiUsageResult(ctx, usage, { gatewayResponseId: completionId, providerRequestId });
-    ctx.state.aiApiStreamResult = { succeeded: false, id: completionId, errorCode: 'stream_error' };
+    ctx.state.aiApiStreamResult = {
+      succeeded: false,
+      id: completionId,
+      errorCode: cancelled ? 'client_disconnected' : 'stream_error',
+    };
   } finally {
     requestAbort.dispose();
     if (!ctx.res.writableEnded && !ctx.res.destroyed) ctx.res.end();