plugin-ai-api 1.0.11 → 1.0.13

package/dist/server/routes/chat-completions.js

@@ -26,6 +26,8 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var chat_completions_exports = {};
 __export(chat_completions_exports, {
+  applyProviderRequestParameters: () => applyProviderRequestParameters,
+  getProviderRequestParameters: () => getProviderRequestParameters,
   handleChatCompletions: () => handleChatCompletions
 });
 module.exports = __toCommonJS(chat_completions_exports);
@@ -108,13 +110,15 @@ async function handleChatCompletions(ctx, plugin) {
       ctx.body = (0, import_openai_format.toOpenAIError)(500, `Provider '${service.provider}' not registered`, "server_error");
       return;
     }
+    const providerRequestParameters = getProviderRequestParameters(body);
     const modelOptions = {
       model: modelId,
       llmService: service.name
     };
     if (body.temperature !== void 0) modelOptions.temperature = body.temperature;
     if (body.top_p !== void 0) modelOptions.topP = body.top_p;
-    if (body.max_tokens !== void 0) modelOptions.maxTokens = body.max_tokens;
+    if (body.max_completion_tokens !== void 0) modelOptions.maxTokens = body.max_completion_tokens;
+    else if (body.max_tokens !== void 0) modelOptions.maxTokens = body.max_tokens;
     if (body.frequency_penalty !== void 0) modelOptions.frequencyPenalty = body.frequency_penalty;
     if (body.presence_penalty !== void 0) modelOptions.presencePenalty = body.presence_penalty;
     if (body.stop !== void 0) modelOptions.stop = body.stop;
@@ -166,11 +170,26 @@ async function handleChatCompletions(ctx, plugin) {
     });
     const completionId = (0, import_openai_format.generateCompletionId)();
     const baseModel = provider.createModel();
-    const chatModel = bindRequestTools(baseModel, body.tools, body.tool_choice);
+    applyProviderRequestParameters(baseModel, providerRequestParameters);
+    const chatModel = bindRequestTools(baseModel, body.tools, body.tool_choice, providerRequestParameters);
     if (stream) {
-      await handleStreamingCompletion(ctx, chatModel, langchainMessages, completionId, body.model);
+      await handleStreamingCompletion(
+        ctx,
+        chatModel,
+        langchainMessages,
+        completionId,
+        body.model,
+        providerRequestParameters
+      );
     } else {
-      await handleNonStreamingCompletion(ctx, chatModel, langchainMessages, completionId, body.model);
+      await handleNonStreamingCompletion(
+        ctx,
+        chatModel,
+        langchainMessages,
+        completionId,
+        body.model,
+        providerRequestParameters
+      );
     }
   } catch (err) {
     ctx.log.error("AI API chat completions error:", err);
@@ -180,8 +199,8 @@ async function handleChatCompletions(ctx, plugin) {
     }
   }
 }
-async function handleNonStreamingCompletion(ctx, chatModel, messages, completionId, modelName) {
-  const result = await chatModel.invoke(messages);
+async function handleNonStreamingCompletion(ctx, chatModel, messages, completionId, modelName, providerRequestParameters) {
+  const result = await chatModel.invoke(messages, providerRequestParameters);
   let content = "";
   if (typeof result.content === "string") {
     content = result.content;
@@ -204,7 +223,7 @@ async function handleNonStreamingCompletion(ctx, chatModel, messages, completion
     toolCalls
   });
 }
-async function handleStreamingCompletion(ctx, chatModel, messages, completionId, modelName) {
+async function handleStreamingCompletion(ctx, chatModel, messages, completionId, modelName, providerRequestParameters) {
   ctx.set({
     "Content-Type": "text/event-stream",
     "Cache-Control": "no-cache",
@@ -227,7 +246,7 @@ async function handleStreamingCompletion(ctx, chatModel, messages, completionId,
   let usage;
   let finishReason = "stop";
   try {
-    const stream = await chatModel.stream(messages, { signal: requestAbort.signal });
+    const stream = await chatModel.stream(messages, { ...providerRequestParameters, signal: requestAbort.signal });
     for await (const chunk of stream) {
       if (requestAbort.signal.aborted) throw requestAbort.signal.reason;
       let content = "";
@@ -300,12 +319,41 @@ async function handleStreamingCompletion(ctx, chatModel, messages, completionId,
 function getErrorMessage(error, fallback) {
   return error instanceof Error && error.message ? error.message : fallback;
 }
-function bindRequestTools(chatModel, tools, toolChoice) {
+const GATEWAY_MANAGED_PARAMETERS = /* @__PURE__ */ new Set(["model", "messages", "tools", "tool_choice", "stream", "n"]);
+function getProviderRequestParameters(body) {
+  return Object.fromEntries(
+    Object.entries(body).filter(([name, value]) => !GATEWAY_MANAGED_PARAMETERS.has(name) && value !== void 0)
+  );
+}
+function applyProviderRequestParameters(chatModel, parameters) {
+  if (!chatModel || typeof chatModel !== "object") return;
+  const model = chatModel;
+  const modelKwargs = { ...model.modelKwargs ?? {} };
+  if (!Object.hasOwn(parameters, "response_format")) {
+    if (isDefaultTextResponseFormat(modelKwargs.response_format)) delete modelKwargs.response_format;
+    if (isDefaultResponsesTextFormat(modelKwargs.text)) delete modelKwargs.text;
+  }
+  model.modelKwargs = { ...modelKwargs, ...parameters };
+}
+function bindRequestTools(chatModel, tools, toolChoice, providerRequestParameters) {
   if (!Array.isArray(tools) || tools.length === 0) return chatModel;
   if (typeof chatModel.bindTools !== "function") {
     throw new Error("The selected LLM provider does not support tool calling");
   }
-  return chatModel.bindTools(tools, toolChoice === void 0 ? void 0 : { tool_choice: toolChoice });
+  return chatModel.bindTools(tools, {
+    ...providerRequestParameters,
+    ...toolChoice === void 0 ? {} : { tool_choice: toolChoice }
+  });
+}
+function isDefaultTextResponseFormat(value) {
+  return isRecord(value) && value.type === "text" && Object.keys(value).length === 1;
+}
+function isDefaultResponsesTextFormat(value) {
+  if (!isRecord(value) || !isRecord(value.format)) return false;
+  return value.format.type === "text" && Object.keys(value.format).length === 1 && Object.keys(value).length === 1;
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function normalizeToolCalls(value) {
   if (!Array.isArray(value) || value.length === 0) return void 0;
@@ -340,5 +388,7 @@ function serializeToolArguments(value) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  applyProviderRequestParameters,
+  getProviderRequestParameters,
   handleChatCompletions
 });

package/dist/server/routes/embeddings.js

@@ -157,7 +157,7 @@ async function handleEmbeddings(ctx, plugin) {
       model: body.model,
       embeddings: vectors,
       // LangChain's EmbeddingsInterface does not expose token counts.
-      promptTokens: 0
+      promptTokens: null
     });
   } catch (err) {
     ctx.log.error("AI API embeddings error:", err);

package/dist/server/routes/router.js

@@ -50,11 +50,12 @@ var import_openai_format = require("../utils/openai-format");
 var import_rate_limit = require("../middleware/rate-limit");
 var import_role_permission = require("../middleware/role-permission");
 var import_usage = require("../usage");
+var import_streaming = require("../utils/streaming");
 const API_PREFIX = "/api/ai-llm/v1";
 function createAiLlmRouter(plugin) {
   const checkRateLimit = (0, import_rate_limit.createRateLimitMiddleware)(plugin.rateLimiter);
   return async (ctx, next) => {
-    var _a, _b, _c, _d;
+    var _a, _b, _c;
     const { path, method } = ctx;
     if (!path.startsWith(API_PREFIX)) {
       return next();
@@ -100,28 +101,35 @@ function createAiLlmRouter(plugin) {
       return;
     }
     const model = ((_a = ctx.request.body) == null ? void 0 : _a.model) ?? "-";
+    const isUsageEndpoint = method === "POST" && (subPath === "/chat/completions" || subPath === "/completions" || subPath === "/embeddings");
+    const resolvedMode = isUsageEndpoint ? await resolveMode(ctx) : "llm";
+    const requestBody = ctx.request.body || {};
+    const streaming = isUsageEndpoint && (0, import_streaming.isStreamingRequested)(requestBody.stream);
+    if (streaming && (subPath === "/chat/completions" || subPath === "/completions")) {
+      const streamOptions = requestBody.stream_options;
+      ctx.request.body = {
+        ...requestBody,
+        stream_options: {
+          ...streamOptions && typeof streamOptions === "object" ? streamOptions : {},
+          include_usage: true
+        }
+      };
+    }
     const t0 = Date.now();
     let usageId;
     try {
-      usageId = await (0, import_usage.startUsageRecord)(
-        ctx,
-        requestId,
-        subPath,
-        String(model),
-        Boolean((_b = ctx.request.body) == null ? void 0 : _b.stream)
-      );
+      usageId = isUsageEndpoint ? await (0, import_usage.startUsageRecord)(ctx, requestId, subPath, String(model), streaming, resolvedMode) : void 0;
     } catch (usageError) {
       ctx.log.error("AI API usage record could not be created:", usageError);
     }
     try {
       if (method === "POST" && subPath === "/chat/completions") {
-        const mode = await resolveMode(ctx);
-        await (mode === "agent" ? (0, import_agent_completions.handleAgentCompletions)(ctx, plugin) : (0, import_chat_completions.handleChatCompletions)(ctx, plugin));
+        await (resolvedMode === "agent" ? (0, import_agent_completions.handleAgentCompletions)(ctx, plugin) : (0, import_chat_completions.handleChatCompletions)(ctx, plugin));
         logRequest(
           ctx,
           requestId,
           model,
-          ((_c = ctx.state.aiApiStreamResult) == null ? void 0 : _c.succeeded) === false ? "error" : "ok",
+          ((_b = ctx.state.aiApiStreamResult) == null ? void 0 : _b.succeeded) === false ? "error" : "ok",
           Date.now() - t0
         );
         return;
@@ -132,7 +140,7 @@ function createAiLlmRouter(plugin) {
         return;
       }
       if (method === "POST" && subPath === "/completions") {
-        const completionsMode = await resolveMode(ctx);
+        const completionsMode = resolvedMode;
         if (completionsMode === "agent") {
           const reqBody = ctx.request.body;
           if ((reqBody == null ? void 0 : reqBody.prompt) !== void 0) {
@@ -147,7 +155,7 @@ function createAiLlmRouter(plugin) {
           ctx,
           requestId,
           model,
-          ((_d = ctx.state.aiApiStreamResult) == null ? void 0 : _d.succeeded) === false ? "error" : "ok",
+          ((_c = ctx.state.aiApiStreamResult) == null ? void 0 : _c.succeeded) === false ? "error" : "ok",
           Date.now() - t0
         );
         return;

package/dist/server/usage.js

@@ -30,7 +30,20 @@ __export(usage_exports, {
   startUsageRecord: () => startUsageRecord
 });
 module.exports = __toCommonJS(usage_exports);
-async function startUsageRecord(ctx, requestId, endpoint, model, streaming) {
+function normalizeUsage(value) {
+  if (!value || typeof value !== "object") return void 0;
+  const source = value;
+  const prompt = source.prompt_tokens ?? source.input_tokens;
+  const completion = source.completion_tokens ?? source.output_tokens;
+  const total = source.total_tokens;
+  if (prompt == null && completion == null && total == null) return void 0;
+  return {
+    prompt_tokens: typeof prompt === "number" ? prompt : null,
+    completion_tokens: typeof completion === "number" ? completion : null,
+    total_tokens: typeof total === "number" ? total : null
+  };
+}
+async function startUsageRecord(ctx, requestId, endpoint, model, streaming, mode) {
   var _a, _b;
   const body = ctx.request.body || {};
   const messages = Array.isArray(body.messages) ? body.messages : void 0;
@@ -45,7 +58,7 @@ async function startUsageRecord(ctx, requestId, endpoint, model, streaming) {
       oauthSubject: oauth == null ? void 0 : oauth.subject,
       oauthScopes: oauth == null ? void 0 : oauth.scopes,
       endpoint,
-      mode: ctx.get("X-AI-Mode") || void 0,
+      mode,
       model: model === "-" ? void 0 : model,
       status: "pending",
       streaming,
@@ -56,21 +69,21 @@ async function startUsageRecord(ctx, requestId, endpoint, model, streaming) {
   return record.id;
 }
 async function finishUsageRecord(ctx, id, startedAt, status) {
-  var _a;
+  var _a, _b;
   const response = ctx.body || {};
   const streamResult = ctx.state.aiApiStreamResult;
-  const usage = response.usage || (streamResult == null ? void 0 : streamResult.usage);
+  const usage = normalizeUsage(response.usage) || normalizeUsage((_a = response.response_metadata) == null ? void 0 : _a.usage) || normalizeUsage(streamResult == null ? void 0 : streamResult.usage);
   const values = {
     status: streamResult ? streamResult.succeeded ? "succeeded" : "failed" : status,
     httpStatus: ctx.status,
-    errorCode: ((_a = response.error) == null ? void 0 : _a.code) || (streamResult == null ? void 0 : streamResult.errorCode),
+    errorCode: ((_b = response.error) == null ? void 0 : _b.code) || (streamResult == null ? void 0 : streamResult.errorCode),
     inputTokens: usage == null ? void 0 : usage.prompt_tokens,
     outputTokens: usage == null ? void 0 : usage.completion_tokens,
     totalTokens: usage == null ? void 0 : usage.total_tokens,
     providerRequestId: response.id || (streamResult == null ? void 0 : streamResult.id),
     completedAt: /* @__PURE__ */ new Date(),
     durationMs: Date.now() - startedAt,
-    responseMetadata: { usageSource: usage ? "response" : "unavailable" }
+    responseMetadata: { usageSource: usage ? "provider" : "unavailable" }
   };
   await ctx.db.getRepository("aiApiUsageRecords").update({ filterByTk: id, values });
 }

package/dist/server/utils/openai-format.js

@@ -98,7 +98,7 @@ function toOpenAIResponse(options) {
         finish_reason: finishReason
       }
     ],
-    usage: usage || { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }
+    usage: usage || { prompt_tokens: null, completion_tokens: null, total_tokens: null }
   };
 }
 function toOpenAIStreamChunk(options) {
@@ -120,7 +120,7 @@ function toOpenAIStreamChunk(options) {
   };
 }
 function toOpenAIEmbeddingsResponse(options) {
-  const { model, embeddings, promptTokens = 0 } = options;
+  const { model, embeddings, promptTokens = null } = options;
   return {
     object: "list",
     data: embeddings.map((embedding, index) => ({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "plugin-ai-api",
-  "version": "1.0.11",
+  "version": "1.0.13",
   "main": "dist/server/index.js",
   "dependencies": {},
   "peerDependencies": {

package/src/server/__tests__/openai-format.test.ts

@@ -1,5 +1,6 @@
 import { toOpenAIResponse, toOpenAIStreamChunk } from '../utils/openai-format';
 import { isStreamingRequested } from '../utils/streaming';
+import { applyProviderRequestParameters, getProviderRequestParameters } from '../routes/chat-completions';
 
 describe('AI API OpenAI tool-call formatting', () => {
   it('streams by default and only disables streaming for an explicit false value', () => {
@@ -50,3 +51,58 @@ describe('AI API OpenAI tool-call formatting', () => {
     expect(chunk.choices[0].delta.tool_calls?.[0].function?.name).toBe('get_weather');
   });
 });
+
+describe('AI API provider parameter forwarding', () => {
+  it('forwards model and tool-call parameters not managed by the gateway', () => {
+    const parameters = getProviderRequestParameters({
+      model: 'service/model',
+      messages: [{ role: 'user', content: 'Use a tool' }],
+      tools: [{ type: 'function', function: { name: 'search' } }],
+      tool_choice: 'auto',
+      stream: true,
+      n: 1,
+      parallel_tool_calls: false,
+      reasoning_effort: 'medium',
+      max_completion_tokens: 4096,
+      seed: 7,
+      service_tier: 'default',
+    });
+
+    expect(parameters).toEqual({
+      parallel_tool_calls: false,
+      reasoning_effort: 'medium',
+      max_completion_tokens: 4096,
+      seed: 7,
+      service_tier: 'default',
+    });
+  });
+
+  it('merges passthrough parameters into model kwargs and removes the synthetic text response format', () => {
+    const model = {
+      modelKwargs: {
+        response_format: { type: 'text' },
+        existing_provider_option: true,
+      },
+    };
+
+    applyProviderRequestParameters(model, {
+      parallel_tool_calls: false,
+      reasoning_effort: 'high',
+    });
+
+    expect(model.modelKwargs).toEqual({
+      existing_provider_option: true,
+      parallel_tool_calls: false,
+      reasoning_effort: 'high',
+    });
+  });
+
+  it('preserves an explicitly requested response format', () => {
+    const model = { modelKwargs: { response_format: { type: 'text' } } };
+    const responseFormat = { type: 'json_schema', json_schema: { name: 'answer', schema: { type: 'object' } } };
+
+    applyProviderRequestParameters(model, { response_format: responseFormat });
+
+    expect(model.modelKwargs.response_format).toEqual(responseFormat);
+  });
+});

package/src/server/routes/chat-completions.ts

@@ -120,7 +120,8 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
       return;
     }
 
-    const modelOptions: Record<string, any> = {
+    const providerRequestParameters = getProviderRequestParameters(body);
+    const modelOptions: Record<string, unknown> = {
       model: modelId,
       llmService: service.name,
     };
@@ -128,7 +129,8 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
     // Pass through optional parameters
     if (body.temperature !== undefined) modelOptions.temperature = body.temperature;
     if (body.top_p !== undefined) modelOptions.topP = body.top_p;
-    if (body.max_tokens !== undefined) modelOptions.maxTokens = body.max_tokens;
+    if (body.max_completion_tokens !== undefined) modelOptions.maxTokens = body.max_completion_tokens;
+    else if (body.max_tokens !== undefined) modelOptions.maxTokens = body.max_tokens;
     if (body.frequency_penalty !== undefined) modelOptions.frequencyPenalty = body.frequency_penalty;
     if (body.presence_penalty !== undefined) modelOptions.presencePenalty = body.presence_penalty;
     if (body.stop !== undefined) modelOptions.stop = body.stop;
@@ -192,14 +194,29 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
 
     const completionId = generateCompletionId();
     const baseModel = provider.createModel();
-    const chatModel = bindRequestTools(baseModel, body.tools, body.tool_choice);
+    applyProviderRequestParameters(baseModel, providerRequestParameters);
+    const chatModel = bindRequestTools(baseModel, body.tools, body.tool_choice, providerRequestParameters);
 
     if (stream) {
       // ─── Streaming mode ───
-      await handleStreamingCompletion(ctx, chatModel, langchainMessages, completionId, body.model);
+      await handleStreamingCompletion(
+        ctx,
+        chatModel,
+        langchainMessages,
+        completionId,
+        body.model,
+        providerRequestParameters,
+      );
     } else {
       // ─── Non-streaming mode ───
-      await handleNonStreamingCompletion(ctx, chatModel, langchainMessages, completionId, body.model);
+      await handleNonStreamingCompletion(
+        ctx,
+        chatModel,
+        langchainMessages,
+        completionId,
+        body.model,
+        providerRequestParameters,
+      );
     }
   } catch (err) {
     ctx.log.error('AI API chat completions error:', err);
@@ -218,8 +235,9 @@ async function handleNonStreamingCompletion(
   messages: any[],
   completionId: string,
   modelName: string,
+  providerRequestParameters: Record<string, unknown>,
 ) {
-  const result = await chatModel.invoke(messages);
+  const result = await chatModel.invoke(messages, providerRequestParameters);
 
   let content = '';
   if (typeof result.content === 'string') {
@@ -258,6 +276,7 @@ async function handleStreamingCompletion(
   messages: any[],
   completionId: string,
   modelName: string,
+  providerRequestParameters: Record<string, unknown>,
 ) {
   // Set SSE headers
   ctx.set({
@@ -284,7 +303,7 @@ async function handleStreamingCompletion(
   let usage: { prompt_tokens: number; completion_tokens: number; total_tokens: number } | undefined;
   let finishReason = 'stop';
   try {
-    const stream = await chatModel.stream(messages, { signal: requestAbort.signal });
+    const stream = await chatModel.stream(messages, { ...providerRequestParameters, signal: requestAbort.signal });
 
     for await (const chunk of stream) {
       if (requestAbort.signal.aborted) throw requestAbort.signal.reason;
@@ -367,12 +386,62 @@ function getErrorMessage(error: unknown, fallback: string) {
   return error instanceof Error && error.message ? error.message : fallback;
 }
 
-function bindRequestTools(chatModel: any, tools: unknown, toolChoice: unknown) {
+const GATEWAY_MANAGED_PARAMETERS = new Set(['model', 'messages', 'tools', 'tool_choice', 'stream', 'n']);
+
+export function getProviderRequestParameters(body: Record<string, unknown>): Record<string, unknown> {
+  return Object.fromEntries(
+    Object.entries(body).filter(([name, value]) => !GATEWAY_MANAGED_PARAMETERS.has(name) && value !== undefined),
+  );
+}
+
+interface ModelWithKwargs {
+  modelKwargs?: Record<string, unknown>;
+}
+
+export function applyProviderRequestParameters(chatModel: unknown, parameters: Record<string, unknown>): void {
+  if (!chatModel || typeof chatModel !== 'object') return;
+
+  const model = chatModel as ModelWithKwargs;
+  const modelKwargs = { ...(model.modelKwargs ?? {}) };
+
+  // OpenAI providers currently install a synthetic text response format even when
+  // the client did not request one. Remove that default so LLM mode matches the
+  // original OpenAI-compatible request more closely.
+  if (!Object.hasOwn(parameters, 'response_format')) {
+    if (isDefaultTextResponseFormat(modelKwargs.response_format)) delete modelKwargs.response_format;
+    if (isDefaultResponsesTextFormat(modelKwargs.text)) delete modelKwargs.text;
+  }
+
+  model.modelKwargs = { ...modelKwargs, ...parameters };
+}
+
+function bindRequestTools(
+  chatModel: any,
+  tools: unknown,
+  toolChoice: unknown,
+  providerRequestParameters: Record<string, unknown>,
+) {
   if (!Array.isArray(tools) || tools.length === 0) return chatModel;
   if (typeof chatModel.bindTools !== 'function') {
     throw new Error('The selected LLM provider does not support tool calling');
   }
-  return chatModel.bindTools(tools, toolChoice === undefined ? undefined : { tool_choice: toolChoice });
+  return chatModel.bindTools(tools, {
+    ...providerRequestParameters,
+    ...(toolChoice === undefined ? {} : { tool_choice: toolChoice }),
+  });
+}
+
+function isDefaultTextResponseFormat(value: unknown): boolean {
+  return isRecord(value) && value.type === 'text' && Object.keys(value).length === 1;
+}
+
+function isDefaultResponsesTextFormat(value: unknown): boolean {
+  if (!isRecord(value) || !isRecord(value.format)) return false;
+  return value.format.type === 'text' && Object.keys(value.format).length === 1 && Object.keys(value).length === 1;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
 
 function normalizeToolCalls(value: unknown): OpenAIToolCall[] | undefined {

package/src/server/routes/embeddings.ts

@@ -1,191 +1,191 @@
-/**
- * This file is part of the NocoBase (R) project.
- * Copyright (c) 2020-2024 NocoBase Co., Ltd.
- * Authors: NocoBase Team.
- *
- * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
- * For more information, please refer to: https://www.nocobase.com/agreement.
- */
-
-import { Context } from '@nocobase/actions';
-import { toOpenAIError, toOpenAIEmbeddingsResponse } from '../utils/openai-format';
-import { resolveModelString } from '../utils/resolve-service';
-import type PluginAiApiServer from '../plugin';
-
-/**
- * POST /api/ai-llm/v1/embeddings
- *
- * OpenAI-compatible embeddings endpoint.
- *
- * Supported providers (those with an `embedding` field in LLMProviderMeta):
- *   - openai            → OpenAiEmbeddingProvider
- *   - openai-completions → OpenAiEmbeddingProvider
- *   - dashscope         → DashscopeEmbeddingProvider
- *   - google-genai      → GoogleGenAIEmbeddingProvider
- *   - ollama            → OllamaEmbeddingProvider
- *
- * Not supported: anthropic, deepseek, kimi (no embedding provider registered).
- *
- * Limitations:
- * - encoding_format 'base64' is not supported (always returns float arrays)
- * - Token counts always return 0 (LangChain embeddings API doesn't expose this)
- * - Token array input (integer[]) is not supported, only string input
- */
-export async function handleEmbeddings(ctx: Context, plugin: PluginAiApiServer) {
-  const body = ctx.request.body as any;
-
-  // ─── Validate ────────────────────────────────────────────────────────────
-  if (!body?.model) {
-    ctx.status = 400;
-    ctx.body = toOpenAIError(400, "'model' is required", 'invalid_request_error', 'missing_model');
-    return;
-  }
-
-  if (body.input === undefined || body.input === null) {
-    ctx.status = 400;
-    ctx.body = toOpenAIError(400, "'input' is required", 'invalid_request_error', 'missing_input');
-    return;
-  }
-
-  if (body.encoding_format === 'base64') {
-    ctx.status = 400;
-    ctx.body = toOpenAIError(
-      400,
-      "encoding_format 'base64' is not supported. Use 'float' (default) or omit the parameter.",
-      'invalid_request_error',
-      'unsupported_encoding_format',
-    );
-    return;
-  }
-
-  // ─── Normalize input to string[] ─────────────────────────────────────────
-  let inputs: string[];
-  if (typeof body.input === 'string') {
-    inputs = [body.input];
-  } else if (Array.isArray(body.input)) {
-    if (body.input.length === 0) {
-      ctx.status = 400;
-      ctx.body = toOpenAIError(400, "'input' array must not be empty", 'invalid_request_error');
-      return;
-    }
-    if (typeof body.input[0] === 'number') {
-      ctx.status = 400;
-      ctx.body = toOpenAIError(
-        400,
-        'Token array input is not supported. Please provide string input.',
-        'invalid_request_error',
-        'unsupported_input_type',
-      );
-      return;
-    }
-    inputs = body.input as string[];
-  } else {
-    ctx.status = 400;
-    ctx.body = toOpenAIError(400, "'input' must be a string or array of strings", 'invalid_request_error');
-    return;
-  }
-
-  // ─── Resolve model ────────────────────────────────────────────────────────
-  const resolved = await resolveModelString(ctx, body.model);
-  if (!resolved) {
-    ctx.status = 404;
-    ctx.body = toOpenAIError(
-      404,
-      `Could not resolve model '${body.model}'. Use GET /v1/models to list available models.`,
-      'invalid_request_error',
-      'model_not_found',
-    );
-    return;
-  }
-
-  const { service, modelId } = resolved;
-
-  if (service.enabled === false) {
-    ctx.status = 404;
-    ctx.body = toOpenAIError(
-      404,
-      `LLM service '${service.title || service.name}' is disabled`,
-      'invalid_request_error',
-      'model_not_found',
-    );
-    return;
-  }
-
-  // ─── Check service whitelist ──────────────────────────────────────────────
-  try {
-    const config = await ctx.db.getRepository('aiApiConfig').findOne();
-    if (config?.enabledLlmServices?.length) {
-      const allowed = config.enabledLlmServices.some((s: string) => s === service.name || s === service.title);
-      if (!allowed) {
-        ctx.status = 403;
-        ctx.body = toOpenAIError(
-          403,
-          `LLM service '${service.title || service.name}' is not enabled for API access`,
-          'invalid_request_error',
-          'model_not_available',
-        );
-        return;
-      }
-    }
-  } catch {
-    // Config read failure: fail open
-  }
-
-  // ─── Get embedding provider ───────────────────────────────────────────────
-  const aiPlugin = ctx.app.pm.get('ai') as any;
-  if (!aiPlugin) {
-    ctx.status = 500;
-    ctx.body = toOpenAIError(500, 'AI plugin not available', 'server_error');
-    return;
-  }
-
-  const providerMeta = aiPlugin.aiManager.llmProviders.get(service.provider);
-  if (!providerMeta) {
-    ctx.status = 500;
-    ctx.body = toOpenAIError(500, `Provider '${service.provider}' not registered`, 'server_error');
-    return;
-  }
-
-  // providerMeta.embedding is the EmbeddingProvider constructor (if supported by this provider)
-  if (!providerMeta.embedding) {
-    ctx.status = 400;
-    ctx.body = toOpenAIError(
-      400,
-      `Provider '${providerMeta.title || service.provider}' does not support embeddings. ` +
-        `Embedding-capable providers: openai, openai-completions, dashscope, google-genai, ollama.`,
-      'invalid_request_error',
-      'model_not_supported',
-    );
-    return;
-  }
-
-  try {
-    // ─── Instantiate and call the embedding provider ──────────────────────
-    const EmbeddingClass = providerMeta.embedding;
-    const embeddingProvider = new EmbeddingClass({
-      app: ctx.app,
-      serviceOptions: service.options, // Contains apiKey, baseURL, etc.
-      modelOptions: { model: modelId }, // The specific embedding model
-    });
-
-    // createEmbedding() returns a LangChain EmbeddingsInterface.
-    // embedDocuments() accepts string[] and returns number[][] (one vector per input).
-    const embeddingModel = embeddingProvider.createEmbedding();
-    const vectors: number[][] = await embeddingModel.embedDocuments(inputs);
-
-    ctx.status = 200;
-    ctx.set('Content-Type', 'application/json');
-    ctx.body = toOpenAIEmbeddingsResponse({
-      model: body.model,
-      embeddings: vectors,
-      // LangChain's EmbeddingsInterface does not expose token counts.
-      promptTokens: 0,
-    });
-  } catch (err) {
-    ctx.log.error('AI API embeddings error:', err);
-    if (!ctx.res.headersSent) {
-      ctx.status = 500;
-      ctx.body = toOpenAIError(500, err.message || 'Failed to generate embeddings', 'server_error');
-    }
-  }
-}
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Context } from '@nocobase/actions';
+import { toOpenAIError, toOpenAIEmbeddingsResponse } from '../utils/openai-format';
+import { resolveModelString } from '../utils/resolve-service';
+import type PluginAiApiServer from '../plugin';
+
+/**
+ * POST /api/ai-llm/v1/embeddings
+ *
+ * OpenAI-compatible embeddings endpoint.
+ *
+ * Supported providers (those with an `embedding` field in LLMProviderMeta):
+ *   - openai            → OpenAiEmbeddingProvider
+ *   - openai-completions → OpenAiEmbeddingProvider
+ *   - dashscope         → DashscopeEmbeddingProvider
+ *   - google-genai      → GoogleGenAIEmbeddingProvider
+ *   - ollama            → OllamaEmbeddingProvider
+ *
+ * Not supported: anthropic, deepseek, kimi (no embedding provider registered).
+ *
+ * Limitations:
+ * - encoding_format 'base64' is not supported (always returns float arrays)
+ * - Token counts always return 0 (LangChain embeddings API doesn't expose this)
+ * - Token array input (integer[]) is not supported, only string input
+ */
+export async function handleEmbeddings(ctx: Context, plugin: PluginAiApiServer) {
+  const body = ctx.request.body as any;
+
+  // ─── Validate ────────────────────────────────────────────────────────────
+  if (!body?.model) {
+    ctx.status = 400;
+    ctx.body = toOpenAIError(400, "'model' is required", 'invalid_request_error', 'missing_model');
+    return;
+  }
+
+  if (body.input === undefined || body.input === null) {
+    ctx.status = 400;
+    ctx.body = toOpenAIError(400, "'input' is required", 'invalid_request_error', 'missing_input');
+    return;
+  }
+
+  if (body.encoding_format === 'base64') {
+    ctx.status = 400;
+    ctx.body = toOpenAIError(
+      400,
+      "encoding_format 'base64' is not supported. Use 'float' (default) or omit the parameter.",
+      'invalid_request_error',
+      'unsupported_encoding_format',
+    );
+    return;
+  }
+
+  // ─── Normalize input to string[] ─────────────────────────────────────────
+  let inputs: string[];
+  if (typeof body.input === 'string') {
+    inputs = [body.input];
+  } else if (Array.isArray(body.input)) {
+    if (body.input.length === 0) {
+      ctx.status = 400;
+      ctx.body = toOpenAIError(400, "'input' array must not be empty", 'invalid_request_error');
+      return;
+    }
+    if (typeof body.input[0] === 'number') {
+      ctx.status = 400;
+      ctx.body = toOpenAIError(
+        400,
+        'Token array input is not supported. Please provide string input.',
+        'invalid_request_error',
+        'unsupported_input_type',
+      );
+      return;
+    }
+    inputs = body.input as string[];
+  } else {
+    ctx.status = 400;
+    ctx.body = toOpenAIError(400, "'input' must be a string or array of strings", 'invalid_request_error');
+    return;
+  }
+
+  // ─── Resolve model ────────────────────────────────────────────────────────
+  const resolved = await resolveModelString(ctx, body.model);
+  if (!resolved) {
+    ctx.status = 404;
+    ctx.body = toOpenAIError(
+      404,
+      `Could not resolve model '${body.model}'. Use GET /v1/models to list available models.`,
+      'invalid_request_error',
+      'model_not_found',
+    );
+    return;
+  }
+
+  const { service, modelId } = resolved;
+
+  if (service.enabled === false) {
+    ctx.status = 404;
+    ctx.body = toOpenAIError(
+      404,
+      `LLM service '${service.title || service.name}' is disabled`,
+      'invalid_request_error',
+      'model_not_found',
+    );
+    return;
+  }
+
+  // ─── Check service whitelist ──────────────────────────────────────────────
+  try {
+    const config = await ctx.db.getRepository('aiApiConfig').findOne();
+    if (config?.enabledLlmServices?.length) {
+      const allowed = config.enabledLlmServices.some((s: string) => s === service.name || s === service.title);
+      if (!allowed) {
+        ctx.status = 403;
+        ctx.body = toOpenAIError(
+          403,
+          `LLM service '${service.title || service.name}' is not enabled for API access`,
+          'invalid_request_error',
+          'model_not_available',
+        );
+        return;
+      }
+    }
+  } catch {
+    // Config read failure: fail open
+  }
+
+  // ─── Get embedding provider ───────────────────────────────────────────────
+  const aiPlugin = ctx.app.pm.get('ai') as any;
+  if (!aiPlugin) {
+    ctx.status = 500;
+    ctx.body = toOpenAIError(500, 'AI plugin not available', 'server_error');
+    return;
+  }
+
+  const providerMeta = aiPlugin.aiManager.llmProviders.get(service.provider);
+  if (!providerMeta) {
+    ctx.status = 500;
+    ctx.body = toOpenAIError(500, `Provider '${service.provider}' not registered`, 'server_error');
+    return;
+  }
+
+  // providerMeta.embedding is the EmbeddingProvider constructor (if supported by this provider)
+  if (!providerMeta.embedding) {
+    ctx.status = 400;
+    ctx.body = toOpenAIError(
+      400,
+      `Provider '${providerMeta.title || service.provider}' does not support embeddings. ` +
+        `Embedding-capable providers: openai, openai-completions, dashscope, google-genai, ollama.`,
+      'invalid_request_error',
+      'model_not_supported',
+    );
+    return;
+  }
+
+  try {
+    // ─── Instantiate and call the embedding provider ──────────────────────
+    const EmbeddingClass = providerMeta.embedding;
+    const embeddingProvider = new EmbeddingClass({
+      app: ctx.app,
+      serviceOptions: service.options, // Contains apiKey, baseURL, etc.
+      modelOptions: { model: modelId }, // The specific embedding model
+    });
+
+    // createEmbedding() returns a LangChain EmbeddingsInterface.
+    // embedDocuments() accepts string[] and returns number[][] (one vector per input).
+    const embeddingModel = embeddingProvider.createEmbedding();
+    const vectors: number[][] = await embeddingModel.embedDocuments(inputs);
+
+    ctx.status = 200;
+    ctx.set('Content-Type', 'application/json');
+    ctx.body = toOpenAIEmbeddingsResponse({
+      model: body.model,
+      embeddings: vectors,
+      // LangChain's EmbeddingsInterface does not expose token counts.
+      promptTokens: null,
+    });
+  } catch (err) {
+    ctx.log.error('AI API embeddings error:', err);
+    if (!ctx.res.headersSent) {
+      ctx.status = 500;
+      ctx.body = toOpenAIError(500, err.message || 'Failed to generate embeddings', 'server_error');
+    }
+  }
+}

package/src/server/routes/router.ts

@@ -19,6 +19,7 @@ import { toOpenAIError } from '../utils/openai-format';
 import { createRateLimitMiddleware } from '../middleware/rate-limit';
 import { checkRolePermission } from '../middleware/role-permission';
 import { startUsageRecord, finishUsageRecord } from '../usage';
+import { isStreamingRequested } from '../utils/streaming';
 import type PluginAiApiServer from '../plugin';
 
 const API_PREFIX = '/api/ai-llm/v1';
@@ -120,16 +121,27 @@ export function createAiLlmRouter(plugin: PluginAiApiServer) {
 
     // ─── Route matching ───────────────────────────────────────────────────
     const model = (ctx.request.body as any)?.model ?? '-';
+    const isUsageEndpoint =
+      method === 'POST' && (subPath === '/chat/completions' || subPath === '/completions' || subPath === '/embeddings');
+    const resolvedMode = isUsageEndpoint ? await resolveMode(ctx) : 'llm';
+    const requestBody = (ctx.request.body || {}) as Record<string, unknown>;
+    const streaming = isUsageEndpoint && isStreamingRequested(requestBody.stream);
+    if (streaming && (subPath === '/chat/completions' || subPath === '/completions')) {
+      const streamOptions = requestBody.stream_options;
+      ctx.request.body = {
+        ...requestBody,
+        stream_options: {
+          ...(streamOptions && typeof streamOptions === 'object' ? streamOptions : {}),
+          include_usage: true,
+        },
+      };
+    }
     const t0 = Date.now();
     let usageId: unknown;
     try {
-      usageId = await startUsageRecord(
-        ctx,
-        requestId,
-        subPath,
-        String(model),
-        Boolean((ctx.request.body as any)?.stream),
-      );
+      usageId = isUsageEndpoint
+        ? await startUsageRecord(ctx, requestId, subPath, String(model), streaming, resolvedMode)
+        : undefined;
     } catch (usageError) {
       ctx.log.error('AI API usage record could not be created:', usageError);
     }
@@ -137,8 +149,7 @@ export function createAiLlmRouter(plugin: PluginAiApiServer) {
     try {
       // POST /v1/chat/completions — route based on mode
       if (method === 'POST' && subPath === '/chat/completions') {
-        const mode = await resolveMode(ctx);
-        await (mode === 'agent' ? handleAgentCompletions(ctx, plugin) : handleChatCompletions(ctx, plugin));
+        await (resolvedMode === 'agent' ? handleAgentCompletions(ctx, plugin) : handleChatCompletions(ctx, plugin));
         logRequest(
           ctx,
           requestId,
@@ -158,7 +169,7 @@ export function createAiLlmRouter(plugin: PluginAiApiServer) {
 
       // POST /v1/completions (legacy text completions — used by LiteLLM)
       if (method === 'POST' && subPath === '/completions') {
-        const completionsMode = await resolveMode(ctx);
+        const completionsMode = resolvedMode;
         if (completionsMode === 'agent') {
           // Convert legacy prompt → messages format for agent handler
           const reqBody = ctx.request.body as any;

package/src/server/usage.ts

@@ -1,6 +1,20 @@
 import { Context } from '@nocobase/actions';
 
-type Usage = { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number };
+export type Usage = { prompt_tokens?: number | null; completion_tokens?: number | null; total_tokens?: number | null };
+
+function normalizeUsage(value: unknown): Usage | undefined {
+  if (!value || typeof value !== 'object') return undefined;
+  const source = value as Record<string, unknown>;
+  const prompt = source.prompt_tokens ?? source.input_tokens;
+  const completion = source.completion_tokens ?? source.output_tokens;
+  const total = source.total_tokens;
+  if (prompt == null && completion == null && total == null) return undefined;
+  return {
+    prompt_tokens: typeof prompt === 'number' ? prompt : null,
+    completion_tokens: typeof completion === 'number' ? completion : null,
+    total_tokens: typeof total === 'number' ? total : null,
+  };
+}
 
 export async function startUsageRecord(
   ctx: Context,
@@ -8,6 +22,7 @@ export async function startUsageRecord(
   endpoint: string,
   model: string,
   streaming: boolean,
+  mode: 'llm' | 'agent',
 ) {
   const body = (ctx.request.body || {}) as Record<string, unknown>;
   const messages = Array.isArray(body.messages) ? body.messages : undefined;
@@ -22,7 +37,7 @@ export async function startUsageRecord(
       oauthSubject: oauth?.subject,
       oauthScopes: oauth?.scopes,
       endpoint,
-      mode: ctx.get('X-AI-Mode') || undefined,
+      mode,
       model: model === '-' ? undefined : model,
       status: 'pending',
       streaming,
@@ -34,11 +49,19 @@ export async function startUsageRecord(
 }
 
 export async function finishUsageRecord(ctx: Context, id: unknown, startedAt: number, status: 'succeeded' | 'failed') {
-  const response = (ctx.body || {}) as { usage?: Usage; id?: string; error?: { code?: string } };
+  const response = (ctx.body || {}) as {
+    usage?: Usage;
+    response_metadata?: { usage?: unknown };
+    id?: string;
+    error?: { code?: string };
+  };
   const streamResult = ctx.state.aiApiStreamResult as
     | { usage?: Usage; id?: string; errorCode?: string; succeeded: boolean }
     | undefined;
-  const usage = response.usage || streamResult?.usage;
+  const usage =
+    normalizeUsage(response.usage) ||
+    normalizeUsage(response.response_metadata?.usage) ||
+    normalizeUsage(streamResult?.usage);
   const values = {
     status: streamResult ? (streamResult.succeeded ? 'succeeded' : 'failed') : status,
     httpStatus: ctx.status,
@@ -49,7 +72,7 @@ export async function finishUsageRecord(ctx: Context, id: unknown, startedAt: nu
     providerRequestId: response.id || streamResult?.id,
     completedAt: new Date(),
     durationMs: Date.now() - startedAt,
-    responseMetadata: { usageSource: usage ? 'response' : 'unavailable' },
+    responseMetadata: { usageSource: usage ? 'provider' : 'unavailable' },
   };
   await ctx.db.getRepository('aiApiUsageRecords').update({ filterByTk: id, values });
 }

package/src/server/utils/openai-format.ts

@@ -87,7 +87,7 @@ export function toOpenAIResponse(options: {
         finish_reason: finishReason,
       },
     ],
-    usage: usage || { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
+    usage: usage || { prompt_tokens: null, completion_tokens: null, total_tokens: null },
   };
 }
 
@@ -132,8 +132,12 @@ export interface OpenAIToolCallChunk {
 
 // ─── OpenAI Embeddings response ───
 
-export function toOpenAIEmbeddingsResponse(options: { model: string; embeddings: number[][]; promptTokens?: number }) {
-  const { model, embeddings, promptTokens = 0 } = options;
+export function toOpenAIEmbeddingsResponse(options: {
+  model: string;
+  embeddings: number[][];
+  promptTokens?: number | null;
+}) {
+  const { model, embeddings, promptTokens = null } = options;
   return {
     object: 'list' as const,
     data: embeddings.map((embedding, index) => ({