plugin-ai-api 1.0.24 → 1.0.25

package/src/server/collections/ai-api-usage-records.ts

@@ -32,6 +32,7 @@ export default defineCollection({
     { name: 'inputTokens', type: 'integer', allowNull: true },
     { name: 'outputTokens', type: 'integer', allowNull: true },
     { name: 'totalTokens', type: 'integer', allowNull: true },
+    { name: 'promptCacheTokens', type: 'integer', allowNull: true },
     { name: 'estimatedCost', type: 'decimal', allowNull: true, precision: 20, scale: 8 },
     { name: 'currency', type: 'string', allowNull: true },
     { name: 'costStatus', type: 'string', allowNull: true, index: true },

package/src/server/index.ts

@@ -1,10 +1,10 @@
-/**
- * This file is part of the NocoBase (R) project.
- * Copyright (c) 2020-2024 NocoBase Co., Ltd.
- * Authors: NocoBase Team.
- *
- * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
- * For more information, please refer to: https://www.nocobase.com/agreement.
- */
-
-export { default } from './plugin';
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+export { default } from './plugin';

package/src/server/middleware/rate-limit.ts

@@ -1,70 +1,70 @@
-/**
- * This file is part of the NocoBase (R) project.
- * Copyright (c) 2020-2024 NocoBase Co., Ltd.
- * Authors: NocoBase Team.
- *
- * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
- * For more information, please refer to: https://www.nocobase.com/agreement.
- */
-
-import { Context } from '@nocobase/actions';
-import { RateLimiter } from '../utils/rate-limiter';
-import { toOpenAIError } from '../utils/openai-format';
-
-/**
- * Creates a rate-limiting check function for use in the AI API router.
- *
- * Must be called AFTER authenticateBearer() so ctx.state.currentUser is set.
- * Reads rateLimitPerMinute from aiApiConfig on each request (cheap single-row read,
- * allows config changes to take effect immediately without restart).
- * Falls back to 60 req/min if the config record is missing or the field is 0/null.
- *
- * Returns false (and writes the 429 response) when the rate limit is exceeded.
- * Returns true when the request is allowed.
- *
- * Sets OpenAI-compatible rate limit response headers on every request:
- *   X-RateLimit-Limit: <limit>
- *   X-RateLimit-Remaining: <remaining> (on 429: 0)
- *   Retry-After: <seconds>  (on 429 only)
- */
-export function createRateLimitMiddleware(limiter: RateLimiter) {
-  return async (ctx: Context): Promise<boolean> => {
-    const userId = ctx.state.currentUser?.id;
-    // Auth runs before this; if somehow missing, fail open (don't block the request).
-    if (userId === undefined || userId === null) return true;
-
-    let limit = 60;
-    try {
-      const config = await ctx.db.getRepository('aiApiConfig').findOne();
-      const configLimit = config?.rateLimitPerMinute;
-      if (configLimit && configLimit > 0) {
-        limit = configLimit;
-      }
-    } catch (configErr) {
-      // Config read failure: fail open — don't block legitimate requests
-      // Log at WARN so admins can detect DB connectivity issues
-      ctx.app?.logger?.warn('[ai-api] Rate limit config read failed, using default (60/min)', configErr);
-    }
-
-    const result = limiter.check(userId, limit);
-
-    if (!result.allowed) {
-      const retryAfterSec = Math.ceil((result as any).retryAfterMs / 1000);
-      ctx.set('Retry-After', String(retryAfterSec));
-      ctx.set('X-RateLimit-Limit', String(limit));
-      ctx.set('X-RateLimit-Remaining', '0');
-      ctx.status = 429;
-      ctx.body = toOpenAIError(
-        429,
-        `Rate limit exceeded. You have used all ${limit} requests allowed per minute. ` +
-          `Please wait ${retryAfterSec} second${retryAfterSec !== 1 ? 's' : ''} before retrying.`,
-        'requests',
-        'rate_limit_exceeded',
-      );
-      return false;
-    }
-
-    ctx.set('X-RateLimit-Limit', String(limit));
-    return true;
-  };
-}
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import { Context } from '@nocobase/actions';
+import { RateLimiter } from '../utils/rate-limiter';
+import { toOpenAIError } from '../utils/openai-format';
+
+/**
+ * Creates a rate-limiting check function for use in the AI API router.
+ *
+ * Must be called AFTER authenticateBearer() so ctx.state.currentUser is set.
+ * Reads rateLimitPerMinute from aiApiConfig on each request (cheap single-row read,
+ * allows config changes to take effect immediately without restart).
+ * Falls back to 60 req/min if the config record is missing or the field is 0/null.
+ *
+ * Returns false (and writes the 429 response) when the rate limit is exceeded.
+ * Returns true when the request is allowed.
+ *
+ * Sets OpenAI-compatible rate limit response headers on every request:
+ *   X-RateLimit-Limit: <limit>
+ *   X-RateLimit-Remaining: <remaining> (on 429: 0)
+ *   Retry-After: <seconds>  (on 429 only)
+ */
+export function createRateLimitMiddleware(limiter: RateLimiter) {
+  return async (ctx: Context): Promise<boolean> => {
+    const userId = ctx.state.currentUser?.id;
+    // Auth runs before this; if somehow missing, fail open (don't block the request).
+    if (userId === undefined || userId === null) return true;
+
+    let limit = 60;
+    try {
+      const config = await ctx.db.getRepository('aiApiConfig').findOne();
+      const configLimit = config?.rateLimitPerMinute;
+      if (configLimit && configLimit > 0) {
+        limit = configLimit;
+      }
+    } catch (configErr) {
+      // Config read failure: fail open — don't block legitimate requests
+      // Log at WARN so admins can detect DB connectivity issues
+      ctx.app?.logger?.warn('[ai-api] Rate limit config read failed, using default (60/min)', configErr);
+    }
+
+    const result = limiter.check(userId, limit);
+
+    if (!result.allowed) {
+      const retryAfterSec = Math.ceil((result as any).retryAfterMs / 1000);
+      ctx.set('Retry-After', String(retryAfterSec));
+      ctx.set('X-RateLimit-Limit', String(limit));
+      ctx.set('X-RateLimit-Remaining', '0');
+      ctx.status = 429;
+      ctx.body = toOpenAIError(
+        429,
+        `Rate limit exceeded. You have used all ${limit} requests allowed per minute. ` +
+          `Please wait ${retryAfterSec} second${retryAfterSec !== 1 ? 's' : ''} before retrying.`,
+        'requests',
+        'rate_limit_exceeded',
+      );
+      return false;
+    }
+
+    ctx.set('X-RateLimit-Limit', String(limit));
+    return true;
+  };
+}

package/src/server/migrations/20260813000000-add-prompt-cache-tokens.ts

@@ -0,0 +1,46 @@
+import { Migration } from '@nocobase/server';
+
+export default class AddPromptCacheTokensToUsageRecords extends Migration {
+  on = 'beforeLoad' as const;
+
+  async up() {
+    const collection = this.db.getCollection('aiApiUsageRecords');
+    if (!collection) return;
+
+    const field = collection.getField('promptCacheTokens');
+    if (!field) {
+      collection.addField('promptCacheTokens', { type: 'integer', allowNull: true });
+    }
+
+    if (await collection.existsInDb()) {
+      const tableName = collection.getTableNameWithSchema();
+      const exists = await this.tableColumnExists(tableName, 'promptCacheTokens');
+      if (!exists) {
+        await this.queryInterface.addColumn(tableName, 'promptCacheTokens', {
+          type: this.db.sequelize.getDialect() === 'sqlite' ? 'INTEGER' : 'INTEGER',
+          allowNull: true,
+        });
+      }
+    }
+  }
+
+  async down() {
+    const collection = this.db.getCollection('aiApiUsageRecords');
+    if (!collection) return;
+
+    if (await collection.existsInDb()) {
+      const tableName = collection.getTableNameWithSchema();
+      const exists = await this.tableColumnExists(tableName, 'promptCacheTokens');
+      if (exists) {
+        await this.queryInterface.removeColumn(tableName, 'promptCacheTokens');
+      }
+    }
+
+    collection.removeField('promptCacheTokens');
+  }
+
+  private async tableColumnExists(tableName: string, columnName: string): Promise<boolean> {
+    const columns = await this.queryInterface.describeTable(tableName);
+    return Object.prototype.hasOwnProperty.call(columns, columnName);
+  }
+}

package/src/server/plugin.ts

@@ -18,6 +18,12 @@ import { invalidateRolePermissionCache } from './middleware/role-permission';
 import { invalidateUserPermissionCache } from './utils/user-permissions';
 import { validateModelPrice, validateModelMetadata, validateQuotaPolicy } from './validation';
 import { AI_API_ACL_SNIPPET, AI_API_USER_PERMISSIONS_SNIPPET } from '../constants';
+import {
+  FileProcessorService,
+  base64FileForwarder,
+  httpFileUrlFetcher,
+  pdfFileProcessor,
+} from './services/file-processor';
 
 // Ensure dayjs timezone + utc plugins are loaded.
 // Some Docker builds ship an older @nocobase/utils whose dayjs.js does not
@@ -39,6 +45,12 @@ export class PluginAiApiServer extends Plugin {
    */
   rateLimiter = new RateLimiter(60_000);
 
+  /**
+   * Extensible file processor service. Other plugins can register custom processors
+   * to transform file/file_url content blocks before they reach the LLM.
+   */
+  fileProcessorService = new FileProcessorService();
+
   private gcInterval: NodeJS.Timeout | null = null;
 
   async afterAdd() {}
@@ -56,6 +68,13 @@ export class PluginAiApiServer extends Plugin {
   }
 
   async load() {
+    // Register default file processors. Custom plugins can register additional
+    // processors by retrieving this plugin instance and calling
+    // `fileProcessorService.register(processor)`.
+    this.fileProcessorService.register(base64FileForwarder);
+    this.fileProcessorService.register(httpFileUrlFetcher);
+    this.fileProcessorService.register(pdfFileProcessor);
+
     // 1. Claim body parsing for our own routes before the core bodyParser runs.
     //    Core registers koa-bodyparser with a global REQUEST_BODY_LIMIT (10mb by
     //    default) much earlier in the stack, so without this the gateway's own
@@ -159,6 +178,7 @@ export class PluginAiApiServer extends Plugin {
           enabledLlmServices: [],
           rateLimitPerMinute: 60,
           quotaEnabled: false,
+          pdfRenderPagesAsImages: false,
           defaultReservationOutputTokens: 4096,
         },
       });

package/src/server/resource/ai-api-config.ts

@@ -48,6 +48,7 @@ const aiApiConfigResource: ResourceOptions = {
             enabledLlmServices: [],
             rateLimitPerMinute: 60,
             maxRequestBodyMb: 10,
+            pdfRenderPagesAsImages: false,
             quotaEnabled: false,
             defaultReservationOutputTokens: 4096,
             options: {},
@@ -72,6 +73,7 @@ const aiApiConfigResource: ResourceOptions = {
             enabledLlmServices: values.enabledLlmServices ?? [],
             rateLimitPerMinute: values.rateLimitPerMinute ?? 60,
             maxRequestBodyMb: coerceMaxRequestBodyMb(values.maxRequestBodyMb ?? DEFAULT_MAX_REQUEST_BODY_MB),
+            pdfRenderPagesAsImages: values.pdfRenderPagesAsImages ?? false,
             quotaEnabled: values.quotaEnabled ?? false,
             defaultReservationOutputTokens: values.defaultReservationOutputTokens ?? 4096,
             options: values.options ?? {},
@@ -87,6 +89,9 @@ const aiApiConfigResource: ResourceOptions = {
         if (values.maxRequestBodyMb !== undefined) {
           updateData.maxRequestBodyMb = coerceMaxRequestBodyMb(values.maxRequestBodyMb);
         }
+        if (values.pdfRenderPagesAsImages !== undefined) {
+          updateData.pdfRenderPagesAsImages = Boolean(values.pdfRenderPagesAsImages);
+        }
         if (values.quotaEnabled !== undefined) updateData.quotaEnabled = values.quotaEnabled;
         if (values.defaultReservationOutputTokens !== undefined) {
           updateData.defaultReservationOutputTokens = values.defaultReservationOutputTokens;

package/src/server/resource/ai-api-usage-monitor.ts

@@ -7,6 +7,7 @@ interface UsageSummaryRow {
   inputTokens?: string | number;
   outputTokens?: string | number;
   totalTokens?: string | number;
+  promptCacheTokens?: string | number;
 }
 
 interface CostSummaryRow {
@@ -45,6 +46,7 @@ const aiApiUsageMonitorResource: ResourceOptions = {
           [fn('COALESCE', fn('SUM', col('inputTokens')), 0), 'inputTokens'],
           [fn('COALESCE', fn('SUM', col('outputTokens')), 0), 'outputTokens'],
           [fn('COALESCE', fn('SUM', col('totalTokens')), 0), 'totalTokens'],
+          [fn('COALESCE', fn('SUM', col('promptCacheTokens')), 0), 'promptCacheTokens'],
         ],
         where,
         raw: true,
@@ -61,6 +63,7 @@ const aiApiUsageMonitorResource: ResourceOptions = {
         inputTokens: Number(totals?.inputTokens ?? 0),
         outputTokens: Number(totals?.outputTokens ?? 0),
         totalTokens: Number(totals?.totalTokens ?? 0),
+        promptCacheTokens: Number(totals?.promptCacheTokens ?? 0),
         costsByCurrency: costs.map((item) => ({
           currency: item.currency || 'USD',
           totalCost: String(item.totalCost ?? 0),

package/src/server/routes/chat-completions.ts

@@ -33,6 +33,7 @@ import type PluginAiApiServer from '../plugin';
 import { AiApiQuotaError, markLlmProviderAttempted, prepareLlmBilling } from '../billing';
 import { DirectLlmContextError, prepareDirectLlmContext, type OpenAIMessage } from '../utils/direct-llm-context';
 import { markAiApiFirstProviderOutput } from '../utils/app-observability';
+import { FileContentBlock, FileProcessorError } from '../services/file-processor';
 
 /**
  * POST /api/ai-llm/v1/chat/completions
@@ -195,6 +196,14 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
       messages.unshift({ role: 'system', content: systemPrompt });
     }
 
+    // ─── Process file / file_url blocks through the file processor service ───
+    messages = await Promise.all(
+      messages.map(async (msg) => ({
+        ...msg,
+        content: await processMessageContentFileBlocks(msg.content, ctx, plugin),
+      })),
+    );
+
     const preparedContext = await prepareDirectLlmContext(ctx, {
       serviceName: service.name,
       modelId,
@@ -271,13 +280,14 @@ export async function handleChatCompletions(ctx: Context, plugin: PluginAiApiSer
     if (!ctx.res?.headersSent) {
       const isQuotaError = err instanceof AiApiQuotaError;
       const isContextError = err instanceof DirectLlmContextError;
-      ctx.status = isQuotaError ? 429 : isContextError ? 400 : 500;
+      const isFileError = err instanceof FileProcessorError;
+      ctx.status = isQuotaError ? 429 : isContextError || isFileError ? 400 : 500;
       if (isQuotaError) ctx.set('X-RateLimit-Reason', err.code);
       ctx.body = toOpenAIError(
         ctx.status,
         getErrorMessage(err, 'Internal server error'),
-        isQuotaError ? 'quota_error' : isContextError ? 'invalid_request_error' : 'server_error',
-        isQuotaError || isContextError ? err.code : undefined,
+        isQuotaError ? 'quota_error' : isContextError || isFileError ? 'invalid_request_error' : 'server_error',
+        isQuotaError || isContextError || isFileError ? err.code : undefined,
       );
     }
   }
@@ -305,10 +315,15 @@ async function handleNonStreamingCompletion(
   }
 
   // Extract usage if available
-  const usage = setAiApiUsageResult(ctx, result.usage_metadata, {
-    gatewayResponseId: completionId,
-    providerRequestId: extractProviderRequestId(result),
-  });
+  const usage = setAiApiUsageResult(
+    ctx,
+    result.usage_metadata,
+    {
+      gatewayResponseId: completionId,
+      providerRequestId: extractProviderRequestId(result),
+    },
+    result.response_metadata,
+  );
 
   ctx.status = 200;
   const toolCalls = normalizeToolCalls(result.tool_calls);
@@ -475,7 +490,7 @@ function getErrorMessage(error: unknown, fallback: string) {
  * the model answers as if the attachment was never sent. A 400 is far easier to
  * debug than a confidently wrong completion.
  */
-const SUPPORTED_CONTENT_BLOCK_TYPES = new Set(['text', 'image_url']);
+const SUPPORTED_CONTENT_BLOCK_TYPES = new Set(['text', 'image_url', 'file', 'file_url']);
 
 /**
  * Deliberately mirrors the exact grammar `@langchain/core`'s `parseBase64DataUrl`
@@ -486,6 +501,14 @@ const SUPPORTED_CONTENT_BLOCK_TYPES = new Set(['text', 'image_url']);
  */
 const BASE64_DATA_URL_PATTERN = /^data:(\w+\/\w+);base64,([A-Za-z0-9+/]+=*)$/;
 
+/**
+ * File blocks accept a wider range of MIME types than `image_url` blocks:
+ * documents such as `application/vnd.openxmlformats-officedocument.wordprocessingml.document`
+ * or `image/svg+xml` are valid attachments. The grammar still requires a proper
+ * `type/subtype` and standard base64 payload.
+ */
+const FILE_BASE64_DATA_URL_PATTERN = /^data:([^;\s]+);base64,([A-Za-z0-9+/]+=*)$/;
+
 /**
  * The regex above is LangChain's, and LangChain's is lenient: `A===`, `A=`,
  * `AAAAA` and `AAAA=` all match it but are not decodable base64. LangChain then
@@ -591,8 +614,8 @@ function describeContentBlockProblem(block: unknown): string | undefined {
   if (!type) return "each content block requires a 'type' field";
   if (!SUPPORTED_CONTENT_BLOCK_TYPES.has(type)) {
     return (
-      `content block type '${type}' is not supported — this gateway forwards 'text' and 'image_url' only. ` +
-      `Send documents as text, or inline them as an 'image_url' data URL if the model reads images`
+      `content block type '${type}' is not supported — this gateway forwards 'text', 'image_url', ` +
+      `'file', and 'file_url' only. Send documents as text, or inline them as a 'file' / 'file_url' block`
     );
   }
 
@@ -600,9 +623,56 @@ function describeContentBlockProblem(block: unknown): string | undefined {
     return typeof block.text === 'string' ? undefined : "a 'text' block requires a string 'text' field";
   }
 
+  if (type === 'file') {
+    return describeFileProblem(block.file);
+  }
+
+  if (type === 'file_url') {
+    return describeFileUrlProblem(block.file_url);
+  }
+
   return describeImageUrlProblem(block.image_url);
 }
 
+function describeFileProblem(file: unknown): string | undefined {
+  if (!isRecord(file)) return "a 'file' block requires an object 'file' field";
+  const fileData = typeof file.file_data === 'string' ? file.file_data : undefined;
+  if (!fileData) return "a 'file' block requires a string 'file.file_data' field";
+  if (!fileData.startsWith('data:')) {
+    return "a 'file' block's 'file_data' must be a base64 data URL starting with 'data:'";
+  }
+  const match = FILE_BASE64_DATA_URL_PATTERN.exec(fileData);
+  if (!match || !match[1].includes('/')) {
+    return (
+      `malformed base64 data URL. Expected 'data:<mime-type>;base64,<base64>' ` +
+      `with a valid type/subtype and standard base64 (no whitespace or URL-safe characters)`
+    );
+  }
+  if (!isDecodableBase64(match[2])) {
+    return (
+      `base64 payload is not decodable. Check the padding and length — ` +
+      `the data must be a multiple of 4 characters with at most two trailing '='`
+    );
+  }
+  return undefined;
+}
+
+function describeFileUrlProblem(fileUrl: unknown): string | undefined {
+  if (!isRecord(fileUrl)) return "a 'file_url' block requires an object 'file_url' field";
+  const url = typeof fileUrl.url === 'string' ? fileUrl.url : undefined;
+  if (!url) return "a 'file_url' block requires a string 'file_url.url' field";
+  let protocol: string;
+  try {
+    protocol = new URL(url).protocol;
+  } catch {
+    return `'${url}' is not a valid URL. Use an http(s) URL`;
+  }
+  if (protocol !== 'http:' && protocol !== 'https:') {
+    return `URL protocol '${protocol}' is not supported. Use an http(s) URL`;
+  }
+  return undefined;
+}
+
 function describeImageUrlProblem(imageUrl: unknown): string | undefined {
   const url = typeof imageUrl === 'string' ? imageUrl : isRecord(imageUrl) ? imageUrl.url : undefined;
   if (typeof url !== 'string' || url === '') {
@@ -681,7 +751,60 @@ export function normalizeMessageContent(content: unknown): MessageContent {
   return JSON.stringify(content);
 }
 
-const GATEWAY_MANAGED_PARAMETERS = new Set(['model', 'messages', 'tools', 'tool_choice', 'stream', 'n']);
+/**
+ * Run any `file` or `file_url` content blocks through the plugin's file
+ * processor service. Custom plugins can register processors to fetch URLs,
+ * extract text, OCR, etc. Other block types are left untouched.
+ *
+ * The service is invoked repeatedly if a processor returns another file-like
+ * block (e.g. a `file_url` becomes a `file` block, which may then be converted
+ * to images by the PDF processor).
+ */
+async function processMessageContentFileBlocks(
+  content: unknown,
+  ctx: Context,
+  plugin: PluginAiApiServer,
+): Promise<unknown> {
+  if (!Array.isArray(content)) return content;
+  const processed: unknown[] = [];
+  for (const block of content) {
+    processed.push(...(await processFileBlockChain(block, ctx, plugin, 0)));
+  }
+  return processed;
+}
+
+const MAX_FILE_PROCESSOR_CHAIN_DEPTH = 3;
+
+async function processFileBlockChain(
+  block: unknown,
+  ctx: Context,
+  plugin: PluginAiApiServer,
+  depth: number,
+): Promise<unknown[]> {
+  if (!isRecord(block) || (block.type !== 'file' && block.type !== 'file_url')) {
+    return [block];
+  }
+  if (depth > MAX_FILE_PROCESSOR_CHAIN_DEPTH) {
+    return [block];
+  }
+
+  const result = await plugin.fileProcessorService.process(block as FileContentBlock, { ctx });
+  const results = Array.isArray(result) ? result : [result];
+
+  const next: unknown[] = [];
+  for (const item of results) {
+    if (isRecord(item) && (item.type === 'file' || item.type === 'file_url')) {
+      // The output is still a file-like block; run it through the chain again
+      // so that a `file_url` -> `file` -> images pipeline can complete.
+      next.push(...(await processFileBlockChain(item, ctx, plugin, depth + 1)));
+    } else {
+      next.push(item);
+    }
+  }
+  return next;
+}
+
+const GATEWAY_MANAGED_PARAMETERS = new Set(['model', 'messages', 'prompt', 'tools', 'tool_choice', 'stream', 'n']);
 
 export function getProviderRequestParameters(body: Record<string, unknown>): Record<string, unknown> {
   return Object.fromEntries(

package/src/server/routes/completions.ts

@@ -23,6 +23,7 @@ import {
   isStreamingRequested,
   writeResponse,
 } from '../utils/streaming';
+import { getProviderRequestParameters, applyProviderRequestParameters } from './chat-completions';
 import { extractProviderRequestId, normalizeUsage, setAiApiUsageResult, type Usage } from '../usage';
 import type PluginAiApiServer from '../plugin';
 import { AiApiQuotaError, markLlmProviderAttempted, prepareLlmBilling } from '../billing';
@@ -169,6 +170,8 @@ export async function handleCompletions(ctx: Context, plugin: PluginAiApiServer)
 
     const completionId = generateCompletionId().replace('chatcmpl-', 'cmpl-');
     const chatModel = provider.createModel();
+    const providerRequestParameters = getProviderRequestParameters(body);
+    applyProviderRequestParameters(chatModel, providerRequestParameters);
     markLlmProviderAttempted(ctx);
 
     if (stream) {
@@ -179,9 +182,17 @@ export async function handleCompletions(ctx: Context, plugin: PluginAiApiServer)
         completionId,
         body.model,
         body.stream_options,
+        providerRequestParameters,
       );
     } else {
-      await handleNonStreamingTextCompletion(ctx, chatModel, langchainMessages, completionId, body.model);
+      await handleNonStreamingTextCompletion(
+        ctx,
+        chatModel,
+        langchainMessages,
+        completionId,
+        body.model,
+        providerRequestParameters,
+      );
     }
   } catch (err) {
     ctx.log.error('AI API completions error:', err);
@@ -208,8 +219,9 @@ async function handleNonStreamingTextCompletion(
   messages: [string, string][],
   completionId: string,
   modelName: string,
+  providerRequestParameters: Record<string, unknown>,
 ) {
-  const result = await chatModel.invoke(messages);
+  const result = await chatModel.invoke(messages, providerRequestParameters);
 
   let text = '';
   if (typeof result.content === 'string') {
@@ -219,10 +231,15 @@ async function handleNonStreamingTextCompletion(
     text = textPart?.text || JSON.stringify(result.content);
   }
 
-  const usage = setAiApiUsageResult(ctx, result.usage_metadata, {
-    gatewayResponseId: completionId,
-    providerRequestId: extractProviderRequestId(result),
-  });
+  const usage = setAiApiUsageResult(
+    ctx,
+    result.usage_metadata,
+    {
+      gatewayResponseId: completionId,
+      providerRequestId: extractProviderRequestId(result),
+    },
+    result.response_metadata,
+  );
 
   ctx.status = 200;
   ctx.body = {
@@ -239,7 +256,14 @@ async function handleNonStreamingTextCompletion(
         finish_reason: 'stop',
       },
     ],
-    usage: usage ?? { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
+    usage: usage
+      ? {
+          prompt_tokens: usage.prompt_tokens,
+          completion_tokens: usage.completion_tokens,
+          total_tokens: usage.total_tokens,
+          prompt_tokens_details: { cached_tokens: usage.prompt_cache_tokens ?? null },
+        }
+      : { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0, prompt_tokens_details: { cached_tokens: null } },
   };
 }
 
@@ -252,6 +276,7 @@ async function handleStreamingTextCompletion(
   completionId: string,
   modelName: string,
   streamOptions: Record<string, unknown> | undefined,
+  providerRequestParameters: Record<string, unknown> | undefined,
 ) {
   ctx.set({
     'Content-Type': 'text/event-stream',
@@ -266,6 +291,7 @@ async function handleStreamingTextCompletion(
   let providerRequestId: string | undefined;
   try {
     const stream = await chatModel.stream(messages, {
+      ...providerRequestParameters,
       stream_options: { ...streamOptions, include_usage: true },
       signal: requestAbort.signal,
     });