qwenproxy-cli 1.0.0

package/src/routes/responses/streaming.ts

@@ -0,0 +1,528 @@
+import crypto from "crypto";
+import type {
+  ResponsesOutputMessage,
+  ResponsesOutputFunctionCall,
+  ResponsesOutputReasoning,
+  ResponsesOutputContentPart,
+  ResponsesUsage,
+  ResponsesStreamEvent,
+} from "./types.ts";
+
+/**
+ * Convert OpenAI Chat Completions streaming chunks to Responses API streaming events.
+ *
+ * The Responses API has more granular streaming events than Chat Completions:
+ * - response.created
+ * - response.in_progress
+ * - response.output_item.added
+ * - response.content_part.added
+ * - response.output_text.delta (multiple)
+ * - response.output_text.done
+ * - response.content_part.done
+ * - response.output_item.done
+ * - response.function_call_arguments.delta (for tool calls)
+ * - response.function_call_arguments.done
+ * - response.completed
+ */
+
+export interface ResponsesStreamState {
+  responseId: string;
+  messageId: string;
+  reasoningId: string;
+  requestModel: string;
+  outputIndex: number;
+  contentIndex: number;
+  currentBlockType: "text" | "function_call" | "reasoning" | null;
+  accumulatedText: string;
+  accumulatedReasoning: string;
+  accumulatedToolCalls: Map<
+    number,
+    {
+      id: string;
+      callId: string;
+      name: string;
+      arguments: string;
+    }
+  >;
+  completedOutput: (
+    | ResponsesOutputMessage
+    | ResponsesOutputFunctionCall
+    | ResponsesOutputReasoning
+  )[];
+  inputTokens: number;
+  /** Real usage from upstream (overwrites estimates) */
+  upstreamUsage: {
+    inputTokens: number | null;
+    outputTokens: number | null;
+    cachedTokens: number | null;
+    reasoningTokens: number | null;
+  };
+  contextMeter: ResponsesUsage["context_meter"];
+}
+
+export function createStreamState(
+  responseId: string,
+  requestModel: string,
+): ResponsesStreamState {
+  return {
+    responseId,
+    messageId: `msg_${crypto.randomBytes(16).toString("hex")}`,
+    reasoningId: `rs_${crypto.randomBytes(16).toString("hex")}`,
+    requestModel,
+    outputIndex: 0,
+    contentIndex: 0,
+    currentBlockType: null,
+    accumulatedText: "",
+    accumulatedReasoning: "",
+    accumulatedToolCalls: new Map(),
+    completedOutput: [],
+    inputTokens: 0,
+    upstreamUsage: {
+      inputTokens: null,
+      outputTokens: null,
+      cachedTokens: null,
+      reasoningTokens: null,
+    },
+    contextMeter: undefined,
+  };
+}
+
+/**
+ * Process a Chat Completions streaming chunk and emit Responses API events.
+ */
+export function processChatChunk(
+  chunk: any,
+  state: ResponsesStreamState,
+): ResponsesStreamEvent[] {
+  const events: ResponsesStreamEvent[] = [];
+
+  // Update token count if available (real upstream usage overwrites)
+  if (chunk.usage) {
+    if (chunk.usage.context_meter) {
+      state.contextMeter = chunk.usage.context_meter;
+    }
+    if (chunk.usage.prompt_tokens !== undefined) {
+      state.inputTokens = chunk.usage.prompt_tokens;
+      state.upstreamUsage.inputTokens = chunk.usage.prompt_tokens;
+    }
+    if (chunk.usage.input_tokens !== undefined) {
+      state.inputTokens = chunk.usage.input_tokens;
+      state.upstreamUsage.inputTokens = chunk.usage.input_tokens;
+    }
+    if (chunk.usage.completion_tokens !== undefined) {
+      state.upstreamUsage.outputTokens = chunk.usage.completion_tokens;
+    }
+    if (chunk.usage.output_tokens !== undefined) {
+      state.upstreamUsage.outputTokens = chunk.usage.output_tokens;
+    }
+    const cached = chunk.usage.prompt_tokens_details?.cached_tokens
+      ?? chunk.usage.input_tokens_details?.cached_tokens;
+    if (cached !== undefined) {
+      state.upstreamUsage.cachedTokens = cached;
+    }
+    const reasoning = chunk.usage.completion_tokens_details?.reasoning_tokens
+      ?? chunk.usage.output_tokens_details?.reasoning_tokens;
+    if (reasoning !== undefined) {
+      state.upstreamUsage.reasoningTokens = reasoning;
+    }
+  }
+
+  const choice = chunk.choices?.[0];
+  if (!choice) return events;
+
+  const delta = choice.delta ?? {};
+
+  // Reasoning content (thinking/reasoning from Qwen models)
+  if (delta.reasoning_content) {
+    // Close non-reasoning block if active
+    if (
+      state.currentBlockType === "text" ||
+      state.currentBlockType === "function_call"
+    ) {
+      if (state.currentBlockType === "text") {
+        events.push(...closeCurrentText(state));
+      } else if (state.currentBlockType === "function_call") {
+        events.push(...closeCurrentFunctionCall(state));
+      }
+    }
+
+    // Start reasoning block if needed
+    if (state.currentBlockType !== "reasoning") {
+      const reasoningItem: ResponsesOutputReasoning = {
+        type: "reasoning",
+        id: state.reasoningId,
+        summary: [],
+      };
+      events.push({
+        type: "response.output_item.added",
+        output_index: state.outputIndex,
+        item: reasoningItem,
+      });
+      // Emit reasoning_summary_part.added
+      events.push({
+        type: "response.reasoning_summary_part.added",
+        item_id: state.reasoningId,
+        output_index: state.outputIndex,
+        part: { type: "summary_text", text: "" },
+      });
+      state.currentBlockType = "reasoning";
+    }
+
+    state.accumulatedReasoning += delta.reasoning_content;
+    events.push({
+      type: "response.reasoning_summary_text.delta",
+      item_id: state.reasoningId,
+      output_index: state.outputIndex,
+      delta: delta.reasoning_content,
+    });
+  }
+
+  // Text content
+  if (delta.content) {
+    // Start text block if needed
+    if (state.currentBlockType !== "text") {
+      // Close previous block if exists
+      if (state.currentBlockType === "function_call") {
+        events.push(...closeCurrentFunctionCall(state));
+      } else if (state.currentBlockType === "reasoning") {
+        events.push(...closeCurrentReasoning(state));
+      }
+
+      // Emit output_item.added for message
+      const messageItem: ResponsesOutputMessage = {
+        type: "message",
+        id: state.messageId,
+        role: "assistant",
+        status: "in_progress",
+        content: [],
+      };
+      events.push({
+        type: "response.output_item.added",
+        output_index: state.outputIndex,
+        item: messageItem,
+      });
+
+      // Emit content_part.added
+      const contentPart: ResponsesOutputContentPart = {
+        type: "output_text",
+        text: "",
+        annotations: [],
+      };
+      events.push({
+        type: "response.content_part.added",
+        item_id: state.messageId,
+        output_index: state.outputIndex,
+        content_index: state.contentIndex,
+        part: contentPart,
+      });
+
+      state.currentBlockType = "text";
+    }
+
+    // Emit text delta
+    state.accumulatedText += delta.content;
+    events.push({
+      type: "response.output_text.delta",
+      item_id: state.messageId,
+      output_index: state.outputIndex,
+      content_index: state.contentIndex,
+      delta: delta.content,
+    });
+  }
+
+  // Tool calls
+  if (delta.tool_calls) {
+    for (const tc of delta.tool_calls) {
+      const index = tc.index;
+
+      // New tool call
+      if (tc.function?.name) {
+        // Close previous text block if exists
+        if (state.currentBlockType === "text") {
+          events.push(...closeCurrentText(state));
+        }
+
+        const callId =
+          tc.id || `call_${crypto.randomBytes(12).toString("hex")}`;
+        const fcId = `fc_${crypto.randomBytes(12).toString("hex")}`;
+
+        state.accumulatedToolCalls.set(index, {
+          id: fcId,
+          callId,
+          name: tc.function.name,
+          arguments: "",
+        });
+
+        // Emit output_item.added for function_call
+        const fcItem: ResponsesOutputFunctionCall = {
+          type: "function_call",
+          id: fcId,
+          call_id: callId,
+          name: tc.function.name,
+          arguments: "",
+          status: "in_progress",
+        };
+        events.push({
+          type: "response.output_item.added",
+          output_index: state.outputIndex,
+          item: fcItem,
+        });
+
+        state.currentBlockType = "function_call";
+      }
+
+      // Tool call arguments
+      if (tc.function?.arguments) {
+        const stored = state.accumulatedToolCalls.get(index);
+        if (stored) {
+          stored.arguments += tc.function.arguments;
+          events.push({
+            type: "response.function_call_arguments.delta",
+            item_id: stored.id,
+            output_index: state.outputIndex,
+            delta: tc.function.arguments,
+          });
+        }
+      }
+    }
+  }
+
+  // Finish reason
+  if (choice.finish_reason) {
+    // Close current block
+    if (state.currentBlockType === "reasoning") {
+      events.push(...closeCurrentReasoning(state));
+    }
+    if (state.currentBlockType === "text") {
+      events.push(...closeCurrentText(state));
+    } else if (state.currentBlockType === "function_call") {
+      events.push(...closeCurrentFunctionCall(state));
+    }
+  }
+
+  return events;
+}
+
+/**
+ * Close the current text content block.
+ */
+function closeCurrentText(state: ResponsesStreamState): ResponsesStreamEvent[] {
+  const events: ResponsesStreamEvent[] = [];
+
+  if (state.currentBlockType !== "text") return events;
+
+  // Emit output_text.done
+  events.push({
+    type: "response.output_text.done",
+    item_id: state.messageId,
+    output_index: state.outputIndex,
+    content_index: state.contentIndex,
+    text: state.accumulatedText,
+  });
+
+  // Emit content_part.done
+  const contentPart: ResponsesOutputContentPart = {
+    type: "output_text",
+    text: state.accumulatedText,
+    annotations: [],
+  };
+  events.push({
+    type: "response.content_part.done",
+    item_id: state.messageId,
+    output_index: state.outputIndex,
+    content_index: state.contentIndex,
+    part: contentPart,
+  });
+
+  // Emit output_item.done for message
+  const messageItem: ResponsesOutputMessage = {
+    type: "message",
+    id: state.messageId,
+    role: "assistant",
+    status: "completed",
+    content: [contentPart],
+  };
+  events.push({
+    type: "response.output_item.done",
+    output_index: state.outputIndex,
+    item: messageItem,
+  });
+  state.completedOutput.push(messageItem);
+
+  state.currentBlockType = null;
+  state.contentIndex++;
+  state.outputIndex++;
+
+  return events;
+}
+
+/**
+ * Close all current function call blocks.
+ */
+function closeCurrentFunctionCall(
+  state: ResponsesStreamState,
+): ResponsesStreamEvent[] {
+  const events: ResponsesStreamEvent[] = [];
+
+  if (state.currentBlockType !== "function_call") return events;
+
+  // Close all accumulated tool calls
+  for (const [_, fc] of state.accumulatedToolCalls) {
+    // Emit function_call_arguments.done
+    events.push({
+      type: "response.function_call_arguments.done",
+      item_id: fc.id,
+      output_index: state.outputIndex,
+      arguments: fc.arguments,
+    });
+
+    // Emit output_item.done for function_call
+    const fcItem: ResponsesOutputFunctionCall = {
+      type: "function_call",
+      id: fc.id,
+      call_id: fc.callId,
+      name: fc.name,
+      arguments: fc.arguments,
+      status: "completed",
+    };
+    events.push({
+      type: "response.output_item.done",
+      output_index: state.outputIndex,
+      item: fcItem,
+    });
+    state.completedOutput.push(fcItem);
+
+    state.outputIndex++;
+  }
+
+  state.currentBlockType = null;
+  state.accumulatedToolCalls.clear();
+
+  return events;
+}
+
+/**
+ * Close the current reasoning block.
+ */
+function closeCurrentReasoning(
+  state: ResponsesStreamState,
+): ResponsesStreamEvent[] {
+  const events: ResponsesStreamEvent[] = [];
+
+  if (state.currentBlockType !== "reasoning") return events;
+
+  // Emit reasoning_summary_text.done
+  events.push({
+    type: "response.reasoning_summary_text.done",
+    item_id: state.reasoningId,
+    output_index: state.outputIndex,
+    text: state.accumulatedReasoning,
+  });
+
+  // Emit reasoning_summary_part.done
+  events.push({
+    type: "response.reasoning_summary_part.done",
+    item_id: state.reasoningId,
+    output_index: state.outputIndex,
+    part: { type: "summary_text", text: state.accumulatedReasoning },
+  });
+
+  const reasoningItem: ResponsesOutputReasoning = {
+    type: "reasoning",
+    id: state.reasoningId,
+    summary: [{ type: "summary_text", text: state.accumulatedReasoning }],
+  };
+  events.push({
+    type: "response.output_item.done",
+    output_index: state.outputIndex,
+    item: reasoningItem,
+  });
+  state.completedOutput.push(reasoningItem);
+
+  state.currentBlockType = null;
+  state.outputIndex++;
+
+  return events;
+}
+
+/**
+ * Build the final output items from accumulated stream state.
+ */
+export function buildFinalOutput(
+  state: ResponsesStreamState,
+): (
+  | ResponsesOutputMessage
+  | ResponsesOutputFunctionCall
+  | ResponsesOutputReasoning
+)[] {
+  const output: (
+    | ResponsesOutputMessage
+    | ResponsesOutputFunctionCall
+    | ResponsesOutputReasoning
+  )[] = [...state.completedOutput];
+
+  // Add an open reasoning block if the stream ended without a finish_reason
+  if (state.currentBlockType === "reasoning" && state.accumulatedReasoning) {
+    output.push({
+      type: "reasoning",
+      id: state.reasoningId,
+      summary: [{ type: "summary_text", text: state.accumulatedReasoning }],
+    });
+  }
+
+  // Add an open text block if the stream ended without a finish_reason
+  if (state.currentBlockType === "text" && state.accumulatedText) {
+    output.push({
+      type: "message",
+      id: state.messageId,
+      role: "assistant",
+      status: "completed",
+      content: [
+        {
+          type: "output_text",
+          text: state.accumulatedText,
+          annotations: [],
+        },
+      ],
+    });
+  }
+
+  // Add open function calls if the stream ended without a finish_reason
+  if (state.currentBlockType === "function_call") {
+    for (const [, fc] of state.accumulatedToolCalls) {
+      output.push({
+        type: "function_call",
+        id: fc.id,
+        call_id: fc.callId,
+        name: fc.name,
+        arguments: fc.arguments,
+        status: "completed",
+      });
+    }
+  }
+
+  return output;
+}
+
+/**
+ * Build final usage from stream state — always includes details (Grok/serde fix).
+ */
+export function buildFinalUsage(
+  state: ResponsesStreamState,
+  completionTokens: number,
+): ResponsesUsage {
+  const inputTokens = state.upstreamUsage.inputTokens ?? state.inputTokens;
+  const outputTokens = state.upstreamUsage.outputTokens ?? completionTokens;
+  return {
+    input_tokens: inputTokens,
+    output_tokens: outputTokens,
+    total_tokens: inputTokens + outputTokens,
+    input_tokens_details: {
+      cached_tokens: state.upstreamUsage.cachedTokens ?? 0,
+    },
+    output_tokens_details: {
+      reasoning_tokens: state.upstreamUsage.reasoningTokens ?? 0,
+    },
+    ...(state.contextMeter ? { context_meter: state.contextMeter } : {}),
+  };
+}