vforclaude2 0.3.87

package/open-sse/utils/stream.js

@@ -0,0 +1,392 @@
+import { translateResponse, initState } from "../translator/index.js";
+import { FORMATS } from "../translator/formats.js";
+import { trackPendingRequest, appendRequestLog } from "@/lib/usageDb.js";
+import { extractUsage, hasValidUsage, estimateUsage, logUsage, addBufferToUsage, filterUsageForFormat, COLORS } from "./usageTracking.js";
+import { parseSSELine, hasValuableContent, fixInvalidId, formatSSE } from "./streamHelpers.js";
+
+export { COLORS, formatSSE };
+
+// sharedEncoder is stateless — safe to share across streams
+const sharedEncoder = new TextEncoder();
+
+/**
+ * Stream modes
+ */
+const STREAM_MODE = {
+  TRANSLATE: "translate",    // Full translation between formats
+  PASSTHROUGH: "passthrough" // No translation, normalize output, extract usage
+};
+
+/**
+ * Create unified SSE transform stream
+ * @param {object} options
+ * @param {string} options.mode - Stream mode: translate, passthrough
+ * @param {string} options.targetFormat - Provider format (for translate mode)
+ * @param {string} options.sourceFormat - Client format (for translate mode)
+ * @param {string} options.provider - Provider name
+ * @param {object} options.reqLogger - Request logger instance
+ * @param {string} options.model - Model name
+ * @param {string} options.connectionId - Connection ID for usage tracking
+ * @param {object} options.body - Request body (for input token estimation)
+ * @param {function} options.onStreamComplete - Callback when stream completes (content, usage)
+ * @param {string} options.apiKey - API key for usage tracking
+ */
+export function createSSEStream(options = {}) {
+  const {
+    mode = STREAM_MODE.TRANSLATE,
+    targetFormat,
+    sourceFormat,
+    provider = null,
+    reqLogger = null,
+    toolNameMap = null,
+    model = null,
+    connectionId = null,
+    body = null,
+    onStreamComplete = null,
+    apiKey = null
+  } = options;
+
+  let buffer = "";
+  let usage = null;
+
+  // Per-stream decoder with stream:true to correctly handle multi-byte chars split across chunks
+  const decoder = new TextDecoder("utf-8", { fatal: false });
+
+  const state = mode === STREAM_MODE.TRANSLATE ? { ...initState(sourceFormat), provider, toolNameMap, model } : null;
+
+  let totalContentLength = 0;
+  let accumulatedContent = "";
+  let accumulatedThinking = "";
+  let ttftAt = null;
+
+  return new TransformStream({
+    transform(chunk, controller) {
+      if (!ttftAt) {
+        ttftAt = Date.now();
+      }
+      const text = decoder.decode(chunk, { stream: true });
+      buffer += text;
+      reqLogger?.appendProviderChunk?.(text);
+
+      const lines = buffer.split("\n");
+      buffer = lines.pop() || "";
+
+      for (const line of lines) {
+        const trimmed = line.trim();
+
+        // Passthrough mode: normalize and forward
+        if (mode === STREAM_MODE.PASSTHROUGH) {
+          let output;
+          let injectedUsage = false;
+
+          if (trimmed.startsWith("data:") && trimmed.slice(5).trim() !== "[DONE]") {
+            try {
+              const parsed = JSON.parse(trimmed.slice(5).trim());
+
+              const idFixed = fixInvalidId(parsed);
+
+              // Ensure OpenAI-required fields are present on streaming chunks (Letta compat)
+              let fieldsInjected = false;
+              if (parsed.choices !== undefined) {
+                if (!parsed.object) { parsed.object = "chat.completion.chunk"; fieldsInjected = true; }
+                if (!parsed.created) { parsed.created = Math.floor(Date.now() / 1000); fieldsInjected = true; }
+              }
+
+              // Strip Azure-specific non-standard fields from streaming chunks
+              if (parsed.prompt_filter_results !== undefined) {
+                delete parsed.prompt_filter_results;
+                fieldsInjected = true;
+              }
+              if (parsed?.choices) {
+                for (const choice of parsed.choices) {
+                  if (choice.content_filter_results !== undefined) {
+                    delete choice.content_filter_results;
+                    fieldsInjected = true;
+                  }
+                }
+              }
+
+              if (!hasValuableContent(parsed, FORMATS.OPENAI)) {
+                continue;
+              }
+
+              const delta = parsed.choices?.[0]?.delta;
+              const content = delta?.content;
+              const reasoning = delta?.reasoning_content;
+              if (content && typeof content === "string") {
+                totalContentLength += content.length;
+                accumulatedContent += content;
+              }
+              if (reasoning && typeof reasoning === "string") {
+                totalContentLength += reasoning.length;
+                accumulatedThinking += reasoning;
+              }
+
+              const extracted = extractUsage(parsed);
+              if (extracted) {
+                usage = extracted;
+              }
+
+              const isFinishChunk = parsed.choices?.[0]?.finish_reason;
+              if (isFinishChunk && !hasValidUsage(parsed.usage)) {
+                const estimated = estimateUsage(body, totalContentLength, FORMATS.OPENAI);
+                parsed.usage = filterUsageForFormat(estimated, FORMATS.OPENAI);
+                output = `data: ${JSON.stringify(parsed)}\n`;
+                usage = estimated;
+                injectedUsage = true;
+              } else if (isFinishChunk && usage) {
+                const buffered = addBufferToUsage(usage);
+                parsed.usage = filterUsageForFormat(buffered, FORMATS.OPENAI);
+                output = `data: ${JSON.stringify(parsed)}\n`;
+                injectedUsage = true;
+              } else if (idFixed || fieldsInjected) {
+                output = `data: ${JSON.stringify(parsed)}\n`;
+                injectedUsage = true;
+              }
+            } catch { }
+          }
+
+          if (!injectedUsage) {
+            if (line.startsWith("data:") && !line.startsWith("data: ")) {
+              output = "data: " + line.slice(5) + "\n";
+            } else {
+              output = line + "\n";
+            }
+          }
+
+          reqLogger?.appendConvertedChunk?.(output);
+          controller.enqueue(sharedEncoder.encode(output));
+          continue;
+        }
+
+        // Translate mode
+        if (!trimmed) continue;
+
+        const parsed = parseSSELine(trimmed, targetFormat);
+        if (!parsed) continue;
+
+        // For Ollama: done=true is the final chunk with finish_reason/usage, must translate
+        // For other formats: done=true is the [DONE] sentinel, skip
+        if (parsed && parsed.done && targetFormat !== FORMATS.OLLAMA) {
+          const output = "data: [DONE]\n\n";
+          reqLogger?.appendConvertedChunk?.(output);
+          controller.enqueue(sharedEncoder.encode(output));
+          continue;
+        }
+
+        // Claude format - content
+        if (parsed.delta?.text) {
+          totalContentLength += parsed.delta.text.length;
+          accumulatedContent += parsed.delta.text;
+        }
+        // Claude format - thinking
+        if (parsed.delta?.thinking) {
+          totalContentLength += parsed.delta.thinking.length;
+          accumulatedThinking += parsed.delta.thinking;
+        }
+        
+        // OpenAI format - content
+        if (parsed.choices?.[0]?.delta?.content) {
+          totalContentLength += parsed.choices[0].delta.content.length;
+          accumulatedContent += parsed.choices[0].delta.content;
+        }
+        // OpenAI format - reasoning
+        if (parsed.choices?.[0]?.delta?.reasoning_content) {
+          totalContentLength += parsed.choices[0].delta.reasoning_content.length;
+          accumulatedThinking += parsed.choices[0].delta.reasoning_content;
+        }
+        
+        // Gemini format
+        if (parsed.candidates?.[0]?.content?.parts) {
+          for (const part of parsed.candidates[0].content.parts) {
+            if (part.text && typeof part.text === "string") {
+              totalContentLength += part.text.length;
+              // Check if this is thinking content
+              if (part.thought === true) {
+                accumulatedThinking += part.text;
+              } else {
+                accumulatedContent += part.text;
+              }
+            }
+          }
+        }
+
+        // Extract usage
+        const extracted = extractUsage(parsed);
+        if (extracted) state.usage = extracted; // Keep original usage for logging
+
+        // Translate: targetFormat -> openai -> sourceFormat
+        const translated = translateResponse(targetFormat, sourceFormat, parsed, state);
+
+        // Log OpenAI intermediate chunks (if available)
+        if (translated?._openaiIntermediate) {
+          for (const item of translated._openaiIntermediate) {
+            const openaiOutput = formatSSE(item, FORMATS.OPENAI);
+            reqLogger?.appendOpenAIChunk?.(openaiOutput);
+          }
+        }
+
+        if (translated?.length > 0) {
+          for (const item of translated) {
+            // Filter empty chunks
+            if (!hasValuableContent(item, sourceFormat)) {
+              continue; // Skip this empty chunk
+            }
+
+            // Inject estimated usage if finish chunk has no valid usage
+            const isFinishChunk = item.type === "message_delta" || item.choices?.[0]?.finish_reason;
+            if (state.finishReason && isFinishChunk && !hasValidUsage(item.usage) && totalContentLength > 0) {
+              const estimated = estimateUsage(body, totalContentLength, sourceFormat);
+              item.usage = filterUsageForFormat(estimated, sourceFormat); // Filter + already has buffer
+              state.usage = estimated;
+            } else if (state.finishReason && isFinishChunk && state.usage) {
+              // Add buffer and filter usage for client (but keep original in state.usage for logging)
+              const buffered = addBufferToUsage(state.usage);
+              item.usage = filterUsageForFormat(buffered, sourceFormat);
+            }
+
+            const output = formatSSE(item, sourceFormat);
+            reqLogger?.appendConvertedChunk?.(output);
+            controller.enqueue(sharedEncoder.encode(output));
+          }
+        }
+      }
+    },
+
+    flush(controller) {
+      trackPendingRequest(model, provider, connectionId, false);
+      try {
+        const remaining = decoder.decode();
+        if (remaining) buffer += remaining;
+
+        if (mode === STREAM_MODE.PASSTHROUGH) {
+          if (buffer) {
+            let output = buffer;
+            if (buffer.startsWith("data:") && !buffer.startsWith("data: ")) {
+              output = "data: " + buffer.slice(5);
+            }
+            reqLogger?.appendConvertedChunk?.(output);
+            controller.enqueue(sharedEncoder.encode(output));
+          }
+
+          if (!hasValidUsage(usage) && totalContentLength > 0) {
+            usage = estimateUsage(body, totalContentLength, FORMATS.OPENAI);
+          }
+
+          if (hasValidUsage(usage)) {
+            logUsage(provider, usage, model, connectionId, apiKey);
+          } else {
+            appendRequestLog({ model, provider, connectionId, tokens: null, status: "200 OK" }).catch(() => { });
+          }
+          
+          // IMPORTANT: In passthrough mode we still must terminate the SSE stream.
+          // Some clients (e.g. OpenClaw) expect the OpenAI-style sentinel:
+          //   data: [DONE]\n\n
+          // Without it they can hang until timeout and trigger failover.
+          const doneOutput = "data: [DONE]\n\n";
+          reqLogger?.appendConvertedChunk?.(doneOutput);
+          controller.enqueue(sharedEncoder.encode(doneOutput));
+
+          if (onStreamComplete) {
+            onStreamComplete({
+              content: accumulatedContent,
+              thinking: accumulatedThinking
+            }, usage, ttftAt);
+          }
+          return;
+        }
+
+        if (buffer.trim()) {
+          const parsed = parseSSELine(buffer.trim());
+          if (parsed && !parsed.done) {
+            const translated = translateResponse(targetFormat, sourceFormat, parsed, state);
+
+            if (translated?._openaiIntermediate) {
+              for (const item of translated._openaiIntermediate) {
+                const openaiOutput = formatSSE(item, FORMATS.OPENAI);
+                reqLogger?.appendOpenAIChunk?.(openaiOutput);
+              }
+            }
+
+            if (translated?.length > 0) {
+              for (const item of translated) {
+                const output = formatSSE(item, sourceFormat);
+                reqLogger?.appendConvertedChunk?.(output);
+                controller.enqueue(sharedEncoder.encode(output));
+              }
+            }
+          }
+        }
+
+        const flushed = translateResponse(targetFormat, sourceFormat, null, state);
+
+        if (flushed?._openaiIntermediate) {
+          for (const item of flushed._openaiIntermediate) {
+            const openaiOutput = formatSSE(item, FORMATS.OPENAI);
+            reqLogger?.appendOpenAIChunk?.(openaiOutput);
+          }
+        }
+
+        if (flushed?.length > 0) {
+          for (const item of flushed) {
+            const output = formatSSE(item, sourceFormat);
+            reqLogger?.appendConvertedChunk?.(output);
+            controller.enqueue(sharedEncoder.encode(output));
+          }
+        }
+
+        const doneOutput = "data: [DONE]\n\n";
+        reqLogger?.appendConvertedChunk?.(doneOutput);
+        controller.enqueue(sharedEncoder.encode(doneOutput));
+
+        if (!hasValidUsage(state?.usage) && totalContentLength > 0) {
+          state.usage = estimateUsage(body, totalContentLength, sourceFormat);
+        }
+
+        if (hasValidUsage(state?.usage)) {
+          logUsage(state.provider || targetFormat, state.usage, model, connectionId, apiKey);
+        } else {
+          appendRequestLog({ model, provider, connectionId, tokens: null, status: "200 OK" }).catch(() => { });
+        }
+        
+        if (onStreamComplete) {
+          onStreamComplete({
+            content: accumulatedContent,
+            thinking: accumulatedThinking
+          }, state?.usage, ttftAt);
+        }
+      } catch (error) {
+        console.log("Error in flush:", error);
+      }
+    }
+  });
+}
+
+export function createSSETransformStreamWithLogger(targetFormat, sourceFormat, provider = null, reqLogger = null, toolNameMap = null, model = null, connectionId = null, body = null, onStreamComplete = null, apiKey = null) {
+  return createSSEStream({
+    mode: STREAM_MODE.TRANSLATE,
+    targetFormat,
+    sourceFormat,
+    provider,
+    reqLogger,
+    toolNameMap,
+    model,
+    connectionId,
+    body,
+    onStreamComplete,
+    apiKey
+  });
+}
+
+export function createPassthroughStreamWithLogger(provider = null, reqLogger = null, model = null, connectionId = null, body = null, onStreamComplete = null, apiKey = null) {
+  return createSSEStream({
+    mode: STREAM_MODE.PASSTHROUGH,
+    provider,
+    reqLogger,
+    model,
+    connectionId,
+    body,
+    onStreamComplete,
+    apiKey
+  });
+}

package/open-sse/utils/streamHandler.js

@@ -0,0 +1,138 @@
+// Stream handler with disconnect detection - shared for all providers
+
+// Get HH:MM:SS timestamp
+function getTimeString() {
+  return new Date().toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" });
+}
+
+/**
+ * Create stream controller with abort and disconnect detection
+ * @param {object} options
+ * @param {function} options.onDisconnect - Callback when client disconnects
+ * @param {object} options.log - Logger instance
+ * @param {string} options.provider - Provider name
+ * @param {string} options.model - Model name
+ */
+export function createStreamController({ onDisconnect, onError, log, provider, model } = {}) {
+  const abortController = new AbortController();
+  const startTime = Date.now();
+  let disconnected = false;
+  let abortTimeout = null;
+
+  const logStream = (status) => {
+    const duration = Date.now() - startTime;
+    const p = provider?.toUpperCase() || "UNKNOWN";
+    console.log(`[${getTimeString()}] 🌊 [STREAM] ${p} | ${model || "unknown"} | ${duration}ms | ${status}`);
+  };
+
+  return {
+    signal: abortController.signal,
+    startTime,
+
+    isConnected: () => !disconnected,
+
+    // Call when client disconnects
+    handleDisconnect: (reason = "client_closed") => {
+      if (disconnected) return;
+      disconnected = true;
+
+      logStream(`disconnect: ${reason}`);
+
+      // Delay abort to allow cleanup
+      abortTimeout = setTimeout(() => {
+        abortController.abort();
+      }, 500);
+
+      onDisconnect?.({ reason, duration: Date.now() - startTime });
+    },
+
+    // Call when stream completes normally
+    handleComplete: () => {
+      if (disconnected) return;
+      disconnected = true;
+
+      logStream("complete");
+
+      if (abortTimeout) {
+        clearTimeout(abortTimeout);
+        abortTimeout = null;
+      }
+    },
+
+    // Call on error
+    handleError: (error) => {
+      if (disconnected) return;
+      disconnected = true;
+
+      if (abortTimeout) {
+        clearTimeout(abortTimeout);
+        abortTimeout = null;
+      }
+
+      if (error.name === "AbortError") {
+        logStream("aborted");
+        return;
+      }
+
+      logStream(`error: ${error.message}`);
+      onError?.(error);
+    },
+
+    abort: () => abortController.abort()
+  };
+}
+
+/**
+ * Create transform stream with disconnect detection
+ * Wraps existing transform stream and adds abort capability
+ */
+export function createDisconnectAwareStream(transformStream, streamController) {
+  const reader = transformStream.readable.getReader();
+  const writer = transformStream.writable.getWriter();
+
+  return new ReadableStream({
+    async pull(controller) {
+      if (!streamController.isConnected()) {
+        controller.close();
+        return;
+      }
+
+      try {
+        const { done, value } = await reader.read();
+        if (done) {
+          streamController.handleComplete();
+          controller.close();
+          return;
+        }
+        controller.enqueue(value);
+      } catch (error) {
+        streamController.handleError(error);
+        // Cleanup reader/writer to avoid orphaned streams
+        reader.cancel().catch(() => {});
+        writer.abort().catch(() => {});
+        controller.error(error);
+      }
+    },
+
+    cancel(reason) {
+      streamController.handleDisconnect(reason || "cancelled");
+      reader.cancel();
+      writer.abort();
+    }
+  });
+}
+
+/**
+ * Pipe provider response through transform with disconnect detection
+ * @param {Response} providerResponse - Response from provider
+ * @param {TransformStream} transformStream - Transform stream for SSE
+ * @param {object} streamController - Stream controller from createStreamController
+ */
+export function pipeWithDisconnect(providerResponse, transformStream, streamController) {
+  const transformedBody = providerResponse.body.pipeThrough(transformStream);
+  return createDisconnectAwareStream(
+    { readable: transformedBody, writable: { getWriter: () => ({ abort: () => Promise.resolve() }) } },
+    streamController
+  );
+}
+

package/open-sse/utils/streamHelpers.js

@@ -0,0 +1,122 @@
+import { FORMATS } from "../translator/formats.js";
+
+// Parse SSE data line
+export function parseSSELine(line, format = null) {
+  if (!line) return null;
+
+  // NDJSON format (Ollama): raw JSON lines without "data:" prefix
+  if (format === FORMATS.OLLAMA) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith("{")) {
+      try {
+        return JSON.parse(trimmed);
+      } catch (error) {
+        return null;
+      }
+    }
+    return null;
+  }
+
+  // Standard SSE format: "data: {...}"
+  if (line.charCodeAt(0) !== 100) return null; // 'd' = 100
+
+  const data = line.slice(5).trim();
+  if (data === "[DONE]") return { done: true };
+
+  try {
+    return JSON.parse(data);
+  } catch (error) {
+    if (data.length > 0 && data.length < 1000) {
+      console.log(`[WARN] Failed to parse SSE line (${data.length} chars): ${data.substring(0, 100)}...`);
+    }
+    return null;
+  }
+}
+
+// Check if chunk has valuable content (not empty)
+export function hasValuableContent(chunk, format) {
+  // OpenAI format
+  if (format === FORMATS.OPENAI && chunk.choices?.[0]?.delta) {
+    const delta = chunk.choices[0].delta;
+    return delta.content && delta.content !== "" ||
+           delta.reasoning_content && delta.reasoning_content !== "" ||
+           delta.tool_calls && delta.tool_calls.length > 0 ||
+           chunk.choices[0].finish_reason ||
+           delta.role;
+  }
+
+  // Claude format
+  if (format === FORMATS.CLAUDE) {
+    const isContentBlockDelta = chunk.type === "content_block_delta";
+    const hasText = chunk.delta?.text && chunk.delta.text !== "";
+    const hasThinking = chunk.delta?.thinking && chunk.delta.thinking !== "";
+    const hasInputJson = chunk.delta?.partial_json && chunk.delta.partial_json !== "";
+    
+    if (isContentBlockDelta && !hasText && !hasThinking && !hasInputJson) {
+      return false;
+    }
+    return true;
+  }
+
+  return true; // Other formats: keep all chunks
+}
+
+// Fix invalid id (generic or too short)
+export function fixInvalidId(parsed) {
+  if (parsed.id && (parsed.id === "chat" || parsed.id === "completion" || parsed.id.length < 8)) {
+    const fallbackId = parsed.extend_fields?.requestId || 
+                      parsed.extend_fields?.traceId || 
+                      Date.now().toString(36);
+    parsed.id = `chatcmpl-${fallbackId}`;
+    return true;
+  }
+  return false;
+}
+
+function cleanUsagePayload(payload) {
+  if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
+    return payload;
+  }
+
+  let cleaned = payload;
+
+  if ("usage" in cleaned) {
+    if (cleaned.usage === null) {
+      const { usage, ...payloadWithoutUsage } = cleaned;
+      cleaned = payloadWithoutUsage;
+    } else if (typeof cleaned.usage === "object" && cleaned.usage.perf_metrics === null) {
+      const { perf_metrics, ...usageWithoutPerf } = cleaned.usage;
+      cleaned = { ...cleaned, usage: usageWithoutPerf };
+    }
+  }
+
+  if (cleaned.response && typeof cleaned.response === "object" && !Array.isArray(cleaned.response)) {
+    const cleanedResponse = cleanUsagePayload(cleaned.response);
+    if (cleanedResponse !== cleaned.response) {
+      cleaned = { ...cleaned, response: cleanedResponse };
+    }
+  }
+
+  return cleaned;
+}
+
+// Format output as SSE
+export function formatSSE(data, sourceFormat) {
+  if (data === null || data === undefined) return "data: null\n\n";
+  if (data && data.done) return "data: [DONE]\n\n";
+
+  // OpenAI Responses API format
+  if (data && data.event && data.data) {
+    const cleanedEventData = cleanUsagePayload(data.data);
+    return `event: ${data.event}\ndata: ${JSON.stringify(cleanedEventData)}\n\n`;
+  }
+
+  data = cleanUsagePayload(data);
+
+  // Claude format
+  if (sourceFormat === FORMATS.CLAUDE && data && data.type) {
+    return `event: ${data.type}\ndata: ${JSON.stringify(data)}\n\n`;
+  }
+
+  return `data: ${JSON.stringify(data)}\n\n`;
+}