ruflo 3.5.2 → 3.5.4

package/src/ruvocal/src/lib/server/endpoints/openai/endpointOai.ts

@@ -0,0 +1,266 @@
+import { z } from "zod";
+import { openAICompletionToTextGenerationStream } from "./openAICompletionToTextGenerationStream";
+import {
+	openAIChatToTextGenerationSingle,
+	openAIChatToTextGenerationStream,
+} from "./openAIChatToTextGenerationStream";
+import type { CompletionCreateParamsStreaming } from "openai/resources/completions";
+import type {
+	ChatCompletionCreateParamsNonStreaming,
+	ChatCompletionCreateParamsStreaming,
+} from "openai/resources/chat/completions";
+import { buildPrompt } from "$lib/buildPrompt";
+import { config } from "$lib/server/config";
+import type { Endpoint } from "../endpoints";
+import type OpenAI from "openai";
+import { createImageProcessorOptionsValidator, makeImageProcessor } from "../images";
+import { prepareMessagesWithFiles } from "$lib/server/textGeneration/utils/prepareFiles";
+// uuid import removed (no tool call ids)
+
+export const endpointOAIParametersSchema = z.object({
+	weight: z.number().int().positive().default(1),
+	model: z.any(),
+	type: z.literal("openai"),
+	baseURL: z.string().url().default("https://api.openai.com/v1"),
+	// Canonical auth token is OPENAI_API_KEY; keep HF_TOKEN as legacy alias
+	apiKey: z.string().default(config.OPENAI_API_KEY || config.HF_TOKEN || "sk-"),
+	completion: z
+		.union([z.literal("completions"), z.literal("chat_completions")])
+		.default("chat_completions"),
+	defaultHeaders: z.record(z.string()).optional(),
+	defaultQuery: z.record(z.string()).optional(),
+	extraBody: z.record(z.any()).optional(),
+	multimodal: z
+		.object({
+			image: createImageProcessorOptionsValidator({
+				supportedMimeTypes: [
+					// Restrict to the most widely-supported formats
+					"image/png",
+					"image/jpeg",
+				],
+				preferredMimeType: "image/jpeg",
+				maxSizeInMB: 1,
+				maxWidth: 1024,
+				maxHeight: 1024,
+			}),
+		})
+		.default({}),
+	/* enable use of max_completion_tokens in place of max_tokens */
+	useCompletionTokens: z.boolean().default(false),
+	streamingSupported: z.boolean().default(true),
+});
+
+export async function endpointOai(
+	input: z.input<typeof endpointOAIParametersSchema>
+): Promise<Endpoint> {
+	const {
+		baseURL,
+		apiKey,
+		completion,
+		model,
+		defaultHeaders,
+		defaultQuery,
+		multimodal,
+		extraBody,
+		useCompletionTokens,
+		streamingSupported,
+	} = endpointOAIParametersSchema.parse(input);
+
+	let OpenAI;
+	try {
+		OpenAI = (await import("openai")).OpenAI;
+	} catch (e) {
+		throw new Error("Failed to import OpenAI", { cause: e });
+	}
+
+	// Store router metadata if captured
+	let routerMetadata: { route?: string; model?: string; provider?: string } = {};
+
+	// Custom fetch wrapper to capture response headers for router metadata
+	const customFetch = async (url: RequestInfo, init?: RequestInit): Promise<Response> => {
+		const response = await fetch(url, init);
+
+		// Capture router headers if present (fallback for non-streaming)
+		const routeHeader = response.headers.get("X-Router-Route");
+		const modelHeader = response.headers.get("X-Router-Model");
+		const providerHeader = response.headers.get("x-inference-provider");
+
+		if (routeHeader && modelHeader) {
+			routerMetadata = {
+				route: routeHeader,
+				model: modelHeader,
+				provider: providerHeader || undefined,
+			};
+		} else if (providerHeader) {
+			// Even without router metadata, capture provider info
+			routerMetadata = {
+				provider: providerHeader,
+			};
+		}
+
+		return response;
+	};
+
+	const openai = new OpenAI({
+		apiKey: apiKey || "sk-",
+		baseURL,
+		defaultHeaders: {
+			...(config.PUBLIC_APP_NAME === "HuggingChat" && { "User-Agent": "huggingchat" }),
+			...defaultHeaders,
+		},
+		defaultQuery,
+		fetch: customFetch,
+	});
+
+	const imageProcessor = makeImageProcessor(multimodal.image);
+
+	if (completion === "completions") {
+		return async ({
+			messages,
+			preprompt,
+			generateSettings,
+			conversationId,
+			locals,
+			abortSignal,
+			provider,
+		}) => {
+			const prompt = await buildPrompt({
+				messages,
+				preprompt,
+				model,
+			});
+
+			// Build model ID with optional provider suffix (e.g., "model:fastest" or "model:together")
+			const baseModelId = model.id ?? model.name;
+			const modelId = provider && provider !== "auto" ? `${baseModelId}:${provider}` : baseModelId;
+
+			const parameters = { ...model.parameters, ...generateSettings };
+			const body: CompletionCreateParamsStreaming = {
+				model: modelId,
+				prompt,
+				stream: true,
+				max_tokens: parameters?.max_tokens,
+				stop: parameters?.stop,
+				temperature: parameters?.temperature,
+				top_p: parameters?.top_p,
+				frequency_penalty: parameters?.frequency_penalty,
+				presence_penalty: parameters?.presence_penalty,
+			};
+
+			const openAICompletion = await openai.completions.create(body, {
+				body: { ...body, ...extraBody },
+				headers: {
+					"ChatUI-Conversation-ID": conversationId?.toString() ?? "",
+					"X-use-cache": "false",
+					...(locals?.token ? { Authorization: `Bearer ${locals.token}` } : {}),
+					// Bill to organization if configured
+					...(locals?.billingOrganization ? { "X-HF-Bill-To": locals.billingOrganization } : {}),
+				},
+				signal: abortSignal,
+			});
+
+			return openAICompletionToTextGenerationStream(openAICompletion);
+		};
+	} else if (completion === "chat_completions") {
+		return async ({
+			messages,
+			preprompt,
+			generateSettings,
+			conversationId,
+			isMultimodal,
+			locals,
+			abortSignal,
+			provider,
+		}) => {
+			// Format messages for the chat API, handling multimodal content if supported
+			let messagesOpenAI: OpenAI.Chat.Completions.ChatCompletionMessageParam[] =
+				await prepareMessagesWithFiles(messages, imageProcessor, isMultimodal ?? model.multimodal);
+
+			// Normalize preprompt and handle empty values
+			const normalizedPreprompt = typeof preprompt === "string" ? preprompt.trim() : "";
+
+			// Check if a system message already exists as the first message
+			const hasSystemMessage = messagesOpenAI.length > 0 && messagesOpenAI[0]?.role === "system";
+
+			if (hasSystemMessage) {
+				// Prepend normalized preprompt to existing system content when non-empty
+				if (normalizedPreprompt) {
+					const userSystemPrompt =
+						(typeof messagesOpenAI[0].content === "string"
+							? (messagesOpenAI[0].content as string)
+							: "") || "";
+					messagesOpenAI[0].content =
+						normalizedPreprompt + (userSystemPrompt ? "\n\n" + userSystemPrompt : "");
+				}
+			} else {
+				// Insert a system message only if the preprompt is non-empty
+				if (normalizedPreprompt) {
+					messagesOpenAI = [{ role: "system", content: normalizedPreprompt }, ...messagesOpenAI];
+				}
+			}
+
+			// Combine model defaults with request-specific parameters
+			const parameters = { ...model.parameters, ...generateSettings };
+
+			// Build model ID with optional provider suffix (e.g., "model:fastest" or "model:together")
+			const baseModelId = model.id ?? model.name;
+			const modelId = provider && provider !== "auto" ? `${baseModelId}:${provider}` : baseModelId;
+
+			const body = {
+				model: modelId,
+				messages: messagesOpenAI,
+				stream: streamingSupported,
+				// Support two different ways of specifying token limits depending on the model
+				...(useCompletionTokens
+					? { max_completion_tokens: parameters?.max_tokens }
+					: { max_tokens: parameters?.max_tokens }),
+				stop: parameters?.stop,
+				temperature: parameters?.temperature,
+				top_p: parameters?.top_p,
+				frequency_penalty: parameters?.frequency_penalty,
+				presence_penalty: parameters?.presence_penalty,
+			};
+
+			// Handle both streaming and non-streaming responses with appropriate processors
+			if (streamingSupported) {
+				const openChatAICompletion = await openai.chat.completions.create(
+					body as ChatCompletionCreateParamsStreaming,
+					{
+						body: { ...body, ...extraBody },
+						headers: {
+							"ChatUI-Conversation-ID": conversationId?.toString() ?? "",
+							"X-use-cache": "false",
+							...(locals?.token ? { Authorization: `Bearer ${locals.token}` } : {}),
+							// Bill to organization if configured
+							...(locals?.billingOrganization
+								? { "X-HF-Bill-To": locals.billingOrganization }
+								: {}),
+						},
+						signal: abortSignal,
+					}
+				);
+				return openAIChatToTextGenerationStream(openChatAICompletion, () => routerMetadata);
+			} else {
+				const openChatAICompletion = await openai.chat.completions.create(
+					body as ChatCompletionCreateParamsNonStreaming,
+					{
+						body: { ...body, ...extraBody },
+						headers: {
+							"ChatUI-Conversation-ID": conversationId?.toString() ?? "",
+							"X-use-cache": "false",
+							...(locals?.token ? { Authorization: `Bearer ${locals.token}` } : {}),
+							// Bill to organization if configured
+							...(locals?.billingOrganization
+								? { "X-HF-Bill-To": locals.billingOrganization }
+								: {}),
+						},
+						signal: abortSignal,
+					}
+				);
+				return openAIChatToTextGenerationSingle(openChatAICompletion, () => routerMetadata);
+			}
+		};
+	} else {
+		throw new Error("Invalid completion type");
+	}
+}

package/src/ruvocal/src/lib/server/endpoints/openai/openAIChatToTextGenerationStream.ts

@@ -0,0 +1,212 @@
+import type { TextGenerationStreamOutput } from "@huggingface/inference";
+import type OpenAI from "openai";
+import type { Stream } from "openai/streaming";
+
+/**
+ * Transform a stream of OpenAI.Chat.ChatCompletion into a stream of TextGenerationStreamOutput
+ */
+export async function* openAIChatToTextGenerationStream(
+	completionStream: Stream<OpenAI.Chat.Completions.ChatCompletionChunk>,
+	getRouterMetadata?: () => { route?: string; model?: string; provider?: string }
+) {
+	let generatedText = "";
+	let tokenId = 0;
+	let toolBuffer = ""; // legacy hack kept harmless
+	let metadataYielded = false;
+	let thinkOpen = false;
+
+	for await (const completion of completionStream) {
+		const retyped = completion as {
+			"x-router-metadata"?: { route: string; model: string; provider?: string };
+		};
+		// Check if this chunk contains router metadata (first chunk from llm-router)
+		if (!metadataYielded && retyped["x-router-metadata"]) {
+			const metadata = retyped["x-router-metadata"];
+			yield {
+				token: {
+					id: tokenId++,
+					text: "",
+					logprob: 0,
+					special: true,
+				},
+				generated_text: null,
+				details: null,
+				routerMetadata: {
+					route: metadata.route,
+					model: metadata.model,
+					provider: metadata.provider,
+				},
+			} as TextGenerationStreamOutput & {
+				routerMetadata: { route: string; model: string; provider?: string };
+			};
+			metadataYielded = true;
+			// Skip processing this chunk as content since it's just metadata
+			if (
+				!completion.choices ||
+				completion.choices.length === 0 ||
+				!completion.choices[0].delta?.content
+			) {
+				continue;
+			}
+		}
+		const { choices } = completion;
+		const delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta & {
+			reasoning?: string;
+			reasoning_content?: string;
+		} = choices?.[0]?.delta ?? {};
+		const content: string = delta.content ?? "";
+		const reasoning: string =
+			typeof delta?.reasoning === "string"
+				? (delta.reasoning as string)
+				: typeof delta?.reasoning_content === "string"
+					? (delta.reasoning_content as string)
+					: "";
+		const last = choices?.[0]?.finish_reason === "stop" || choices?.[0]?.finish_reason === "length";
+
+		// if the last token is a stop and the tool buffer is not empty, yield it as a generated_text
+		if (choices?.[0]?.finish_reason === "stop" && toolBuffer.length > 0) {
+			yield {
+				token: {
+					id: tokenId++,
+					special: true,
+					logprob: 0,
+					text: "",
+				},
+				generated_text: toolBuffer,
+				details: null,
+			} as TextGenerationStreamOutput;
+			break;
+		}
+
+		// weird bug where the parameters are streamed in like this
+		if (choices?.[0]?.delta?.tool_calls) {
+			const calls = Array.isArray(choices[0].delta.tool_calls)
+				? choices[0].delta.tool_calls
+				: [choices[0].delta.tool_calls];
+
+			if (
+				calls.length === 1 &&
+				calls[0].index === 0 &&
+				calls[0].id === "" &&
+				calls[0].type === "function" &&
+				!!calls[0].function &&
+				calls[0].function.name === null
+			) {
+				toolBuffer += calls[0].function.arguments;
+				continue;
+			}
+		}
+
+		let combined = "";
+		if (reasoning && reasoning.length > 0) {
+			if (!thinkOpen) {
+				combined += "<think>" + reasoning;
+				thinkOpen = true;
+			} else {
+				combined += reasoning;
+			}
+		}
+
+		if (content && content.length > 0) {
+			const trimmed = content.trim();
+			// Allow <think> tags in content to pass through (for models like DeepSeek R1)
+			if (thinkOpen && trimmed === "</think>") {
+				// close once without duplicating the tag
+				combined += "</think>";
+				thinkOpen = false;
+			} else if (thinkOpen) {
+				combined += "</think>" + content;
+				thinkOpen = false;
+			} else {
+				combined += content;
+			}
+		}
+
+		// Accumulate the combined token into the full text
+		generatedText += combined;
+		const output: TextGenerationStreamOutput = {
+			token: {
+				id: tokenId++,
+				text: combined,
+				logprob: 0,
+				special: last,
+			},
+			generated_text: last ? generatedText : null,
+			details: null,
+		};
+		yield output;
+
+		// Tools removed: ignore tool_calls deltas
+	}
+
+	// If metadata wasn't yielded from chunks (e.g., from headers), yield it at the end
+	if (!metadataYielded && getRouterMetadata) {
+		const routerMetadata = getRouterMetadata();
+		// Yield if we have either complete router metadata OR just provider info
+		if (
+			(routerMetadata && routerMetadata.route && routerMetadata.model) ||
+			routerMetadata?.provider
+		) {
+			yield {
+				token: {
+					id: tokenId++,
+					text: "",
+					logprob: 0,
+					special: true,
+				},
+				generated_text: null,
+				details: null,
+				routerMetadata,
+			} as TextGenerationStreamOutput & {
+				routerMetadata: { route?: string; model?: string; provider?: string };
+			};
+		}
+	}
+}
+
+/**
+ * Transform a non-streaming OpenAI chat completion into a stream of TextGenerationStreamOutput
+ */
+export async function* openAIChatToTextGenerationSingle(
+	completion: OpenAI.Chat.Completions.ChatCompletion,
+	getRouterMetadata?: () => { route?: string; model?: string; provider?: string }
+) {
+	const message: NonNullable<OpenAI.Chat.Completions.ChatCompletion.Choice>["message"] & {
+		reasoning?: string;
+		reasoning_content?: string;
+	} = completion.choices?.[0]?.message ?? {};
+	let content: string = message?.content || "";
+	// Provider-dependent reasoning shapes (non-streaming)
+	const r: string =
+		typeof message?.reasoning === "string"
+			? (message.reasoning as string)
+			: typeof message?.reasoning_content === "string"
+				? (message.reasoning_content as string)
+				: "";
+	if (r && r.length > 0) {
+		content = `<think>${r}</think>` + content;
+	}
+	const tokenId = 0;
+
+	// Yield the content as a single token
+	yield {
+		token: {
+			id: tokenId,
+			text: content,
+			logprob: 0,
+			special: false,
+		},
+		generated_text: content,
+		details: null,
+		...(getRouterMetadata
+			? (() => {
+					const metadata = getRouterMetadata();
+					return (metadata && metadata.route && metadata.model) || metadata?.provider
+						? { routerMetadata: metadata }
+						: {};
+				})()
+			: {}),
+	} as TextGenerationStreamOutput & {
+		routerMetadata?: { route?: string; model?: string; provider?: string };
+	};
+}

package/src/ruvocal/src/lib/server/endpoints/openai/openAICompletionToTextGenerationStream.ts

@@ -0,0 +1,32 @@
+import type { TextGenerationStreamOutput } from "@huggingface/inference";
+import type OpenAI from "openai";
+import type { Stream } from "openai/streaming";
+
+/**
+ * Transform a stream of OpenAI.Completions.Completion into a stream of TextGenerationStreamOutput
+ */
+export async function* openAICompletionToTextGenerationStream(
+	completionStream: Stream<OpenAI.Completions.Completion>
+) {
+	let generatedText = "";
+	let tokenId = 0;
+	for await (const completion of completionStream) {
+		const { choices } = completion;
+		const text = choices?.[0]?.text ?? "";
+		const last = choices?.[0]?.finish_reason === "stop" || choices?.[0]?.finish_reason === "length";
+		if (text) {
+			generatedText = generatedText + text;
+		}
+		const output: TextGenerationStreamOutput = {
+			token: {
+				id: tokenId++,
+				text,
+				logprob: 0,
+				special: last,
+			},
+			generated_text: last ? generatedText : null,
+			details: null,
+		};
+		yield output;
+	}
+}

package/src/ruvocal/src/lib/server/endpoints/preprocessMessages.ts

@@ -0,0 +1,61 @@
+import type { Message } from "$lib/types/Message";
+import type { EndpointMessage } from "./endpoints";
+import { downloadFile } from "../files/downloadFile";
+import type { ObjectId } from "mongodb";
+
+export async function preprocessMessages(
+	messages: Message[],
+	convId: ObjectId
+): Promise<EndpointMessage[]> {
+	return Promise.resolve(messages)
+		.then((msgs) => downloadFiles(msgs, convId))
+		.then((msgs) => injectClipboardFiles(msgs))
+		.then(stripEmptyInitialSystemMessage);
+}
+
+async function downloadFiles(messages: Message[], convId: ObjectId): Promise<EndpointMessage[]> {
+	return Promise.all(
+		messages.map<Promise<EndpointMessage>>((message) =>
+			Promise.all((message.files ?? []).map((file) => downloadFile(file.value, convId))).then(
+				(files) => ({ ...message, files })
+			)
+		)
+	);
+}
+
+async function injectClipboardFiles(messages: EndpointMessage[]) {
+	return Promise.all(
+		messages.map((message) => {
+			const plaintextFiles = message.files
+				?.filter((file) => file.mime === "application/vnd.chatui.clipboard")
+				.map((file) => Buffer.from(file.value, "base64").toString("utf-8"));
+
+			if (!plaintextFiles || plaintextFiles.length === 0) return message;
+
+			return {
+				...message,
+				content: `${plaintextFiles.join("\n\n")}\n\n${message.content}`,
+				files: message.files?.filter((file) => file.mime !== "application/vnd.chatui.clipboard"),
+			};
+		})
+	);
+}
+
+/**
+ * Remove an initial system message if its content is empty/whitespace only.
+ * This prevents sending an empty system prompt to any provider.
+ */
+function stripEmptyInitialSystemMessage(messages: EndpointMessage[]): EndpointMessage[] {
+	if (!messages?.length) return messages;
+	const first = messages[0];
+	if (first?.from !== "system") return messages;
+
+	const content = first?.content as unknown;
+	const isEmpty = typeof content === "string" ? content.trim().length === 0 : false;
+
+	if (isEmpty) {
+		return messages.slice(1);
+	}
+
+	return messages;
+}

package/src/ruvocal/src/lib/server/exitHandler.ts

@@ -0,0 +1,59 @@
+import { randomUUID } from "$lib/utils/randomUuid";
+import { timeout } from "$lib/utils/timeout";
+import { logger } from "./logger";
+
+type ExitHandler = () => void | Promise<void>;
+type ExitHandlerUnsubscribe = () => void;
+
+const listeners = new Map<string, ExitHandler>();
+
+export function onExit(cb: ExitHandler): ExitHandlerUnsubscribe {
+	const uuid = randomUUID();
+	listeners.set(uuid, cb);
+	return () => {
+		listeners.delete(uuid);
+	};
+}
+
+async function runExitHandler(handler: ExitHandler): Promise<void> {
+	return timeout(Promise.resolve().then(handler), 30_000).catch((err) => {
+		logger.error(err, "Exit handler failed to run");
+	});
+}
+
+export function initExitHandler() {
+	let signalCount = 0;
+	const exitHandler = async () => {
+		if (signalCount === 1) {
+			logger.info("Received signal... Exiting");
+			await Promise.all(Array.from(listeners.values()).map(runExitHandler));
+			logger.info("All exit handlers ran... Waiting for svelte server to exit");
+		}
+	};
+
+	process.on("SIGINT", () => {
+		signalCount++;
+
+		if (signalCount >= 2) {
+			process.kill(process.pid, "SIGKILL");
+		} else {
+			exitHandler().catch((err) => {
+				logger.error(err, "Error in exit handler on SIGINT:");
+				process.kill(process.pid, "SIGKILL");
+			});
+		}
+	});
+
+	process.on("SIGTERM", () => {
+		signalCount++;
+
+		if (signalCount >= 2) {
+			process.kill(process.pid, "SIGKILL");
+		} else {
+			exitHandler().catch((err) => {
+				logger.error(err, "Error in exit handler on SIGTERM:");
+				process.kill(process.pid, "SIGKILL");
+			});
+		}
+	});
+}

package/src/ruvocal/src/lib/server/files/downloadFile.ts

@@ -0,0 +1,34 @@
+import { error } from "@sveltejs/kit";
+import { collections } from "$lib/server/database";
+import type { Conversation } from "$lib/types/Conversation";
+import type { SharedConversation } from "$lib/types/SharedConversation";
+import type { MessageFile } from "$lib/types/Message";
+
+export async function downloadFile(
+	sha256: string,
+	convId: Conversation["_id"] | SharedConversation["_id"]
+): Promise<MessageFile & { type: "base64" }> {
+	const fileId = collections.bucket.find({ filename: `${convId.toString()}-${sha256}` });
+
+	const file = await fileId.next();
+	if (!file) {
+		error(404, "File not found");
+	}
+	if (file.metadata?.conversation !== convId.toString()) {
+		error(403, "You don't have access to this file.");
+	}
+
+	const mime = file.metadata?.mime;
+	const name = file.filename;
+
+	const fileStream = collections.bucket.openDownloadStream(file._id);
+
+	const buffer = await new Promise<Buffer>((resolve, reject) => {
+		const chunks: Uint8Array[] = [];
+		fileStream.on("data", (chunk) => chunks.push(chunk));
+		fileStream.on("error", reject);
+		fileStream.on("end", () => resolve(Buffer.concat(chunks)));
+	});
+
+	return { type: "base64", name, value: buffer.toString("base64"), mime };
+}

package/src/ruvocal/src/lib/server/files/uploadFile.ts

@@ -0,0 +1,29 @@
+import type { Conversation } from "$lib/types/Conversation";
+import type { MessageFile } from "$lib/types/Message";
+import { sha256 } from "$lib/utils/sha256";
+import { fileTypeFromBuffer } from "file-type";
+import { collections } from "$lib/server/database";
+
+export async function uploadFile(file: File, conv: Conversation): Promise<MessageFile> {
+	const sha = await sha256(await file.text());
+	const buffer = await file.arrayBuffer();
+
+	// Attempt to detect the mime type of the file, fallback to the uploaded mime
+	const mime = await fileTypeFromBuffer(buffer).then((fileType) => fileType?.mime ?? file.type);
+
+	const upload = collections.bucket.openUploadStream(`${conv._id}-${sha}`, {
+		metadata: { conversation: conv._id.toString(), mime },
+	});
+
+	upload.write((await file.arrayBuffer()) as unknown as Buffer);
+	upload.end();
+
+	// only return the filename when upload throws a finish event or a 20s time out occurs
+	return new Promise((resolve, reject) => {
+		upload.once("finish", () =>
+			resolve({ type: "hash", value: sha, mime: file.type, name: file.name })
+		);
+		upload.once("error", reject);
+		setTimeout(() => reject(new Error("Upload timed out")), 20_000);
+	});
+}