ruflo 3.5.2 → 3.5.3

package/src/ruvocal/src/lib/server/models.ts

@@ -0,0 +1,518 @@
+import { config } from "$lib/server/config";
+import type { ChatTemplateInput } from "$lib/types/Template";
+import { z } from "zod";
+import endpoints, { endpointSchema, type Endpoint } from "./endpoints/endpoints";
+
+import JSON5 from "json5";
+import { logger } from "$lib/server/logger";
+import { makeRouterEndpoint } from "$lib/server/router/endpoint";
+
+type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
+
+const sanitizeJSONEnv = (val: string, fallback: string) => {
+	const raw = (val ?? "").trim();
+	const unquoted = raw.startsWith("`") && raw.endsWith("`") ? raw.slice(1, -1) : raw;
+	return unquoted || fallback;
+};
+
+const modelConfig = z.object({
+	/** Used as an identifier in DB */
+	id: z.string().optional(),
+	/** Used to link to the model page, and for inference */
+	name: z.string().default(""),
+	displayName: z.string().min(1).optional(),
+	description: z.string().min(1).optional(),
+	logoUrl: z.string().url().optional(),
+	websiteUrl: z.string().url().optional(),
+	modelUrl: z.string().url().optional(),
+	tokenizer: z.never().optional(),
+	datasetName: z.string().min(1).optional(),
+	datasetUrl: z.string().url().optional(),
+	preprompt: z.string().default(""),
+	prepromptUrl: z.string().url().optional(),
+	chatPromptTemplate: z.never().optional(),
+	promptExamples: z
+		.array(
+			z.object({
+				title: z.string().min(1),
+				prompt: z.string().min(1),
+			})
+		)
+		.optional(),
+	endpoints: z.array(endpointSchema).optional(),
+	providers: z.array(z.object({ supports_tools: z.boolean().optional() }).passthrough()).optional(),
+	parameters: z
+		.object({
+			temperature: z.number().min(0).max(2).optional(),
+			truncate: z.number().int().positive().optional(),
+			max_tokens: z.number().int().positive().optional(),
+			stop: z.array(z.string()).optional(),
+			top_p: z.number().positive().optional(),
+			top_k: z.number().positive().optional(),
+			frequency_penalty: z.number().min(-2).max(2).optional(),
+			presence_penalty: z.number().min(-2).max(2).optional(),
+		})
+		.passthrough()
+		.optional(),
+	multimodal: z.boolean().default(false),
+	multimodalAcceptedMimetypes: z.array(z.string()).optional(),
+	// Aggregated tool-calling capability across providers (HF router)
+	supportsTools: z.boolean().default(false),
+	unlisted: z.boolean().default(false),
+	embeddingModel: z.never().optional(),
+	/** Used to enable/disable system prompt usage */
+	systemRoleSupported: z.boolean().default(true),
+});
+
+type ModelConfig = z.infer<typeof modelConfig>;
+
+const overrideEntrySchema = modelConfig
+	.partial()
+	.extend({
+		id: z.string().optional(),
+		name: z.string().optional(),
+	})
+	.refine((value) => Boolean((value.id ?? value.name)?.trim()), {
+		message: "Model override entry must provide an id or name",
+	});
+
+type ModelOverride = z.infer<typeof overrideEntrySchema>;
+
+const openaiBaseUrl = config.OPENAI_BASE_URL
+	? config.OPENAI_BASE_URL.replace(/\/$/, "")
+	: undefined;
+const isHFRouter = openaiBaseUrl === "https://router.huggingface.co/v1";
+
+const listSchema = z
+	.object({
+		data: z.array(
+			z.object({
+				id: z.string(),
+				description: z.string().optional(),
+				providers: z
+					.array(z.object({ supports_tools: z.boolean().optional() }).passthrough())
+					.optional(),
+				architecture: z
+					.object({
+						input_modalities: z.array(z.string()).optional(),
+					})
+					.passthrough()
+					.optional(),
+			})
+		),
+	})
+	.passthrough();
+
+function getChatPromptRender(_m: ModelConfig): (inputs: ChatTemplateInput) => string {
+	// Minimal template to support legacy "completions" flow if ever used.
+	// We avoid any tokenizer/Jinja usage in this build.
+	return ({ messages, preprompt }) => {
+		const parts: string[] = [];
+		if (preprompt) parts.push(`[SYSTEM]\n${preprompt}`);
+		for (const msg of messages) {
+			const role = msg.from === "assistant" ? "ASSISTANT" : msg.from.toUpperCase();
+			parts.push(`[${role}]\n${msg.content}`);
+		}
+		parts.push(`[ASSISTANT]`);
+		return parts.join("\n\n");
+	};
+}
+
+const processModel = async (m: ModelConfig) => ({
+	...m,
+	chatPromptRender: await getChatPromptRender(m),
+	id: m.id || m.name,
+	displayName: m.displayName || m.name,
+	preprompt: m.prepromptUrl ? await fetch(m.prepromptUrl).then((r) => r.text()) : m.preprompt,
+	parameters: { ...m.parameters, stop_sequences: m.parameters?.stop },
+	unlisted: m.unlisted ?? false,
+});
+
+const addEndpoint = (m: Awaited<ReturnType<typeof processModel>>) => ({
+	...m,
+	getEndpoint: async (): Promise<Endpoint> => {
+		if (!m.endpoints || m.endpoints.length === 0) {
+			throw new Error("No endpoints configured. This build requires OpenAI-compatible endpoints.");
+		}
+		// Only support OpenAI-compatible endpoints in this build
+		const endpoint = m.endpoints[0];
+		if (endpoint.type !== "openai") {
+			throw new Error("Only 'openai' endpoint type is supported in this build");
+		}
+		return await endpoints.openai({ ...endpoint, model: m });
+	},
+});
+
+type InternalProcessedModel = Awaited<ReturnType<typeof addEndpoint>> & {
+	isRouter: boolean;
+	hasInferenceAPI: boolean;
+};
+
+const inferenceApiIds: string[] = [];
+
+const getModelOverrides = (): ModelOverride[] => {
+	const overridesEnv = (Reflect.get(config, "MODELS") as string | undefined) ?? "";
+
+	if (!overridesEnv.trim()) {
+		return [];
+	}
+
+	try {
+		return z.array(overrideEntrySchema).parse(JSON5.parse(sanitizeJSONEnv(overridesEnv, "[]")));
+	} catch (error) {
+		logger.error(error, "[models] Failed to parse MODELS overrides");
+		return [];
+	}
+};
+
+export type ModelsRefreshSummary = {
+	refreshedAt: Date;
+	durationMs: number;
+	added: string[];
+	removed: string[];
+	changed: string[];
+	total: number;
+};
+
+export type ProcessedModel = InternalProcessedModel;
+
+export let models: ProcessedModel[] = [];
+export let defaultModel!: ProcessedModel;
+export let taskModel!: ProcessedModel;
+export let validModelIdSchema: z.ZodType<string> = z.string();
+export let lastModelRefresh = new Date(0);
+export let lastModelRefreshDurationMs = 0;
+export let lastModelRefreshSummary: ModelsRefreshSummary = {
+	refreshedAt: new Date(0),
+	durationMs: 0,
+	added: [],
+	removed: [],
+	changed: [],
+	total: 0,
+};
+
+let inflightRefresh: Promise<ModelsRefreshSummary> | null = null;
+
+const createValidModelIdSchema = (modelList: ProcessedModel[]): z.ZodType<string> => {
+	if (modelList.length === 0) {
+		throw new Error("No models available to build validation schema");
+	}
+	const ids = new Set(modelList.map((m) => m.id));
+	return z.string().refine((value) => ids.has(value), "Invalid model id");
+};
+
+const resolveTaskModel = (modelList: ProcessedModel[]) => {
+	if (modelList.length === 0) {
+		throw new Error("No models available to select task model");
+	}
+
+	if (config.TASK_MODEL) {
+		const preferred = modelList.find(
+			(m) => m.name === config.TASK_MODEL || m.id === config.TASK_MODEL
+		);
+		if (preferred) {
+			return preferred;
+		}
+	}
+
+	return modelList[0];
+};
+
+const signatureForModel = (model: ProcessedModel) =>
+	JSON.stringify({
+		description: model.description,
+		displayName: model.displayName,
+		providers: model.providers,
+		parameters: model.parameters,
+		preprompt: model.preprompt,
+		prepromptUrl: model.prepromptUrl,
+		endpoints:
+			model.endpoints?.map((endpoint) => {
+				if (endpoint.type === "openai") {
+					const { type, baseURL } = endpoint;
+					return { type, baseURL };
+				}
+				return { type: endpoint.type };
+			}) ?? null,
+		multimodal: model.multimodal,
+		multimodalAcceptedMimetypes: model.multimodalAcceptedMimetypes,
+		supportsTools: (model as unknown as { supportsTools?: boolean }).supportsTools ?? false,
+		isRouter: model.isRouter,
+		hasInferenceAPI: model.hasInferenceAPI,
+	});
+
+const applyModelState = (newModels: ProcessedModel[], startedAt: number): ModelsRefreshSummary => {
+	if (newModels.length === 0) {
+		throw new Error("Failed to load any models from upstream");
+	}
+
+	const previousIds = new Set(models.map((m) => m.id));
+	const previousSignatures = new Map(models.map((m) => [m.id, signatureForModel(m)]));
+	const refreshedAt = new Date();
+	const durationMs = Date.now() - startedAt;
+
+	models = newModels;
+	defaultModel = models[0];
+	taskModel = resolveTaskModel(models);
+	validModelIdSchema = createValidModelIdSchema(models);
+	lastModelRefresh = refreshedAt;
+	lastModelRefreshDurationMs = durationMs;
+
+	const added = newModels.map((m) => m.id).filter((id) => !previousIds.has(id));
+	const removed = Array.from(previousIds).filter(
+		(id) => !newModels.some((model) => model.id === id)
+	);
+	const changed = newModels
+		.filter((model) => {
+			const previousSignature = previousSignatures.get(model.id);
+			return previousSignature !== undefined && previousSignature !== signatureForModel(model);
+		})
+		.map((model) => model.id);
+
+	const summary: ModelsRefreshSummary = {
+		refreshedAt,
+		durationMs,
+		added,
+		removed,
+		changed,
+		total: models.length,
+	};
+
+	lastModelRefreshSummary = summary;
+
+	logger.info(
+		{
+			total: summary.total,
+			added: summary.added,
+			removed: summary.removed,
+			changed: summary.changed,
+			durationMs: summary.durationMs,
+		},
+		"[models] Model cache refreshed"
+	);
+
+	return summary;
+};
+
+const buildModels = async (): Promise<ProcessedModel[]> => {
+	if (!openaiBaseUrl) {
+		logger.error(
+			"OPENAI_BASE_URL is required. Set it to an OpenAI-compatible base (e.g., https://router.huggingface.co/v1)."
+		);
+		throw new Error("OPENAI_BASE_URL not set");
+	}
+
+	try {
+		const baseURL = openaiBaseUrl;
+		logger.info({ baseURL }, "[models] Using OpenAI-compatible base URL");
+
+		// Canonical auth token is OPENAI_API_KEY; keep HF_TOKEN as legacy alias
+		const authToken = config.OPENAI_API_KEY || config.HF_TOKEN;
+
+		// Use auth token from the start if available to avoid rate limiting issues
+		// Some APIs rate-limit unauthenticated requests more aggressively
+		const response = await fetch(`${baseURL}/models`, {
+			headers: authToken ? { Authorization: `Bearer ${authToken}` } : undefined,
+		});
+		logger.info({ status: response.status }, "[models] First fetch status");
+		if (!response.ok && response.status === 401 && !authToken) {
+			// If we get 401 and didn't have a token, there's nothing we can do
+			throw new Error(
+				`Failed to fetch ${baseURL}/models: ${response.status} ${response.statusText} (no auth token available)`
+			);
+		}
+		if (!response.ok) {
+			throw new Error(
+				`Failed to fetch ${baseURL}/models: ${response.status} ${response.statusText}`
+			);
+		}
+		const json = await response.json();
+		logger.info({ keys: Object.keys(json || {}) }, "[models] Response keys");
+
+		const parsed = listSchema.parse(json);
+		logger.info({ count: parsed.data.length }, "[models] Parsed models count");
+
+		let modelsRaw = parsed.data.map((m) => {
+			let logoUrl: string | undefined = undefined;
+			if (isHFRouter && m.id.includes("/")) {
+				const org = m.id.split("/")[0];
+				logoUrl = `https://huggingface.co/api/avatars/${encodeURIComponent(org)}`;
+			}
+
+			const inputModalities = (m.architecture?.input_modalities ?? []).map((modality) =>
+				modality.toLowerCase()
+			);
+			const supportsImageInput =
+				inputModalities.includes("image") || inputModalities.includes("vision");
+
+			// If any provider supports tools, consider the model as supporting tools
+			const supportsTools = Boolean((m.providers ?? []).some((p) => p?.supports_tools === true));
+			return {
+				id: m.id,
+				name: m.id,
+				displayName: m.id,
+				description: m.description,
+				logoUrl,
+				providers: m.providers,
+				multimodal: supportsImageInput,
+				multimodalAcceptedMimetypes: supportsImageInput ? ["image/*"] : undefined,
+				supportsTools,
+				endpoints: [
+					{
+						type: "openai" as const,
+						baseURL,
+						// apiKey will be taken from OPENAI_API_KEY or HF_TOKEN automatically
+					},
+				],
+			} as ModelConfig;
+		}) as ModelConfig[];
+
+		const overrides = getModelOverrides();
+
+		if (overrides.length) {
+			const overrideMap = new Map<string, ModelOverride>();
+			for (const override of overrides) {
+				for (const key of [override.id, override.name]) {
+					const trimmed = key?.trim();
+					if (trimmed) overrideMap.set(trimmed, override);
+				}
+			}
+
+			// Filter to only configured models and apply overrides, preserving MODELS order
+			const filteredAndOrdered: ModelConfig[] = [];
+			for (const override of overrides) {
+				const matchKey = override.name?.trim() || override.id?.trim() || "";
+				const found = modelsRaw.find(
+					(model) => model.id === matchKey || model.name === matchKey
+				);
+				if (found) {
+					const { id, name, ...rest } = override;
+					void id;
+					void name;
+					filteredAndOrdered.push({ ...found, ...rest });
+				}
+			}
+
+			// If we matched at least one, use filtered list; otherwise fall back to all models with overrides
+			if (filteredAndOrdered.length > 0) {
+				modelsRaw = filteredAndOrdered;
+			} else {
+				modelsRaw = modelsRaw.map((model) => {
+					const override = overrideMap.get(model.id ?? "") ?? overrideMap.get(model.name ?? "");
+					if (!override) return model;
+
+					const { id, name, ...rest } = override;
+					void id;
+					void name;
+
+					return {
+						...model,
+						...rest,
+					};
+				});
+			}
+		}
+
+		const builtModels = await Promise.all(
+			modelsRaw.map((e) =>
+				processModel(e)
+					.then(addEndpoint)
+					.then(async (m) => ({
+						...m,
+						hasInferenceAPI: inferenceApiIds.includes(m.id ?? m.name),
+						// router decoration added later
+						isRouter: false as boolean,
+					}))
+			)
+		);
+
+		const archBase = (config.LLM_ROUTER_ARCH_BASE_URL || "").trim();
+		const routerLabel = (config.PUBLIC_LLM_ROUTER_DISPLAY_NAME || "Omni").trim() || "Omni";
+		const routerLogo = (config.PUBLIC_LLM_ROUTER_LOGO_URL || "").trim();
+		const routerAliasId = (config.PUBLIC_LLM_ROUTER_ALIAS_ID || "omni").trim() || "omni";
+		const routerMultimodalEnabled =
+			(config.LLM_ROUTER_ENABLE_MULTIMODAL || "").toLowerCase() === "true";
+		const routerToolsEnabled = (config.LLM_ROUTER_ENABLE_TOOLS || "").toLowerCase() === "true";
+
+		let decorated = builtModels as ProcessedModel[];
+
+		if (archBase) {
+			// Build a minimal model config for the alias
+			const aliasRaw = {
+				id: routerAliasId,
+				name: routerAliasId,
+				displayName: routerLabel,
+				description: "Automatically routes your messages to the best model for your request.",
+				logoUrl: routerLogo || undefined,
+				preprompt: "",
+				endpoints: [
+					{
+						type: "openai" as const,
+						baseURL: openaiBaseUrl,
+					},
+				],
+				// Keep the alias visible
+				unlisted: false,
+			} as ModelConfig;
+
+			if (routerMultimodalEnabled) {
+				aliasRaw.multimodal = true;
+				aliasRaw.multimodalAcceptedMimetypes = ["image/*"];
+			}
+
+			if (routerToolsEnabled) {
+				aliasRaw.supportsTools = true;
+			}
+
+			const aliasBase = await processModel(aliasRaw);
+			// Create a self-referential ProcessedModel for the router endpoint
+			const aliasModel: ProcessedModel = {
+				...aliasBase,
+				isRouter: true,
+				hasInferenceAPI: false,
+				// getEndpoint uses the router wrapper regardless of the endpoints array
+				getEndpoint: async (): Promise<Endpoint> => makeRouterEndpoint(aliasModel),
+			} as ProcessedModel;
+
+			// Put alias first
+			decorated = [aliasModel, ...decorated];
+		}
+
+		return decorated;
+	} catch (e) {
+		logger.error(e, "Failed to load models from OpenAI base URL");
+		throw e;
+	}
+};
+
+const rebuildModels = async (): Promise<ModelsRefreshSummary> => {
+	const startedAt = Date.now();
+	const newModels = await buildModels();
+	return applyModelState(newModels, startedAt);
+};
+
+await rebuildModels();
+
+export const refreshModels = async (): Promise<ModelsRefreshSummary> => {
+	if (inflightRefresh) {
+		return inflightRefresh;
+	}
+
+	inflightRefresh = rebuildModels().finally(() => {
+		inflightRefresh = null;
+	});
+
+	return inflightRefresh;
+};
+
+export const validateModel = (_models: BackendModel[]) => {
+	// Zod enum function requires 2 parameters
+	return z.enum([_models[0].id, ..._models.slice(1).map((m) => m.id)]);
+};
+
+// if `TASK_MODEL` is string & name of a model in `MODELS`, then we use `MODELS[TASK_MODEL]`, else we try to parse `TASK_MODEL` as a model config itself
+
+export type BackendModel = Optional<
+	typeof defaultModel,
+	"preprompt" | "parameters" | "multimodal" | "unlisted" | "hasInferenceAPI"
+>;

package/src/ruvocal/src/lib/server/requestContext.ts

@@ -0,0 +1,55 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { randomUUID } from "node:crypto";
+
+export interface RequestContext {
+	requestId: string;
+	url?: string;
+	ip?: string;
+	user?: string;
+	statusCode?: number;
+}
+
+const asyncLocalStorage = new AsyncLocalStorage<RequestContext>();
+
+/**
+ * Run a function within a request context.
+ * All logs within this context will automatically include the requestId.
+ */
+export function runWithRequestContext<T>(
+	fn: () => T,
+	context: Partial<RequestContext> & { requestId?: string } = {}
+): T {
+	const fullContext: RequestContext = {
+		requestId: context.requestId ?? randomUUID(),
+		url: context.url,
+		ip: context.ip,
+		user: context.user,
+		statusCode: context.statusCode,
+	};
+	return asyncLocalStorage.run(fullContext, fn);
+}
+
+/**
+ * Update the current request context with additional information.
+ * Useful for adding user information after authentication.
+ */
+export function updateRequestContext(updates: Partial<Omit<RequestContext, "requestId">>): void {
+	const store = asyncLocalStorage.getStore();
+	if (store) {
+		Object.assign(store, updates);
+	}
+}
+
+/**
+ * Get the current request context, if any.
+ */
+export function getRequestContext(): RequestContext | undefined {
+	return asyncLocalStorage.getStore();
+}
+
+/**
+ * Get the current request ID, or undefined if not in a request context.
+ */
+export function getRequestId(): string | undefined {
+	return asyncLocalStorage.getStore()?.requestId;
+}