ruflo 3.5.2 → 3.5.3

package/src/ruvocal/src/routes/admin/export/+server.ts

@@ -0,0 +1,159 @@
+import { config } from "$lib/server/config";
+import { collections } from "$lib/server/database";
+import type { Message } from "$lib/types/Message";
+import { error } from "@sveltejs/kit";
+import { pathToFileURL } from "node:url";
+import { unlink } from "node:fs/promises";
+import { uploadFile } from "@huggingface/hub";
+import parquet from "parquetjs";
+import { z } from "zod";
+import { logger } from "$lib/server/logger.js";
+
+// Triger like this:
+// curl -X POST "http://localhost:5173/chat/admin/export" -H "Authorization: Bearer <ADMIN_API_SECRET>" -H "Content-Type: application/json" -d '{"model": "OpenAssistant/oasst-sft-6-llama-30b-xor"}'
+
+export async function POST({ request }) {
+	if (!config.PARQUET_EXPORT_DATASET || !config.PARQUET_EXPORT_HF_TOKEN) {
+		error(500, "Parquet export is not configured.");
+	}
+
+	const { model } = z
+		.object({
+			model: z.string(),
+		})
+		.parse(await request.json());
+
+	const schema = new parquet.ParquetSchema({
+		title: { type: "UTF8" },
+		created_at: { type: "TIMESTAMP_MILLIS" },
+		updated_at: { type: "TIMESTAMP_MILLIS" },
+		messages: {
+			repeated: true,
+			fields: {
+				from: { type: "UTF8" },
+				content: { type: "UTF8" },
+				score: { type: "INT_8", optional: true },
+			},
+		},
+	});
+
+	const fileName = `/tmp/conversations-${new Date().toJSON().slice(0, 10)}-${Date.now()}.parquet`;
+
+	const writer = await parquet.ParquetWriter.openFile(schema, fileName);
+
+	let count = 0;
+	logger.info("Exporting conversations for model", model);
+
+	for await (const conversation of collections.settings.aggregate<{
+		title: string;
+		created_at: Date;
+		updated_at: Date;
+		messages: Message[];
+	}>([
+		{
+			$match: {
+				shareConversationsWithModelAuthors: true,
+				sessionId: { $exists: true },
+				userId: { $exists: false },
+			},
+		},
+		{
+			$lookup: {
+				from: "conversations",
+				localField: "sessionId",
+				foreignField: "sessionId",
+				as: "conversations",
+				pipeline: [{ $match: { model, userId: { $exists: false } } }],
+			},
+		},
+		{ $unwind: "$conversations" },
+		{
+			$project: {
+				title: "$conversations.title",
+				created_at: "$conversations.createdAt",
+				updated_at: "$conversations.updatedAt",
+				messages: "$conversations.messages",
+			},
+		},
+	])) {
+		await writer.appendRow({
+			title: conversation.title,
+			created_at: conversation.created_at,
+			updated_at: conversation.updated_at,
+			messages: conversation.messages.map((message: Message) => ({
+				from: message.from,
+				content: message.content,
+				...(message.score ? { score: message.score } : undefined),
+			})),
+		});
+		++count;
+
+		if (count % 1_000 === 0) {
+			logger.info("Exported", count, "conversations");
+		}
+	}
+
+	logger.info("exporting convos with userId");
+
+	for await (const conversation of collections.settings.aggregate<{
+		title: string;
+		created_at: Date;
+		updated_at: Date;
+		messages: Message[];
+	}>([
+		{ $match: { shareConversationsWithModelAuthors: true, userId: { $exists: true } } },
+		{
+			$lookup: {
+				from: "conversations",
+				localField: "userId",
+				foreignField: "userId",
+				as: "conversations",
+				pipeline: [{ $match: { model } }],
+			},
+		},
+		{ $unwind: "$conversations" },
+		{
+			$project: {
+				title: "$conversations.title",
+				created_at: "$conversations.createdAt",
+				updated_at: "$conversations.updatedAt",
+				messages: "$conversations.messages",
+			},
+		},
+	])) {
+		await writer.appendRow({
+			title: conversation.title,
+			created_at: conversation.created_at,
+			updated_at: conversation.updated_at,
+			messages: conversation.messages.map((message: Message) => ({
+				from: message.from,
+				content: message.content,
+				...(message.score ? { score: message.score } : undefined),
+			})),
+		});
+		++count;
+
+		if (count % 1_000 === 0) {
+			logger.info("Exported", count, "conversations");
+		}
+	}
+
+	await writer.close();
+
+	logger.info("Uploading", fileName, "to Hugging Face Hub");
+
+	await uploadFile({
+		file: pathToFileURL(fileName) as URL,
+		credentials: { accessToken: config.PARQUET_EXPORT_HF_TOKEN },
+		repo: {
+			type: "dataset",
+			name: config.PARQUET_EXPORT_DATASET,
+		},
+	});
+
+	logger.info("Upload done");
+
+	await unlink(fileName);
+
+	return new Response();
+}

package/src/ruvocal/src/routes/admin/stats/compute/+server.ts

@@ -0,0 +1,16 @@
+import { json } from "@sveltejs/kit";
+import { logger } from "$lib/server/logger";
+import { computeAllStats } from "$lib/jobs/refresh-conversation-stats";
+
+// Triger like this:
+// curl -X POST "http://localhost:5173/chat/admin/stats/compute" -H "Authorization: Bearer <ADMIN_API_SECRET>"
+
+export async function POST() {
+	computeAllStats().catch((e) => logger.error(e, "Error computing all stats"));
+	return json(
+		{
+			message: "Stats job started",
+		},
+		{ status: 202 }
+	);
+}

package/src/ruvocal/src/routes/api/conversation/[id]/+server.ts

@@ -0,0 +1,40 @@
+import { collections } from "$lib/server/database";
+import { authCondition } from "$lib/server/auth";
+import { z } from "zod";
+import { ObjectId } from "mongodb";
+
+export async function GET({ locals, params }) {
+	const id = z.string().parse(params.id);
+	const convId = new ObjectId(id);
+
+	if (locals.user?._id || locals.sessionId) {
+		const conv = await collections.conversations.findOne({
+			_id: convId,
+			...authCondition(locals),
+		});
+
+		if (conv) {
+			const res = {
+				id: conv._id,
+				title: conv.title,
+				updatedAt: conv.updatedAt,
+				modelId: conv.model,
+				messages: conv.messages.map((message) => ({
+					content: message.content,
+					from: message.from,
+					id: message.id,
+					createdAt: message.createdAt,
+					updatedAt: message.updatedAt,
+					// websearch removed
+					files: message.files,
+					updates: message.updates,
+				})),
+			};
+			return Response.json(res);
+		} else {
+			return Response.json({ message: "Conversation not found" }, { status: 404 });
+		}
+	} else {
+		return Response.json({ message: "Must have session cookie" }, { status: 401 });
+	}
+}

package/src/ruvocal/src/routes/api/conversation/[id]/message/[messageId]/+server.ts

@@ -0,0 +1,42 @@
+import { authCondition } from "$lib/server/auth";
+import { collections } from "$lib/server/database";
+import { error } from "@sveltejs/kit";
+import { ObjectId } from "mongodb";
+
+export async function DELETE({ locals, params }) {
+	const messageId = params.messageId;
+
+	if (!messageId || typeof messageId !== "string") {
+		error(400, "Invalid message id");
+	}
+
+	const conversation = await collections.conversations.findOne({
+		...authCondition(locals),
+		_id: new ObjectId(params.id),
+	});
+
+	if (!conversation) {
+		error(404, "Conversation not found");
+	}
+
+	const filteredMessages = conversation.messages
+		.filter(
+			(message) =>
+				// not the message AND the message is not in ancestors
+				!(message.id === messageId) && message.ancestors && !message.ancestors.includes(messageId)
+		)
+		.map((message) => {
+			// remove the message from children if it's there
+			if (message.children && message.children.includes(messageId)) {
+				message.children = message.children.filter((child) => child !== messageId);
+			}
+			return message;
+		});
+
+	await collections.conversations.updateOne(
+		{ _id: conversation._id, ...authCondition(locals) },
+		{ $set: { messages: filteredMessages } }
+	);
+
+	return new Response();
+}

package/src/ruvocal/src/routes/api/conversations/+server.ts

@@ -0,0 +1,48 @@
+import { collections } from "$lib/server/database";
+import { authCondition } from "$lib/server/auth";
+import type { Conversation } from "$lib/types/Conversation";
+import { CONV_NUM_PER_PAGE } from "$lib/constants/pagination";
+
+export async function GET({ locals, url }) {
+	const p = parseInt(url.searchParams.get("p") ?? "0");
+	if (locals.user?._id || locals.sessionId) {
+		const convs = await collections.conversations
+			.find({
+				...authCondition(locals),
+			})
+			.project<Pick<Conversation, "_id" | "title" | "updatedAt" | "model" | never>>({
+				title: 1,
+				updatedAt: 1,
+				model: 1,
+			})
+			.sort({ updatedAt: -1 })
+			.skip(p * CONV_NUM_PER_PAGE)
+			.limit(CONV_NUM_PER_PAGE)
+			.toArray();
+
+		if (convs.length === 0) {
+			return Response.json([]);
+		}
+		const res = convs.map((conv) => ({
+			_id: conv._id,
+			id: conv._id, // legacy param iOS
+			title: conv.title,
+			updatedAt: conv.updatedAt,
+			model: conv.model,
+			modelId: conv.model, // legacy param iOS
+		}));
+		return Response.json(res);
+	} else {
+		return Response.json({ message: "Must have session cookie" }, { status: 401 });
+	}
+}
+
+export async function DELETE({ locals }) {
+	if (locals.user?._id || locals.sessionId) {
+		await collections.conversations.deleteMany({
+			...authCondition(locals),
+		});
+	}
+
+	return new Response();
+}

package/src/ruvocal/src/routes/api/fetch-url/+server.ts

@@ -0,0 +1,147 @@
+import { error } from "@sveltejs/kit";
+import { logger } from "$lib/server/logger.js";
+import { Agent, fetch } from "undici";
+import { isValidUrl, assertSafeIp } from "$lib/server/urlSafety";
+import dns from "node:dns";
+
+const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB
+const FETCH_TIMEOUT = 30000; // 30 seconds
+const MAX_REDIRECTS = 5;
+const SECURITY_HEADERS: HeadersInit = {
+	// Prevent any active content from executing if someone navigates directly to this endpoint.
+	"Content-Security-Policy":
+		"default-src 'none'; frame-ancestors 'none'; sandbox; script-src 'none'; img-src 'none'; style-src 'none'; connect-src 'none'; media-src 'none'; object-src 'none'; base-uri 'none'; form-action 'none'",
+	"X-Content-Type-Options": "nosniff",
+	"X-Frame-Options": "DENY",
+	"Referrer-Policy": "no-referrer",
+};
+
+/**
+ * Undici dispatcher that validates resolved IPs at connection time,
+ * preventing TOCTOU DNS rebinding attacks.
+ */
+const ssrfSafeAgent = new Agent({
+	connect: {
+		lookup: (hostname, options, callback) => {
+			dns.lookup(hostname, options, (err, address, family) => {
+				if (err) return callback(err, "", 4);
+				if (typeof address === "string") {
+					try {
+						assertSafeIp(address, hostname);
+					} catch (e) {
+						return callback(e as Error, "", 4);
+					}
+				} else if (Array.isArray(address)) {
+					for (const entry of address) {
+						try {
+							assertSafeIp(entry.address, hostname);
+						} catch (e) {
+							return callback(e as Error, "", 4);
+						}
+					}
+				}
+				return callback(null, address, family);
+			});
+		},
+	},
+});
+
+export async function GET({ url }) {
+	const targetUrl = url.searchParams.get("url");
+
+	if (!targetUrl) {
+		logger.warn("Missing 'url' parameter");
+		throw error(400, "Missing 'url' parameter");
+	}
+
+	if (!isValidUrl(targetUrl)) {
+		logger.warn({ targetUrl }, "Invalid or unsafe URL (only HTTPS is supported)");
+		throw error(400, "Invalid or unsafe URL (only HTTPS is supported)");
+	}
+
+	// Fetch with timeout, following redirects manually to validate each hop
+	const controller = new AbortController();
+	const timeoutId = setTimeout(() => controller.abort(), FETCH_TIMEOUT);
+
+	let currentUrl = targetUrl;
+	let response: Awaited<ReturnType<typeof fetch>>;
+	let redirectCount = 0;
+
+	try {
+		// eslint-disable-next-line no-constant-condition
+		while (true) {
+			response = await fetch(currentUrl, {
+				signal: controller.signal,
+				redirect: "manual",
+				dispatcher: ssrfSafeAgent,
+				headers: {
+					"User-Agent": "HuggingChat-Attachment-Fetcher/1.0",
+				},
+			});
+
+			if (response.status >= 300 && response.status < 400) {
+				redirectCount++;
+				if (redirectCount > MAX_REDIRECTS) {
+					throw error(502, "Too many redirects");
+				}
+
+				const location = response.headers.get("location");
+				if (!location) {
+					throw error(502, "Redirect without Location header");
+				}
+
+				// Resolve relative redirects against the current URL
+				const redirectUrl = new URL(location, currentUrl).toString();
+
+				if (!isValidUrl(redirectUrl)) {
+					logger.warn(
+						{ redirectUrl, originalUrl: targetUrl },
+						"Redirect to unsafe URL blocked (SSRF)"
+					);
+					throw error(403, "Redirect target is not allowed");
+				}
+
+				currentUrl = redirectUrl;
+				continue;
+			}
+
+			break;
+		}
+	} finally {
+		clearTimeout(timeoutId);
+	}
+
+	if (!response.ok) {
+		logger.error({ targetUrl, response }, "Error fetching URL. Response not ok.");
+		throw error(response.status, `Failed to fetch: ${response.statusText}`);
+	}
+
+	// Check content length if available
+	const contentLength = response.headers.get("content-length");
+	if (contentLength && parseInt(contentLength) > MAX_FILE_SIZE) {
+		throw error(413, "File too large (max 10MB)");
+	}
+
+	// Stream the response back
+	const originalContentType = response.headers.get("content-type") || "application/octet-stream";
+	// Send as text/plain for safety; expose the original type via secondary header
+	const safeContentType = "text/plain; charset=utf-8";
+	const contentDisposition = response.headers.get("content-disposition");
+
+	const headers: HeadersInit = {
+		"Content-Type": safeContentType,
+		"X-Forwarded-Content-Type": originalContentType,
+		"Cache-Control": "public, max-age=3600",
+		...(contentDisposition ? { "Content-Disposition": contentDisposition } : {}),
+		...SECURITY_HEADERS,
+	};
+
+	// Get the body as array buffer to check size
+	const arrayBuffer = await response.arrayBuffer();
+
+	if (arrayBuffer.byteLength > MAX_FILE_SIZE) {
+		throw error(413, "File too large (max 10MB)");
+	}
+
+	return new Response(arrayBuffer, { headers });
+}