ruflo 3.10.36 → 3.10.38

package/src/ruvocal/src/routes/admin/export/+server.ts

@@ -1,159 +1,159 @@
-import { config } from "$lib/server/config";
-import { collections } from "$lib/server/database";
-import type { Message } from "$lib/types/Message";
-import { error } from "@sveltejs/kit";
-import { pathToFileURL } from "node:url";
-import { unlink } from "node:fs/promises";
-import { uploadFile } from "@huggingface/hub";
-import parquet from "parquetjs";
-import { z } from "zod";
-import { logger } from "$lib/server/logger.js";
-
-// Triger like this:
-// curl -X POST "http://localhost:5173/chat/admin/export" -H "Authorization: Bearer <ADMIN_API_SECRET>" -H "Content-Type: application/json" -d '{"model": "OpenAssistant/oasst-sft-6-llama-30b-xor"}'
-
-export async function POST({ request }) {
-	if (!config.PARQUET_EXPORT_DATASET || !config.PARQUET_EXPORT_HF_TOKEN) {
-		error(500, "Parquet export is not configured.");
-	}
-
-	const { model } = z
-		.object({
-			model: z.string(),
-		})
-		.parse(await request.json());
-
-	const schema = new parquet.ParquetSchema({
-		title: { type: "UTF8" },
-		created_at: { type: "TIMESTAMP_MILLIS" },
-		updated_at: { type: "TIMESTAMP_MILLIS" },
-		messages: {
-			repeated: true,
-			fields: {
-				from: { type: "UTF8" },
-				content: { type: "UTF8" },
-				score: { type: "INT_8", optional: true },
-			},
-		},
-	});
-
-	const fileName = `/tmp/conversations-${new Date().toJSON().slice(0, 10)}-${Date.now()}.parquet`;
-
-	const writer = await parquet.ParquetWriter.openFile(schema, fileName);
-
-	let count = 0;
-	logger.info("Exporting conversations for model", model);
-
-	for await (const conversation of collections.settings.aggregate<{
-		title: string;
-		created_at: Date;
-		updated_at: Date;
-		messages: Message[];
-	}>([
-		{
-			$match: {
-				shareConversationsWithModelAuthors: true,
-				sessionId: { $exists: true },
-				userId: { $exists: false },
-			},
-		},
-		{
-			$lookup: {
-				from: "conversations",
-				localField: "sessionId",
-				foreignField: "sessionId",
-				as: "conversations",
-				pipeline: [{ $match: { model, userId: { $exists: false } } }],
-			},
-		},
-		{ $unwind: "$conversations" },
-		{
-			$project: {
-				title: "$conversations.title",
-				created_at: "$conversations.createdAt",
-				updated_at: "$conversations.updatedAt",
-				messages: "$conversations.messages",
-			},
-		},
-	])) {
-		await writer.appendRow({
-			title: conversation.title,
-			created_at: conversation.created_at,
-			updated_at: conversation.updated_at,
-			messages: conversation.messages.map((message: Message) => ({
-				from: message.from,
-				content: message.content,
-				...(message.score ? { score: message.score } : undefined),
-			})),
-		});
-		++count;
-
-		if (count % 1_000 === 0) {
-			logger.info("Exported", count, "conversations");
-		}
-	}
-
-	logger.info("exporting convos with userId");
-
-	for await (const conversation of collections.settings.aggregate<{
-		title: string;
-		created_at: Date;
-		updated_at: Date;
-		messages: Message[];
-	}>([
-		{ $match: { shareConversationsWithModelAuthors: true, userId: { $exists: true } } },
-		{
-			$lookup: {
-				from: "conversations",
-				localField: "userId",
-				foreignField: "userId",
-				as: "conversations",
-				pipeline: [{ $match: { model } }],
-			},
-		},
-		{ $unwind: "$conversations" },
-		{
-			$project: {
-				title: "$conversations.title",
-				created_at: "$conversations.createdAt",
-				updated_at: "$conversations.updatedAt",
-				messages: "$conversations.messages",
-			},
-		},
-	])) {
-		await writer.appendRow({
-			title: conversation.title,
-			created_at: conversation.created_at,
-			updated_at: conversation.updated_at,
-			messages: conversation.messages.map((message: Message) => ({
-				from: message.from,
-				content: message.content,
-				...(message.score ? { score: message.score } : undefined),
-			})),
-		});
-		++count;
-
-		if (count % 1_000 === 0) {
-			logger.info("Exported", count, "conversations");
-		}
-	}
-
-	await writer.close();
-
-	logger.info("Uploading", fileName, "to Hugging Face Hub");
-
-	await uploadFile({
-		file: pathToFileURL(fileName) as URL,
-		credentials: { accessToken: config.PARQUET_EXPORT_HF_TOKEN },
-		repo: {
-			type: "dataset",
-			name: config.PARQUET_EXPORT_DATASET,
-		},
-	});
-
-	logger.info("Upload done");
-
-	await unlink(fileName);
-
-	return new Response();
-}
+import { config } from "$lib/server/config";
+import { collections } from "$lib/server/database";
+import type { Message } from "$lib/types/Message";
+import { error } from "@sveltejs/kit";
+import { pathToFileURL } from "node:url";
+import { unlink } from "node:fs/promises";
+import { uploadFile } from "@huggingface/hub";
+import parquet from "parquetjs";
+import { z } from "zod";
+import { logger } from "$lib/server/logger.js";
+
+// Triger like this:
+// curl -X POST "http://localhost:5173/chat/admin/export" -H "Authorization: Bearer <ADMIN_API_SECRET>" -H "Content-Type: application/json" -d '{"model": "OpenAssistant/oasst-sft-6-llama-30b-xor"}'
+
+export async function POST({ request }) {
+	if (!config.PARQUET_EXPORT_DATASET || !config.PARQUET_EXPORT_HF_TOKEN) {
+		error(500, "Parquet export is not configured.");
+	}
+
+	const { model } = z
+		.object({
+			model: z.string(),
+		})
+		.parse(await request.json());
+
+	const schema = new parquet.ParquetSchema({
+		title: { type: "UTF8" },
+		created_at: { type: "TIMESTAMP_MILLIS" },
+		updated_at: { type: "TIMESTAMP_MILLIS" },
+		messages: {
+			repeated: true,
+			fields: {
+				from: { type: "UTF8" },
+				content: { type: "UTF8" },
+				score: { type: "INT_8", optional: true },
+			},
+		},
+	});
+
+	const fileName = `/tmp/conversations-${new Date().toJSON().slice(0, 10)}-${Date.now()}.parquet`;
+
+	const writer = await parquet.ParquetWriter.openFile(schema, fileName);
+
+	let count = 0;
+	logger.info("Exporting conversations for model", model);
+
+	for await (const conversation of collections.settings.aggregate<{
+		title: string;
+		created_at: Date;
+		updated_at: Date;
+		messages: Message[];
+	}>([
+		{
+			$match: {
+				shareConversationsWithModelAuthors: true,
+				sessionId: { $exists: true },
+				userId: { $exists: false },
+			},
+		},
+		{
+			$lookup: {
+				from: "conversations",
+				localField: "sessionId",
+				foreignField: "sessionId",
+				as: "conversations",
+				pipeline: [{ $match: { model, userId: { $exists: false } } }],
+			},
+		},
+		{ $unwind: "$conversations" },
+		{
+			$project: {
+				title: "$conversations.title",
+				created_at: "$conversations.createdAt",
+				updated_at: "$conversations.updatedAt",
+				messages: "$conversations.messages",
+			},
+		},
+	])) {
+		await writer.appendRow({
+			title: conversation.title,
+			created_at: conversation.created_at,
+			updated_at: conversation.updated_at,
+			messages: conversation.messages.map((message: Message) => ({
+				from: message.from,
+				content: message.content,
+				...(message.score ? { score: message.score } : undefined),
+			})),
+		});
+		++count;
+
+		if (count % 1_000 === 0) {
+			logger.info("Exported", count, "conversations");
+		}
+	}
+
+	logger.info("exporting convos with userId");
+
+	for await (const conversation of collections.settings.aggregate<{
+		title: string;
+		created_at: Date;
+		updated_at: Date;
+		messages: Message[];
+	}>([
+		{ $match: { shareConversationsWithModelAuthors: true, userId: { $exists: true } } },
+		{
+			$lookup: {
+				from: "conversations",
+				localField: "userId",
+				foreignField: "userId",
+				as: "conversations",
+				pipeline: [{ $match: { model } }],
+			},
+		},
+		{ $unwind: "$conversations" },
+		{
+			$project: {
+				title: "$conversations.title",
+				created_at: "$conversations.createdAt",
+				updated_at: "$conversations.updatedAt",
+				messages: "$conversations.messages",
+			},
+		},
+	])) {
+		await writer.appendRow({
+			title: conversation.title,
+			created_at: conversation.created_at,
+			updated_at: conversation.updated_at,
+			messages: conversation.messages.map((message: Message) => ({
+				from: message.from,
+				content: message.content,
+				...(message.score ? { score: message.score } : undefined),
+			})),
+		});
+		++count;
+
+		if (count % 1_000 === 0) {
+			logger.info("Exported", count, "conversations");
+		}
+	}
+
+	await writer.close();
+
+	logger.info("Uploading", fileName, "to Hugging Face Hub");
+
+	await uploadFile({
+		file: pathToFileURL(fileName) as URL,
+		credentials: { accessToken: config.PARQUET_EXPORT_HF_TOKEN },
+		repo: {
+			type: "dataset",
+			name: config.PARQUET_EXPORT_DATASET,
+		},
+	});
+
+	logger.info("Upload done");
+
+	await unlink(fileName);
+
+	return new Response();
+}

package/src/ruvocal/src/routes/admin/stats/compute/+server.ts

@@ -1,16 +1,16 @@
-import { json } from "@sveltejs/kit";
-import { logger } from "$lib/server/logger";
-import { computeAllStats } from "$lib/jobs/refresh-conversation-stats";
-
-// Triger like this:
-// curl -X POST "http://localhost:5173/chat/admin/stats/compute" -H "Authorization: Bearer <ADMIN_API_SECRET>"
-
-export async function POST() {
-	computeAllStats().catch((e) => logger.error(e, "Error computing all stats"));
-	return json(
-		{
-			message: "Stats job started",
-		},
-		{ status: 202 }
-	);
-}
+import { json } from "@sveltejs/kit";
+import { logger } from "$lib/server/logger";
+import { computeAllStats } from "$lib/jobs/refresh-conversation-stats";
+
+// Triger like this:
+// curl -X POST "http://localhost:5173/chat/admin/stats/compute" -H "Authorization: Bearer <ADMIN_API_SECRET>"
+
+export async function POST() {
+	computeAllStats().catch((e) => logger.error(e, "Error computing all stats"));
+	return json(
+		{
+			message: "Stats job started",
+		},
+		{ status: 202 }
+	);
+}

package/src/ruvocal/src/routes/api/conversation/[id]/+server.ts

@@ -1,40 +1,40 @@
-import { collections } from "$lib/server/database";
-import { authCondition } from "$lib/server/auth";
-import { z } from "zod";
-import { ObjectId } from "mongodb";
-
-export async function GET({ locals, params }) {
-	const id = z.string().parse(params.id);
-	const convId = new ObjectId(id);
-
-	if (locals.user?._id || locals.sessionId) {
-		const conv = await collections.conversations.findOne({
-			_id: convId,
-			...authCondition(locals),
-		});
-
-		if (conv) {
-			const res = {
-				id: conv._id,
-				title: conv.title,
-				updatedAt: conv.updatedAt,
-				modelId: conv.model,
-				messages: conv.messages.map((message) => ({
-					content: message.content,
-					from: message.from,
-					id: message.id,
-					createdAt: message.createdAt,
-					updatedAt: message.updatedAt,
-					// websearch removed
-					files: message.files,
-					updates: message.updates,
-				})),
-			};
-			return Response.json(res);
-		} else {
-			return Response.json({ message: "Conversation not found" }, { status: 404 });
-		}
-	} else {
-		return Response.json({ message: "Must have session cookie" }, { status: 401 });
-	}
-}
+import { collections } from "$lib/server/database";
+import { authCondition } from "$lib/server/auth";
+import { z } from "zod";
+import { ObjectId } from "mongodb";
+
+export async function GET({ locals, params }) {
+	const id = z.string().parse(params.id);
+	const convId = new ObjectId(id);
+
+	if (locals.user?._id || locals.sessionId) {
+		const conv = await collections.conversations.findOne({
+			_id: convId,
+			...authCondition(locals),
+		});
+
+		if (conv) {
+			const res = {
+				id: conv._id,
+				title: conv.title,
+				updatedAt: conv.updatedAt,
+				modelId: conv.model,
+				messages: conv.messages.map((message) => ({
+					content: message.content,
+					from: message.from,
+					id: message.id,
+					createdAt: message.createdAt,
+					updatedAt: message.updatedAt,
+					// websearch removed
+					files: message.files,
+					updates: message.updates,
+				})),
+			};
+			return Response.json(res);
+		} else {
+			return Response.json({ message: "Conversation not found" }, { status: 404 });
+		}
+	} else {
+		return Response.json({ message: "Must have session cookie" }, { status: 401 });
+	}
+}

package/src/ruvocal/src/routes/api/conversation/[id]/message/[messageId]/+server.ts

@@ -1,42 +1,42 @@
-import { authCondition } from "$lib/server/auth";
-import { collections } from "$lib/server/database";
-import { error } from "@sveltejs/kit";
-import { ObjectId } from "mongodb";
-
-export async function DELETE({ locals, params }) {
-	const messageId = params.messageId;
-
-	if (!messageId || typeof messageId !== "string") {
-		error(400, "Invalid message id");
-	}
-
-	const conversation = await collections.conversations.findOne({
-		...authCondition(locals),
-		_id: new ObjectId(params.id),
-	});
-
-	if (!conversation) {
-		error(404, "Conversation not found");
-	}
-
-	const filteredMessages = conversation.messages
-		.filter(
-			(message) =>
-				// not the message AND the message is not in ancestors
-				!(message.id === messageId) && message.ancestors && !message.ancestors.includes(messageId)
-		)
-		.map((message) => {
-			// remove the message from children if it's there
-			if (message.children && message.children.includes(messageId)) {
-				message.children = message.children.filter((child) => child !== messageId);
-			}
-			return message;
-		});
-
-	await collections.conversations.updateOne(
-		{ _id: conversation._id, ...authCondition(locals) },
-		{ $set: { messages: filteredMessages } }
-	);
-
-	return new Response();
-}
+import { authCondition } from "$lib/server/auth";
+import { collections } from "$lib/server/database";
+import { error } from "@sveltejs/kit";
+import { ObjectId } from "mongodb";
+
+export async function DELETE({ locals, params }) {
+	const messageId = params.messageId;
+
+	if (!messageId || typeof messageId !== "string") {
+		error(400, "Invalid message id");
+	}
+
+	const conversation = await collections.conversations.findOne({
+		...authCondition(locals),
+		_id: new ObjectId(params.id),
+	});
+
+	if (!conversation) {
+		error(404, "Conversation not found");
+	}
+
+	const filteredMessages = conversation.messages
+		.filter(
+			(message) =>
+				// not the message AND the message is not in ancestors
+				!(message.id === messageId) && message.ancestors && !message.ancestors.includes(messageId)
+		)
+		.map((message) => {
+			// remove the message from children if it's there
+			if (message.children && message.children.includes(messageId)) {
+				message.children = message.children.filter((child) => child !== messageId);
+			}
+			return message;
+		});
+
+	await collections.conversations.updateOne(
+		{ _id: conversation._id, ...authCondition(locals) },
+		{ $set: { messages: filteredMessages } }
+	);
+
+	return new Response();
+}

package/src/ruvocal/src/routes/api/conversations/+server.ts

@@ -1,48 +1,48 @@
-import { collections } from "$lib/server/database";
-import { authCondition } from "$lib/server/auth";
-import type { Conversation } from "$lib/types/Conversation";
-import { CONV_NUM_PER_PAGE } from "$lib/constants/pagination";
-
-export async function GET({ locals, url }) {
-	const p = parseInt(url.searchParams.get("p") ?? "0");
-	if (locals.user?._id || locals.sessionId) {
-		const convs = await collections.conversations
-			.find({
-				...authCondition(locals),
-			})
-			.project<Pick<Conversation, "_id" | "title" | "updatedAt" | "model" | never>>({
-				title: 1,
-				updatedAt: 1,
-				model: 1,
-			})
-			.sort({ updatedAt: -1 })
-			.skip(p * CONV_NUM_PER_PAGE)
-			.limit(CONV_NUM_PER_PAGE)
-			.toArray();
-
-		if (convs.length === 0) {
-			return Response.json([]);
-		}
-		const res = convs.map((conv) => ({
-			_id: conv._id,
-			id: conv._id, // legacy param iOS
-			title: conv.title,
-			updatedAt: conv.updatedAt,
-			model: conv.model,
-			modelId: conv.model, // legacy param iOS
-		}));
-		return Response.json(res);
-	} else {
-		return Response.json({ message: "Must have session cookie" }, { status: 401 });
-	}
-}
-
-export async function DELETE({ locals }) {
-	if (locals.user?._id || locals.sessionId) {
-		await collections.conversations.deleteMany({
-			...authCondition(locals),
-		});
-	}
-
-	return new Response();
-}
+import { collections } from "$lib/server/database";
+import { authCondition } from "$lib/server/auth";
+import type { Conversation } from "$lib/types/Conversation";
+import { CONV_NUM_PER_PAGE } from "$lib/constants/pagination";
+
+export async function GET({ locals, url }) {
+	const p = parseInt(url.searchParams.get("p") ?? "0");
+	if (locals.user?._id || locals.sessionId) {
+		const convs = await collections.conversations
+			.find({
+				...authCondition(locals),
+			})
+			.project<Pick<Conversation, "_id" | "title" | "updatedAt" | "model" | never>>({
+				title: 1,
+				updatedAt: 1,
+				model: 1,
+			})
+			.sort({ updatedAt: -1 })
+			.skip(p * CONV_NUM_PER_PAGE)
+			.limit(CONV_NUM_PER_PAGE)
+			.toArray();
+
+		if (convs.length === 0) {
+			return Response.json([]);
+		}
+		const res = convs.map((conv) => ({
+			_id: conv._id,
+			id: conv._id, // legacy param iOS
+			title: conv.title,
+			updatedAt: conv.updatedAt,
+			model: conv.model,
+			modelId: conv.model, // legacy param iOS
+		}));
+		return Response.json(res);
+	} else {
+		return Response.json({ message: "Must have session cookie" }, { status: 401 });
+	}
+}
+
+export async function DELETE({ locals }) {
+	if (locals.user?._id || locals.sessionId) {
+		await collections.conversations.deleteMany({
+			...authCondition(locals),
+		});
+	}
+
+	return new Response();
+}