ruflo 3.5.2 → 3.5.3

package/src/ruvocal/src/lib/constants/mime.ts

@@ -0,0 +1,11 @@
+// Centralized MIME allowlists used across client and server
+// Keep these lists minimal and consistent with server processing.
+
+export const TEXT_MIME_ALLOWLIST = [
+	"text/*",
+	"application/json",
+	"application/xml",
+	"application/csv",
+] as const;
+
+export const IMAGE_MIME_ALLOWLIST_DEFAULT = ["image/jpeg", "image/png"] as const;

package/src/ruvocal/src/lib/constants/pagination.ts

@@ -0,0 +1 @@
+export const CONV_NUM_PER_PAGE = 30;

package/src/ruvocal/src/lib/constants/publicSepToken.ts

@@ -0,0 +1 @@
+export const PUBLIC_SEP_TOKEN = "</s>";

package/src/ruvocal/src/lib/constants/routerExamples.ts

@@ -0,0 +1,209 @@
+export type RouterFollowUp = {
+	title: string;
+	prompt: string;
+};
+
+export type RouterExampleAttachment = {
+	src: string;
+};
+
+export type RouterExample = {
+	title: string;
+	prompt: string;
+	followUps?: RouterFollowUp[];
+	attachments?: RouterExampleAttachment[];
+};
+
+export const routerExamples: RouterExample[] = [
+	{
+		title: "HTML game",
+		prompt: "Code a minimal Flappy Bird game using HTML and Canvas",
+		followUps: [
+			{
+				title: "README.md file",
+				prompt: "Create a comprehensive README.md for the Flappy Bird game project.",
+			},
+			{
+				title: "CRT Screen",
+				prompt: "Add a CRT screen effect to the game",
+			},
+			{
+				title: "Add power-ups",
+				prompt:
+					"Add collectible coins between pipes that award bonus points and a shield power-up that allows one collision.",
+			},
+			{
+				title: "Explain collision detection",
+				prompt:
+					"Explain the collision detection algorithm for the bird and pipes in simple terms with examples.",
+			},
+		],
+	},
+	{
+		title: "Weird painting",
+		prompt: "is this a real painting?",
+		attachments: [
+			{
+				src: "huggingchat/castle-example.jpg",
+			},
+		],
+	},
+	{
+		title: "Landing page",
+		prompt:
+			"Build a responsive SaaS landing page for my AI coding assitant using Tailwind CSS. With a hero, features, testimonials, and pricing sections.",
+		followUps: [
+			{
+				title: "Dark mode",
+				prompt: "Add dark mode and make it the default",
+			},
+			{
+				title: "Write blog post",
+				prompt: "Write a blog post introducing my service.",
+			},
+			{
+				title: "Translate to Italian",
+				prompt: "Translate only the text content displayed to users into Italian.",
+			},
+			{
+				title: "Architecture review",
+				prompt:
+					"Review the architecture and suggest improvements for scalability, SEO optimization, and performance.",
+			},
+		],
+	},
+	{
+		title: "Eminem song",
+		prompt:
+			"Write an Eminem-style rap battling AI taking over hip-hop, with two energetic verses and a catchy hook.",
+		followUps: [
+			{
+				title: "Psychological analysis",
+				prompt: "Provide a psychological analysis of Eminem's emotions in this song.",
+			},
+			{
+				title: "Wired Article",
+				prompt: "Write an article in the style of Wired explaining this Eminem release.",
+			},
+			{
+				title: "Roleplay",
+				prompt: "Roleplay as Eminem so I can discuss the song with him.",
+			},
+			{
+				title: "Translate to Spanish",
+				prompt: "Translate the rap lyrics to Spanish while maintaining the rhyme scheme and flow.",
+			},
+		],
+	},
+	{
+		title: "Act as Yoda",
+		prompt: "Act as Yoda",
+		followUps: [
+			{
+				title: "Give advice",
+				prompt:
+					"Continue acting as Yoda and offer three pieces of life advice for staying focused under pressure.",
+			},
+			{
+				title: "Explain the Force",
+				prompt:
+					"In Yoda's voice, explain the concept of the Force to a young padawan using modern language.",
+			},
+			{
+				title: "Plain English",
+				prompt:
+					"Rewrite the previous response from Yoda into plain English while keeping the same meaning.",
+			},
+			{
+				title: "Compare philosophies",
+				prompt:
+					"Compare Yoda's Jedi philosophy to Stoic philosophy from ancient Greece and explain the similarities and differences.",
+			},
+		],
+	},
+	{
+		title: "Generate prompts",
+		prompt: `Generate 5 creative prompts Text-to-image prompts like: "Cyberpunk cityscape at night, neon lights, flying cars, rain-slicked streets, blade runner aesthetic, highly detailed`,
+		followUps: [
+			{
+				title: "Turn into JSON",
+				prompt: `Generate a detailed JSON object for each prompt. Include fields for subjects (list of objects), scene (setting, environment, background details), actions (what's happening), style (artistic style or medium)`,
+			},
+			{
+				title: "Sci-fi portraits",
+				prompt:
+					"Produce five futuristic character portrait prompts with unique professions and settings.",
+			},
+			{
+				title: "Explain image generation",
+				prompt:
+					"Explain how text-to-image diffusion models work, covering the denoising process and how text prompts guide generation.",
+			},
+		],
+	},
+	{
+		title: "Explain LLMs",
+		prompt:
+			"Explain how large language models based on transformers work, covering attention, embeddings, and training objectives.",
+		followUps: [
+			{
+				title: "Generate a Quiz",
+				prompt: "Craft a 5-question multiple-choice quiz to validate what I learned.",
+			},
+			{
+				title: "Compare to RNNs",
+				prompt:
+					"Compare transformer-based large language models to recurrent neural networks, focusing on training efficiency and capabilities.",
+			},
+			{
+				title: "Student summary",
+				prompt:
+					"Summarize the explanation of large language models for a high school student using relatable analogies.",
+			},
+			{
+				title: "Write a blog post",
+				prompt:
+					"Write a blog post about how transformers revolutionized NLP, targeting software engineers who are new to AI.",
+			},
+		],
+	},
+	{
+		title: "Translate in Italian",
+		prompt: `Translate in Italian: Some are born great, some achieve greatness, and some have greatness thrust upon 'em`,
+		followUps: [
+			{
+				title: "Back to English",
+				prompt:
+					"Translate the Italian version back into English while keeping Shakespeare's tone intact.",
+			},
+			{
+				title: "Explain choices",
+				prompt: "Explain your translation choices for each key phrase from the Italian version.",
+			},
+			{
+				title: "Modernize",
+				prompt:
+					"Modernize the Italian translation into contemporary informal Italian suitable for social media.",
+			},
+			{
+				title: "Teach me Italian",
+				prompt:
+					"Help me practice Italian by conversing about this Shakespeare quote, correcting my grammar when needed.",
+			},
+		],
+	},
+	{
+		title: "Pelican on a bicycle",
+		prompt: "Draw an SVG of a pelican riding a bicycle",
+		followUps: [
+			{
+				title: "Add a top hat",
+				prompt: "Add a fancy top hat to the pelican and make it look distinguished",
+			},
+			{
+				title: "Make it animated",
+				prompt: "Add CSS animations to make the bicycle wheels spin and the pelican's wings flap",
+			},
+		],
+	},
+];

package/src/ruvocal/src/lib/createShareLink.ts

@@ -0,0 +1,27 @@
+import { base } from "$app/paths";
+import { page } from "$app/state";
+
+// Returns a public share URL for a conversation id.
+// If `id` is already a 7-char share id, no network call is made.
+export async function createShareLink(id: string): Promise<string> {
+	const prefix =
+		page.data.publicConfig.PUBLIC_SHARE_PREFIX ||
+		`${page.data.publicConfig.PUBLIC_ORIGIN || page.url.origin}${base}`;
+
+	if (id.length === 7) {
+		return `${prefix}/r/${id}`;
+	}
+
+	const res = await fetch(`${base}/conversation/${id}/share`, {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+	});
+
+	if (!res.ok) {
+		const text = await res.text().catch(() => "");
+		throw new Error(text || "Failed to create share link");
+	}
+
+	const { shareId } = await res.json();
+	return `${prefix}/r/${shareId}`;
+}

package/src/ruvocal/src/lib/jobs/refresh-conversation-stats.ts

@@ -0,0 +1,297 @@
+import type { ConversationStats } from "$lib/types/ConversationStats";
+import { CONVERSATION_STATS_COLLECTION, collections } from "$lib/server/database";
+import { logger } from "$lib/server/logger";
+import type { ObjectId } from "mongodb";
+import { acquireLock, refreshLock } from "$lib/migrations/lock";
+import { Semaphores } from "$lib/types/Semaphore";
+
+async function getLastComputationTime(): Promise<Date> {
+	const lastStats = await collections.conversationStats.findOne({}, { sort: { "date.at": -1 } });
+	return lastStats?.date?.at || new Date(0);
+}
+
+async function shouldComputeStats(): Promise<boolean> {
+	const lastComputationTime = await getLastComputationTime();
+	const oneDayAgo = new Date(Date.now() - 24 * 3_600_000);
+	return lastComputationTime < oneDayAgo;
+}
+
+export async function computeAllStats() {
+	for (const span of ["day", "week", "month"] as const) {
+		computeStats({ dateField: "updatedAt", type: "conversation", span }).catch((e) =>
+			logger.error(e, "Error computing conversation stats for updatedAt")
+		);
+		computeStats({ dateField: "createdAt", type: "conversation", span }).catch((e) =>
+			logger.error(e, "Error computing conversation stats for createdAt")
+		);
+		computeStats({ dateField: "createdAt", type: "message", span }).catch((e) =>
+			logger.error(e, "Error computing message stats for createdAt")
+		);
+	}
+}
+
+async function computeStats(params: {
+	dateField: ConversationStats["date"]["field"];
+	span: ConversationStats["date"]["span"];
+	type: ConversationStats["type"];
+}) {
+	const indexes = await collections.semaphores.listIndexes().toArray();
+	if (indexes.length <= 2) {
+		logger.info("Indexes not created, skipping stats computation");
+		return;
+	}
+
+	const lastComputed = await collections.conversationStats.findOne(
+		{ "date.field": params.dateField, "date.span": params.span, type: params.type },
+		{ sort: { "date.at": -1 } }
+	);
+
+	// If the last computed week is at the beginning of the last computed month, we need to include some days from the previous month
+	// In those cases we need to compute the stats from before the last month as everything is one aggregation
+	const minDate = lastComputed ? lastComputed.date.at : new Date(0);
+
+	logger.debug(
+		{ minDate, dateField: params.dateField, span: params.span, type: params.type },
+		"Computing conversation stats"
+	);
+
+	const dateField = params.type === "message" ? "messages." + params.dateField : params.dateField;
+
+	const pipeline = [
+		{
+			$match: {
+				[dateField]: { $gte: minDate },
+			},
+		},
+		// For message stats: use $filter to reduce data before $unwind (optimization)
+		// For conversation stats: simple projection
+		...(params.type === "message"
+			? [
+					{
+						$project: {
+							// Filter messages by date, then map to only keep the date field
+							// This avoids carrying large message payloads (content, files, etc.) through the pipeline
+							messages: {
+								$map: {
+									input: {
+										$filter: {
+											input: "$messages",
+											as: "msg",
+											cond: { $gte: [`$$msg.${params.dateField}`, minDate] },
+										},
+									},
+									as: "msg",
+									in: { [params.dateField]: `$$msg.${params.dateField}` },
+								},
+							},
+							sessionId: 1,
+							userId: 1,
+						},
+					},
+					{
+						$unwind: "$messages",
+					},
+				]
+			: [
+					{
+						$project: {
+							[dateField]: 1,
+							sessionId: 1,
+							userId: 1,
+						},
+					},
+				]),
+		{
+			$sort: {
+				[dateField]: 1,
+			},
+		},
+		{
+			$facet: {
+				userId: [
+					{
+						$match: {
+							userId: { $exists: true },
+						},
+					},
+					{
+						$group: {
+							_id: {
+								at: { $dateTrunc: { date: `$${dateField}`, unit: params.span } },
+								userId: "$userId",
+							},
+						},
+					},
+					{
+						$group: {
+							_id: "$_id.at",
+							count: { $sum: 1 },
+						},
+					},
+					{
+						$project: {
+							_id: 0,
+							date: {
+								at: "$_id",
+								field: params.dateField,
+								span: params.span,
+							},
+							distinct: "userId",
+							count: 1,
+						},
+					},
+				],
+				sessionId: [
+					{
+						$match: {
+							sessionId: { $exists: true },
+						},
+					},
+					{
+						$group: {
+							_id: {
+								at: { $dateTrunc: { date: `$${dateField}`, unit: params.span } },
+								sessionId: "$sessionId",
+							},
+						},
+					},
+					{
+						$group: {
+							_id: "$_id.at",
+							count: { $sum: 1 },
+						},
+					},
+					{
+						$project: {
+							_id: 0,
+							date: {
+								at: "$_id",
+								field: params.dateField,
+								span: params.span,
+							},
+							distinct: "sessionId",
+							count: 1,
+						},
+					},
+				],
+				userOrSessionId: [
+					{
+						$group: {
+							_id: {
+								at: { $dateTrunc: { date: `$${dateField}`, unit: params.span } },
+								userOrSessionId: { $ifNull: ["$userId", "$sessionId"] },
+							},
+						},
+					},
+					{
+						$group: {
+							_id: "$_id.at",
+							count: { $sum: 1 },
+						},
+					},
+					{
+						$project: {
+							_id: 0,
+							date: {
+								at: "$_id",
+								field: params.dateField,
+								span: params.span,
+							},
+							distinct: "userOrSessionId",
+							count: 1,
+						},
+					},
+				],
+				_id: [
+					{
+						$group: {
+							_id: { $dateTrunc: { date: `$${dateField}`, unit: params.span } },
+							count: { $sum: 1 },
+						},
+					},
+					{
+						$project: {
+							_id: 0,
+							date: {
+								at: "$_id",
+								field: params.dateField,
+								span: params.span,
+							},
+							distinct: "_id",
+							count: 1,
+						},
+					},
+				],
+			},
+		},
+		{
+			$project: {
+				stats: {
+					$concatArrays: ["$userId", "$sessionId", "$userOrSessionId", "$_id"],
+				},
+			},
+		},
+		{
+			$unwind: "$stats",
+		},
+		{
+			$replaceRoot: {
+				newRoot: "$stats",
+			},
+		},
+		{
+			$set: {
+				type: params.type,
+			},
+		},
+		{
+			$merge: {
+				into: CONVERSATION_STATS_COLLECTION,
+				on: ["date.at", "type", "date.span", "date.field", "distinct"],
+				whenMatched: "replace",
+				whenNotMatched: "insert",
+			},
+		},
+	];
+
+	await collections.conversations.aggregate(pipeline, { allowDiskUse: true }).next();
+
+	logger.debug(
+		{ minDate, dateField: params.dateField, span: params.span, type: params.type },
+		"Computed conversation stats"
+	);
+}
+
+let hasLock = false;
+let lockId: ObjectId | null = null;
+
+async function maintainLock() {
+	if (hasLock && lockId) {
+		hasLock = await refreshLock(Semaphores.CONVERSATION_STATS, lockId);
+
+		if (!hasLock) {
+			lockId = null;
+		}
+	} else if (!hasLock) {
+		lockId = (await acquireLock(Semaphores.CONVERSATION_STATS)) || null;
+		hasLock = !!lockId;
+	}
+
+	setTimeout(maintainLock, 10_000);
+}
+
+export function refreshConversationStats() {
+	const ONE_HOUR_MS = 3_600_000;
+
+	maintainLock().then(async () => {
+		if (await shouldComputeStats()) {
+			computeAllStats();
+		}
+
+		setInterval(async () => {
+			if (await shouldComputeStats()) {
+				computeAllStats();
+			}
+		}, 24 * ONE_HOUR_MS);
+	});
+}

package/src/ruvocal/src/lib/migrations/lock.ts

@@ -0,0 +1,56 @@
+import { collections } from "$lib/server/database";
+import { ObjectId } from "mongodb";
+import type { Semaphores } from "$lib/types/Semaphore";
+
+/**
+ * Returns the lock id if the lock was acquired, false otherwise
+ */
+export async function acquireLock(key: Semaphores | string): Promise<ObjectId | false> {
+	try {
+		const id = new ObjectId();
+
+		const insert = await collections.semaphores.insertOne({
+			_id: id,
+			key,
+			createdAt: new Date(),
+			updatedAt: new Date(),
+			deleteAt: new Date(Date.now() + 1000 * 60 * 3), // 3 minutes
+		});
+
+		return insert.acknowledged ? id : false; // true if the document was inserted
+	} catch (e) {
+		// unique index violation, so there must already be a lock
+		return false;
+	}
+}
+
+export async function releaseLock(key: Semaphores | string, lockId: ObjectId) {
+	await collections.semaphores.deleteOne({
+		_id: lockId,
+		key,
+	});
+}
+
+export async function isDBLocked(key: Semaphores | string): Promise<boolean> {
+	const res = await collections.semaphores.countDocuments({
+		key,
+	});
+	return res > 0;
+}
+
+export async function refreshLock(key: Semaphores | string, lockId: ObjectId): Promise<boolean> {
+	const result = await collections.semaphores.updateOne(
+		{
+			_id: lockId,
+			key,
+		},
+		{
+			$set: {
+				updatedAt: new Date(),
+				deleteAt: new Date(Date.now() + 1000 * 60 * 3), // 3 minutes
+			},
+		}
+	);
+
+	return result.matchedCount > 0;
+}

package/src/ruvocal/src/lib/migrations/migrations.spec.ts

@@ -0,0 +1,74 @@
+import { afterEach, assert, beforeAll, describe, expect, it } from "vitest";
+import { migrations } from "./routines";
+import { acquireLock, isDBLocked, refreshLock, releaseLock } from "./lock";
+import { Semaphores } from "$lib/types/Semaphore";
+import { collections, ready } from "$lib/server/database";
+
+describe(
+	"migrations",
+	{
+		retry: 3,
+	},
+	() => {
+		beforeAll(async () => {
+			await ready;
+			try {
+				await collections.semaphores.createIndex({ key: 1 }, { unique: true });
+			} catch (e) {
+				// Index might already exist, ignore error
+			}
+		}, 20000);
+
+		it("should not have duplicates guid", async () => {
+			const guids = migrations.map((m) => m._id.toString());
+			const uniqueGuids = [...new Set(guids)];
+			expect(uniqueGuids.length).toBe(guids.length);
+		});
+
+		it("should acquire only one lock on DB", async () => {
+			const results = await Promise.all(
+				new Array(1000).fill(0).map(() => acquireLock(Semaphores.TEST_MIGRATION))
+			);
+			const locks = results.filter((r) => r);
+
+			const semaphores = await collections.semaphores.find({}).toArray();
+
+			expect(locks.length).toBe(1);
+			expect(semaphores).toBeDefined();
+			expect(semaphores.length).toBe(1);
+			expect(semaphores?.[0].key).toBe(Semaphores.TEST_MIGRATION);
+		});
+
+		it("should read the lock correctly", async () => {
+			const lockId = await acquireLock(Semaphores.TEST_MIGRATION);
+			assert(lockId);
+			expect(await isDBLocked(Semaphores.TEST_MIGRATION)).toBe(true);
+			expect(!!(await acquireLock(Semaphores.TEST_MIGRATION))).toBe(false);
+			await releaseLock(Semaphores.TEST_MIGRATION, lockId);
+			expect(await isDBLocked(Semaphores.TEST_MIGRATION)).toBe(false);
+		});
+
+		it("should refresh the lock", async () => {
+			const lockId = await acquireLock(Semaphores.TEST_MIGRATION);
+
+			assert(lockId);
+
+			// get the updatedAt time
+
+			const updatedAtInitially = (await collections.semaphores.findOne({}))?.updatedAt;
+
+			await refreshLock(Semaphores.TEST_MIGRATION, lockId);
+
+			const updatedAtAfterRefresh = (await collections.semaphores.findOne({}))?.updatedAt;
+
+			expect(updatedAtInitially).toBeDefined();
+			expect(updatedAtAfterRefresh).toBeDefined();
+			expect(updatedAtInitially).not.toBe(updatedAtAfterRefresh);
+		});
+
+		afterEach(async () => {
+			await collections.semaphores.deleteMany({});
+			await collections.migrationResults.deleteMany({});
+		});
+	}
+);