ruflo 3.5.2 → 3.5.4

package/src/ruvocal/src/lib/migrations/migrations.ts

@@ -0,0 +1,109 @@
+import { Database } from "$lib/server/database";
+import { migrations } from "./routines";
+import { acquireLock, releaseLock, isDBLocked, refreshLock } from "./lock";
+import { Semaphores } from "$lib/types/Semaphore";
+import { logger } from "$lib/server/logger";
+import { config } from "$lib/server/config";
+
+export async function checkAndRunMigrations() {
+	// make sure all GUIDs are unique
+	if (new Set(migrations.map((m) => m._id.toString())).size !== migrations.length) {
+		throw new Error("Duplicate migration GUIDs found.");
+	}
+
+	// check if all migrations have already been run
+	const migrationResults = await (await Database.getInstance())
+		.getCollections()
+		.migrationResults.find()
+		.toArray();
+
+	logger.debug("[MIGRATIONS] Begin check...");
+
+	const lockId = await acquireLock(Semaphores.MIGRATION);
+
+	if (!lockId) {
+		// another instance already has the lock, so we exit early
+		logger.debug(
+			"[MIGRATIONS] Another instance already has the lock. Waiting for DB to be unlocked."
+		);
+
+		// block until the lock is released
+		while (await isDBLocked(Semaphores.MIGRATION)) {
+			await new Promise((resolve) => setTimeout(resolve, 1000));
+		}
+		return;
+	}
+
+	// once here, we have the lock
+	// make sure to refresh it regularly while it's running
+	const refreshInterval = setInterval(async () => {
+		await refreshLock(Semaphores.MIGRATION, lockId);
+	}, 1000 * 10);
+
+	// iterate over all migrations
+	for (const migration of migrations) {
+		// check if the migration has already been applied
+		const shouldRun =
+			migration.runEveryTime ||
+			!migrationResults.find((m) => m._id.toString() === migration._id.toString());
+
+		// check if the migration has already been applied
+		if (!shouldRun) {
+			logger.debug(`[MIGRATIONS] "${migration.name}" already applied. Skipping...`);
+		} else {
+			// check the modifiers to see if some cases match
+			if (
+				(migration.runForHuggingChat === "only" && !config.isHuggingChat) ||
+				(migration.runForHuggingChat === "never" && config.isHuggingChat)
+			) {
+				logger.debug(
+					`[MIGRATIONS] "${migration.name}" should not be applied for this run. Skipping...`
+				);
+				continue;
+			}
+
+			// otherwise all is good and we can run the migration
+			logger.debug(
+				`[MIGRATIONS] "${migration.name}" ${
+					migration.runEveryTime ? "should run every time" : "not applied yet"
+				}. Applying...`
+			);
+
+			await (await Database.getInstance()).getCollections().migrationResults.updateOne(
+				{ _id: migration._id },
+				{
+					$set: {
+						name: migration.name,
+						status: "ongoing",
+					},
+				},
+				{ upsert: true }
+			);
+
+			let result = false;
+
+			try {
+				// RVF store: no transactions needed, run migration directly
+				result = await migration.up(await Database.getInstance());
+			} catch (e) {
+				logger.error(e, `[MIGRATIONS]  "${migration.name}" failed!`);
+			}
+
+			await (await Database.getInstance()).getCollections().migrationResults.updateOne(
+				{ _id: migration._id },
+				{
+					$set: {
+						name: migration.name,
+						status: result ? "success" : "failure",
+					},
+				},
+				{ upsert: true }
+			);
+		}
+	}
+
+	logger.debug("[MIGRATIONS] All migrations applied. Releasing lock");
+
+	clearInterval(refreshInterval);
+	await releaseLock(Semaphores.MIGRATION, lockId);
+}

package/src/ruvocal/src/lib/migrations/routines/01-update-search-assistants.ts

@@ -0,0 +1,50 @@
+import type { Migration } from ".";
+import { collections } from "$lib/server/database";
+import { ObjectId, type AnyBulkWriteOperation } from "mongodb";
+import type { Assistant } from "$lib/types/Assistant";
+import { generateSearchTokens } from "$lib/utils/searchTokens";
+
+const migration: Migration = {
+	_id: new ObjectId("5f9f3e3e3e3e3e3e3e3e3e3e"),
+	name: "Update search assistants",
+	up: async () => {
+		const { assistants } = collections;
+		let ops: AnyBulkWriteOperation<Assistant>[] = [];
+
+		for await (const assistant of assistants
+			.find()
+			.project<Pick<Assistant, "_id" | "name">>({ _id: 1, name: 1 })) {
+			ops.push({
+				updateOne: {
+					filter: {
+						_id: assistant._id,
+					},
+					update: {
+						$set: {
+							searchTokens: generateSearchTokens(assistant.name),
+						},
+					},
+				},
+			});
+
+			if (ops.length >= 1000) {
+				process.stdout.write(".");
+				await assistants.bulkWrite(ops, { ordered: false });
+				ops = [];
+			}
+		}
+
+		if (ops.length) {
+			await assistants.bulkWrite(ops, { ordered: false });
+		}
+
+		return true;
+	},
+	down: async () => {
+		const { assistants } = collections;
+		await assistants.updateMany({}, { $unset: { searchTokens: "" } });
+		return true;
+	},
+};
+
+export default migration;

package/src/ruvocal/src/lib/migrations/routines/02-update-assistants-models.ts

@@ -0,0 +1,48 @@
+import type { Migration } from ".";
+import { collections } from "$lib/server/database";
+import { ObjectId } from "mongodb";
+
+const updateAssistantsModels: Migration = {
+	_id: new ObjectId("5f9f3f3f3f3f3f3f3f3f3f3f"),
+	name: "Update deprecated models in assistants with the default model",
+	up: async () => {
+		const models = (await import("$lib/server/models")).models;
+		//@ts-expect-error the property doesn't exist anymore, keeping the script for reference
+		const oldModels = (await import("$lib/server/models")).oldModels;
+		const { assistants } = collections;
+
+		const modelIds = models.map((el) => el.id);
+		const defaultModelId = models[0].id;
+
+		// Find all assistants whose modelId is not in modelIds, and update it
+		const bulkOps = await assistants
+			.find({ modelId: { $nin: modelIds } })
+			.map((assistant) => {
+				// has an old model
+				let newModelId = defaultModelId;
+
+				const oldModel = oldModels.find((m: (typeof models)[number]) => m.id === assistant.modelId);
+				if (oldModel && oldModel.transferTo && !!models.find((m) => m.id === oldModel.transferTo)) {
+					newModelId = oldModel.transferTo;
+				}
+
+				return {
+					updateOne: {
+						filter: { _id: assistant._id },
+						update: { $set: { modelId: newModelId } },
+					},
+				};
+			})
+			.toArray();
+
+		if (bulkOps.length > 0) {
+			await assistants.bulkWrite(bulkOps);
+		}
+
+		return true;
+	},
+	runEveryTime: true,
+	runForHuggingChat: "only",
+};
+
+export default updateAssistantsModels;

package/src/ruvocal/src/lib/migrations/routines/04-update-message-updates.ts

@@ -0,0 +1,151 @@
+import type { Migration } from ".";
+import { collections } from "$lib/server/database";
+import { ObjectId, type WithId } from "mongodb";
+import type { Conversation } from "$lib/types/Conversation";
+import {
+	MessageUpdateStatus,
+	MessageUpdateType,
+	type MessageUpdate,
+} from "$lib/types/MessageUpdate";
+import type { Message } from "$lib/types/Message";
+// isMessageWebSearchSourcesUpdate removed from utils; use inline predicate
+
+// -----------
+// Copy of the previous message update types
+export type FinalAnswer = {
+	type: "finalAnswer";
+	text: string;
+};
+
+export type TextStreamUpdate = {
+	type: "stream";
+	token: string;
+};
+
+type WebSearchUpdate = {
+	type: "webSearch";
+	messageType: "update" | "error" | "sources";
+	message: string;
+	args?: string[];
+	sources?: { title?: string; link: string }[];
+};
+
+type StatusUpdate = {
+	type: "status";
+	status: "started" | "pending" | "finished" | "error" | "title";
+	message?: string;
+};
+
+type ErrorUpdate = {
+	type: "error";
+	message: string;
+	name: string;
+};
+
+type FileUpdate = {
+	type: "file";
+	sha: string;
+};
+
+type OldMessageUpdate =
+	| FinalAnswer
+	| TextStreamUpdate
+	| WebSearchUpdate
+	| StatusUpdate
+	| ErrorUpdate
+	| FileUpdate;
+
+/** Converts the old message update to the new schema */
+function convertMessageUpdate(message: Message, update: OldMessageUpdate): MessageUpdate | null {
+	try {
+		// Text and files
+		if (update.type === "finalAnswer") {
+			return {
+				type: MessageUpdateType.FinalAnswer,
+				text: update.text,
+				interrupted: message.interrupted ?? false,
+			};
+		} else if (update.type === "stream") {
+			return {
+				type: MessageUpdateType.Stream,
+				token: update.token,
+			};
+		} else if (update.type === "file") {
+			return {
+				type: MessageUpdateType.File,
+				name: "Unknown",
+				sha: update.sha,
+				// assume jpeg but could be any image. should be harmless
+				mime: "image/jpeg",
+			};
+		}
+
+		// Status
+		else if (update.type === "status") {
+			if (update.status === "title") {
+				return {
+					type: MessageUpdateType.Title,
+					title: update.message ?? "New Chat",
+				};
+			}
+			if (update.status === "pending") return null;
+
+			const status =
+				update.status === "started"
+					? MessageUpdateStatus.Started
+					: update.status === "finished"
+						? MessageUpdateStatus.Finished
+						: MessageUpdateStatus.Error;
+			return {
+				type: MessageUpdateType.Status,
+				status,
+				message: update.message,
+			};
+		} else if (update.type === "error") {
+			// Treat it as an error status update
+			return {
+				type: MessageUpdateType.Status,
+				status: MessageUpdateStatus.Error,
+				message: update.message,
+			};
+		}
+
+		// Web Search
+		else if (update.type === "webSearch") {
+			return null; // Web search updates are no longer supported
+		}
+		console.warn("Unknown message update during migration:", update);
+		return null;
+	} catch (error) {
+		console.error("Error converting message update during migration. Skipping it... Error:", error);
+		return null;
+	}
+}
+
+const updateMessageUpdates: Migration = {
+	_id: new ObjectId("5f9f7f7f7f7f7f7f7f7f7f7f"),
+	name: "Convert message updates to the new schema",
+	up: async () => {
+		const allConversations = collections.conversations.find({});
+
+		let conversation: WithId<Pick<Conversation, "messages">> | null = null;
+		while ((conversation = await allConversations.tryNext())) {
+			const messages = conversation.messages.map((message) => {
+				// Convert all of the existing updates to the new schema
+				const updates = message.updates
+					?.map((update) => convertMessageUpdate(message, update as OldMessageUpdate))
+					.filter((update): update is MessageUpdate => Boolean(update));
+
+				return { ...message, updates };
+			});
+
+			// Set the new messages array
+			await collections.conversations.updateOne({ _id: conversation._id }, { $set: { messages } });
+		}
+
+		return true;
+	},
+	runEveryTime: false,
+};
+
+export default updateMessageUpdates;

package/src/ruvocal/src/lib/migrations/routines/05-update-message-files.ts

@@ -0,0 +1,56 @@
+import { ObjectId, type WithId } from "mongodb";
+import { collections } from "$lib/server/database";
+
+import type { Migration } from ".";
+import type { Conversation } from "$lib/types/Conversation";
+import type { MessageFile } from "$lib/types/Message";
+
+const updateMessageFiles: Migration = {
+	_id: new ObjectId("5f9f5f5f5f5f5f5f5f5f5f5f"),
+	name: "Convert message files to the new schema",
+	up: async () => {
+		const allConversations = collections.conversations.find({}, { projection: { messages: 1 } });
+
+		let conversation: WithId<Pick<Conversation, "messages">> | null = null;
+		while ((conversation = await allConversations.tryNext())) {
+			const messages = conversation.messages.map((message) => {
+				const files = (message.files as string[] | undefined)?.map<MessageFile>((file) => {
+					// File is already in the new format
+					if (typeof file !== "string") return file;
+
+					// File was a hash pointing to a file in the bucket
+					if (file.length === 64) {
+						return {
+							type: "hash",
+							name: "unknown.jpg",
+							value: file,
+							mime: "image/jpeg",
+						};
+					}
+					// File was a base64 string
+					else {
+						return {
+							type: "base64",
+							name: "unknown.jpg",
+							value: file,
+							mime: "image/jpeg",
+						};
+					}
+				});
+
+				return {
+					...message,
+					files,
+				};
+			});
+
+			// Set the new messages array
+			await collections.conversations.updateOne({ _id: conversation._id }, { $set: { messages } });
+		}
+
+		return true;
+	},
+	runEveryTime: false,
+};
+
+export default updateMessageFiles;

package/src/ruvocal/src/lib/migrations/routines/06-trim-message-updates.ts

@@ -0,0 +1,56 @@
+import type { Migration } from ".";
+import { collections } from "$lib/server/database";
+import { ObjectId, type WithId } from "mongodb";
+import type { Conversation } from "$lib/types/Conversation";
+import type { Message } from "$lib/types/Message";
+import type { MessageUpdate } from "$lib/types/MessageUpdate";
+import { logger } from "$lib/server/logger";
+
+// -----------
+
+/** Converts the old message update to the new schema */
+function convertMessageUpdate(message: Message, update: unknown): MessageUpdate | null {
+	try {
+		// Trim legacy web search updates entirely
+		if (
+			typeof update === "object" &&
+			update !== null &&
+			(update as { type: string }).type === "webSearch"
+		) {
+			return null;
+		}
+
+		return update as MessageUpdate;
+	} catch (error) {
+		logger.error(error, "Error converting message update during migration. Skipping it..");
+		return null;
+	}
+}
+
+const trimMessageUpdates: Migration = {
+	_id: new ObjectId("000000000000000000000006"),
+	name: "Trim message updates to reduce stored size",
+	up: async () => {
+		const allConversations = collections.conversations.find({});
+
+		let conversation: WithId<Pick<Conversation, "messages">> | null = null;
+		while ((conversation = await allConversations.tryNext())) {
+			const messages = conversation.messages.map((message) => {
+				// Convert all of the existing updates to the new schema
+				const updates = message.updates
+					?.map((update) => convertMessageUpdate(message, update))
+					.filter((update): update is MessageUpdate => Boolean(update));
+
+				return { ...message, updates };
+			});
+
+			// Set the new messages array
+			await collections.conversations.updateOne({ _id: conversation._id }, { $set: { messages } });
+		}
+
+		return true;
+	},
+	runEveryTime: false,
+};
+
+export default trimMessageUpdates;

package/src/ruvocal/src/lib/migrations/routines/08-update-featured-to-review.ts

@@ -0,0 +1,32 @@
+import type { Migration } from ".";
+import { collections } from "$lib/server/database";
+import { ObjectId } from "mongodb";
+import { ReviewStatus } from "$lib/types/Review";
+
+const updateFeaturedToReview: Migration = {
+	_id: new ObjectId("000000000000000000000008"),
+	name: "Update featured to review",
+	up: async () => {
+		const { assistants, tools } = collections;
+
+		// Update assistants
+		await assistants.updateMany({ featured: true }, { $set: { review: ReviewStatus.APPROVED } });
+		await assistants.updateMany(
+			{ featured: { $ne: true } },
+			{ $set: { review: ReviewStatus.PRIVATE } }
+		);
+
+		await assistants.updateMany({}, { $unset: { featured: "" } });
+
+		// Update tools
+		await tools.updateMany({ featured: true }, { $set: { review: ReviewStatus.APPROVED } });
+		await tools.updateMany({ featured: { $ne: true } }, { $set: { review: ReviewStatus.PRIVATE } });
+
+		await tools.updateMany({}, { $unset: { featured: "" } });
+
+		return true;
+	},
+	runEveryTime: false,
+};
+
+export default updateFeaturedToReview;