ruflo 3.7.0-alpha.11 → 3.7.0-alpha.13

package/src/ruvocal/src/lib/migrations/lock.ts

@@ -1,56 +1,56 @@
-import { collections } from "$lib/server/database";
-import { ObjectId } from "mongodb";
-import type { Semaphores } from "$lib/types/Semaphore";
-
-/**
- * Returns the lock id if the lock was acquired, false otherwise
- */
-export async function acquireLock(key: Semaphores | string): Promise<ObjectId | false> {
-	try {
-		const id = new ObjectId();
-
-		const insert = await collections.semaphores.insertOne({
-			_id: id,
-			key,
-			createdAt: new Date(),
-			updatedAt: new Date(),
-			deleteAt: new Date(Date.now() + 1000 * 60 * 3), // 3 minutes
-		});
-
-		return insert.acknowledged ? id : false; // true if the document was inserted
-	} catch (e) {
-		// unique index violation, so there must already be a lock
-		return false;
-	}
-}
-
-export async function releaseLock(key: Semaphores | string, lockId: ObjectId) {
-	await collections.semaphores.deleteOne({
-		_id: lockId,
-		key,
-	});
-}
-
-export async function isDBLocked(key: Semaphores | string): Promise<boolean> {
-	const res = await collections.semaphores.countDocuments({
-		key,
-	});
-	return res > 0;
-}
-
-export async function refreshLock(key: Semaphores | string, lockId: ObjectId): Promise<boolean> {
-	const result = await collections.semaphores.updateOne(
-		{
-			_id: lockId,
-			key,
-		},
-		{
-			$set: {
-				updatedAt: new Date(),
-				deleteAt: new Date(Date.now() + 1000 * 60 * 3), // 3 minutes
-			},
-		}
-	);
-
-	return result.matchedCount > 0;
-}
+import { collections } from "$lib/server/database";
+import { ObjectId } from "mongodb";
+import type { Semaphores } from "$lib/types/Semaphore";
+
+/**
+ * Returns the lock id if the lock was acquired, false otherwise
+ */
+export async function acquireLock(key: Semaphores | string): Promise<ObjectId | false> {
+	try {
+		const id = new ObjectId();
+
+		const insert = await collections.semaphores.insertOne({
+			_id: id,
+			key,
+			createdAt: new Date(),
+			updatedAt: new Date(),
+			deleteAt: new Date(Date.now() + 1000 * 60 * 3), // 3 minutes
+		});
+
+		return insert.acknowledged ? id : false; // true if the document was inserted
+	} catch (e) {
+		// unique index violation, so there must already be a lock
+		return false;
+	}
+}
+
+export async function releaseLock(key: Semaphores | string, lockId: ObjectId) {
+	await collections.semaphores.deleteOne({
+		_id: lockId,
+		key,
+	});
+}
+
+export async function isDBLocked(key: Semaphores | string): Promise<boolean> {
+	const res = await collections.semaphores.countDocuments({
+		key,
+	});
+	return res > 0;
+}
+
+export async function refreshLock(key: Semaphores | string, lockId: ObjectId): Promise<boolean> {
+	const result = await collections.semaphores.updateOne(
+		{
+			_id: lockId,
+			key,
+		},
+		{
+			$set: {
+				updatedAt: new Date(),
+				deleteAt: new Date(Date.now() + 1000 * 60 * 3), // 3 minutes
+			},
+		}
+	);
+
+	return result.matchedCount > 0;
+}

package/src/ruvocal/src/lib/migrations/migrations.spec.ts

@@ -1,74 +1,74 @@
-import { afterEach, assert, beforeAll, describe, expect, it } from "vitest";
-import { migrations } from "./routines";
-import { acquireLock, isDBLocked, refreshLock, releaseLock } from "./lock";
-import { Semaphores } from "$lib/types/Semaphore";
-import { collections, ready } from "$lib/server/database";
-
-describe(
-	"migrations",
-	{
-		retry: 3,
-	},
-	() => {
-		beforeAll(async () => {
-			await ready;
-			try {
-				await collections.semaphores.createIndex({ key: 1 }, { unique: true });
-			} catch (e) {
-				// Index might already exist, ignore error
-			}
-		}, 20000);
-
-		it("should not have duplicates guid", async () => {
-			const guids = migrations.map((m) => m._id.toString());
-			const uniqueGuids = [...new Set(guids)];
-			expect(uniqueGuids.length).toBe(guids.length);
-		});
-
-		it("should acquire only one lock on DB", async () => {
-			const results = await Promise.all(
-				new Array(1000).fill(0).map(() => acquireLock(Semaphores.TEST_MIGRATION))
-			);
-			const locks = results.filter((r) => r);
-
-			const semaphores = await collections.semaphores.find({}).toArray();
-
-			expect(locks.length).toBe(1);
-			expect(semaphores).toBeDefined();
-			expect(semaphores.length).toBe(1);
-			expect(semaphores?.[0].key).toBe(Semaphores.TEST_MIGRATION);
-		});
-
-		it("should read the lock correctly", async () => {
-			const lockId = await acquireLock(Semaphores.TEST_MIGRATION);
-			assert(lockId);
-			expect(await isDBLocked(Semaphores.TEST_MIGRATION)).toBe(true);
-			expect(!!(await acquireLock(Semaphores.TEST_MIGRATION))).toBe(false);
-			await releaseLock(Semaphores.TEST_MIGRATION, lockId);
-			expect(await isDBLocked(Semaphores.TEST_MIGRATION)).toBe(false);
-		});
-
-		it("should refresh the lock", async () => {
-			const lockId = await acquireLock(Semaphores.TEST_MIGRATION);
-
-			assert(lockId);
-
-			// get the updatedAt time
-
-			const updatedAtInitially = (await collections.semaphores.findOne({}))?.updatedAt;
-
-			await refreshLock(Semaphores.TEST_MIGRATION, lockId);
-
-			const updatedAtAfterRefresh = (await collections.semaphores.findOne({}))?.updatedAt;
-
-			expect(updatedAtInitially).toBeDefined();
-			expect(updatedAtAfterRefresh).toBeDefined();
-			expect(updatedAtInitially).not.toBe(updatedAtAfterRefresh);
-		});
-
-		afterEach(async () => {
-			await collections.semaphores.deleteMany({});
-			await collections.migrationResults.deleteMany({});
-		});
-	}
-);
+import { afterEach, assert, beforeAll, describe, expect, it } from "vitest";
+import { migrations } from "./routines";
+import { acquireLock, isDBLocked, refreshLock, releaseLock } from "./lock";
+import { Semaphores } from "$lib/types/Semaphore";
+import { collections, ready } from "$lib/server/database";
+
+describe(
+	"migrations",
+	{
+		retry: 3,
+	},
+	() => {
+		beforeAll(async () => {
+			await ready;
+			try {
+				await collections.semaphores.createIndex({ key: 1 }, { unique: true });
+			} catch (e) {
+				// Index might already exist, ignore error
+			}
+		}, 20000);
+
+		it("should not have duplicates guid", async () => {
+			const guids = migrations.map((m) => m._id.toString());
+			const uniqueGuids = [...new Set(guids)];
+			expect(uniqueGuids.length).toBe(guids.length);
+		});
+
+		it("should acquire only one lock on DB", async () => {
+			const results = await Promise.all(
+				new Array(1000).fill(0).map(() => acquireLock(Semaphores.TEST_MIGRATION))
+			);
+			const locks = results.filter((r) => r);
+
+			const semaphores = await collections.semaphores.find({}).toArray();
+
+			expect(locks.length).toBe(1);
+			expect(semaphores).toBeDefined();
+			expect(semaphores.length).toBe(1);
+			expect(semaphores?.[0].key).toBe(Semaphores.TEST_MIGRATION);
+		});
+
+		it("should read the lock correctly", async () => {
+			const lockId = await acquireLock(Semaphores.TEST_MIGRATION);
+			assert(lockId);
+			expect(await isDBLocked(Semaphores.TEST_MIGRATION)).toBe(true);
+			expect(!!(await acquireLock(Semaphores.TEST_MIGRATION))).toBe(false);
+			await releaseLock(Semaphores.TEST_MIGRATION, lockId);
+			expect(await isDBLocked(Semaphores.TEST_MIGRATION)).toBe(false);
+		});
+
+		it("should refresh the lock", async () => {
+			const lockId = await acquireLock(Semaphores.TEST_MIGRATION);
+
+			assert(lockId);
+
+			// get the updatedAt time
+
+			const updatedAtInitially = (await collections.semaphores.findOne({}))?.updatedAt;
+
+			await refreshLock(Semaphores.TEST_MIGRATION, lockId);
+
+			const updatedAtAfterRefresh = (await collections.semaphores.findOne({}))?.updatedAt;
+
+			expect(updatedAtInitially).toBeDefined();
+			expect(updatedAtAfterRefresh).toBeDefined();
+			expect(updatedAtInitially).not.toBe(updatedAtAfterRefresh);
+		});
+
+		afterEach(async () => {
+			await collections.semaphores.deleteMany({});
+			await collections.migrationResults.deleteMany({});
+		});
+	}
+);

package/src/ruvocal/src/lib/migrations/migrations.ts

@@ -1,109 +1,109 @@
-import { Database } from "$lib/server/database";
-import { migrations } from "./routines";
-import { acquireLock, releaseLock, isDBLocked, refreshLock } from "./lock";
-import { Semaphores } from "$lib/types/Semaphore";
-import { logger } from "$lib/server/logger";
-import { config } from "$lib/server/config";
-
-export async function checkAndRunMigrations() {
-	// make sure all GUIDs are unique
-	if (new Set(migrations.map((m) => m._id.toString())).size !== migrations.length) {
-		throw new Error("Duplicate migration GUIDs found.");
-	}
-
-	// check if all migrations have already been run
-	const migrationResults = await (await Database.getInstance())
-		.getCollections()
-		.migrationResults.find()
-		.toArray();
-
-	logger.debug("[MIGRATIONS] Begin check...");
-
-	const lockId = await acquireLock(Semaphores.MIGRATION);
-
-	if (!lockId) {
-		// another instance already has the lock, so we exit early
-		logger.debug(
-			"[MIGRATIONS] Another instance already has the lock. Waiting for DB to be unlocked."
-		);
-
-		// block until the lock is released
-		while (await isDBLocked(Semaphores.MIGRATION)) {
-			await new Promise((resolve) => setTimeout(resolve, 1000));
-		}
-		return;
-	}
-
-	// once here, we have the lock
-	// make sure to refresh it regularly while it's running
-	const refreshInterval = setInterval(async () => {
-		await refreshLock(Semaphores.MIGRATION, lockId);
-	}, 1000 * 10);
-
-	// iterate over all migrations
-	for (const migration of migrations) {
-		// check if the migration has already been applied
-		const shouldRun =
-			migration.runEveryTime ||
-			!migrationResults.find((m) => m._id.toString() === migration._id.toString());
-
-		// check if the migration has already been applied
-		if (!shouldRun) {
-			logger.debug(`[MIGRATIONS] "${migration.name}" already applied. Skipping...`);
-		} else {
-			// check the modifiers to see if some cases match
-			if (
-				(migration.runForHuggingChat === "only" && !config.isHuggingChat) ||
-				(migration.runForHuggingChat === "never" && config.isHuggingChat)
-			) {
-				logger.debug(
-					`[MIGRATIONS] "${migration.name}" should not be applied for this run. Skipping...`
-				);
-				continue;
-			}
-
-			// otherwise all is good and we can run the migration
-			logger.debug(
-				`[MIGRATIONS] "${migration.name}" ${
-					migration.runEveryTime ? "should run every time" : "not applied yet"
-				}. Applying...`
-			);
-
-			await (await Database.getInstance()).getCollections().migrationResults.updateOne(
-				{ _id: migration._id },
-				{
-					$set: {
-						name: migration.name,
-						status: "ongoing",
-					},
-				},
-				{ upsert: true }
-			);
-
-			let result = false;
-
-			try {
-				// RVF store: no transactions needed, run migration directly
-				result = await migration.up(await Database.getInstance());
-			} catch (e) {
-				logger.error(e, `[MIGRATIONS]  "${migration.name}" failed!`);
-			}
-
-			await (await Database.getInstance()).getCollections().migrationResults.updateOne(
-				{ _id: migration._id },
-				{
-					$set: {
-						name: migration.name,
-						status: result ? "success" : "failure",
-					},
-				},
-				{ upsert: true }
-			);
-		}
-	}
-
-	logger.debug("[MIGRATIONS] All migrations applied. Releasing lock");
-
-	clearInterval(refreshInterval);
-	await releaseLock(Semaphores.MIGRATION, lockId);
-}
+import { Database } from "$lib/server/database";
+import { migrations } from "./routines";
+import { acquireLock, releaseLock, isDBLocked, refreshLock } from "./lock";
+import { Semaphores } from "$lib/types/Semaphore";
+import { logger } from "$lib/server/logger";
+import { config } from "$lib/server/config";
+
+export async function checkAndRunMigrations() {
+	// make sure all GUIDs are unique
+	if (new Set(migrations.map((m) => m._id.toString())).size !== migrations.length) {
+		throw new Error("Duplicate migration GUIDs found.");
+	}
+
+	// check if all migrations have already been run
+	const migrationResults = await (await Database.getInstance())
+		.getCollections()
+		.migrationResults.find()
+		.toArray();
+
+	logger.debug("[MIGRATIONS] Begin check...");
+
+	const lockId = await acquireLock(Semaphores.MIGRATION);
+
+	if (!lockId) {
+		// another instance already has the lock, so we exit early
+		logger.debug(
+			"[MIGRATIONS] Another instance already has the lock. Waiting for DB to be unlocked."
+		);
+
+		// block until the lock is released
+		while (await isDBLocked(Semaphores.MIGRATION)) {
+			await new Promise((resolve) => setTimeout(resolve, 1000));
+		}
+		return;
+	}
+
+	// once here, we have the lock
+	// make sure to refresh it regularly while it's running
+	const refreshInterval = setInterval(async () => {
+		await refreshLock(Semaphores.MIGRATION, lockId);
+	}, 1000 * 10);
+
+	// iterate over all migrations
+	for (const migration of migrations) {
+		// check if the migration has already been applied
+		const shouldRun =
+			migration.runEveryTime ||
+			!migrationResults.find((m) => m._id.toString() === migration._id.toString());
+
+		// check if the migration has already been applied
+		if (!shouldRun) {
+			logger.debug(`[MIGRATIONS] "${migration.name}" already applied. Skipping...`);
+		} else {
+			// check the modifiers to see if some cases match
+			if (
+				(migration.runForHuggingChat === "only" && !config.isHuggingChat) ||
+				(migration.runForHuggingChat === "never" && config.isHuggingChat)
+			) {
+				logger.debug(
+					`[MIGRATIONS] "${migration.name}" should not be applied for this run. Skipping...`
+				);
+				continue;
+			}
+
+			// otherwise all is good and we can run the migration
+			logger.debug(
+				`[MIGRATIONS] "${migration.name}" ${
+					migration.runEveryTime ? "should run every time" : "not applied yet"
+				}. Applying...`
+			);
+
+			await (await Database.getInstance()).getCollections().migrationResults.updateOne(
+				{ _id: migration._id },
+				{
+					$set: {
+						name: migration.name,
+						status: "ongoing",
+					},
+				},
+				{ upsert: true }
+			);
+
+			let result = false;
+
+			try {
+				// RVF store: no transactions needed, run migration directly
+				result = await migration.up(await Database.getInstance());
+			} catch (e) {
+				logger.error(e, `[MIGRATIONS]  "${migration.name}" failed!`);
+			}
+
+			await (await Database.getInstance()).getCollections().migrationResults.updateOne(
+				{ _id: migration._id },
+				{
+					$set: {
+						name: migration.name,
+						status: result ? "success" : "failure",
+					},
+				},
+				{ upsert: true }
+			);
+		}
+	}
+
+	logger.debug("[MIGRATIONS] All migrations applied. Releasing lock");
+
+	clearInterval(refreshInterval);
+	await releaseLock(Semaphores.MIGRATION, lockId);
+}

package/src/ruvocal/src/lib/migrations/routines/01-update-search-assistants.ts

@@ -1,50 +1,50 @@
-import type { Migration } from ".";
-import { collections } from "$lib/server/database";
-import { ObjectId, type AnyBulkWriteOperation } from "mongodb";
-import type { Assistant } from "$lib/types/Assistant";
-import { generateSearchTokens } from "$lib/utils/searchTokens";
-
-const migration: Migration = {
-	_id: new ObjectId("5f9f3e3e3e3e3e3e3e3e3e3e"),
-	name: "Update search assistants",
-	up: async () => {
-		const { assistants } = collections;
-		let ops: AnyBulkWriteOperation<Assistant>[] = [];
-
-		for await (const assistant of assistants
-			.find()
-			.project<Pick<Assistant, "_id" | "name">>({ _id: 1, name: 1 })) {
-			ops.push({
-				updateOne: {
-					filter: {
-						_id: assistant._id,
-					},
-					update: {
-						$set: {
-							searchTokens: generateSearchTokens(assistant.name),
-						},
-					},
-				},
-			});
-
-			if (ops.length >= 1000) {
-				process.stdout.write(".");
-				await assistants.bulkWrite(ops, { ordered: false });
-				ops = [];
-			}
-		}
-
-		if (ops.length) {
-			await assistants.bulkWrite(ops, { ordered: false });
-		}
-
-		return true;
-	},
-	down: async () => {
-		const { assistants } = collections;
-		await assistants.updateMany({}, { $unset: { searchTokens: "" } });
-		return true;
-	},
-};
-
-export default migration;
+import type { Migration } from ".";
+import { collections } from "$lib/server/database";
+import { ObjectId, type AnyBulkWriteOperation } from "mongodb";
+import type { Assistant } from "$lib/types/Assistant";
+import { generateSearchTokens } from "$lib/utils/searchTokens";
+
+const migration: Migration = {
+	_id: new ObjectId("5f9f3e3e3e3e3e3e3e3e3e3e"),
+	name: "Update search assistants",
+	up: async () => {
+		const { assistants } = collections;
+		let ops: AnyBulkWriteOperation<Assistant>[] = [];
+
+		for await (const assistant of assistants
+			.find()
+			.project<Pick<Assistant, "_id" | "name">>({ _id: 1, name: 1 })) {
+			ops.push({
+				updateOne: {
+					filter: {
+						_id: assistant._id,
+					},
+					update: {
+						$set: {
+							searchTokens: generateSearchTokens(assistant.name),
+						},
+					},
+				},
+			});
+
+			if (ops.length >= 1000) {
+				process.stdout.write(".");
+				await assistants.bulkWrite(ops, { ordered: false });
+				ops = [];
+			}
+		}
+
+		if (ops.length) {
+			await assistants.bulkWrite(ops, { ordered: false });
+		}
+
+		return true;
+	},
+	down: async () => {
+		const { assistants } = collections;
+		await assistants.updateMany({}, { $unset: { searchTokens: "" } });
+		return true;
+	},
+};
+
+export default migration;