rivetkit 2.0.42 → 2.1.0-rc.1

package/src/drivers/file-system/global-state.ts

@@ -5,6 +5,7 @@ import type { AnyActorInstance } from "@/actor/instance/mod";
 import type { ActorKey } from "@/actor/mod";
 import type { AnyClient } from "@/client/client";
 import { type ActorDriver, getInitialActorKvState } from "@/driver-helpers/mod";
+import type { RegistryConfig } from "@/registry/config";
 import type * as schema from "@/schemas/file-system-driver/mod";
 import {
 	ACTOR_ALARM_VERSIONED,
@@ -12,8 +13,6 @@ import {
 	CURRENT_VERSION as FILE_SYSTEM_DRIVER_CURRENT_VERSION,
 } from "@/schemas/file-system-driver/versioned";
 import {
-	arrayBuffersEqual,
-	bufferToArrayBuffer,
 	type LongTimeoutHandle,
 	promiseWithResolvers,
 	setLongTimeout,
@@ -31,7 +30,19 @@ import {
 	ensureDirectoryExistsSync,
 	getStoragePath,
 } from "./utils";
-import { RegistryConfig } from "@/registry/config";
+import {
+	computePrefixUpperBound,
+	ensureUint8Array,
+	loadSqliteRuntime,
+	type SqliteRuntime,
+	type SqliteRuntimeDatabase,
+} from "./sqlite-runtime";
+import {
+	estimateKvSize,
+	validateKvEntries,
+	validateKvKey,
+	validateKvKeys,
+} from "./kv-limits";
 
 // Actor handler to track running instances
 
@@ -54,6 +65,8 @@ interface ActorEntry {
 	actor?: AnyActorInstance;
 	/** Promise for starting the actor. */
 	startPromise?: ReturnType<typeof promiseWithResolvers<void>>;
+	/** Promise for stopping the actor. */
+	stopPromise?: PromiseWithResolvers<void>;
 
 	alarmTimeout?: LongTimeoutHandle;
 	/** The timestamp currently scheduled for this actor's alarm (ms since epoch). */
@@ -70,6 +83,15 @@ interface ActorEntry {
 	generation: string;
 }
 
+export interface FileSystemDriverOptions {
+	/** Whether to persist data to disk */
+	persist?: boolean;
+	/** Custom path for storage */
+	customPath?: string;
+	/** Deprecated option retained for explicit migration to sqlite-only KV. */
+	useNativeSqlite?: boolean;
+}
+
 /**
  * Global state for the file system driver
  */
@@ -80,6 +102,8 @@ export class FileSystemGlobalState {
 	#alarmsDir: string;
 
 	#persist: boolean;
+	#sqliteRuntime: SqliteRuntime;
+	#actorKvDatabases = new Map<string, SqliteRuntimeDatabase>();
 
 	// IMPORTANT: Never delete from this map. Doing so will result in race
 	// conditions since the actor generation will cease to be tracked
@@ -106,8 +130,15 @@ export class FileSystemGlobalState {
 		return this.#actorCountOnStartup;
 	}
 
-	constructor(persist: boolean = true, customPath?: string) {
+	constructor(options: FileSystemDriverOptions = {}) {
+		const { persist = true, customPath, useNativeSqlite = true } = options;
+		if (!useNativeSqlite) {
+			throw new Error(
+				"File-system driver no longer supports non-SQLite KV storage.",
+			);
+		}
 		this.#persist = persist;
+		this.#sqliteRuntime = loadSqliteRuntime();
 		this.#storagePath = persist ? (customPath ?? getStoragePath()) : "/tmp";
 		const path = getNodePath();
 		this.#stateDir = path.join(this.#storagePath, "state");
@@ -132,6 +163,7 @@ export class FileSystemGlobalState {
 				msg: "file system driver ready",
 				dir: this.#storagePath,
 				actorCount: this.#actorCountOnStartup,
+				sqliteRuntime: this.#sqliteRuntime.kind,
 			});
 
 			// Cleanup stale temp files on startup
@@ -143,8 +175,21 @@ export class FileSystemGlobalState {
 					error: err,
 				});
 			}
+
+			try {
+				this.#migrateLegacyKvToSqliteOnStartupSync();
+			} catch (error) {
+				logger().error({
+					msg: "failed legacy kv startup migration",
+					error,
+				});
+				throw error;
+			}
 		} else {
-			logger().debug({ msg: "memory driver ready" });
+			logger().debug({
+				msg: "memory driver ready",
+				sqliteRuntime: this.#sqliteRuntime.kind,
+			});
 		}
 	}
 
@@ -160,6 +205,154 @@ export class FileSystemGlobalState {
 		return getNodePath().join(this.#alarmsDir, actorId);
 	}
 
+	#getActorKvDatabasePath(actorId: string): string {
+		if (this.#persist) {
+			return this.getActorDbPath(actorId);
+		}
+		return ":memory:";
+	}
+
+	#ensureActorKvTables(db: SqliteRuntimeDatabase): void {
+		db.exec(`
+			CREATE TABLE IF NOT EXISTS kv (
+				key BLOB PRIMARY KEY NOT NULL,
+				value BLOB NOT NULL
+			)
+		`);
+	}
+
+	#getOrCreateActorKvDatabase(actorId: string): SqliteRuntimeDatabase {
+		const existing = this.#actorKvDatabases.get(actorId);
+		if (existing) {
+			return existing;
+		}
+
+		const dbPath = this.#getActorKvDatabasePath(actorId);
+		if (this.#persist) {
+			const path = getNodePath();
+			ensureDirectoryExistsSync(path.dirname(dbPath));
+		}
+
+		let db: SqliteRuntimeDatabase;
+		try {
+			db = this.#sqliteRuntime.open(dbPath);
+		} catch (error) {
+			throw new Error(
+				`failed to open actor kv database for actor ${actorId} at ${dbPath}: ${error}`,
+			);
+		}
+
+		this.#ensureActorKvTables(db);
+		this.#actorKvDatabases.set(actorId, db);
+		return db;
+	}
+
+	#closeActorKvDatabase(actorId: string): void {
+		const db = this.#actorKvDatabases.get(actorId);
+		if (!db) {
+			return;
+		}
+
+		try {
+			db.close();
+		} finally {
+			this.#actorKvDatabases.delete(actorId);
+		}
+	}
+
+	#putKvEntriesInDb(
+		db: SqliteRuntimeDatabase,
+		entries: [Uint8Array, Uint8Array][],
+	): void {
+		if (entries.length === 0) {
+			return;
+		}
+
+		db.exec("BEGIN");
+		try {
+			for (const [key, value] of entries) {
+				db.run("INSERT OR REPLACE INTO kv (key, value) VALUES (?, ?)", [
+					key,
+					value,
+				]);
+			}
+			db.exec("COMMIT");
+		} catch (error) {
+			try {
+				db.exec("ROLLBACK");
+			} catch {
+				// Ignore rollback errors, original error is more actionable.
+			}
+			throw error;
+		}
+	}
+
+	#isKvDbPopulated(db: SqliteRuntimeDatabase): boolean {
+		const row = db.get<{ count: number | bigint }>(
+			"SELECT COUNT(*) AS count FROM kv",
+		);
+		const count = row ? Number(row.count) : 0;
+		return count > 0;
+	}
+
+	#migrateLegacyKvToSqliteOnStartupSync(): void {
+		const fsSync = getNodeFsSync();
+		if (!fsSync.existsSync(this.#stateDir)) {
+			return;
+		}
+
+		const actorIds = fsSync
+			.readdirSync(this.#stateDir)
+			.filter((id) => !id.includes(".tmp."));
+
+		for (const actorId of actorIds) {
+			const statePath = this.getActorStatePath(actorId);
+			let state: schema.ActorState;
+			try {
+				const stateBytes = fsSync.readFileSync(statePath);
+				state = ACTOR_STATE_VERSIONED.deserializeWithEmbeddedVersion(
+					new Uint8Array(stateBytes),
+				);
+			} catch (error) {
+				logger().warn({
+					msg: "failed to parse actor state during startup migration",
+					actorId,
+					error,
+				});
+				continue;
+			}
+
+			if (!state.kvStorage || state.kvStorage.length === 0) {
+				continue;
+			}
+
+			const dbPath = this.getActorDbPath(actorId);
+			const path = getNodePath();
+			ensureDirectoryExistsSync(path.dirname(dbPath));
+			const db = this.#sqliteRuntime.open(dbPath);
+			try {
+				this.#ensureActorKvTables(db);
+				if (this.#isKvDbPopulated(db)) {
+					continue;
+				}
+
+				const legacyEntries = state.kvStorage.map((entry) => [
+					new Uint8Array(entry.key),
+					new Uint8Array(entry.value),
+				]) as [Uint8Array, Uint8Array][];
+				this.#putKvEntriesInDb(db, legacyEntries);
+
+				logger().info({
+					msg: "migrated legacy actor kv storage to sqlite",
+					actorId,
+					entryCount: legacyEntries.length,
+				});
+			} finally {
+				db.close();
+			}
+		}
+	}
+
 	async *getActorsIterator(params: {
 		cursor?: string;
 	}): AsyncGenerator<schema.ActorState> {
@@ -228,14 +421,16 @@ export class FileSystemGlobalState {
 	): Promise<ActorEntry> {
 		// TODO: Does not check if actor already exists on fs
 
-		const entry = this.#upsertEntry(actorId);
+		await this.#waitForActorStop(actorId);
+		let entry = this.#upsertEntry(actorId);
 
 		// Check if actor already exists (has state or is being stopped)
 		if (entry.state) {
 			throw new ActorDuplicateKey(name, key);
 		}
 		if (this.isActorStopping(actorId)) {
-			throw new Error(`Actor ${actorId} is stopping`);
+			await this.#waitForActorStop(actorId);
+			entry = this.#upsertEntry(actorId);
 		}
 
 		// If actor was destroyed, reset to NONEXISTENT and increment generation
@@ -244,31 +439,35 @@ export class FileSystemGlobalState {
 			entry.generation = crypto.randomUUID();
 		}
 
-		// Initialize storage
-		const kvStorage: schema.ActorKvEntry[] = [];
+		// Initialize storage (runtime KV is stored in SQLite; state.kvStorage is legacy-only)
 		const initialKvState = getInitialActorKvState(input);
-		for (const [key, value] of initialKvState) {
-			kvStorage.push({
-				key: bufferToArrayBuffer(key),
-				value: bufferToArrayBuffer(value),
-			});
-		}
 
 		// Initialize metadata
-		entry.state = {
-			actorId,
-			name,
-			key,
-			createdAt: BigInt(Date.now()),
-			kvStorage,
-			startTs: null,
-			connectableTs: null,
-			sleepTs: null,
-			destroyTs: null,
-		};
-		entry.lifecycleState = ActorLifecycleState.AWAKE;
-
-		await this.writeActor(actorId, entry.generation, entry.state);
+		await this.#withActorWrite(actorId, async (lockedEntry) => {
+			lockedEntry.state = {
+				actorId,
+				name,
+				key,
+				createdAt: BigInt(Date.now()),
+				kvStorage: [],
+				startTs: null,
+				connectableTs: null,
+				sleepTs: null,
+				destroyTs: null,
+			};
+			lockedEntry.lifecycleState = ActorLifecycleState.AWAKE;
+			if (this.#persist) {
+				await this.#performWrite(
+					actorId,
+					lockedEntry.generation,
+					lockedEntry.state,
+				);
+			}
+			if (initialKvState.length > 0) {
+				const db = this.#getOrCreateActorKvDatabase(actorId);
+				this.#putKvEntriesInDb(db, initialKvState);
+			}
+		});
 
 		return entry;
 	}
@@ -313,10 +512,16 @@ export class FileSystemGlobalState {
 			const fs = getNodeFs();
 			const stateData = await fs.readFile(stateFilePath);
 
-			// Cache the loaded state in handler
-			entry.state = ACTOR_STATE_VERSIONED.deserializeWithEmbeddedVersion(
-				new Uint8Array(stateData),
-			);
+			const loadedState =
+				ACTOR_STATE_VERSIONED.deserializeWithEmbeddedVersion(
+					new Uint8Array(stateData),
+				);
+
+			// Runtime reads/writes are SQLite-only; legacy kvStorage is for one-time startup migration.
+			entry.state = {
+				...loadedState,
+				kvStorage: [],
+			};
 
 			return entry;
 		} catch (innerError: any) {
@@ -340,13 +545,16 @@ export class FileSystemGlobalState {
 		key: ActorKey,
 		input: unknown | undefined,
 	): Promise<ActorEntry> {
+		await this.#waitForActorStop(actorId);
+
 		// Attempt to load actor
 		const entry = await this.loadActor(actorId);
 
 		// If no state for this actor, then create & write state
 		if (!entry.state) {
 			if (this.isActorStopping(actorId)) {
-				throw new Error(`Actor ${actorId} stopping`);
+				await this.#waitForActorStop(actorId);
+				return await this.loadOrCreateActor(actorId, name, key, input);
 			}
 
 			// If actor was destroyed, reset to NONEXISTENT and increment generation
@@ -355,31 +563,36 @@ export class FileSystemGlobalState {
 				entry.generation = crypto.randomUUID();
 			}
 
-			// Initialize kvStorage with the initial persist data
-			const kvStorage: schema.ActorKvEntry[] = [];
-			const initialKvState = getInitialActorKvState(input);
-			for (const [key, value] of initialKvState) {
-				kvStorage.push({
-					key: bufferToArrayBuffer(key),
-					value: bufferToArrayBuffer(value),
+				// Initialize storage (runtime KV is stored in SQLite; state.kvStorage is legacy-only)
+				const initialKvState = getInitialActorKvState(input);
+
+				await this.#withActorWrite(actorId, async (lockedEntry) => {
+					lockedEntry.state = {
+						actorId,
+						name,
+						key: key as readonly string[],
+						createdAt: BigInt(Date.now()),
+						kvStorage: [],
+						startTs: null,
+						connectableTs: null,
+						sleepTs: null,
+						destroyTs: null,
+					};
+					if (this.#persist) {
+						await this.#performWrite(
+							actorId,
+							lockedEntry.generation,
+							lockedEntry.state,
+						);
+					}
+					if (initialKvState.length > 0) {
+						const db = this.#getOrCreateActorKvDatabase(actorId);
+						this.#putKvEntriesInDb(db, initialKvState);
+					}
 				});
 			}
-
-			entry.state = {
-				actorId,
-				name,
-				key: key as readonly string[],
-				createdAt: BigInt(Date.now()),
-				kvStorage,
-				startTs: null,
-				connectableTs: null,
-				sleepTs: null,
-				destroyTs: null,
-			};
-			await this.writeActor(actorId, entry.generation, entry.state);
+			return entry;
 		}
-		return entry;
-	}
 
 	async sleepActor(actorId: string) {
 		invariant(
@@ -396,27 +609,47 @@ export class FileSystemGlobalState {
 			return;
 		}
 		actor.lifecycleState = ActorLifecycleState.STARTING_SLEEP;
+		actor.stopPromise = promiseWithResolvers((reason) => logger().warn({ msg: "unhandled actor sleep stop promise rejection", reason }));
 
 		// Wait for actor to fully start before stopping it to avoid race conditions
 		if (actor.loadPromise) await actor.loadPromise.catch();
 		if (actor.startPromise?.promise)
 			await actor.startPromise.promise.catch();
 
-		// Update state with sleep timestamp
-		if (actor.state) {
-			actor.state = {
-				...actor.state,
-				sleepTs: BigInt(Date.now()),
-			};
-			await this.writeActor(actorId, actor.generation, actor.state);
-		}
-
-		// Stop actor
-		invariant(actor.actor, "actor should be loaded");
-		await actor.actor.onStop("sleep");
+		try {
+			// Update state with sleep timestamp
+			if (actor.state) {
+				await this.#withActorWrite(actorId, async (lockedEntry) => {
+					if (!lockedEntry.state) {
+						return;
+					}
+					lockedEntry.state = {
+						...lockedEntry.state,
+						sleepTs: BigInt(Date.now()),
+					};
+					if (this.#persist) {
+						await this.#performWrite(
+							actorId,
+							lockedEntry.generation,
+							lockedEntry.state,
+						);
+					}
+				});
+			}
 
-		// Remove from map after stop is complete
-		this.#actors.delete(actorId);
+			// Stop actor
+			invariant(actor.actor, "actor should be loaded");
+			await actor.actor.onStop("sleep");
+			} finally {
+				// Ensure any pending KV writes finish before removing the entry.
+				await this.#withActorWrite(actorId, async () => {});
+				this.#closeActorKvDatabase(actorId);
+				actor.stopPromise?.resolve();
+				actor.stopPromise = undefined;
+
+			// Remove from map after stop is complete
+			this.#actors.delete(actorId);
+		}
 	}
 
 	async destroyActor(actorId: string) {
@@ -429,94 +662,117 @@ export class FileSystemGlobalState {
 			return;
 		}
 		actor.lifecycleState = ActorLifecycleState.STARTING_DESTROY;
+		actor.stopPromise = promiseWithResolvers((reason) => logger().warn({ msg: "unhandled actor destroy stop promise rejection", reason }));
 
 		// Wait for actor to fully start before stopping it to avoid race conditions
 		if (actor.loadPromise) await actor.loadPromise.catch();
 		if (actor.startPromise?.promise)
 			await actor.startPromise.promise.catch();
 
-		// Update state with destroy timestamp
-		if (actor.state) {
-			actor.state = {
-				...actor.state,
-				destroyTs: BigInt(Date.now()),
-			};
-			await this.writeActor(actorId, actor.generation, actor.state);
-		}
+		try {
+			// Update state with destroy timestamp
+			if (actor.state) {
+				await this.#withActorWrite(actorId, async (lockedEntry) => {
+					if (!lockedEntry.state) {
+						return;
+					}
+					lockedEntry.state = {
+						...lockedEntry.state,
+						destroyTs: BigInt(Date.now()),
+					};
+					if (this.#persist) {
+						await this.#performWrite(
+							actorId,
+							lockedEntry.generation,
+							lockedEntry.state,
+						);
+					}
+				});
+			}
 
-		// Stop actor if it's running
-		if (actor.actor) {
-			await actor.actor.onStop("destroy");
-		}
+			// Stop actor if it's running
+			if (actor.actor) {
+				await actor.actor.onStop("destroy");
+			}
 
-		// Clear alarm timeout if exists
-		if (actor.alarmTimeout) {
-			actor.alarmTimeout.abort();
-		}
+				// Ensure any pending KV writes finish before deleting files.
+				await this.#withActorWrite(actorId, async () => {});
+				this.#closeActorKvDatabase(actorId);
 
-		// Delete persisted files if using file system driver
-		if (this.#persist) {
-			const fs = getNodeFs();
+			// Clear alarm timeout if exists
+			if (actor.alarmTimeout) {
+				actor.alarmTimeout.abort();
+			}
 
-			// Delete all actor files in parallel
-			await Promise.all([
-				// Delete actor state file
-				(async () => {
-					try {
-						await fs.unlink(this.getActorStatePath(actorId));
-					} catch (err: any) {
-						if (err?.code !== "ENOENT") {
-							logger().error({
-								msg: "failed to delete actor state file",
-								actorId,
-								error: stringifyError(err),
-							});
+			// Delete persisted files if using file system driver
+			if (this.#persist) {
+				const fs = getNodeFs();
+
+				// Delete all actor files in parallel
+				await Promise.all([
+					// Delete actor state file
+					(async () => {
+						try {
+							await fs.unlink(this.getActorStatePath(actorId));
+						} catch (err: any) {
+							if (err?.code !== "ENOENT") {
+								logger().error({
+									msg: "failed to delete actor state file",
+									actorId,
+									error: stringifyError(err),
+								});
+							}
 						}
-					}
-				})(),
-				// Delete actor database file
-				(async () => {
-					try {
-						await fs.unlink(this.getActorDbPath(actorId));
-					} catch (err: any) {
-						if (err?.code !== "ENOENT") {
-							logger().error({
-								msg: "failed to delete actor database file",
-								actorId,
-								error: stringifyError(err),
-							});
+					})(),
+					// Delete actor database file
+					(async () => {
+						try {
+							await fs.unlink(this.getActorDbPath(actorId));
+						} catch (err: any) {
+							if (err?.code !== "ENOENT") {
+								logger().error({
+									msg: "failed to delete actor database file",
+									actorId,
+									error: stringifyError(err),
+								});
+							}
 						}
-					}
-				})(),
-				// Delete actor alarm file
-				(async () => {
-					try {
-						await fs.unlink(this.getActorAlarmPath(actorId));
-					} catch (err: any) {
-						if (err?.code !== "ENOENT") {
-							logger().error({
-								msg: "failed to delete actor alarm file",
-								actorId,
-								error: stringifyError(err),
-							});
+					})(),
+					// Delete actor alarm file
+					(async () => {
+						try {
+							await fs.unlink(this.getActorAlarmPath(actorId));
+						} catch (err: any) {
+							if (err?.code !== "ENOENT") {
+								logger().error({
+									msg: "failed to delete actor alarm file",
+									actorId,
+									error: stringifyError(err),
+								});
+							}
 						}
-					}
-				})(),
-			]);
+					})(),
+				]);
+			}
+		} finally {
+			// Ensure any pending KV writes finish before clearing the entry.
+			await this.#withActorWrite(actorId, async () => {});
+			actor.stopPromise?.resolve();
+			actor.stopPromise = undefined;
+
+			// Reset the entry
+			//
+			// Do not remove entry in order to avoid race condition with
+			// destroying. Next actor creation will increment the generation.
+			actor.state = undefined;
+			actor.loadPromise = undefined;
+			actor.actor = undefined;
+			actor.startPromise = undefined;
+			actor.alarmTimeout = undefined;
+			actor.alarmTimeout = undefined;
+			actor.pendingWriteResolver = undefined;
+			actor.lifecycleState = ActorLifecycleState.DESTROYED;
 		}
-
-		// Reset the entry
-		//
-		// Do not remove entry in order to avoid race condition with
-		// destroying. Next actor creation will increment the generation.
-		actor.state = undefined;
-		actor.loadPromise = undefined;
-		actor.actor = undefined;
-		actor.startPromise = undefined;
-		actor.alarmTimeout = undefined;
-		actor.alarmTimeout = undefined;
-		actor.pendingWriteResolver = undefined;
-		actor.lifecycleState = ActorLifecycleState.DESTROYED;
 	}
 
 	/**
@@ -531,10 +787,9 @@ export class FileSystemGlobalState {
 			return;
 		}
 
-		const entry = this.#actors.get(actorId);
-		invariant(entry, "actor entry does not exist");
-
-		await this.#performWrite(actorId, generation, state);
+		await this.#withActorWrite(actorId, async () => {
+			await this.#performWrite(actorId, generation, state);
+		});
 	}
 
 	isGenerationCurrentAndNotDestroyed(
@@ -558,6 +813,65 @@ export class FileSystemGlobalState {
 		);
 	}
 
+	async #waitForActorStop(actorId: string): Promise<void> {
+		while (true) {
+			const entry = this.#actors.get(actorId);
+			if (!entry?.stopPromise) {
+				return;
+			}
+			try {
+				await entry.stopPromise.promise;
+			} catch {
+				return;
+			}
+		}
+	}
+
+	async #withActorWrite<T>(
+		actorId: string,
+		fn: (entry: ActorEntry) => Promise<T>,
+	): Promise<T> {
+		const entry = this.#actors.get(actorId);
+		invariant(entry, "actor entry does not exist");
+
+		const previousWrite = entry.pendingWriteResolver;
+		const currentWrite = promiseWithResolvers<void>((reason) => logger().warn({ msg: "unhandled kv write promise rejection", reason }));
+		entry.pendingWriteResolver = currentWrite;
+
+		if (previousWrite) {
+			try {
+				await previousWrite.promise;
+			} catch {
+				// Ignore failed previous writes so later writes can proceed.
+			}
+		}
+
+		try {
+			return await fn(entry);
+		} finally {
+			currentWrite.resolve();
+			if (entry.pendingWriteResolver === currentWrite) {
+				entry.pendingWriteResolver = undefined;
+			}
+		}
+	}
+
+	async #waitForPendingWrite(actorId: string): Promise<void> {
+		const entry = this.#actors.get(actorId);
+		if (!entry?.pendingWriteResolver) {
+			return;
+		}
+
+		while (entry.pendingWriteResolver) {
+			const pending = entry.pendingWriteResolver;
+			try {
+				await pending.promise;
+			} catch {
+				// Ignore write failures to avoid blocking reads forever.
+			}
+		}
+	}
+
 	async setActorAlarm(actorId: string, timestamp: number) {
 		const entry = this.#actors.get(actorId);
 		invariant(entry, "actor entry does not exist");
@@ -725,8 +1039,10 @@ export class FileSystemGlobalState {
 		actorDriver: ActorDriver,
 		actorId: string,
 	): Promise<AnyActorInstance> {
+		await this.#waitForActorStop(actorId);
+
 		// Get the actor metadata
-		const entry = await this.loadActor(actorId);
+		let entry = await this.loadActor(actorId);
 		if (!entry.state) {
 			throw new Error(
 				`Actor does not exist and cannot be started: "${actorId}"`,
@@ -742,19 +1058,27 @@ export class FileSystemGlobalState {
 
 		// Actor already loaded
 		if (entry.actor) {
-			return entry.actor;
+			if (entry.actor.isStopping || this.isActorStopping(actorId)) {
+				await this.#waitForActorStop(actorId);
+				entry = await this.loadActor(actorId);
+				if (!entry.state) {
+					throw new Error(
+						`Actor does not exist and cannot be started: "${actorId}"`,
+					);
+				}
+			} else {
+				return entry.actor;
+			}
 		}
 
 		// Create start promise
-		entry.startPromise = promiseWithResolvers();
+		entry.startPromise = promiseWithResolvers((reason) => logger().warn({ msg: "unhandled actor start promise rejection", reason }));
 
 		try {
 			// Create actor
-			const definition = lookupInRegistry(
-				config,
-				entry.state.name,
-			);
-			entry.actor = definition.instantiate();
+			const definition = lookupInRegistry(config, entry.state.name);
+			entry.actor = await definition.instantiate();
+			entry.lifecycleState = ActorLifecycleState.AWAKE;
 
 			// Start actor
 			await entry.actor.start(
@@ -769,24 +1093,37 @@ export class FileSystemGlobalState {
 			// Update state with start timestamp
 			// NOTE: connectableTs is always in sync with startTs since actors become connectable immediately after starting
 			const now = BigInt(Date.now());
-			entry.state = {
-				...entry.state,
-				startTs: now,
-				connectableTs: now,
-				sleepTs: null, // Clear sleep timestamp when actor wakes up
-			};
-			await this.writeActor(actorId, entry.generation, entry.state);
+			await this.#withActorWrite(actorId, async (lockedEntry) => {
+				if (!lockedEntry.state) {
+					throw new Error(
+						`Actor does not exist and cannot be started: "${actorId}"`,
+					);
+				}
+				lockedEntry.state = {
+					...lockedEntry.state,
+					startTs: now,
+					connectableTs: now,
+					sleepTs: null, // Clear sleep timestamp when actor wakes up
+				};
+					if (this.#persist) {
+						await this.#performWrite(
+							actorId,
+							lockedEntry.generation,
+							lockedEntry.state,
+						);
+					}
+				});
 
 			// Finish
 			entry.startPromise.resolve();
 			entry.startPromise = undefined;
 
 			return entry.actor;
-		} catch (innerError) {
-			const error = new Error(
-				`Failed to start actor ${actorId}: ${innerError}`,
-				{ cause: innerError },
-			);
+			} catch (innerError) {
+				const error = new Error(
+					`Failed to start actor ${actorId}: ${innerError}`,
+					{ cause: innerError },
+				);
 			entry.startPromise?.reject(error);
 			entry.startPromise = undefined;
 			throw error;
@@ -983,48 +1320,20 @@ export class FileSystemGlobalState {
 		actorId: string,
 		entries: [Uint8Array, Uint8Array][],
 	): Promise<void> {
-		const entry = await this.loadActor(actorId);
-		if (!entry.state) {
-			if (this.isActorStopping(actorId)) {
-				return;
-			} else {
+		await this.loadActor(actorId);
+		await this.#withActorWrite(actorId, async (entry) => {
+			if (!entry.state) {
+				if (this.isActorStopping(actorId)) {
+					return;
+				}
 				throw new Error(`Actor ${actorId} state not loaded`);
 			}
-		}
-
-		// Create a mutable copy of kvStorage
-		const newKvStorage = [...entry.state.kvStorage];
-
-		// Update kvStorage with new entries
-		for (const [key, value] of entries) {
-			// Find existing entry with the same key
-			const existingIndex = newKvStorage.findIndex((e) =>
-				arrayBuffersEqual(e.key, bufferToArrayBuffer(key)),
-			);
-
-			if (existingIndex >= 0) {
-				// Replace existing entry with new one
-				newKvStorage[existingIndex] = {
-					key: bufferToArrayBuffer(key),
-					value: bufferToArrayBuffer(value),
-				};
-			} else {
-				// Add new entry
-				newKvStorage.push({
-					key: bufferToArrayBuffer(key),
-					value: bufferToArrayBuffer(value),
-				});
-			}
-		}
-
-		// Update state with new kvStorage
-		entry.state = {
-			...entry.state,
-			kvStorage: newKvStorage,
-		};
 
-		// Save state to disk
-		await this.writeActor(actorId, entry.generation, entry.state);
+			const db = this.#getOrCreateActorKvDatabase(actorId);
+			const totalSize = estimateKvSize(db);
+			validateKvEntries(entries, totalSize);
+			this.#putKvEntriesInDb(db, entries);
+		});
 	}
 
 	/**
@@ -1035,6 +1344,7 @@ export class FileSystemGlobalState {
 		keys: Uint8Array[],
 	): Promise<(Uint8Array | null)[]> {
 		const entry = await this.loadActor(actorId);
+		await this.#waitForPendingWrite(actorId);
 		if (!entry.state) {
 			if (this.isActorStopping(actorId)) {
 				throw new Error(`Actor ${actorId} is stopping`);
@@ -1043,18 +1353,20 @@ export class FileSystemGlobalState {
 			}
 		}
 
+		validateKvKeys(keys);
+
+		const db = this.#getOrCreateActorKvDatabase(actorId);
 		const results: (Uint8Array | null)[] = [];
 		for (const key of keys) {
-			// Find entry with the same key
-			const foundEntry = entry.state.kvStorage.find((e) =>
-				arrayBuffersEqual(e.key, bufferToArrayBuffer(key)),
+			const row = db.get<{ value: Uint8Array | ArrayBuffer }>(
+				"SELECT value FROM kv WHERE key = ?",
+				[key],
 			);
-
-			if (foundEntry) {
-				results.push(new Uint8Array(foundEntry.value));
-			} else {
+			if (!row) {
 				results.push(null);
+				continue;
 			}
+			results.push(ensureUint8Array(row.value, "value"));
 		}
 		return results;
 	}
@@ -1063,37 +1375,36 @@ export class FileSystemGlobalState {
 	 * Batch delete KV entries for an actor.
 	 */
 	async kvBatchDelete(actorId: string, keys: Uint8Array[]): Promise<void> {
-		const entry = await this.loadActor(actorId);
-		if (!entry.state) {
-			if (this.isActorStopping(actorId)) {
-				return;
-			} else {
+		await this.loadActor(actorId);
+		await this.#withActorWrite(actorId, async (entry) => {
+			if (!entry.state) {
+				if (this.isActorStopping(actorId)) {
+					return;
+				}
 				throw new Error(`Actor ${actorId} state not loaded`);
 			}
-		}
-
-		// Create a mutable copy of kvStorage
-		const newKvStorage = [...entry.state.kvStorage];
-
-		// Delete entries from kvStorage
-		for (const key of keys) {
-			const indexToDelete = newKvStorage.findIndex((e) =>
-				arrayBuffersEqual(e.key, bufferToArrayBuffer(key)),
-			);
 
-			if (indexToDelete >= 0) {
-				newKvStorage.splice(indexToDelete, 1);
+			if (keys.length === 0) {
+				return;
 			}
-		}
+			validateKvKeys(keys);
 
-		// Update state with new kvStorage
-		entry.state = {
-			...entry.state,
-			kvStorage: newKvStorage,
-		};
-
-		// Save state to disk
-		await this.writeActor(actorId, entry.generation, entry.state);
+			const db = this.#getOrCreateActorKvDatabase(actorId);
+			db.exec("BEGIN");
+			try {
+				for (const key of keys) {
+					db.run("DELETE FROM kv WHERE key = ?", [key]);
+				}
+				db.exec("COMMIT");
+			} catch (error) {
+				try {
+					db.exec("ROLLBACK");
+				} catch {
+					// Ignore rollback errors, original error is more actionable.
+				}
+				throw error;
+			}
+		});
 	}
 
 	/**
@@ -1104,6 +1415,7 @@ export class FileSystemGlobalState {
 		prefix: Uint8Array,
 	): Promise<[Uint8Array, Uint8Array][]> {
 		const entry = await this.loadActor(actorId);
+		await this.#waitForPendingWrite(actorId);
 		if (!entry.state) {
 			if (this.isActorStopping(actorId)) {
 				throw new Error(`Actor ${actorId} is destroying`);
@@ -1111,24 +1423,23 @@ export class FileSystemGlobalState {
 				throw new Error(`Actor ${actorId} state not loaded`);
 			}
 		}
+		validateKvKey(prefix, "prefix key");
+
+		const db = this.#getOrCreateActorKvDatabase(actorId);
+		const upperBound = computePrefixUpperBound(prefix);
+		const rows = upperBound
+			? db.all<{ key: Uint8Array | ArrayBuffer; value: Uint8Array | ArrayBuffer }>(
+					"SELECT key, value FROM kv WHERE key >= ? AND key < ? ORDER BY key ASC",
+					[prefix, upperBound],
+				)
+			: db.all<{ key: Uint8Array | ArrayBuffer; value: Uint8Array | ArrayBuffer }>(
+					"SELECT key, value FROM kv WHERE key >= ? ORDER BY key ASC",
+					[prefix],
+				);
 
-		const results: [Uint8Array, Uint8Array][] = [];
-		for (const kvEntry of entry.state.kvStorage) {
-			const keyBytes = new Uint8Array(kvEntry.key);
-			// Check if key starts with prefix
-			if (keyBytes.length >= prefix.length) {
-				let hasPrefix = true;
-				for (let i = 0; i < prefix.length; i++) {
-					if (keyBytes[i] !== prefix[i]) {
-						hasPrefix = false;
-						break;
-					}
-				}
-				if (hasPrefix) {
-					results.push([keyBytes, new Uint8Array(kvEntry.value)]);
-				}
-			}
-		}
-		return results;
+		return rows.map((row) => [
+			ensureUint8Array(row.key, "key"),
+			ensureUint8Array(row.value, "value"),
+		]);
 	}
 }