rivetkit 2.0.22-rc.2 → 2.0.23

package/src/actor/instance.ts

@@ -19,6 +19,7 @@ import {
 	bufferToArrayBuffer,
 	EXTRA_ERROR_LOG,
 	getEnvUniversal,
+	idToStr,
 	promiseWithResolvers,
 	SinglePromiseQueue,
 } from "@/utils";
@@ -53,7 +54,7 @@ import type {
 import { processMessage } from "./protocol/old";
 import { CachedSerializer } from "./protocol/serde";
 import { Schedule } from "./schedule";
-import { DeadlineError, deadline } from "./utils";
+import { DeadlineError, deadline, isConnStatePath, isStatePath } from "./utils";
 
 export const PERSIST_SYMBOL = Symbol("persist");
 
@@ -244,7 +245,10 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 						lastSeen: conn.lastSeen,
 						stateEnabled: conn.__stateEnabled,
 						isHibernatable: conn.isHibernatable,
-						requestId: conn.__socket?.requestId,
+						hibernatableRequestId: conn.__persist
+							.hibernatableRequestId
+							? idToStr(conn.__persist.hibernatableRequestId)
+							: undefined,
 						driver: conn.__driverState
 							? getConnDriverKindFromState(conn.__driverState)
 							: undefined,
@@ -267,6 +271,7 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 				const conn = await this.createConn(
 					{
 						requestId: requestId,
+						hibernatable: false,
 						driverState: { [ConnDriverKind.HTTP]: {} },
 					},
 					undefined,
@@ -676,6 +681,10 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 
 			this.#onPersistSavedPromise?.resolve();
 		} catch (error) {
+			this.#rLog.error({
+				msg: "error saving persist",
+				error: stringifyError(error),
+			});
 			this.#onPersistSavedPromise?.reject(error);
 			throw error;
 		}
@@ -694,7 +703,6 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 		// Set raw persist object
 		this.#persistRaw = target;
 
-		// TODO: Only validate this for conn state
 		// TODO: Allow disabling in production
 		// If this can't be proxied, return raw value
 		if (target === null || typeof target !== "object") {
@@ -728,35 +736,46 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 				_previousValue: any,
 				_applyData: any,
 			) => {
-				if (path !== "state" && !path.startsWith("state.")) {
-					return;
+				const actorStatePath = isStatePath(path);
+				const connStatePath = isConnStatePath(path);
+
+				// Validate CBOR serializability for state changes
+				if (actorStatePath || connStatePath) {
+					let invalidPath = "";
+					if (
+						!isCborSerializable(
+							value,
+							(invalidPathPart) => {
+								invalidPath = invalidPathPart;
+							},
+							"",
+						)
+					) {
+						throw new errors.InvalidStateType({
+							path: path + (invalidPath ? `.${invalidPath}` : ""),
+						});
+					}
 				}
 
-				let invalidPath = "";
-				if (
-					!isCborSerializable(
-						value,
-						(invalidPathPart) => {
-							invalidPath = invalidPathPart;
-						},
-						"",
-					)
-				) {
-					throw new errors.InvalidStateType({
-						path: path + (invalidPath ? `.${invalidPath}` : ""),
-					});
-				}
+				this.#rLog.debug({
+					msg: "onChange triggered, setting persistChanged=true",
+					path,
+				});
 				this.#persistChanged = true;
 
-				// Inform the inspector about state changes
-				this.inspector.emitter.emit(
-					"stateUpdated",
-					this.#persist.state,
-				);
+				// Inform the inspector about state changes (only for state path)
+				if (actorStatePath) {
+					this.inspector.emitter.emit(
+						"stateUpdated",
+						this.#persist.state,
+					);
+				}
 
 				// Call onStateChange if it exists
+				//
 				// Skip if we're already inside onStateChange to prevent infinite recursion
 				if (
+					actorStatePath &&
 					this.#config.onStateChange &&
 					this.#ready &&
 					!this.#isInOnStateChange
@@ -802,6 +821,8 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 			this.#rLog.info({
 				msg: "actor restoring",
 				connections: persistData.connections.length,
+				hibernatableWebSockets:
+					persistData.hibernatableWebSocket.length,
 			});
 
 			// Set initial state
@@ -1000,6 +1021,74 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 	): Promise<Conn<S, CP, CS, V, I, DB>> {
 		this.#assertReady();
 
+		// Check for hibernatable websocket reconnection
+		if (socket.requestIdBuf && socket.hibernatable) {
+			this.rLog.debug({
+				msg: "checking for hibernatable websocket connection",
+				requestId: socket.requestId,
+				existingConnectionsCount: this.#connections.size,
+			});
+
+			// Find existing connection with matching hibernatableRequestId
+			const existingConn = Array.from(this.#connections.values()).find(
+				(conn) =>
+					conn.__persist.hibernatableRequestId &&
+					arrayBuffersEqual(
+						conn.__persist.hibernatableRequestId,
+						socket.requestIdBuf!,
+					),
+			);
+
+			if (existingConn) {
+				this.rLog.debug({
+					msg: "reconnecting hibernatable websocket connection",
+					connectionId: existingConn.id,
+					requestId: socket.requestId,
+				});
+
+				// If there's an existing driver state, clean it up without marking as clean disconnect
+				if (existingConn.__driverState) {
+					this.#rLog.warn({
+						msg: "found existing driver state on hibernatable websocket",
+						connectionId: existingConn.id,
+						requestId: socket.requestId,
+					});
+					const driverKind = getConnDriverKindFromState(
+						existingConn.__driverState,
+					);
+					const driver = CONN_DRIVERS[driverKind];
+					if (driver.disconnect) {
+						// Call driver disconnect to clean up directly. Don't use Conn.disconnect since that will remove the connection entirely.
+						driver.disconnect(
+							this,
+							existingConn,
+							(existingConn.__driverState as any)[driverKind],
+							"Reconnecting hibernatable websocket with new driver state",
+						);
+					}
+				}
+
+				// Update with new driver state
+				existingConn.__socket = socket;
+				existingConn.__persist.lastSeen = Date.now();
+
+				// Update sleep timer since connection is now active
+				this.#resetSleepTimer();
+
+				this.inspector.emitter.emit("connectionUpdated");
+
+				// We don't need to send a new init message since this is a
+				// hibernated request that has already been initialized
+
+				return existingConn;
+			} else {
+				this.rLog.debug({
+					msg: "no existing hibernatable connection found, creating new connection",
+					requestId: socket.requestId,
+				});
+			}
+		}
+
 		// If connection ID and token are provided, try to reconnect
 		if (connectionId && connectionToken) {
 			this.rLog.debug({
@@ -1058,14 +1147,12 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 				);
 
 				return existingConn;
+			} else {
+				this.rLog.debug({
+					msg: "connection not found or token mismatch, creating new connection",
+					connectionId,
+				});
 			}
-
-			// If we get here, either connection doesn't exist or token doesn't match
-			// Fall through to create new connection with new IDs
-			this.rLog.debug({
-				msg: "connection not found or token mismatch, creating new connection",
-				connectionId,
-			});
 		}
 
 		// Generate new connection ID and token if not provided or if reconnection failed
@@ -1131,6 +1218,19 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 			lastSeen: Date.now(),
 			subscriptions: [],
 		};
+
+		// Check if this connection is for a hibernatable websocket
+		if (socket.requestIdBuf) {
+			const isHibernatable =
+				this.#persist.hibernatableWebSocket.findIndex((ws) =>
+					arrayBuffersEqual(ws.requestId, socket.requestIdBuf!),
+				) !== -1;
+
+			if (isHibernatable) {
+				persist.hibernatableRequestId = socket.requestIdBuf;
+			}
+		}
+
 		const conn = new Conn<S, CP, CS, V, I, DB>(this, persist);
 		conn.__socket = socket;
 		this.#connections.set(conn.id, conn);
@@ -1864,6 +1964,13 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 	async saveState(opts: SaveStateOptions) {
 		this.#assertReady(opts.allowStoppingState);
 
+		this.#rLog.debug({
+			msg: "saveState called",
+			persistChanged: this.#persistChanged,
+			allowStoppingState: opts.allowStoppingState,
+			immediate: opts.immediate,
+		});
+
 		if (this.#persistChanged) {
 			if (opts.immediate) {
 				// Save immediately
@@ -1920,6 +2027,9 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 	#resetSleepTimer() {
 		if (this.#config.options.noSleep || !this.#sleepingSupported) return;
 
+		// Don't sleep if already stopping
+		if (this.#stopCalled) return;
+
 		const canSleep = this.#canSleep();
 
 		this.#rLog.debug({
@@ -1979,11 +2089,20 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 	 * 4. Engine runner will publish EventActorStateUpdate with ActorStateSTop
 	 **/
 	_startSleep() {
+		if (this.#stopCalled) {
+			this.#rLog.debug({
+				msg: "cannot call _startSleep if actor already stopping",
+			});
+			return;
+		}
+
 		// IMPORTANT: #sleepCalled should have no effect on the actor's
 		// behavior aside from preventing calling _startSleep twice. Wait for
 		// `_onStop` before putting in a stopping state.
 		if (this.#sleepCalled) {
-			this.#rLog.warn({ msg: "already sleeping actor" });
+			this.#rLog.warn({
+				msg: "cannot call _startSleep twice, actor already sleeping",
+			});
 			return;
 		}
 		this.#sleepCalled = true;
@@ -2054,12 +2173,20 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 			}
 		}
 
-		// Disconnect existing connections
 		const promises: Promise<unknown>[] = [];
+
+		// Disconnect existing non-hibernatable connections
 		for (const connection of this.#connections.values()) {
-			promises.push(connection.disconnect());
+			if (!connection.isHibernatable) {
+				this.#rLog.debug({
+					msg: "disconnecting non-hibernatable connection on actor stop",
+					connId: connection.id,
+				});
+				promises.push(connection.disconnect());
+			}
 
-			// TODO: Figure out how to abort HTTP requests on shutdown
+			// TODO: Figure out how to abort HTTP requests on shutdown. This
+			// might already be handled by the engine runner tunnel shutdown.
 		}
 
 		// Wait for any background tasks to finish, with timeout
@@ -2069,7 +2196,6 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 
 		// Clear timeouts
 		if (this.#pendingSaveTimeout) clearTimeout(this.#pendingSaveTimeout);
-		if (this.#sleepTimeout) clearTimeout(this.#sleepTimeout);
 		if (this.#checkConnLivenessInterval)
 			clearInterval(this.#checkConnLivenessInterval);
 
@@ -2149,6 +2275,7 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 					eventName: sub.eventName,
 				})),
 				lastSeen: BigInt(conn.lastSeen),
+				hibernatableRequestId: conn.hibernatableRequestId ?? null,
 			})),
 			scheduledEvents: persist.scheduledEvents.map((event) => ({
 				eventId: event.eventId,
@@ -2187,6 +2314,7 @@ export class ActorInstance<S, CP, CS, V, I, DB extends AnyDatabaseProvider> {
 					eventName: sub.eventName,
 				})),
 				lastSeen: Number(conn.lastSeen),
+				hibernatableRequestId: conn.hibernatableRequestId ?? undefined,
 			})),
 			scheduledEvents: bareData.scheduledEvents.map((event) => ({
 				eventId: event.eventId,

package/src/actor/persisted.ts

@@ -16,8 +16,11 @@ export interface PersistedConn<CP, CS> {
 	state: CS;
 	subscriptions: PersistedSubscription[];
 
-	/** Last time the socket was seen. This is set when disconencted so we can determine when we need to clean this up. */
+	/** Last time the socket was seen. This is set when disconnected so we can determine when we need to clean this up. */
 	lastSeen: number;
+
+	/** Request ID of the hibernatable WebSocket. See PersistedActor.hibernatableWebSocket */
+	hibernatableRequestId?: ArrayBuffer;
 }
 
 export interface PersistedSubscription {

package/src/actor/router-endpoints.ts

@@ -116,6 +116,7 @@ export async function handleWebSocketConnect(
 	encoding: Encoding,
 	parameters: unknown,
 	requestId: string,
+	requestIdBuf: ArrayBuffer | undefined,
 	connId: string | undefined,
 	connToken: string | undefined,
 ): Promise<UpgradeWebSocketArgs> {
@@ -184,9 +185,19 @@ export async function handleWebSocketConnect(
 						actorId,
 					});
 
+					// Check if this is a hibernatable websocket
+					const isHibernatable =
+						!!requestIdBuf &&
+						actor[PERSIST_SYMBOL].hibernatableWebSocket.findIndex(
+							(ws) =>
+								arrayBuffersEqual(ws.requestId, requestIdBuf),
+						) !== -1;
+
 					conn = await actor.createConn(
 						{
 							requestId: requestId,
+							requestIdBuf: requestIdBuf,
+							hibernatable: isHibernatable,
 							driverState: {
 								[ConnDriverKind.WEBSOCKET]: {
 									encoding,
@@ -365,6 +376,7 @@ export async function handleSseConnect(
 			conn = await actor.createConn(
 				{
 					requestId: requestId,
+					hibernatable: false,
 					driverState: {
 						[ConnDriverKind.SSE]: {
 							encoding,
@@ -479,6 +491,7 @@ export async function handleAction(
 		conn = await actor.createConn(
 			{
 				requestId: requestId,
+				hibernatable: false,
 				driverState: { [ConnDriverKind.HTTP]: {} },
 			},
 			parameters,
@@ -593,6 +606,7 @@ export async function handleRawWebSocketHandler(
 	path: string,
 	actorDriver: ActorDriver,
 	actorId: string,
+	requestIdBuf: ArrayBuffer | undefined,
 ): Promise<UpgradeWebSocketArgs> {
 	const actor = await actorDriver.loadActor(actorId);
 

package/src/actor/router.ts

@@ -187,6 +187,7 @@ export function createActorRouter(
 					encoding,
 					connParams,
 					generateConnRequestId(),
+					undefined,
 					connIdRaw,
 					connTokenRaw,
 				);
@@ -303,6 +304,7 @@ export function createActorRouter(
 					pathWithQuery,
 					actorDriver,
 					c.env.actorId,
+					undefined,
 				);
 			})(c, noopNext());
 		} else {

package/src/actor/utils.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, test } from "vitest";
+import { isConnStatePath, isStatePath } from "./utils";
+
+describe("isStatePath", () => {
+	test("matches exact state", () => {
+		expect(isStatePath("state")).toBe(true);
+	});
+
+	test("matches nested state paths", () => {
+		expect(isStatePath("state.foo")).toBe(true);
+		expect(isStatePath("state.foo.bar")).toBe(true);
+	});
+
+	test("does not match other paths", () => {
+		expect(isStatePath("connections")).toBe(false);
+		expect(isStatePath("stateX")).toBe(false);
+		expect(isStatePath("mystate")).toBe(false);
+	});
+});
+
+describe("isConnStatePath", () => {
+	test("matches connection state paths", () => {
+		expect(isConnStatePath("connections.0.state")).toBe(true);
+		expect(isConnStatePath("connections.123.state")).toBe(true);
+	});
+
+	test("matches nested connection state paths", () => {
+		expect(isConnStatePath("connections.0.state.foo")).toBe(true);
+		expect(isConnStatePath("connections.5.state.bar.baz")).toBe(true);
+	});
+
+	test("does not match non-state connection paths", () => {
+		expect(isConnStatePath("connections.0.params")).toBe(false);
+		expect(isConnStatePath("connections.0.token")).toBe(false);
+		expect(isConnStatePath("connections.0")).toBe(false);
+	});
+
+	test("does not match other paths", () => {
+		expect(isConnStatePath("state")).toBe(false);
+		expect(isConnStatePath("connections")).toBe(false);
+		expect(isConnStatePath("other.0.state")).toBe(false);
+	});
+
+	test("does not match malformed paths", () => {
+		expect(isConnStatePath("connections.state")).toBe(false);
+		expect(isConnStatePath("connections.0.stateX")).toBe(false);
+	});
+});

package/src/actor/utils.ts

@@ -104,3 +104,26 @@ export function generateRandomString(length = 32) {
 	}
 	return result;
 }
+
+/**
+ * Checks if a path is an actor state path within the persisted actor data.
+ */
+export function isStatePath(path: string): boolean {
+	return path === "state" || path.startsWith("state.");
+}
+
+/**
+ * Checks if a path is a connection state path within the persisted actor data.
+ */
+export function isConnStatePath(path: string): boolean {
+	if (!path.startsWith("connections.")) {
+		return false;
+	}
+	const stateIndex = path.indexOf(".state", 12); // Start after "connections."
+	if (stateIndex === -1) {
+		return false;
+	}
+	const afterState = stateIndex + 6; // ".state".length = 6
+	// Check if ".state" is followed by end of string or "."
+	return path.length === afterState || path[afterState] === ".";
+}

package/src/drivers/engine/actor-driver.ts

@@ -111,7 +111,6 @@ export class EngineActorDriver implements ActorDriver {
 		);
 
 		// Create runner configuration
-		let hasDisconnected = false;
 		const engineRunnerConfig: EngineRunnerConfig = {
 			version: this.#version,
 			endpoint: getEndpoint(runConfig),
@@ -125,32 +124,9 @@ export class EngineActorDriver implements ActorDriver {
 			},
 			prepopulateActorNames: buildActorNames(registryConfig),
 			onConnected: () => {
-				if (hasDisconnected) {
-					logger().info({
-						msg: "runner reconnected",
-						namespace: this.#runConfig.namespace,
-						runnerName: this.#runConfig.runnerName,
-					});
-				} else {
-					logger().debug({
-						msg: "runner connected",
-						namespace: this.#runConfig.namespace,
-						runnerName: this.#runConfig.runnerName,
-					});
-				}
-
 				this.#runnerStarted.resolve(undefined);
 			},
-			onDisconnected: (code, reason) => {
-				logger().warn({
-					msg: "runner disconnected",
-					namespace: this.#runConfig.namespace,
-					runnerName: this.#runConfig.runnerName,
-					code,
-					reason,
-				});
-				hasDisconnected = true;
-			},
+			onDisconnected: (_code, _reason) => {},
 			onShutdown: () => {
 				this.#runnerStopped.resolve(undefined);
 				this.#isRunnerStopped = true;
@@ -196,20 +172,34 @@ export class EngineActorDriver implements ActorDriver {
 				}
 
 				// Check for existing WS
-				const existingWs = handler.actor[
-					PERSIST_SYMBOL
-				].hibernatableWebSocket.find((ws) =>
+				const hibernatableArray =
+					handler.actor[PERSIST_SYMBOL].hibernatableWebSocket;
+				logger().debug({
+					msg: "checking hibernatable websockets",
+					requestId: idToStr(requestId),
+					existingHibernatableWebSockets: hibernatableArray.length,
+				});
+				const existingWs = hibernatableArray.find((ws) =>
 					arrayBuffersEqual(ws.requestId, requestId),
 				);
 
 				// Determine configuration for new WS
 				let hibernationConfig: HibernationConfig;
 				if (existingWs) {
+					logger().debug({
+						msg: "found existing hibernatable websocket",
+						requestId: idToStr(requestId),
+						lastMsgIndex: existingWs.msgIndex,
+					});
 					hibernationConfig = {
 						enabled: true,
 						lastMsgIndex: Number(existingWs.msgIndex),
 					};
 				} else {
+					logger().debug({
+						msg: "no existing hibernatable websocket found",
+						requestId: idToStr(requestId),
+					});
 					if (path === PATH_CONNECT_WEBSOCKET) {
 						hibernationConfig = {
 							enabled: true,
@@ -275,12 +265,24 @@ export class EngineActorDriver implements ActorDriver {
 					}
 				}
 
-				// Save hibernatable WebSocket
-				handler.actor[PERSIST_SYMBOL].hibernatableWebSocket.push({
-					requestId,
-					lastSeenTimestamp: BigInt(Date.now()),
-					msgIndex: -1n,
-				});
+				// Save or update hibernatable WebSocket
+				if (existingWs) {
+					logger().debug({
+						msg: "updated existing hibernatable websocket timestamp",
+						requestId: idToStr(requestId),
+					});
+					existingWs.lastSeenTimestamp = BigInt(Date.now());
+				} else {
+					logger().debug({
+						msg: "created new hibernatable websocket entry",
+						requestId: idToStr(requestId),
+					});
+					handler.actor[PERSIST_SYMBOL].hibernatableWebSocket.push({
+						requestId,
+						lastSeenTimestamp: BigInt(Date.now()),
+						msgIndex: -1n,
+					});
+				}
 
 				return hibernationConfig;
 			},
@@ -353,6 +355,12 @@ export class EngineActorDriver implements ActorDriver {
 
 		handler.persistedData = data;
 
+		logger().debug({
+			msg: "writing persisted data for actor",
+			actorId,
+			dataSize: data.byteLength,
+		});
+
 		await this.#runner.kvPut(actorId, [[KEYS.PERSIST_DATA, data]]);
 	}
 
@@ -427,6 +435,13 @@ export class EngineActorDriver implements ActorDriver {
 				persistedValue !== null
 					? persistedValue
 					: serializeEmptyPersistData(input);
+
+			logger().debug({
+				msg: "loaded persisted data for actor",
+				actorId,
+				dataSize: handler.persistedData?.byteLength,
+				wasInStorage: persistedValue !== null,
+			});
 		}
 
 		const name = actorConfig.name as string;
@@ -547,6 +562,7 @@ export class EngineActorDriver implements ActorDriver {
 				encoding,
 				connParams,
 				requestId,
+				requestIdBuf,
 				// Extract connId and connToken from protocols if needed
 				undefined,
 				undefined,
@@ -557,6 +573,7 @@ export class EngineActorDriver implements ActorDriver {
 				url.pathname + url.search,
 				this,
 				actorId,
+				requestIdBuf,
 			);
 		} else {
 			throw new Error(`Unreachable path: ${url.pathname}`);
@@ -626,7 +643,52 @@ export class EngineActorDriver implements ActorDriver {
 	}
 
 	async shutdownRunner(immediate: boolean): Promise<void> {
-		logger().info({ msg: "stopping engine actor driver" });
+		logger().info({ msg: "stopping engine actor driver", immediate });
+
+		// TODO: We need to update the runner to have a draining state so:
+		// 1. Send ToServerDraining
+		//		- This causes Pegboard to stop allocating actors to this runner
+		// 2. Pegboard sends ToClientStopActor for all actors on this runner which handles the graceful migration of each actor independently
+		// 3. Send ToServerStopping once all actors have successfully stopped
+		//
+		// What's happening right now is:
+		// 1. All actors enter stopped state
+		// 2. Actors still respond to requests because only RivetKit knows it's
+		//    stopping, this causes all requests to issue errors that the actor is
+		//    stopping. (This will NOT return a 503 bc the runner has no idea the
+		//    actors are stopping.)
+		// 3. Once the last actor stops, then the runner finally stops + actors
+		//    reschedule
+		//
+		// This means that:
+		// - All actors on this runner are bricked until the slowest _onStop finishes
+		// - Guard will not gracefully handle requests bc it's not receiving a 503
+		// - Actors can still be scheduled to this runner while the other
+		//   actors are stopping, meaning that those actors will NOT get _onStop
+		//   and will potentiall corrupt their state
+		//
+		// HACK: Stop all actors to allow state to be saved
+		// NOTE: _onStop is only supposed to be called by the runner, we're
+		// abusing it here
+		logger().debug({
+			msg: "stopping all actors before shutdown",
+			actorCount: this.#actors.size,
+		});
+		const stopPromises: Promise<void>[] = [];
+		for (const [_actorId, handler] of this.#actors.entries()) {
+			if (handler.actor) {
+				stopPromises.push(
+					handler.actor._onStop().catch((err) => {
+						handler.actor?.rLog.error({
+							msg: "_onStop errored",
+							error: stringifyError(err),
+						});
+					}),
+				);
+			}
+		}
+		await Promise.all(stopPromises);
+		logger().debug({ msg: "all actors stopped" });
 
 		// Clear the ack flush interval
 		if (this.#wsAckFlushInterval) {