rivetkit 2.3.30 → 3.0.0-preview.2

package/src/client/actor-handle.ts

@@ -61,6 +61,7 @@ import {
 import { type ClientRaw, CREATE_ACTOR_CONN_PROXY } from "./client";
 import { ActorError, isSchedulingError } from "./errors";
 import { retryOnLifecycleBoundary } from "./lifecycle-errors";
+import { RetryBudget, retryAfterMs, retryDelayMs } from "./retry-budget";
 import { logger } from "./log";
 import { outboundTelemetryHeaders } from "./outbound-telemetry";
 import {
@@ -94,6 +95,7 @@ export class ActorHandleRaw {
 	#queueSendMutex = new AsyncMutex();
 	#currentActorInvocation?: CurrentActorInvocation;
 	#startCallSpan?: StartCallSpan;
+	#retryBudget: RetryBudget;
 
 	/**
 	 * Do not call this directly.
@@ -113,6 +115,7 @@ export class ActorHandleRaw {
 		signal?: AbortSignal,
 		currentActorInvocation?: CurrentActorInvocation,
 		startCallSpan?: StartCallSpan,
+		retryBudget: RetryBudget = new RetryBudget(),
 	) {
 		this.#client = client;
 		this.#driver = driver;
@@ -124,6 +127,7 @@ export class ActorHandleRaw {
 		this.#signal = signal;
 		this.#currentActorInvocation = currentActorInvocation;
 		this.#startCallSpan = startCallSpan;
+		this.#retryBudget = retryBudget;
 	}
 
 	async #resolveConnectionParams(): Promise<unknown> {
@@ -166,6 +170,7 @@ export class ActorHandleRaw {
 				this.#actorResolutionState,
 			);
 
+			this.#retryBudget.deposit();
 			for (let attempt = 0; attempt < maxAttempts; attempt++) {
 				let actorId: string | undefined;
 				try {
@@ -205,9 +210,19 @@ export class ActorHandleRaw {
 							metadata,
 							attempt,
 							maxAttempts,
-						)
+						) &&
+						this.#retryBudget.tryWithdraw()
 					) {
-						await this.#waitForRetryWindow();
+						await new Promise((resolve) =>
+							setTimeout(
+								resolve,
+								retryDelayMs(attempt, {
+									initialDelayMs: 25,
+									maxDelayMs: 1_000,
+									hintMs: retryAfterMs(err),
+								}),
+							),
+						);
 						continue;
 					}
 
@@ -309,7 +324,10 @@ export class ActorHandleRaw {
 			if (opts.name === "destroy") {
 				return await run();
 			}
-			return await retryOnLifecycleBoundary(run, { signal });
+			return await retryOnLifecycleBoundary(run, {
+				signal,
+				budget: this.#retryBudget,
+			});
 		};
 		if (!call) {
 			return await send();

package/src/client/client.ts

@@ -14,6 +14,7 @@ import {
 import { type ActorHandle, ActorHandleRaw } from "./actor-handle";
 import { logger } from "./log";
 import { resolveGatewayTarget } from "./resolve-gateway-target";
+import { type RetryBudgetOptions, RetryBudget } from "./retry-budget";
 
 export type { ClientConfig, ClientConfigInput } from "./config";
 
@@ -181,6 +182,8 @@ export const CREATE_ACTOR_CONN_PROXY = Symbol("createActorConnProxy");
 export interface ClientRawOptions {
 	encoding?: Encoding;
 	gateway?: ActorGatewayOptions;
+	/** Bounds retries of refused requests across every handle of this client. */
+	retryBudget?: RetryBudgetOptions;
 }
 
 /** Runtime-supplied wiring for clients created inside an actor. Set only by the native runtime. */
@@ -211,6 +214,7 @@ export class ClientRaw {
 	#gatewayOptions: ActorGatewayOptions;
 	#currentActorInvocation?: CurrentActorInvocation;
 	#startCallSpan?: StartCallSpan;
+	#retryBudget: RetryBudget;
 
 	/**
 	 * Creates an instance of Client.
@@ -228,6 +232,7 @@ export class ClientRaw {
 		this.#gatewayOptions = options.gateway ?? {};
 		this.#currentActorInvocation = options.currentActorInvocation;
 		this.#startCallSpan = options.startCallSpan;
+		this.#retryBudget = new RetryBudget(options.retryBudget);
 	}
 
 	/**
@@ -431,6 +436,7 @@ export class ClientRaw {
 			signal,
 			this.#currentActorInvocation,
 			this.#startCallSpan,
+			this.#retryBudget,
 		);
 	}
 

package/src/client/config.ts

@@ -88,6 +88,17 @@ export const ClientConfigSchemaBase = z.object({
 	// `proxyWebSocket`
 	getUpgradeWebSocket: z.custom<GetUpgradeWebSocket>().optional(),
 
+	/**
+	 * Bounds retries of refused requests: `percent` retries per 100 requests (default 10), with
+	 * up to `maxTokens` banked for bursts (default 100). A refusal past the budget is thrown.
+	 */
+	retryBudget: z
+		.object({
+			percent: z.number().min(0).max(100).optional(),
+			maxTokens: z.number().int().min(0).optional(),
+		})
+		.optional(),
+
 	/** Replaces global fetch for actor HTTP gateway requests. */
 	fetch: z.custom<typeof globalThis.fetch>().optional(),
 
@@ -166,7 +177,7 @@ export function convertRegistryConfigToClientConfig(
 		token: config.token,
 		getToken: undefined,
 		namespace: config.namespace,
-		poolName: config.envoy.poolName,
+		poolName: config.node.poolName,
 		headers: config.headers,
 		gateway: { skipReadyWait: false },
 		encoding: "bare",

package/src/client/lifecycle-errors.ts

@@ -1,4 +1,5 @@
 import { ActorError, HttpRequestError } from "./errors";
+import { type RetryBudget, retryAfterMs, retryDelayMs } from "./retry-budget";
 
 export interface LifecycleBoundaryInfo {
 	kind: "request_retry" | "reconnect_only";
@@ -271,6 +272,11 @@ async function waitWithSignal(ms: number, signal?: AbortSignal) {
 	});
 }
 
+/**
+ * Runs `run`, retrying retryable lifecycle and refusal errors with jittered exponential backoff
+ * that honors the service's `retry_after_ms` hint. With a `budget`, the call credits it once and
+ * every retry spends a token; a spent budget returns the error instead of retrying.
+ */
 export async function retryOnLifecycleBoundary<T>(
 	run: () => Promise<T>,
 	opts?: {
@@ -278,11 +284,13 @@ export async function retryOnLifecycleBoundary<T>(
 		initialDelayMs?: number;
 		maxDelayMs?: number;
 		signal?: AbortSignal;
+		budget?: RetryBudget;
 	},
 ): Promise<T> {
 	const maxAttempts = opts?.maxAttempts ?? 5;
 	const initialDelayMs = opts?.initialDelayMs ?? 25;
 	const maxDelayMs = opts?.maxDelayMs ?? 200;
+	opts?.budget?.deposit();
 
 	let lastError: unknown;
 	for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
@@ -299,8 +307,15 @@ export async function retryOnLifecycleBoundary<T>(
 			if (attempt === maxAttempts - 1) {
 				break;
 			}
+			if (opts?.budget && !opts.budget.tryWithdraw()) {
+				break;
+			}
 
-			const delayMs = Math.min(initialDelayMs * 2 ** attempt, maxDelayMs);
+			const delayMs = retryDelayMs(attempt, {
+				initialDelayMs,
+				maxDelayMs,
+				hintMs: retryAfterMs(error),
+			});
 			await waitWithSignal(delayMs, opts?.signal);
 		}
 	}

package/src/client/retry-budget.ts

@@ -0,0 +1,78 @@
+import { ActorError } from "./errors";
+
+/** How many retries a client may spend, as a share of the requests it sends. */
+export interface RetryBudgetOptions {
+	/** Retries allowed per 100 requests (default 10). */
+	percent?: number;
+	/** Retries the client can bank for a burst of refusals; the budget starts full (default 100). */
+	maxTokens?: number;
+}
+
+const DEFAULT_PERCENT = 10;
+const DEFAULT_MAX_TOKENS = 100;
+
+/**
+ * Bounds the client's retries of refused requests. Each request adds a fraction of a token and
+ * each retry spends one, so retries stay a fixed share of traffic while the service sheds load
+ * instead of multiplying it.
+ */
+export class RetryBudget {
+	#tokens: number;
+	readonly #maxTokens: number;
+	readonly #deposit: number;
+
+	constructor(options: RetryBudgetOptions = {}) {
+		const percent = options.percent ?? DEFAULT_PERCENT;
+		const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+		if (!(percent >= 0 && percent <= 100)) {
+			throw new Error("retryBudget.percent must be in 0..=100");
+		}
+		if (!(maxTokens >= 0)) {
+			throw new Error("retryBudget.maxTokens must be at least 0");
+		}
+		this.#maxTokens = maxTokens;
+		this.#tokens = maxTokens;
+		this.#deposit = percent / 100;
+	}
+
+	/** Credits one request sent. */
+	deposit(): void {
+		this.#tokens = Math.min(this.#maxTokens, this.#tokens + this.#deposit);
+	}
+
+	/** Spends one token. Returns `false` when the budget is spent. */
+	tryWithdraw(): boolean {
+		// Rounding guards against float drift: ten 0.1 deposits must make one token.
+		if (this.#tokens + 1e-9 < 1) {
+			return false;
+		}
+		this.#tokens = Math.max(0, this.#tokens - 1);
+		return true;
+	}
+}
+
+/** The service's retry hint for a refused request, in milliseconds, when it sent one. */
+export function retryAfterMs(error: unknown): number | undefined {
+	if (!(error instanceof ActorError)) {
+		return undefined;
+	}
+	const metadata = error.metadata;
+	if (metadata === null || typeof metadata !== "object") {
+		return undefined;
+	}
+	const hint = (metadata as { retry_after_ms?: unknown }).retry_after_ms;
+	return typeof hint === "number" && hint >= 0 ? hint : undefined;
+}
+
+/**
+ * The wait before retry `attempt` (0-based): jittered exponential backoff over
+ * `[initialDelayMs, maxDelayMs]`, never shorter than the service's hint.
+ */
+export function retryDelayMs(
+	attempt: number,
+	opts: { initialDelayMs: number; maxDelayMs: number; hintMs?: number },
+): number {
+	const backoff = Math.min(opts.initialDelayMs * 2 ** attempt, opts.maxDelayMs);
+	const jittered = backoff / 2 + Math.random() * (backoff / 2);
+	return Math.max(jittered, opts.hintMs ?? 0);
+}

package/src/client/utils.ts

@@ -19,21 +19,27 @@ export interface ParsedCloseReason {
 	group: string;
 	code: string;
 	rayId?: string;
+	/** The service's retry hint, from a `#retry_after_ms=<ms>` suffix. */
+	retryAfterMs?: number;
 }
 
+const RETRY_AFTER_SUFFIX = /^retry_after_ms=(\d+)$/;
+
 /**
  * Parses WebSocket close reason string into structured data.
  *
  * Expected format examples:
  *   - "guard.actor_runner_failed#t1s80so6h3irenp8ymzltfoittcl00"
  *   - "ws.client_closed"
+ *   - "guard.service_unavailable#retry_after_ms=1000", a refusal with its retry hint. The hint
+ *     rides after `#` so older clients still read the retryable `<group>.<code>`.
  *
  * Returns undefined if the format is invalid
  */
 export function parseWebSocketCloseReason(
 	reason: string,
 ): ParsedCloseReason | undefined {
-	const [mainPart, rayId] = reason.split("#");
+	const [mainPart, suffix] = reason.split("#");
 	const [group, code] = mainPart.split(".");
 
 	if (!group || !code) {
@@ -41,10 +47,15 @@ export function parseWebSocketCloseReason(
 		return undefined;
 	}
 
+	const retryAfter = suffix?.match(RETRY_AFTER_SUFFIX);
+	if (retryAfter) {
+		return { group, code, retryAfterMs: Number(retryAfter[1]) };
+	}
+
 	return {
 		group,
 		code,
-		rayId,
+		rayId: suffix,
 	};
 }
 

package/src/common/database/config.ts

@@ -52,7 +52,13 @@ export interface SqliteTransactionOptions {
  * to change without notice.
  */
 export interface SqliteProfilingOptions {
-	enabled?: boolean;
+	/** Exact SQL statements to give fingerprint labels. Profiling defaults to actor
+	 * aggregates without fingerprint labels. Profile logs remain enabled.
+	 * RIVETKIT_SQLITE_PROFILING overrides selection with a JSON object containing
+	 * queries and/or fingerprints arrays. An empty object clears fingerprint selection. */
+	queries?: string[];
+	/** Exact query fingerprints to label separately, as reported in profile logs. */
+	fingerprints?: string[];
 	maxTrackedStatementFingerprints?: number;
 	maxTrackedTransactionFingerprints?: number;
 	maxPrometheusSeries?: number;

package/src/common/database/native-database.ts

@@ -108,6 +108,11 @@ export type StateAwareSqliteDatabase = SqliteDatabase & {
 	): Promise<SqliteTransactionDatabase>;
 };
 
+type NativeSqliteDatabase = StateAwareSqliteDatabase & {
+	/** Stop JS calls while leaving the runtime-owned backend open. */
+	detachForShutdown(): Promise<void>;
+};
+
 export interface JsNativeTransactionLike {
 	exec(sql: string): Promise<NativeExecResult>;
 	execute(
@@ -311,7 +316,7 @@ class NativeCloseGate {
 		};
 	}
 
-	async close(callback: () => Promise<void>): Promise<void> {
+	async close(callback?: () => Promise<void>): Promise<void> {
 		if (this.#closed) {
 			return;
 		}
@@ -319,15 +324,17 @@ class NativeCloseGate {
 		if (this.#active > 0) {
 			await new Promise<void>((resolve) => this.#waiters.push(resolve));
 		}
-		await callback();
+		await callback?.();
 	}
 }
 
 export function wrapJsNativeDatabase(
 	database: JsNativeDatabaseLike,
-): StateAwareSqliteDatabase {
+): NativeSqliteDatabase {
 	const gate = new NativeCloseGate();
-	let closePromise: Promise<void> | undefined;
+	// Detach and explicit close are mutually exclusive terminal operations.
+	// Whichever starts first defines whether the backend is physically closed.
+	let terminalPromise: Promise<void> | undefined;
 	let lastInsertRowId: number | null = null;
 
 	const executeNative = async (
@@ -485,8 +492,12 @@ export function wrapJsNativeDatabase(
 			return normalizeNativeMetrics(database.metrics?.());
 		},
 		async close(): Promise<void> {
-			closePromise ??= gate.close(() => database.close());
-			await closePromise;
+			terminalPromise ??= gate.close(() => database.close());
+			await terminalPromise;
+		},
+		async detachForShutdown(): Promise<void> {
+			terminalPromise ??= gate.close();
+			await terminalPromise;
 		},
 	};
 }

package/src/common/router.ts

@@ -182,7 +182,7 @@ export function handleMetadataRequest(
 		version: VERSION,
 		envoy: {
 			kind: envoyKind,
-			version: config.envoy.version,
+			version: config.node.version,
 		},
 		envoyProtocolVersion: envoyProtocol.VERSION,
 		actorNames: buildActorNames(config),

package/src/engine-api/actors.ts

@@ -6,7 +6,7 @@ export const ActorSchema = z.object({
 	name: z.string(),
 	key: z.string(),
 	namespace_id: RivetIdSchema,
-	runner_name_selector: z.string(),
+	pool: z.string(),
 	create_ts: z.number(),
 	connectable_ts: z.number().nullable().optional(),
 	destroy_ts: z.number().nullable().optional(),
@@ -31,7 +31,7 @@ export type ActorsListResponse = z.infer<typeof ActorsListResponseSchema>;
 export const ActorsCreateRequestSchema = z.object({
 	datacenter: z.string().optional(),
 	name: z.string(),
-	runner_name_selector: z.string(),
+	pool: z.string(),
 	crash_policy: z.string(),
 	key: z.string().nullable().optional(),
 	input: z.string().nullable().optional(),
@@ -48,7 +48,7 @@ export const ActorsGetOrCreateRequestSchema = z.object({
 	datacenter: z.string().optional(),
 	name: z.string(),
 	key: z.string(),
-	runner_name_selector: z.string(),
+	pool: z.string(),
 	crash_policy: z.string(),
 	input: z.string().nullable().optional(),
 });

package/src/engine-client/actor-websocket-client.ts

@@ -1,3 +1,4 @@
+import { actorKeyToGatewayQuery } from "@/actor/keys";
 import {
 	type ClientConfig,
 	DEFAULT_MAX_QUERY_INPUT_SIZE,
@@ -289,8 +290,9 @@ export function buildActorQueryGatewayUrl(
 }
 
 function pushKeyQueryParams(params: URLSearchParams, key: string[]): void {
-	if (key.length > 0) {
-		params.append("rvt-key", key.join(","));
+	const rvtKey = actorKeyToGatewayQuery(key);
+	if (rvtKey !== undefined) {
+		params.append("rvt-key", rvtKey);
 	}
 }
 

package/src/engine-client/api-endpoints.ts

@@ -167,17 +167,6 @@ export interface RunnerConfigsResponse {
 	runner_configs: Record<string, RunnerConfigDatacenters>;
 }
 
-export async function getRunnerConfig(
-	config: ClientConfig,
-	name: string,
-): Promise<RunnerConfigsResponse> {
-	return apiCall<never, RunnerConfigsResponse>(
-		config,
-		"GET",
-		`/runner-configs?runner_name=${name}`,
-	);
-}
-
 // MARK: Update runner config
 export interface RegistryConfigRequest {
 	datacenters: Record<
@@ -212,11 +201,11 @@ export async function updateRunnerConfig(
 	runnerName: string,
 	request: RegistryConfigRequest,
 ): Promise<void> {
-	return apiCall<RegistryConfigRequest, void>(
+	return apiCall<unknown, void>(
 		config,
 		"PUT",
-		`/runner-configs/${runnerName}`,
-		request,
+		`/pools/${encodeURIComponent(runnerName)}`,
+		Object.values(request.datacenters)[0],
 	);
 }
 
@@ -237,3 +226,44 @@ export async function kvGet(
 		`/actors/${encodeURIComponent(actorId)}/kv/keys/${encodeURIComponent(key)}`,
 	);
 }
+
+/** Pool changes apply when each node next connects. */
+export interface PoolConfig {
+	drain_on_version_upgrade?: boolean;
+	actor_eviction_delay?: number;
+	actor_eviction_period?: number;
+	actor_eviction_rate?: number;
+	metadata?: unknown;
+}
+
+export function upsertPool(
+	config: ClientConfig,
+	name: string,
+	pool: PoolConfig,
+): Promise<{ pool: PoolConfig & { name: string } }> {
+	return apiCall(config, "PUT", `/pools/${encodeURIComponent(name)}`, pool);
+}
+
+export function getPool(
+	config: ClientConfig,
+	name: string,
+): Promise<{ pool: PoolConfig & { name: string } }> {
+	return apiCall(config, "GET", `/pools/${encodeURIComponent(name)}`);
+}
+
+export function listPools(
+	config: ClientConfig,
+	page?: { limit?: number; cursor?: string },
+): Promise<{
+	pools: Array<PoolConfig & { name: string }>;
+	pagination: { cursor: string | null };
+}> {
+	const query = new URLSearchParams();
+	if (page?.limit !== undefined) query.set("limit", String(page.limit));
+	if (page?.cursor !== undefined) query.set("cursor", page.cursor);
+	return apiCall(config, "GET", `/pools?${query}`);
+}
+
+export function deletePool(config: ClientConfig, name: string): Promise<void> {
+	return apiCall(config, "DELETE", `/pools/${encodeURIComponent(name)}`);
+}

package/src/engine-client/mod.ts

@@ -254,7 +254,7 @@ export class RemoteEngineControlClient implements EngineControlClient {
 					datacenter: region,
 					name,
 					key: serializeActorKey(key),
-					runner_name_selector: poolName ?? this.#config.poolName,
+					pool: poolName ?? this.#config.poolName,
 					input: actorInput
 						? uint8ArrayToBase64(
 								encodeCborCompat(actorInput as JsonCompatValue),
@@ -325,7 +325,7 @@ export class RemoteEngineControlClient implements EngineControlClient {
 			createActor(config, {
 				datacenter: region,
 				name,
-				runner_name_selector: poolName ?? this.#config.poolName,
+				pool: poolName ?? this.#config.poolName,
 				key: serializeActorKey(key),
 				input: input
 					? uint8ArrayToBase64(

package/src/registry/config/envoy.ts

@@ -11,7 +11,7 @@ export const EnvoyConfigSchema = z.object({
 
 	// Deprecated.
 	totalSlots: z.number().default(() => getRivetTotalSlots() ?? 100000),
-	envoyKey: z.string().optional(),
+	nodeKey: z.string().optional(),
 });
 export type EnvoyConfigInput = z.input<typeof EnvoyConfigSchema>;
 export type EnvoyConfig = z.infer<typeof EnvoyConfigSchema>;