rivetkit 2.3.3 → 2.3.5-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/client.d.ts +157 -9
- package/dist/browser/client.js +518 -542
- package/dist/browser/client.js.map +1 -1
- package/dist/browser/inspector/client.d.ts +101 -2
- package/dist/browser/inspector/client.js +1245 -227
- package/dist/browser/inspector/client.js.map +1 -1
- package/dist/tsup/agent-os/index.cjs +103 -36
- package/dist/tsup/agent-os/index.cjs.map +1 -1
- package/dist/tsup/agent-os/index.d.cts +159 -11
- package/dist/tsup/agent-os/index.d.ts +159 -11
- package/dist/tsup/agent-os/index.js +103 -36
- package/dist/tsup/agent-os/index.js.map +1 -1
- package/dist/tsup/{chunk-34P567PO.cjs → chunk-4XKED5P5.cjs} +145 -176
- package/dist/tsup/chunk-4XKED5P5.cjs.map +1 -0
- package/dist/tsup/{chunk-PYO2U53V.cjs → chunk-6HQCLWAW.cjs} +384 -4
- package/dist/tsup/chunk-6HQCLWAW.cjs.map +1 -0
- package/dist/tsup/{chunk-JZ7TWV65.js → chunk-7AFZMFPQ.js} +15 -1
- package/dist/tsup/chunk-7AFZMFPQ.js.map +1 -0
- package/dist/tsup/{chunk-BLDO2FKO.js → chunk-B4IZU537.js} +382 -2
- package/dist/tsup/chunk-B4IZU537.js.map +1 -0
- package/dist/tsup/{chunk-3MHDOUD7.cjs → chunk-G3CCRCTK.cjs} +14 -9
- package/dist/tsup/chunk-G3CCRCTK.cjs.map +1 -0
- package/dist/tsup/{chunk-CXRPENO3.js → chunk-GAV4PKFT.js} +15 -46
- package/dist/tsup/{chunk-CXRPENO3.js.map → chunk-GAV4PKFT.js.map} +1 -1
- package/dist/tsup/{chunk-HXUEHHJF.js → chunk-JI6GZ2C2.js} +2 -11
- package/dist/tsup/{chunk-HXUEHHJF.js.map → chunk-JI6GZ2C2.js.map} +1 -1
- package/dist/tsup/{chunk-YUEC5SEY.cjs → chunk-M3FCXL3Q.cjs} +3 -3
- package/dist/tsup/{chunk-YUEC5SEY.cjs.map → chunk-M3FCXL3Q.cjs.map} +1 -1
- package/dist/tsup/{chunk-KEKWJECX.js → chunk-MGTSVSUC.js} +2 -2
- package/dist/tsup/{chunk-VE2X4KMG.cjs → chunk-MYWZCDML.cjs} +2 -11
- package/dist/tsup/chunk-MYWZCDML.cjs.map +1 -0
- package/dist/tsup/{chunk-DFA7SYSW.js → chunk-NIC3QLKL.js} +3 -3
- package/dist/tsup/{chunk-BZM4PMFS.js → chunk-P4OGQ4JL.js} +5 -7
- package/dist/tsup/{chunk-BZM4PMFS.js.map → chunk-P4OGQ4JL.js.map} +1 -1
- package/dist/tsup/{chunk-NT2NK3VT.js → chunk-PAG6G46L.js} +2 -2
- package/dist/tsup/{chunk-XIX5DOZN.js → chunk-SHBE77IT.js} +14 -9
- package/dist/tsup/chunk-SHBE77IT.js.map +1 -0
- package/dist/tsup/{chunk-TYLP3U4Y.cjs → chunk-TNIWIUNV.cjs} +9 -11
- package/dist/tsup/chunk-TNIWIUNV.cjs.map +1 -0
- package/dist/tsup/{chunk-S644FL73.cjs → chunk-TR5NLR5V.cjs} +4 -4
- package/dist/tsup/{chunk-S644FL73.cjs.map → chunk-TR5NLR5V.cjs.map} +1 -1
- package/dist/tsup/{chunk-ZA7FLHKH.cjs → chunk-UEULFXUI.cjs} +16 -2
- package/dist/tsup/chunk-UEULFXUI.cjs.map +1 -0
- package/dist/tsup/{chunk-X63ROSOG.cjs → chunk-UJ2FQY63.cjs} +7 -7
- package/dist/tsup/{chunk-X63ROSOG.cjs.map → chunk-UJ2FQY63.cjs.map} +1 -1
- package/dist/tsup/client/mod.cjs +7 -8
- package/dist/tsup/client/mod.cjs.map +1 -1
- package/dist/tsup/client/mod.d.cts +5 -4
- package/dist/tsup/client/mod.d.ts +5 -4
- package/dist/tsup/client/mod.js +6 -7
- package/dist/tsup/common/log.cjs +2 -2
- package/dist/tsup/common/log.js +1 -1
- package/dist/tsup/common/websocket.cjs +3 -3
- package/dist/tsup/common/websocket.js +2 -2
- package/dist/tsup/{config-D49x8NpL.d.cts → config-BxOimoGX.d.cts} +16 -0
- package/dist/tsup/{config-D49x8NpL.d.ts → config-BxOimoGX.d.ts} +16 -0
- package/dist/tsup/{config-ZkIsRFsN.d.ts → config-BzdqbLAP.d.ts} +164 -27
- package/dist/tsup/{config-BfBhjEbk.d.cts → config-CPPEwn_e.d.cts} +164 -27
- package/dist/tsup/{context-sZ9GlaBK.d.cts → context-DSA9gFXB.d.cts} +2 -2
- package/dist/tsup/{context-B7-84u9K.d.ts → context-wWv-K2K_.d.ts} +2 -2
- package/dist/tsup/db/drizzle.cjs +120 -68
- package/dist/tsup/db/drizzle.cjs.map +1 -1
- package/dist/tsup/db/drizzle.d.cts +8 -3
- package/dist/tsup/db/drizzle.d.ts +8 -3
- package/dist/tsup/db/drizzle.js +120 -68
- package/dist/tsup/db/drizzle.js.map +1 -1
- package/dist/tsup/db/mod.cjs +74 -22
- package/dist/tsup/db/mod.cjs.map +1 -1
- package/dist/tsup/db/mod.d.cts +4 -3
- package/dist/tsup/db/mod.d.ts +4 -3
- package/dist/tsup/db/mod.js +74 -22
- package/dist/tsup/db/mod.js.map +1 -1
- package/dist/tsup/dynamic/mod.cjs +2 -2
- package/dist/tsup/dynamic/mod.d.cts +3 -3
- package/dist/tsup/dynamic/mod.d.ts +3 -3
- package/dist/tsup/dynamic/mod.js +1 -1
- package/dist/tsup/inspector/mod.cjs +4 -5
- package/dist/tsup/inspector/mod.cjs.map +1 -1
- package/dist/tsup/inspector/mod.js +3 -4
- package/dist/tsup/inspector/mod.js.map +1 -1
- package/dist/tsup/inspector-tab/mod.cjs +2 -2
- package/dist/tsup/inspector-tab/mod.cjs.map +1 -1
- package/dist/tsup/inspector-tab/mod.d.cts +5 -5
- package/dist/tsup/inspector-tab/mod.d.ts +5 -5
- package/dist/tsup/inspector-tab/mod.js +1 -1
- package/dist/tsup/inspector-tab/mod.js.map +1 -1
- package/dist/tsup/mod.cjs +605 -158
- package/dist/tsup/mod.cjs.map +1 -1
- package/dist/tsup/mod.d.cts +5 -5
- package/dist/tsup/mod.d.ts +5 -5
- package/dist/tsup/mod.js +528 -81
- package/dist/tsup/mod.js.map +1 -1
- package/dist/tsup/test/mod.cjs +10 -11
- package/dist/tsup/test/mod.cjs.map +1 -1
- package/dist/tsup/test/mod.d.cts +3 -3
- package/dist/tsup/test/mod.d.ts +3 -3
- package/dist/tsup/test/mod.js +6 -7
- package/dist/tsup/test/mod.js.map +1 -1
- package/dist/tsup/utils-CV0akuDD.d.ts +162 -0
- package/dist/tsup/utils-DQDCphrl.d.cts +162 -0
- package/dist/tsup/utils.cjs +6 -2
- package/dist/tsup/utils.cjs.map +1 -1
- package/dist/tsup/utils.d.cts +3 -146
- package/dist/tsup/utils.d.ts +3 -146
- package/dist/tsup/utils.js +5 -1
- package/dist/tsup/workflow/mod.cjs +198 -151
- package/dist/tsup/workflow/mod.cjs.map +1 -1
- package/dist/tsup/workflow/mod.d.cts +5 -5
- package/dist/tsup/workflow/mod.d.ts +5 -5
- package/dist/tsup/workflow/mod.js +187 -140
- package/dist/tsup/workflow/mod.js.map +1 -1
- package/package.json +8 -8
- package/src/actor/config.test.ts +14 -0
- package/src/actor/config.ts +104 -16
- package/src/actor/definition.ts +18 -7
- package/src/agent-os/actor/index.ts +2 -2
- package/src/common/bare/generated/actor-runtime-socket-protocol/v1.ts +848 -0
- package/src/common/bare/generated/inspector/v6.ts +1367 -0
- package/src/common/database/config.ts +24 -0
- package/src/common/database/mod.test.ts +148 -59
- package/src/common/database/mod.ts +93 -23
- package/src/common/database/native-database.test.ts +9 -0
- package/src/common/database/native-database.ts +151 -0
- package/src/common/database/shared.ts +15 -0
- package/src/common/encoding.ts +28 -21
- package/src/db/drizzle.test.ts +157 -0
- package/src/db/drizzle.ts +170 -79
- package/src/dynamic/isolate-runtime.ts +0 -2
- package/src/engine-client/driver.ts +0 -15
- package/src/engine-client/mod.ts +0 -41
- package/src/inspector/client.browser.ts +69 -8
- package/src/inspector-tab/mod.ts +1 -0
- package/src/registry/config/index.ts +1 -35
- package/src/registry/napi-runtime.ts +204 -16
- package/src/registry/native.ts +200 -30
- package/src/registry/runtime.ts +146 -3
- package/src/registry/wasm-runtime.ts +217 -6
- package/src/utils.ts +3 -0
- package/src/workflow/context.ts +5 -27
- package/src/workflow/driver.test.ts +62 -0
- package/src/workflow/driver.ts +231 -113
- package/src/workflow/mod.ts +3 -3
- package/dist/tsup/chunk-34P567PO.cjs.map +0 -1
- package/dist/tsup/chunk-3MHDOUD7.cjs.map +0 -1
- package/dist/tsup/chunk-APW73HCH.cjs +0 -382
- package/dist/tsup/chunk-APW73HCH.cjs.map +0 -1
- package/dist/tsup/chunk-BLDO2FKO.js.map +0 -1
- package/dist/tsup/chunk-DHXWD4DF.js +0 -382
- package/dist/tsup/chunk-DHXWD4DF.js.map +0 -1
- package/dist/tsup/chunk-JZ7TWV65.js.map +0 -1
- package/dist/tsup/chunk-PYO2U53V.cjs.map +0 -1
- package/dist/tsup/chunk-TYLP3U4Y.cjs.map +0 -1
- package/dist/tsup/chunk-VE2X4KMG.cjs.map +0 -1
- package/dist/tsup/chunk-XIX5DOZN.js.map +0 -1
- package/dist/tsup/chunk-ZA7FLHKH.cjs.map +0 -1
- package/src/actor/driver.ts +0 -141
- package/src/actor/instance/mod.ts +0 -2484
- package/src/drivers/engine/actor-driver.ts +0 -2473
- /package/dist/tsup/{chunk-KEKWJECX.js.map → chunk-MGTSVSUC.js.map} +0 -0
- /package/dist/tsup/{chunk-DFA7SYSW.js.map → chunk-NIC3QLKL.js.map} +0 -0
- /package/dist/tsup/{chunk-NT2NK3VT.js.map → chunk-PAG6G46L.js.map} +0 -0
|
@@ -1,2473 +0,0 @@
|
|
|
1
|
-
import type { EnvoyConfig } from "@rivetkit/rivetkit-native/wrapper";
|
|
2
|
-
import {
|
|
3
|
-
type EnvoyHandle,
|
|
4
|
-
type HibernatingWebSocketMetadata,
|
|
5
|
-
openDatabaseFromEnvoy,
|
|
6
|
-
type protocol,
|
|
7
|
-
startEnvoySync,
|
|
8
|
-
} from "@rivetkit/rivetkit-native/wrapper";
|
|
9
|
-
import * as cbor from "cbor-x";
|
|
10
|
-
import type { Context as HonoContext } from "hono";
|
|
11
|
-
import { streamSSE } from "hono/streaming";
|
|
12
|
-
import { WSContext, type WSContextInit } from "hono/ws";
|
|
13
|
-
import invariant from "invariant";
|
|
14
|
-
import { CONN_STATE_MANAGER_SYMBOL } from "@/actor/conn/mod";
|
|
15
|
-
import { isStaticActorDefinition, lookupInRegistry } from "@/actor/definition";
|
|
16
|
-
import type { ActorDriver } from "@/actor/driver";
|
|
17
|
-
import { KEYS } from "@/actor/instance/keys";
|
|
18
|
-
import type { AnyActorInstance } from "@/actor/instance/mod";
|
|
19
|
-
import {
|
|
20
|
-
type AnyStaticActorInstance,
|
|
21
|
-
isStaticActorInstance,
|
|
22
|
-
} from "@/actor/instance/mod";
|
|
23
|
-
import {
|
|
24
|
-
compareBytes,
|
|
25
|
-
createPreloadMap,
|
|
26
|
-
type PreloadMap,
|
|
27
|
-
} from "@/actor/instance/preload-map";
|
|
28
|
-
import { deserializeActorKey } from "@/actor/keys";
|
|
29
|
-
import type { Encoding } from "@/actor/protocol/serde";
|
|
30
|
-
import { type ActorRouter, createActorRouter } from "@/actor/router";
|
|
31
|
-
import {
|
|
32
|
-
parseWebSocketProtocols,
|
|
33
|
-
routeWebSocket,
|
|
34
|
-
truncateRawWebSocketPathPrefix,
|
|
35
|
-
type UpgradeWebSocketArgs,
|
|
36
|
-
} from "@/actor/router-websocket-endpoints";
|
|
37
|
-
import type { Client } from "@/client/client";
|
|
38
|
-
import {
|
|
39
|
-
PATH_CONNECT,
|
|
40
|
-
PATH_INSPECTOR_CONNECT,
|
|
41
|
-
PATH_WEBSOCKET_BASE,
|
|
42
|
-
PATH_WEBSOCKET_PREFIX,
|
|
43
|
-
} from "@/common/actor-router-consts";
|
|
44
|
-
import { getLogger } from "@/common/log";
|
|
45
|
-
import { deconstructError } from "@/common/utils";
|
|
46
|
-
import type {
|
|
47
|
-
RivetMessageEvent,
|
|
48
|
-
UniversalWebSocket,
|
|
49
|
-
} from "@/common/websocket-interface";
|
|
50
|
-
import {
|
|
51
|
-
buildHibernatableWebSocketAckStateTestResponse,
|
|
52
|
-
type IndexedWebSocketPayload,
|
|
53
|
-
parseHibernatableWebSocketAckStateTestRequest,
|
|
54
|
-
registerRemoteHibernatableWebSocketAckHooks,
|
|
55
|
-
setHibernatableWebSocketAckTestHooks,
|
|
56
|
-
unregisterRemoteHibernatableWebSocketAckHooks,
|
|
57
|
-
} from "@/common/websocket-test-hooks";
|
|
58
|
-
import {
|
|
59
|
-
type JsNativeDatabaseLike,
|
|
60
|
-
wrapJsNativeDatabase,
|
|
61
|
-
} from "@/db/native-database";
|
|
62
|
-
import {
|
|
63
|
-
type EngineControlClient,
|
|
64
|
-
getInitialActorKvState,
|
|
65
|
-
} from "@/driver-helpers/mod";
|
|
66
|
-
import { DynamicActorInstance } from "@/dynamic/instance";
|
|
67
|
-
import { isDynamicActorDefinition } from "@/dynamic/internal";
|
|
68
|
-
import { DynamicActorIsolateRuntime } from "@/dynamic/isolate-runtime";
|
|
69
|
-
import { getEndpoint } from "@/engine-client/api-utils";
|
|
70
|
-
import { buildActorNames, type RegistryConfig } from "@/registry/config";
|
|
71
|
-
import {
|
|
72
|
-
type LongTimeoutHandle,
|
|
73
|
-
promiseWithResolvers,
|
|
74
|
-
setLongTimeout,
|
|
75
|
-
stringifyError,
|
|
76
|
-
VERSION,
|
|
77
|
-
} from "@/utils";
|
|
78
|
-
import { logger } from "./log";
|
|
79
|
-
|
|
80
|
-
const ENVOY_SSE_PING_INTERVAL = 1000;
|
|
81
|
-
const FALLBACK_ENVOY_STOP_WAIT_MS = 15_000;
|
|
82
|
-
const INITIAL_SLEEP_TIMEOUT_MS = 250;
|
|
83
|
-
const REMOTE_ACK_HOOK_QUERY_PARAM = "__rivetkitAckHook";
|
|
84
|
-
|
|
85
|
-
// Message ack deadline is 30s on the gateway, but we will ack more frequently
|
|
86
|
-
// in order to minimize the message buffer size on the gateway and to give
|
|
87
|
-
// generous breathing room for the timeout.
|
|
88
|
-
//
|
|
89
|
-
// See engine/packages/pegboard-gateway/src/shared_state.rs
|
|
90
|
-
// (HWS_MESSAGE_ACK_TIMEOUT)
|
|
91
|
-
const _CONN_MESSAGE_ACK_DEADLINE = 5_000;
|
|
92
|
-
|
|
93
|
-
// Force saveState when cumulative message size reaches this threshold (0.5 MB)
|
|
94
|
-
//
|
|
95
|
-
// See engine/packages/pegboard-gateway/src/shared_state.rs
|
|
96
|
-
// (HWS_MAX_PENDING_MSGS_SIZE_PER_REQ)
|
|
97
|
-
const _CONN_BUFFERED_MESSAGE_SIZE_THRESHOLD = 500_000;
|
|
98
|
-
|
|
99
|
-
interface ActorHandler {
|
|
100
|
-
actor?: AnyActorInstance;
|
|
101
|
-
actorName?: string;
|
|
102
|
-
actorStartPromise?: ReturnType<typeof promiseWithResolvers<void>>;
|
|
103
|
-
actorStartError?: Error;
|
|
104
|
-
alarmTimeout?: LongTimeoutHandle;
|
|
105
|
-
alarmTimestamp?: number;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
interface HibernatableWebSocketAckState {
|
|
109
|
-
lastSentIndex: number;
|
|
110
|
-
lastAckedIndex: number;
|
|
111
|
-
pendingIndexes: number[];
|
|
112
|
-
ackWaiters: Map<number, Array<() => void>>;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
interface HibernatableConnectBinding {
|
|
116
|
-
actorId: string;
|
|
117
|
-
websocket: UniversalWebSocket;
|
|
118
|
-
request: Request;
|
|
119
|
-
requestPath: string;
|
|
120
|
-
requestHeaders: Record<string, string>;
|
|
121
|
-
encoding: Encoding;
|
|
122
|
-
connParams: unknown;
|
|
123
|
-
gatewayId: ArrayBuffer;
|
|
124
|
-
requestId: ArrayBuffer;
|
|
125
|
-
remoteAckHookToken?: string;
|
|
126
|
-
detach?: () => void;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
interface HibernatableRunnerWebSocketBinding {
|
|
130
|
-
actorId: string;
|
|
131
|
-
websocket: UniversalWebSocket;
|
|
132
|
-
requestPath: string;
|
|
133
|
-
requestHeaders: Record<string, string>;
|
|
134
|
-
encoding: Encoding;
|
|
135
|
-
connParams: unknown;
|
|
136
|
-
gatewayId: ArrayBuffer;
|
|
137
|
-
requestId: ArrayBuffer;
|
|
138
|
-
remoteAckHookToken?: string;
|
|
139
|
-
proxyToActorWs?: UniversalWebSocket;
|
|
140
|
-
detach?: () => void;
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
export type DriverContext = {};
|
|
144
|
-
|
|
145
|
-
export class EngineActorDriver implements ActorDriver {
|
|
146
|
-
#config: RegistryConfig;
|
|
147
|
-
#inlineClient: Client<any>;
|
|
148
|
-
#envoy: EnvoyHandle;
|
|
149
|
-
#actors: Map<string, ActorHandler> = new Map();
|
|
150
|
-
#dynamicRuntimes = new Map<string, DynamicActorIsolateRuntime>();
|
|
151
|
-
#hibernatableWebSocketAcks = new Map<
|
|
152
|
-
string,
|
|
153
|
-
HibernatableWebSocketAckState
|
|
154
|
-
>();
|
|
155
|
-
#hibernatableConnectBindings = new Map<
|
|
156
|
-
string,
|
|
157
|
-
HibernatableConnectBinding
|
|
158
|
-
>();
|
|
159
|
-
#hibernatableRunnerWebSocketBindings = new Map<
|
|
160
|
-
string,
|
|
161
|
-
HibernatableRunnerWebSocketBinding
|
|
162
|
-
>();
|
|
163
|
-
#actorRouter: ActorRouter;
|
|
164
|
-
|
|
165
|
-
#envoyStarted: PromiseWithResolvers<void> = promiseWithResolvers((reason) =>
|
|
166
|
-
logger().warn({
|
|
167
|
-
msg: "unhandled envoy started promise rejection",
|
|
168
|
-
reason,
|
|
169
|
-
}),
|
|
170
|
-
);
|
|
171
|
-
#envoyStopped: PromiseWithResolvers<void> = promiseWithResolvers((reason) =>
|
|
172
|
-
logger().warn({
|
|
173
|
-
msg: "unhandled envoy stopped promise rejection",
|
|
174
|
-
reason,
|
|
175
|
-
}),
|
|
176
|
-
);
|
|
177
|
-
#isEnvoyStopped: boolean = false;
|
|
178
|
-
#isShuttingDown: boolean = false;
|
|
179
|
-
|
|
180
|
-
// HACK: Track actor stop intent locally since the envoy protocol doesn't
|
|
181
|
-
// pass the stop reason to onActorStop. This will be fixed when the envoy
|
|
182
|
-
// protocol is updated to send the intent directly (see RVT-5284)
|
|
183
|
-
#actorStopIntent: Map<string, "sleep" | "destroy" | "crash"> = new Map();
|
|
184
|
-
|
|
185
|
-
constructor(
|
|
186
|
-
config: RegistryConfig,
|
|
187
|
-
engineClient: EngineControlClient,
|
|
188
|
-
inlineClient: Client<any>,
|
|
189
|
-
) {
|
|
190
|
-
this.#config = config;
|
|
191
|
-
this.#engineClient = engineClient;
|
|
192
|
-
this.#inlineClient = inlineClient;
|
|
193
|
-
|
|
194
|
-
// HACK: Override inspector token (which are likely to be
|
|
195
|
-
// removed later on) with token from x-rivet-token header
|
|
196
|
-
// TODO:
|
|
197
|
-
// if (token && runConfig.inspector && runConfig.inspector.enabled) {
|
|
198
|
-
// runConfig.inspector.token = () => token;
|
|
199
|
-
// }
|
|
200
|
-
|
|
201
|
-
this.#actorRouter = createActorRouter(
|
|
202
|
-
config,
|
|
203
|
-
this,
|
|
204
|
-
undefined,
|
|
205
|
-
config.test.enabled,
|
|
206
|
-
);
|
|
207
|
-
|
|
208
|
-
// Create configuration
|
|
209
|
-
const envoyConfig: EnvoyConfig = {
|
|
210
|
-
version: config.envoy.version,
|
|
211
|
-
endpoint: getEndpoint(config),
|
|
212
|
-
token: config.token,
|
|
213
|
-
namespace: config.namespace,
|
|
214
|
-
poolName: config.envoy.poolName,
|
|
215
|
-
notGlobal: true,
|
|
216
|
-
metadata: {
|
|
217
|
-
rivetkit: { version: VERSION },
|
|
218
|
-
},
|
|
219
|
-
prepopulateActorNames: buildActorNames(config),
|
|
220
|
-
onShutdown: () => {
|
|
221
|
-
this.#envoyStopped.resolve();
|
|
222
|
-
this.#isEnvoyStopped = true;
|
|
223
|
-
},
|
|
224
|
-
fetch: this.#envoyFetch.bind(this),
|
|
225
|
-
websocket: this.#envoyWebSocket.bind(this),
|
|
226
|
-
hibernatableWebSocket: {
|
|
227
|
-
canHibernate: this.#hwsCanHibernate.bind(this),
|
|
228
|
-
},
|
|
229
|
-
onActorStart: this.#envoyOnActorStart.bind(this),
|
|
230
|
-
onActorStop: this.#envoyOnActorStop.bind(this),
|
|
231
|
-
logger: getLogger("envoy-client"),
|
|
232
|
-
debugLatencyMs: process.env._RIVET_DEBUG_LATENCY_MS
|
|
233
|
-
? Number.parseInt(process.env._RIVET_DEBUG_LATENCY_MS, 10)
|
|
234
|
-
: undefined,
|
|
235
|
-
};
|
|
236
|
-
|
|
237
|
-
// Create and start envoy
|
|
238
|
-
const envoy = startEnvoySync(envoyConfig);
|
|
239
|
-
|
|
240
|
-
this.#envoy = envoy;
|
|
241
|
-
|
|
242
|
-
envoy.started().then(
|
|
243
|
-
() => {
|
|
244
|
-
this.#envoyStarted.resolve();
|
|
245
|
-
},
|
|
246
|
-
(error) => {
|
|
247
|
-
this.#envoyStarted.reject(error);
|
|
248
|
-
},
|
|
249
|
-
);
|
|
250
|
-
|
|
251
|
-
logger().debug({
|
|
252
|
-
msg: "envoy client started",
|
|
253
|
-
endpoint: config.endpoint,
|
|
254
|
-
namespace: config.namespace,
|
|
255
|
-
poolName: config.envoy.poolName,
|
|
256
|
-
});
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
async #discardCrashedActorState(actorId: string) {
|
|
260
|
-
const handler = this.#actors.get(actorId);
|
|
261
|
-
if (!handler) {
|
|
262
|
-
return;
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
if (handler.alarmTimeout) {
|
|
266
|
-
handler.alarmTimeout.abort();
|
|
267
|
-
handler.alarmTimeout = undefined;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
if (handler.actor && isStaticActorInstance(handler.actor)) {
|
|
271
|
-
try {
|
|
272
|
-
await handler.actor.debugForceCrash();
|
|
273
|
-
} catch (err) {
|
|
274
|
-
logger().debug({
|
|
275
|
-
msg: "actor crash cleanup errored",
|
|
276
|
-
actorId,
|
|
277
|
-
error: stringifyError(err),
|
|
278
|
-
});
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
this.#actors.delete(actorId);
|
|
283
|
-
this.#actorStopIntent.delete(actorId);
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
getExtraActorLogParams(): Record<string, string> {
|
|
287
|
-
return { envoyKey: this.#envoy.getEnvoyKey() ?? "-" };
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
#hibernatableWebSocketAckKey(
|
|
291
|
-
gatewayId: ArrayBuffer,
|
|
292
|
-
requestId: ArrayBuffer,
|
|
293
|
-
): string {
|
|
294
|
-
return `${Buffer.from(gatewayId).toString("hex")}:${Buffer.from(requestId).toString("hex")}`;
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
#ensureHibernatableWebSocketAckState(
|
|
298
|
-
gatewayId: ArrayBuffer,
|
|
299
|
-
requestId: ArrayBuffer,
|
|
300
|
-
): HibernatableWebSocketAckState {
|
|
301
|
-
const key = this.#hibernatableWebSocketAckKey(gatewayId, requestId);
|
|
302
|
-
let state = this.#hibernatableWebSocketAcks.get(key);
|
|
303
|
-
if (!state) {
|
|
304
|
-
state = {
|
|
305
|
-
lastSentIndex: 0,
|
|
306
|
-
lastAckedIndex: 0,
|
|
307
|
-
pendingIndexes: [],
|
|
308
|
-
ackWaiters: new Map(),
|
|
309
|
-
};
|
|
310
|
-
this.#hibernatableWebSocketAcks.set(key, state);
|
|
311
|
-
}
|
|
312
|
-
return state;
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
#deleteHibernatableWebSocketAckState(
|
|
316
|
-
gatewayId: ArrayBuffer,
|
|
317
|
-
requestId: ArrayBuffer,
|
|
318
|
-
): void {
|
|
319
|
-
this.#hibernatableWebSocketAcks.delete(
|
|
320
|
-
this.#hibernatableWebSocketAckKey(gatewayId, requestId),
|
|
321
|
-
);
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
#detachHibernatableConnectBinding(
|
|
325
|
-
gatewayId: ArrayBuffer,
|
|
326
|
-
requestId: ArrayBuffer,
|
|
327
|
-
): void {
|
|
328
|
-
const key = this.#hibernatableWebSocketAckKey(gatewayId, requestId);
|
|
329
|
-
const binding = this.#hibernatableConnectBindings.get(key);
|
|
330
|
-
if (!binding) {
|
|
331
|
-
return;
|
|
332
|
-
}
|
|
333
|
-
binding.detach?.();
|
|
334
|
-
binding.detach = undefined;
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
#deleteHibernatableConnectBinding(
|
|
338
|
-
gatewayId: ArrayBuffer,
|
|
339
|
-
requestId: ArrayBuffer,
|
|
340
|
-
): void {
|
|
341
|
-
const key = this.#hibernatableWebSocketAckKey(gatewayId, requestId);
|
|
342
|
-
const binding = this.#hibernatableConnectBindings.get(key);
|
|
343
|
-
binding?.detach?.();
|
|
344
|
-
this.#hibernatableConnectBindings.delete(key);
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
#cleanupFailedHibernatableConnectBinding(
|
|
348
|
-
binding: HibernatableConnectBinding,
|
|
349
|
-
): void {
|
|
350
|
-
this.#deleteHibernatableWebSocketAckState(
|
|
351
|
-
binding.gatewayId,
|
|
352
|
-
binding.requestId,
|
|
353
|
-
);
|
|
354
|
-
unregisterRemoteHibernatableWebSocketAckHooks(
|
|
355
|
-
binding.remoteAckHookToken,
|
|
356
|
-
this.#config.test.enabled,
|
|
357
|
-
);
|
|
358
|
-
this.#deleteHibernatableConnectBinding(
|
|
359
|
-
binding.gatewayId,
|
|
360
|
-
binding.requestId,
|
|
361
|
-
);
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
#detachHibernatableRunnerWebSocketBinding(
|
|
365
|
-
gatewayId: ArrayBuffer,
|
|
366
|
-
requestId: ArrayBuffer,
|
|
367
|
-
): void {
|
|
368
|
-
const key = this.#hibernatableWebSocketAckKey(gatewayId, requestId);
|
|
369
|
-
const binding = this.#hibernatableRunnerWebSocketBindings.get(key);
|
|
370
|
-
if (!binding) {
|
|
371
|
-
return;
|
|
372
|
-
}
|
|
373
|
-
binding.detach?.();
|
|
374
|
-
binding.detach = undefined;
|
|
375
|
-
// Close the runtime-side ws so it does not outlive the binding when
|
|
376
|
-
// rebinding overwrites `binding.proxyToActorWs` with a fresh socket.
|
|
377
|
-
const previous = binding.proxyToActorWs;
|
|
378
|
-
if (previous && previous.readyState !== previous.CLOSED) {
|
|
379
|
-
previous.close(1011, "dynamic.rebind");
|
|
380
|
-
}
|
|
381
|
-
binding.proxyToActorWs = undefined;
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
#deleteHibernatableRunnerWebSocketBinding(
|
|
385
|
-
gatewayId: ArrayBuffer,
|
|
386
|
-
requestId: ArrayBuffer,
|
|
387
|
-
): void {
|
|
388
|
-
const key = this.#hibernatableWebSocketAckKey(gatewayId, requestId);
|
|
389
|
-
const binding = this.#hibernatableRunnerWebSocketBindings.get(key);
|
|
390
|
-
binding?.detach?.();
|
|
391
|
-
this.#hibernatableRunnerWebSocketBindings.delete(key);
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
#cleanupFailedHibernatableRunnerWebSocketBinding(
|
|
395
|
-
binding: HibernatableRunnerWebSocketBinding,
|
|
396
|
-
): void {
|
|
397
|
-
const proxyToActorWs = binding.proxyToActorWs;
|
|
398
|
-
if (
|
|
399
|
-
proxyToActorWs &&
|
|
400
|
-
proxyToActorWs.readyState !== proxyToActorWs.CLOSED
|
|
401
|
-
) {
|
|
402
|
-
proxyToActorWs.close(1011, "dynamic.bind_failed");
|
|
403
|
-
}
|
|
404
|
-
this.#deleteHibernatableWebSocketAckState(
|
|
405
|
-
binding.gatewayId,
|
|
406
|
-
binding.requestId,
|
|
407
|
-
);
|
|
408
|
-
unregisterRemoteHibernatableWebSocketAckHooks(
|
|
409
|
-
binding.remoteAckHookToken,
|
|
410
|
-
this.#config.test.enabled,
|
|
411
|
-
);
|
|
412
|
-
this.#deleteHibernatableRunnerWebSocketBinding(
|
|
413
|
-
binding.gatewayId,
|
|
414
|
-
binding.requestId,
|
|
415
|
-
);
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
#recordInboundHibernatableWebSocketMessage(
|
|
419
|
-
gatewayId: ArrayBuffer,
|
|
420
|
-
requestId: ArrayBuffer,
|
|
421
|
-
rivetMessageIndex: number,
|
|
422
|
-
): void {
|
|
423
|
-
const state = this.#ensureHibernatableWebSocketAckState(
|
|
424
|
-
gatewayId,
|
|
425
|
-
requestId,
|
|
426
|
-
);
|
|
427
|
-
state.lastSentIndex = Math.max(state.lastSentIndex, rivetMessageIndex);
|
|
428
|
-
if (!state.pendingIndexes.includes(rivetMessageIndex)) {
|
|
429
|
-
state.pendingIndexes.push(rivetMessageIndex);
|
|
430
|
-
state.pendingIndexes.sort((a, b) => a - b);
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
#recordAckedHibernatableWebSocketMessage(
|
|
435
|
-
gatewayId: ArrayBuffer,
|
|
436
|
-
requestId: ArrayBuffer,
|
|
437
|
-
serverMessageIndex: number,
|
|
438
|
-
): void {
|
|
439
|
-
const state = this.#ensureHibernatableWebSocketAckState(
|
|
440
|
-
gatewayId,
|
|
441
|
-
requestId,
|
|
442
|
-
);
|
|
443
|
-
state.lastAckedIndex = Math.max(
|
|
444
|
-
state.lastAckedIndex,
|
|
445
|
-
serverMessageIndex,
|
|
446
|
-
);
|
|
447
|
-
state.pendingIndexes = state.pendingIndexes.filter(
|
|
448
|
-
(index) => index > serverMessageIndex,
|
|
449
|
-
);
|
|
450
|
-
for (const [index, waiters] of state.ackWaiters) {
|
|
451
|
-
if (index > serverMessageIndex) {
|
|
452
|
-
continue;
|
|
453
|
-
}
|
|
454
|
-
state.ackWaiters.delete(index);
|
|
455
|
-
for (const resolve of waiters) {
|
|
456
|
-
resolve();
|
|
457
|
-
}
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
#registerHibernatableWebSocketAckTestHooks(
|
|
462
|
-
websocket: UniversalWebSocket,
|
|
463
|
-
gatewayId: ArrayBuffer,
|
|
464
|
-
requestId: ArrayBuffer,
|
|
465
|
-
remoteHookToken?: string,
|
|
466
|
-
): void {
|
|
467
|
-
setHibernatableWebSocketAckTestHooks(
|
|
468
|
-
websocket,
|
|
469
|
-
{
|
|
470
|
-
getState: () => {
|
|
471
|
-
const state = this.#ensureHibernatableWebSocketAckState(
|
|
472
|
-
gatewayId,
|
|
473
|
-
requestId,
|
|
474
|
-
);
|
|
475
|
-
return {
|
|
476
|
-
lastSentIndex: state.lastSentIndex,
|
|
477
|
-
lastAckedIndex: state.lastAckedIndex,
|
|
478
|
-
pendingIndexes: [...state.pendingIndexes],
|
|
479
|
-
};
|
|
480
|
-
},
|
|
481
|
-
waitForAck: async (serverMessageIndex) => {
|
|
482
|
-
const state = this.#ensureHibernatableWebSocketAckState(
|
|
483
|
-
gatewayId,
|
|
484
|
-
requestId,
|
|
485
|
-
);
|
|
486
|
-
if (state.lastAckedIndex >= serverMessageIndex) {
|
|
487
|
-
return;
|
|
488
|
-
}
|
|
489
|
-
await new Promise<void>((resolve) => {
|
|
490
|
-
const waiters =
|
|
491
|
-
state.ackWaiters.get(serverMessageIndex) ?? [];
|
|
492
|
-
waiters.push(resolve);
|
|
493
|
-
state.ackWaiters.set(serverMessageIndex, waiters);
|
|
494
|
-
});
|
|
495
|
-
},
|
|
496
|
-
},
|
|
497
|
-
this.#config.test.enabled,
|
|
498
|
-
);
|
|
499
|
-
registerRemoteHibernatableWebSocketAckHooks(
|
|
500
|
-
remoteHookToken ?? "",
|
|
501
|
-
{
|
|
502
|
-
getState: () => {
|
|
503
|
-
const state = this.#ensureHibernatableWebSocketAckState(
|
|
504
|
-
gatewayId,
|
|
505
|
-
requestId,
|
|
506
|
-
);
|
|
507
|
-
return {
|
|
508
|
-
lastSentIndex: state.lastSentIndex,
|
|
509
|
-
lastAckedIndex: state.lastAckedIndex,
|
|
510
|
-
pendingIndexes: [...state.pendingIndexes],
|
|
511
|
-
};
|
|
512
|
-
},
|
|
513
|
-
waitForAck: async (serverMessageIndex) => {
|
|
514
|
-
const state = this.#ensureHibernatableWebSocketAckState(
|
|
515
|
-
gatewayId,
|
|
516
|
-
requestId,
|
|
517
|
-
);
|
|
518
|
-
if (state.lastAckedIndex >= serverMessageIndex) {
|
|
519
|
-
return;
|
|
520
|
-
}
|
|
521
|
-
await new Promise<void>((resolve) => {
|
|
522
|
-
const waiters =
|
|
523
|
-
state.ackWaiters.get(serverMessageIndex) ?? [];
|
|
524
|
-
waiters.push(resolve);
|
|
525
|
-
state.ackWaiters.set(serverMessageIndex, waiters);
|
|
526
|
-
});
|
|
527
|
-
},
|
|
528
|
-
},
|
|
529
|
-
this.#config.test.enabled && Boolean(remoteHookToken),
|
|
530
|
-
);
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
#maybeRespondToHibernatableAckStateProbe(
|
|
534
|
-
websocket: UniversalWebSocket,
|
|
535
|
-
data: IndexedWebSocketPayload,
|
|
536
|
-
gatewayId: ArrayBuffer,
|
|
537
|
-
requestId: ArrayBuffer,
|
|
538
|
-
): boolean {
|
|
539
|
-
if (
|
|
540
|
-
!parseHibernatableWebSocketAckStateTestRequest(
|
|
541
|
-
data,
|
|
542
|
-
this.#config.test.enabled,
|
|
543
|
-
)
|
|
544
|
-
) {
|
|
545
|
-
return false;
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
const state = this.#ensureHibernatableWebSocketAckState(
|
|
549
|
-
gatewayId,
|
|
550
|
-
requestId,
|
|
551
|
-
);
|
|
552
|
-
const response = buildHibernatableWebSocketAckStateTestResponse(
|
|
553
|
-
{
|
|
554
|
-
lastSentIndex: state.lastSentIndex,
|
|
555
|
-
lastAckedIndex: state.lastAckedIndex,
|
|
556
|
-
pendingIndexes: [...state.pendingIndexes],
|
|
557
|
-
},
|
|
558
|
-
this.#config.test.enabled,
|
|
559
|
-
);
|
|
560
|
-
invariant(response, "missing hibernatable websocket ack test response");
|
|
561
|
-
websocket.send(response);
|
|
562
|
-
return true;
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
async #loadActorHandler(actorId: string): Promise<ActorHandler> {
|
|
566
|
-
// Check if actor is already loaded
|
|
567
|
-
const handler = this.#actors.get(actorId);
|
|
568
|
-
if (!handler)
|
|
569
|
-
throw new Error(`Actor handler does not exist ${actorId}`);
|
|
570
|
-
if (handler.actorStartPromise) await handler.actorStartPromise.promise;
|
|
571
|
-
if (handler.actorStartError) throw handler.actorStartError;
|
|
572
|
-
if (!handler.actor) throw new Error("Actor should be loaded");
|
|
573
|
-
return handler;
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
getContext(_actorId: string): DriverContext {
|
|
577
|
-
return {};
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
cancelAlarm(actorId: string): void {
|
|
581
|
-
const handler = this.#actors.get(actorId);
|
|
582
|
-
if (handler?.alarmTimeout) {
|
|
583
|
-
handler.alarmTimeout.abort();
|
|
584
|
-
handler.alarmTimeout = undefined;
|
|
585
|
-
}
|
|
586
|
-
}
|
|
587
|
-
|
|
588
|
-
#isDynamicActor(actorId: string): boolean {
|
|
589
|
-
return this.#dynamicRuntimes.has(actorId);
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
#requireDynamicRuntime(actorId: string): DynamicActorIsolateRuntime {
|
|
593
|
-
const runtime = this.#dynamicRuntimes.get(actorId);
|
|
594
|
-
if (!runtime) {
|
|
595
|
-
throw new Error(
|
|
596
|
-
`dynamic runtime is not loaded for actor ${actorId}`,
|
|
597
|
-
);
|
|
598
|
-
}
|
|
599
|
-
return runtime;
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
async setAlarm(actor: AnyActorInstance, timestamp: number): Promise<void> {
|
|
603
|
-
const handler = this.#actors.get(actor.id);
|
|
604
|
-
if (!handler) {
|
|
605
|
-
logger().warn({
|
|
606
|
-
msg: "no handler for actor to set alarm",
|
|
607
|
-
});
|
|
608
|
-
|
|
609
|
-
return;
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
// Clear prev timeout
|
|
613
|
-
if (handler.alarmTimeout && handler.alarmTimestamp === timestamp) {
|
|
614
|
-
return;
|
|
615
|
-
}
|
|
616
|
-
|
|
617
|
-
if (handler.alarmTimeout) {
|
|
618
|
-
handler.alarmTimeout.abort();
|
|
619
|
-
handler.alarmTimeout = undefined;
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
// Set alarm
|
|
623
|
-
const delay = Math.max(0, timestamp - Date.now());
|
|
624
|
-
handler.alarmTimestamp = timestamp;
|
|
625
|
-
handler.alarmTimeout = setLongTimeout(() => {
|
|
626
|
-
void (async () => {
|
|
627
|
-
const currentHandler = this.#actors.get(actor.id);
|
|
628
|
-
if (!currentHandler) {
|
|
629
|
-
logger().debug({
|
|
630
|
-
msg: "alarm fired without loaded actor",
|
|
631
|
-
actorId: actor.id,
|
|
632
|
-
});
|
|
633
|
-
return;
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
if (currentHandler.actorStartPromise) {
|
|
637
|
-
try {
|
|
638
|
-
await currentHandler.actorStartPromise.promise;
|
|
639
|
-
} catch (error) {
|
|
640
|
-
logger().debug({
|
|
641
|
-
msg: "alarm skipped after actor failed to start",
|
|
642
|
-
actorId: actor.id,
|
|
643
|
-
error: stringifyError(error),
|
|
644
|
-
});
|
|
645
|
-
return;
|
|
646
|
-
}
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
const alarmActor = this.#actors.get(actor.id)?.actor;
|
|
650
|
-
if (!alarmActor || alarmActor.isStopping) {
|
|
651
|
-
logger().debug({
|
|
652
|
-
msg: "alarm fired without ready actor",
|
|
653
|
-
actorId: actor.id,
|
|
654
|
-
});
|
|
655
|
-
return;
|
|
656
|
-
}
|
|
657
|
-
|
|
658
|
-
await alarmActor.onAlarm();
|
|
659
|
-
})().catch((error) => {
|
|
660
|
-
logger().error({
|
|
661
|
-
msg: "actor alarm failed",
|
|
662
|
-
actorId: actor.id,
|
|
663
|
-
error: stringifyError(error),
|
|
664
|
-
});
|
|
665
|
-
});
|
|
666
|
-
handler.alarmTimeout = undefined;
|
|
667
|
-
handler.alarmTimestamp = undefined;
|
|
668
|
-
}, delay);
|
|
669
|
-
|
|
670
|
-
// TODO: This call may not be needed on ActorInstance.start, but it does help ensure that the local state is synced with the alarm state
|
|
671
|
-
// Set alarm on Rivet
|
|
672
|
-
//
|
|
673
|
-
// This does not call an "alarm" event like Durable Objects.
|
|
674
|
-
// Instead, it just wakes the actor on the alarm (if not
|
|
675
|
-
// already awake).
|
|
676
|
-
//
|
|
677
|
-
// onAlarm is automatically called on `ActorInstance.start` when waking
|
|
678
|
-
// again.
|
|
679
|
-
this.#envoy.setAlarm(actor.id, timestamp);
|
|
680
|
-
}
|
|
681
|
-
|
|
682
|
-
// Engine drivers expose the native SQLite provider directly.
|
|
683
|
-
|
|
684
|
-
getInitialSleepTimeoutMs(
|
|
685
|
-
_actor: AnyActorInstance,
|
|
686
|
-
defaultTimeoutMs: number,
|
|
687
|
-
): number {
|
|
688
|
-
return Math.max(defaultTimeoutMs, INITIAL_SLEEP_TIMEOUT_MS);
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
getNativeDatabaseProvider() {
|
|
692
|
-
const envoy = this.#envoy;
|
|
693
|
-
return {
|
|
694
|
-
open: async (actorId: string) => {
|
|
695
|
-
const database: JsNativeDatabaseLike =
|
|
696
|
-
await openDatabaseFromEnvoy(envoy, actorId);
|
|
697
|
-
return wrapJsNativeDatabase(database);
|
|
698
|
-
},
|
|
699
|
-
};
|
|
700
|
-
}
|
|
701
|
-
|
|
702
|
-
// MARK: - Batch KV operations
|
|
703
|
-
async kvBatchPut(
|
|
704
|
-
actorId: string,
|
|
705
|
-
entries: [Uint8Array, Uint8Array][],
|
|
706
|
-
): Promise<void> {
|
|
707
|
-
await this.#envoy.kvPut(actorId, entries);
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
async kvBatchGet(
|
|
711
|
-
actorId: string,
|
|
712
|
-
keys: Uint8Array[],
|
|
713
|
-
): Promise<(Uint8Array | null)[]> {
|
|
714
|
-
return await this.#envoy.kvGet(actorId, keys);
|
|
715
|
-
}
|
|
716
|
-
|
|
717
|
-
async kvBatchDelete(actorId: string, keys: Uint8Array[]): Promise<void> {
|
|
718
|
-
await this.#envoy.kvDelete(actorId, keys);
|
|
719
|
-
}
|
|
720
|
-
|
|
721
|
-
async kvDeleteRange(
|
|
722
|
-
actorId: string,
|
|
723
|
-
start: Uint8Array,
|
|
724
|
-
end: Uint8Array,
|
|
725
|
-
): Promise<void> {
|
|
726
|
-
await this.#envoy.kvDeleteRange(actorId, start, end);
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
async kvList(actorId: string): Promise<Uint8Array[]> {
|
|
730
|
-
const entries = await this.#envoy.kvListPrefix(
|
|
731
|
-
actorId,
|
|
732
|
-
new Uint8Array(),
|
|
733
|
-
);
|
|
734
|
-
const keys = entries.map(([key]) => key);
|
|
735
|
-
return keys;
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
async kvListPrefix(
|
|
739
|
-
actorId: string,
|
|
740
|
-
prefix: Uint8Array,
|
|
741
|
-
options?: {
|
|
742
|
-
reverse?: boolean;
|
|
743
|
-
limit?: number;
|
|
744
|
-
},
|
|
745
|
-
): Promise<[Uint8Array, Uint8Array][]> {
|
|
746
|
-
const result = await this.#envoy.kvListPrefix(actorId, prefix, options);
|
|
747
|
-
logger().info({
|
|
748
|
-
msg: "kvListPrefix called",
|
|
749
|
-
actorId,
|
|
750
|
-
prefixStr: new TextDecoder().decode(prefix),
|
|
751
|
-
entriesCount: result.length,
|
|
752
|
-
keys: result.map(([key]: [Uint8Array, ...unknown[]]) =>
|
|
753
|
-
new TextDecoder().decode(key),
|
|
754
|
-
),
|
|
755
|
-
});
|
|
756
|
-
return result;
|
|
757
|
-
}
|
|
758
|
-
|
|
759
|
-
async kvListRange(
|
|
760
|
-
actorId: string,
|
|
761
|
-
start: Uint8Array,
|
|
762
|
-
end: Uint8Array,
|
|
763
|
-
options?: {
|
|
764
|
-
reverse?: boolean;
|
|
765
|
-
limit?: number;
|
|
766
|
-
},
|
|
767
|
-
): Promise<[Uint8Array, Uint8Array][]> {
|
|
768
|
-
return await this.#envoy.kvListRange(
|
|
769
|
-
actorId,
|
|
770
|
-
start,
|
|
771
|
-
end,
|
|
772
|
-
true,
|
|
773
|
-
options,
|
|
774
|
-
);
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
ackHibernatableWebSocketMessage(
|
|
778
|
-
gatewayId: ArrayBuffer,
|
|
779
|
-
requestId: ArrayBuffer,
|
|
780
|
-
serverMessageIndex: number,
|
|
781
|
-
): void {
|
|
782
|
-
this.#recordAckedHibernatableWebSocketMessage(
|
|
783
|
-
gatewayId,
|
|
784
|
-
requestId,
|
|
785
|
-
serverMessageIndex,
|
|
786
|
-
);
|
|
787
|
-
this.#envoy.sendHibernatableWebSocketMessageAck(
|
|
788
|
-
gatewayId,
|
|
789
|
-
requestId,
|
|
790
|
-
serverMessageIndex,
|
|
791
|
-
);
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
// MARK: - Actor Lifecycle
|
|
795
|
-
async loadActor(actorId: string): Promise<AnyActorInstance> {
|
|
796
|
-
const handler = await this.#loadActorHandler(actorId);
|
|
797
|
-
if (!handler.actor) throw new Error(`Actor ${actorId} failed to load`);
|
|
798
|
-
return handler.actor;
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
startSleep(actorId: string) {
|
|
802
|
-
// HACK: Track intent for onActorStop (see RVT-5284)
|
|
803
|
-
this.#actorStopIntent.set(actorId, "sleep");
|
|
804
|
-
this.#envoy.sleepActor(actorId);
|
|
805
|
-
}
|
|
806
|
-
|
|
807
|
-
startDestroy(actorId: string) {
|
|
808
|
-
// HACK: Track intent for onActorStop (see RVT-5284)
|
|
809
|
-
this.#actorStopIntent.set(actorId, "destroy");
|
|
810
|
-
this.#envoy.destroyActor(actorId);
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
async hardCrashActor(actorId: string): Promise<void> {
|
|
814
|
-
const handler = this.#actors.get(actorId);
|
|
815
|
-
if (!handler) {
|
|
816
|
-
return;
|
|
817
|
-
}
|
|
818
|
-
|
|
819
|
-
if (handler.actorStartPromise) {
|
|
820
|
-
await handler.actorStartPromise.promise.catch(() => undefined);
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
logger().info({
|
|
824
|
-
msg: "simulating hard crash for actor",
|
|
825
|
-
actorId,
|
|
826
|
-
});
|
|
827
|
-
|
|
828
|
-
await this.#discardCrashedActorState(actorId);
|
|
829
|
-
this.#actorStopIntent.set(actorId, "crash");
|
|
830
|
-
this.#envoy.stopActor(actorId, undefined, "simulated hard crash");
|
|
831
|
-
}
|
|
832
|
-
|
|
833
|
-
async shutdown(immediate: boolean): Promise<void> {
|
|
834
|
-
if (this.#isShuttingDown) {
|
|
835
|
-
return;
|
|
836
|
-
}
|
|
837
|
-
this.#isShuttingDown = true;
|
|
838
|
-
const envoyStopWaitMs = this.#envoyStopWaitMs();
|
|
839
|
-
|
|
840
|
-
logger().info({ msg: "stopping engine actor driver", immediate });
|
|
841
|
-
if (!immediate) {
|
|
842
|
-
// Put actors through the normal sleep intent path before draining the
|
|
843
|
-
// runner. This ensures Pegboard marks the actor workflow as sleeping
|
|
844
|
-
// and preserves wakeability across runner handoff.
|
|
845
|
-
logger().debug({
|
|
846
|
-
msg: "sending sleep intent to actors before runner drain",
|
|
847
|
-
actorCount: this.#actors.size,
|
|
848
|
-
});
|
|
849
|
-
for (const actorId of this.#actors.keys()) {
|
|
850
|
-
this.startSleep(actorId);
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
const actorSleepDeadline = Date.now() + envoyStopWaitMs;
|
|
854
|
-
while (this.#actors.size > 0 && Date.now() < actorSleepDeadline) {
|
|
855
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
if (this.#actors.size > 0) {
|
|
859
|
-
logger().warn({
|
|
860
|
-
msg: "timed out waiting for actors to stop before envoy drain",
|
|
861
|
-
remainingActors: this.#actors.size,
|
|
862
|
-
waitMs: envoyStopWaitMs,
|
|
863
|
-
});
|
|
864
|
-
// Snapshot so concurrent removals from `stopActor` do not
|
|
865
|
-
// invalidate the iterator.
|
|
866
|
-
for (const actorId of Array.from(this.#actors.keys())) {
|
|
867
|
-
logger().warn({
|
|
868
|
-
msg: "force stopping actor during driver shutdown",
|
|
869
|
-
actorId,
|
|
870
|
-
});
|
|
871
|
-
this.#envoy.stopActor(
|
|
872
|
-
actorId,
|
|
873
|
-
undefined,
|
|
874
|
-
"driver shutdown after sleep timeout",
|
|
875
|
-
);
|
|
876
|
-
}
|
|
877
|
-
} else {
|
|
878
|
-
logger().debug({
|
|
879
|
-
msg: "all actors stopped before envoy drain",
|
|
880
|
-
});
|
|
881
|
-
}
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
try {
|
|
885
|
-
await this.#envoy.shutdown(immediate);
|
|
886
|
-
} catch (error) {
|
|
887
|
-
const message =
|
|
888
|
-
error instanceof Error ? error.message : String(error);
|
|
889
|
-
if (
|
|
890
|
-
message.includes("WebSocket connection closed during shutdown")
|
|
891
|
-
) {
|
|
892
|
-
logger().debug({
|
|
893
|
-
msg: "ignoring shutdown websocket close race",
|
|
894
|
-
error: message,
|
|
895
|
-
});
|
|
896
|
-
} else {
|
|
897
|
-
throw error;
|
|
898
|
-
}
|
|
899
|
-
}
|
|
900
|
-
|
|
901
|
-
const stopped = await Promise.race([
|
|
902
|
-
this.#envoyStopped.promise.then(() => true),
|
|
903
|
-
new Promise<false>((resolve) =>
|
|
904
|
-
setTimeout(() => resolve(false), envoyStopWaitMs),
|
|
905
|
-
),
|
|
906
|
-
]);
|
|
907
|
-
if (!stopped) {
|
|
908
|
-
logger().warn({
|
|
909
|
-
msg: "timed out waiting for envoy shutdown",
|
|
910
|
-
waitMs: envoyStopWaitMs,
|
|
911
|
-
});
|
|
912
|
-
}
|
|
913
|
-
|
|
914
|
-
await this.#disposeAllDynamicRuntimes("driver shutdown");
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
async waitForReady(): Promise<void> {
|
|
918
|
-
await this.#envoy.started();
|
|
919
|
-
}
|
|
920
|
-
|
|
921
|
-
#envoyStopWaitMs(): number {
|
|
922
|
-
const actorStopThreshold = Number(
|
|
923
|
-
this.#envoy.getProtocolMetadata()?.actorStopThreshold,
|
|
924
|
-
);
|
|
925
|
-
if (Number.isFinite(actorStopThreshold) && actorStopThreshold > 0) {
|
|
926
|
-
return actorStopThreshold;
|
|
927
|
-
}
|
|
928
|
-
return FALLBACK_ENVOY_STOP_WAIT_MS;
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
async #bindHibernatableConnectSocket(
|
|
932
|
-
binding: HibernatableConnectBinding,
|
|
933
|
-
isRestoringHibernatable: boolean,
|
|
934
|
-
): Promise<void> {
|
|
935
|
-
this.#detachHibernatableConnectBinding(
|
|
936
|
-
binding.gatewayId,
|
|
937
|
-
binding.requestId,
|
|
938
|
-
);
|
|
939
|
-
this.#hibernatableConnectBindings.set(
|
|
940
|
-
this.#hibernatableWebSocketAckKey(
|
|
941
|
-
binding.gatewayId,
|
|
942
|
-
binding.requestId,
|
|
943
|
-
),
|
|
944
|
-
binding,
|
|
945
|
-
);
|
|
946
|
-
|
|
947
|
-
if (this.#isDynamicActor(binding.actorId)) {
|
|
948
|
-
try {
|
|
949
|
-
await this.#bindDynamicHibernatableConnectSocket(
|
|
950
|
-
binding,
|
|
951
|
-
isRestoringHibernatable,
|
|
952
|
-
);
|
|
953
|
-
} catch (error) {
|
|
954
|
-
this.#cleanupFailedHibernatableConnectBinding(binding);
|
|
955
|
-
throw error;
|
|
956
|
-
}
|
|
957
|
-
return;
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
const wsHandler = await routeWebSocket(
|
|
961
|
-
binding.request,
|
|
962
|
-
binding.requestPath,
|
|
963
|
-
binding.requestHeaders,
|
|
964
|
-
this.#config,
|
|
965
|
-
this,
|
|
966
|
-
binding.actorId,
|
|
967
|
-
binding.encoding,
|
|
968
|
-
binding.connParams,
|
|
969
|
-
binding.gatewayId,
|
|
970
|
-
binding.requestId,
|
|
971
|
-
true,
|
|
972
|
-
isRestoringHibernatable,
|
|
973
|
-
).catch((error) => {
|
|
974
|
-
this.#cleanupFailedHibernatableConnectBinding(binding);
|
|
975
|
-
throw error;
|
|
976
|
-
});
|
|
977
|
-
|
|
978
|
-
(binding.websocket as WSContextInit).raw = binding.websocket;
|
|
979
|
-
const wsContext = new WSContext(binding.websocket);
|
|
980
|
-
|
|
981
|
-
const onOpen = (event: Event) => {
|
|
982
|
-
wsHandler.onOpen(event, wsContext);
|
|
983
|
-
};
|
|
984
|
-
const onMessage = (event: RivetMessageEvent) => {
|
|
985
|
-
if (
|
|
986
|
-
this.#maybeRespondToHibernatableAckStateProbe(
|
|
987
|
-
binding.websocket,
|
|
988
|
-
event.data,
|
|
989
|
-
binding.gatewayId,
|
|
990
|
-
binding.requestId,
|
|
991
|
-
)
|
|
992
|
-
) {
|
|
993
|
-
return;
|
|
994
|
-
}
|
|
995
|
-
|
|
996
|
-
wsHandler.onMessage(event, wsContext);
|
|
997
|
-
|
|
998
|
-
const actor = this.#actors.get(binding.actorId)?.actor;
|
|
999
|
-
if (!actor || !isStaticActorInstance(actor) || !wsHandler.conn) {
|
|
1000
|
-
return;
|
|
1001
|
-
}
|
|
1002
|
-
|
|
1003
|
-
const conn = actor.connectionManager.findHibernatableConn(
|
|
1004
|
-
binding.gatewayId,
|
|
1005
|
-
binding.requestId,
|
|
1006
|
-
);
|
|
1007
|
-
if (!conn) {
|
|
1008
|
-
return;
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
|
-
if (typeof event.rivetMessageIndex === "number") {
|
|
1012
|
-
this.#recordInboundHibernatableWebSocketMessage(
|
|
1013
|
-
binding.gatewayId,
|
|
1014
|
-
binding.requestId,
|
|
1015
|
-
event.rivetMessageIndex,
|
|
1016
|
-
);
|
|
1017
|
-
}
|
|
1018
|
-
actor.handleInboundHibernatableWebSocketMessage(
|
|
1019
|
-
conn,
|
|
1020
|
-
event.data,
|
|
1021
|
-
event.rivetMessageIndex,
|
|
1022
|
-
);
|
|
1023
|
-
};
|
|
1024
|
-
const onClose = (event: CloseEvent) => {
|
|
1025
|
-
wsHandler.onClose(event, wsContext);
|
|
1026
|
-
this.#deleteHibernatableWebSocketAckState(
|
|
1027
|
-
binding.gatewayId,
|
|
1028
|
-
binding.requestId,
|
|
1029
|
-
);
|
|
1030
|
-
unregisterRemoteHibernatableWebSocketAckHooks(
|
|
1031
|
-
binding.remoteAckHookToken,
|
|
1032
|
-
this.#config.test.enabled,
|
|
1033
|
-
);
|
|
1034
|
-
this.#deleteHibernatableConnectBinding(
|
|
1035
|
-
binding.gatewayId,
|
|
1036
|
-
binding.requestId,
|
|
1037
|
-
);
|
|
1038
|
-
};
|
|
1039
|
-
const onError = (event: Event) => {
|
|
1040
|
-
wsHandler.onError(event, wsContext);
|
|
1041
|
-
};
|
|
1042
|
-
|
|
1043
|
-
binding.websocket.addEventListener("message", onMessage);
|
|
1044
|
-
binding.websocket.addEventListener("close", onClose);
|
|
1045
|
-
binding.websocket.addEventListener("error", onError);
|
|
1046
|
-
if (isRestoringHibernatable) {
|
|
1047
|
-
wsHandler.onRestore?.(wsContext);
|
|
1048
|
-
} else {
|
|
1049
|
-
binding.websocket.addEventListener("open", onOpen);
|
|
1050
|
-
}
|
|
1051
|
-
|
|
1052
|
-
binding.detach = () => {
|
|
1053
|
-
binding.websocket.removeEventListener("message", onMessage);
|
|
1054
|
-
binding.websocket.removeEventListener("close", onClose);
|
|
1055
|
-
binding.websocket.removeEventListener("error", onError);
|
|
1056
|
-
if (!isRestoringHibernatable) {
|
|
1057
|
-
binding.websocket.removeEventListener("open", onOpen);
|
|
1058
|
-
}
|
|
1059
|
-
};
|
|
1060
|
-
}
|
|
1061
|
-
|
|
1062
|
-
async #bindDynamicHibernatableConnectSocket(
|
|
1063
|
-
binding: HibernatableConnectBinding,
|
|
1064
|
-
isRestoringHibernatable: boolean,
|
|
1065
|
-
): Promise<void> {
|
|
1066
|
-
const runtime = this.#requireDynamicRuntime(binding.actorId);
|
|
1067
|
-
const proxyToActorWs = await runtime.openWebSocket(
|
|
1068
|
-
binding.requestPath,
|
|
1069
|
-
binding.encoding,
|
|
1070
|
-
binding.connParams,
|
|
1071
|
-
{
|
|
1072
|
-
headers: binding.requestHeaders,
|
|
1073
|
-
gatewayId: binding.gatewayId,
|
|
1074
|
-
requestId: binding.requestId,
|
|
1075
|
-
isHibernatable: true,
|
|
1076
|
-
isRestoringHibernatable,
|
|
1077
|
-
},
|
|
1078
|
-
);
|
|
1079
|
-
|
|
1080
|
-
const onProxyMessage = (event: RivetMessageEvent) => {
|
|
1081
|
-
if (binding.websocket.readyState !== binding.websocket.OPEN) {
|
|
1082
|
-
return;
|
|
1083
|
-
}
|
|
1084
|
-
binding.websocket.send(event.data as any);
|
|
1085
|
-
};
|
|
1086
|
-
const onProxyClose = (event: CloseEvent) => {
|
|
1087
|
-
if (event.reason === "dynamic.runtime.disposed") {
|
|
1088
|
-
return;
|
|
1089
|
-
}
|
|
1090
|
-
if (binding.websocket.readyState !== binding.websocket.CLOSED) {
|
|
1091
|
-
binding.websocket.close(event.code, event.reason);
|
|
1092
|
-
}
|
|
1093
|
-
};
|
|
1094
|
-
const onProxyError = () => {
|
|
1095
|
-
if (binding.websocket.readyState !== binding.websocket.CLOSED) {
|
|
1096
|
-
binding.websocket.close(1011, "dynamic.websocket_error");
|
|
1097
|
-
}
|
|
1098
|
-
};
|
|
1099
|
-
const onMessage = (event: RivetMessageEvent) => {
|
|
1100
|
-
if (
|
|
1101
|
-
this.#maybeRespondToHibernatableAckStateProbe(
|
|
1102
|
-
binding.websocket,
|
|
1103
|
-
event.data,
|
|
1104
|
-
binding.gatewayId,
|
|
1105
|
-
binding.requestId,
|
|
1106
|
-
)
|
|
1107
|
-
) {
|
|
1108
|
-
return;
|
|
1109
|
-
}
|
|
1110
|
-
|
|
1111
|
-
if (typeof event.rivetMessageIndex === "number") {
|
|
1112
|
-
this.#recordInboundHibernatableWebSocketMessage(
|
|
1113
|
-
binding.gatewayId,
|
|
1114
|
-
binding.requestId,
|
|
1115
|
-
event.rivetMessageIndex,
|
|
1116
|
-
);
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
void runtime
|
|
1120
|
-
.forwardIncomingWebSocketMessage(
|
|
1121
|
-
proxyToActorWs,
|
|
1122
|
-
event.data as any,
|
|
1123
|
-
event.rivetMessageIndex,
|
|
1124
|
-
)
|
|
1125
|
-
.catch((error) => {
|
|
1126
|
-
logger().error({
|
|
1127
|
-
msg: "failed forwarding websocket message to dynamic actor",
|
|
1128
|
-
actorId: binding.actorId,
|
|
1129
|
-
error: stringifyError(error),
|
|
1130
|
-
});
|
|
1131
|
-
binding.websocket.close(
|
|
1132
|
-
1011,
|
|
1133
|
-
"dynamic.websocket_forward_failed",
|
|
1134
|
-
);
|
|
1135
|
-
});
|
|
1136
|
-
};
|
|
1137
|
-
const onClose = (event: CloseEvent) => {
|
|
1138
|
-
if (proxyToActorWs.readyState !== proxyToActorWs.CLOSED) {
|
|
1139
|
-
proxyToActorWs.close(event.code, event.reason);
|
|
1140
|
-
}
|
|
1141
|
-
this.#deleteHibernatableWebSocketAckState(
|
|
1142
|
-
binding.gatewayId,
|
|
1143
|
-
binding.requestId,
|
|
1144
|
-
);
|
|
1145
|
-
unregisterRemoteHibernatableWebSocketAckHooks(
|
|
1146
|
-
binding.remoteAckHookToken,
|
|
1147
|
-
this.#config.test.enabled,
|
|
1148
|
-
);
|
|
1149
|
-
this.#deleteHibernatableConnectBinding(
|
|
1150
|
-
binding.gatewayId,
|
|
1151
|
-
binding.requestId,
|
|
1152
|
-
);
|
|
1153
|
-
};
|
|
1154
|
-
const onError = () => {
|
|
1155
|
-
if (proxyToActorWs.readyState !== proxyToActorWs.CLOSED) {
|
|
1156
|
-
proxyToActorWs.close(1011, "dynamic.gateway_error");
|
|
1157
|
-
}
|
|
1158
|
-
};
|
|
1159
|
-
|
|
1160
|
-
proxyToActorWs.addEventListener("message", onProxyMessage);
|
|
1161
|
-
proxyToActorWs.addEventListener("close", onProxyClose);
|
|
1162
|
-
proxyToActorWs.addEventListener("error", onProxyError);
|
|
1163
|
-
binding.websocket.addEventListener("message", onMessage);
|
|
1164
|
-
binding.websocket.addEventListener("close", onClose);
|
|
1165
|
-
binding.websocket.addEventListener("error", onError);
|
|
1166
|
-
|
|
1167
|
-
binding.detach = () => {
|
|
1168
|
-
proxyToActorWs.removeEventListener("message", onProxyMessage);
|
|
1169
|
-
proxyToActorWs.removeEventListener("close", onProxyClose);
|
|
1170
|
-
proxyToActorWs.removeEventListener("error", onProxyError);
|
|
1171
|
-
binding.websocket.removeEventListener("message", onMessage);
|
|
1172
|
-
binding.websocket.removeEventListener("close", onClose);
|
|
1173
|
-
binding.websocket.removeEventListener("error", onError);
|
|
1174
|
-
if (proxyToActorWs.readyState !== proxyToActorWs.CLOSED) {
|
|
1175
|
-
proxyToActorWs.close(1011, "dynamic.rebind");
|
|
1176
|
-
}
|
|
1177
|
-
};
|
|
1178
|
-
}
|
|
1179
|
-
|
|
1180
|
-
async #bindDynamicHibernatableRunnerWebSocket(
|
|
1181
|
-
binding: HibernatableRunnerWebSocketBinding,
|
|
1182
|
-
isRestoringHibernatable: boolean,
|
|
1183
|
-
): Promise<void> {
|
|
1184
|
-
this.#detachHibernatableRunnerWebSocketBinding(
|
|
1185
|
-
binding.gatewayId,
|
|
1186
|
-
binding.requestId,
|
|
1187
|
-
);
|
|
1188
|
-
this.#hibernatableRunnerWebSocketBindings.set(
|
|
1189
|
-
this.#hibernatableWebSocketAckKey(
|
|
1190
|
-
binding.gatewayId,
|
|
1191
|
-
binding.requestId,
|
|
1192
|
-
),
|
|
1193
|
-
binding,
|
|
1194
|
-
);
|
|
1195
|
-
|
|
1196
|
-
let runtime: DynamicActorIsolateRuntime;
|
|
1197
|
-
let proxyToActorWs: UniversalWebSocket;
|
|
1198
|
-
try {
|
|
1199
|
-
runtime = this.#requireDynamicRuntime(binding.actorId);
|
|
1200
|
-
proxyToActorWs = await runtime.openWebSocket(
|
|
1201
|
-
binding.requestPath,
|
|
1202
|
-
binding.encoding,
|
|
1203
|
-
binding.connParams,
|
|
1204
|
-
{
|
|
1205
|
-
headers: binding.requestHeaders,
|
|
1206
|
-
gatewayId: binding.gatewayId,
|
|
1207
|
-
requestId: binding.requestId,
|
|
1208
|
-
isHibernatable: true,
|
|
1209
|
-
isRestoringHibernatable,
|
|
1210
|
-
},
|
|
1211
|
-
);
|
|
1212
|
-
} catch (error) {
|
|
1213
|
-
this.#cleanupFailedHibernatableRunnerWebSocketBinding(binding);
|
|
1214
|
-
throw error;
|
|
1215
|
-
}
|
|
1216
|
-
binding.proxyToActorWs = proxyToActorWs;
|
|
1217
|
-
|
|
1218
|
-
const onProxyMessage = (event: RivetMessageEvent) => {
|
|
1219
|
-
if (binding.websocket.readyState !== binding.websocket.OPEN) {
|
|
1220
|
-
return;
|
|
1221
|
-
}
|
|
1222
|
-
binding.websocket.send(event.data as any);
|
|
1223
|
-
};
|
|
1224
|
-
const onProxyClose = (event: CloseEvent) => {
|
|
1225
|
-
if (event.reason === "dynamic.runtime.disposed") {
|
|
1226
|
-
return;
|
|
1227
|
-
}
|
|
1228
|
-
if (binding.websocket.readyState !== binding.websocket.CLOSED) {
|
|
1229
|
-
binding.websocket.close(event.code, event.reason);
|
|
1230
|
-
}
|
|
1231
|
-
};
|
|
1232
|
-
const onProxyError = () => {
|
|
1233
|
-
if (binding.websocket.readyState !== binding.websocket.CLOSED) {
|
|
1234
|
-
binding.websocket.close(1011, "dynamic.websocket_error");
|
|
1235
|
-
}
|
|
1236
|
-
};
|
|
1237
|
-
const onMessage = (event: RivetMessageEvent) => {
|
|
1238
|
-
if (
|
|
1239
|
-
this.#maybeRespondToHibernatableAckStateProbe(
|
|
1240
|
-
binding.websocket,
|
|
1241
|
-
event.data,
|
|
1242
|
-
binding.gatewayId,
|
|
1243
|
-
binding.requestId,
|
|
1244
|
-
)
|
|
1245
|
-
) {
|
|
1246
|
-
return;
|
|
1247
|
-
}
|
|
1248
|
-
|
|
1249
|
-
if (typeof event.rivetMessageIndex === "number") {
|
|
1250
|
-
this.#recordInboundHibernatableWebSocketMessage(
|
|
1251
|
-
binding.gatewayId,
|
|
1252
|
-
binding.requestId,
|
|
1253
|
-
event.rivetMessageIndex,
|
|
1254
|
-
);
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
const currentRuntime = this.#dynamicRuntimes.get(binding.actorId);
|
|
1258
|
-
const currentProxyToActorWs = binding.proxyToActorWs;
|
|
1259
|
-
if (!currentRuntime || !currentProxyToActorWs) {
|
|
1260
|
-
logger().error({
|
|
1261
|
-
msg: "dynamic runtime websocket binding is missing after restore",
|
|
1262
|
-
actorId: binding.actorId,
|
|
1263
|
-
});
|
|
1264
|
-
binding.websocket.close(
|
|
1265
|
-
1011,
|
|
1266
|
-
"dynamic.websocket_forward_failed",
|
|
1267
|
-
);
|
|
1268
|
-
return;
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
|
-
void currentRuntime
|
|
1272
|
-
.forwardIncomingWebSocketMessage(
|
|
1273
|
-
currentProxyToActorWs,
|
|
1274
|
-
event.data as any,
|
|
1275
|
-
event.rivetMessageIndex,
|
|
1276
|
-
)
|
|
1277
|
-
.catch((error) => {
|
|
1278
|
-
logger().error({
|
|
1279
|
-
msg: "failed forwarding websocket message to dynamic actor",
|
|
1280
|
-
actorId: binding.actorId,
|
|
1281
|
-
error: stringifyError(error),
|
|
1282
|
-
});
|
|
1283
|
-
binding.websocket.close(
|
|
1284
|
-
1011,
|
|
1285
|
-
"dynamic.websocket_forward_failed",
|
|
1286
|
-
);
|
|
1287
|
-
});
|
|
1288
|
-
};
|
|
1289
|
-
const onClose = (event: CloseEvent) => {
|
|
1290
|
-
const currentProxyToActorWs = binding.proxyToActorWs;
|
|
1291
|
-
if (
|
|
1292
|
-
currentProxyToActorWs &&
|
|
1293
|
-
currentProxyToActorWs.readyState !==
|
|
1294
|
-
currentProxyToActorWs.CLOSED
|
|
1295
|
-
) {
|
|
1296
|
-
currentProxyToActorWs.close(event.code, event.reason);
|
|
1297
|
-
}
|
|
1298
|
-
this.#deleteHibernatableWebSocketAckState(
|
|
1299
|
-
binding.gatewayId,
|
|
1300
|
-
binding.requestId,
|
|
1301
|
-
);
|
|
1302
|
-
unregisterRemoteHibernatableWebSocketAckHooks(
|
|
1303
|
-
binding.remoteAckHookToken,
|
|
1304
|
-
this.#config.test.enabled,
|
|
1305
|
-
);
|
|
1306
|
-
this.#deleteHibernatableRunnerWebSocketBinding(
|
|
1307
|
-
binding.gatewayId,
|
|
1308
|
-
binding.requestId,
|
|
1309
|
-
);
|
|
1310
|
-
};
|
|
1311
|
-
const onError = () => {
|
|
1312
|
-
const currentProxyToActorWs = binding.proxyToActorWs;
|
|
1313
|
-
if (
|
|
1314
|
-
currentProxyToActorWs &&
|
|
1315
|
-
currentProxyToActorWs.readyState !==
|
|
1316
|
-
currentProxyToActorWs.CLOSED
|
|
1317
|
-
) {
|
|
1318
|
-
currentProxyToActorWs.close(1011, "dynamic.gateway_error");
|
|
1319
|
-
}
|
|
1320
|
-
};
|
|
1321
|
-
|
|
1322
|
-
proxyToActorWs.addEventListener("message", onProxyMessage);
|
|
1323
|
-
proxyToActorWs.addEventListener("close", onProxyClose);
|
|
1324
|
-
proxyToActorWs.addEventListener("error", onProxyError);
|
|
1325
|
-
binding.websocket.addEventListener("message", onMessage);
|
|
1326
|
-
binding.websocket.addEventListener("close", onClose);
|
|
1327
|
-
binding.websocket.addEventListener("error", onError);
|
|
1328
|
-
|
|
1329
|
-
binding.detach = () => {
|
|
1330
|
-
proxyToActorWs.removeEventListener("message", onProxyMessage);
|
|
1331
|
-
proxyToActorWs.removeEventListener("close", onProxyClose);
|
|
1332
|
-
proxyToActorWs.removeEventListener("error", onProxyError);
|
|
1333
|
-
binding.websocket.removeEventListener("message", onMessage);
|
|
1334
|
-
binding.websocket.removeEventListener("close", onClose);
|
|
1335
|
-
binding.websocket.removeEventListener("error", onError);
|
|
1336
|
-
};
|
|
1337
|
-
}
|
|
1338
|
-
|
|
1339
|
-
async #rebindDynamicHibernatableRunnerWebSockets(
|
|
1340
|
-
actorId: string,
|
|
1341
|
-
): Promise<void> {
|
|
1342
|
-
const bindings = Array.from(
|
|
1343
|
-
this.#hibernatableRunnerWebSocketBindings.values(),
|
|
1344
|
-
).filter((binding) => binding.actorId === actorId);
|
|
1345
|
-
// Bind every binding even if some throw, so a single failed rebind
|
|
1346
|
-
// does not strand the rest in an idle state.
|
|
1347
|
-
const results = await Promise.allSettled(
|
|
1348
|
-
bindings.map((binding) =>
|
|
1349
|
-
this.#bindDynamicHibernatableRunnerWebSocket(binding, true),
|
|
1350
|
-
),
|
|
1351
|
-
);
|
|
1352
|
-
for (const result of results) {
|
|
1353
|
-
if (result.status === "rejected") {
|
|
1354
|
-
logger().warn({
|
|
1355
|
-
msg: "failed to rebind dynamic hibernatable runner websocket",
|
|
1356
|
-
actorId,
|
|
1357
|
-
error: stringifyError(result.reason),
|
|
1358
|
-
});
|
|
1359
|
-
}
|
|
1360
|
-
}
|
|
1361
|
-
}
|
|
1362
|
-
|
|
1363
|
-
async #rebindHibernatableConnectSockets(actorId: string): Promise<void> {
|
|
1364
|
-
const bindings = Array.from(
|
|
1365
|
-
this.#hibernatableConnectBindings.values(),
|
|
1366
|
-
).filter((binding) => binding.actorId === actorId);
|
|
1367
|
-
|
|
1368
|
-
// Same reasoning as `#rebindDynamicHibernatableRunnerWebSockets`: a
|
|
1369
|
-
// failure in one binding should not block the others from rebinding.
|
|
1370
|
-
const results = await Promise.allSettled(
|
|
1371
|
-
bindings.map((binding) =>
|
|
1372
|
-
this.#bindHibernatableConnectSocket(binding, true),
|
|
1373
|
-
),
|
|
1374
|
-
);
|
|
1375
|
-
for (const result of results) {
|
|
1376
|
-
if (result.status === "rejected") {
|
|
1377
|
-
logger().warn({
|
|
1378
|
-
msg: "failed to rebind hibernatable connect socket",
|
|
1379
|
-
actorId,
|
|
1380
|
-
error: stringifyError(result.reason),
|
|
1381
|
-
});
|
|
1382
|
-
}
|
|
1383
|
-
}
|
|
1384
|
-
}
|
|
1385
|
-
|
|
1386
|
-
async serverlessHandleStart(c: HonoContext): Promise<Response> {
|
|
1387
|
-
const payload = await c.req.arrayBuffer();
|
|
1388
|
-
|
|
1389
|
-
return streamSSE(c, async (stream) => {
|
|
1390
|
-
// NOTE: onAbort does not work reliably
|
|
1391
|
-
stream.onAbort(() => {});
|
|
1392
|
-
c.req.raw.signal.addEventListener("abort", () => {
|
|
1393
|
-
logger().debug("SSE aborted");
|
|
1394
|
-
});
|
|
1395
|
-
|
|
1396
|
-
await this.#envoyStarted.promise;
|
|
1397
|
-
|
|
1398
|
-
if (this.#isShuttingDown) {
|
|
1399
|
-
logger().debug({
|
|
1400
|
-
msg: "ignoring serverless start because driver is shutting down",
|
|
1401
|
-
});
|
|
1402
|
-
return;
|
|
1403
|
-
}
|
|
1404
|
-
|
|
1405
|
-
await this.#envoy.startServerlessActor(payload);
|
|
1406
|
-
|
|
1407
|
-
// Send ping every second to keep the connection alive
|
|
1408
|
-
while (true) {
|
|
1409
|
-
if (this.#isEnvoyStopped) {
|
|
1410
|
-
logger().debug({
|
|
1411
|
-
msg: "envoy is stopped",
|
|
1412
|
-
});
|
|
1413
|
-
break;
|
|
1414
|
-
}
|
|
1415
|
-
|
|
1416
|
-
if (stream.closed || stream.aborted) {
|
|
1417
|
-
logger().debug({
|
|
1418
|
-
msg: "envoy sse stream closed",
|
|
1419
|
-
closed: stream.closed,
|
|
1420
|
-
aborted: stream.aborted,
|
|
1421
|
-
});
|
|
1422
|
-
break;
|
|
1423
|
-
}
|
|
1424
|
-
|
|
1425
|
-
await stream.writeSSE({ event: "ping", data: "" });
|
|
1426
|
-
await stream.sleep(ENVOY_SSE_PING_INTERVAL);
|
|
1427
|
-
}
|
|
1428
|
-
});
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
#buildStartupPreloadMap(
|
|
1432
|
-
preloadedKv: protocol.PreloadedKv | null,
|
|
1433
|
-
persistDataOverride?: Uint8Array,
|
|
1434
|
-
): { preloadMap: PreloadMap | undefined; entries: number } {
|
|
1435
|
-
if (preloadedKv == null) {
|
|
1436
|
-
return { preloadMap: undefined, entries: 0 };
|
|
1437
|
-
}
|
|
1438
|
-
|
|
1439
|
-
const entries: [Uint8Array, Uint8Array][] = preloadedKv.entries.map(
|
|
1440
|
-
(entry) => [new Uint8Array(entry.key), new Uint8Array(entry.value)],
|
|
1441
|
-
);
|
|
1442
|
-
|
|
1443
|
-
if (persistDataOverride) {
|
|
1444
|
-
let replaced = false;
|
|
1445
|
-
for (const entry of entries) {
|
|
1446
|
-
if (compareBytes(entry[0], KEYS.PERSIST_DATA) === 0) {
|
|
1447
|
-
entry[1] = persistDataOverride;
|
|
1448
|
-
replaced = true;
|
|
1449
|
-
break;
|
|
1450
|
-
}
|
|
1451
|
-
}
|
|
1452
|
-
|
|
1453
|
-
if (!replaced) {
|
|
1454
|
-
entries.push([KEYS.PERSIST_DATA, persistDataOverride]);
|
|
1455
|
-
}
|
|
1456
|
-
}
|
|
1457
|
-
|
|
1458
|
-
entries.sort((a, b) => compareBytes(a[0], b[0]));
|
|
1459
|
-
|
|
1460
|
-
const requestedGetKeys = preloadedKv.requestedGetKeys
|
|
1461
|
-
.map((key) => new Uint8Array(key))
|
|
1462
|
-
.sort(compareBytes);
|
|
1463
|
-
const requestedPrefixes = preloadedKv.requestedPrefixes
|
|
1464
|
-
.map((prefix) => new Uint8Array(prefix))
|
|
1465
|
-
.sort(compareBytes);
|
|
1466
|
-
|
|
1467
|
-
return {
|
|
1468
|
-
preloadMap: createPreloadMap(
|
|
1469
|
-
entries,
|
|
1470
|
-
requestedGetKeys,
|
|
1471
|
-
requestedPrefixes,
|
|
1472
|
-
),
|
|
1473
|
-
entries: entries.length,
|
|
1474
|
-
};
|
|
1475
|
-
}
|
|
1476
|
-
|
|
1477
|
-
async #envoyOnActorStart(
|
|
1478
|
-
_envoy: EnvoyHandle,
|
|
1479
|
-
actorId: string,
|
|
1480
|
-
generation: number,
|
|
1481
|
-
actorConfig: protocol.ActorConfig,
|
|
1482
|
-
preloadedKv: protocol.PreloadedKv | null,
|
|
1483
|
-
_sqliteStartupData: protocol.SqliteStartupData | null,
|
|
1484
|
-
): Promise<void> {
|
|
1485
|
-
if (this.#isShuttingDown) {
|
|
1486
|
-
logger().debug({
|
|
1487
|
-
msg: "rejecting actor start because driver is shutting down",
|
|
1488
|
-
actorId,
|
|
1489
|
-
name: actorConfig.name,
|
|
1490
|
-
generation,
|
|
1491
|
-
});
|
|
1492
|
-
throw new Error("engine actor driver is shutting down");
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
|
-
logger().debug({
|
|
1496
|
-
msg: "engine actor starting",
|
|
1497
|
-
actorId,
|
|
1498
|
-
name: actorConfig.name,
|
|
1499
|
-
key: actorConfig.key,
|
|
1500
|
-
generation,
|
|
1501
|
-
});
|
|
1502
|
-
|
|
1503
|
-
// Deserialize input
|
|
1504
|
-
let input: any;
|
|
1505
|
-
if (actorConfig.input && actorConfig.input.byteLength > 0) {
|
|
1506
|
-
input = cbor.decode(new Uint8Array(actorConfig.input));
|
|
1507
|
-
}
|
|
1508
|
-
|
|
1509
|
-
// Get or create handler
|
|
1510
|
-
let handler = this.#actors.get(actorId);
|
|
1511
|
-
if (!handler) {
|
|
1512
|
-
// IMPORTANT: We must set the handler in the map synchronously before doing any
|
|
1513
|
-
// async operations to avoid race conditions where multiple calls might try to
|
|
1514
|
-
// create the same handler simultaneously.
|
|
1515
|
-
handler = {
|
|
1516
|
-
actorStartPromise: promiseWithResolvers((reason) =>
|
|
1517
|
-
logger().warn({
|
|
1518
|
-
msg: "unhandled actor start promise rejection",
|
|
1519
|
-
reason,
|
|
1520
|
-
}),
|
|
1521
|
-
),
|
|
1522
|
-
};
|
|
1523
|
-
this.#actors.set(actorId, handler);
|
|
1524
|
-
}
|
|
1525
|
-
handler.actorStartError = undefined;
|
|
1526
|
-
|
|
1527
|
-
const name = actorConfig.name as string;
|
|
1528
|
-
invariant(actorConfig.key, "actor should have a key");
|
|
1529
|
-
const key = deserializeActorKey(actorConfig.key);
|
|
1530
|
-
handler.actorName = name;
|
|
1531
|
-
|
|
1532
|
-
try {
|
|
1533
|
-
let preloadMap: PreloadMap | undefined;
|
|
1534
|
-
let persistDataBuffer: Uint8Array | null | undefined;
|
|
1535
|
-
let checkPersistDataMs = 0;
|
|
1536
|
-
let initNewActorMs = 0;
|
|
1537
|
-
let preloadKvMs = 0;
|
|
1538
|
-
let preloadKvEntries = 0;
|
|
1539
|
-
let driverKvRoundTrips = 0;
|
|
1540
|
-
|
|
1541
|
-
if (preloadedKv) {
|
|
1542
|
-
const preloadStart = performance.now();
|
|
1543
|
-
const preloaded = this.#buildStartupPreloadMap(preloadedKv);
|
|
1544
|
-
preloadMap = preloaded.preloadMap;
|
|
1545
|
-
preloadKvEntries = preloaded.entries;
|
|
1546
|
-
preloadKvMs = performance.now() - preloadStart;
|
|
1547
|
-
persistDataBuffer = preloadMap?.get(KEYS.PERSIST_DATA)?.value;
|
|
1548
|
-
logger().debug({
|
|
1549
|
-
msg: "received startup kv preload from start command",
|
|
1550
|
-
actorId,
|
|
1551
|
-
entries: preloadKvEntries,
|
|
1552
|
-
durationMs: preloadKvMs,
|
|
1553
|
-
});
|
|
1554
|
-
}
|
|
1555
|
-
|
|
1556
|
-
if (persistDataBuffer === undefined) {
|
|
1557
|
-
const checkStart = performance.now();
|
|
1558
|
-
const [persistData] = await this.#envoy.kvGet(actorId, [
|
|
1559
|
-
KEYS.PERSIST_DATA,
|
|
1560
|
-
]);
|
|
1561
|
-
persistDataBuffer = persistData;
|
|
1562
|
-
checkPersistDataMs = performance.now() - checkStart;
|
|
1563
|
-
driverKvRoundTrips++;
|
|
1564
|
-
}
|
|
1565
|
-
|
|
1566
|
-
if (persistDataBuffer === null) {
|
|
1567
|
-
const initStart = performance.now();
|
|
1568
|
-
const initialKvState = getInitialActorKvState(input);
|
|
1569
|
-
const persistData = initialKvState[0]?.[1];
|
|
1570
|
-
await this.#envoy.kvPut(actorId, initialKvState);
|
|
1571
|
-
initNewActorMs = performance.now() - initStart;
|
|
1572
|
-
driverKvRoundTrips++;
|
|
1573
|
-
if (preloadedKv && persistData) {
|
|
1574
|
-
const preloadStart = performance.now();
|
|
1575
|
-
const preloaded = this.#buildStartupPreloadMap(
|
|
1576
|
-
preloadedKv,
|
|
1577
|
-
persistData,
|
|
1578
|
-
);
|
|
1579
|
-
preloadMap = preloaded.preloadMap;
|
|
1580
|
-
preloadKvEntries = preloaded.entries;
|
|
1581
|
-
preloadKvMs += performance.now() - preloadStart;
|
|
1582
|
-
}
|
|
1583
|
-
logger().debug({
|
|
1584
|
-
msg: "initialized persist data for new actor",
|
|
1585
|
-
actorId,
|
|
1586
|
-
durationMs: initNewActorMs,
|
|
1587
|
-
});
|
|
1588
|
-
}
|
|
1589
|
-
|
|
1590
|
-
// Create actor instance
|
|
1591
|
-
const definition = lookupInRegistry(this.#config, actorConfig.name);
|
|
1592
|
-
if (isDynamicActorDefinition(definition)) {
|
|
1593
|
-
let runtime = this.#dynamicRuntimes.get(actorId);
|
|
1594
|
-
if (!runtime) {
|
|
1595
|
-
runtime = new DynamicActorIsolateRuntime({
|
|
1596
|
-
actorId,
|
|
1597
|
-
actorName: name,
|
|
1598
|
-
actorKey: key,
|
|
1599
|
-
input,
|
|
1600
|
-
region: "unknown",
|
|
1601
|
-
loader: definition.loader,
|
|
1602
|
-
actorDriver: this,
|
|
1603
|
-
inlineClient: this.#inlineClient,
|
|
1604
|
-
test: this.#config.test,
|
|
1605
|
-
});
|
|
1606
|
-
await runtime.start();
|
|
1607
|
-
this.#dynamicRuntimes.set(actorId, runtime);
|
|
1608
|
-
}
|
|
1609
|
-
|
|
1610
|
-
const dynamicActor = new DynamicActorInstance(actorId, runtime);
|
|
1611
|
-
handler.actor = dynamicActor;
|
|
1612
|
-
|
|
1613
|
-
handler.actorStartError = undefined;
|
|
1614
|
-
handler.actorStartPromise?.resolve();
|
|
1615
|
-
handler.actorStartPromise = undefined;
|
|
1616
|
-
|
|
1617
|
-
try {
|
|
1618
|
-
await this.#rebindHibernatableConnectSockets(actorId);
|
|
1619
|
-
await this.#rebindDynamicHibernatableRunnerWebSockets(
|
|
1620
|
-
actorId,
|
|
1621
|
-
);
|
|
1622
|
-
const rawMetaEntries =
|
|
1623
|
-
await dynamicActor.getHibernatingWebSockets();
|
|
1624
|
-
const metaEntries = rawMetaEntries.map((entry) => ({
|
|
1625
|
-
gatewayId: entry.gatewayId,
|
|
1626
|
-
requestId: entry.requestId,
|
|
1627
|
-
rivetMessageIndex: entry.serverMessageIndex,
|
|
1628
|
-
envoyMessageIndex: entry.clientMessageIndex,
|
|
1629
|
-
path: entry.path,
|
|
1630
|
-
headers: entry.headers,
|
|
1631
|
-
}));
|
|
1632
|
-
await this.#envoy.restoreHibernatingRequests(
|
|
1633
|
-
actorId,
|
|
1634
|
-
metaEntries,
|
|
1635
|
-
);
|
|
1636
|
-
} catch (error) {
|
|
1637
|
-
logger().warn({
|
|
1638
|
-
msg: "failed to restore dynamic hibernating requests after actor start",
|
|
1639
|
-
actorId,
|
|
1640
|
-
error: stringifyError(error),
|
|
1641
|
-
});
|
|
1642
|
-
}
|
|
1643
|
-
} else if (isStaticActorDefinition(definition)) {
|
|
1644
|
-
const instantiateStart = performance.now();
|
|
1645
|
-
const staticActor =
|
|
1646
|
-
(await definition.instantiate()) as AnyStaticActorInstance;
|
|
1647
|
-
const instantiateMs = performance.now() - instantiateStart;
|
|
1648
|
-
handler.actor = staticActor;
|
|
1649
|
-
|
|
1650
|
-
// Record driver-level startup metrics on the actor.
|
|
1651
|
-
staticActor.metrics.startup.checkPersistDataMs =
|
|
1652
|
-
checkPersistDataMs;
|
|
1653
|
-
staticActor.metrics.startup.initNewActorMs = initNewActorMs;
|
|
1654
|
-
staticActor.metrics.startup.preloadKvMs = preloadKvMs;
|
|
1655
|
-
staticActor.metrics.startup.preloadKvEntries = preloadKvEntries;
|
|
1656
|
-
staticActor.metrics.startup.instantiateMs = instantiateMs;
|
|
1657
|
-
staticActor.metrics.startup.kvRoundTrips = driverKvRoundTrips;
|
|
1658
|
-
|
|
1659
|
-
// Cap the actor's graceful shutdown window to fit inside the
|
|
1660
|
-
// engine's stop deadline, leaving a 1s safety margin. On
|
|
1661
|
-
// serverless this also adds the drain grace period so sleep
|
|
1662
|
-
// can run during the drain window.
|
|
1663
|
-
const protocolMetadata = this.#envoy.getProtocolMetadata();
|
|
1664
|
-
if (protocolMetadata) {
|
|
1665
|
-
const stopThresholdMax = Math.max(
|
|
1666
|
-
Number(protocolMetadata.actorStopThreshold) - 1000,
|
|
1667
|
-
0,
|
|
1668
|
-
);
|
|
1669
|
-
const drainMax = protocolMetadata.serverlessDrainGracePeriod
|
|
1670
|
-
? Math.max(
|
|
1671
|
-
Number(
|
|
1672
|
-
protocolMetadata.serverlessDrainGracePeriod,
|
|
1673
|
-
) - 1000,
|
|
1674
|
-
0,
|
|
1675
|
-
)
|
|
1676
|
-
: 0;
|
|
1677
|
-
staticActor.overrides.sleepGracePeriod =
|
|
1678
|
-
stopThresholdMax + drainMax;
|
|
1679
|
-
}
|
|
1680
|
-
|
|
1681
|
-
// Start actor
|
|
1682
|
-
await staticActor.start(
|
|
1683
|
-
this,
|
|
1684
|
-
this.#inlineClient,
|
|
1685
|
-
actorId,
|
|
1686
|
-
name,
|
|
1687
|
-
key,
|
|
1688
|
-
"unknown", // TODO: Add regions
|
|
1689
|
-
preloadMap,
|
|
1690
|
-
);
|
|
1691
|
-
} else {
|
|
1692
|
-
throw new Error(
|
|
1693
|
-
`actor definition for ${actorConfig.name} is not instantiable`,
|
|
1694
|
-
);
|
|
1695
|
-
}
|
|
1696
|
-
|
|
1697
|
-
logger().debug({ msg: "engine actor started", actorId, name, key });
|
|
1698
|
-
} catch (innerError) {
|
|
1699
|
-
const dynamicRuntime = this.#dynamicRuntimes.get(actorId);
|
|
1700
|
-
if (dynamicRuntime) {
|
|
1701
|
-
try {
|
|
1702
|
-
await dynamicRuntime.dispose();
|
|
1703
|
-
} catch (disposeError) {
|
|
1704
|
-
logger().debug({
|
|
1705
|
-
msg: "failed to dispose dynamic runtime after actor start failure",
|
|
1706
|
-
actorId,
|
|
1707
|
-
error: stringifyError(disposeError),
|
|
1708
|
-
});
|
|
1709
|
-
}
|
|
1710
|
-
this.#dynamicRuntimes.delete(actorId);
|
|
1711
|
-
}
|
|
1712
|
-
const error =
|
|
1713
|
-
innerError instanceof Error
|
|
1714
|
-
? new Error(
|
|
1715
|
-
`Failed to start actor ${actorId}: ${innerError.message}`,
|
|
1716
|
-
{ cause: innerError },
|
|
1717
|
-
)
|
|
1718
|
-
: new Error(
|
|
1719
|
-
`Failed to start actor ${actorId}: ${String(innerError)}`,
|
|
1720
|
-
);
|
|
1721
|
-
handler.actor = undefined;
|
|
1722
|
-
handler.actorStartError = error;
|
|
1723
|
-
handler.actorStartPromise?.reject(error);
|
|
1724
|
-
handler.actorStartPromise = undefined;
|
|
1725
|
-
logger().error({
|
|
1726
|
-
msg: "engine actor failed to start",
|
|
1727
|
-
actorId,
|
|
1728
|
-
name,
|
|
1729
|
-
key,
|
|
1730
|
-
error: stringifyError(error),
|
|
1731
|
-
});
|
|
1732
|
-
|
|
1733
|
-
try {
|
|
1734
|
-
this.#envoy.stopActor(actorId, undefined);
|
|
1735
|
-
} catch (stopError) {
|
|
1736
|
-
logger().debug({
|
|
1737
|
-
msg: "failed to stop actor after start failure",
|
|
1738
|
-
actorId,
|
|
1739
|
-
error: stringifyError(stopError),
|
|
1740
|
-
});
|
|
1741
|
-
}
|
|
1742
|
-
}
|
|
1743
|
-
}
|
|
1744
|
-
|
|
1745
|
-
async #envoyOnActorStop(
|
|
1746
|
-
_envoyHandle: EnvoyHandle,
|
|
1747
|
-
actorId: string,
|
|
1748
|
-
generation: number,
|
|
1749
|
-
_reason: protocol.StopActorReason,
|
|
1750
|
-
): Promise<void> {
|
|
1751
|
-
logger().debug({ msg: "engine actor stopping", actorId, generation });
|
|
1752
|
-
|
|
1753
|
-
// HACK: Retrieve the stop intent we tracked locally (see RVT-5284)
|
|
1754
|
-
// Default to "sleep" if no intent was recorded (e.g., if the runner
|
|
1755
|
-
// initiated the stop)
|
|
1756
|
-
//
|
|
1757
|
-
// TODO: This will not work if the actor is destroyed from the API
|
|
1758
|
-
// correctly. Currently, it will use the sleep intent, but it's
|
|
1759
|
-
// actually a destroy intent.
|
|
1760
|
-
const reason = this.#actorStopIntent.get(actorId) ?? "sleep";
|
|
1761
|
-
this.#actorStopIntent.delete(actorId);
|
|
1762
|
-
|
|
1763
|
-
const handler = this.#actors.get(actorId);
|
|
1764
|
-
if (!handler) {
|
|
1765
|
-
logger().debug({
|
|
1766
|
-
msg: "no engine actor handler to stop",
|
|
1767
|
-
actorId,
|
|
1768
|
-
reason,
|
|
1769
|
-
});
|
|
1770
|
-
return;
|
|
1771
|
-
}
|
|
1772
|
-
|
|
1773
|
-
if (handler.actorStartPromise) {
|
|
1774
|
-
try {
|
|
1775
|
-
logger().debug({
|
|
1776
|
-
msg: "engine actor stopping before it started, waiting",
|
|
1777
|
-
actorId,
|
|
1778
|
-
generation,
|
|
1779
|
-
});
|
|
1780
|
-
await handler.actorStartPromise.promise;
|
|
1781
|
-
} catch (err) {
|
|
1782
|
-
// Start failed, but we still want to clean up the handler
|
|
1783
|
-
logger().debug({
|
|
1784
|
-
msg: "actor start failed during stop, cleaning up handler",
|
|
1785
|
-
actorId,
|
|
1786
|
-
error: stringifyError(err),
|
|
1787
|
-
});
|
|
1788
|
-
}
|
|
1789
|
-
}
|
|
1790
|
-
|
|
1791
|
-
if (handler.actor) {
|
|
1792
|
-
try {
|
|
1793
|
-
if (
|
|
1794
|
-
reason === "crash" &&
|
|
1795
|
-
isStaticActorInstance(handler.actor)
|
|
1796
|
-
) {
|
|
1797
|
-
await handler.actor.debugForceCrash();
|
|
1798
|
-
} else if (reason !== "crash") {
|
|
1799
|
-
await handler.actor.onStop(reason);
|
|
1800
|
-
}
|
|
1801
|
-
} catch (err) {
|
|
1802
|
-
logger().error({
|
|
1803
|
-
msg: "error in onStop, proceeding with removing actor",
|
|
1804
|
-
error: stringifyError(err),
|
|
1805
|
-
});
|
|
1806
|
-
}
|
|
1807
|
-
}
|
|
1808
|
-
await this.#disposeDynamicRuntime(actorId, "actor stop");
|
|
1809
|
-
|
|
1810
|
-
if (handler.alarmTimeout) {
|
|
1811
|
-
handler.alarmTimeout.abort();
|
|
1812
|
-
handler.alarmTimeout = undefined;
|
|
1813
|
-
}
|
|
1814
|
-
|
|
1815
|
-
this.#actors.delete(actorId);
|
|
1816
|
-
|
|
1817
|
-
logger().debug({ msg: "engine actor stopped", actorId, reason });
|
|
1818
|
-
}
|
|
1819
|
-
|
|
1820
|
-
async #disposeDynamicRuntime(
|
|
1821
|
-
actorId: string,
|
|
1822
|
-
reason: string,
|
|
1823
|
-
): Promise<void> {
|
|
1824
|
-
const runtime = this.#dynamicRuntimes.get(actorId);
|
|
1825
|
-
if (!runtime) {
|
|
1826
|
-
return;
|
|
1827
|
-
}
|
|
1828
|
-
|
|
1829
|
-
try {
|
|
1830
|
-
await runtime.dispose();
|
|
1831
|
-
} catch (error) {
|
|
1832
|
-
logger().warn({
|
|
1833
|
-
msg: "failed to dispose dynamic runtime",
|
|
1834
|
-
actorId,
|
|
1835
|
-
reason,
|
|
1836
|
-
error: stringifyError(error),
|
|
1837
|
-
});
|
|
1838
|
-
} finally {
|
|
1839
|
-
this.#dynamicRuntimes.delete(actorId);
|
|
1840
|
-
}
|
|
1841
|
-
}
|
|
1842
|
-
|
|
1843
|
-
async #disposeAllDynamicRuntimes(reason: string): Promise<void> {
|
|
1844
|
-
await Promise.all(
|
|
1845
|
-
Array.from(this.#dynamicRuntimes.keys(), (actorId) =>
|
|
1846
|
-
this.#disposeDynamicRuntime(actorId, reason),
|
|
1847
|
-
),
|
|
1848
|
-
);
|
|
1849
|
-
}
|
|
1850
|
-
|
|
1851
|
-
// MARK: - Envoy Networking
|
|
1852
|
-
async #envoyFetch(
|
|
1853
|
-
_envoy: EnvoyHandle,
|
|
1854
|
-
actorId: string,
|
|
1855
|
-
_gatewayIdBuf: ArrayBuffer,
|
|
1856
|
-
_requestIdBuf: ArrayBuffer,
|
|
1857
|
-
request: Request,
|
|
1858
|
-
): Promise<Response> {
|
|
1859
|
-
logger().debug({
|
|
1860
|
-
msg: "envoy fetch",
|
|
1861
|
-
actorId,
|
|
1862
|
-
url: request.url,
|
|
1863
|
-
method: request.method,
|
|
1864
|
-
});
|
|
1865
|
-
const overlayResponse = this.#routeOverlayRequest(actorId, request);
|
|
1866
|
-
if (overlayResponse) {
|
|
1867
|
-
return overlayResponse;
|
|
1868
|
-
}
|
|
1869
|
-
|
|
1870
|
-
if (this.#isDynamicActor(actorId)) {
|
|
1871
|
-
return await this.#requireDynamicRuntime(actorId).fetch(request);
|
|
1872
|
-
}
|
|
1873
|
-
return await this.#actorRouter.fetch(request, { actorId });
|
|
1874
|
-
}
|
|
1875
|
-
|
|
1876
|
-
#routeOverlayRequest(actorId: string, request: Request): Response | null {
|
|
1877
|
-
const url = new URL(request.url);
|
|
1878
|
-
switch (`${request.method} ${url.pathname}`) {
|
|
1879
|
-
case "PUT /dynamic/reload":
|
|
1880
|
-
return this.#handleDynamicReloadOverlay(actorId);
|
|
1881
|
-
default:
|
|
1882
|
-
return null;
|
|
1883
|
-
}
|
|
1884
|
-
}
|
|
1885
|
-
|
|
1886
|
-
#handleDynamicReloadOverlay(actorId: string): Response {
|
|
1887
|
-
if (!this.#isDynamicActor(actorId)) {
|
|
1888
|
-
return new Response("not a dynamic actor", { status: 404 });
|
|
1889
|
-
}
|
|
1890
|
-
this.startSleep(actorId);
|
|
1891
|
-
return new Response(null, { status: 200 });
|
|
1892
|
-
}
|
|
1893
|
-
|
|
1894
|
-
async #envoyWebSocket(
|
|
1895
|
-
_envoy: EnvoyHandle,
|
|
1896
|
-
actorId: string,
|
|
1897
|
-
websocketRaw: any,
|
|
1898
|
-
gatewayIdBuf: ArrayBuffer,
|
|
1899
|
-
requestIdBuf: ArrayBuffer,
|
|
1900
|
-
request: Request,
|
|
1901
|
-
requestPath: string,
|
|
1902
|
-
requestHeaders: Record<string, string>,
|
|
1903
|
-
isHibernatable: boolean,
|
|
1904
|
-
isRestoringHibernatable: boolean,
|
|
1905
|
-
): Promise<void> {
|
|
1906
|
-
const websocket = websocketRaw as UniversalWebSocket;
|
|
1907
|
-
|
|
1908
|
-
// Parse configuration from Sec-WebSocket-Protocol header (optional for path-based routing)
|
|
1909
|
-
const protocols = request.headers.get("sec-websocket-protocol");
|
|
1910
|
-
const { encoding, connParams, ackHookToken } =
|
|
1911
|
-
parseWebSocketProtocols(protocols);
|
|
1912
|
-
const remoteAckHookToken =
|
|
1913
|
-
ackHookToken ??
|
|
1914
|
-
new URL(request.url).searchParams.get(
|
|
1915
|
-
REMOTE_ACK_HOOK_QUERY_PARAM,
|
|
1916
|
-
) ??
|
|
1917
|
-
undefined;
|
|
1918
|
-
const requestPathWithoutQuery = requestPath.split("?")[0];
|
|
1919
|
-
|
|
1920
|
-
if (isHibernatable && requestPathWithoutQuery === PATH_CONNECT) {
|
|
1921
|
-
this.#registerHibernatableWebSocketAckTestHooks(
|
|
1922
|
-
websocket,
|
|
1923
|
-
gatewayIdBuf,
|
|
1924
|
-
requestIdBuf,
|
|
1925
|
-
remoteAckHookToken,
|
|
1926
|
-
);
|
|
1927
|
-
try {
|
|
1928
|
-
await this.#bindHibernatableConnectSocket(
|
|
1929
|
-
{
|
|
1930
|
-
actorId,
|
|
1931
|
-
websocket,
|
|
1932
|
-
request,
|
|
1933
|
-
requestPath,
|
|
1934
|
-
requestHeaders,
|
|
1935
|
-
encoding,
|
|
1936
|
-
connParams,
|
|
1937
|
-
gatewayId: gatewayIdBuf,
|
|
1938
|
-
requestId: requestIdBuf,
|
|
1939
|
-
remoteAckHookToken,
|
|
1940
|
-
},
|
|
1941
|
-
isRestoringHibernatable,
|
|
1942
|
-
);
|
|
1943
|
-
} catch (error) {
|
|
1944
|
-
logger().error({
|
|
1945
|
-
msg: "failed to bind hibernatable connect websocket",
|
|
1946
|
-
actorId,
|
|
1947
|
-
error: stringifyError(error),
|
|
1948
|
-
});
|
|
1949
|
-
websocket.close(1011, "ws.route_error");
|
|
1950
|
-
}
|
|
1951
|
-
return;
|
|
1952
|
-
}
|
|
1953
|
-
|
|
1954
|
-
if (this.#isDynamicActor(actorId)) {
|
|
1955
|
-
await this.#runnerDynamicWebSocket(
|
|
1956
|
-
actorId,
|
|
1957
|
-
websocket,
|
|
1958
|
-
gatewayIdBuf,
|
|
1959
|
-
requestIdBuf,
|
|
1960
|
-
requestPath,
|
|
1961
|
-
requestHeaders,
|
|
1962
|
-
encoding,
|
|
1963
|
-
connParams,
|
|
1964
|
-
isHibernatable,
|
|
1965
|
-
isRestoringHibernatable,
|
|
1966
|
-
);
|
|
1967
|
-
return;
|
|
1968
|
-
}
|
|
1969
|
-
|
|
1970
|
-
// Fetch WS handler
|
|
1971
|
-
//
|
|
1972
|
-
// We store the promise since we need to add WebSocket event listeners immediately that will wait for the promise to resolve
|
|
1973
|
-
let wsHandler: UpgradeWebSocketArgs;
|
|
1974
|
-
try {
|
|
1975
|
-
wsHandler = await routeWebSocket(
|
|
1976
|
-
request,
|
|
1977
|
-
requestPath,
|
|
1978
|
-
requestHeaders,
|
|
1979
|
-
this.#config,
|
|
1980
|
-
this,
|
|
1981
|
-
actorId,
|
|
1982
|
-
encoding,
|
|
1983
|
-
connParams,
|
|
1984
|
-
gatewayIdBuf,
|
|
1985
|
-
requestIdBuf,
|
|
1986
|
-
isHibernatable,
|
|
1987
|
-
isRestoringHibernatable,
|
|
1988
|
-
);
|
|
1989
|
-
} catch (error) {
|
|
1990
|
-
logger().error({
|
|
1991
|
-
msg: "building websocket handlers errored",
|
|
1992
|
-
error,
|
|
1993
|
-
});
|
|
1994
|
-
websocketRaw.close(1011, "ws.route_error");
|
|
1995
|
-
return;
|
|
1996
|
-
}
|
|
1997
|
-
|
|
1998
|
-
// Connect the Hono WS hook to the adapter
|
|
1999
|
-
//
|
|
2000
|
-
// We need to assign to `raw` in order for WSContext to expose it on
|
|
2001
|
-
// `ws.raw`
|
|
2002
|
-
(websocket as WSContextInit).raw = websocket;
|
|
2003
|
-
const wsContext = new WSContext(websocket);
|
|
2004
|
-
|
|
2005
|
-
// Get connection and actor from wsHandler (may be undefined for inspector endpoint)
|
|
2006
|
-
const conn = wsHandler.conn;
|
|
2007
|
-
const actor = wsHandler.actor;
|
|
2008
|
-
const _connStateManager = conn?.[CONN_STATE_MANAGER_SYMBOL];
|
|
2009
|
-
|
|
2010
|
-
// Bind event listeners to Hono WebSocket handlers
|
|
2011
|
-
//
|
|
2012
|
-
// We update the HWS data after calling handlers in order to ensure
|
|
2013
|
-
// that the handler ran successfully. By doing this, we ensure at least
|
|
2014
|
-
// once delivery of events to the event handlers.
|
|
2015
|
-
|
|
2016
|
-
if (isHibernatable) {
|
|
2017
|
-
this.#registerHibernatableWebSocketAckTestHooks(
|
|
2018
|
-
websocket,
|
|
2019
|
-
gatewayIdBuf,
|
|
2020
|
-
requestIdBuf,
|
|
2021
|
-
remoteAckHookToken,
|
|
2022
|
-
);
|
|
2023
|
-
}
|
|
2024
|
-
|
|
2025
|
-
if (isRestoringHibernatable) {
|
|
2026
|
-
wsHandler.onRestore?.(wsContext);
|
|
2027
|
-
}
|
|
2028
|
-
|
|
2029
|
-
const isRawWebSocketPath =
|
|
2030
|
-
requestPath === PATH_WEBSOCKET_BASE ||
|
|
2031
|
-
requestPath.startsWith(PATH_WEBSOCKET_PREFIX);
|
|
2032
|
-
const handleMessageEvent = (event: RivetMessageEvent) => {
|
|
2033
|
-
if (
|
|
2034
|
-
isHibernatable &&
|
|
2035
|
-
this.#maybeRespondToHibernatableAckStateProbe(
|
|
2036
|
-
websocket,
|
|
2037
|
-
event.data,
|
|
2038
|
-
gatewayIdBuf,
|
|
2039
|
-
requestIdBuf,
|
|
2040
|
-
)
|
|
2041
|
-
) {
|
|
2042
|
-
return;
|
|
2043
|
-
}
|
|
2044
|
-
|
|
2045
|
-
const currentActor = this.#actors.get(actorId)?.actor;
|
|
2046
|
-
const actorForDispatch =
|
|
2047
|
-
currentActor && isStaticActorInstance(currentActor)
|
|
2048
|
-
? currentActor
|
|
2049
|
-
: actor;
|
|
2050
|
-
const connForDispatch =
|
|
2051
|
-
isHibernatable && actorForDispatch
|
|
2052
|
-
? (actorForDispatch.connectionManager.findHibernatableConn(
|
|
2053
|
-
gatewayIdBuf,
|
|
2054
|
-
requestIdBuf,
|
|
2055
|
-
) ?? conn)
|
|
2056
|
-
: conn;
|
|
2057
|
-
|
|
2058
|
-
if (actorForDispatch?.isStopping) {
|
|
2059
|
-
logger().debug({
|
|
2060
|
-
msg: "ignoring ws message, actor is stopping",
|
|
2061
|
-
connId: connForDispatch?.id,
|
|
2062
|
-
actorId: actorForDispatch?.id,
|
|
2063
|
-
messageIndex: event.rivetMessageIndex,
|
|
2064
|
-
});
|
|
2065
|
-
if (
|
|
2066
|
-
!isRawWebSocketPath &&
|
|
2067
|
-
websocket.readyState !== websocket.CLOSED
|
|
2068
|
-
) {
|
|
2069
|
-
websocket.close(1011, "actor.stopping");
|
|
2070
|
-
}
|
|
2071
|
-
return;
|
|
2072
|
-
}
|
|
2073
|
-
|
|
2074
|
-
const run = async () => {
|
|
2075
|
-
// Process message
|
|
2076
|
-
if (
|
|
2077
|
-
isHibernatable &&
|
|
2078
|
-
typeof event.rivetMessageIndex === "number"
|
|
2079
|
-
) {
|
|
2080
|
-
this.#recordInboundHibernatableWebSocketMessage(
|
|
2081
|
-
gatewayIdBuf,
|
|
2082
|
-
requestIdBuf,
|
|
2083
|
-
event.rivetMessageIndex,
|
|
2084
|
-
);
|
|
2085
|
-
}
|
|
2086
|
-
wsHandler.onMessage(event, wsContext);
|
|
2087
|
-
|
|
2088
|
-
// Runtime-owned hibernatable websocket bookkeeping lives on the
|
|
2089
|
-
// actor instance so static and dynamic paths share the same logic.
|
|
2090
|
-
if (connForDispatch && actorForDispatch) {
|
|
2091
|
-
actorForDispatch.handleInboundHibernatableWebSocketMessage(
|
|
2092
|
-
connForDispatch,
|
|
2093
|
-
event.data,
|
|
2094
|
-
event.rivetMessageIndex,
|
|
2095
|
-
);
|
|
2096
|
-
}
|
|
2097
|
-
};
|
|
2098
|
-
|
|
2099
|
-
if (isRawWebSocketPath && actorForDispatch) {
|
|
2100
|
-
void actorForDispatch.internalKeepAwake(run);
|
|
2101
|
-
} else {
|
|
2102
|
-
void run();
|
|
2103
|
-
}
|
|
2104
|
-
};
|
|
2105
|
-
const attachMessageListener = () => {
|
|
2106
|
-
websocket.addEventListener("message", handleMessageEvent);
|
|
2107
|
-
};
|
|
2108
|
-
let postOpenListenersAttached = false;
|
|
2109
|
-
const attachPostOpenListeners = () => {
|
|
2110
|
-
if (postOpenListenersAttached) {
|
|
2111
|
-
return;
|
|
2112
|
-
}
|
|
2113
|
-
postOpenListenersAttached = true;
|
|
2114
|
-
|
|
2115
|
-
if (!isRawWebSocketPath) {
|
|
2116
|
-
attachMessageListener();
|
|
2117
|
-
}
|
|
2118
|
-
|
|
2119
|
-
websocket.addEventListener("close", (event) => {
|
|
2120
|
-
if (isRawWebSocketPath && actor) {
|
|
2121
|
-
void actor.internalKeepAwake(async () => {
|
|
2122
|
-
await Promise.resolve();
|
|
2123
|
-
wsHandler.onClose(event, wsContext);
|
|
2124
|
-
});
|
|
2125
|
-
} else {
|
|
2126
|
-
wsHandler.onClose(event, wsContext);
|
|
2127
|
-
}
|
|
2128
|
-
if (isHibernatable) {
|
|
2129
|
-
this.#deleteHibernatableWebSocketAckState(
|
|
2130
|
-
gatewayIdBuf,
|
|
2131
|
-
requestIdBuf,
|
|
2132
|
-
);
|
|
2133
|
-
unregisterRemoteHibernatableWebSocketAckHooks(
|
|
2134
|
-
remoteAckHookToken,
|
|
2135
|
-
this.#config.test.enabled,
|
|
2136
|
-
);
|
|
2137
|
-
}
|
|
2138
|
-
});
|
|
2139
|
-
|
|
2140
|
-
websocket.addEventListener("error", (event) => {
|
|
2141
|
-
wsHandler.onError(event, wsContext);
|
|
2142
|
-
});
|
|
2143
|
-
};
|
|
2144
|
-
|
|
2145
|
-
websocket.addEventListener("open", (event) => {
|
|
2146
|
-
if (isRawWebSocketPath) {
|
|
2147
|
-
attachMessageListener();
|
|
2148
|
-
}
|
|
2149
|
-
|
|
2150
|
-
// Attach close and error listeners before onOpen so an actor that
|
|
2151
|
-
// immediately rejects the connection does not lose its close event.
|
|
2152
|
-
attachPostOpenListeners();
|
|
2153
|
-
|
|
2154
|
-
wsHandler.onOpen(event, wsContext);
|
|
2155
|
-
});
|
|
2156
|
-
|
|
2157
|
-
if (!isRawWebSocketPath) {
|
|
2158
|
-
attachPostOpenListeners();
|
|
2159
|
-
}
|
|
2160
|
-
}
|
|
2161
|
-
|
|
2162
|
-
async #runnerDynamicWebSocket(
|
|
2163
|
-
actorId: string,
|
|
2164
|
-
websocket: UniversalWebSocket,
|
|
2165
|
-
gatewayIdBuf: ArrayBuffer,
|
|
2166
|
-
requestIdBuf: ArrayBuffer,
|
|
2167
|
-
requestPath: string,
|
|
2168
|
-
requestHeaders: Record<string, string>,
|
|
2169
|
-
encoding: Encoding,
|
|
2170
|
-
connParams: unknown,
|
|
2171
|
-
isHibernatable: boolean,
|
|
2172
|
-
isRestoringHibernatable: boolean,
|
|
2173
|
-
): Promise<void> {
|
|
2174
|
-
let runtime: DynamicActorIsolateRuntime;
|
|
2175
|
-
const remoteAckHookToken =
|
|
2176
|
-
parseWebSocketProtocols(
|
|
2177
|
-
requestHeaders["sec-websocket-protocol"] ?? undefined,
|
|
2178
|
-
).ackHookToken ??
|
|
2179
|
-
new URL(`http://actor${requestPath}`).searchParams.get(
|
|
2180
|
-
REMOTE_ACK_HOOK_QUERY_PARAM,
|
|
2181
|
-
) ??
|
|
2182
|
-
undefined;
|
|
2183
|
-
if (isHibernatable) {
|
|
2184
|
-
this.#registerHibernatableWebSocketAckTestHooks(
|
|
2185
|
-
websocket,
|
|
2186
|
-
gatewayIdBuf,
|
|
2187
|
-
requestIdBuf,
|
|
2188
|
-
remoteAckHookToken,
|
|
2189
|
-
);
|
|
2190
|
-
try {
|
|
2191
|
-
await this.#bindDynamicHibernatableRunnerWebSocket(
|
|
2192
|
-
{
|
|
2193
|
-
actorId,
|
|
2194
|
-
websocket,
|
|
2195
|
-
requestPath,
|
|
2196
|
-
requestHeaders,
|
|
2197
|
-
encoding,
|
|
2198
|
-
connParams,
|
|
2199
|
-
gatewayId: gatewayIdBuf,
|
|
2200
|
-
requestId: requestIdBuf,
|
|
2201
|
-
remoteAckHookToken,
|
|
2202
|
-
},
|
|
2203
|
-
isRestoringHibernatable,
|
|
2204
|
-
);
|
|
2205
|
-
} catch (error) {
|
|
2206
|
-
const { group, code } = deconstructError(error, false);
|
|
2207
|
-
logger().error({
|
|
2208
|
-
msg: "failed to bind dynamic hibernatable websocket",
|
|
2209
|
-
actorId,
|
|
2210
|
-
error: stringifyError(error),
|
|
2211
|
-
});
|
|
2212
|
-
websocket.close(1011, `${group}.${code}`);
|
|
2213
|
-
}
|
|
2214
|
-
return;
|
|
2215
|
-
}
|
|
2216
|
-
|
|
2217
|
-
try {
|
|
2218
|
-
runtime = this.#requireDynamicRuntime(actorId);
|
|
2219
|
-
} catch (error) {
|
|
2220
|
-
logger().error({
|
|
2221
|
-
msg: "dynamic runtime missing for websocket",
|
|
2222
|
-
actorId,
|
|
2223
|
-
error: stringifyError(error),
|
|
2224
|
-
});
|
|
2225
|
-
websocket.close(1011, "dynamic.runtime_missing");
|
|
2226
|
-
return;
|
|
2227
|
-
}
|
|
2228
|
-
|
|
2229
|
-
let proxyToActorWs: UniversalWebSocket;
|
|
2230
|
-
try {
|
|
2231
|
-
proxyToActorWs = await runtime.openWebSocket(
|
|
2232
|
-
requestPath,
|
|
2233
|
-
encoding,
|
|
2234
|
-
connParams,
|
|
2235
|
-
{
|
|
2236
|
-
headers: requestHeaders,
|
|
2237
|
-
gatewayId: gatewayIdBuf,
|
|
2238
|
-
requestId: requestIdBuf,
|
|
2239
|
-
isHibernatable,
|
|
2240
|
-
isRestoringHibernatable,
|
|
2241
|
-
},
|
|
2242
|
-
);
|
|
2243
|
-
} catch (error) {
|
|
2244
|
-
const { group, code } = deconstructError(error, false);
|
|
2245
|
-
logger().error({
|
|
2246
|
-
msg: "failed to open dynamic websocket",
|
|
2247
|
-
actorId,
|
|
2248
|
-
error: stringifyError(error),
|
|
2249
|
-
});
|
|
2250
|
-
websocket.close(1011, `${group}.${code}`);
|
|
2251
|
-
return;
|
|
2252
|
-
}
|
|
2253
|
-
|
|
2254
|
-
proxyToActorWs.addEventListener(
|
|
2255
|
-
"message",
|
|
2256
|
-
(event: RivetMessageEvent) => {
|
|
2257
|
-
if (websocket.readyState !== websocket.OPEN) {
|
|
2258
|
-
return;
|
|
2259
|
-
}
|
|
2260
|
-
websocket.send(event.data as any);
|
|
2261
|
-
},
|
|
2262
|
-
);
|
|
2263
|
-
|
|
2264
|
-
proxyToActorWs.addEventListener("close", (event) => {
|
|
2265
|
-
if (isHibernatable && event.reason === "dynamic.runtime.disposed") {
|
|
2266
|
-
logger().debug({
|
|
2267
|
-
msg: "ignoring dynamic runtime dispose close for hibernatable websocket",
|
|
2268
|
-
actorId,
|
|
2269
|
-
code: event.code,
|
|
2270
|
-
reason: event.reason,
|
|
2271
|
-
});
|
|
2272
|
-
return;
|
|
2273
|
-
}
|
|
2274
|
-
if (websocket.readyState !== websocket.CLOSED) {
|
|
2275
|
-
websocket.close(event.code, event.reason);
|
|
2276
|
-
}
|
|
2277
|
-
});
|
|
2278
|
-
|
|
2279
|
-
proxyToActorWs.addEventListener("error", (_event) => {
|
|
2280
|
-
if (websocket.readyState !== websocket.CLOSED) {
|
|
2281
|
-
websocket.close(1011, "dynamic.websocket_error");
|
|
2282
|
-
}
|
|
2283
|
-
});
|
|
2284
|
-
|
|
2285
|
-
websocket.addEventListener("message", (event: RivetMessageEvent) => {
|
|
2286
|
-
if (
|
|
2287
|
-
isHibernatable &&
|
|
2288
|
-
this.#maybeRespondToHibernatableAckStateProbe(
|
|
2289
|
-
websocket,
|
|
2290
|
-
event.data,
|
|
2291
|
-
gatewayIdBuf,
|
|
2292
|
-
requestIdBuf,
|
|
2293
|
-
)
|
|
2294
|
-
) {
|
|
2295
|
-
return;
|
|
2296
|
-
}
|
|
2297
|
-
|
|
2298
|
-
const actorHandler = this.#actors.get(actorId);
|
|
2299
|
-
if (actorHandler?.actor?.isStopping) {
|
|
2300
|
-
return;
|
|
2301
|
-
}
|
|
2302
|
-
if (isHibernatable && typeof event.rivetMessageIndex === "number") {
|
|
2303
|
-
this.#recordInboundHibernatableWebSocketMessage(
|
|
2304
|
-
gatewayIdBuf,
|
|
2305
|
-
requestIdBuf,
|
|
2306
|
-
event.rivetMessageIndex,
|
|
2307
|
-
);
|
|
2308
|
-
}
|
|
2309
|
-
void runtime
|
|
2310
|
-
.forwardIncomingWebSocketMessage(
|
|
2311
|
-
proxyToActorWs,
|
|
2312
|
-
event.data as any,
|
|
2313
|
-
event.rivetMessageIndex,
|
|
2314
|
-
)
|
|
2315
|
-
.catch((error) => {
|
|
2316
|
-
logger().error({
|
|
2317
|
-
msg: "failed forwarding websocket message to dynamic actor",
|
|
2318
|
-
actorId,
|
|
2319
|
-
error: stringifyError(error),
|
|
2320
|
-
});
|
|
2321
|
-
websocket.close(1011, "dynamic.websocket_forward_failed");
|
|
2322
|
-
});
|
|
2323
|
-
});
|
|
2324
|
-
|
|
2325
|
-
websocket.addEventListener("close", (event) => {
|
|
2326
|
-
if (proxyToActorWs.readyState !== proxyToActorWs.CLOSED) {
|
|
2327
|
-
proxyToActorWs.close(event.code, event.reason);
|
|
2328
|
-
}
|
|
2329
|
-
});
|
|
2330
|
-
|
|
2331
|
-
websocket.addEventListener("error", () => {
|
|
2332
|
-
if (proxyToActorWs.readyState !== proxyToActorWs.CLOSED) {
|
|
2333
|
-
proxyToActorWs.close(1011, "dynamic.gateway_error");
|
|
2334
|
-
}
|
|
2335
|
-
});
|
|
2336
|
-
}
|
|
2337
|
-
|
|
2338
|
-
// MARK: - Hibernating WebSockets
|
|
2339
|
-
#hwsCanHibernate(
|
|
2340
|
-
actorId: string,
|
|
2341
|
-
gatewayId: ArrayBuffer,
|
|
2342
|
-
requestId: ArrayBuffer,
|
|
2343
|
-
request: Request,
|
|
2344
|
-
): boolean {
|
|
2345
|
-
const url = new URL(request.url);
|
|
2346
|
-
const path = url.pathname;
|
|
2347
|
-
|
|
2348
|
-
// Resolve actor name from either the envoy's actor view or the local
|
|
2349
|
-
// handler. WebSocket opens can race with actor startup, so the local
|
|
2350
|
-
// handler may know the actor name slightly earlier than the envoy.
|
|
2351
|
-
const actorInstance = this.#envoy.getActor(actorId);
|
|
2352
|
-
const handler = this.#actors.get(actorId);
|
|
2353
|
-
const actorName =
|
|
2354
|
-
actorInstance &&
|
|
2355
|
-
"config" in actorInstance &&
|
|
2356
|
-
actorInstance.config &&
|
|
2357
|
-
typeof actorInstance.config === "object" &&
|
|
2358
|
-
"name" in actorInstance.config &&
|
|
2359
|
-
typeof actorInstance.config.name === "string"
|
|
2360
|
-
? actorInstance.config.name
|
|
2361
|
-
: handler?.actorName;
|
|
2362
|
-
if (!actorName) {
|
|
2363
|
-
logger().warn({
|
|
2364
|
-
msg: "actor name unavailable in #hwsCanHibernate",
|
|
2365
|
-
actorId,
|
|
2366
|
-
});
|
|
2367
|
-
return false;
|
|
2368
|
-
}
|
|
2369
|
-
|
|
2370
|
-
// Determine configuration for new WS
|
|
2371
|
-
logger().debug({
|
|
2372
|
-
msg: "no existing hibernatable websocket found",
|
|
2373
|
-
gatewayId: Buffer.from(gatewayId).toString("hex"),
|
|
2374
|
-
requestId: Buffer.from(requestId).toString("hex"),
|
|
2375
|
-
});
|
|
2376
|
-
if (path === PATH_CONNECT) {
|
|
2377
|
-
return true;
|
|
2378
|
-
} else if (
|
|
2379
|
-
path === PATH_WEBSOCKET_BASE ||
|
|
2380
|
-
path.startsWith(PATH_WEBSOCKET_PREFIX)
|
|
2381
|
-
) {
|
|
2382
|
-
// Find actor config
|
|
2383
|
-
// Hibernation capability is a definition-level property, so the
|
|
2384
|
-
// envoy can decide it before the actor has fully started.
|
|
2385
|
-
const definition = lookupInRegistry(this.#config, actorName);
|
|
2386
|
-
|
|
2387
|
-
// Check if can hibernate
|
|
2388
|
-
const canHibernateWebSocket =
|
|
2389
|
-
definition.config.options?.canHibernateWebSocket;
|
|
2390
|
-
if (canHibernateWebSocket === true) {
|
|
2391
|
-
return true;
|
|
2392
|
-
} else if (typeof canHibernateWebSocket === "function") {
|
|
2393
|
-
try {
|
|
2394
|
-
// Truncate the path to match the behavior on onRawWebSocket
|
|
2395
|
-
const newPath = truncateRawWebSocketPathPrefix(
|
|
2396
|
-
url.pathname,
|
|
2397
|
-
);
|
|
2398
|
-
const truncatedRequest = new Request(
|
|
2399
|
-
`http://actor${newPath}`,
|
|
2400
|
-
request,
|
|
2401
|
-
);
|
|
2402
|
-
|
|
2403
|
-
const canHibernate =
|
|
2404
|
-
canHibernateWebSocket(truncatedRequest);
|
|
2405
|
-
return canHibernate;
|
|
2406
|
-
} catch (error) {
|
|
2407
|
-
logger().error({
|
|
2408
|
-
msg: "error calling canHibernateWebSocket",
|
|
2409
|
-
error,
|
|
2410
|
-
});
|
|
2411
|
-
return false;
|
|
2412
|
-
}
|
|
2413
|
-
} else {
|
|
2414
|
-
return false;
|
|
2415
|
-
}
|
|
2416
|
-
} else if (path === PATH_INSPECTOR_CONNECT) {
|
|
2417
|
-
return false;
|
|
2418
|
-
} else {
|
|
2419
|
-
logger().warn({
|
|
2420
|
-
msg: "unexpected path for getActorHibernationConfig",
|
|
2421
|
-
path,
|
|
2422
|
-
});
|
|
2423
|
-
return false;
|
|
2424
|
-
}
|
|
2425
|
-
}
|
|
2426
|
-
|
|
2427
|
-
async #hwsLoadAll(
|
|
2428
|
-
actorId: string,
|
|
2429
|
-
): Promise<HibernatingWebSocketMetadata[]> {
|
|
2430
|
-
const actor = await this.loadActor(actorId);
|
|
2431
|
-
if (!isStaticActorInstance(actor)) {
|
|
2432
|
-
const runtime = this.#dynamicRuntimes.get(actorId);
|
|
2433
|
-
if (!runtime) {
|
|
2434
|
-
return [];
|
|
2435
|
-
}
|
|
2436
|
-
const entries = await runtime.getHibernatingWebSockets();
|
|
2437
|
-
return entries.map((entry) => ({
|
|
2438
|
-
gatewayId: entry.gatewayId,
|
|
2439
|
-
requestId: entry.requestId,
|
|
2440
|
-
rivetMessageIndex: entry.serverMessageIndex,
|
|
2441
|
-
envoyMessageIndex: entry.clientMessageIndex,
|
|
2442
|
-
path: entry.path,
|
|
2443
|
-
headers: entry.headers,
|
|
2444
|
-
}));
|
|
2445
|
-
}
|
|
2446
|
-
return actor.getHibernatingWebSocketMetadata().map((entry) => ({
|
|
2447
|
-
gatewayId: entry.gatewayId,
|
|
2448
|
-
requestId: entry.requestId,
|
|
2449
|
-
rivetMessageIndex: entry.serverMessageIndex,
|
|
2450
|
-
envoyMessageIndex: entry.clientMessageIndex,
|
|
2451
|
-
path: entry.path,
|
|
2452
|
-
headers: entry.headers,
|
|
2453
|
-
}));
|
|
2454
|
-
}
|
|
2455
|
-
|
|
2456
|
-
async onBeforeActorStart(actor: AnyStaticActorInstance): Promise<void> {
|
|
2457
|
-
// Resolve promise if waiting.
|
|
2458
|
-
//
|
|
2459
|
-
// The websocket restore path needs to be able to load the actor while
|
|
2460
|
-
// rebinding persisted sockets, so this promise cannot wait on restore.
|
|
2461
|
-
const handler = this.#actors.get(actor.id);
|
|
2462
|
-
invariant(handler, "missing actor handler in onBeforeActorReady");
|
|
2463
|
-
handler.actorStartError = undefined;
|
|
2464
|
-
handler.actorStartPromise?.resolve();
|
|
2465
|
-
handler.actorStartPromise = undefined;
|
|
2466
|
-
|
|
2467
|
-
await this.#rebindHibernatableConnectSockets(actor.id);
|
|
2468
|
-
|
|
2469
|
-
// Restore hibernating requests
|
|
2470
|
-
const metaEntries = await this.#hwsLoadAll(actor.id);
|
|
2471
|
-
await this.#envoy.restoreHibernatingRequests(actor.id, metaEntries);
|
|
2472
|
-
}
|
|
2473
|
-
}
|