socket-function 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +50 -50
- package/SocketFunction.ts +280 -280
- package/SocketFunctionTypes.ts +90 -90
- package/hot/HotReloadController.ts +105 -105
- package/mobx/UrlParam.ts +39 -39
- package/mobx/observer.tsx +49 -49
- package/mobx/promiseToObservable.tsx +41 -41
- package/package.json +30 -28
- package/require/CSSShim.ts +19 -19
- package/require/RequireController.ts +252 -252
- package/require/buffer.js +2368 -2368
- package/require/compileFlags.ts +44 -44
- package/require/require.html +13 -13
- package/require/require.js +462 -456
- package/spec.txt +115 -115
- package/src/CallFactory.ts +389 -389
- package/src/JSONLACKS/JSONLACKS.generated.js +17 -17
- package/src/JSONLACKS/JSONLACKS.pegjs +247 -247
- package/src/JSONLACKS/JSONLACKS.ts +429 -375
- package/src/args.ts +21 -21
- package/src/batching.ts +170 -129
- package/src/caching.ts +318 -314
- package/src/callHTTPHandler.ts +203 -203
- package/src/callManager.ts +134 -134
- package/src/certStore.ts +29 -29
- package/src/fixLargeNetworkCalls.ts +8 -8
- package/src/formatting/colors.ts +78 -78
- package/src/formatting/format.ts +160 -156
- package/src/formatting/logColors.ts +17 -17
- package/src/misc.ts +302 -171
- package/src/nodeCache.ts +92 -92
- package/src/nodeProxy.ts +54 -54
- package/src/profiling/getOwnTime.ts +142 -142
- package/src/profiling/measure.ts +273 -244
- package/src/profiling/stats.ts +212 -212
- package/src/profiling/tcpLagProxy.ts +63 -63
- package/src/storagePath.ts +10 -10
- package/src/tlsParsing.ts +96 -96
- package/src/types.ts +8 -8
- package/src/webSocketServer.ts +250 -250
- package/test/client.css +2 -2
- package/test/client.ts +46 -46
- package/test/server.ts +43 -43
- package/test/shared.ts +52 -52
- package/tsconfig.json +26 -26
package/src/CallFactory.ts
CHANGED
|
@@ -1,390 +1,390 @@
|
|
|
1
|
-
import { CallerContext, CallerContextBase, CallType, FullCallType } from "../SocketFunctionTypes";
|
|
2
|
-
import * as ws from "ws";
|
|
3
|
-
import { performLocalCall } from "./callManager";
|
|
4
|
-
import { convertErrorStackToError, formatNumberSuffixed, isNode, list } from "./misc";
|
|
5
|
-
import { createWebsocketFactory, getTLSSocket } from "./websocketFactory";
|
|
6
|
-
import { SocketFunction } from "../SocketFunction";
|
|
7
|
-
import { gzip } from "zlib";
|
|
8
|
-
import * as tls from "tls";
|
|
9
|
-
import { getClientNodeId, getNodeIdLocation, registerNodeClient } from "./nodeCache";
|
|
10
|
-
import debugbreak from "debugbreak";
|
|
11
|
-
import { lazy } from "./caching";
|
|
12
|
-
import { JSONLACKS } from "./JSONLACKS/JSONLACKS";
|
|
13
|
-
import { red, yellow } from "./formatting/logColors";
|
|
14
|
-
import { isSplitableArray, markArrayAsSplitable } from "./fixLargeNetworkCalls";
|
|
15
|
-
import { delay } from "./batching";
|
|
16
|
-
|
|
17
|
-
const MIN_RETRY_DELAY = 1000;
|
|
18
|
-
|
|
19
|
-
type InternalCallType = FullCallType & {
|
|
20
|
-
seqNum: number;
|
|
21
|
-
isReturn: false;
|
|
22
|
-
compress: boolean;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
type InternalReturnType = {
|
|
26
|
-
isReturn: true;
|
|
27
|
-
result: unknown;
|
|
28
|
-
error?: string;
|
|
29
|
-
seqNum: number;
|
|
30
|
-
resultSize: number;
|
|
31
|
-
compressed: boolean;
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
export interface CallFactory {
|
|
36
|
-
nodeId: string;
|
|
37
|
-
lastClosed: number;
|
|
38
|
-
closedForever?: boolean;
|
|
39
|
-
isConnected?: boolean;
|
|
40
|
-
// NOTE: May or may not have reconnection or retry logic inside of performCall.
|
|
41
|
-
// Trigger performLocalCall on the other side of the connection
|
|
42
|
-
performCall(call: CallType): Promise<unknown>;
|
|
43
|
-
onNextDisconnect(callback: () => void): void;
|
|
44
|
-
connectionId: { nodeId: string };
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
export interface SenderInterface {
|
|
48
|
-
nodeId?: string;
|
|
49
|
-
// Only set AFTER "open" (if set at all, as in the browser we don't have access to the socket).
|
|
50
|
-
socket?: tls.TLSSocket;
|
|
51
|
-
|
|
52
|
-
send(data: string | Buffer): void;
|
|
53
|
-
|
|
54
|
-
addEventListener(event: "open", listener: () => void): void;
|
|
55
|
-
addEventListener(event: "close", listener: () => void): void;
|
|
56
|
-
addEventListener(event: "error", listener: (err: { message: string }) => void): void;
|
|
57
|
-
addEventListener(event: "message", listener: (data: ws.RawData | ws.MessageEvent | string) => void): void;
|
|
58
|
-
|
|
59
|
-
readyState: number;
|
|
60
|
-
|
|
61
|
-
ping?(): void;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export async function createCallFactory(
|
|
65
|
-
webSocketBase: SenderInterface | undefined,
|
|
66
|
-
// The node id we are connecting to (or that connected to us)
|
|
67
|
-
nodeId: string,
|
|
68
|
-
// The node id that we were contacted on
|
|
69
|
-
localNodeId = "",
|
|
70
|
-
): Promise<CallFactory> {
|
|
71
|
-
let niceConnectionName = nodeId;
|
|
72
|
-
|
|
73
|
-
const createWebsocket = createWebsocketFactory();
|
|
74
|
-
const registerOnce = lazy(() => registerNodeClient(callFactory));
|
|
75
|
-
|
|
76
|
-
const canReconnect = !!getNodeIdLocation(nodeId);
|
|
77
|
-
|
|
78
|
-
let pendingCalls: Map<number, {
|
|
79
|
-
data: Buffer;
|
|
80
|
-
call: InternalCallType;
|
|
81
|
-
callback: (resultJSON: InternalReturnType) => void;
|
|
82
|
-
}> = new Map();
|
|
83
|
-
// NOTE: It is important to make this as random as possible, to prevent
|
|
84
|
-
// reconnections dues to a process being reset causing seqNum collisions
|
|
85
|
-
// in return calls.
|
|
86
|
-
let nextSeqNum = Date.now() + Math.random();
|
|
87
|
-
|
|
88
|
-
// NOTE: I'm not sure if this is needed, I thought it was, but... now I think
|
|
89
|
-
// it probably isn't...
|
|
90
|
-
// if (webSocketBase?.readyState === 1 /* OPEN */ && webSocketBase.ping) {
|
|
91
|
-
// // Heartbeat loop, otherwise onDisconnect is never called.
|
|
92
|
-
// ((async () => {
|
|
93
|
-
// while (webSocketBase?.readyState === 1 /* OPEN */ && webSocketBase.ping) {
|
|
94
|
-
// await delay(1000 * 60);
|
|
95
|
-
// webSocketBase.ping?.();
|
|
96
|
-
// }
|
|
97
|
-
// }))().catch(() => { });
|
|
98
|
-
// }
|
|
99
|
-
|
|
100
|
-
let lastConnectionAttempt = 0;
|
|
101
|
-
|
|
102
|
-
let callerContext: CallerContextBase = {
|
|
103
|
-
nodeId,
|
|
104
|
-
localNodeId
|
|
105
|
-
};
|
|
106
|
-
|
|
107
|
-
let disconnectCallbacks: (() => void)[] = [];
|
|
108
|
-
function onNextDisconnect(callback: () => void): void {
|
|
109
|
-
disconnectCallbacks.push(callback);
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
let callFactory: CallFactory = {
|
|
113
|
-
nodeId,
|
|
114
|
-
lastClosed: 0,
|
|
115
|
-
connectionId: { nodeId },
|
|
116
|
-
onNextDisconnect,
|
|
117
|
-
async performCall(call: CallType) {
|
|
118
|
-
let seqNum = nextSeqNum++;
|
|
119
|
-
let fullCall: InternalCallType = {
|
|
120
|
-
nodeId,
|
|
121
|
-
isReturn: false,
|
|
122
|
-
args: call.args,
|
|
123
|
-
classGuid: call.classGuid,
|
|
124
|
-
functionName: call.functionName,
|
|
125
|
-
seqNum,
|
|
126
|
-
compress: !!SocketFunction.compression,
|
|
127
|
-
};
|
|
128
|
-
let time = Date.now();
|
|
129
|
-
let data = Buffer.from(JSONLACKS.stringify(fullCall));
|
|
130
|
-
time = Date.now() - time;
|
|
131
|
-
if (time > SocketFunction.WIRE_WARN_TIME) {
|
|
132
|
-
console.log(red(`Slow serialize, took ${time}ms to serialize ${data.byteLength} bytes. For ${call.classGuid}.${call.functionName}`));
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
if (data.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
|
|
136
|
-
let splitArgIndex = call.args.findIndex(isSplitableArray);
|
|
137
|
-
if (splitArgIndex >= 0) {
|
|
138
|
-
console.log(yellow(`Splitting large call due to large args: ${call.classGuid}.${call.functionName}`));
|
|
139
|
-
let SPLIT_GROUPS = 10;
|
|
140
|
-
let splitArg = call.args[splitArgIndex] as unknown[];
|
|
141
|
-
let subCalls = list(SPLIT_GROUPS).map(index => {
|
|
142
|
-
let start = Math.floor(index / SPLIT_GROUPS * splitArg.length);
|
|
143
|
-
let end = Math.floor((index + 1) / SPLIT_GROUPS * splitArg.length);
|
|
144
|
-
return splitArg.slice(start, end);
|
|
145
|
-
}).filter(x => x.length > 0);
|
|
146
|
-
|
|
147
|
-
let calls = subCalls.map(async splitList => {
|
|
148
|
-
let subCall = { ...call };
|
|
149
|
-
subCall.args = subCall.args.slice();
|
|
150
|
-
subCall.args[splitArgIndex] = markArrayAsSplitable(splitList);
|
|
151
|
-
await callFactory.performCall(subCall);
|
|
152
|
-
});
|
|
153
|
-
await Promise.allSettled(calls);
|
|
154
|
-
await Promise.all(calls);
|
|
155
|
-
// Eh... we COULD return the array of results, but... then the result would sometimes be an array,
|
|
156
|
-
// some times not, so, it is better to return a string which will make it more clear why it sometimes varies.
|
|
157
|
-
return "CALLS_SPLIT_DUE_TO_LARGE_ARGS";
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${data.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
let resultPromise = new Promise((resolve, reject) => {
|
|
164
|
-
let callback = (result: InternalReturnType) => {
|
|
165
|
-
if (SocketFunction.logMessages) {
|
|
166
|
-
console.log(`SIZE\t${(formatNumberSuffixed(result.resultSize) + "B").padEnd(4, " ")}\t${call.classGuid}.${call.functionName}`);
|
|
167
|
-
}
|
|
168
|
-
pendingCalls.delete(seqNum);
|
|
169
|
-
if (result.error) {
|
|
170
|
-
reject(convertErrorStackToError(result.error));
|
|
171
|
-
} else {
|
|
172
|
-
resolve(result.result);
|
|
173
|
-
}
|
|
174
|
-
};
|
|
175
|
-
pendingCalls.set(seqNum, { callback, data, call: fullCall });
|
|
176
|
-
});
|
|
177
|
-
|
|
178
|
-
await send(data);
|
|
179
|
-
|
|
180
|
-
return await resultPromise;
|
|
181
|
-
}
|
|
182
|
-
};
|
|
183
|
-
|
|
184
|
-
let webSocketPromise: Promise<SenderInterface> | undefined;
|
|
185
|
-
if (webSocketBase) {
|
|
186
|
-
webSocketPromise = Promise.resolve(webSocketBase);
|
|
187
|
-
await initializeWebsocket(webSocketBase);
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
async function initializeWebsocket(newWebSocket: SenderInterface) {
|
|
191
|
-
registerOnce();
|
|
192
|
-
|
|
193
|
-
function onClose(error: string) {
|
|
194
|
-
callFactory.connectionId = { nodeId };
|
|
195
|
-
callFactory.lastClosed = Date.now();
|
|
196
|
-
webSocketPromise = undefined;
|
|
197
|
-
if (!canReconnect) {
|
|
198
|
-
callFactory.closedForever = true;
|
|
199
|
-
}
|
|
200
|
-
for (let [key, call] of pendingCalls) {
|
|
201
|
-
pendingCalls.delete(key);
|
|
202
|
-
call.callback({
|
|
203
|
-
isReturn: true,
|
|
204
|
-
result: undefined,
|
|
205
|
-
error: error,
|
|
206
|
-
seqNum: call.call.seqNum,
|
|
207
|
-
resultSize: 0,
|
|
208
|
-
compressed: false,
|
|
209
|
-
});
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
let callbacks = disconnectCallbacks;
|
|
213
|
-
disconnectCallbacks = [];
|
|
214
|
-
for (let callback of callbacks) {
|
|
215
|
-
try {
|
|
216
|
-
callback();
|
|
217
|
-
} catch { }
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
newWebSocket.addEventListener("error", e => {
|
|
222
|
-
// NOTE: No more logging, as we throw, so the caller should be logging the
|
|
223
|
-
// error (or swallowing it, if that is what it wants to do).
|
|
224
|
-
//console.log(`Websocket error for ${niceConnectionName}`, e.message);
|
|
225
|
-
onClose(`Connection error for ${niceConnectionName}: ${e.message}`);
|
|
226
|
-
});
|
|
227
|
-
|
|
228
|
-
newWebSocket.addEventListener("close", async () => {
|
|
229
|
-
//console.log(`Websocket closed ${niceConnectionName}`);
|
|
230
|
-
onClose(`Connection closed to ${niceConnectionName}`);
|
|
231
|
-
});
|
|
232
|
-
|
|
233
|
-
newWebSocket.addEventListener("message", onMessage);
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
if (newWebSocket.readyState === 0 /* CONNECTING */) {
|
|
237
|
-
await new Promise<void>(resolve => {
|
|
238
|
-
newWebSocket.addEventListener("open", () => {
|
|
239
|
-
if (!SocketFunction.silent) {
|
|
240
|
-
console.log(`Connection established to ${niceConnectionName}`);
|
|
241
|
-
}
|
|
242
|
-
callFactory.isConnected = true;
|
|
243
|
-
resolve();
|
|
244
|
-
});
|
|
245
|
-
newWebSocket.addEventListener("close", () => resolve());
|
|
246
|
-
newWebSocket.addEventListener("error", () => resolve());
|
|
247
|
-
});
|
|
248
|
-
} else if (newWebSocket.readyState !== 1 /* OPEN */) {
|
|
249
|
-
onClose(`Websocket received in closed state`);
|
|
250
|
-
callFactory.isConnected = true;
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
async function send(data: Buffer) {
|
|
255
|
-
if (!webSocketPromise) {
|
|
256
|
-
if (canReconnect) {
|
|
257
|
-
webSocketPromise = tryToReconnect();
|
|
258
|
-
} else {
|
|
259
|
-
throw new Error(`Cannot send data to ${niceConnectionName} as the connection has closed`);
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
let webSocket = await webSocketPromise;
|
|
263
|
-
webSocket.send(data);
|
|
264
|
-
}
|
|
265
|
-
async function tryToReconnect(): Promise<SenderInterface> {
|
|
266
|
-
// Don't try to reconnect too often!
|
|
267
|
-
let timeSinceLastAttempt = Date.now() - lastConnectionAttempt;
|
|
268
|
-
if (timeSinceLastAttempt < MIN_RETRY_DELAY) {
|
|
269
|
-
await new Promise(r => setTimeout(r, MIN_RETRY_DELAY - timeSinceLastAttempt));
|
|
270
|
-
}
|
|
271
|
-
lastConnectionAttempt = Date.now();
|
|
272
|
-
|
|
273
|
-
let newWebSocket = createWebsocket(nodeId);
|
|
274
|
-
await initializeWebsocket(newWebSocket);
|
|
275
|
-
|
|
276
|
-
return newWebSocket;
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
async function onMessage(message: ws.RawData | ws.MessageEvent | string) {
|
|
281
|
-
try {
|
|
282
|
-
if (typeof message === "object" && "data" in message) {
|
|
283
|
-
message = message.data;
|
|
284
|
-
}
|
|
285
|
-
if (!isNode()) {
|
|
286
|
-
if (message instanceof Blob) {
|
|
287
|
-
message = Buffer.from(await message.arrayBuffer());
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
if (message instanceof Buffer || typeof message === "string") {
|
|
291
|
-
|
|
292
|
-
let resultSize = message.length;
|
|
293
|
-
|
|
294
|
-
if (message instanceof Buffer && message[0] === 0) {
|
|
295
|
-
// First byte of 0 means it is decompressed (as JSON can't have a first byte of 0).
|
|
296
|
-
(message as any) = message.slice(1);
|
|
297
|
-
|
|
298
|
-
// TODO: Add typings for DecompressionStream
|
|
299
|
-
let DecompressionStream = (window as any).DecompressionStream;
|
|
300
|
-
// https://stackoverflow.com/a/68829631/1117119
|
|
301
|
-
let stream = new DecompressionStream("gzip");
|
|
302
|
-
let blob = new Blob([message]);
|
|
303
|
-
let decompressedStream = (await (blob.stream() as any).pipeThrough(stream));
|
|
304
|
-
let arrayBuffer = await new Response(decompressedStream).arrayBuffer();
|
|
305
|
-
(message as any) = Buffer.from(arrayBuffer);
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
let time = Date.now();
|
|
309
|
-
let call = JSONLACKS.parse(message.toString(), { extended: false }) as InternalCallType | InternalReturnType;
|
|
310
|
-
time = Date.now() - time;
|
|
311
|
-
|
|
312
|
-
if (call.isReturn) {
|
|
313
|
-
let callbackObj = pendingCalls.get(call.seqNum);
|
|
314
|
-
if (time > SocketFunction.WIRE_WARN_TIME) {
|
|
315
|
-
console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for receieving result of call to ${callbackObj?.call.classGuid}.${callbackObj?.call.functionName}`));
|
|
316
|
-
}
|
|
317
|
-
if (!callbackObj) {
|
|
318
|
-
console.log(`Got return for unknown call ${call.seqNum}`);
|
|
319
|
-
return;
|
|
320
|
-
}
|
|
321
|
-
call.resultSize = resultSize;
|
|
322
|
-
callbackObj.callback(call);
|
|
323
|
-
} else {
|
|
324
|
-
if (time > SocketFunction.WIRE_WARN_TIME) {
|
|
325
|
-
console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for call to ${call.classGuid}.${call.functionName}`));
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
let response: InternalReturnType;
|
|
329
|
-
try {
|
|
330
|
-
let result = await performLocalCall({ call, caller: callerContext });
|
|
331
|
-
response = {
|
|
332
|
-
isReturn: true,
|
|
333
|
-
result,
|
|
334
|
-
seqNum: call.seqNum,
|
|
335
|
-
resultSize: resultSize,
|
|
336
|
-
compressed: false,
|
|
337
|
-
};
|
|
338
|
-
} catch (e: any) {
|
|
339
|
-
response = {
|
|
340
|
-
isReturn: true,
|
|
341
|
-
result: undefined,
|
|
342
|
-
seqNum: call.seqNum,
|
|
343
|
-
error: e.stack,
|
|
344
|
-
resultSize: resultSize,
|
|
345
|
-
compressed: false,
|
|
346
|
-
};
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
let result: Buffer;
|
|
350
|
-
if (isNode() && call.compress && SocketFunction.compression?.type === "gzip") {
|
|
351
|
-
response.compressed = true;
|
|
352
|
-
result = Buffer.from(JSONLACKS.stringify(response));
|
|
353
|
-
result = await new Promise<Buffer>((resolve, reject) =>
|
|
354
|
-
gzip(result, (err, result) => err ? reject(err) : resolve(result))
|
|
355
|
-
);
|
|
356
|
-
result = Buffer.concat([new Uint8Array([0]), result]);
|
|
357
|
-
} else {
|
|
358
|
-
result = Buffer.from(JSONLACKS.stringify(response));
|
|
359
|
-
}
|
|
360
|
-
if (result.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
|
|
361
|
-
response = {
|
|
362
|
-
isReturn: true,
|
|
363
|
-
result: undefined,
|
|
364
|
-
seqNum: call.seqNum,
|
|
365
|
-
error: new Error(`Response too large to send (${call.classGuid}.${call.functionName}, size: ${result.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`).stack,
|
|
366
|
-
resultSize: resultSize,
|
|
367
|
-
compressed: false,
|
|
368
|
-
};
|
|
369
|
-
result = Buffer.from(JSONLACKS.stringify(response));
|
|
370
|
-
}
|
|
371
|
-
await send(result);
|
|
372
|
-
}
|
|
373
|
-
return;
|
|
374
|
-
}
|
|
375
|
-
throw new Error(`Unhandled data type ${typeof message}`);
|
|
376
|
-
} catch (e: any) {
|
|
377
|
-
// NOTE: I'm looking for all types of errors here (specifically, .send errors), in case
|
|
378
|
-
// there are errors I should be handling.
|
|
379
|
-
if (e.stack.startsWith("Error: Cannot send data to") && e.stack.includes("as the connection has closed")) {
|
|
380
|
-
// This is fine, just ignore it
|
|
381
|
-
} else {
|
|
382
|
-
debugbreak(2);
|
|
383
|
-
debugger;
|
|
384
|
-
console.error(e.stack);
|
|
385
|
-
}
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
return callFactory;
|
|
1
|
+
import { CallerContext, CallerContextBase, CallType, FullCallType } from "../SocketFunctionTypes";
|
|
2
|
+
import * as ws from "ws";
|
|
3
|
+
import { performLocalCall } from "./callManager";
|
|
4
|
+
import { convertErrorStackToError, formatNumberSuffixed, isNode, list } from "./misc";
|
|
5
|
+
import { createWebsocketFactory, getTLSSocket } from "./websocketFactory";
|
|
6
|
+
import { SocketFunction } from "../SocketFunction";
|
|
7
|
+
import { gzip } from "zlib";
|
|
8
|
+
import * as tls from "tls";
|
|
9
|
+
import { getClientNodeId, getNodeIdLocation, registerNodeClient } from "./nodeCache";
|
|
10
|
+
import debugbreak from "debugbreak";
|
|
11
|
+
import { lazy } from "./caching";
|
|
12
|
+
import { JSONLACKS } from "./JSONLACKS/JSONLACKS";
|
|
13
|
+
import { red, yellow } from "./formatting/logColors";
|
|
14
|
+
import { isSplitableArray, markArrayAsSplitable } from "./fixLargeNetworkCalls";
|
|
15
|
+
import { delay } from "./batching";
|
|
16
|
+
|
|
17
|
+
const MIN_RETRY_DELAY = 1000;
|
|
18
|
+
|
|
19
|
+
type InternalCallType = FullCallType & {
|
|
20
|
+
seqNum: number;
|
|
21
|
+
isReturn: false;
|
|
22
|
+
compress: boolean;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
type InternalReturnType = {
|
|
26
|
+
isReturn: true;
|
|
27
|
+
result: unknown;
|
|
28
|
+
error?: string;
|
|
29
|
+
seqNum: number;
|
|
30
|
+
resultSize: number;
|
|
31
|
+
compressed: boolean;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
export interface CallFactory {
|
|
36
|
+
nodeId: string;
|
|
37
|
+
lastClosed: number;
|
|
38
|
+
closedForever?: boolean;
|
|
39
|
+
isConnected?: boolean;
|
|
40
|
+
// NOTE: May or may not have reconnection or retry logic inside of performCall.
|
|
41
|
+
// Trigger performLocalCall on the other side of the connection
|
|
42
|
+
performCall(call: CallType): Promise<unknown>;
|
|
43
|
+
onNextDisconnect(callback: () => void): void;
|
|
44
|
+
connectionId: { nodeId: string };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface SenderInterface {
|
|
48
|
+
nodeId?: string;
|
|
49
|
+
// Only set AFTER "open" (if set at all, as in the browser we don't have access to the socket).
|
|
50
|
+
socket?: tls.TLSSocket;
|
|
51
|
+
|
|
52
|
+
send(data: string | Buffer): void;
|
|
53
|
+
|
|
54
|
+
addEventListener(event: "open", listener: () => void): void;
|
|
55
|
+
addEventListener(event: "close", listener: () => void): void;
|
|
56
|
+
addEventListener(event: "error", listener: (err: { message: string }) => void): void;
|
|
57
|
+
addEventListener(event: "message", listener: (data: ws.RawData | ws.MessageEvent | string) => void): void;
|
|
58
|
+
|
|
59
|
+
readyState: number;
|
|
60
|
+
|
|
61
|
+
ping?(): void;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export async function createCallFactory(
|
|
65
|
+
webSocketBase: SenderInterface | undefined,
|
|
66
|
+
// The node id we are connecting to (or that connected to us)
|
|
67
|
+
nodeId: string,
|
|
68
|
+
// The node id that we were contacted on
|
|
69
|
+
localNodeId = "",
|
|
70
|
+
): Promise<CallFactory> {
|
|
71
|
+
let niceConnectionName = nodeId;
|
|
72
|
+
|
|
73
|
+
const createWebsocket = createWebsocketFactory();
|
|
74
|
+
const registerOnce = lazy(() => registerNodeClient(callFactory));
|
|
75
|
+
|
|
76
|
+
const canReconnect = !!getNodeIdLocation(nodeId);
|
|
77
|
+
|
|
78
|
+
let pendingCalls: Map<number, {
|
|
79
|
+
data: Buffer;
|
|
80
|
+
call: InternalCallType;
|
|
81
|
+
callback: (resultJSON: InternalReturnType) => void;
|
|
82
|
+
}> = new Map();
|
|
83
|
+
// NOTE: It is important to make this as random as possible, to prevent
|
|
84
|
+
// reconnections dues to a process being reset causing seqNum collisions
|
|
85
|
+
// in return calls.
|
|
86
|
+
let nextSeqNum = Date.now() + Math.random();
|
|
87
|
+
|
|
88
|
+
// NOTE: I'm not sure if this is needed, I thought it was, but... now I think
|
|
89
|
+
// it probably isn't...
|
|
90
|
+
// if (webSocketBase?.readyState === 1 /* OPEN */ && webSocketBase.ping) {
|
|
91
|
+
// // Heartbeat loop, otherwise onDisconnect is never called.
|
|
92
|
+
// ((async () => {
|
|
93
|
+
// while (webSocketBase?.readyState === 1 /* OPEN */ && webSocketBase.ping) {
|
|
94
|
+
// await delay(1000 * 60);
|
|
95
|
+
// webSocketBase.ping?.();
|
|
96
|
+
// }
|
|
97
|
+
// }))().catch(() => { });
|
|
98
|
+
// }
|
|
99
|
+
|
|
100
|
+
let lastConnectionAttempt = 0;
|
|
101
|
+
|
|
102
|
+
let callerContext: CallerContextBase = {
|
|
103
|
+
nodeId,
|
|
104
|
+
localNodeId
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
let disconnectCallbacks: (() => void)[] = [];
|
|
108
|
+
function onNextDisconnect(callback: () => void): void {
|
|
109
|
+
disconnectCallbacks.push(callback);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
let callFactory: CallFactory = {
|
|
113
|
+
nodeId,
|
|
114
|
+
lastClosed: 0,
|
|
115
|
+
connectionId: { nodeId },
|
|
116
|
+
onNextDisconnect,
|
|
117
|
+
async performCall(call: CallType) {
|
|
118
|
+
let seqNum = nextSeqNum++;
|
|
119
|
+
let fullCall: InternalCallType = {
|
|
120
|
+
nodeId,
|
|
121
|
+
isReturn: false,
|
|
122
|
+
args: call.args,
|
|
123
|
+
classGuid: call.classGuid,
|
|
124
|
+
functionName: call.functionName,
|
|
125
|
+
seqNum,
|
|
126
|
+
compress: !!SocketFunction.compression,
|
|
127
|
+
};
|
|
128
|
+
let time = Date.now();
|
|
129
|
+
let data = Buffer.from(JSONLACKS.stringify(fullCall));
|
|
130
|
+
time = Date.now() - time;
|
|
131
|
+
if (time > SocketFunction.WIRE_WARN_TIME) {
|
|
132
|
+
console.log(red(`Slow serialize, took ${time}ms to serialize ${data.byteLength} bytes. For ${call.classGuid}.${call.functionName}`));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (data.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
|
|
136
|
+
let splitArgIndex = call.args.findIndex(isSplitableArray);
|
|
137
|
+
if (splitArgIndex >= 0) {
|
|
138
|
+
console.log(yellow(`Splitting large call due to large args: ${call.classGuid}.${call.functionName}`));
|
|
139
|
+
let SPLIT_GROUPS = 10;
|
|
140
|
+
let splitArg = call.args[splitArgIndex] as unknown[];
|
|
141
|
+
let subCalls = list(SPLIT_GROUPS).map(index => {
|
|
142
|
+
let start = Math.floor(index / SPLIT_GROUPS * splitArg.length);
|
|
143
|
+
let end = Math.floor((index + 1) / SPLIT_GROUPS * splitArg.length);
|
|
144
|
+
return splitArg.slice(start, end);
|
|
145
|
+
}).filter(x => x.length > 0);
|
|
146
|
+
|
|
147
|
+
let calls = subCalls.map(async splitList => {
|
|
148
|
+
let subCall = { ...call };
|
|
149
|
+
subCall.args = subCall.args.slice();
|
|
150
|
+
subCall.args[splitArgIndex] = markArrayAsSplitable(splitList);
|
|
151
|
+
await callFactory.performCall(subCall);
|
|
152
|
+
});
|
|
153
|
+
await Promise.allSettled(calls);
|
|
154
|
+
await Promise.all(calls);
|
|
155
|
+
// Eh... we COULD return the array of results, but... then the result would sometimes be an array,
|
|
156
|
+
// some times not, so, it is better to return a string which will make it more clear why it sometimes varies.
|
|
157
|
+
return "CALLS_SPLIT_DUE_TO_LARGE_ARGS";
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
throw new Error(`Call too large to send (${call.classGuid}.${call.functionName}, size: ${data.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
let resultPromise = new Promise((resolve, reject) => {
|
|
164
|
+
let callback = (result: InternalReturnType) => {
|
|
165
|
+
if (SocketFunction.logMessages) {
|
|
166
|
+
console.log(`SIZE\t${(formatNumberSuffixed(result.resultSize) + "B").padEnd(4, " ")}\t${call.classGuid}.${call.functionName} at ${Date.now()}`);
|
|
167
|
+
}
|
|
168
|
+
pendingCalls.delete(seqNum);
|
|
169
|
+
if (result.error) {
|
|
170
|
+
reject(convertErrorStackToError(result.error));
|
|
171
|
+
} else {
|
|
172
|
+
resolve(result.result);
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
pendingCalls.set(seqNum, { callback, data, call: fullCall });
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
await send(data);
|
|
179
|
+
|
|
180
|
+
return await resultPromise;
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
let webSocketPromise: Promise<SenderInterface> | undefined;
|
|
185
|
+
if (webSocketBase) {
|
|
186
|
+
webSocketPromise = Promise.resolve(webSocketBase);
|
|
187
|
+
await initializeWebsocket(webSocketBase);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async function initializeWebsocket(newWebSocket: SenderInterface) {
|
|
191
|
+
registerOnce();
|
|
192
|
+
|
|
193
|
+
function onClose(error: string) {
|
|
194
|
+
callFactory.connectionId = { nodeId };
|
|
195
|
+
callFactory.lastClosed = Date.now();
|
|
196
|
+
webSocketPromise = undefined;
|
|
197
|
+
if (!canReconnect) {
|
|
198
|
+
callFactory.closedForever = true;
|
|
199
|
+
}
|
|
200
|
+
for (let [key, call] of pendingCalls) {
|
|
201
|
+
pendingCalls.delete(key);
|
|
202
|
+
call.callback({
|
|
203
|
+
isReturn: true,
|
|
204
|
+
result: undefined,
|
|
205
|
+
error: error,
|
|
206
|
+
seqNum: call.call.seqNum,
|
|
207
|
+
resultSize: 0,
|
|
208
|
+
compressed: false,
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
let callbacks = disconnectCallbacks;
|
|
213
|
+
disconnectCallbacks = [];
|
|
214
|
+
for (let callback of callbacks) {
|
|
215
|
+
try {
|
|
216
|
+
callback();
|
|
217
|
+
} catch { }
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
newWebSocket.addEventListener("error", e => {
|
|
222
|
+
// NOTE: No more logging, as we throw, so the caller should be logging the
|
|
223
|
+
// error (or swallowing it, if that is what it wants to do).
|
|
224
|
+
//console.log(`Websocket error for ${niceConnectionName}`, e.message);
|
|
225
|
+
onClose(`Connection error for ${niceConnectionName}: ${e.message}`);
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
newWebSocket.addEventListener("close", async () => {
|
|
229
|
+
//console.log(`Websocket closed ${niceConnectionName}`);
|
|
230
|
+
onClose(`Connection closed to ${niceConnectionName}`);
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
newWebSocket.addEventListener("message", onMessage);
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
if (newWebSocket.readyState === 0 /* CONNECTING */) {
|
|
237
|
+
await new Promise<void>(resolve => {
|
|
238
|
+
newWebSocket.addEventListener("open", () => {
|
|
239
|
+
if (!SocketFunction.silent) {
|
|
240
|
+
console.log(`Connection established to ${niceConnectionName}`);
|
|
241
|
+
}
|
|
242
|
+
callFactory.isConnected = true;
|
|
243
|
+
resolve();
|
|
244
|
+
});
|
|
245
|
+
newWebSocket.addEventListener("close", () => resolve());
|
|
246
|
+
newWebSocket.addEventListener("error", () => resolve());
|
|
247
|
+
});
|
|
248
|
+
} else if (newWebSocket.readyState !== 1 /* OPEN */) {
|
|
249
|
+
onClose(`Websocket received in closed state`);
|
|
250
|
+
callFactory.isConnected = true;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
async function send(data: Buffer) {
|
|
255
|
+
if (!webSocketPromise) {
|
|
256
|
+
if (canReconnect) {
|
|
257
|
+
webSocketPromise = tryToReconnect();
|
|
258
|
+
} else {
|
|
259
|
+
throw new Error(`Cannot send data to ${niceConnectionName} as the connection has closed`);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
let webSocket = await webSocketPromise;
|
|
263
|
+
webSocket.send(data);
|
|
264
|
+
}
|
|
265
|
+
async function tryToReconnect(): Promise<SenderInterface> {
|
|
266
|
+
// Don't try to reconnect too often!
|
|
267
|
+
let timeSinceLastAttempt = Date.now() - lastConnectionAttempt;
|
|
268
|
+
if (timeSinceLastAttempt < MIN_RETRY_DELAY) {
|
|
269
|
+
await new Promise(r => setTimeout(r, MIN_RETRY_DELAY - timeSinceLastAttempt));
|
|
270
|
+
}
|
|
271
|
+
lastConnectionAttempt = Date.now();
|
|
272
|
+
|
|
273
|
+
let newWebSocket = createWebsocket(nodeId);
|
|
274
|
+
await initializeWebsocket(newWebSocket);
|
|
275
|
+
|
|
276
|
+
return newWebSocket;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
async function onMessage(message: ws.RawData | ws.MessageEvent | string) {
|
|
281
|
+
try {
|
|
282
|
+
if (typeof message === "object" && "data" in message) {
|
|
283
|
+
message = message.data;
|
|
284
|
+
}
|
|
285
|
+
if (!isNode()) {
|
|
286
|
+
if (message instanceof Blob) {
|
|
287
|
+
message = Buffer.from(await message.arrayBuffer());
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
if (message instanceof Buffer || typeof message === "string") {
|
|
291
|
+
|
|
292
|
+
let resultSize = message.length;
|
|
293
|
+
|
|
294
|
+
if (message instanceof Buffer && message[0] === 0) {
|
|
295
|
+
// First byte of 0 means it is decompressed (as JSON can't have a first byte of 0).
|
|
296
|
+
(message as any) = message.slice(1);
|
|
297
|
+
|
|
298
|
+
// TODO: Add typings for DecompressionStream
|
|
299
|
+
let DecompressionStream = (window as any).DecompressionStream;
|
|
300
|
+
// https://stackoverflow.com/a/68829631/1117119
|
|
301
|
+
let stream = new DecompressionStream("gzip");
|
|
302
|
+
let blob = new Blob([message]);
|
|
303
|
+
let decompressedStream = (await (blob.stream() as any).pipeThrough(stream));
|
|
304
|
+
let arrayBuffer = await new Response(decompressedStream).arrayBuffer();
|
|
305
|
+
(message as any) = Buffer.from(arrayBuffer);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
let time = Date.now();
|
|
309
|
+
let call = JSONLACKS.parse(message.toString(), { extended: false }) as InternalCallType | InternalReturnType;
|
|
310
|
+
time = Date.now() - time;
|
|
311
|
+
|
|
312
|
+
if (call.isReturn) {
|
|
313
|
+
let callbackObj = pendingCalls.get(call.seqNum);
|
|
314
|
+
if (time > SocketFunction.WIRE_WARN_TIME) {
|
|
315
|
+
console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for receieving result of call to ${callbackObj?.call.classGuid}.${callbackObj?.call.functionName}`));
|
|
316
|
+
}
|
|
317
|
+
if (!callbackObj) {
|
|
318
|
+
console.log(`Got return for unknown call ${call.seqNum}`);
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
call.resultSize = resultSize;
|
|
322
|
+
callbackObj.callback(call);
|
|
323
|
+
} else {
|
|
324
|
+
if (time > SocketFunction.WIRE_WARN_TIME) {
|
|
325
|
+
console.log(red(`Slow parse, took ${time}ms to parse ${message.length} bytes, for call to ${call.classGuid}.${call.functionName}`));
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
let response: InternalReturnType;
|
|
329
|
+
try {
|
|
330
|
+
let result = await performLocalCall({ call, caller: callerContext });
|
|
331
|
+
response = {
|
|
332
|
+
isReturn: true,
|
|
333
|
+
result,
|
|
334
|
+
seqNum: call.seqNum,
|
|
335
|
+
resultSize: resultSize,
|
|
336
|
+
compressed: false,
|
|
337
|
+
};
|
|
338
|
+
} catch (e: any) {
|
|
339
|
+
response = {
|
|
340
|
+
isReturn: true,
|
|
341
|
+
result: undefined,
|
|
342
|
+
seqNum: call.seqNum,
|
|
343
|
+
error: e.stack,
|
|
344
|
+
resultSize: resultSize,
|
|
345
|
+
compressed: false,
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
let result: Buffer;
|
|
350
|
+
if (isNode() && call.compress && SocketFunction.compression?.type === "gzip") {
|
|
351
|
+
response.compressed = true;
|
|
352
|
+
result = Buffer.from(JSONLACKS.stringify(response));
|
|
353
|
+
result = await new Promise<Buffer>((resolve, reject) =>
|
|
354
|
+
gzip(result, (err, result) => err ? reject(err) : resolve(result))
|
|
355
|
+
);
|
|
356
|
+
result = Buffer.concat([new Uint8Array([0]), result]);
|
|
357
|
+
} else {
|
|
358
|
+
result = Buffer.from(JSONLACKS.stringify(response));
|
|
359
|
+
}
|
|
360
|
+
if (result.byteLength > SocketFunction.MAX_MESSAGE_SIZE * 1.5) {
|
|
361
|
+
response = {
|
|
362
|
+
isReturn: true,
|
|
363
|
+
result: undefined,
|
|
364
|
+
seqNum: call.seqNum,
|
|
365
|
+
error: new Error(`Response too large to send (${call.classGuid}.${call.functionName}, size: ${result.byteLength} > ${SocketFunction.MAX_MESSAGE_SIZE}). If you need to handle very large static data use some external service, such as Backblaze B2 or AWS S3. Or consider fragmenting data at an application level, because sending large data will cause large lag spikes for other clients using this server. Or, if absolutely required, set SocketFunction.MAX_MESSAGE_SIZE to a higher value.`).stack,
|
|
366
|
+
resultSize: resultSize,
|
|
367
|
+
compressed: false,
|
|
368
|
+
};
|
|
369
|
+
result = Buffer.from(JSONLACKS.stringify(response));
|
|
370
|
+
}
|
|
371
|
+
await send(result);
|
|
372
|
+
}
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
throw new Error(`Unhandled data type ${typeof message}`);
|
|
376
|
+
} catch (e: any) {
|
|
377
|
+
// NOTE: I'm looking for all types of errors here (specifically, .send errors), in case
|
|
378
|
+
// there are errors I should be handling.
|
|
379
|
+
if (e.stack.startsWith("Error: Cannot send data to") && e.stack.includes("as the connection has closed")) {
|
|
380
|
+
// This is fine, just ignore it
|
|
381
|
+
} else {
|
|
382
|
+
debugbreak(2);
|
|
383
|
+
debugger;
|
|
384
|
+
console.error(e.stack);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
return callFactory;
|
|
390
390
|
}
|