spooder 5.1.12 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +522 -566
- package/bun.lock +5 -34
- package/package.json +2 -4
- package/src/api.ts +656 -56
- package/src/cli.ts +201 -52
- package/src/config.ts +10 -5
- package/src/dispatch.ts +3 -0
- package/test.ts +10 -0
- package/src/api_db.ts +0 -732
package/src/api.ts
CHANGED
|
@@ -4,96 +4,505 @@ import path from 'node:path';
|
|
|
4
4
|
import fs from 'node:fs/promises';
|
|
5
5
|
import crypto from 'crypto';
|
|
6
6
|
import { Blob } from 'node:buffer';
|
|
7
|
-
import { ColorInput } from 'bun';
|
|
7
|
+
import { ColorInput, SQL } from 'bun';
|
|
8
8
|
import packageJson from '../package.json' with { type: 'json' };
|
|
9
9
|
|
|
10
|
-
// region
|
|
11
|
-
export
|
|
10
|
+
// region exit codes
|
|
11
|
+
export const EXIT_CODE = {
|
|
12
|
+
SUCCESS: 0,
|
|
13
|
+
GENERAL_ERROR: 1,
|
|
14
|
+
|
|
15
|
+
// 3-125 are free for application errors
|
|
16
|
+
SPOODER_AUTO_UPDATE: 42
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export const EXIT_CODE_NAMES = Object.fromEntries(
|
|
20
|
+
Object.entries(EXIT_CODE).map(([key, value]) => [value, key])
|
|
21
|
+
);
|
|
12
22
|
// endregion
|
|
13
23
|
|
|
14
24
|
// region workers
|
|
15
25
|
type WorkerMessageData = Record<string, any>;
|
|
16
|
-
type
|
|
17
|
-
|
|
26
|
+
type WorkerMessage = {
|
|
27
|
+
id: string;
|
|
28
|
+
peer: string;
|
|
29
|
+
data?: WorkerMessageData;
|
|
30
|
+
uuid: string;
|
|
31
|
+
response_to?: string;
|
|
18
32
|
};
|
|
19
33
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
34
|
+
const RESPONSE_TIMEOUT_MS = 5000;
|
|
35
|
+
|
|
36
|
+
export interface WorkerPool {
|
|
37
|
+
id: string;
|
|
38
|
+
send(peer: string, id: string, data?: WorkerMessageData, expect_response?: false): void;
|
|
39
|
+
send(peer: string, id: string, data: WorkerMessageData | undefined, expect_response: true): Promise<WorkerMessage>;
|
|
40
|
+
broadcast: (id: string, data?: WorkerMessageData) => void;
|
|
41
|
+
respond: (message: WorkerMessage, data?: WorkerMessageData) => void;
|
|
42
|
+
on: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => void;
|
|
43
|
+
once: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => void;
|
|
24
44
|
off: (event: string) => void;
|
|
25
45
|
}
|
|
26
46
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
throw new ErrorWithMetadata('invalid worker message type', { message });
|
|
30
|
-
|
|
31
|
-
if (typeof message.id !== 'string')
|
|
32
|
-
throw new Error('missing worker message .id');
|
|
33
|
-
}
|
|
47
|
+
export const WORKER_EXIT_NO_RESTART = 42;
|
|
48
|
+
const log_worker = log_create_logger('worker_pool', 'spooder');
|
|
34
49
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
type AutoRestartConfig = {
|
|
51
|
+
backoff_max?: number;
|
|
52
|
+
backoff_grace?: number;
|
|
53
|
+
max_attempts?: number;
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
type WorkerPoolOptions = {
|
|
57
|
+
id?: string;
|
|
58
|
+
worker: string | string[];
|
|
59
|
+
size?: number;
|
|
60
|
+
auto_restart?: boolean | AutoRestartConfig;
|
|
61
|
+
response_timeout?: number;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
type WorkerState = {
|
|
65
|
+
worker: Worker;
|
|
66
|
+
worker_id?: string;
|
|
67
|
+
restart_delay: number;
|
|
68
|
+
restart_attempts: number;
|
|
69
|
+
restart_success_timer: Timer | null;
|
|
70
|
+
worker_path: string;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
export async function worker_pool(options: WorkerPoolOptions): Promise<WorkerPool> {
|
|
74
|
+
const pipe_workers = new BiMap<string, Worker>();
|
|
75
|
+
const worker_promises = new WeakMap<Worker, (value: void | PromiseLike<void>) => void>();
|
|
76
|
+
|
|
77
|
+
const peer_id = options.id ?? 'main';
|
|
78
|
+
const response_timeout = options.response_timeout ?? RESPONSE_TIMEOUT_MS;
|
|
79
|
+
|
|
80
|
+
const auto_restart_enabled = options.auto_restart !== undefined && options.auto_restart !== false;
|
|
81
|
+
const auto_restart_config = typeof options.auto_restart === 'object' ? options.auto_restart : {};
|
|
82
|
+
const backoff_max = auto_restart_config.backoff_max ?? 5 * 60 * 1000; // 5 min
|
|
83
|
+
const backoff_grace = auto_restart_config.backoff_grace ?? 30000; // 30 seconds
|
|
84
|
+
const max_attempts = auto_restart_config.max_attempts ?? 5;
|
|
85
|
+
|
|
86
|
+
const worker_states = new WeakMap<Worker, WorkerState>();
|
|
87
|
+
|
|
88
|
+
const worker_paths: string[] = options.size !== undefined
|
|
89
|
+
? Array(options.size).fill(options.worker)
|
|
90
|
+
: Array.isArray(options.worker) ? options.worker : [options.worker];
|
|
91
|
+
|
|
92
|
+
log_worker(`created worker pool {${peer_id}}`);
|
|
93
|
+
|
|
94
|
+
const callbacks = new Map<string, (data: WorkerMessage) => Promise<void> | void>();
|
|
95
|
+
const pending_responses = new Map<string, { resolve: (message: WorkerMessage) => void, reject: (error: Error) => void, timeout: Timer | undefined }>();
|
|
96
|
+
|
|
97
|
+
async function restart_worker(worker: Worker) {
|
|
98
|
+
if (!auto_restart_enabled)
|
|
99
|
+
return;
|
|
100
|
+
|
|
101
|
+
const state = worker_states.get(worker);
|
|
102
|
+
if (!state)
|
|
103
|
+
return;
|
|
104
|
+
|
|
105
|
+
if (state.restart_success_timer) {
|
|
106
|
+
clearTimeout(state.restart_success_timer);
|
|
107
|
+
state.restart_success_timer = null;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (max_attempts !== -1 && state.restart_attempts >= max_attempts) {
|
|
111
|
+
log_worker(`worker {${state.worker_id ?? 'unknown'}} maximum restart attempts ({${max_attempts}}) reached, stopping auto-restart`);
|
|
112
|
+
return;
|
|
53
113
|
}
|
|
114
|
+
|
|
115
|
+
state.restart_attempts++;
|
|
116
|
+
const current_delay = Math.min(state.restart_delay, backoff_max);
|
|
117
|
+
const max_attempt_str = max_attempts === -1 ? '∞' : max_attempts;
|
|
118
|
+
|
|
119
|
+
log_worker(`restarting worker {${state.worker_id ?? 'unknown'}} in {${current_delay}ms} (attempt {${state.restart_attempts}}/{${max_attempt_str}}, delay capped at {${backoff_max}ms})`);
|
|
120
|
+
|
|
121
|
+
setTimeout(() => {
|
|
122
|
+
const new_worker = new Worker(state.worker_path);
|
|
123
|
+
|
|
124
|
+
state.worker = new_worker;
|
|
125
|
+
state.restart_delay = Math.min(state.restart_delay * 2, backoff_max);
|
|
126
|
+
|
|
127
|
+
state.restart_success_timer = setTimeout(() => {
|
|
128
|
+
state.restart_delay = 100;
|
|
129
|
+
state.restart_attempts = 0;
|
|
130
|
+
state.restart_success_timer = null;
|
|
131
|
+
}, backoff_grace);
|
|
132
|
+
|
|
133
|
+
worker_states.delete(worker);
|
|
134
|
+
worker_states.set(new_worker, state);
|
|
135
|
+
|
|
136
|
+
setup_worker_listeners(new_worker);
|
|
137
|
+
}, current_delay);
|
|
54
138
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
worker.
|
|
139
|
+
|
|
140
|
+
function setup_worker_listeners(worker: Worker) {
|
|
141
|
+
worker.addEventListener('message', (event) => {
|
|
142
|
+
handle_worker_message(worker, event);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
worker.addEventListener('close', (event: CloseEvent) => {
|
|
146
|
+
const worker_id = pipe_workers.getByValue(worker);
|
|
147
|
+
const exit_code = event.code;
|
|
148
|
+
log_worker(`worker {${worker_id ?? 'unknown'}} closed, exit code {${exit_code}}`);
|
|
149
|
+
|
|
150
|
+
if (worker_id)
|
|
151
|
+
pipe_workers.deleteByKey(worker_id);
|
|
152
|
+
|
|
153
|
+
if (auto_restart_enabled && exit_code !== WORKER_EXIT_NO_RESTART)
|
|
154
|
+
restart_worker(worker);
|
|
155
|
+
else if (exit_code === WORKER_EXIT_NO_RESTART)
|
|
156
|
+
log_worker(`worker {${worker_id ?? 'unknown'}} exited with {WORKER_EXIT_NO_RESTART}, skipping auto-restart`);
|
|
157
|
+
});
|
|
62
158
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
159
|
+
|
|
160
|
+
function handle_worker_message(worker: Worker, event: MessageEvent) {
|
|
161
|
+
const message = event.data as WorkerMessage;
|
|
162
|
+
|
|
163
|
+
if (message.id === '__register__') {
|
|
164
|
+
const worker_id = message.data?.worker_id;
|
|
165
|
+
if (worker_id === undefined) {
|
|
166
|
+
log_error('cannot register worker without ID');
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
if (pipe_workers.hasKey(worker_id)) {
|
|
171
|
+
log_error(`worker ID {${worker_id}} already in-use`);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
pipe_workers.set(message.data?.worker_id, worker);
|
|
176
|
+
|
|
177
|
+
const state = worker_states.get(worker);
|
|
178
|
+
if (state)
|
|
179
|
+
state.worker_id = worker_id;
|
|
180
|
+
|
|
181
|
+
worker_promises.get(worker)?.();
|
|
182
|
+
worker_promises.delete(worker);
|
|
183
|
+
} else if (message.peer === '__broadcast__') {
|
|
184
|
+
const worker_id = pipe_workers.getByValue(worker);
|
|
185
|
+
if (worker_id === undefined)
|
|
186
|
+
return;
|
|
187
|
+
|
|
188
|
+
message.peer = worker_id;
|
|
189
|
+
callbacks.get(message.id)?.(message);
|
|
190
|
+
|
|
191
|
+
for (const target_worker of pipe_workers.values()) {
|
|
192
|
+
if (target_worker === worker)
|
|
193
|
+
continue;
|
|
194
|
+
|
|
195
|
+
target_worker.postMessage(message);
|
|
196
|
+
}
|
|
197
|
+
} else {
|
|
198
|
+
const target_peer = message.peer;
|
|
199
|
+
const target_worker = pipe_workers.getByKey(target_peer);
|
|
200
|
+
const worker_id = pipe_workers.getByValue(worker);
|
|
201
|
+
|
|
202
|
+
if (worker_id === undefined)
|
|
203
|
+
return;
|
|
204
|
+
|
|
205
|
+
message.peer = worker_id;
|
|
206
|
+
|
|
207
|
+
if (target_peer === peer_id) {
|
|
208
|
+
if (message.response_to && pending_responses.has(message.response_to)) {
|
|
209
|
+
const pending = pending_responses.get(message.response_to)!;
|
|
210
|
+
if (pending.timeout)
|
|
211
|
+
clearTimeout(pending.timeout);
|
|
212
|
+
|
|
213
|
+
pending_responses.delete(message.response_to);
|
|
214
|
+
pending.resolve(message);
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
callbacks.get(message.id)?.(message);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
target_worker?.postMessage(message);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const pool: WorkerPool = {
|
|
226
|
+
id: peer_id,
|
|
227
|
+
|
|
228
|
+
send(peer: string, id: string, data?: WorkerMessageData, expect_response?: boolean): any {
|
|
229
|
+
const message: WorkerMessage = { id, peer: peer_id, data, uuid: Bun.randomUUIDv7() };
|
|
230
|
+
|
|
231
|
+
if (expect_response) {
|
|
232
|
+
return new Promise<WorkerMessage>((resolve, reject) => {
|
|
233
|
+
let timeout: Timer | undefined;
|
|
234
|
+
|
|
235
|
+
if (response_timeout !== -1) {
|
|
236
|
+
timeout = setTimeout(() => {
|
|
237
|
+
pending_responses.delete(message.uuid);
|
|
238
|
+
reject(new Error(`Response timeout after ${response_timeout}ms`));
|
|
239
|
+
}, response_timeout);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
pending_responses.set(message.uuid, { resolve, reject, timeout });
|
|
243
|
+
|
|
244
|
+
const target_worker = pipe_workers.getByKey(peer);
|
|
245
|
+
target_worker?.postMessage(message);
|
|
246
|
+
});
|
|
247
|
+
} else {
|
|
248
|
+
const target_worker = pipe_workers.getByKey(peer);
|
|
249
|
+
target_worker?.postMessage(message);
|
|
250
|
+
}
|
|
67
251
|
},
|
|
68
|
-
|
|
69
|
-
|
|
252
|
+
|
|
253
|
+
broadcast: (id: string, data?: WorkerMessageData) => {
|
|
254
|
+
const message: WorkerMessage = { peer: peer_id, id, data, uuid: Bun.randomUUIDv7() };
|
|
255
|
+
|
|
256
|
+
for (const target_worker of pipe_workers.values())
|
|
257
|
+
target_worker.postMessage(message);
|
|
258
|
+
},
|
|
259
|
+
|
|
260
|
+
respond: (message: WorkerMessage, data?: WorkerMessageData) => {
|
|
261
|
+
const response: WorkerMessage = {
|
|
262
|
+
id: message.id,
|
|
263
|
+
peer: peer_id,
|
|
264
|
+
data,
|
|
265
|
+
uuid: Bun.randomUUIDv7(),
|
|
266
|
+
response_to: message.uuid
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
const target_worker = pipe_workers.getByKey(message.peer);
|
|
270
|
+
target_worker?.postMessage(response);
|
|
271
|
+
},
|
|
272
|
+
|
|
273
|
+
on: (event: string, callback: (data: WorkerMessage) => Promise<void> | void) => {
|
|
70
274
|
callbacks.set(event, callback);
|
|
71
275
|
},
|
|
72
|
-
|
|
276
|
+
|
|
73
277
|
off: (event: string) => {
|
|
74
278
|
callbacks.delete(event);
|
|
75
279
|
},
|
|
76
|
-
|
|
77
|
-
once: (event: string, callback: (data:
|
|
78
|
-
callbacks.set(event, async (data:
|
|
280
|
+
|
|
281
|
+
once: (event: string, callback: (data: WorkerMessage) => Promise<void> | void) => {
|
|
282
|
+
callbacks.set(event, async (data: WorkerMessage) => {
|
|
79
283
|
await callback(data);
|
|
80
284
|
callbacks.delete(event);
|
|
81
285
|
});
|
|
82
286
|
}
|
|
83
287
|
};
|
|
288
|
+
|
|
289
|
+
const promises = [];
|
|
290
|
+
for (const path of worker_paths) {
|
|
291
|
+
const worker = new Worker(path);
|
|
292
|
+
|
|
293
|
+
if (auto_restart_enabled) {
|
|
294
|
+
worker_states.set(worker, {
|
|
295
|
+
worker,
|
|
296
|
+
restart_delay: 100,
|
|
297
|
+
restart_attempts: 0,
|
|
298
|
+
restart_success_timer: null,
|
|
299
|
+
worker_path: path
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
setup_worker_listeners(worker);
|
|
304
|
+
|
|
305
|
+
promises.push(new Promise<void>(resolve => {
|
|
306
|
+
worker_promises.set(worker, resolve);
|
|
307
|
+
}));
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
await Promise.all(promises);
|
|
311
|
+
|
|
312
|
+
return pool;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
export function worker_connect(peer_id?: string, response_timeout?: number): WorkerPool {
|
|
316
|
+
const listeners = new Map<string, (message: WorkerMessage) => Promise<void> | void>();
|
|
317
|
+
const pending_responses = new Map<string, { resolve: (message: WorkerMessage) => void, reject: (error: Error) => void, timeout: Timer | undefined }>();
|
|
318
|
+
|
|
319
|
+
if (peer_id === undefined) {
|
|
320
|
+
// normally we would increment 'worker1', 'worker2' etc but we
|
|
321
|
+
// have no simple way of keeping track of global worker count.
|
|
322
|
+
|
|
323
|
+
// in normal circumstances, users should provide an ID via the
|
|
324
|
+
// parameter, this is just a sensible fallback.
|
|
325
|
+
peer_id = 'worker-' + Bun.randomUUIDv7();
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
response_timeout = response_timeout ?? RESPONSE_TIMEOUT_MS;
|
|
329
|
+
|
|
330
|
+
log_worker(`worker {${peer_id}} connected to pool`);
|
|
331
|
+
|
|
332
|
+
const worker = globalThis as unknown as Worker;
|
|
333
|
+
worker.onmessage = event => {
|
|
334
|
+
const message = event.data as WorkerMessage;
|
|
335
|
+
|
|
336
|
+
if (message.response_to && pending_responses.has(message.response_to)) {
|
|
337
|
+
const pending = pending_responses.get(message.response_to)!;
|
|
338
|
+
if (pending.timeout)
|
|
339
|
+
clearTimeout(pending.timeout);
|
|
340
|
+
|
|
341
|
+
pending_responses.delete(message.response_to);
|
|
342
|
+
pending.resolve(message);
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
listeners.get(message.id)?.(message);
|
|
347
|
+
};
|
|
348
|
+
|
|
349
|
+
worker.postMessage({
|
|
350
|
+
id: '__register__',
|
|
351
|
+
data: {
|
|
352
|
+
worker_id: peer_id
|
|
353
|
+
}
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
return {
|
|
357
|
+
id: peer_id,
|
|
358
|
+
|
|
359
|
+
send(peer: string, id: string, data?: WorkerMessageData, expect_response?: boolean): any {
|
|
360
|
+
const message: WorkerMessage = { id, peer, data, uuid: Bun.randomUUIDv7() };
|
|
361
|
+
|
|
362
|
+
if (expect_response) {
|
|
363
|
+
return new Promise<WorkerMessage>((resolve, reject) => {
|
|
364
|
+
let timeout: Timer | undefined;
|
|
365
|
+
|
|
366
|
+
if (response_timeout !== -1) {
|
|
367
|
+
timeout = setTimeout(() => {
|
|
368
|
+
pending_responses.delete(message.uuid);
|
|
369
|
+
reject(new Error(`Response timeout after ${response_timeout}ms`));
|
|
370
|
+
}, response_timeout);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
pending_responses.set(message.uuid, { resolve, reject, timeout });
|
|
374
|
+
worker.postMessage(message);
|
|
375
|
+
});
|
|
376
|
+
} else {
|
|
377
|
+
worker.postMessage(message);
|
|
378
|
+
}
|
|
379
|
+
},
|
|
380
|
+
|
|
381
|
+
broadcast: (id: string, data?: WorkerMessageData) => {
|
|
382
|
+
const message: WorkerMessage = { peer: '__broadcast__', id, data, uuid: Bun.randomUUIDv7() };
|
|
383
|
+
worker.postMessage(message);
|
|
384
|
+
},
|
|
385
|
+
|
|
386
|
+
respond: (message: WorkerMessage, data?: WorkerMessageData) => {
|
|
387
|
+
const response: WorkerMessage = {
|
|
388
|
+
id: message.id,
|
|
389
|
+
peer: message.peer,
|
|
390
|
+
data,
|
|
391
|
+
uuid: Bun.randomUUIDv7(),
|
|
392
|
+
response_to: message.uuid
|
|
393
|
+
};
|
|
394
|
+
|
|
395
|
+
worker.postMessage(response);
|
|
396
|
+
},
|
|
397
|
+
|
|
398
|
+
on: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => {
|
|
399
|
+
listeners.set(event, callback);
|
|
400
|
+
},
|
|
401
|
+
|
|
402
|
+
off: (event: string) => {
|
|
403
|
+
listeners.delete(event);
|
|
404
|
+
},
|
|
405
|
+
|
|
406
|
+
once: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => {
|
|
407
|
+
listeners.set(event, async (message: WorkerMessage) => {
|
|
408
|
+
await callback(message);
|
|
409
|
+
listeners.delete(event);
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
};
|
|
84
413
|
}
|
|
414
|
+
|
|
85
415
|
// endregion
|
|
86
416
|
|
|
87
417
|
// region utility
|
|
418
|
+
export class BiMap<K, V> {
|
|
419
|
+
private ktv = new Map<K, V>();
|
|
420
|
+
private vtk = new Map<V, K>();
|
|
421
|
+
|
|
422
|
+
set(key: K, value: V): void {
|
|
423
|
+
const old_val = this.ktv.get(key);
|
|
424
|
+
if (old_val !== undefined)
|
|
425
|
+
this.vtk.delete(old_val);
|
|
426
|
+
|
|
427
|
+
const old_key = this.vtk.get(value);
|
|
428
|
+
if (old_key !== undefined)
|
|
429
|
+
this.ktv.delete(old_key);
|
|
430
|
+
|
|
431
|
+
this.ktv.set(key, value);
|
|
432
|
+
this.vtk.set(value, key);
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
getByKey(key: K): V | undefined {
|
|
436
|
+
return this.ktv.get(key);
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
getByValue(value: V): K | undefined {
|
|
440
|
+
return this.vtk.get(value);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
hasKey(key: K): boolean {
|
|
444
|
+
return this.ktv.has(key);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
hasValue(value: V): boolean {
|
|
448
|
+
return this.vtk.has(value);
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
deleteByKey(key: K): boolean {
|
|
452
|
+
const value = this.ktv.get(key);
|
|
453
|
+
if (value === undefined)
|
|
454
|
+
return false;
|
|
455
|
+
|
|
456
|
+
this.ktv.delete(key);
|
|
457
|
+
this.vtk.delete(value);
|
|
458
|
+
return true;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
deleteByValue(value: V): boolean {
|
|
462
|
+
const key = this.vtk.get(value);
|
|
463
|
+
if (key === undefined)
|
|
464
|
+
return false;
|
|
465
|
+
|
|
466
|
+
this.vtk.delete(value);
|
|
467
|
+
this.ktv.delete(key);
|
|
468
|
+
return true;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
clear(): void {
|
|
472
|
+
this.ktv.clear();
|
|
473
|
+
this.vtk.clear();
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
get size(): number {
|
|
477
|
+
return this.ktv.size;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
entries(): IterableIterator<[K, V]> {
|
|
481
|
+
return this.ktv.entries();
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
keys(): IterableIterator<K> {
|
|
485
|
+
return this.ktv.keys();
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
values(): IterableIterator<V> {
|
|
489
|
+
return this.ktv.values();
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
[Symbol.iterator](): IterableIterator<[K, V]> {
|
|
493
|
+
return this.ktv.entries();
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
88
497
|
const FILESIZE_UNITS = ['bytes', 'kb', 'mb', 'gb', 'tb'];
|
|
89
498
|
|
|
90
|
-
function filesize(bytes: number): string {
|
|
499
|
+
export function filesize(bytes: number): string {
|
|
91
500
|
if (bytes === 0)
|
|
92
501
|
return '0 bytes';
|
|
93
|
-
|
|
502
|
+
|
|
94
503
|
const i = Math.floor(Math.log(bytes) / Math.log(1024));
|
|
95
504
|
const size = bytes / Math.pow(1024, i);
|
|
96
|
-
|
|
505
|
+
|
|
97
506
|
return `${size.toFixed(i === 0 ? 0 : 1)} ${FILESIZE_UNITS[i]}`;
|
|
98
507
|
}
|
|
99
508
|
// endregion
|
|
@@ -106,8 +515,20 @@ export function log_create_logger(label: string, color: ColorInput = 'blue') {
|
|
|
106
515
|
const ansi = Bun.color(color, 'ansi-256') ?? '\x1b[38;5;6m';
|
|
107
516
|
const prefix = `[${ansi}${label}\x1b[0m] `;
|
|
108
517
|
|
|
109
|
-
return (
|
|
110
|
-
|
|
518
|
+
return (strings: TemplateStringsArray | string, ...values: any[]) => {
|
|
519
|
+
if (typeof strings === 'string') {
|
|
520
|
+
// regular string with { } syntax
|
|
521
|
+
console.log(prefix + strings.replace(/\{([^}]+)\}/g, `${ansi}$1\x1b[0m`), ...values);
|
|
522
|
+
} else {
|
|
523
|
+
// tagged template literal
|
|
524
|
+
let message = '';
|
|
525
|
+
for (let i = 0; i < strings.length; i++) {
|
|
526
|
+
message += strings[i];
|
|
527
|
+
if (i < values.length)
|
|
528
|
+
message += `${ansi}${values[i]}\x1b[0m`;
|
|
529
|
+
}
|
|
530
|
+
console.log(prefix + message);
|
|
531
|
+
}
|
|
111
532
|
};
|
|
112
533
|
}
|
|
113
534
|
|
|
@@ -121,6 +542,71 @@ export const log_error = log_create_logger('error', 'red');
|
|
|
121
542
|
|
|
122
543
|
// endregion
|
|
123
544
|
|
|
545
|
+
// region spooder ipc
|
|
546
|
+
export const IPC_OP = {
|
|
547
|
+
CMSG_TRIGGER_UPDATE: -1,
|
|
548
|
+
SMSG_UPDATE_READY: -2,
|
|
549
|
+
CMSG_REGISTER_LISTENER: -3,
|
|
550
|
+
};
|
|
551
|
+
|
|
552
|
+
// internal targets should always use __X__ as this format is
|
|
553
|
+
// reserved; userland instances cannot be named this way
|
|
554
|
+
export const IPC_TARGET = {
|
|
555
|
+
SPOODER: '__spooder__',
|
|
556
|
+
BROADCAST: '__broadcast__'
|
|
557
|
+
};
|
|
558
|
+
|
|
559
|
+
type IPC_Callback = (data: IPC_Message) => void;
|
|
560
|
+
type IPC_Message = {
|
|
561
|
+
op: number;
|
|
562
|
+
peer: string;
|
|
563
|
+
data?: object
|
|
564
|
+
};
|
|
565
|
+
|
|
566
|
+
let ipc_fail_announced = false;
|
|
567
|
+
let ipc_listener_attached = false;
|
|
568
|
+
|
|
569
|
+
const ipc_listeners = new Map<number, Set<IPC_Callback>>();
|
|
570
|
+
|
|
571
|
+
function ipc_on_message(payload: IPC_Message) {
|
|
572
|
+
const listeners = ipc_listeners.get(payload.op);
|
|
573
|
+
if (!listeners)
|
|
574
|
+
return;
|
|
575
|
+
|
|
576
|
+
for (const callback of listeners)
|
|
577
|
+
callback(payload);
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
export function ipc_send(peer: string, op: number, data?: object) {
|
|
581
|
+
if (!process.send) {
|
|
582
|
+
if (!ipc_fail_announced) {
|
|
583
|
+
log_spooder(`{ipc_send} failed, process not spawned with ipc channel`);
|
|
584
|
+
caution('ipc_send failed', { e: 'process not spawned with ipc channel' });
|
|
585
|
+
ipc_fail_announced = true;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
process.send({ peer, op, data });
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
export function ipc_register(op: number, callback: IPC_Callback) {
|
|
595
|
+
if (!ipc_listener_attached) {
|
|
596
|
+
process.on('message', ipc_on_message);
|
|
597
|
+
ipc_listener_attached = true;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
const listeners = ipc_listeners.get(op);
|
|
601
|
+
if (listeners)
|
|
602
|
+
listeners.add(callback);
|
|
603
|
+
else
|
|
604
|
+
ipc_listeners.set(op, new Set([callback]));
|
|
605
|
+
|
|
606
|
+
ipc_send(IPC_TARGET.SPOODER, IPC_OP.CMSG_REGISTER_LISTENER, { op });
|
|
607
|
+
}
|
|
608
|
+
// endregion
|
|
609
|
+
|
|
124
610
|
// region cache
|
|
125
611
|
type CacheOptions = {
|
|
126
612
|
ttl?: number;
|
|
@@ -1624,4 +2110,118 @@ export function http_serve(port: number, hostname?: string) {
|
|
|
1624
2110
|
}
|
|
1625
2111
|
};
|
|
1626
2112
|
}
|
|
2113
|
+
// endregion
|
|
2114
|
+
|
|
2115
|
+
// region db
|
|
2116
|
+
type SchemaOptions = {
|
|
2117
|
+
schema_table: string;
|
|
2118
|
+
recursive: boolean;
|
|
2119
|
+
throw_on_skip: boolean;
|
|
2120
|
+
};
|
|
2121
|
+
|
|
2122
|
+
type TableRevision = {
|
|
2123
|
+
revision_number: number;
|
|
2124
|
+
file_path: string;
|
|
2125
|
+
filename: string;
|
|
2126
|
+
};
|
|
2127
|
+
|
|
2128
|
+
const db_log = log_create_logger('db', 'spooder');
|
|
2129
|
+
|
|
2130
|
+
export function db_set_cast<T extends string>(set: string | null): Set<T> {
|
|
2131
|
+
return new Set(set?.split(',') as T[] ?? []);
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
export function db_set_serialize<T extends string>(set: Iterable<T> | null): string {
|
|
2135
|
+
return set ? Array.from(set).join(',') : '';
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
export async function db_get_schema_revision(db: SQL): Promise<number|null> {
|
|
2139
|
+
try {
|
|
2140
|
+
const [result] = await db`SELECT MAX(revision_number) as latest_revision FROM db_schema`;
|
|
2141
|
+
return result.latest_revision ?? 0;
|
|
2142
|
+
} catch (e) {
|
|
2143
|
+
return null;
|
|
2144
|
+
}
|
|
2145
|
+
}
|
|
2146
|
+
|
|
2147
|
+
export async function db_schema(db: SQL, schema_path: string, options?: SchemaOptions): Promise<boolean> {
|
|
2148
|
+
const schema_table = options?.schema_table ?? 'db_schema';
|
|
2149
|
+
const recursive = options?.recursive ?? true;
|
|
2150
|
+
|
|
2151
|
+
db_log`applying schema revisions from ${schema_path}`;
|
|
2152
|
+
let current_revision = await db_get_schema_revision(db);
|
|
2153
|
+
|
|
2154
|
+
if (current_revision === null) {
|
|
2155
|
+
db_log`initiating schema database table ${schema_table}`;
|
|
2156
|
+
await db`CREATE TABLE ${db(schema_table)} (
|
|
2157
|
+
revision_number INTEGER PRIMARY KEY,
|
|
2158
|
+
filename VARCHAR(255) NOT NULL,
|
|
2159
|
+
applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
2160
|
+
);`;
|
|
2161
|
+
}
|
|
2162
|
+
|
|
2163
|
+
current_revision ??= 0;
|
|
2164
|
+
|
|
2165
|
+
const revisions = Array<TableRevision>();
|
|
2166
|
+
const files = await fs.readdir(schema_path, { recursive, encoding: 'utf8' });
|
|
2167
|
+
for (const file of files) {
|
|
2168
|
+
const filename = path.basename(file);
|
|
2169
|
+
if (!filename.toLowerCase().endsWith('.sql'))
|
|
2170
|
+
continue;
|
|
2171
|
+
|
|
2172
|
+
const match = filename.match(/^(\d+)/);
|
|
2173
|
+
const revision_number = match ? Number(match[1]) : null;
|
|
2174
|
+
if (revision_number === null || revision_number < 1) {
|
|
2175
|
+
log_error`skipping sql file ${file}, invalid revision number`;
|
|
2176
|
+
continue;
|
|
2177
|
+
}
|
|
2178
|
+
|
|
2179
|
+
const file_path = path.join(schema_path, file);
|
|
2180
|
+
if (revision_number > current_revision) {
|
|
2181
|
+
revisions.push({
|
|
2182
|
+
revision_number,
|
|
2183
|
+
file_path,
|
|
2184
|
+
filename
|
|
2185
|
+
});
|
|
2186
|
+
}
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
// sort revisions in ascending order before applying
|
|
2190
|
+
// for recursive trees or unreliable OS sort ordering
|
|
2191
|
+
revisions.sort((a, b) => a.revision_number - b.revision_number);
|
|
2192
|
+
|
|
2193
|
+
const revisions_applied = Array<string>();
|
|
2194
|
+
for (const rev of revisions) {
|
|
2195
|
+
db_log`applying revision ${rev.revision_number} from ${rev.filename}`;
|
|
2196
|
+
|
|
2197
|
+
try {
|
|
2198
|
+
await db.begin(async tx => {
|
|
2199
|
+
await tx.file(rev.file_path);
|
|
2200
|
+
await tx`INSERT INTO ${db(schema_table)} ${db(rev, 'revision_number', 'filename')}`;
|
|
2201
|
+
revisions_applied.push(rev.filename);
|
|
2202
|
+
});
|
|
2203
|
+
} catch (err) {
|
|
2204
|
+
|
|
2205
|
+
log_error`failed to apply revisions from ${rev.filename}: ${err}`;
|
|
2206
|
+
log_error`${'warning'}: if ${rev.filename} contained DDL statements, they will ${'not'} be rolled back automatically`;
|
|
2207
|
+
log_error`verify the current database state ${'before'} running ammended revisions`;
|
|
2208
|
+
|
|
2209
|
+
const last_revision = await db_get_schema_revision(db);
|
|
2210
|
+
db_log`database schema revision is now ${last_revision ?? 0}`;
|
|
2211
|
+
|
|
2212
|
+
caution('db_schema failed', { rev, err, last_revision, revisions_applied });
|
|
2213
|
+
|
|
2214
|
+
return false;
|
|
2215
|
+
}
|
|
2216
|
+
}
|
|
2217
|
+
|
|
2218
|
+
if (revisions_applied.length > 0) {
|
|
2219
|
+
const new_revision = await db_get_schema_revision(db);
|
|
2220
|
+
db_log`applied ${revisions_applied.length} database schema revisions (${current_revision} >> ${new_revision})`;
|
|
2221
|
+
} else {
|
|
2222
|
+
db_log`no database schema revisions to apply (current: ${current_revision})`;
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
return true;
|
|
2226
|
+
}
|
|
1627
2227
|
// endregion
|