spooder 5.1.12 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/api.ts CHANGED
@@ -4,96 +4,505 @@ import path from 'node:path';
4
4
  import fs from 'node:fs/promises';
5
5
  import crypto from 'crypto';
6
6
  import { Blob } from 'node:buffer';
7
- import { ColorInput } from 'bun';
7
+ import { ColorInput, SQL } from 'bun';
8
8
  import packageJson from '../package.json' with { type: 'json' };
9
9
 
10
- // region api forwarding
11
- export * from './api_db';
10
+ // region exit codes
11
+ export const EXIT_CODE = {
12
+ SUCCESS: 0,
13
+ GENERAL_ERROR: 1,
14
+
15
+ // 3-125 are free for application errors
16
+ SPOODER_AUTO_UPDATE: 42
17
+ };
18
+
19
+ export const EXIT_CODE_NAMES = Object.fromEntries(
20
+ Object.entries(EXIT_CODE).map(([key, value]) => [value, key])
21
+ );
12
22
  // endregion
13
23
 
14
24
  // region workers
15
25
  type WorkerMessageData = Record<string, any>;
16
- type WorkerEventPipeOptions = {
17
- use_canary_reporting?: boolean;
26
+ type WorkerMessage = {
27
+ id: string;
28
+ peer: string;
29
+ data?: WorkerMessageData;
30
+ uuid: string;
31
+ response_to?: string;
18
32
  };
19
33
 
20
- export interface WorkerEventPipe {
21
- send: (id: string, data?: object) => void;
22
- on: (event: string, callback: (data: WorkerMessageData) => Promise<void> | void) => void;
23
- once: (event: string, callback: (data: WorkerMessageData) => Promise<void> | void) => void;
34
+ const RESPONSE_TIMEOUT_MS = 5000;
35
+
36
+ export interface WorkerPool {
37
+ id: string;
38
+ send(peer: string, id: string, data?: WorkerMessageData, expect_response?: false): void;
39
+ send(peer: string, id: string, data: WorkerMessageData | undefined, expect_response: true): Promise<WorkerMessage>;
40
+ broadcast: (id: string, data?: WorkerMessageData) => void;
41
+ respond: (message: WorkerMessage, data?: WorkerMessageData) => void;
42
+ on: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => void;
43
+ once: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => void;
24
44
  off: (event: string) => void;
25
45
  }
26
46
 
27
- function worker_validate_message(message: any) {
28
- if (typeof message !== 'object' || message === null)
29
- throw new ErrorWithMetadata('invalid worker message type', { message });
30
-
31
- if (typeof message.id !== 'string')
32
- throw new Error('missing worker message .id');
33
- }
47
+ export const WORKER_EXIT_NO_RESTART = 42;
48
+ const log_worker = log_create_logger('worker_pool', 'spooder');
34
49
 
35
- const log_worker = log_create_logger('worker', 'spooder');
36
- export function worker_event_pipe(worker: Worker, options?: WorkerEventPipeOptions): WorkerEventPipe {
37
- const use_canary_reporting = options?.use_canary_reporting ?? false;
38
- const callbacks = new Map<string, (data: Record<string, any>) => Promise<void> | void>();
39
-
40
- function handle_message(event: MessageEvent) {
41
- try {
42
- const message = JSON.parse(event.data);
43
- worker_validate_message(message);
44
-
45
- const callback = callbacks.get(message.id);
46
- if (callback !== undefined)
47
- callback(message.data ?? {});
48
- } catch (e) {
49
- log_error(`exception in worker: ${(e as Error).message}`);
50
-
51
- if (use_canary_reporting)
52
- caution('worker: exception handling payload', { exception: e });
50
+ type AutoRestartConfig = {
51
+ backoff_max?: number;
52
+ backoff_grace?: number;
53
+ max_attempts?: number;
54
+ };
55
+
56
+ type WorkerPoolOptions = {
57
+ id?: string;
58
+ worker: string | string[];
59
+ size?: number;
60
+ auto_restart?: boolean | AutoRestartConfig;
61
+ response_timeout?: number;
62
+ };
63
+
64
+ type WorkerState = {
65
+ worker: Worker;
66
+ worker_id?: string;
67
+ restart_delay: number;
68
+ restart_attempts: number;
69
+ restart_success_timer: Timer | null;
70
+ worker_path: string;
71
+ };
72
+
73
+ export async function worker_pool(options: WorkerPoolOptions): Promise<WorkerPool> {
74
+ const pipe_workers = new BiMap<string, Worker>();
75
+ const worker_promises = new WeakMap<Worker, (value: void | PromiseLike<void>) => void>();
76
+
77
+ const peer_id = options.id ?? 'main';
78
+ const response_timeout = options.response_timeout ?? RESPONSE_TIMEOUT_MS;
79
+
80
+ const auto_restart_enabled = options.auto_restart !== undefined && options.auto_restart !== false;
81
+ const auto_restart_config = typeof options.auto_restart === 'object' ? options.auto_restart : {};
82
+ const backoff_max = auto_restart_config.backoff_max ?? 5 * 60 * 1000; // 5 min
83
+ const backoff_grace = auto_restart_config.backoff_grace ?? 30000; // 30 seconds
84
+ const max_attempts = auto_restart_config.max_attempts ?? 5;
85
+
86
+ const worker_states = new WeakMap<Worker, WorkerState>();
87
+
88
+ const worker_paths: string[] = options.size !== undefined
89
+ ? Array(options.size).fill(options.worker)
90
+ : Array.isArray(options.worker) ? options.worker : [options.worker];
91
+
92
+ log_worker(`created worker pool {${peer_id}}`);
93
+
94
+ const callbacks = new Map<string, (data: WorkerMessage) => Promise<void> | void>();
95
+ const pending_responses = new Map<string, { resolve: (message: WorkerMessage) => void, reject: (error: Error) => void, timeout: Timer | undefined }>();
96
+
97
+ async function restart_worker(worker: Worker) {
98
+ if (!auto_restart_enabled)
99
+ return;
100
+
101
+ const state = worker_states.get(worker);
102
+ if (!state)
103
+ return;
104
+
105
+ if (state.restart_success_timer) {
106
+ clearTimeout(state.restart_success_timer);
107
+ state.restart_success_timer = null;
108
+ }
109
+
110
+ if (max_attempts !== -1 && state.restart_attempts >= max_attempts) {
111
+ log_worker(`worker {${state.worker_id ?? 'unknown'}} maximum restart attempts ({${max_attempts}}) reached, stopping auto-restart`);
112
+ return;
53
113
  }
114
+
115
+ state.restart_attempts++;
116
+ const current_delay = Math.min(state.restart_delay, backoff_max);
117
+ const max_attempt_str = max_attempts === -1 ? '∞' : max_attempts;
118
+
119
+ log_worker(`restarting worker {${state.worker_id ?? 'unknown'}} in {${current_delay}ms} (attempt {${state.restart_attempts}}/{${max_attempt_str}}, delay capped at {${backoff_max}ms})`);
120
+
121
+ setTimeout(() => {
122
+ const new_worker = new Worker(state.worker_path);
123
+
124
+ state.worker = new_worker;
125
+ state.restart_delay = Math.min(state.restart_delay * 2, backoff_max);
126
+
127
+ state.restart_success_timer = setTimeout(() => {
128
+ state.restart_delay = 100;
129
+ state.restart_attempts = 0;
130
+ state.restart_success_timer = null;
131
+ }, backoff_grace);
132
+
133
+ worker_states.delete(worker);
134
+ worker_states.set(new_worker, state);
135
+
136
+ setup_worker_listeners(new_worker);
137
+ }, current_delay);
54
138
  }
55
-
56
- if (Bun.isMainThread) {
57
- log_worker(`event pipe connected {main thread} ⇄ {worker}`);
58
- worker.addEventListener('message', handle_message);
59
- } else {
60
- log_worker(`event pipe connected {worker} ⇄ {main thread}`);
61
- worker.onmessage = handle_message;
139
+
140
+ function setup_worker_listeners(worker: Worker) {
141
+ worker.addEventListener('message', (event) => {
142
+ handle_worker_message(worker, event);
143
+ });
144
+
145
+ worker.addEventListener('close', (event: CloseEvent) => {
146
+ const worker_id = pipe_workers.getByValue(worker);
147
+ const exit_code = event.code;
148
+ log_worker(`worker {${worker_id ?? 'unknown'}} closed, exit code {${exit_code}}`);
149
+
150
+ if (worker_id)
151
+ pipe_workers.deleteByKey(worker_id);
152
+
153
+ if (auto_restart_enabled && exit_code !== WORKER_EXIT_NO_RESTART)
154
+ restart_worker(worker);
155
+ else if (exit_code === WORKER_EXIT_NO_RESTART)
156
+ log_worker(`worker {${worker_id ?? 'unknown'}} exited with {WORKER_EXIT_NO_RESTART}, skipping auto-restart`);
157
+ });
62
158
  }
63
-
64
- return {
65
- send: (id: string, data: object = {}) => {
66
- worker.postMessage(JSON.stringify({ id, data }));
159
+
160
+ function handle_worker_message(worker: Worker, event: MessageEvent) {
161
+ const message = event.data as WorkerMessage;
162
+
163
+ if (message.id === '__register__') {
164
+ const worker_id = message.data?.worker_id;
165
+ if (worker_id === undefined) {
166
+ log_error('cannot register worker without ID');
167
+ return;
168
+ }
169
+
170
+ if (pipe_workers.hasKey(worker_id)) {
171
+ log_error(`worker ID {${worker_id}} already in-use`);
172
+ return;
173
+ }
174
+
175
+ pipe_workers.set(message.data?.worker_id, worker);
176
+
177
+ const state = worker_states.get(worker);
178
+ if (state)
179
+ state.worker_id = worker_id;
180
+
181
+ worker_promises.get(worker)?.();
182
+ worker_promises.delete(worker);
183
+ } else if (message.peer === '__broadcast__') {
184
+ const worker_id = pipe_workers.getByValue(worker);
185
+ if (worker_id === undefined)
186
+ return;
187
+
188
+ message.peer = worker_id;
189
+ callbacks.get(message.id)?.(message);
190
+
191
+ for (const target_worker of pipe_workers.values()) {
192
+ if (target_worker === worker)
193
+ continue;
194
+
195
+ target_worker.postMessage(message);
196
+ }
197
+ } else {
198
+ const target_peer = message.peer;
199
+ const target_worker = pipe_workers.getByKey(target_peer);
200
+ const worker_id = pipe_workers.getByValue(worker);
201
+
202
+ if (worker_id === undefined)
203
+ return;
204
+
205
+ message.peer = worker_id;
206
+
207
+ if (target_peer === peer_id) {
208
+ if (message.response_to && pending_responses.has(message.response_to)) {
209
+ const pending = pending_responses.get(message.response_to)!;
210
+ if (pending.timeout)
211
+ clearTimeout(pending.timeout);
212
+
213
+ pending_responses.delete(message.response_to);
214
+ pending.resolve(message);
215
+ return;
216
+ }
217
+
218
+ callbacks.get(message.id)?.(message);
219
+ }
220
+
221
+ target_worker?.postMessage(message);
222
+ }
223
+ }
224
+
225
+ const pool: WorkerPool = {
226
+ id: peer_id,
227
+
228
+ send(peer: string, id: string, data?: WorkerMessageData, expect_response?: boolean): any {
229
+ const message: WorkerMessage = { id, peer: peer_id, data, uuid: Bun.randomUUIDv7() };
230
+
231
+ if (expect_response) {
232
+ return new Promise<WorkerMessage>((resolve, reject) => {
233
+ let timeout: Timer | undefined;
234
+
235
+ if (response_timeout !== -1) {
236
+ timeout = setTimeout(() => {
237
+ pending_responses.delete(message.uuid);
238
+ reject(new Error(`Response timeout after ${response_timeout}ms`));
239
+ }, response_timeout);
240
+ }
241
+
242
+ pending_responses.set(message.uuid, { resolve, reject, timeout });
243
+
244
+ const target_worker = pipe_workers.getByKey(peer);
245
+ target_worker?.postMessage(message);
246
+ });
247
+ } else {
248
+ const target_worker = pipe_workers.getByKey(peer);
249
+ target_worker?.postMessage(message);
250
+ }
67
251
  },
68
-
69
- on: (event: string, callback: (data: WorkerMessageData) => Promise<void> | void) => {
252
+
253
+ broadcast: (id: string, data?: WorkerMessageData) => {
254
+ const message: WorkerMessage = { peer: peer_id, id, data, uuid: Bun.randomUUIDv7() };
255
+
256
+ for (const target_worker of pipe_workers.values())
257
+ target_worker.postMessage(message);
258
+ },
259
+
260
+ respond: (message: WorkerMessage, data?: WorkerMessageData) => {
261
+ const response: WorkerMessage = {
262
+ id: message.id,
263
+ peer: peer_id,
264
+ data,
265
+ uuid: Bun.randomUUIDv7(),
266
+ response_to: message.uuid
267
+ };
268
+
269
+ const target_worker = pipe_workers.getByKey(message.peer);
270
+ target_worker?.postMessage(response);
271
+ },
272
+
273
+ on: (event: string, callback: (data: WorkerMessage) => Promise<void> | void) => {
70
274
  callbacks.set(event, callback);
71
275
  },
72
-
276
+
73
277
  off: (event: string) => {
74
278
  callbacks.delete(event);
75
279
  },
76
-
77
- once: (event: string, callback: (data: WorkerMessageData) => Promise<void> | void) => {
78
- callbacks.set(event, async (data: WorkerMessageData) => {
280
+
281
+ once: (event: string, callback: (data: WorkerMessage) => Promise<void> | void) => {
282
+ callbacks.set(event, async (data: WorkerMessage) => {
79
283
  await callback(data);
80
284
  callbacks.delete(event);
81
285
  });
82
286
  }
83
287
  };
288
+
289
+ const promises = [];
290
+ for (const path of worker_paths) {
291
+ const worker = new Worker(path);
292
+
293
+ if (auto_restart_enabled) {
294
+ worker_states.set(worker, {
295
+ worker,
296
+ restart_delay: 100,
297
+ restart_attempts: 0,
298
+ restart_success_timer: null,
299
+ worker_path: path
300
+ });
301
+ }
302
+
303
+ setup_worker_listeners(worker);
304
+
305
+ promises.push(new Promise<void>(resolve => {
306
+ worker_promises.set(worker, resolve);
307
+ }));
308
+ }
309
+
310
+ await Promise.all(promises);
311
+
312
+ return pool;
313
+ }
314
+
315
+ export function worker_connect(peer_id?: string, response_timeout?: number): WorkerPool {
316
+ const listeners = new Map<string, (message: WorkerMessage) => Promise<void> | void>();
317
+ const pending_responses = new Map<string, { resolve: (message: WorkerMessage) => void, reject: (error: Error) => void, timeout: Timer | undefined }>();
318
+
319
+ if (peer_id === undefined) {
320
+ // normally we would increment 'worker1', 'worker2' etc but we
321
+ // have no simple way of keeping track of global worker count.
322
+
323
+ // in normal circumstances, users should provide an ID via the
324
+ // parameter, this is just a sensible fallback.
325
+ peer_id = 'worker-' + Bun.randomUUIDv7();
326
+ }
327
+
328
+ response_timeout = response_timeout ?? RESPONSE_TIMEOUT_MS;
329
+
330
+ log_worker(`worker {${peer_id}} connected to pool`);
331
+
332
+ const worker = globalThis as unknown as Worker;
333
+ worker.onmessage = event => {
334
+ const message = event.data as WorkerMessage;
335
+
336
+ if (message.response_to && pending_responses.has(message.response_to)) {
337
+ const pending = pending_responses.get(message.response_to)!;
338
+ if (pending.timeout)
339
+ clearTimeout(pending.timeout);
340
+
341
+ pending_responses.delete(message.response_to);
342
+ pending.resolve(message);
343
+ return;
344
+ }
345
+
346
+ listeners.get(message.id)?.(message);
347
+ };
348
+
349
+ worker.postMessage({
350
+ id: '__register__',
351
+ data: {
352
+ worker_id: peer_id
353
+ }
354
+ });
355
+
356
+ return {
357
+ id: peer_id,
358
+
359
+ send(peer: string, id: string, data?: WorkerMessageData, expect_response?: boolean): any {
360
+ const message: WorkerMessage = { id, peer, data, uuid: Bun.randomUUIDv7() };
361
+
362
+ if (expect_response) {
363
+ return new Promise<WorkerMessage>((resolve, reject) => {
364
+ let timeout: Timer | undefined;
365
+
366
+ if (response_timeout !== -1) {
367
+ timeout = setTimeout(() => {
368
+ pending_responses.delete(message.uuid);
369
+ reject(new Error(`Response timeout after ${response_timeout}ms`));
370
+ }, response_timeout);
371
+ }
372
+
373
+ pending_responses.set(message.uuid, { resolve, reject, timeout });
374
+ worker.postMessage(message);
375
+ });
376
+ } else {
377
+ worker.postMessage(message);
378
+ }
379
+ },
380
+
381
+ broadcast: (id: string, data?: WorkerMessageData) => {
382
+ const message: WorkerMessage = { peer: '__broadcast__', id, data, uuid: Bun.randomUUIDv7() };
383
+ worker.postMessage(message);
384
+ },
385
+
386
+ respond: (message: WorkerMessage, data?: WorkerMessageData) => {
387
+ const response: WorkerMessage = {
388
+ id: message.id,
389
+ peer: message.peer,
390
+ data,
391
+ uuid: Bun.randomUUIDv7(),
392
+ response_to: message.uuid
393
+ };
394
+
395
+ worker.postMessage(response);
396
+ },
397
+
398
+ on: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => {
399
+ listeners.set(event, callback);
400
+ },
401
+
402
+ off: (event: string) => {
403
+ listeners.delete(event);
404
+ },
405
+
406
+ once: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => {
407
+ listeners.set(event, async (message: WorkerMessage) => {
408
+ await callback(message);
409
+ listeners.delete(event);
410
+ });
411
+ }
412
+ };
84
413
  }
414
+
85
415
  // endregion
86
416
 
87
417
  // region utility
418
+ export class BiMap<K, V> {
419
+ private ktv = new Map<K, V>();
420
+ private vtk = new Map<V, K>();
421
+
422
+ set(key: K, value: V): void {
423
+ const old_val = this.ktv.get(key);
424
+ if (old_val !== undefined)
425
+ this.vtk.delete(old_val);
426
+
427
+ const old_key = this.vtk.get(value);
428
+ if (old_key !== undefined)
429
+ this.ktv.delete(old_key);
430
+
431
+ this.ktv.set(key, value);
432
+ this.vtk.set(value, key);
433
+ }
434
+
435
+ getByKey(key: K): V | undefined {
436
+ return this.ktv.get(key);
437
+ }
438
+
439
+ getByValue(value: V): K | undefined {
440
+ return this.vtk.get(value);
441
+ }
442
+
443
+ hasKey(key: K): boolean {
444
+ return this.ktv.has(key);
445
+ }
446
+
447
+ hasValue(value: V): boolean {
448
+ return this.vtk.has(value);
449
+ }
450
+
451
+ deleteByKey(key: K): boolean {
452
+ const value = this.ktv.get(key);
453
+ if (value === undefined)
454
+ return false;
455
+
456
+ this.ktv.delete(key);
457
+ this.vtk.delete(value);
458
+ return true;
459
+ }
460
+
461
+ deleteByValue(value: V): boolean {
462
+ const key = this.vtk.get(value);
463
+ if (key === undefined)
464
+ return false;
465
+
466
+ this.vtk.delete(value);
467
+ this.ktv.delete(key);
468
+ return true;
469
+ }
470
+
471
+ clear(): void {
472
+ this.ktv.clear();
473
+ this.vtk.clear();
474
+ }
475
+
476
+ get size(): number {
477
+ return this.ktv.size;
478
+ }
479
+
480
+ entries(): IterableIterator<[K, V]> {
481
+ return this.ktv.entries();
482
+ }
483
+
484
+ keys(): IterableIterator<K> {
485
+ return this.ktv.keys();
486
+ }
487
+
488
+ values(): IterableIterator<V> {
489
+ return this.ktv.values();
490
+ }
491
+
492
+ [Symbol.iterator](): IterableIterator<[K, V]> {
493
+ return this.ktv.entries();
494
+ }
495
+ }
496
+
88
497
  const FILESIZE_UNITS = ['bytes', 'kb', 'mb', 'gb', 'tb'];
89
498
 
90
- function filesize(bytes: number): string {
499
+ export function filesize(bytes: number): string {
91
500
  if (bytes === 0)
92
501
  return '0 bytes';
93
-
502
+
94
503
  const i = Math.floor(Math.log(bytes) / Math.log(1024));
95
504
  const size = bytes / Math.pow(1024, i);
96
-
505
+
97
506
  return `${size.toFixed(i === 0 ? 0 : 1)} ${FILESIZE_UNITS[i]}`;
98
507
  }
99
508
  // endregion
@@ -106,8 +515,20 @@ export function log_create_logger(label: string, color: ColorInput = 'blue') {
106
515
  const ansi = Bun.color(color, 'ansi-256') ?? '\x1b[38;5;6m';
107
516
  const prefix = `[${ansi}${label}\x1b[0m] `;
108
517
 
109
- return (message: string, ...params: any[]) => {
110
- console.log(prefix + message.replace(/\{([^}]+)\}/g, `${ansi}$1\x1b[0m`), ...params);
518
+ return (strings: TemplateStringsArray | string, ...values: any[]) => {
519
+ if (typeof strings === 'string') {
520
+ // regular string with { } syntax
521
+ console.log(prefix + strings.replace(/\{([^}]+)\}/g, `${ansi}$1\x1b[0m`), ...values);
522
+ } else {
523
+ // tagged template literal
524
+ let message = '';
525
+ for (let i = 0; i < strings.length; i++) {
526
+ message += strings[i];
527
+ if (i < values.length)
528
+ message += `${ansi}${values[i]}\x1b[0m`;
529
+ }
530
+ console.log(prefix + message);
531
+ }
111
532
  };
112
533
  }
113
534
 
@@ -121,6 +542,71 @@ export const log_error = log_create_logger('error', 'red');
121
542
 
122
543
  // endregion
123
544
 
545
+ // region spooder ipc
546
+ export const IPC_OP = {
547
+ CMSG_TRIGGER_UPDATE: -1,
548
+ SMSG_UPDATE_READY: -2,
549
+ CMSG_REGISTER_LISTENER: -3,
550
+ };
551
+
552
+ // internal targets should always use __X__ as this format is
553
+ // reserved; userland instances cannot be named this way
554
+ export const IPC_TARGET = {
555
+ SPOODER: '__spooder__',
556
+ BROADCAST: '__broadcast__'
557
+ };
558
+
559
+ type IPC_Callback = (data: IPC_Message) => void;
560
+ type IPC_Message = {
561
+ op: number;
562
+ peer: string;
563
+ data?: object
564
+ };
565
+
566
+ let ipc_fail_announced = false;
567
+ let ipc_listener_attached = false;
568
+
569
+ const ipc_listeners = new Map<number, Set<IPC_Callback>>();
570
+
571
+ function ipc_on_message(payload: IPC_Message) {
572
+ const listeners = ipc_listeners.get(payload.op);
573
+ if (!listeners)
574
+ return;
575
+
576
+ for (const callback of listeners)
577
+ callback(payload);
578
+ }
579
+
580
+ export function ipc_send(peer: string, op: number, data?: object) {
581
+ if (!process.send) {
582
+ if (!ipc_fail_announced) {
583
+ log_spooder(`{ipc_send} failed, process not spawned with ipc channel`);
584
+ caution('ipc_send failed', { e: 'process not spawned with ipc channel' });
585
+ ipc_fail_announced = true;
586
+ }
587
+
588
+ return;
589
+ }
590
+
591
+ process.send({ peer, op, data });
592
+ }
593
+
594
+ export function ipc_register(op: number, callback: IPC_Callback) {
595
+ if (!ipc_listener_attached) {
596
+ process.on('message', ipc_on_message);
597
+ ipc_listener_attached = true;
598
+ }
599
+
600
+ const listeners = ipc_listeners.get(op);
601
+ if (listeners)
602
+ listeners.add(callback);
603
+ else
604
+ ipc_listeners.set(op, new Set([callback]));
605
+
606
+ ipc_send(IPC_TARGET.SPOODER, IPC_OP.CMSG_REGISTER_LISTENER, { op });
607
+ }
608
+ // endregion
609
+
124
610
  // region cache
125
611
  type CacheOptions = {
126
612
  ttl?: number;
@@ -1624,4 +2110,118 @@ export function http_serve(port: number, hostname?: string) {
1624
2110
  }
1625
2111
  };
1626
2112
  }
2113
+ // endregion
2114
+
2115
+ // region db
2116
+ type SchemaOptions = {
2117
+ schema_table: string;
2118
+ recursive: boolean;
2119
+ throw_on_skip: boolean;
2120
+ };
2121
+
2122
+ type TableRevision = {
2123
+ revision_number: number;
2124
+ file_path: string;
2125
+ filename: string;
2126
+ };
2127
+
2128
+ const db_log = log_create_logger('db', 'spooder');
2129
+
2130
+ export function db_set_cast<T extends string>(set: string | null): Set<T> {
2131
+ return new Set(set?.split(',') as T[] ?? []);
2132
+ }
2133
+
2134
+ export function db_set_serialize<T extends string>(set: Iterable<T> | null): string {
2135
+ return set ? Array.from(set).join(',') : '';
2136
+ }
2137
+
2138
+ export async function db_get_schema_revision(db: SQL): Promise<number|null> {
2139
+ try {
2140
+ const [result] = await db`SELECT MAX(revision_number) as latest_revision FROM db_schema`;
2141
+ return result.latest_revision ?? 0;
2142
+ } catch (e) {
2143
+ return null;
2144
+ }
2145
+ }
2146
+
2147
+ export async function db_schema(db: SQL, schema_path: string, options?: SchemaOptions): Promise<boolean> {
2148
+ const schema_table = options?.schema_table ?? 'db_schema';
2149
+ const recursive = options?.recursive ?? true;
2150
+
2151
+ db_log`applying schema revisions from ${schema_path}`;
2152
+ let current_revision = await db_get_schema_revision(db);
2153
+
2154
+ if (current_revision === null) {
2155
+ db_log`initiating schema database table ${schema_table}`;
2156
+ await db`CREATE TABLE ${db(schema_table)} (
2157
+ revision_number INTEGER PRIMARY KEY,
2158
+ filename VARCHAR(255) NOT NULL,
2159
+ applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
2160
+ );`;
2161
+ }
2162
+
2163
+ current_revision ??= 0;
2164
+
2165
+ const revisions = Array<TableRevision>();
2166
+ const files = await fs.readdir(schema_path, { recursive, encoding: 'utf8' });
2167
+ for (const file of files) {
2168
+ const filename = path.basename(file);
2169
+ if (!filename.toLowerCase().endsWith('.sql'))
2170
+ continue;
2171
+
2172
+ const match = filename.match(/^(\d+)/);
2173
+ const revision_number = match ? Number(match[1]) : null;
2174
+ if (revision_number === null || revision_number < 1) {
2175
+ log_error`skipping sql file ${file}, invalid revision number`;
2176
+ continue;
2177
+ }
2178
+
2179
+ const file_path = path.join(schema_path, file);
2180
+ if (revision_number > current_revision) {
2181
+ revisions.push({
2182
+ revision_number,
2183
+ file_path,
2184
+ filename
2185
+ });
2186
+ }
2187
+ }
2188
+
2189
+ // sort revisions in ascending order before applying
2190
+ // for recursive trees or unreliable OS sort ordering
2191
+ revisions.sort((a, b) => a.revision_number - b.revision_number);
2192
+
2193
+ const revisions_applied = Array<string>();
2194
+ for (const rev of revisions) {
2195
+ db_log`applying revision ${rev.revision_number} from ${rev.filename}`;
2196
+
2197
+ try {
2198
+ await db.begin(async tx => {
2199
+ await tx.file(rev.file_path);
2200
+ await tx`INSERT INTO ${db(schema_table)} ${db(rev, 'revision_number', 'filename')}`;
2201
+ revisions_applied.push(rev.filename);
2202
+ });
2203
+ } catch (err) {
2204
+
2205
+ log_error`failed to apply revisions from ${rev.filename}: ${err}`;
2206
+ log_error`${'warning'}: if ${rev.filename} contained DDL statements, they will ${'not'} be rolled back automatically`;
2207
+ log_error`verify the current database state ${'before'} running ammended revisions`;
2208
+
2209
+ const last_revision = await db_get_schema_revision(db);
2210
+ db_log`database schema revision is now ${last_revision ?? 0}`;
2211
+
2212
+ caution('db_schema failed', { rev, err, last_revision, revisions_applied });
2213
+
2214
+ return false;
2215
+ }
2216
+ }
2217
+
2218
+ if (revisions_applied.length > 0) {
2219
+ const new_revision = await db_get_schema_revision(db);
2220
+ db_log`applied ${revisions_applied.length} database schema revisions (${current_revision} >> ${new_revision})`;
2221
+ } else {
2222
+ db_log`no database schema revisions to apply (current: ${current_revision})`;
2223
+ }
2224
+
2225
+ return true;
2226
+ }
1627
2227
  // endregion