spooder 5.1.11 → 6.0.0

package/src/api.ts

@@ -7,93 +7,506 @@ import { Blob } from 'node:buffer';
 import { ColorInput } from 'bun';
 import packageJson from '../package.json' with { type: 'json' };
 
+// region exit codes
+export const EXIT_CODE = {
+	SUCCESS: 0,
+	GENERAL_ERROR: 1,
+
+	// 3-125 are free for application errors
+	SPOODER_AUTO_UPDATE: 42
+};
+
+export const EXIT_CODE_NAMES = Object.fromEntries(
+	Object.entries(EXIT_CODE).map(([key, value]) => [value, key])
+);
+// endregion
+
 // region api forwarding
 export * from './api_db';
 // endregion
 
 // region workers
 type WorkerMessageData = Record<string, any>;
-type WorkerEventPipeOptions = {
-	use_canary_reporting?: boolean;
+type WorkerMessage = {
+	id: string;
+	peer: string;
+	data?: WorkerMessageData;
+	uuid: string;
+	response_to?: string;
 };
 
-export interface WorkerEventPipe {
-	send: (id: string, data?: object) => void;
-	on: (event: string, callback: (data: WorkerMessageData) => Promise<void> | void) => void;
-	once: (event: string, callback: (data: WorkerMessageData) => Promise<void> | void) => void;
+const RESPONSE_TIMEOUT_MS = 5000;
+
+export interface WorkerPool {
+	id: string;
+	send(peer: string, id: string, data?: WorkerMessageData, expect_response?: false): void;
+	send(peer: string, id: string, data: WorkerMessageData | undefined, expect_response: true): Promise<WorkerMessage>;
+	broadcast: (id: string, data?: WorkerMessageData) => void;
+	respond: (message: WorkerMessage, data?: WorkerMessageData) => void;
+	on: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => void;
+	once: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => void;
 	off: (event: string) => void;
 }
 
-function worker_validate_message(message: any) {
-	if (typeof message !== 'object' || message === null)
-		throw new ErrorWithMetadata('invalid worker message type', { message });
-	
-	if (typeof message.id !== 'string')
-		throw new Error('missing worker message .id');
-}
+export const WORKER_EXIT_NO_RESTART = 42;
+const log_worker = log_create_logger('worker_pool', 'spooder');
 
-const log_worker = log_create_logger('worker', 'spooder');
-export function worker_event_pipe(worker: Worker, options?: WorkerEventPipeOptions): WorkerEventPipe {
-	const use_canary_reporting = options?.use_canary_reporting ?? false;
-	const callbacks = new Map<string, (data: Record<string, any>) => Promise<void> | void>();
-	
-	function handle_message(event: MessageEvent) {
-		try {
-			const message = JSON.parse(event.data);
-			worker_validate_message(message);
-			
-			const callback = callbacks.get(message.id);
-			if (callback !== undefined)
-				callback(message.data ?? {});
-		} catch (e) {
-			log_error(`exception in worker: ${(e as Error).message}`);
-			
-			if (use_canary_reporting)
-				caution('worker: exception handling payload', { exception: e });
+type AutoRestartConfig = {
+	backoff_max?: number;
+	backoff_grace?: number;
+	max_attempts?: number;
+};
+
+type WorkerPoolOptions = {
+	id?: string;
+	worker: string | string[];
+	size?: number;
+	auto_restart?: boolean | AutoRestartConfig;
+	response_timeout?: number;
+};
+
+type WorkerState = {
+	worker: Worker;
+	worker_id?: string;
+	restart_delay: number;
+	restart_attempts: number;
+	restart_success_timer: Timer | null;
+	worker_path: string;
+};
+
+export async function worker_pool(options: WorkerPoolOptions): Promise<WorkerPool> {
+	const pipe_workers = new BiMap<string, Worker>();
+	const worker_promises = new WeakMap<Worker, (value: void | PromiseLike<void>) => void>();
+
+	const peer_id = options.id ?? 'main';
+	const response_timeout = options.response_timeout ?? RESPONSE_TIMEOUT_MS;
+
+	const auto_restart_enabled = options.auto_restart !== undefined && options.auto_restart !== false;
+	const auto_restart_config = typeof options.auto_restart === 'object' ? options.auto_restart : {};
+	const backoff_max = auto_restart_config.backoff_max ?? 5 * 60 * 1000; // 5 min
+	const backoff_grace = auto_restart_config.backoff_grace ?? 30000; // 30 seconds
+	const max_attempts = auto_restart_config.max_attempts ?? 5;
+
+	const worker_states = new WeakMap<Worker, WorkerState>();
+
+	const worker_paths: string[] = options.size !== undefined
+		? Array(options.size).fill(options.worker)
+		: Array.isArray(options.worker) ? options.worker : [options.worker];
+
+	log_worker(`created worker pool {${peer_id}}`);
+
+	const callbacks = new Map<string, (data: WorkerMessage) => Promise<void> | void>();
+	const pending_responses = new Map<string, { resolve: (message: WorkerMessage) => void, reject: (error: Error) => void, timeout: Timer | undefined }>();
+
+	async function restart_worker(worker: Worker) {
+		if (!auto_restart_enabled)
+			return;
+
+		const state = worker_states.get(worker);
+		if (!state)
+			return;
+
+		if (state.restart_success_timer) {
+			clearTimeout(state.restart_success_timer);
+			state.restart_success_timer = null;
 		}
+
+		if (max_attempts !== -1 && state.restart_attempts >= max_attempts) {
+			log_worker(`worker {${state.worker_id ?? 'unknown'}} maximum restart attempts ({${max_attempts}}) reached, stopping auto-restart`);
+			return;
+		}
+
+		state.restart_attempts++;
+		const current_delay = Math.min(state.restart_delay, backoff_max);
+		const max_attempt_str = max_attempts === -1 ? '∞' : max_attempts;
+
+		log_worker(`restarting worker {${state.worker_id ?? 'unknown'}} in {${current_delay}ms} (attempt {${state.restart_attempts}}/{${max_attempt_str}}, delay capped at {${backoff_max}ms})`);
+
+		setTimeout(() => {
+			const new_worker = new Worker(state.worker_path);
+
+			state.worker = new_worker;
+			state.restart_delay = Math.min(state.restart_delay * 2, backoff_max);
+
+			state.restart_success_timer = setTimeout(() => {
+				state.restart_delay = 100;
+				state.restart_attempts = 0;
+				state.restart_success_timer = null;
+			}, backoff_grace);
+
+			worker_states.delete(worker);
+			worker_states.set(new_worker, state);
+
+			setup_worker_listeners(new_worker);
+		}, current_delay);
 	}
-	
-	if (Bun.isMainThread) {
-		log_worker(`event pipe connected {main thread} ⇄ {worker}`);
-		worker.addEventListener('message', handle_message);
-	} else {
-		log_worker(`event pipe connected {worker} ⇄ {main thread}`);
-		worker.onmessage = handle_message;
+
+	function setup_worker_listeners(worker: Worker) {
+		worker.addEventListener('message', (event) => {
+			handle_worker_message(worker, event);
+		});
+
+		worker.addEventListener('close', (event: CloseEvent) => {
+			const worker_id = pipe_workers.getByValue(worker);
+			const exit_code = event.code;
+			log_worker(`worker {${worker_id ?? 'unknown'}} closed, exit code {${exit_code}}`);
+
+			if (worker_id)
+				pipe_workers.deleteByKey(worker_id);
+
+			if (auto_restart_enabled && exit_code !== WORKER_EXIT_NO_RESTART)
+				restart_worker(worker);
+			else if (exit_code === WORKER_EXIT_NO_RESTART)
+				log_worker(`worker {${worker_id ?? 'unknown'}} exited with {WORKER_EXIT_NO_RESTART}, skipping auto-restart`);
+		});
 	}
-	
-	return {
-		send: (id: string, data: object = {}) => {
-			worker.postMessage(JSON.stringify({ id, data }));
+
+	function handle_worker_message(worker: Worker, event: MessageEvent) {
+		const message = event.data as WorkerMessage;
+
+		if (message.id === '__register__') {
+			const worker_id = message.data?.worker_id;
+			if (worker_id === undefined) {
+				log_error('cannot register worker without ID');
+				return;
+			}
+
+			if (pipe_workers.hasKey(worker_id)) {
+				log_error(`worker ID {${worker_id}} already in-use`);
+				return;
+			}
+
+			pipe_workers.set(message.data?.worker_id, worker);
+
+			const state = worker_states.get(worker);
+			if (state)
+				state.worker_id = worker_id;
+
+			worker_promises.get(worker)?.();
+			worker_promises.delete(worker);
+		} else if (message.peer === '__broadcast__') {
+			const worker_id = pipe_workers.getByValue(worker);
+			if (worker_id === undefined)
+				return;
+
+			message.peer = worker_id;
+			callbacks.get(message.id)?.(message);
+
+			for (const target_worker of pipe_workers.values()) {
+				if (target_worker === worker)
+					continue;
+
+				target_worker.postMessage(message);
+			}
+		} else {
+			const target_peer = message.peer;
+			const target_worker = pipe_workers.getByKey(target_peer);
+			const worker_id = pipe_workers.getByValue(worker);
+
+			if (worker_id === undefined)
+				return;
+
+			message.peer = worker_id;
+
+			if (target_peer === peer_id) {
+				if (message.response_to && pending_responses.has(message.response_to)) {
+					const pending = pending_responses.get(message.response_to)!;
+					if (pending.timeout)
+						clearTimeout(pending.timeout);
+
+					pending_responses.delete(message.response_to);
+					pending.resolve(message);
+					return;
+				}
+
+				callbacks.get(message.id)?.(message);
+			}
+
+			target_worker?.postMessage(message);
+		}
+	}
+
+	const pool: WorkerPool = {
+		id: peer_id,
+
+		send(peer: string, id: string, data?: WorkerMessageData, expect_response?: boolean): any {
+			const message: WorkerMessage = { id, peer: peer_id, data, uuid: Bun.randomUUIDv7() };
+
+			if (expect_response) {
+				return new Promise<WorkerMessage>((resolve, reject) => {
+					let timeout: Timer | undefined;
+
+					if (response_timeout !== -1) {
+						timeout = setTimeout(() => {
+							pending_responses.delete(message.uuid);
+							reject(new Error(`Response timeout after ${response_timeout}ms`));
+						}, response_timeout);
+					}
+
+					pending_responses.set(message.uuid, { resolve, reject, timeout });
+
+					const target_worker = pipe_workers.getByKey(peer);
+					target_worker?.postMessage(message);
+				});
+			} else {
+				const target_worker = pipe_workers.getByKey(peer);
+				target_worker?.postMessage(message);
+			}
 		},
-		
-		on: (event: string, callback: (data: WorkerMessageData) => Promise<void> | void) => {
+
+		broadcast: (id: string, data?: WorkerMessageData) => {
+			const message: WorkerMessage = { peer: peer_id, id, data, uuid: Bun.randomUUIDv7() };
+			
+			for (const target_worker of pipe_workers.values())
+				target_worker.postMessage(message);
+		},
+
+		respond: (message: WorkerMessage, data?: WorkerMessageData) => {
+			const response: WorkerMessage = {
+				id: message.id,
+				peer: peer_id,
+				data,
+				uuid: Bun.randomUUIDv7(),
+				response_to: message.uuid
+			};
+
+			const target_worker = pipe_workers.getByKey(message.peer);
+			target_worker?.postMessage(response);
+		},
+
+		on: (event: string, callback: (data: WorkerMessage) => Promise<void> | void) => {
 			callbacks.set(event, callback);
 		},
-		
+
 		off: (event: string) => {
 			callbacks.delete(event);
 		},
-		
-		once: (event: string, callback: (data: WorkerMessageData) => Promise<void> | void) => {
-			callbacks.set(event, async (data: WorkerMessageData) => {
+
+		once: (event: string, callback: (data: WorkerMessage) => Promise<void> | void) => {
+			callbacks.set(event, async (data: WorkerMessage) => {
 				await callback(data);
 				callbacks.delete(event);
 			});
 		}
 	};
+
+	const promises = [];
+	for (const path of worker_paths) {
+		const worker = new Worker(path);
+
+		if (auto_restart_enabled) {
+			worker_states.set(worker, {
+				worker,
+				restart_delay: 100,
+				restart_attempts: 0,
+				restart_success_timer: null,
+				worker_path: path
+			});
+		}
+
+		setup_worker_listeners(worker);
+
+		promises.push(new Promise<void>(resolve => {
+			worker_promises.set(worker, resolve);
+		}));
+	}
+
+	await Promise.all(promises);
+
+	return pool;
+}
+
+export function worker_connect(peer_id?: string, response_timeout?: number): WorkerPool {
+	const listeners = new Map<string, (message: WorkerMessage) => Promise<void> | void>();
+	const pending_responses = new Map<string, { resolve: (message: WorkerMessage) => void, reject: (error: Error) => void, timeout: Timer | undefined }>();
+
+	if (peer_id === undefined) {
+		// normally we would increment 'worker1', 'worker2' etc but we
+		// have no simple way of keeping track of global worker count.
+
+		// in normal circumstances, users should provide an ID via the
+		// parameter, this is just a sensible fallback.
+		peer_id = 'worker-' + Bun.randomUUIDv7();
+	}
+
+	response_timeout = response_timeout ?? RESPONSE_TIMEOUT_MS;
+
+	log_worker(`worker {${peer_id}} connected to pool`);
+
+	const worker = globalThis as unknown as Worker;
+	worker.onmessage = event => {
+		const message = event.data as WorkerMessage;
+
+		if (message.response_to && pending_responses.has(message.response_to)) {
+			const pending = pending_responses.get(message.response_to)!;
+			if (pending.timeout)
+				clearTimeout(pending.timeout);
+
+			pending_responses.delete(message.response_to);
+			pending.resolve(message);
+			return;
+		}
+
+		listeners.get(message.id)?.(message);
+	};
+
+	worker.postMessage({
+		id: '__register__',
+		data: {
+			worker_id: peer_id
+		}
+	});
+
+	return {
+		id: peer_id,
+
+		send(peer: string, id: string, data?: WorkerMessageData, expect_response?: boolean): any {
+			const message: WorkerMessage = { id, peer, data, uuid: Bun.randomUUIDv7() };
+
+			if (expect_response) {
+				return new Promise<WorkerMessage>((resolve, reject) => {
+					let timeout: Timer | undefined;
+
+					if (response_timeout !== -1) {
+						timeout = setTimeout(() => {
+							pending_responses.delete(message.uuid);
+							reject(new Error(`Response timeout after ${response_timeout}ms`));
+						}, response_timeout);
+					}
+
+					pending_responses.set(message.uuid, { resolve, reject, timeout });
+					worker.postMessage(message);
+				});
+			} else {
+				worker.postMessage(message);
+			}
+		},
+
+		broadcast: (id: string, data?: WorkerMessageData) => {
+			const message: WorkerMessage = { peer: '__broadcast__', id, data, uuid: Bun.randomUUIDv7() };
+			worker.postMessage(message);
+		},
+
+		respond: (message: WorkerMessage, data?: WorkerMessageData) => {
+			const response: WorkerMessage = {
+				id: message.id,
+				peer: message.peer,
+				data,
+				uuid: Bun.randomUUIDv7(),
+				response_to: message.uuid
+			};
+
+			worker.postMessage(response);
+		},
+
+		on: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => {
+			listeners.set(event, callback);
+		},
+
+		off: (event: string) => {
+			listeners.delete(event);
+		},
+
+		once: (event: string, callback: (message: WorkerMessage) => Promise<void> | void) => {
+			listeners.set(event, async (message: WorkerMessage) => {
+				await callback(message);
+				listeners.delete(event);
+			});
+		}
+	};
 }
+
 // endregion
 
 // region utility
+export class BiMap<K, V> {
+	private ktv = new Map<K, V>();
+	private vtk = new Map<V, K>();
+
+	set(key: K, value: V): void {
+		const old_val = this.ktv.get(key);
+		if (old_val !== undefined)
+			this.vtk.delete(old_val);
+
+		const old_key = this.vtk.get(value);
+		if (old_key !== undefined)
+			this.ktv.delete(old_key);
+
+		this.ktv.set(key, value);
+		this.vtk.set(value, key);
+	}
+
+	getByKey(key: K): V | undefined {
+		return this.ktv.get(key);
+	}
+
+	getByValue(value: V): K | undefined {
+		return this.vtk.get(value);
+	}
+
+	hasKey(key: K): boolean {
+		return this.ktv.has(key);
+	}
+
+	hasValue(value: V): boolean {
+		return this.vtk.has(value);
+	}
+
+	deleteByKey(key: K): boolean {
+		const value = this.ktv.get(key);
+		if (value === undefined)
+			return false;
+
+		this.ktv.delete(key);
+		this.vtk.delete(value);
+		return true;
+	}
+
+	deleteByValue(value: V): boolean {
+		const key = this.vtk.get(value);
+		if (key === undefined)
+			return false;
+
+		this.vtk.delete(value);
+		this.ktv.delete(key);
+		return true;
+	}
+
+	clear(): void {
+		this.ktv.clear();
+		this.vtk.clear();
+	}
+
+	get size(): number {
+		return this.ktv.size;
+	}
+
+	entries(): IterableIterator<[K, V]> {
+		return this.ktv.entries();
+	}
+
+	keys(): IterableIterator<K> {
+		return this.ktv.keys();
+	}
+
+	values(): IterableIterator<V> {
+		return this.ktv.values();
+	}
+
+	[Symbol.iterator](): IterableIterator<[K, V]> {
+		return this.ktv.entries();
+	}
+}
+
 const FILESIZE_UNITS = ['bytes', 'kb', 'mb', 'gb', 'tb'];
 
-function filesize(bytes: number): string {
+export function filesize(bytes: number): string {
 	if (bytes === 0)
 		return '0 bytes';
-	
+
 	const i = Math.floor(Math.log(bytes) / Math.log(1024));
 	const size = bytes / Math.pow(1024, i);
-	
+
 	return `${size.toFixed(i === 0 ? 0 : 1)} ${FILESIZE_UNITS[i]}`;
 }
 // endregion
@@ -121,6 +534,71 @@ export const log_error = log_create_logger('error', 'red');
 
 // endregion
 
+// region spooder ipc
+export const IPC_OP = {
+	CMSG_TRIGGER_UPDATE: -1,
+	SMSG_UPDATE_READY: -2,
+	CMSG_REGISTER_LISTENER: -3,
+};
+
+// internal targets should always use __X__ as this format is
+// reserved; userland instances cannot be named this way
+export const IPC_TARGET = {
+	SPOODER: '__spooder__',
+	BROADCAST: '__broadcast__'
+};
+
+type IPC_Callback = (data: IPC_Message) => void;
+type IPC_Message = {
+	op: number;
+	peer: string;
+	data?: object
+};
+
+let ipc_fail_announced = false;
+let ipc_listener_attached = false;
+
+const ipc_listeners = new Map<number, Set<IPC_Callback>>();
+
+function ipc_on_message(payload: IPC_Message) {
+	const listeners = ipc_listeners.get(payload.op);
+	if (!listeners)
+		return;
+
+	for (const callback of listeners)
+		callback(payload);
+}
+
+export function ipc_send(peer: string, op: number, data?: object) {
+	if (!process.send) {
+		if (!ipc_fail_announced) {
+			log_spooder(`{ipc_send} failed, process not spawned with ipc channel`);
+			caution('ipc_send failed', { e: 'process not spawned with ipc channel' });
+			ipc_fail_announced = true;
+		}
+
+		return;
+	}
+
+	process.send({ peer, op, data });
+}
+
+export function ipc_register(op: number, callback: IPC_Callback) {
+	if (!ipc_listener_attached) {
+		process.on('message', ipc_on_message);
+		ipc_listener_attached = true;
+	}
+
+	const listeners = ipc_listeners.get(op);
+	if (listeners)
+		listeners.add(callback);
+	else
+		ipc_listeners.set(op, new Set([callback]));
+
+	ipc_send(IPC_TARGET.SPOODER, IPC_OP.CMSG_REGISTER_LISTENER, { op });
+}
+// endregion
+
 // region cache
 type CacheOptions = {
 	ttl?: number;
@@ -128,6 +606,7 @@ type CacheOptions = {
 	use_etags?: boolean;
 	headers?: Record<string, string>,
 	use_canary_reporting?: boolean;
+	enabled?: boolean;
 };
 
 type CacheEntry = {
@@ -154,6 +633,7 @@ export function cache_http(options?: CacheOptions) {
 	const use_etags = options?.use_etags ?? true;
 	const cache_headers = options?.headers ?? {};
 	const canary_report = options?.use_canary_reporting ?? false;
+	const enabled = options?.enabled ?? true;
 	
 	const entries = new Map<string, CacheEntry>();
 	let total_cache_size = 0;
@@ -267,7 +747,8 @@ export function cache_http(options?: CacheOptions) {
 						cached_ts: now_ts
 					};
 					
-					store_cache_entry(file_path, entry, now_ts);
+					if (enabled)
+						store_cache_entry(file_path, entry, now_ts);
 				}
 				
 				return build_response(entry, req, 200);
@@ -290,7 +771,8 @@ export function cache_http(options?: CacheOptions) {
 					cached_ts: now_ts
 				};
 				
-				store_cache_entry(cache_key, entry, now_ts);
+				if (enabled)
+					store_cache_entry(cache_key, entry, now_ts);
 			}
 			
 			return build_response(entry, req, status_code);