threadsx 2.0.2 → 2.2.0

package/esm/master/implementation.node.mjs

@@ -0,0 +1,186 @@
+import { createRequire as __createRequire } from "module";
+import { fileURLToPath as __fileURLToPath } from "url";
+import { dirname as __pathDirname } from "path";
+const require = __createRequire(import.meta.url);
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+import getCallsites from 'callsites';
+import { cpus } from 'os';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { isWebpackBundle, getNodeRequire, getModuleDirname } from '../node-require.mjs';
+import getImplementation from '../worker_threads.mjs';
+
+/// <reference lib="dom" />
+// NOTE: `callsites` is intentionally pinned to ^3. v4+ is ESM-only and would
+// break the CommonJS (dist/) build. Do not bump it to v4 without moving the
+// package to ESM-first output. (See .github/dependabot.yml, which also ignores
+// its major updates.)
+let detectedTsRuntime;
+const defaultPoolSize = cpus().length;
+/**
+ * Whether the application opted out of threadsx installing SIGINT/SIGTERM
+ * handlers (which call `process.exit()`), via `THREADS_SKIP_SIGNAL_HANDLERS`.
+ */
+function signalHandlersDisabled() {
+    const value = typeof process !== "undefined" && process.env
+        ? process.env.THREADS_SKIP_SIGNAL_HANDLERS
+        : undefined;
+    return value === "1" || value === "true";
+}
+/**
+ * Detects an available TypeScript runtime so `.ts`/`.tsx` worker files can be
+ * spawned directly during development. Prefers `tsx`, falls back to `ts-node`.
+ */
+function detectTsRuntime() {
+    if (isWebpackBundle()) {
+        // Webpack build: => No TS runtime required or possible
+        return null;
+    }
+    if (detectedTsRuntime !== undefined) {
+        return detectedTsRuntime;
+    }
+    detectedTsRuntime = null;
+    for (const candidate of ["tsx", "ts-node"]) {
+        try {
+            getNodeRequire().resolve(candidate);
+            detectedTsRuntime = candidate;
+            break;
+        }
+        catch (error) {
+            if (error && error.code === "MODULE_NOT_FOUND") {
+                continue;
+            }
+            // Re-throw
+            throw error;
+        }
+    }
+    return detectedTsRuntime;
+}
+function createTsRuntimeModule(scriptPath, runtime) {
+    const register = runtime === "tsx"
+        ? `require("tsx/cjs");`
+        : `require("ts-node/register/transpile-only");`;
+    const content = `
+    ${register}
+    require(${JSON.stringify(scriptPath)});
+  `;
+    return content;
+}
+function rebaseScriptPath(scriptPath, ignoreRegex) {
+    // An already-absolute path (e.g. from `new Worker(new URL("./w", import.meta.url))`,
+    // which threadsx normalizes to a filesystem path) must not be rebased onto the
+    // caller's directory — doing so would prepend it and corrupt the path.
+    if (path.isAbsolute(scriptPath)) {
+        return scriptPath;
+    }
+    const parentCallSite = getCallsites().find((callsite) => {
+        const filename = callsite.getFileName();
+        return Boolean(filename &&
+            !filename.match(ignoreRegex) &&
+            !filename.match(/[/\\]master[/\\]implementation/) &&
+            !filename.match(/^internal\/process/));
+    });
+    const rawCallerPath = parentCallSite ? parentCallSite.getFileName() : null;
+    let callerPath = rawCallerPath ? rawCallerPath : null;
+    if (callerPath && callerPath.startsWith('file:')) {
+        callerPath = fileURLToPath(callerPath);
+    }
+    const rebasedScriptPath = callerPath ? path.join(path.dirname(callerPath), scriptPath) : scriptPath;
+    return rebasedScriptPath;
+}
+function resolveScriptPath(scriptPath, baseURL) {
+    const makeRelative = (filePath) => {
+        return path.isAbsolute(filePath) ? filePath : path.join(baseURL || getModuleDirname(), filePath);
+    };
+    // In a webpack bundle, walking the call stack is useless (every caller lives
+    // inside the bundle), so relative paths resolve without rebasing.
+    const workerFilePath = isWebpackBundle()
+        ? getNodeRequire().resolve(makeRelative(scriptPath))
+        : getNodeRequire().resolve(makeRelative(rebaseScriptPath(scriptPath, /[/\\]worker_threads[/\\]/)));
+    return workerFilePath;
+}
+function initWorkerThreadsWorker() {
+    const NativeWorker = getImplementation().Worker;
+    let allWorkers = [];
+    class Worker extends NativeWorker {
+        constructor(scriptPath, options) {
+            // Bundlers like webpack 5 pass a `URL` (from `new Worker(new URL(...))`)
+            // pointing at the emitted worker chunk.
+            const normalizedScriptPath = scriptPath instanceof URL ? fileURLToPath(scriptPath) : scriptPath;
+            const resolvedScriptPath = options && options.fromSource
+                ? null
+                : resolveScriptPath(normalizedScriptPath, (options || {})._baseURL);
+            const tsRuntime = resolvedScriptPath && /\.tsx?$/i.test(resolvedScriptPath) ? detectTsRuntime() : null;
+            if (!resolvedScriptPath) {
+                // `options.fromSource` is true
+                const sourceCode = scriptPath;
+                super(sourceCode, Object.assign(Object.assign({}, options), { eval: true }));
+            }
+            else if (tsRuntime) {
+                super(createTsRuntimeModule(resolvedScriptPath, tsRuntime), Object.assign(Object.assign({}, options), { eval: true }));
+            }
+            else if (resolvedScriptPath.match(/\.asar[/\\]/)) {
+                // See <https://github.com/andywer/threads-plugin/issues/17>
+                super(resolvedScriptPath.replace(/\.asar([/\\])/, ".asar.unpacked$1"), options);
+            }
+            else {
+                super(resolvedScriptPath, options);
+            }
+            this.mappedEventListeners = new WeakMap();
+            allWorkers.push(this);
+        }
+        addEventListener(eventName, rawListener) {
+            const listener = (message) => {
+                rawListener({ data: message });
+            };
+            this.mappedEventListeners.set(rawListener, listener);
+            this.on(eventName, listener);
+        }
+        removeEventListener(eventName, rawListener) {
+            const listener = this.mappedEventListeners.get(rawListener) || rawListener;
+            this.off(eventName, listener);
+        }
+    }
+    const terminateWorkersAndMaster = () => {
+        // we should terminate all workers and then gracefully shutdown self process
+        Promise.all(allWorkers.map(worker => worker.terminate())).then(() => process.exit(0), () => process.exit(1));
+        allWorkers = [];
+    };
+    // Take care to not leave orphaned processes behind. See #147.
+    //
+    // These handlers call process.exit(), which hijacks the host application's
+    // own shutdown. Applications that manage their own graceful shutdown can opt
+    // out by setting THREADS_SKIP_SIGNAL_HANDLERS. See upstream #388 / #484.
+    if (!signalHandlersDisabled()) {
+        process.on("SIGINT", () => terminateWorkersAndMaster());
+        process.on("SIGTERM", () => terminateWorkersAndMaster());
+    }
+    class BlobWorker extends Worker {
+        constructor(blob, options) {
+            super(Buffer.from(blob).toString("utf-8"), Object.assign(Object.assign({}, options), { fromSource: true }));
+        }
+        static fromText(source, options) {
+            return new Worker(source, Object.assign(Object.assign({}, options), { fromSource: true }));
+        }
+    }
+    return {
+        blob: BlobWorker,
+        default: Worker
+    };
+}
+let implementation;
+function selectWorkerImplementation() {
+    return initWorkerThreadsWorker();
+}
+function getWorkerImplementation() {
+    if (!implementation) {
+        implementation = selectWorkerImplementation();
+    }
+    return implementation;
+}
+function isWorkerRuntime() {
+    return !getImplementation().isMainThread;
+}
+
+export { defaultPoolSize, getWorkerImplementation, isWorkerRuntime };

package/esm/master/index.mjs

@@ -0,0 +1,17 @@
+import { createRequire as __createRequire } from "module";
+import { fileURLToPath as __fileURLToPath } from "url";
+import { dirname as __pathDirname } from "path";
+const require = __createRequire(import.meta.url);
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+import { getWorkerImplementation } from './implementation.mjs';
+export { isWorkerRuntime } from './implementation.mjs';
+export { Pool } from './pool.mjs';
+export { spawn } from './spawn.mjs';
+
+/** Separate class to spawn workers from source code blobs or strings. */
+const BlobWorker = getWorkerImplementation().blob;
+/** Worker implementation. Either web worker or a node.js Worker class. */
+const Worker = getWorkerImplementation().default;
+
+export { BlobWorker, Worker };

package/esm/master/invocation-proxy.mjs

@@ -0,0 +1,164 @@
+import { createRequire as __createRequire } from "module";
+import { fileURLToPath as __fileURLToPath } from "url";
+import { dirname as __pathDirname } from "path";
+const require = __createRequire(import.meta.url);
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+import DebugLogger from 'debug';
+import { multicast, Observable } from 'observable-fns';
+import { serialize, deserialize } from '../common.mjs';
+import { isDataCloneError, ThreadCloneError } from '../errors.mjs';
+import { ObservablePromise } from '../observable-promise.mjs';
+import { isTransferDescriptor } from '../transferable.mjs';
+import { MasterMessageType, WorkerMessageType } from '../types/messages.mjs';
+
+/*
+ * This source file contains the code for proxying calls in the master thread to calls in the workers
+ * by `.postMessage()`-ing.
+ *
+ * Keep in mind that this code can make or break the program's performance! Need to optimize more…
+ */
+const debugMessages = DebugLogger("threads:master:messages");
+let nextJobUID = 1;
+const dedupe = (array) => Array.from(new Set(array));
+const isJobErrorMessage = (data) => data && data.type === WorkerMessageType.error;
+const isJobResultMessage = (data) => data && data.type === WorkerMessageType.result;
+const isJobStartMessage = (data) => data && data.type === WorkerMessageType.running;
+function createObservableForJob(worker, jobUID) {
+    return new Observable(observer => {
+        let asyncType;
+        let settled = false;
+        const cleanup = () => {
+            worker.removeEventListener("message", messageHandler);
+            worker.removeEventListener("error", errorHandler);
+            worker.removeEventListener("exit", exitHandler);
+        };
+        const messageHandler = ((event) => {
+            debugMessages("Message from worker:", event.data);
+            if (!event.data || event.data.uid !== jobUID)
+                return;
+            if (isJobStartMessage(event.data)) {
+                asyncType = event.data.resultType;
+            }
+            else if (isJobResultMessage(event.data)) {
+                if (asyncType === "promise") {
+                    if (typeof event.data.payload !== "undefined") {
+                        observer.next(deserialize(event.data.payload));
+                    }
+                    settled = true;
+                    observer.complete();
+                    cleanup();
+                }
+                else {
+                    if (typeof event.data.payload !== "undefined") {
+                        observer.next(deserialize(event.data.payload));
+                    }
+                    if (event.data.complete) {
+                        settled = true;
+                        observer.complete();
+                        cleanup();
+                    }
+                }
+            }
+            else if (isJobErrorMessage(event.data)) {
+                const error = deserialize(event.data.error);
+                settled = true;
+                observer.error(error);
+                cleanup();
+            }
+        });
+        // If the worker crashes or is terminated before the job produces a result,
+        // reject the pending job instead of leaving the promise hanging forever.
+        // See #386.
+        const errorHandler = ((event) => {
+            if (settled)
+                return;
+            settled = true;
+            const error = event && event.data instanceof Error
+                ? event.data
+                : Error(String((event && event.data) || "Worker errored before the job completed."));
+            observer.error(error);
+            cleanup();
+        });
+        const exitHandler = ((event) => {
+            if (settled)
+                return;
+            settled = true;
+            const exitCode = event ? event.data : undefined;
+            observer.error(Error(`Worker terminated before the job completed (exit code: ${exitCode}).`));
+            cleanup();
+        });
+        worker.addEventListener("message", messageHandler);
+        worker.addEventListener("error", errorHandler);
+        worker.addEventListener("exit", exitHandler);
+        return () => {
+            if (asyncType === "observable" || !asyncType) {
+                const cancelMessage = {
+                    type: MasterMessageType.cancel,
+                    uid: jobUID
+                };
+                worker.postMessage(cancelMessage);
+            }
+            cleanup();
+        };
+    });
+}
+function prepareArguments(rawArgs) {
+    if (rawArgs.length === 0) {
+        // Exit early if possible
+        return {
+            args: [],
+            transferables: []
+        };
+    }
+    const args = [];
+    const transferables = [];
+    for (const arg of rawArgs) {
+        if (isTransferDescriptor(arg)) {
+            args.push(serialize(arg.send));
+            transferables.push(...arg.transferables);
+        }
+        else {
+            args.push(serialize(arg));
+        }
+    }
+    return {
+        args,
+        transferables: transferables.length === 0 ? transferables : dedupe(transferables)
+    };
+}
+function createProxyFunction(worker, method) {
+    return ((...rawArgs) => {
+        const uid = nextJobUID++;
+        const { args, transferables } = prepareArguments(rawArgs);
+        const runMessage = {
+            type: MasterMessageType.run,
+            uid,
+            method,
+            args
+        };
+        debugMessages("Sending command to run function to worker:", runMessage);
+        try {
+            worker.postMessage(runMessage, transferables);
+        }
+        catch (error) {
+            if (isDataCloneError(error)) {
+                const cloneError = new ThreadCloneError(`Cannot send arguments to the worker thread: a value is not structured-cloneable. ` +
+                    `Functions, class instances and other non-serializable values cannot be passed to a thread. ` +
+                    `Original error: ${error.message}`, error);
+                return ObservablePromise.from(Promise.reject(cloneError));
+            }
+            return ObservablePromise.from(Promise.reject(error));
+        }
+        return ObservablePromise.from(multicast(createObservableForJob(worker, uid)));
+    });
+}
+function createProxyModule(worker, methodNames) {
+    const proxy = {};
+    for (const methodName of methodNames) {
+        proxy[methodName] = createProxyFunction(worker, methodName);
+    }
+    return proxy;
+}
+
+export { createProxyFunction, createProxyModule };

package/esm/master/pool-types.mjs

@@ -0,0 +1,20 @@
+import { createRequire as __createRequire } from "module";
+import { fileURLToPath as __fileURLToPath } from "url";
+import { dirname as __pathDirname } from "path";
+const require = __createRequire(import.meta.url);
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+/** Pool event type. Specifies the type of each `PoolEvent`. */
+var PoolEventType;
+(function (PoolEventType) {
+    PoolEventType["initialized"] = "initialized";
+    PoolEventType["taskCanceled"] = "taskCanceled";
+    PoolEventType["taskCompleted"] = "taskCompleted";
+    PoolEventType["taskFailed"] = "taskFailed";
+    PoolEventType["taskQueued"] = "taskQueued";
+    PoolEventType["taskQueueDrained"] = "taskQueueDrained";
+    PoolEventType["taskStart"] = "taskStart";
+    PoolEventType["terminated"] = "terminated";
+})(PoolEventType || (PoolEventType = {}));
+
+export { PoolEventType };