xhs-mp-pack 1.1.0-beta.0

package/dist/thread/WorkerPool.js

@@ -0,0 +1,370 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable no-console */
+const child_process_1 = __importDefault(require("child_process"));
+const queue_1 = __importDefault(require("neo-async/queue"));
+const mapSeries_1 = __importDefault(require("neo-async/mapSeries"));
+const readBuffer_1 = __importDefault(require("./readBuffer"));
+const WorkerError_1 = __importDefault(require("./WorkerError"));
+const serializer_1 = require("./serializer");
+const workerPath = require.resolve('./worker');
+let workerId = 0;
+class PoolWorker {
+    constructor(options, onJobDone) {
+        this.state = '';
+        this.disposed = false;
+        this.nextJobId = 0;
+        this.jobs = Object.create(null);
+        this.activeJobs = 0;
+        this.onJobDone = onJobDone;
+        this.id = workerId;
+        workerId += 1;
+        // Empty or invalid node args would break the child process
+        const sanitizedNodeArgs = (options.nodeArgs || []).filter(opt => !!opt);
+        this.worker = child_process_1.default.spawn(process.execPath, 
+        // @ts-ignore
+        [].concat(sanitizedNodeArgs).concat(workerPath, options.parallelJobs), {
+            detached: true,
+            stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'],
+        });
+        this.worker.unref();
+        // This prevents a problem where the worker stdio can be undefined
+        // when the kernel hits the limit of open files.
+        // More info can be found on: https://github.com/webpack-contrib/thread-loader/issues/2
+        if (!this.worker.stdio) {
+            throw new Error(`Failed to create the worker pool with workerId: ${workerId} and ${''}configuration: ${JSON.stringify(options)}. Please verify if you hit the OS open files limit.`);
+        }
+        const [, , , readPipe, writePipe] = this.worker.stdio;
+        this.readPipe = readPipe;
+        this.writePipe = writePipe;
+        this.listenStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr);
+        this.readNextMessage();
+    }
+    listenStdOutAndErrFromWorker(workerStdout, workerStderr) {
+        if (workerStdout) {
+            workerStdout.on('data', this.writeToStdout);
+        }
+        if (workerStderr) {
+            workerStderr.on('data', this.writeToStderr);
+        }
+    }
+    ignoreStdOutAndErrFromWorker(workerStdout, workerStderr) {
+        if (workerStdout) {
+            workerStdout.removeListener('data', this.writeToStdout);
+        }
+        if (workerStderr) {
+            workerStderr.removeListener('data', this.writeToStderr);
+        }
+    }
+    writeToStdout(data) {
+        if (!this.disposed) {
+            process.stdout.write(data);
+        }
+    }
+    writeToStderr(data) {
+        if (!this.disposed) {
+            process.stderr.write(data);
+        }
+    }
+    run(data, callback) {
+        const jobId = this.nextJobId;
+        this.nextJobId += 1;
+        this.jobs[jobId] = { data, callback };
+        this.activeJobs += 1;
+        this.writeJson({
+            type: 'job',
+            id: jobId,
+            data,
+        });
+    }
+    warmup(requires) {
+        this.writeJson({
+            type: 'warmup',
+            requires,
+        });
+    }
+    writeJson(data) {
+        const lengthBuffer = Buffer.alloc(4);
+        const messageBuffer = Buffer.from(JSON.stringify(data, serializer_1.replacer), 'utf-8');
+        lengthBuffer.writeInt32BE(messageBuffer.length, 0);
+        this.writePipe.write(lengthBuffer);
+        this.writePipe.write(messageBuffer);
+    }
+    writeEnd() {
+        const lengthBuffer = Buffer.alloc(4);
+        lengthBuffer.writeInt32BE(0, 0);
+        this.writePipe.write(lengthBuffer);
+    }
+    readNextMessage() {
+        this.state = 'read length';
+        this.readBuffer(4, (lengthReadError, lengthBuffer) => {
+            if (lengthReadError) {
+                console.error(`Failed to communicate with worker (read length) ${lengthReadError}`);
+                return;
+            }
+            this.state = 'length read';
+            const length = lengthBuffer.readInt32BE(0);
+            this.state = 'read message';
+            this.readBuffer(length, (messageError, messageBuffer) => {
+                if (messageError) {
+                    console.error(`Failed to communicate with worker (read message) ${messageError}`);
+                    return;
+                }
+                this.state = 'message read';
+                const messageString = messageBuffer.toString('utf-8');
+                const message = JSON.parse(messageString, serializer_1.reviver);
+                this.state = 'process message';
+                this.onWorkerMessage(message, err => {
+                    if (err) {
+                        console.error(`Failed to communicate with worker (process message) ${err}`);
+                        return;
+                    }
+                    this.state = 'soon next';
+                    setImmediate(() => this.readNextMessage());
+                });
+            });
+        });
+    }
+    onWorkerMessage(message, finalCallback) {
+        const { type, id } = message;
+        switch (type) {
+            case 'job': {
+                const { data, error, result } = message;
+                (0, mapSeries_1.default)(data, (length, callback) => this.readBuffer(length, callback), (eachErr, buffers) => {
+                    const { callback: jobCallback } = this.jobs[id];
+                    const callback = (err, arg) => {
+                        if (jobCallback) {
+                            delete this.jobs[id];
+                            this.activeJobs -= 1;
+                            this.onJobDone();
+                            if (err) {
+                                jobCallback(err instanceof Error ? err : new Error(err), arg);
+                            }
+                            else {
+                                jobCallback(null, arg);
+                            }
+                        }
+                        finalCallback();
+                    };
+                    if (eachErr) {
+                        callback(eachErr);
+                        return;
+                    }
+                    let bufferPosition = 0;
+                    if (result.result) {
+                        result.result = result.result.map(r => {
+                            if (r.buffer) {
+                                const buffer = buffers[bufferPosition];
+                                bufferPosition += 1;
+                                if (r.string) {
+                                    return buffer.toString('utf-8');
+                                }
+                                return buffer;
+                            }
+                            return r.data;
+                        });
+                    }
+                    if (error) {
+                        callback(this.fromErrorObj(error), result);
+                        return;
+                    }
+                    callback(null, result);
+                });
+                break;
+            }
+            case 'loadModule': {
+                const { request, questionId } = message;
+                const { data } = this.jobs[id];
+                // eslint-disable-next-line no-unused-vars
+                data.loadModule(request, (error, source, sourceMap, module) => {
+                    this.writeJson({
+                        type: 'result',
+                        id: questionId,
+                        error: error
+                            ? {
+                                message: error.message,
+                                details: error.details,
+                                missing: error.missing,
+                            }
+                            : null,
+                        result: [
+                            source,
+                            sourceMap,
+                            // TODO: Serialize module?
+                            // module,
+                        ],
+                    });
+                });
+                finalCallback();
+                break;
+            }
+            case 'resolve': {
+                const { context, request, options, questionId } = message;
+                const { data } = this.jobs[id];
+                if (options) {
+                    data.getResolve(options)(context, request, (error, result) => {
+                        this.writeJson({
+                            type: 'result',
+                            id: questionId,
+                            error: error
+                                ? {
+                                    message: error.message,
+                                    details: error.details,
+                                    missing: error.missing,
+                                }
+                                : null,
+                            result,
+                        });
+                    });
+                }
+                else {
+                    data.resolve(context, request, (error, result) => {
+                        this.writeJson({
+                            type: 'result',
+                            id: questionId,
+                            error: error
+                                ? {
+                                    message: error.message,
+                                    details: error.details,
+                                    missing: error.missing,
+                                }
+                                : null,
+                            result,
+                        });
+                    });
+                }
+                finalCallback();
+                break;
+            }
+            case 'emitWarning': {
+                const { data } = message;
+                const { data: jobData } = this.jobs[id];
+                jobData.emitWarning(this.fromErrorObj(data));
+                finalCallback();
+                break;
+            }
+            case 'emitError': {
+                const { data } = message;
+                const { data: jobData } = this.jobs[id];
+                jobData.emitError(this.fromErrorObj(data));
+                finalCallback();
+                break;
+            }
+            default: {
+                console.error(`Unexpected worker message ${type} in WorkerPool.`);
+                finalCallback();
+                break;
+            }
+        }
+    }
+    fromErrorObj(arg) {
+        let obj;
+        if (typeof arg === 'string') {
+            obj = { message: arg };
+        }
+        else {
+            obj = arg;
+        }
+        return new WorkerError_1.default(obj, this.id);
+    }
+    readBuffer(length, callback) {
+        (0, readBuffer_1.default)(this.readPipe, length, callback);
+    }
+    dispose() {
+        if (!this.disposed) {
+            this.disposed = true;
+            this.ignoreStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr);
+            this.writeEnd();
+        }
+    }
+}
+class WorkerPool {
+    constructor(options) {
+        this.options = options || {};
+        this.numberOfWorkers = options.numberOfWorkers;
+        this.poolTimeout = options.poolTimeout;
+        this.workerNodeArgs = options.workerNodeArgs;
+        this.workerParallelJobs = options.workerParallelJobs;
+        this.workers = new Set();
+        this.activeJobs = 0;
+        this.timeout = null;
+        this.poolQueue = (0, queue_1.default)(this.distributeJob.bind(this), options.poolParallelJobs);
+        this.terminated = false;
+        this.setupLifeCycle();
+    }
+    isAbleToRun() {
+        return !this.terminated;
+    }
+    terminate() {
+        if (this.terminated) {
+            return;
+        }
+        this.terminated = true;
+        this.poolQueue.kill();
+        this.disposeWorkers(true);
+    }
+    setupLifeCycle() {
+        process.on('exit', () => {
+            this.terminate();
+        });
+    }
+    run(data, callback) {
+        if (this.timeout) {
+            clearTimeout(this.timeout);
+            this.timeout = null;
+        }
+        this.activeJobs += 1;
+        this.poolQueue.push(data, callback);
+    }
+    distributeJob(data, callback) {
+        // use worker with the fewest jobs
+        let bestWorker;
+        for (const worker of this.workers) {
+            if (!bestWorker || worker.activeJobs < bestWorker.activeJobs) {
+                bestWorker = worker;
+            }
+        }
+        if (bestWorker && (bestWorker.activeJobs === 0 || this.workers.size >= this.numberOfWorkers)) {
+            bestWorker.run(data, callback);
+            return;
+        }
+        const newWorker = this.createWorker();
+        newWorker.run(data, callback);
+    }
+    createWorker() {
+        // spin up a new worker
+        const newWorker = new PoolWorker({
+            nodeArgs: this.workerNodeArgs,
+            parallelJobs: this.workerParallelJobs,
+        }, () => this.onJobDone());
+        this.workers.add(newWorker);
+        return newWorker;
+    }
+    warmup(requires) {
+        while (this.workers.size < this.numberOfWorkers) {
+            this.createWorker().warmup(requires);
+        }
+    }
+    onJobDone() {
+        this.activeJobs -= 1;
+        if (this.activeJobs === 0 && isFinite(this.poolTimeout)) {
+            this.timeout = setTimeout(() => this.disposeWorkers(), this.poolTimeout);
+        }
+    }
+    disposeWorkers(fromTerminate) {
+        if (!this.options.poolRespawn && !fromTerminate) {
+            this.terminate();
+            return;
+        }
+        if (this.activeJobs === 0 || fromTerminate) {
+            for (const worker of this.workers) {
+                worker.dispose();
+            }
+            this.workers.clear();
+        }
+    }
+}
+exports.default = WorkerPool;

package/dist/thread/readBuffer.d.ts

@@ -0,0 +1 @@
+export default function readBuffer(pipe: any, length: any, callback: any): void;

package/dist/thread/readBuffer.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+function readBuffer(pipe, length, callback) {
+    if (length === 0) {
+        callback(null, Buffer.alloc(0));
+        return;
+    }
+    let remainingLength = length;
+    const buffers = [];
+    const readChunk = () => {
+        const onChunk = arg => {
+            let chunk = arg;
+            let overflow;
+            if (chunk.length > remainingLength) {
+                overflow = chunk.slice(remainingLength);
+                chunk = chunk.slice(0, remainingLength);
+                remainingLength = 0;
+            }
+            else {
+                remainingLength -= chunk.length;
+            }
+            buffers.push(chunk);
+            if (remainingLength === 0) {
+                pipe.removeListener('data', onChunk);
+                pipe.pause();
+                if (overflow) {
+                    pipe.unshift(overflow);
+                }
+                callback(null, Buffer.concat(buffers, length));
+            }
+        };
+        pipe.on('data', onChunk);
+        pipe.resume();
+    };
+    readChunk();
+}
+exports.default = readBuffer;

package/dist/thread/serializer.d.ts

@@ -0,0 +1,2 @@
+export declare function replacer(_key: any, value: any): any;
+export declare function reviver(_key: any, value: any): any;

package/dist/thread/serializer.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.reviver = exports.replacer = void 0;
+function replacer(_key, value) {
+    if (value instanceof RegExp) {
+        return {
+            __serialized_type: 'RegExp',
+            source: value.source,
+            flags: value.flags,
+        };
+    }
+    return value;
+}
+exports.replacer = replacer;
+function reviver(_key, value) {
+    if (typeof value === 'object' && value !== null) {
+        // eslint-disable-next-line no-underscore-dangle
+        if (value.__serialized_type === 'RegExp') {
+            return new RegExp(value.source, value.flags);
+        }
+    }
+    return value;
+}
+exports.reviver = reviver;

package/dist/thread/worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/thread/worker.js

@@ -0,0 +1,185 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const queue_1 = __importDefault(require("neo-async/queue"));
+const BaseError_1 = __importDefault(require("@/base/errors/BaseError"));
+const readBuffer_1 = __importDefault(require("./readBuffer"));
+const serializer_1 = require("./serializer");
+// @ts-ignore
+const writePipe = fs_1.default.createWriteStream(null, { fd: 3 });
+// @ts-ignore
+const readPipe = fs_1.default.createReadStream(null, { fd: 4 });
+const PARALLEL_JOBS = +process.argv[2] || 20;
+let terminated = false;
+const nextQuestionId = 0;
+const callbackMap = Object.create(null);
+function onError(error) {
+    console.error(error);
+}
+function terminateRead() {
+    terminated = true;
+    readPipe.removeAllListeners();
+}
+function terminateWrite() {
+    terminated = true;
+    writePipe.removeAllListeners();
+}
+function onTerminateRead() {
+    terminateRead();
+}
+function onTerminateWrite() {
+    terminateWrite();
+}
+function writePipeWrite(...args) {
+    if (!terminated) {
+        // @ts-ignore
+        writePipe.write(...args);
+    }
+}
+function writePipeCork() {
+    if (!terminated) {
+        writePipe.cork();
+    }
+}
+function writePipeUncork() {
+    if (!terminated) {
+        writePipe.uncork();
+    }
+}
+function terminate() {
+    terminateRead();
+    terminateWrite();
+}
+function toErrorObj(err) {
+    return {
+        message: err.message,
+        details: err.details,
+        stack: err.stack,
+        hideStack: err.hideStack,
+    };
+}
+function toNativeError(obj) {
+    if (!obj)
+        return null;
+    const err = new BaseError_1.default(obj.message);
+    err.details = obj.details;
+    err.missing = obj.missing;
+    return err;
+}
+function writeJson(data) {
+    writePipeCork();
+    process.nextTick(() => {
+        writePipeUncork();
+    });
+    const lengthBuffer = Buffer.alloc(4);
+    const messageBuffer = Buffer.from(JSON.stringify(data, serializer_1.replacer), 'utf-8');
+    lengthBuffer.writeInt32BE(messageBuffer.length, 0);
+    writePipeWrite(lengthBuffer);
+    writePipeWrite(messageBuffer);
+}
+writePipe.on('finish', onTerminateWrite);
+readPipe.on('end', onTerminateRead);
+writePipe.on('close', onTerminateWrite);
+readPipe.on('close', onTerminateRead);
+readPipe.on('error', onError);
+writePipe.on('error', onError);
+const queue = (0, queue_1.default)(({ id, data }, taskCallback) => {
+    try {
+        // writeJson({
+        //   type: 'job',
+        //   id,
+        // error: err && toErrorObj(err),
+        // result: {
+        //   result: convertedResult,
+        //   cacheable,
+        //   fileDependencies,
+        //   contextDependencies,
+        //   missingDependencies,
+        //   buildDependencies,
+        // },
+        // data: buffersToSend.map(buffer => buffer.length),
+        // })
+    }
+    catch (e) {
+        writeJson({
+            type: 'job',
+            id,
+            error: toErrorObj(e),
+        });
+        taskCallback();
+    }
+}, PARALLEL_JOBS);
+function dispose() {
+    terminate();
+    queue.kill();
+    process.exit(0);
+}
+function onMessage(message) {
+    try {
+        const { type, id } = message;
+        switch (type) {
+            case 'job': {
+                queue.push(message);
+                break;
+            }
+            case 'result': {
+                const { error, result } = message;
+                const callback = callbackMap[id];
+                if (callback) {
+                    const nativeError = toNativeError(error);
+                    callback(nativeError, result);
+                }
+                else {
+                    console.error(`Worker got unexpected result id ${id}`);
+                }
+                delete callbackMap[id];
+                break;
+            }
+            case 'warmup': {
+                const { requires } = message;
+                // load modules into process
+                requires.forEach(r => require(r));
+                break;
+            }
+            default: {
+                console.error(`Worker got unexpected job type ${type}`);
+                break;
+            }
+        }
+    }
+    catch (e) {
+        console.error(`Error in worker ${e}`);
+    }
+}
+function readNextMessage() {
+    (0, readBuffer_1.default)(readPipe, 4, (lengthReadError, lengthBuffer) => {
+        if (lengthReadError) {
+            console.error(`Failed to communicate with main process (read length) ${lengthReadError}`);
+            return;
+        }
+        const length = lengthBuffer.length && lengthBuffer.readInt32BE(0);
+        if (length === 0) {
+            // worker should dispose and exit
+            dispose();
+            return;
+        }
+        (0, readBuffer_1.default)(readPipe, length, (messageError, messageBuffer) => {
+            if (terminated) {
+                return;
+            }
+            if (messageError) {
+                console.error(`Failed to communicate with main process (read message) ${messageError}`);
+                return;
+            }
+            const messageString = messageBuffer.toString('utf-8');
+            const message = JSON.parse(messageString, serializer_1.reviver);
+            onMessage(message);
+            setImmediate(() => readNextMessage());
+        });
+    });
+}
+// start reading messages from main process
+readNextMessage();

package/dist/thread/workerPools.d.ts

@@ -0,0 +1,2 @@
+declare function getPool(options: any): any;
+export { getPool };

package/dist/thread/workerPools.js

@@ -0,0 +1,31 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPool = void 0;
+const os_1 = __importDefault(require("os"));
+const WorkerPool_1 = __importDefault(require("./WorkerPool"));
+const workerPools = Object.create(null);
+function calculateNumberOfWorkers() {
+    // There are situations when this call will return undefined so
+    // we are fallback here to 1.
+    // More info on: https://github.com/nodejs/node/issues/19022
+    const cpus = os_1.default.cpus() || { length: 1 };
+    return Math.max(1, cpus.length - 1);
+}
+function getPool(options) {
+    const workerPoolOptions = {
+        name: options.name || '',
+        numberOfWorkers: options.workers || calculateNumberOfWorkers(),
+        workerNodeArgs: options.workerNodeArgs,
+        workerParallelJobs: options.workerParallelJobs || 20,
+        poolTimeout: options.poolTimeout || 500,
+        poolParallelJobs: options.poolParallelJobs || 200,
+        poolRespawn: options.poolRespawn || false,
+    };
+    const tpKey = JSON.stringify(workerPoolOptions);
+    workerPools[tpKey] = workerPools[tpKey] || new WorkerPool_1.default(workerPoolOptions);
+    return workerPools[tpKey];
+}
+exports.getPool = getPool;