parallel-park 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/dist/index.js +10 -4
- package/package.json +6 -16
- package/src/index.ts +7 -3
- package/tsconfig.json +3 -20
- package/.node-version +0 -1
- package/.npm-version +0 -1
- package/dist/child-process-worker.js +0 -66
- package/dist/in-child-process.js +0 -124
- package/dist/read-until-end.js +0 -25
- package/dist/run-jobs.js +0 -100
- package/src/child-process-worker.ts +0 -86
- package/src/in-child-process.ts +0 -161
- package/src/read-until-end.ts +0 -24
- package/src/run-jobs.ts +0 -136
package/LICENSE
CHANGED
package/dist/index.js
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
const
|
|
5
|
-
Object.defineProperty(exports, "runJobs", { enumerable: true, get: function () { return run_jobs_1.runJobs; } });
|
|
6
|
-
const in_child_process_1 = require("./in-child-process");
|
|
6
|
+
exports.runJobs = exports.inChildProcess = void 0;
|
|
7
|
+
const in_child_process_1 = require("@parallel-park/in-child-process");
|
|
7
8
|
Object.defineProperty(exports, "inChildProcess", { enumerable: true, get: function () { return in_child_process_1.inChildProcess; } });
|
|
9
|
+
const run_jobs_1 = require("@parallel-park/run-jobs");
|
|
10
|
+
Object.defineProperty(exports, "runJobs", { enumerable: true, get: function () { return run_jobs_1.runJobs; } });
|
|
11
|
+
const debug_1 = __importDefault(require("debug"));
|
|
12
|
+
const runJobsDebug = (0, debug_1.default)("parallel-park:run-jobs");
|
|
13
|
+
(0, run_jobs_1.setDebug)(runJobsDebug);
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "parallel-park",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Parallel/concurrent async work, optionally using multiple processes",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "src/index.ts",
|
|
7
7
|
"scripts": {
|
|
8
|
-
"test": "
|
|
9
|
-
"build": "tsc"
|
|
8
|
+
"test": "vitest run",
|
|
9
|
+
"build": "rm -rf ./dist && tsc"
|
|
10
10
|
},
|
|
11
11
|
"keywords": [
|
|
12
12
|
"parallel",
|
|
@@ -27,19 +27,9 @@
|
|
|
27
27
|
"type": "git",
|
|
28
28
|
"url": "git+https://github.com/suchipi/parallel-park.git"
|
|
29
29
|
},
|
|
30
|
-
"devDependencies": {
|
|
31
|
-
"@babel/core": "^7.24.5",
|
|
32
|
-
"@babel/preset-env": "^7.24.5",
|
|
33
|
-
"@babel/preset-typescript": "^7.24.1",
|
|
34
|
-
"@types/debug": "^4.1.12",
|
|
35
|
-
"@types/jest": "^29.5.12",
|
|
36
|
-
"@types/node": "^20.12.10",
|
|
37
|
-
"babel-jest": "^29.7.0",
|
|
38
|
-
"jest": "^29.7.0",
|
|
39
|
-
"typescript": "^5.4.5"
|
|
40
|
-
},
|
|
41
30
|
"dependencies": {
|
|
42
|
-
"@
|
|
43
|
-
"
|
|
31
|
+
"@parallel-park/in-child-process": "0.3.0",
|
|
32
|
+
"@parallel-park/run-jobs": "0.3.0",
|
|
33
|
+
"debug": "^4.4.3"
|
|
44
34
|
}
|
|
45
35
|
}
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { inChildProcess } from "@parallel-park/in-child-process";
|
|
2
|
+
import { runJobs, setDebug as setRunJobsDebug } from "@parallel-park/run-jobs";
|
|
3
|
+
import makeDebug from "debug";
|
|
3
4
|
|
|
4
|
-
|
|
5
|
+
const runJobsDebug = makeDebug("parallel-park:run-jobs");
|
|
6
|
+
setRunJobsDebug(runJobsDebug);
|
|
7
|
+
|
|
8
|
+
export { inChildProcess, runJobs };
|
package/tsconfig.json
CHANGED
|
@@ -1,27 +1,10 @@
|
|
|
1
1
|
{
|
|
2
|
+
"extends": "../../tsconfig.json",
|
|
3
|
+
|
|
2
4
|
"include": ["./src/**/*.ts"],
|
|
3
5
|
"exclude": ["./**/*.test.ts"],
|
|
4
6
|
|
|
5
7
|
"compilerOptions": {
|
|
6
|
-
"
|
|
7
|
-
"target": "es2018",
|
|
8
|
-
"module": "CommonJS",
|
|
9
|
-
"outDir": "./dist",
|
|
10
|
-
|
|
11
|
-
"strict": true,
|
|
12
|
-
"noImplicitAny": false,
|
|
13
|
-
"strictNullChecks": true,
|
|
14
|
-
"strictFunctionTypes": true,
|
|
15
|
-
"strictPropertyInitialization": true,
|
|
16
|
-
"noImplicitThis": true,
|
|
17
|
-
"alwaysStrict": true,
|
|
18
|
-
"noUnusedLocals": false,
|
|
19
|
-
"noUnusedParameters": false,
|
|
20
|
-
"noImplicitReturns": true,
|
|
21
|
-
"noFallthroughCasesInSwitch": true,
|
|
22
|
-
"downlevelIteration": true,
|
|
23
|
-
|
|
24
|
-
"moduleResolution": "node",
|
|
25
|
-
"esModuleInterop": true
|
|
8
|
+
"outDir": "./dist"
|
|
26
9
|
}
|
|
27
10
|
}
|
package/.node-version
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
v20.11.1
|
package/.npm-version
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
10.2.4
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const fs_1 = __importDefault(require("fs"));
|
|
7
|
-
const vm_1 = __importDefault(require("vm"));
|
|
8
|
-
const make_module_env_1 = __importDefault(require("make-module-env"));
|
|
9
|
-
const debug_1 = __importDefault(require("debug"));
|
|
10
|
-
const read_until_end_1 = require("./read-until-end");
|
|
11
|
-
const path_1 = __importDefault(require("path"));
|
|
12
|
-
const debug = (0, debug_1.default)("parallel-park:child-process-worker");
|
|
13
|
-
const commsIn = fs_1.default.createReadStream(
|
|
14
|
-
// @ts-ignore
|
|
15
|
-
null, { fd: 3 });
|
|
16
|
-
const commsOut = fs_1.default.createWriteStream(
|
|
17
|
-
// @ts-ignore
|
|
18
|
-
null, { fd: 4 });
|
|
19
|
-
debug("reading input data...");
|
|
20
|
-
(0, read_until_end_1.readUntilEnd)(commsIn)
|
|
21
|
-
.then((data) => {
|
|
22
|
-
debug("parsing input data...");
|
|
23
|
-
try {
|
|
24
|
-
const [inputs, fnString, callingFile] = JSON.parse(data);
|
|
25
|
-
onReady(inputs, fnString, callingFile);
|
|
26
|
-
}
|
|
27
|
-
catch (err) {
|
|
28
|
-
onError(err);
|
|
29
|
-
}
|
|
30
|
-
})
|
|
31
|
-
.catch(onError);
|
|
32
|
-
function onReady(inputs, fnString, callingFile) {
|
|
33
|
-
debug("in onReady", { inputs, fnString, callingFile });
|
|
34
|
-
// Relevant when callingFile is eg. "REPL2" (from Node.js repl)
|
|
35
|
-
if (!path_1.default.isAbsolute(callingFile)) {
|
|
36
|
-
callingFile = path_1.default.join(process.cwd(), "fake-path.js");
|
|
37
|
-
}
|
|
38
|
-
const wrapperFn = vm_1.default.runInThisContext(`(function moduleWrapper(exports, require, module, __filename, __dirname) {
|
|
39
|
-
return ${fnString};})`);
|
|
40
|
-
const env = (0, make_module_env_1.default)(callingFile);
|
|
41
|
-
const fn = wrapperFn(env.exports, env.require, env.module, env.__filename, env.__dirname);
|
|
42
|
-
const result = fn(inputs);
|
|
43
|
-
if (typeof result === "object" &&
|
|
44
|
-
result != null &&
|
|
45
|
-
typeof result.then === "function") {
|
|
46
|
-
result.then(onSuccess, onError);
|
|
47
|
-
}
|
|
48
|
-
else {
|
|
49
|
-
onSuccess(result);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
function onSuccess(data) {
|
|
53
|
-
debug("in onSuccess", { data });
|
|
54
|
-
commsOut.end(JSON.stringify({ type: "success", data }));
|
|
55
|
-
}
|
|
56
|
-
function onError(error) {
|
|
57
|
-
debug("in onError", { error });
|
|
58
|
-
commsOut.end(JSON.stringify({
|
|
59
|
-
type: "error",
|
|
60
|
-
error: {
|
|
61
|
-
name: error.name,
|
|
62
|
-
message: error.message,
|
|
63
|
-
stack: error.stack,
|
|
64
|
-
},
|
|
65
|
-
}));
|
|
66
|
-
}
|
package/dist/in-child-process.js
DELETED
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.inChildProcess = void 0;
|
|
7
|
-
const child_process_1 = __importDefault(require("child_process"));
|
|
8
|
-
const error_utils_1 = require("@suchipi/error-utils");
|
|
9
|
-
const debug_1 = __importDefault(require("debug"));
|
|
10
|
-
const read_until_end_1 = require("./read-until-end");
|
|
11
|
-
const debug = (0, debug_1.default)("parallel-park:in-child-process");
|
|
12
|
-
const runnerPath = require.resolve("../dist/child-process-worker");
|
|
13
|
-
const inChildProcess = (...args) => {
|
|
14
|
-
var _a;
|
|
15
|
-
const inputs = typeof args[0] === "function" ? {} : args[0];
|
|
16
|
-
const functionToRun = typeof args[0] === "function" ? args[0] : args[1];
|
|
17
|
-
if (typeof inputs !== "object") {
|
|
18
|
-
throw new Error("The first argument to inChildProcess should be an object of input data to pass to the child process.");
|
|
19
|
-
}
|
|
20
|
-
if (typeof functionToRun !== "function") {
|
|
21
|
-
throw new Error("The second argument to inChildProcess should be a function to run in the child process.");
|
|
22
|
-
}
|
|
23
|
-
const here = new error_utils_1.ParsedError(new Error("here"));
|
|
24
|
-
const callingFrame = here.stackFrames[1];
|
|
25
|
-
const callingFile = (_a = callingFrame === null || callingFrame === void 0 ? void 0 : callingFrame.fileName) !== null && _a !== void 0 ? _a : "unknown file";
|
|
26
|
-
debug("spawning child process:", [process.argv[0], runnerPath]);
|
|
27
|
-
const child = child_process_1.default.spawn(process.argv[0], [runnerPath], {
|
|
28
|
-
stdio: ["inherit", "inherit", "inherit", "pipe", "pipe"],
|
|
29
|
-
});
|
|
30
|
-
return new Promise((resolve, reject) => {
|
|
31
|
-
child.on("error", reject);
|
|
32
|
-
const commsOut = child.stdio[3];
|
|
33
|
-
const commsIn = child.stdio[4];
|
|
34
|
-
child.on("spawn", () => {
|
|
35
|
-
const dataToSend = JSON.stringify([
|
|
36
|
-
inputs,
|
|
37
|
-
functionToRun.toString(),
|
|
38
|
-
callingFile,
|
|
39
|
-
]);
|
|
40
|
-
debug("sending inputs to child process:", dataToSend);
|
|
41
|
-
commsOut.end(dataToSend, "utf-8");
|
|
42
|
-
});
|
|
43
|
-
let receivedData = "";
|
|
44
|
-
(0, read_until_end_1.readUntilEnd)(commsIn).then((data) => {
|
|
45
|
-
debug("received data from child process");
|
|
46
|
-
receivedData = data;
|
|
47
|
-
});
|
|
48
|
-
child.on("close", (code, signal) => {
|
|
49
|
-
debug("child process closed:", { code, signal });
|
|
50
|
-
if (code !== 0) {
|
|
51
|
-
reject(new Error(`Child process exited with nonzero status code: ${JSON.stringify({
|
|
52
|
-
code,
|
|
53
|
-
signal,
|
|
54
|
-
})}`));
|
|
55
|
-
}
|
|
56
|
-
else {
|
|
57
|
-
debug("parsing received data from child process...");
|
|
58
|
-
const result = JSON.parse(receivedData);
|
|
59
|
-
switch (result.type) {
|
|
60
|
-
case "success": {
|
|
61
|
-
resolve(result.data);
|
|
62
|
-
break;
|
|
63
|
-
}
|
|
64
|
-
case "error": {
|
|
65
|
-
const error = new Error(result.error.message);
|
|
66
|
-
Object.defineProperty(error, "name", { value: result.error.name });
|
|
67
|
-
Object.defineProperty(error, "stack", {
|
|
68
|
-
value: result.error.name +
|
|
69
|
-
": " +
|
|
70
|
-
result.error.message +
|
|
71
|
-
"\n" +
|
|
72
|
-
result.error.stack
|
|
73
|
-
.split("\n")
|
|
74
|
-
.slice(1)
|
|
75
|
-
.filter((line) => !/node:internal|node:events/.test(line))
|
|
76
|
-
.map((line) => {
|
|
77
|
-
if (/evalmachine/.test(line)) {
|
|
78
|
-
const lineWithoutEvalMachine = line.replace(/evalmachine(?:\.<anonymous>)?/, "<function passed into inChildProcess>");
|
|
79
|
-
const matches = line.match(/:(\d+):(\d+)\)?$/);
|
|
80
|
-
if (!matches) {
|
|
81
|
-
return lineWithoutEvalMachine;
|
|
82
|
-
}
|
|
83
|
-
else {
|
|
84
|
-
let [_, row, col] = matches;
|
|
85
|
-
// subtract 1 from row to skip the module wrapper function line
|
|
86
|
-
row = row - 1;
|
|
87
|
-
// subtract the length of the `return ` keywords in front of the function
|
|
88
|
-
if (row === 1) {
|
|
89
|
-
col = col - `return `.length;
|
|
90
|
-
}
|
|
91
|
-
const hadParen = /\)$/.test(lineWithoutEvalMachine);
|
|
92
|
-
return lineWithoutEvalMachine.replace(/:\d+:\d+\)?$/, `:${row}:${col - 1}${hadParen ? ")" : ""}`);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
else {
|
|
96
|
-
return line;
|
|
97
|
-
}
|
|
98
|
-
})
|
|
99
|
-
.join("\n") +
|
|
100
|
-
"\n" +
|
|
101
|
-
error
|
|
102
|
-
.stack.split("\n")
|
|
103
|
-
.slice(1)
|
|
104
|
-
.filter((line) => !/node:internal|node:events/.test(line))
|
|
105
|
-
.join("\n") +
|
|
106
|
-
"\n" +
|
|
107
|
-
here
|
|
108
|
-
.stack.split("\n")
|
|
109
|
-
.slice(2)
|
|
110
|
-
.filter((line) => !/node:internal|node:events/.test(line))
|
|
111
|
-
.join("\n"),
|
|
112
|
-
});
|
|
113
|
-
reject(error);
|
|
114
|
-
break;
|
|
115
|
-
}
|
|
116
|
-
default: {
|
|
117
|
-
reject(new Error(`Internal parallel-park error: unhandled result type: ${result.type}`));
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
});
|
|
122
|
-
});
|
|
123
|
-
};
|
|
124
|
-
exports.inChildProcess = inChildProcess;
|
package/dist/read-until-end.js
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.readUntilEnd = void 0;
|
|
7
|
-
const debug_1 = __importDefault(require("debug"));
|
|
8
|
-
const debug = (0, debug_1.default)("parallel-park:read-until-end");
|
|
9
|
-
let streamId = 0;
|
|
10
|
-
function readUntilEnd(stream) {
|
|
11
|
-
const id = streamId;
|
|
12
|
-
streamId++;
|
|
13
|
-
return new Promise((resolve) => {
|
|
14
|
-
let data = "";
|
|
15
|
-
stream.on("data", (chunk) => {
|
|
16
|
-
debug("received data chunk from stream", id);
|
|
17
|
-
data += chunk.toString("utf-8");
|
|
18
|
-
});
|
|
19
|
-
stream.on("close", () => {
|
|
20
|
-
debug(`stream ${id} closed; resolving`);
|
|
21
|
-
resolve(data);
|
|
22
|
-
});
|
|
23
|
-
});
|
|
24
|
-
}
|
|
25
|
-
exports.readUntilEnd = readUntilEnd;
|
package/dist/run-jobs.js
DELETED
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.runJobs = void 0;
|
|
7
|
-
const debug_1 = __importDefault(require("debug"));
|
|
8
|
-
const debug = (0, debug_1.default)("parallel-park:run-jobs");
|
|
9
|
-
function isThenable(value) {
|
|
10
|
-
return (typeof value === "object" &&
|
|
11
|
-
value != null &&
|
|
12
|
-
// @ts-ignore accessing .then
|
|
13
|
-
typeof value.then === "function");
|
|
14
|
-
}
|
|
15
|
-
const NOTHING = Symbol("NOTHING");
|
|
16
|
-
let runJobsCallId = 0;
|
|
17
|
-
async function runJobs(inputs, mapper, {
|
|
18
|
-
/**
|
|
19
|
-
* How many jobs are allowed to run at once.
|
|
20
|
-
*/
|
|
21
|
-
concurrency = 8, } = {}) {
|
|
22
|
-
const callId = runJobsCallId;
|
|
23
|
-
runJobsCallId++;
|
|
24
|
-
debug(`runJobs called (callId: ${callId})`, { inputs, mapper, concurrency });
|
|
25
|
-
if (concurrency < 1) {
|
|
26
|
-
throw new Error("Concurrency can't be less than one; that doesn't make any sense.");
|
|
27
|
-
}
|
|
28
|
-
const inputsArray = [];
|
|
29
|
-
const inputIteratorFactory = inputs[Symbol.asyncIterator || NOTHING] || inputs[Symbol.iterator];
|
|
30
|
-
const inputIterator = inputIteratorFactory.call(inputs);
|
|
31
|
-
const maybeLength = Array.isArray(inputs) ? inputs.length : null;
|
|
32
|
-
let iteratorDone = false;
|
|
33
|
-
async function readInput() {
|
|
34
|
-
debug(`reading next input (callId: ${callId})`);
|
|
35
|
-
let nextResult = inputIterator.next();
|
|
36
|
-
if (isThenable(nextResult)) {
|
|
37
|
-
nextResult = await nextResult;
|
|
38
|
-
}
|
|
39
|
-
if (nextResult.done) {
|
|
40
|
-
iteratorDone = true;
|
|
41
|
-
return false;
|
|
42
|
-
}
|
|
43
|
-
else {
|
|
44
|
-
let value = nextResult.value;
|
|
45
|
-
if (isThenable(value)) {
|
|
46
|
-
value = await value;
|
|
47
|
-
}
|
|
48
|
-
inputsArray.push(value);
|
|
49
|
-
return true;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
let unstartedIndex = 0;
|
|
53
|
-
const results = new Array(maybeLength || 0);
|
|
54
|
-
const runningPromises = new Set();
|
|
55
|
-
let error = null;
|
|
56
|
-
async function takeInput() {
|
|
57
|
-
const read = await readInput();
|
|
58
|
-
if (!read)
|
|
59
|
-
return;
|
|
60
|
-
const inputIndex = unstartedIndex;
|
|
61
|
-
unstartedIndex++;
|
|
62
|
-
const input = inputsArray[inputIndex];
|
|
63
|
-
debug(`mapping input into Promise (callId: ${callId})`);
|
|
64
|
-
const promise = mapper(input, inputIndex, maybeLength || Infinity);
|
|
65
|
-
if (!isThenable(promise)) {
|
|
66
|
-
throw new Error("Mapper function passed into runJobs didn't return a Promise. The mapper function should always return a Promise. The easiest way to ensure this is the case is to make your mapper function an async function.");
|
|
67
|
-
}
|
|
68
|
-
const promiseWithMore = promise.then((result) => {
|
|
69
|
-
debug(`child Promise resolved for input (callId: ${callId}):`, input);
|
|
70
|
-
results[inputIndex] = result;
|
|
71
|
-
runningPromises.delete(promiseWithMore);
|
|
72
|
-
}, (err) => {
|
|
73
|
-
debug(`child Promise rejected for input (callId: ${callId}):`, input, "with error:", err);
|
|
74
|
-
runningPromises.delete(promiseWithMore);
|
|
75
|
-
error = err;
|
|
76
|
-
});
|
|
77
|
-
runningPromises.add(promiseWithMore);
|
|
78
|
-
}
|
|
79
|
-
async function proceed() {
|
|
80
|
-
while (!iteratorDone && runningPromises.size < concurrency) {
|
|
81
|
-
await takeInput();
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
await proceed();
|
|
85
|
-
while (runningPromises.size > 0 && !error) {
|
|
86
|
-
await Promise.race(runningPromises.values());
|
|
87
|
-
if (error) {
|
|
88
|
-
debug(`throwing error (callId: ${callId})`);
|
|
89
|
-
throw error;
|
|
90
|
-
}
|
|
91
|
-
await proceed();
|
|
92
|
-
}
|
|
93
|
-
if (error) {
|
|
94
|
-
debug(`throwing error (callId: ${callId})`);
|
|
95
|
-
throw error;
|
|
96
|
-
}
|
|
97
|
-
debug(`all done (callId: ${callId})`);
|
|
98
|
-
return results;
|
|
99
|
-
}
|
|
100
|
-
exports.runJobs = runJobs;
|
|
@@ -1,86 +0,0 @@
|
|
|
1
|
-
import fs from "fs";
|
|
2
|
-
import vm from "vm";
|
|
3
|
-
import makeModuleEnv from "make-module-env";
|
|
4
|
-
import makeDebug from "debug";
|
|
5
|
-
import { readUntilEnd } from "./read-until-end";
|
|
6
|
-
import path from "path";
|
|
7
|
-
|
|
8
|
-
const debug = makeDebug("parallel-park:child-process-worker");
|
|
9
|
-
|
|
10
|
-
const commsIn = fs.createReadStream(
|
|
11
|
-
// @ts-ignore
|
|
12
|
-
null,
|
|
13
|
-
{ fd: 3 }
|
|
14
|
-
);
|
|
15
|
-
const commsOut = fs.createWriteStream(
|
|
16
|
-
// @ts-ignore
|
|
17
|
-
null,
|
|
18
|
-
{ fd: 4 }
|
|
19
|
-
);
|
|
20
|
-
|
|
21
|
-
debug("reading input data...");
|
|
22
|
-
readUntilEnd(commsIn)
|
|
23
|
-
.then((data) => {
|
|
24
|
-
debug("parsing input data...");
|
|
25
|
-
try {
|
|
26
|
-
const [inputs, fnString, callingFile] = JSON.parse(data);
|
|
27
|
-
onReady(inputs, fnString, callingFile);
|
|
28
|
-
} catch (err) {
|
|
29
|
-
onError(err as Error);
|
|
30
|
-
}
|
|
31
|
-
})
|
|
32
|
-
.catch(onError);
|
|
33
|
-
|
|
34
|
-
function onReady(inputs: any, fnString: string, callingFile: string) {
|
|
35
|
-
debug("in onReady", { inputs, fnString, callingFile });
|
|
36
|
-
|
|
37
|
-
// Relevant when callingFile is eg. "REPL2" (from Node.js repl)
|
|
38
|
-
if (!path.isAbsolute(callingFile)) {
|
|
39
|
-
callingFile = path.join(process.cwd(), "fake-path.js");
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
const wrapperFn = vm.runInThisContext(
|
|
43
|
-
`(function moduleWrapper(exports, require, module, __filename, __dirname) {
|
|
44
|
-
return ${fnString};})`
|
|
45
|
-
);
|
|
46
|
-
const env = makeModuleEnv(callingFile);
|
|
47
|
-
const fn = wrapperFn(
|
|
48
|
-
env.exports,
|
|
49
|
-
env.require,
|
|
50
|
-
env.module,
|
|
51
|
-
env.__filename,
|
|
52
|
-
env.__dirname
|
|
53
|
-
);
|
|
54
|
-
const result = fn(inputs);
|
|
55
|
-
|
|
56
|
-
if (
|
|
57
|
-
typeof result === "object" &&
|
|
58
|
-
result != null &&
|
|
59
|
-
typeof result.then === "function"
|
|
60
|
-
) {
|
|
61
|
-
result.then(onSuccess, onError);
|
|
62
|
-
} else {
|
|
63
|
-
onSuccess(result);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
function onSuccess(data: any) {
|
|
68
|
-
debug("in onSuccess", { data });
|
|
69
|
-
|
|
70
|
-
commsOut.end(JSON.stringify({ type: "success", data }));
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
function onError(error: Error) {
|
|
74
|
-
debug("in onError", { error });
|
|
75
|
-
|
|
76
|
-
commsOut.end(
|
|
77
|
-
JSON.stringify({
|
|
78
|
-
type: "error",
|
|
79
|
-
error: {
|
|
80
|
-
name: error.name,
|
|
81
|
-
message: error.message,
|
|
82
|
-
stack: error.stack,
|
|
83
|
-
},
|
|
84
|
-
})
|
|
85
|
-
);
|
|
86
|
-
}
|
package/src/in-child-process.ts
DELETED
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
import child_process from "child_process";
|
|
2
|
-
import type * as stream from "stream";
|
|
3
|
-
import { ParsedError } from "@suchipi/error-utils";
|
|
4
|
-
import makeDebug from "debug";
|
|
5
|
-
import { readUntilEnd } from "./read-until-end";
|
|
6
|
-
|
|
7
|
-
const debug = makeDebug("parallel-park:in-child-process");
|
|
8
|
-
|
|
9
|
-
const runnerPath = require.resolve("../dist/child-process-worker");
|
|
10
|
-
|
|
11
|
-
type InChildProcess = {
|
|
12
|
-
<Inputs extends { [key: string]: any }, Result>(
|
|
13
|
-
inputs: Inputs,
|
|
14
|
-
functionToRun: (inputs: Inputs) => Result | Promise<Result>
|
|
15
|
-
): Promise<Result>;
|
|
16
|
-
|
|
17
|
-
<Result>(functionToRun: () => Result | Promise<Result>): Promise<Result>;
|
|
18
|
-
};
|
|
19
|
-
|
|
20
|
-
export const inChildProcess: InChildProcess = (...args: Array<any>) => {
|
|
21
|
-
const inputs = typeof args[0] === "function" ? {} : args[0];
|
|
22
|
-
const functionToRun = typeof args[0] === "function" ? args[0] : args[1];
|
|
23
|
-
|
|
24
|
-
if (typeof inputs !== "object") {
|
|
25
|
-
throw new Error(
|
|
26
|
-
"The first argument to inChildProcess should be an object of input data to pass to the child process."
|
|
27
|
-
);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
if (typeof functionToRun !== "function") {
|
|
31
|
-
throw new Error(
|
|
32
|
-
"The second argument to inChildProcess should be a function to run in the child process."
|
|
33
|
-
);
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
const here = new ParsedError(new Error("here"));
|
|
37
|
-
|
|
38
|
-
const callingFrame = here.stackFrames[1];
|
|
39
|
-
const callingFile = callingFrame?.fileName ?? "unknown file";
|
|
40
|
-
|
|
41
|
-
debug("spawning child process:", [process.argv[0], runnerPath]);
|
|
42
|
-
|
|
43
|
-
const child = child_process.spawn(process.argv[0], [runnerPath], {
|
|
44
|
-
stdio: ["inherit", "inherit", "inherit", "pipe", "pipe"],
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
return new Promise((resolve, reject) => {
|
|
48
|
-
child.on("error", reject);
|
|
49
|
-
|
|
50
|
-
const commsOut: stream.Writable = child.stdio![3] as any;
|
|
51
|
-
const commsIn: stream.Readable = child.stdio![4] as any;
|
|
52
|
-
|
|
53
|
-
child.on("spawn", () => {
|
|
54
|
-
const dataToSend = JSON.stringify([
|
|
55
|
-
inputs,
|
|
56
|
-
functionToRun.toString(),
|
|
57
|
-
callingFile,
|
|
58
|
-
]);
|
|
59
|
-
debug("sending inputs to child process:", dataToSend);
|
|
60
|
-
commsOut.end(dataToSend, "utf-8");
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
let receivedData = "";
|
|
64
|
-
readUntilEnd(commsIn).then((data) => {
|
|
65
|
-
debug("received data from child process");
|
|
66
|
-
receivedData = data;
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
child.on("close", (code, signal) => {
|
|
70
|
-
debug("child process closed:", { code, signal });
|
|
71
|
-
|
|
72
|
-
if (code !== 0) {
|
|
73
|
-
reject(
|
|
74
|
-
new Error(
|
|
75
|
-
`Child process exited with nonzero status code: ${JSON.stringify({
|
|
76
|
-
code,
|
|
77
|
-
signal,
|
|
78
|
-
})}`
|
|
79
|
-
)
|
|
80
|
-
);
|
|
81
|
-
} else {
|
|
82
|
-
debug("parsing received data from child process...");
|
|
83
|
-
const result = JSON.parse(receivedData);
|
|
84
|
-
switch (result.type) {
|
|
85
|
-
case "success": {
|
|
86
|
-
resolve(result.data);
|
|
87
|
-
break;
|
|
88
|
-
}
|
|
89
|
-
case "error": {
|
|
90
|
-
const error = new Error(result.error.message);
|
|
91
|
-
Object.defineProperty(error, "name", { value: result.error.name });
|
|
92
|
-
Object.defineProperty(error, "stack", {
|
|
93
|
-
value:
|
|
94
|
-
result.error.name +
|
|
95
|
-
": " +
|
|
96
|
-
result.error.message +
|
|
97
|
-
"\n" +
|
|
98
|
-
result.error.stack
|
|
99
|
-
.split("\n")
|
|
100
|
-
.slice(1)
|
|
101
|
-
.filter((line) => !/node:internal|node:events/.test(line))
|
|
102
|
-
.map((line) => {
|
|
103
|
-
if (/evalmachine/.test(line)) {
|
|
104
|
-
const lineWithoutEvalMachine = line.replace(
|
|
105
|
-
/evalmachine(?:\.<anonymous>)?/,
|
|
106
|
-
"<function passed into inChildProcess>"
|
|
107
|
-
);
|
|
108
|
-
|
|
109
|
-
const matches = line.match(/:(\d+):(\d+)\)?$/);
|
|
110
|
-
if (!matches) {
|
|
111
|
-
return lineWithoutEvalMachine;
|
|
112
|
-
} else {
|
|
113
|
-
let [_, row, col] = matches;
|
|
114
|
-
// subtract 1 from row to skip the module wrapper function line
|
|
115
|
-
row = row - 1;
|
|
116
|
-
|
|
117
|
-
// subtract the length of the `return ` keywords in front of the function
|
|
118
|
-
if (row === 1) {
|
|
119
|
-
col = col - `return `.length;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
const hadParen = /\)$/.test(lineWithoutEvalMachine);
|
|
123
|
-
|
|
124
|
-
return lineWithoutEvalMachine.replace(
|
|
125
|
-
/:\d+:\d+\)?$/,
|
|
126
|
-
`:${row}:${col - 1}${hadParen ? ")" : ""}`
|
|
127
|
-
);
|
|
128
|
-
}
|
|
129
|
-
} else {
|
|
130
|
-
return line;
|
|
131
|
-
}
|
|
132
|
-
})
|
|
133
|
-
.join("\n") +
|
|
134
|
-
"\n" +
|
|
135
|
-
error
|
|
136
|
-
.stack!.split("\n")
|
|
137
|
-
.slice(1)
|
|
138
|
-
.filter((line) => !/node:internal|node:events/.test(line))
|
|
139
|
-
.join("\n") +
|
|
140
|
-
"\n" +
|
|
141
|
-
here
|
|
142
|
-
.stack!.split("\n")
|
|
143
|
-
.slice(2)
|
|
144
|
-
.filter((line) => !/node:internal|node:events/.test(line))
|
|
145
|
-
.join("\n"),
|
|
146
|
-
});
|
|
147
|
-
reject(error);
|
|
148
|
-
break;
|
|
149
|
-
}
|
|
150
|
-
default: {
|
|
151
|
-
reject(
|
|
152
|
-
new Error(
|
|
153
|
-
`Internal parallel-park error: unhandled result type: ${result.type}`
|
|
154
|
-
)
|
|
155
|
-
);
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
});
|
|
160
|
-
});
|
|
161
|
-
};
|
package/src/read-until-end.ts
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import type stream from "stream";
|
|
2
|
-
import makeDebug from "debug";
|
|
3
|
-
const debug = makeDebug("parallel-park:read-until-end");
|
|
4
|
-
|
|
5
|
-
let streamId = 0;
|
|
6
|
-
|
|
7
|
-
export function readUntilEnd(stream: stream.Readable): Promise<string> {
|
|
8
|
-
const id = streamId;
|
|
9
|
-
streamId++;
|
|
10
|
-
|
|
11
|
-
return new Promise((resolve) => {
|
|
12
|
-
let data = "";
|
|
13
|
-
|
|
14
|
-
stream.on("data", (chunk) => {
|
|
15
|
-
debug("received data chunk from stream", id);
|
|
16
|
-
data += chunk.toString("utf-8");
|
|
17
|
-
});
|
|
18
|
-
|
|
19
|
-
stream.on("close", () => {
|
|
20
|
-
debug(`stream ${id} closed; resolving`);
|
|
21
|
-
resolve(data);
|
|
22
|
-
});
|
|
23
|
-
});
|
|
24
|
-
}
|
package/src/run-jobs.ts
DELETED
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
import makeDebug from "debug";
|
|
2
|
-
const debug = makeDebug("parallel-park:run-jobs");
|
|
3
|
-
|
|
4
|
-
function isThenable<T>(value: unknown): value is Promise<T> {
|
|
5
|
-
return (
|
|
6
|
-
typeof value === "object" &&
|
|
7
|
-
value != null &&
|
|
8
|
-
// @ts-ignore accessing .then
|
|
9
|
-
typeof value.then === "function"
|
|
10
|
-
);
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
const NOTHING = Symbol("NOTHING");
|
|
14
|
-
|
|
15
|
-
let runJobsCallId = 0;
|
|
16
|
-
|
|
17
|
-
export async function runJobs<T, U>(
|
|
18
|
-
inputs: Iterable<T | Promise<T>> | AsyncIterable<T | Promise<T>>,
|
|
19
|
-
mapper: (input: T, index: number, length: number) => Promise<U>,
|
|
20
|
-
{
|
|
21
|
-
/**
|
|
22
|
-
* How many jobs are allowed to run at once.
|
|
23
|
-
*/
|
|
24
|
-
concurrency = 8,
|
|
25
|
-
}: {
|
|
26
|
-
/**
|
|
27
|
-
* How many jobs are allowed to run at once.
|
|
28
|
-
*/
|
|
29
|
-
concurrency?: number;
|
|
30
|
-
} = {}
|
|
31
|
-
): Promise<Array<U>> {
|
|
32
|
-
const callId = runJobsCallId;
|
|
33
|
-
runJobsCallId++;
|
|
34
|
-
|
|
35
|
-
debug(`runJobs called (callId: ${callId})`, { inputs, mapper, concurrency });
|
|
36
|
-
|
|
37
|
-
if (concurrency < 1) {
|
|
38
|
-
throw new Error(
|
|
39
|
-
"Concurrency can't be less than one; that doesn't make any sense."
|
|
40
|
-
);
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
const inputsArray: Array<T> = [];
|
|
44
|
-
const inputIteratorFactory =
|
|
45
|
-
inputs[Symbol.asyncIterator || NOTHING] || inputs[Symbol.iterator];
|
|
46
|
-
const inputIterator = inputIteratorFactory.call(inputs);
|
|
47
|
-
const maybeLength = Array.isArray(inputs) ? inputs.length : null;
|
|
48
|
-
|
|
49
|
-
let iteratorDone = false;
|
|
50
|
-
|
|
51
|
-
async function readInput(): Promise<boolean> {
|
|
52
|
-
debug(`reading next input (callId: ${callId})`);
|
|
53
|
-
let nextResult = inputIterator.next();
|
|
54
|
-
if (isThenable(nextResult)) {
|
|
55
|
-
nextResult = await nextResult;
|
|
56
|
-
}
|
|
57
|
-
if (nextResult.done) {
|
|
58
|
-
iteratorDone = true;
|
|
59
|
-
return false;
|
|
60
|
-
} else {
|
|
61
|
-
let value = nextResult.value;
|
|
62
|
-
if (isThenable<T>(value)) {
|
|
63
|
-
value = await value;
|
|
64
|
-
}
|
|
65
|
-
inputsArray.push(value);
|
|
66
|
-
return true;
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
let unstartedIndex = 0;
|
|
71
|
-
|
|
72
|
-
const results = new Array(maybeLength || 0);
|
|
73
|
-
const runningPromises = new Set();
|
|
74
|
-
let error: Error | null = null;
|
|
75
|
-
|
|
76
|
-
async function takeInput() {
|
|
77
|
-
const read = await readInput();
|
|
78
|
-
if (!read) return;
|
|
79
|
-
|
|
80
|
-
const inputIndex = unstartedIndex;
|
|
81
|
-
unstartedIndex++;
|
|
82
|
-
|
|
83
|
-
const input = inputsArray[inputIndex];
|
|
84
|
-
debug(`mapping input into Promise (callId: ${callId})`);
|
|
85
|
-
const promise = mapper(input, inputIndex, maybeLength || Infinity);
|
|
86
|
-
|
|
87
|
-
if (!isThenable(promise)) {
|
|
88
|
-
throw new Error(
|
|
89
|
-
"Mapper function passed into runJobs didn't return a Promise. The mapper function should always return a Promise. The easiest way to ensure this is the case is to make your mapper function an async function."
|
|
90
|
-
);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
const promiseWithMore = promise.then(
|
|
94
|
-
(result) => {
|
|
95
|
-
debug(`child Promise resolved for input (callId: ${callId}):`, input);
|
|
96
|
-
results[inputIndex] = result;
|
|
97
|
-
runningPromises.delete(promiseWithMore);
|
|
98
|
-
},
|
|
99
|
-
(err) => {
|
|
100
|
-
debug(
|
|
101
|
-
`child Promise rejected for input (callId: ${callId}):`,
|
|
102
|
-
input,
|
|
103
|
-
"with error:",
|
|
104
|
-
err
|
|
105
|
-
);
|
|
106
|
-
runningPromises.delete(promiseWithMore);
|
|
107
|
-
error = err;
|
|
108
|
-
}
|
|
109
|
-
);
|
|
110
|
-
runningPromises.add(promiseWithMore);
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
async function proceed() {
|
|
114
|
-
while (!iteratorDone && runningPromises.size < concurrency) {
|
|
115
|
-
await takeInput();
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
await proceed();
|
|
120
|
-
while (runningPromises.size > 0 && !error) {
|
|
121
|
-
await Promise.race(runningPromises.values());
|
|
122
|
-
if (error) {
|
|
123
|
-
debug(`throwing error (callId: ${callId})`);
|
|
124
|
-
throw error;
|
|
125
|
-
}
|
|
126
|
-
await proceed();
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
if (error) {
|
|
130
|
-
debug(`throwing error (callId: ${callId})`);
|
|
131
|
-
throw error;
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
debug(`all done (callId: ${callId})`);
|
|
135
|
-
return results;
|
|
136
|
-
}
|