parallel-park 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.node-version +1 -0
- package/.npm-version +1 -0
- package/LICENSE +21 -0
- package/README.md +183 -0
- package/dist/child-process-worker.js +44 -0
- package/dist/in-child-process.js +109 -0
- package/dist/index.js +7 -0
- package/dist/run-jobs.js +59 -0
- package/package.json +40 -2
- package/src/child-process-worker.ts +58 -0
- package/src/in-child-process.ts +148 -0
- package/src/index.ts +4 -0
- package/src/run-jobs.ts +86 -0
- package/tsconfig.json +27 -0
package/.node-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
v16.14.0
|
package/.npm-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
8.3.1
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License Copyright (c) 2022 Lily Scott
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of
|
|
4
|
+
charge, to any person obtaining a copy of this software and associated
|
|
5
|
+
documentation files (the "Software"), to deal in the Software without
|
|
6
|
+
restriction, including without limitation the rights to use, copy, modify, merge,
|
|
7
|
+
publish, distribute, sublicense, and/or sell copies of the Software, and to
|
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to the
|
|
9
|
+
following conditions:
|
|
10
|
+
|
|
11
|
+
The above copyright notice and this permission notice
|
|
12
|
+
(including the next paragraph) shall be included in all copies or substantial
|
|
13
|
+
portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
|
|
16
|
+
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
17
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
|
18
|
+
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
19
|
+
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
20
|
+
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# parallel-park
|
|
2
|
+
|
|
3
|
+
Parallel/concurrent async work, optionally using multiple processes
|
|
4
|
+
|
|
5
|
+
## Usage
|
|
6
|
+
|
|
7
|
+
`parallel-park` exports two functions: `runJobs` and `inChildProcess`.
|
|
8
|
+
|
|
9
|
+
### `runJobs`
|
|
10
|
+
|
|
11
|
+
`runJobs` is kinda like `Promise.all`, but instead of running everything at once, it'll only run a few Promises at a time (you can choose how many to run at once). It's inspired by [Bluebird's Promise.map function](http://bluebirdjs.com/docs/api/promise.map.html).
|
|
12
|
+
|
|
13
|
+
To use it, you pass in an array of inputs and a mapper function that transforms each input into a Promise. You can also optionally specify the maximum number of Promises to wait on at a time by passing an object with a `concurrency` property, which is a number. The concurrency defaults to 8.
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { runJobs } from "parallel-park";
|
|
17
|
+
|
|
18
|
+
const inputs = ["alice", "bob", "carl", "dorsey", "edith"];
|
|
19
|
+
const results = await runJobs(
|
|
20
|
+
inputs,
|
|
21
|
+
async (name, index, inputsCount) => {
|
|
22
|
+
// Do whatever async work you want inside this mapper function.
|
|
23
|
+
// In this case, we use a hypothetical "getUser" function to
|
|
24
|
+
// retrieve data about a user from some web API.
|
|
25
|
+
console.log(`Getting fullName for ${name}...`);
|
|
26
|
+
const user = await getUser(name);
|
|
27
|
+
return user.fullName;
|
|
28
|
+
},
|
|
29
|
+
// This options object with concurrency is an optional argument.
|
|
30
|
+
// If unspecified, it defaults to { concurrency: 8 }
|
|
31
|
+
{
|
|
32
|
+
// This number specifies how many times to call the mapper
|
|
33
|
+
// function before waiting for one of the returned Promises
|
|
34
|
+
// to resolve. Ie. "How many promises to have in-flight concurrently"
|
|
35
|
+
concurrency: 2,
|
|
36
|
+
}
|
|
37
|
+
);
|
|
38
|
+
// Logs these two immediately:
|
|
39
|
+
//
|
|
40
|
+
// Getting fullName for alice...
|
|
41
|
+
// Getting fullName for bob...
|
|
42
|
+
//
|
|
43
|
+
// But, it doesn't log anything else yet, because we told it to only run two things at a time.
|
|
44
|
+
// Then, after one of those Promises has finished, it logs:
|
|
45
|
+
//
|
|
46
|
+
// Getting fullName for carl...
|
|
47
|
+
//
|
|
48
|
+
// And so forth, until all of them are done.
|
|
49
|
+
|
|
50
|
+
// `results` is an Array of the resolved value you returned from your mapper function.
|
|
51
|
+
// The indices in the array correspond to the indices of your inputs.
|
|
52
|
+
console.log(results);
|
|
53
|
+
// Logs:
|
|
54
|
+
// [
|
|
55
|
+
// "Alice Smith",
|
|
56
|
+
// "Bob Eriksson",
|
|
57
|
+
// "Carl Martinez",
|
|
58
|
+
// "Dorsey Toth",
|
|
59
|
+
// "Edith Skalla"
|
|
60
|
+
// ]
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### `inChildProcess`
|
|
64
|
+
|
|
65
|
+
`inChildProcess` is a function that you pass a function into, and it spawns a separate node process to run your function in. Once your function has completed, its return/resolved value will be sent back to the node process you called `inChildProcess` from.
|
|
66
|
+
|
|
67
|
+
```ts
|
|
68
|
+
import { inChildProcess } from "parallel-park";
|
|
69
|
+
|
|
70
|
+
const result = await inChildProcess(() => {
|
|
71
|
+
return 2 + 2;
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
console.log(result); // 4
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Your function can also return a Promise:
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
// Either by returning a Promise directly...
|
|
81
|
+
await inChildProcess(() => {
|
|
82
|
+
return Promise.resolve(2 + 2);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
// Or by using an async function, which returns a Promise that resolves to the function's return value
|
|
86
|
+
await inChildProcess(async () => {
|
|
87
|
+
return 2 + 2;
|
|
88
|
+
});
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
> ⚠️ NOTE: The return value of your function must be JSON-serializable, or else it won't make it across the gap between the parent node process and the child one.
|
|
92
|
+
|
|
93
|
+
The function you pass into `inChildProcess` will be executed in a separate node process; as such, it won't be able to access variables defined in the file calling `inChildProcess`:
|
|
94
|
+
|
|
95
|
+
```ts
|
|
96
|
+
const myName = "Lily";
|
|
97
|
+
|
|
98
|
+
await inChildProcess(() => {
|
|
99
|
+
// Throws an error: myName is not defined
|
|
100
|
+
return myName + "!";
|
|
101
|
+
});
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
To work around this, you can pass an object into `inChildProcess` as its first argument, before the function. When called this way, the function will receive that object:
|
|
105
|
+
|
|
106
|
+
```ts
|
|
107
|
+
await inChildProcess({ myName: "Lily" }, (data) => {
|
|
108
|
+
const myName = data.myName;
|
|
109
|
+
|
|
110
|
+
// No longer throws an error
|
|
111
|
+
return myName + "!";
|
|
112
|
+
});
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
It's common to use object shorthand and destructuring syntax when passing values in:
|
|
116
|
+
|
|
117
|
+
```ts
|
|
118
|
+
const myName = "Lily";
|
|
119
|
+
|
|
120
|
+
await inChildProcess({ myName }, ({ myName }) => {
|
|
121
|
+
return myName + "!";
|
|
122
|
+
});
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
> ⚠️ NOTE: The values in your input object must be JSON-serializable, or else they won't make it across the gap between the parent node process and the child one.
|
|
126
|
+
|
|
127
|
+
Because the inputs have to be JSON-serializable, you may run into an issue if trying to use an external module within the child process:
|
|
128
|
+
|
|
129
|
+
```ts
|
|
130
|
+
const util = require("util");
|
|
131
|
+
|
|
132
|
+
await inChildProcess({ util }, ({ util }) => {
|
|
133
|
+
const someData = { something: true };
|
|
134
|
+
// Throws an error: util.inspect is not a function
|
|
135
|
+
return util.inspect(someData);
|
|
136
|
+
});
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
To work around this, call `require` inside the child process function:
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
await inChildProcess(() => {
|
|
143
|
+
const util = require("util"); // the require is inside the function now
|
|
144
|
+
|
|
145
|
+
const someData = { something: true };
|
|
146
|
+
|
|
147
|
+
// No longer throws an error
|
|
148
|
+
return util.inspect(someData);
|
|
149
|
+
});
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
If you want to use the external module both inside of the child process and outside of it, `require` it in both places:
|
|
153
|
+
|
|
154
|
+
```ts
|
|
155
|
+
const util = require("util");
|
|
156
|
+
|
|
157
|
+
await inChildProcess(() => {
|
|
158
|
+
const util = require("util");
|
|
159
|
+
|
|
160
|
+
const someData = { something: true };
|
|
161
|
+
|
|
162
|
+
// No longer throws an error
|
|
163
|
+
return util.inspect(someData);
|
|
164
|
+
});
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
The `require` inside of the child process can also be used to load stuff from your own code into the child process:
|
|
168
|
+
|
|
169
|
+
```ts
|
|
170
|
+
const file = "/home/lily/hello.txt";
|
|
171
|
+
|
|
172
|
+
await inChildProcess({ file }, async ({ file }) => {
|
|
173
|
+
const processFile = require("./process-file");
|
|
174
|
+
|
|
175
|
+
const results = await processFile(file);
|
|
176
|
+
console.log(results);
|
|
177
|
+
return results;
|
|
178
|
+
});
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
MIT
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const fs_1 = __importDefault(require("fs"));
|
|
7
|
+
const vm_1 = __importDefault(require("vm"));
|
|
8
|
+
const make_module_env_1 = __importDefault(require("make-module-env"));
|
|
9
|
+
const comms = fs_1.default.createWriteStream(
|
|
10
|
+
// @ts-ignore
|
|
11
|
+
null, { fd: 3 });
|
|
12
|
+
const [inputsString, fnString, callingFile] = process.argv.slice(2);
|
|
13
|
+
function onSuccess(data) {
|
|
14
|
+
comms.write(JSON.stringify({ type: "success", data }));
|
|
15
|
+
}
|
|
16
|
+
function onError(error) {
|
|
17
|
+
comms.write(JSON.stringify({
|
|
18
|
+
type: "error",
|
|
19
|
+
error: {
|
|
20
|
+
name: error.name,
|
|
21
|
+
message: error.message,
|
|
22
|
+
stack: error.stack,
|
|
23
|
+
},
|
|
24
|
+
}));
|
|
25
|
+
}
|
|
26
|
+
try {
|
|
27
|
+
const wrapperFn = vm_1.default.runInThisContext(`(function moduleWrapper(exports, require, module, __filename, __dirname) {
|
|
28
|
+
return ${fnString};})`);
|
|
29
|
+
const inputs = JSON.parse(inputsString);
|
|
30
|
+
const env = (0, make_module_env_1.default)(callingFile);
|
|
31
|
+
const fn = wrapperFn(env.exports, env.require, env.module, env.__filename, env.__dirname);
|
|
32
|
+
const result = fn(inputs);
|
|
33
|
+
if (typeof result === "object" &&
|
|
34
|
+
result != null &&
|
|
35
|
+
typeof result.then === "function") {
|
|
36
|
+
result.then(onSuccess, onError);
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
onSuccess(result);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
onError(err);
|
|
44
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.inChildProcess = void 0;
|
|
7
|
+
const child_process_1 = __importDefault(require("child_process"));
|
|
8
|
+
const runnerPath = require.resolve("../dist/child-process-worker");
|
|
9
|
+
const inChildProcess = (...args) => {
|
|
10
|
+
const inputs = typeof args[0] === "function" ? {} : args[0];
|
|
11
|
+
const functionToRun = typeof args[0] === "function" ? args[0] : args[1];
|
|
12
|
+
if (typeof inputs !== "object") {
|
|
13
|
+
throw new Error("The first argument to inChildProcess should be an object of input data to pass to the child process.");
|
|
14
|
+
}
|
|
15
|
+
if (typeof functionToRun !== "function") {
|
|
16
|
+
throw new Error("The second argument to inChildProcess should be a function to run in the child process.");
|
|
17
|
+
}
|
|
18
|
+
const here = new Error("in inChildProcess");
|
|
19
|
+
const callingFileLine = here.stack.split("\n").slice(2)[0];
|
|
20
|
+
const matches = callingFileLine.match(/\(([^\)]+):\d+:\d+\)/);
|
|
21
|
+
const [_, callingFile] = matches;
|
|
22
|
+
const child = child_process_1.default.spawn(process.argv[0], [runnerPath, JSON.stringify(inputs), functionToRun.toString(), callingFile], {
|
|
23
|
+
stdio: ["inherit", "inherit", "inherit", "pipe"],
|
|
24
|
+
});
|
|
25
|
+
return new Promise((resolve, reject) => {
|
|
26
|
+
child.on("error", reject);
|
|
27
|
+
let receivedData = "";
|
|
28
|
+
const comms = child.stdio && child.stdio[3];
|
|
29
|
+
if (!comms) {
|
|
30
|
+
reject(new Error("Failed to establish communication pipe with child process"));
|
|
31
|
+
}
|
|
32
|
+
comms.on("data", (chunk) => {
|
|
33
|
+
receivedData += chunk.toString();
|
|
34
|
+
});
|
|
35
|
+
child.on("close", (code, signal) => {
|
|
36
|
+
if (code !== 0) {
|
|
37
|
+
reject(new Error(`Child process exited with nonzero status code: ${JSON.stringify({
|
|
38
|
+
code,
|
|
39
|
+
signal,
|
|
40
|
+
})}`));
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
const result = JSON.parse(receivedData);
|
|
44
|
+
switch (result.type) {
|
|
45
|
+
case "success": {
|
|
46
|
+
resolve(result.data);
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
case "error": {
|
|
50
|
+
const error = new Error(result.error.message);
|
|
51
|
+
Object.defineProperty(error, "name", { value: result.error.name });
|
|
52
|
+
Object.defineProperty(error, "stack", {
|
|
53
|
+
value: result.error.name +
|
|
54
|
+
": " +
|
|
55
|
+
result.error.message +
|
|
56
|
+
"\n" +
|
|
57
|
+
result.error.stack
|
|
58
|
+
.split("\n")
|
|
59
|
+
.slice(1)
|
|
60
|
+
.filter((line) => !/node:internal|node:events/.test(line))
|
|
61
|
+
.map((line) => {
|
|
62
|
+
if (/evalmachine/.test(line)) {
|
|
63
|
+
const lineWithoutEvalMachine = line.replace(/evalmachine(\.<anonymous>)?/, "<function passed into inChildProcess>");
|
|
64
|
+
const matches = line.match(/:(\d+):(\d+)\)?$/);
|
|
65
|
+
if (!matches) {
|
|
66
|
+
return lineWithoutEvalMachine;
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
let [_, row, col] = matches;
|
|
70
|
+
// subtract 1 from row to skip the module wrapper function line
|
|
71
|
+
row = row - 1;
|
|
72
|
+
// subtract the length of the `return ` keywords in front of the function
|
|
73
|
+
if (row === 1) {
|
|
74
|
+
col = col - `return `.length;
|
|
75
|
+
}
|
|
76
|
+
const hadParen = /\)$/.test(lineWithoutEvalMachine);
|
|
77
|
+
return lineWithoutEvalMachine.replace(/:\d+:\d+\)?$/, `:${row}:${col - 1}${hadParen ? ")" : ""}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
return line;
|
|
82
|
+
}
|
|
83
|
+
})
|
|
84
|
+
.join("\n") +
|
|
85
|
+
"\n" +
|
|
86
|
+
error
|
|
87
|
+
.stack.split("\n")
|
|
88
|
+
.slice(1)
|
|
89
|
+
.filter((line) => !/node:internal|node:events/.test(line))
|
|
90
|
+
.join("\n") +
|
|
91
|
+
"\n" +
|
|
92
|
+
here
|
|
93
|
+
.stack.split("\n")
|
|
94
|
+
.slice(2)
|
|
95
|
+
.filter((line) => !/node:internal|node:events/.test(line))
|
|
96
|
+
.join("\n"),
|
|
97
|
+
});
|
|
98
|
+
reject(error);
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
default: {
|
|
102
|
+
reject(new Error(`Internal parallel-park error: unhandled result type: ${result.type}`));
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
};
|
|
109
|
+
exports.inChildProcess = inChildProcess;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.inChildProcess = exports.runJobs = void 0;
|
|
4
|
+
const run_jobs_1 = require("./run-jobs");
|
|
5
|
+
Object.defineProperty(exports, "runJobs", { enumerable: true, get: function () { return run_jobs_1.runJobs; } });
|
|
6
|
+
const in_child_process_1 = require("./in-child-process");
|
|
7
|
+
Object.defineProperty(exports, "inChildProcess", { enumerable: true, get: function () { return in_child_process_1.inChildProcess; } });
|
package/dist/run-jobs.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runJobs = void 0;
|
|
4
|
+
async function runJobs(inputs, mapper, {
|
|
5
|
+
/**
|
|
6
|
+
* How many jobs are allowed to run at once.
|
|
7
|
+
*/
|
|
8
|
+
concurrency = 8, } = {}) {
|
|
9
|
+
if (concurrency < 1) {
|
|
10
|
+
throw new Error("Concurrency can't be less than one; that doesn't make any sense.");
|
|
11
|
+
}
|
|
12
|
+
if (inputs.length === 0) {
|
|
13
|
+
return Promise.resolve([]);
|
|
14
|
+
}
|
|
15
|
+
concurrency = Math.min(concurrency, inputs.length);
|
|
16
|
+
let unstartedIndex = 0;
|
|
17
|
+
const results = new Array(inputs.length);
|
|
18
|
+
const runningPromises = new Set();
|
|
19
|
+
let error = null;
|
|
20
|
+
function takeInput() {
|
|
21
|
+
const inputIndex = unstartedIndex;
|
|
22
|
+
unstartedIndex++;
|
|
23
|
+
const input = inputs[inputIndex];
|
|
24
|
+
const promise = mapper(input, inputIndex, inputs.length);
|
|
25
|
+
if (typeof promise !== "object" ||
|
|
26
|
+
promise == null ||
|
|
27
|
+
typeof promise.then !== "function") {
|
|
28
|
+
throw new Error("Mapper function passed into runJobs didn't return a Promise. The mapper function should always return a Promise. The easiest way to ensure this is the case is to make your mapper function an async function.");
|
|
29
|
+
}
|
|
30
|
+
const promiseWithMore = promise.then((result) => {
|
|
31
|
+
results[inputIndex] = result;
|
|
32
|
+
runningPromises.delete(promiseWithMore);
|
|
33
|
+
}, (err) => {
|
|
34
|
+
runningPromises.delete(promiseWithMore);
|
|
35
|
+
error = err;
|
|
36
|
+
});
|
|
37
|
+
runningPromises.add(promiseWithMore);
|
|
38
|
+
}
|
|
39
|
+
function proceed() {
|
|
40
|
+
if (unstartedIndex < inputs.length) {
|
|
41
|
+
while (runningPromises.size < concurrency) {
|
|
42
|
+
takeInput();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
proceed();
|
|
47
|
+
while (runningPromises.size > 0 && !error) {
|
|
48
|
+
await Promise.race(runningPromises.values());
|
|
49
|
+
if (error) {
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
proceed();
|
|
53
|
+
}
|
|
54
|
+
if (error) {
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
return results;
|
|
58
|
+
}
|
|
59
|
+
exports.runJobs = runJobs;
|
package/package.json
CHANGED
|
@@ -1,5 +1,43 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "parallel-park",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Parallel/concurrent async work, optionally using multiple processes",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"types": "src/index.ts",
|
|
7
|
+
"scripts": {
|
|
8
|
+
"test": "jest",
|
|
9
|
+
"build": "tsc"
|
|
10
|
+
},
|
|
11
|
+
"keywords": [
|
|
12
|
+
"parallel",
|
|
13
|
+
"concurrent",
|
|
14
|
+
"thread",
|
|
15
|
+
"multithread",
|
|
16
|
+
"worker",
|
|
17
|
+
"process",
|
|
18
|
+
"child",
|
|
19
|
+
"concurrency",
|
|
20
|
+
"co-operative",
|
|
21
|
+
"promise",
|
|
22
|
+
"map"
|
|
23
|
+
],
|
|
24
|
+
"author": "Lily Scott <me@suchipi.com>",
|
|
25
|
+
"license": "MIT",
|
|
26
|
+
"repository": {
|
|
27
|
+
"type": "git",
|
|
28
|
+
"url": "https://github.com/suchipi/parallel-park.git"
|
|
29
|
+
},
|
|
30
|
+
"devDependencies": {
|
|
31
|
+
"@babel/core": "^7.17.5",
|
|
32
|
+
"@babel/preset-env": "^7.16.11",
|
|
33
|
+
"@babel/preset-typescript": "^7.16.7",
|
|
34
|
+
"@types/jest": "^27.4.0",
|
|
35
|
+
"@types/node": "^17.0.18",
|
|
36
|
+
"babel-jest": "^27.5.1",
|
|
37
|
+
"jest": "^27.5.1",
|
|
38
|
+
"typescript": "^4.5.5"
|
|
39
|
+
},
|
|
40
|
+
"dependencies": {
|
|
41
|
+
"make-module-env": "^1.0.1"
|
|
42
|
+
}
|
|
5
43
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import vm from "vm";
|
|
3
|
+
import makeModuleEnv from "make-module-env";
|
|
4
|
+
|
|
5
|
+
const comms = fs.createWriteStream(
|
|
6
|
+
// @ts-ignore
|
|
7
|
+
null,
|
|
8
|
+
{ fd: 3 }
|
|
9
|
+
);
|
|
10
|
+
|
|
11
|
+
const [inputsString, fnString, callingFile] = process.argv.slice(2);
|
|
12
|
+
|
|
13
|
+
function onSuccess(data: any) {
|
|
14
|
+
comms.write(JSON.stringify({ type: "success", data }));
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function onError(error: Error) {
|
|
18
|
+
comms.write(
|
|
19
|
+
JSON.stringify({
|
|
20
|
+
type: "error",
|
|
21
|
+
error: {
|
|
22
|
+
name: error.name,
|
|
23
|
+
message: error.message,
|
|
24
|
+
stack: error.stack,
|
|
25
|
+
},
|
|
26
|
+
})
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
try {
|
|
31
|
+
const wrapperFn = vm.runInThisContext(
|
|
32
|
+
`(function moduleWrapper(exports, require, module, __filename, __dirname) {
|
|
33
|
+
return ${fnString};})`
|
|
34
|
+
);
|
|
35
|
+
const inputs = JSON.parse(inputsString);
|
|
36
|
+
|
|
37
|
+
const env = makeModuleEnv(callingFile);
|
|
38
|
+
const fn = wrapperFn(
|
|
39
|
+
env.exports,
|
|
40
|
+
env.require,
|
|
41
|
+
env.module,
|
|
42
|
+
env.__filename,
|
|
43
|
+
env.__dirname
|
|
44
|
+
);
|
|
45
|
+
const result = fn(inputs);
|
|
46
|
+
|
|
47
|
+
if (
|
|
48
|
+
typeof result === "object" &&
|
|
49
|
+
result != null &&
|
|
50
|
+
typeof result.then === "function"
|
|
51
|
+
) {
|
|
52
|
+
result.then(onSuccess, onError);
|
|
53
|
+
} else {
|
|
54
|
+
onSuccess(result);
|
|
55
|
+
}
|
|
56
|
+
} catch (err) {
|
|
57
|
+
onError(err as Error);
|
|
58
|
+
}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import child_process from "child_process";
|
|
2
|
+
|
|
3
|
+
const runnerPath = require.resolve("../dist/child-process-worker");
|
|
4
|
+
|
|
5
|
+
type InChildProcess = {
|
|
6
|
+
<Inputs extends { [key: string]: any }, Result>(
|
|
7
|
+
inputs: Inputs,
|
|
8
|
+
functionToRun: (inputs: Inputs) => Result | Promise<Result>
|
|
9
|
+
): Promise<Result>;
|
|
10
|
+
|
|
11
|
+
<Result>(functionToRun: () => Result | Promise<Result>): Promise<Result>;
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export const inChildProcess: InChildProcess = (...args) => {
|
|
15
|
+
const inputs = typeof args[0] === "function" ? {} : args[0];
|
|
16
|
+
const functionToRun = typeof args[0] === "function" ? args[0] : args[1];
|
|
17
|
+
|
|
18
|
+
if (typeof inputs !== "object") {
|
|
19
|
+
throw new Error(
|
|
20
|
+
"The first argument to inChildProcess should be an object of input data to pass to the child process."
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
if (typeof functionToRun !== "function") {
|
|
25
|
+
throw new Error(
|
|
26
|
+
"The second argument to inChildProcess should be a function to run in the child process."
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const here = new Error("in inChildProcess");
|
|
31
|
+
|
|
32
|
+
const callingFileLine = here.stack!.split("\n").slice(2)[0];
|
|
33
|
+
const matches = callingFileLine.match(/\(([^\)]+):\d+:\d+\)/);
|
|
34
|
+
const [_, callingFile] = matches!;
|
|
35
|
+
|
|
36
|
+
const child = child_process.spawn(
|
|
37
|
+
process.argv[0],
|
|
38
|
+
[runnerPath, JSON.stringify(inputs), functionToRun.toString(), callingFile],
|
|
39
|
+
{
|
|
40
|
+
stdio: ["inherit", "inherit", "inherit", "pipe"],
|
|
41
|
+
}
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
return new Promise((resolve, reject) => {
|
|
45
|
+
child.on("error", reject);
|
|
46
|
+
|
|
47
|
+
let receivedData = "";
|
|
48
|
+
const comms = child.stdio && child.stdio[3];
|
|
49
|
+
if (!comms) {
|
|
50
|
+
reject(
|
|
51
|
+
new Error("Failed to establish communication pipe with child process")
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
comms!.on("data", (chunk) => {
|
|
56
|
+
receivedData += chunk.toString();
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
child.on("close", (code, signal) => {
|
|
60
|
+
if (code !== 0) {
|
|
61
|
+
reject(
|
|
62
|
+
new Error(
|
|
63
|
+
`Child process exited with nonzero status code: ${JSON.stringify({
|
|
64
|
+
code,
|
|
65
|
+
signal,
|
|
66
|
+
})}`
|
|
67
|
+
)
|
|
68
|
+
);
|
|
69
|
+
} else {
|
|
70
|
+
const result = JSON.parse(receivedData);
|
|
71
|
+
switch (result.type) {
|
|
72
|
+
case "success": {
|
|
73
|
+
resolve(result.data);
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
76
|
+
case "error": {
|
|
77
|
+
const error = new Error(result.error.message);
|
|
78
|
+
Object.defineProperty(error, "name", { value: result.error.name });
|
|
79
|
+
Object.defineProperty(error, "stack", {
|
|
80
|
+
value:
|
|
81
|
+
result.error.name +
|
|
82
|
+
": " +
|
|
83
|
+
result.error.message +
|
|
84
|
+
"\n" +
|
|
85
|
+
result.error.stack
|
|
86
|
+
.split("\n")
|
|
87
|
+
.slice(1)
|
|
88
|
+
.filter((line) => !/node:internal|node:events/.test(line))
|
|
89
|
+
.map((line) => {
|
|
90
|
+
if (/evalmachine/.test(line)) {
|
|
91
|
+
const lineWithoutEvalMachine = line.replace(
|
|
92
|
+
/evalmachine(\.<anonymous>)?/,
|
|
93
|
+
"<function passed into inChildProcess>"
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
const matches = line.match(/:(\d+):(\d+)\)?$/);
|
|
97
|
+
if (!matches) {
|
|
98
|
+
return lineWithoutEvalMachine;
|
|
99
|
+
} else {
|
|
100
|
+
let [_, row, col] = matches;
|
|
101
|
+
// subtract 1 from row to skip the module wrapper function line
|
|
102
|
+
row = row - 1;
|
|
103
|
+
|
|
104
|
+
// subtract the length of the `return ` keywords in front of the function
|
|
105
|
+
if (row === 1) {
|
|
106
|
+
col = col - `return `.length;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const hadParen = /\)$/.test(lineWithoutEvalMachine);
|
|
110
|
+
|
|
111
|
+
return lineWithoutEvalMachine.replace(
|
|
112
|
+
/:\d+:\d+\)?$/,
|
|
113
|
+
`:${row}:${col - 1}${hadParen ? ")" : ""}`
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
} else {
|
|
117
|
+
return line;
|
|
118
|
+
}
|
|
119
|
+
})
|
|
120
|
+
.join("\n") +
|
|
121
|
+
"\n" +
|
|
122
|
+
error
|
|
123
|
+
.stack!.split("\n")
|
|
124
|
+
.slice(1)
|
|
125
|
+
.filter((line) => !/node:internal|node:events/.test(line))
|
|
126
|
+
.join("\n") +
|
|
127
|
+
"\n" +
|
|
128
|
+
here
|
|
129
|
+
.stack!.split("\n")
|
|
130
|
+
.slice(2)
|
|
131
|
+
.filter((line) => !/node:internal|node:events/.test(line))
|
|
132
|
+
.join("\n"),
|
|
133
|
+
});
|
|
134
|
+
reject(error);
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
default: {
|
|
138
|
+
reject(
|
|
139
|
+
new Error(
|
|
140
|
+
`Internal parallel-park error: unhandled result type: ${result.type}`
|
|
141
|
+
)
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
};
|
package/src/index.ts
ADDED
package/src/run-jobs.ts
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
export async function runJobs<T, U>(
|
|
2
|
+
inputs: Array<T>,
|
|
3
|
+
mapper: (input: T, index: number, length: number) => Promise<U>,
|
|
4
|
+
{
|
|
5
|
+
/**
|
|
6
|
+
* How many jobs are allowed to run at once.
|
|
7
|
+
*/
|
|
8
|
+
concurrency = 8,
|
|
9
|
+
}: {
|
|
10
|
+
/**
|
|
11
|
+
* How many jobs are allowed to run at once.
|
|
12
|
+
*/
|
|
13
|
+
concurrency?: number;
|
|
14
|
+
} = {}
|
|
15
|
+
): Promise<Array<U>> {
|
|
16
|
+
if (concurrency < 1) {
|
|
17
|
+
throw new Error(
|
|
18
|
+
"Concurrency can't be less than one; that doesn't make any sense."
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
if (inputs.length === 0) {
|
|
23
|
+
return Promise.resolve([]);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
concurrency = Math.min(concurrency, inputs.length);
|
|
27
|
+
|
|
28
|
+
let unstartedIndex = 0;
|
|
29
|
+
|
|
30
|
+
const results = new Array(inputs.length);
|
|
31
|
+
const runningPromises = new Set();
|
|
32
|
+
let error: Error | null = null;
|
|
33
|
+
|
|
34
|
+
function takeInput() {
|
|
35
|
+
const inputIndex = unstartedIndex;
|
|
36
|
+
unstartedIndex++;
|
|
37
|
+
|
|
38
|
+
const input = inputs[inputIndex];
|
|
39
|
+
const promise = mapper(input, inputIndex, inputs.length);
|
|
40
|
+
|
|
41
|
+
if (
|
|
42
|
+
typeof promise !== "object" ||
|
|
43
|
+
promise == null ||
|
|
44
|
+
typeof promise.then !== "function"
|
|
45
|
+
) {
|
|
46
|
+
throw new Error(
|
|
47
|
+
"Mapper function passed into runJobs didn't return a Promise. The mapper function should always return a Promise. The easiest way to ensure this is the case is to make your mapper function an async function."
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const promiseWithMore = promise.then(
|
|
52
|
+
(result) => {
|
|
53
|
+
results[inputIndex] = result;
|
|
54
|
+
runningPromises.delete(promiseWithMore);
|
|
55
|
+
},
|
|
56
|
+
(err) => {
|
|
57
|
+
runningPromises.delete(promiseWithMore);
|
|
58
|
+
error = err;
|
|
59
|
+
}
|
|
60
|
+
);
|
|
61
|
+
runningPromises.add(promiseWithMore);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function proceed() {
|
|
65
|
+
if (unstartedIndex < inputs.length) {
|
|
66
|
+
while (runningPromises.size < concurrency) {
|
|
67
|
+
takeInput();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
proceed();
|
|
73
|
+
while (runningPromises.size > 0 && !error) {
|
|
74
|
+
await Promise.race(runningPromises.values());
|
|
75
|
+
if (error) {
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
proceed();
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (error) {
|
|
82
|
+
throw error;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return results;
|
|
86
|
+
}
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"include": ["./src/**/*.ts"],
|
|
3
|
+
"exclude": ["./**/*.test.ts"],
|
|
4
|
+
|
|
5
|
+
"compilerOptions": {
|
|
6
|
+
"lib": ["es2019", "dom"],
|
|
7
|
+
"target": "es2018",
|
|
8
|
+
"module": "CommonJS",
|
|
9
|
+
"outDir": "./dist",
|
|
10
|
+
|
|
11
|
+
"strict": true,
|
|
12
|
+
"noImplicitAny": false,
|
|
13
|
+
"strictNullChecks": true,
|
|
14
|
+
"strictFunctionTypes": true,
|
|
15
|
+
"strictPropertyInitialization": true,
|
|
16
|
+
"noImplicitThis": true,
|
|
17
|
+
"alwaysStrict": true,
|
|
18
|
+
"noUnusedLocals": false,
|
|
19
|
+
"noUnusedParameters": false,
|
|
20
|
+
"noImplicitReturns": true,
|
|
21
|
+
"noFallthroughCasesInSwitch": true,
|
|
22
|
+
"downlevelIteration": true,
|
|
23
|
+
|
|
24
|
+
"moduleResolution": "node",
|
|
25
|
+
"esModuleInterop": true
|
|
26
|
+
}
|
|
27
|
+
}
|