parallel-park 0.2.0 → 0.2.1

package/.node-version

@@ -1 +1 @@
-v16.14.0
+v20.11.1

package/.npm-version

@@ -1 +1 @@
-8.3.1
+10.2.4

package/LICENSE

@@ -1,4 +1,4 @@
-MIT License Copyright (c) 2022 Lily Scott
+MIT License Copyright (c) 2022-2024 Lily Skye
 
 Permission is hereby granted, free of
 charge, to any person obtaining a copy of this software and associated
@@ -18,4 +18,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
 EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE.

package/dist/child-process-worker.js

@@ -6,27 +6,37 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
 const vm_1 = __importDefault(require("vm"));
 const make_module_env_1 = __importDefault(require("make-module-env"));
-const comms = fs_1.default.createWriteStream(
+const debug_1 = __importDefault(require("debug"));
+const read_until_end_1 = require("./read-until-end");
+const path_1 = __importDefault(require("path"));
+const debug = (0, debug_1.default)("parallel-park:child-process-worker");
+const commsIn = fs_1.default.createReadStream(
 // @ts-ignore
 null, { fd: 3 });
-const [inputsString, fnString, callingFile] = process.argv.slice(2);
-function onSuccess(data) {
-    comms.write(JSON.stringify({ type: "success", data }));
-}
-function onError(error) {
-    comms.write(JSON.stringify({
-        type: "error",
-        error: {
-            name: error.name,
-            message: error.message,
-            stack: error.stack,
-        },
-    }));
-}
-try {
+const commsOut = fs_1.default.createWriteStream(
+// @ts-ignore
+null, { fd: 4 });
+debug("reading input data...");
+(0, read_until_end_1.readUntilEnd)(commsIn)
+    .then((data) => {
+    debug("parsing input data...");
+    try {
+        const [inputs, fnString, callingFile] = JSON.parse(data);
+        onReady(inputs, fnString, callingFile);
+    }
+    catch (err) {
+        onError(err);
+    }
+})
+    .catch(onError);
+function onReady(inputs, fnString, callingFile) {
+    debug("in onReady", { inputs, fnString, callingFile });
+    // Relevant when callingFile is eg. "REPL2" (from Node.js repl)
+    if (!path_1.default.isAbsolute(callingFile)) {
+        callingFile = path_1.default.join(process.cwd(), "fake-path.js");
+    }
     const wrapperFn = vm_1.default.runInThisContext(`(function moduleWrapper(exports, require, module, __filename, __dirname) {
-return ${fnString};})`);
-    const inputs = JSON.parse(inputsString);
+  return ${fnString};})`);
     const env = (0, make_module_env_1.default)(callingFile);
     const fn = wrapperFn(env.exports, env.require, env.module, env.__filename, env.__dirname);
     const result = fn(inputs);
@@ -39,6 +49,18 @@ return ${fnString};})`);
         onSuccess(result);
     }
 }
-catch (err) {
-    onError(err);
+function onSuccess(data) {
+    debug("in onSuccess", { data });
+    commsOut.end(JSON.stringify({ type: "success", data }));
+}
+function onError(error) {
+    debug("in onError", { error });
+    commsOut.end(JSON.stringify({
+        type: "error",
+        error: {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+        },
+    }));
 }

package/dist/in-child-process.js

@@ -5,8 +5,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.inChildProcess = void 0;
 const child_process_1 = __importDefault(require("child_process"));
+const error_utils_1 = require("@suchipi/error-utils");
+const debug_1 = __importDefault(require("debug"));
+const read_until_end_1 = require("./read-until-end");
+const debug = (0, debug_1.default)("parallel-park:in-child-process");
 const runnerPath = require.resolve("../dist/child-process-worker");
 const inChildProcess = (...args) => {
+    var _a;
     const inputs = typeof args[0] === "function" ? {} : args[0];
     const functionToRun = typeof args[0] === "function" ? args[0] : args[1];
     if (typeof inputs !== "object") {
@@ -15,24 +20,33 @@ const inChildProcess = (...args) => {
     if (typeof functionToRun !== "function") {
         throw new Error("The second argument to inChildProcess should be a function to run in the child process.");
     }
-    const here = new Error("in inChildProcess");
-    const callingFileLine = here.stack.split("\n").slice(2)[0];
-    const matches = callingFileLine.match(/\(([^\)]+):\d+:\d+\)/);
-    const [_, callingFile] = matches;
-    const child = child_process_1.default.spawn(process.argv[0], [runnerPath, JSON.stringify(inputs), functionToRun.toString(), callingFile], {
-        stdio: ["inherit", "inherit", "inherit", "pipe"],
+    const here = new error_utils_1.ParsedError(new Error("here"));
+    const callingFrame = here.stackFrames[1];
+    const callingFile = (_a = callingFrame === null || callingFrame === void 0 ? void 0 : callingFrame.fileName) !== null && _a !== void 0 ? _a : "unknown file";
+    debug("spawning child process:", [process.argv[0], runnerPath]);
+    const child = child_process_1.default.spawn(process.argv[0], [runnerPath], {
+        stdio: ["inherit", "inherit", "inherit", "pipe", "pipe"],
     });
     return new Promise((resolve, reject) => {
         child.on("error", reject);
+        const commsOut = child.stdio[3];
+        const commsIn = child.stdio[4];
+        child.on("spawn", () => {
+            const dataToSend = JSON.stringify([
+                inputs,
+                functionToRun.toString(),
+                callingFile,
+            ]);
+            debug("sending inputs to child process:", dataToSend);
+            commsOut.end(dataToSend, "utf-8");
+        });
         let receivedData = "";
-        const comms = child.stdio && child.stdio[3];
-        if (!comms) {
-            reject(new Error("Failed to establish communication pipe with child process"));
-        }
-        comms.on("data", (chunk) => {
-            receivedData += chunk.toString();
+        (0, read_until_end_1.readUntilEnd)(commsIn).then((data) => {
+            debug("received data from child process");
+            receivedData = data;
         });
         child.on("close", (code, signal) => {
+            debug("child process closed:", { code, signal });
             if (code !== 0) {
                 reject(new Error(`Child process exited with nonzero status code: ${JSON.stringify({
                     code,
@@ -40,6 +54,7 @@ const inChildProcess = (...args) => {
                 })}`));
             }
             else {
+                debug("parsing received data from child process...");
                 const result = JSON.parse(receivedData);
                 switch (result.type) {
                     case "success": {
@@ -60,7 +75,7 @@ const inChildProcess = (...args) => {
                                     .filter((line) => !/node:internal|node:events/.test(line))
                                     .map((line) => {
                                     if (/evalmachine/.test(line)) {
-                                        const lineWithoutEvalMachine = line.replace(/evalmachine(\.<anonymous>)?/, "<function passed into inChildProcess>");
+                                        const lineWithoutEvalMachine = line.replace(/evalmachine(?:\.<anonymous>)?/, "<function passed into inChildProcess>");
                                         const matches = line.match(/:(\d+):(\d+)\)?$/);
                                         if (!matches) {
                                             return lineWithoutEvalMachine;

package/dist/read-until-end.js

@@ -0,0 +1,25 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readUntilEnd = void 0;
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)("parallel-park:read-until-end");
+let streamId = 0;
+function readUntilEnd(stream) {
+    const id = streamId;
+    streamId++;
+    return new Promise((resolve) => {
+        let data = "";
+        stream.on("data", (chunk) => {
+            debug("received data chunk from stream", id);
+            data += chunk.toString("utf-8");
+        });
+        stream.on("close", () => {
+            debug(`stream ${id} closed; resolving`);
+            resolve(data);
+        });
+    });
+}
+exports.readUntilEnd = readUntilEnd;

package/dist/run-jobs.js

@@ -1,6 +1,11 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runJobs = void 0;
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)("parallel-park:run-jobs");
 function isThenable(value) {
     return (typeof value === "object" &&
         value != null &&
@@ -8,11 +13,15 @@ function isThenable(value) {
         typeof value.then === "function");
 }
 const NOTHING = Symbol("NOTHING");
+let runJobsCallId = 0;
 async function runJobs(inputs, mapper, { 
 /**
  * How many jobs are allowed to run at once.
  */
 concurrency = 8, } = {}) {
+    const callId = runJobsCallId;
+    runJobsCallId++;
+    debug(`runJobs called (callId: ${callId})`, { inputs, mapper, concurrency });
     if (concurrency < 1) {
         throw new Error("Concurrency can't be less than one; that doesn't make any sense.");
     }
@@ -22,6 +31,7 @@ concurrency = 8, } = {}) {
     const maybeLength = Array.isArray(inputs) ? inputs.length : null;
     let iteratorDone = false;
     async function readInput() {
+        debug(`reading next input (callId: ${callId})`);
         let nextResult = inputIterator.next();
         if (isThenable(nextResult)) {
             nextResult = await nextResult;
@@ -50,14 +60,17 @@ concurrency = 8, } = {}) {
         const inputIndex = unstartedIndex;
         unstartedIndex++;
         const input = inputsArray[inputIndex];
+        debug(`mapping input into Promise (callId: ${callId})`);
         const promise = mapper(input, inputIndex, maybeLength || Infinity);
         if (!isThenable(promise)) {
             throw new Error("Mapper function passed into runJobs didn't return a Promise. The mapper function should always return a Promise. The easiest way to ensure this is the case is to make your mapper function an async function.");
         }
         const promiseWithMore = promise.then((result) => {
+            debug(`child Promise resolved for input (callId: ${callId}):`, input);
             results[inputIndex] = result;
             runningPromises.delete(promiseWithMore);
         }, (err) => {
+            debug(`child Promise rejected for input (callId: ${callId}):`, input, "with error:", err);
             runningPromises.delete(promiseWithMore);
             error = err;
         });
@@ -72,13 +85,16 @@ concurrency = 8, } = {}) {
     while (runningPromises.size > 0 && !error) {
         await Promise.race(runningPromises.values());
         if (error) {
+            debug(`throwing error (callId: ${callId})`);
             throw error;
         }
         await proceed();
     }
     if (error) {
+        debug(`throwing error (callId: ${callId})`);
         throw error;
     }
+    debug(`all done (callId: ${callId})`);
     return results;
 }
 exports.runJobs = runJobs;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parallel-park",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Parallel/concurrent async work, optionally using multiple processes",
   "main": "dist/index.js",
   "types": "src/index.ts",
@@ -21,23 +21,25 @@
     "promise",
     "map"
   ],
-  "author": "Lily Scott <me@suchipi.com>",
+  "author": "Lily Skye <me@suchipi.com>",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/suchipi/parallel-park.git"
+    "url": "git+https://github.com/suchipi/parallel-park.git"
   },
   "devDependencies": {
-    "@babel/core": "^7.17.5",
-    "@babel/preset-env": "^7.16.11",
-    "@babel/preset-typescript": "^7.16.7",
-    "@types/jest": "^27.4.0",
-    "@types/node": "^17.0.18",
-    "babel-jest": "^27.5.1",
-    "jest": "^27.5.1",
-    "typescript": "^4.5.5"
+    "@babel/core": "^7.24.5",
+    "@babel/preset-env": "^7.24.5",
+    "@babel/preset-typescript": "^7.24.1",
+    "@types/debug": "^4.1.12",
+    "@types/jest": "^29.5.12",
+    "@types/node": "^20.12.10",
+    "babel-jest": "^29.7.0",
+    "jest": "^29.7.0",
+    "typescript": "^5.4.5"
   },
   "dependencies": {
-    "make-module-env": "^1.0.1"
+    "@suchipi/error-utils": "^0.2.0",
+    "make-module-env": "^1.1.3"
   }
 }

package/src/child-process-worker.ts

@@ -1,39 +1,48 @@
 import fs from "fs";
 import vm from "vm";
 import makeModuleEnv from "make-module-env";
+import makeDebug from "debug";
+import { readUntilEnd } from "./read-until-end";
+import path from "path";
 
-const comms = fs.createWriteStream(
+const debug = makeDebug("parallel-park:child-process-worker");
+
+const commsIn = fs.createReadStream(
   // @ts-ignore
   null,
   { fd: 3 }
 );
+const commsOut = fs.createWriteStream(
+  // @ts-ignore
+  null,
+  { fd: 4 }
+);
 
-const [inputsString, fnString, callingFile] = process.argv.slice(2);
+debug("reading input data...");
+readUntilEnd(commsIn)
+  .then((data) => {
+    debug("parsing input data...");
+    try {
+      const [inputs, fnString, callingFile] = JSON.parse(data);
+      onReady(inputs, fnString, callingFile);
+    } catch (err) {
+      onError(err as Error);
+    }
+  })
+  .catch(onError);
 
-function onSuccess(data: any) {
-  comms.write(JSON.stringify({ type: "success", data }));
-}
+function onReady(inputs: any, fnString: string, callingFile: string) {
+  debug("in onReady", { inputs, fnString, callingFile });
 
-function onError(error: Error) {
-  comms.write(
-    JSON.stringify({
-      type: "error",
-      error: {
-        name: error.name,
-        message: error.message,
-        stack: error.stack,
-      },
-    })
-  );
-}
+  // Relevant when callingFile is eg. "REPL2" (from Node.js repl)
+  if (!path.isAbsolute(callingFile)) {
+    callingFile = path.join(process.cwd(), "fake-path.js");
+  }
 
-try {
   const wrapperFn = vm.runInThisContext(
     `(function moduleWrapper(exports, require, module, __filename, __dirname) {
-return ${fnString};})`
+  return ${fnString};})`
   );
-  const inputs = JSON.parse(inputsString);
-
   const env = makeModuleEnv(callingFile);
   const fn = wrapperFn(
     env.exports,
@@ -53,6 +62,25 @@ return ${fnString};})`
   } else {
     onSuccess(result);
   }
-} catch (err) {
-  onError(err as Error);
+}
+
+function onSuccess(data: any) {
+  debug("in onSuccess", { data });
+
+  commsOut.end(JSON.stringify({ type: "success", data }));
+}
+
+function onError(error: Error) {
+  debug("in onError", { error });
+
+  commsOut.end(
+    JSON.stringify({
+      type: "error",
+      error: {
+        name: error.name,
+        message: error.message,
+        stack: error.stack,
+      },
+    })
+  );
 }

package/src/in-child-process.ts

@@ -1,4 +1,10 @@
 import child_process from "child_process";
+import type * as stream from "stream";
+import { ParsedError } from "@suchipi/error-utils";
+import makeDebug from "debug";
+import { readUntilEnd } from "./read-until-end";
+
+const debug = makeDebug("parallel-park:in-child-process");
 
 const runnerPath = require.resolve("../dist/child-process-worker");
 
@@ -11,7 +17,7 @@ type InChildProcess = {
   <Result>(functionToRun: () => Result | Promise<Result>): Promise<Result>;
 };
 
-export const inChildProcess: InChildProcess = (...args) => {
+export const inChildProcess: InChildProcess = (...args: Array<any>) => {
   const inputs = typeof args[0] === "function" ? {} : args[0];
   const functionToRun = typeof args[0] === "function" ? args[0] : args[1];
 
@@ -27,36 +33,42 @@ export const inChildProcess: InChildProcess = (...args) => {
     );
   }
 
-  const here = new Error("in inChildProcess");
+  const here = new ParsedError(new Error("here"));
+
+  const callingFrame = here.stackFrames[1];
+  const callingFile = callingFrame?.fileName ?? "unknown file";
 
-  const callingFileLine = here.stack!.split("\n").slice(2)[0];
-  const matches = callingFileLine.match(/\(([^\)]+):\d+:\d+\)/);
-  const [_, callingFile] = matches!;
+  debug("spawning child process:", [process.argv[0], runnerPath]);
 
-  const child = child_process.spawn(
-    process.argv[0],
-    [runnerPath, JSON.stringify(inputs), functionToRun.toString(), callingFile],
-    {
-      stdio: ["inherit", "inherit", "inherit", "pipe"],
-    }
-  );
+  const child = child_process.spawn(process.argv[0], [runnerPath], {
+    stdio: ["inherit", "inherit", "inherit", "pipe", "pipe"],
+  });
 
   return new Promise((resolve, reject) => {
     child.on("error", reject);
 
+    const commsOut: stream.Writable = child.stdio![3] as any;
+    const commsIn: stream.Readable = child.stdio![4] as any;
+
+    child.on("spawn", () => {
+      const dataToSend = JSON.stringify([
+        inputs,
+        functionToRun.toString(),
+        callingFile,
+      ]);
+      debug("sending inputs to child process:", dataToSend);
+      commsOut.end(dataToSend, "utf-8");
+    });
+
     let receivedData = "";
-    const comms = child.stdio && child.stdio[3];
-    if (!comms) {
-      reject(
-        new Error("Failed to establish communication pipe with child process")
-      );
-    }
-
-    comms!.on("data", (chunk) => {
-      receivedData += chunk.toString();
+    readUntilEnd(commsIn).then((data) => {
+      debug("received data from child process");
+      receivedData = data;
     });
 
     child.on("close", (code, signal) => {
+      debug("child process closed:", { code, signal });
+
       if (code !== 0) {
         reject(
           new Error(
@@ -67,6 +79,7 @@ export const inChildProcess: InChildProcess = (...args) => {
           )
         );
       } else {
+        debug("parsing received data from child process...");
         const result = JSON.parse(receivedData);
         switch (result.type) {
           case "success": {
@@ -89,7 +102,7 @@ export const inChildProcess: InChildProcess = (...args) => {
                   .map((line) => {
                     if (/evalmachine/.test(line)) {
                       const lineWithoutEvalMachine = line.replace(
-                        /evalmachine(\.<anonymous>)?/,
+                        /evalmachine(?:\.<anonymous>)?/,
                         "<function passed into inChildProcess>"
                       );
 

package/src/read-until-end.ts

@@ -0,0 +1,24 @@
+import type stream from "stream";
+import makeDebug from "debug";
+const debug = makeDebug("parallel-park:read-until-end");
+
+let streamId = 0;
+
+export function readUntilEnd(stream: stream.Readable): Promise<string> {
+  const id = streamId;
+  streamId++;
+
+  return new Promise((resolve) => {
+    let data = "";
+
+    stream.on("data", (chunk) => {
+      debug("received data chunk from stream", id);
+      data += chunk.toString("utf-8");
+    });
+
+    stream.on("close", () => {
+      debug(`stream ${id} closed; resolving`);
+      resolve(data);
+    });
+  });
+}

package/src/run-jobs.ts

@@ -1,3 +1,6 @@
+import makeDebug from "debug";
+const debug = makeDebug("parallel-park:run-jobs");
+
 function isThenable<T>(value: unknown): value is Promise<T> {
   return (
     typeof value === "object" &&
@@ -9,6 +12,8 @@ function isThenable<T>(value: unknown): value is Promise<T> {
 
 const NOTHING = Symbol("NOTHING");
 
+let runJobsCallId = 0;
+
 export async function runJobs<T, U>(
   inputs: Iterable<T | Promise<T>> | AsyncIterable<T | Promise<T>>,
   mapper: (input: T, index: number, length: number) => Promise<U>,
@@ -24,6 +29,11 @@ export async function runJobs<T, U>(
     concurrency?: number;
   } = {}
 ): Promise<Array<U>> {
+  const callId = runJobsCallId;
+  runJobsCallId++;
+
+  debug(`runJobs called (callId: ${callId})`, { inputs, mapper, concurrency });
+
   if (concurrency < 1) {
     throw new Error(
       "Concurrency can't be less than one; that doesn't make any sense."
@@ -39,6 +49,7 @@ export async function runJobs<T, U>(
   let iteratorDone = false;
 
   async function readInput(): Promise<boolean> {
+    debug(`reading next input (callId: ${callId})`);
     let nextResult = inputIterator.next();
     if (isThenable(nextResult)) {
       nextResult = await nextResult;
@@ -70,6 +81,7 @@ export async function runJobs<T, U>(
     unstartedIndex++;
 
     const input = inputsArray[inputIndex];
+    debug(`mapping input into Promise (callId: ${callId})`);
     const promise = mapper(input, inputIndex, maybeLength || Infinity);
 
     if (!isThenable(promise)) {
@@ -80,10 +92,17 @@ export async function runJobs<T, U>(
 
     const promiseWithMore = promise.then(
       (result) => {
+        debug(`child Promise resolved for input (callId: ${callId}):`, input);
         results[inputIndex] = result;
         runningPromises.delete(promiseWithMore);
       },
       (err) => {
+        debug(
+          `child Promise rejected for input (callId: ${callId}):`,
+          input,
+          "with error:",
+          err
+        );
         runningPromises.delete(promiseWithMore);
         error = err;
       }
@@ -101,14 +120,17 @@ export async function runJobs<T, U>(
   while (runningPromises.size > 0 && !error) {
     await Promise.race(runningPromises.values());
     if (error) {
+      debug(`throwing error (callId: ${callId})`);
       throw error;
     }
     await proceed();
   }
 
   if (error) {
+    debug(`throwing error (callId: ${callId})`);
     throw error;
   }
 
+  debug(`all done (callId: ${callId})`);
   return results;
 }