parallel-park 0.1.0 → 0.2.0

package/README.md

@@ -10,7 +10,9 @@ Parallel/concurrent async work, optionally using multiple processes
 
 `runJobs` is kinda like `Promise.all`, but instead of running everything at once, it'll only run a few Promises at a time (you can choose how many to run at once). It's inspired by [Bluebird's Promise.map function](http://bluebirdjs.com/docs/api/promise.map.html).
 
-To use it, you pass in an array of inputs and a mapper function that transforms each input into a Promise. You can also optionally specify the maximum number of Promises to wait on at a time by passing an object with a `concurrency` property, which is a number. The concurrency defaults to 8.
+To use it, you pass in an iterable (array, set, generator function, etc) of inputs and a mapper function that transforms each input into a Promise. You can also optionally specify the maximum number of Promises to wait on at a time by passing an object with a `concurrency` property, which is a number. The concurrency defaults to 8.
+
+When using an iterable, if the iterable yields a Promise (ie. `iterable.next() returns { done: false, value: Promise }`), then the yielded Promise will be awaited before being passed into your mapper function. Additionally, async iterables are supported; if `iterable.next()` returns a Promise, it will be awaited.
 
 ```ts
 import { runJobs } from "parallel-park";

package/dist/run-jobs.js

@@ -1,6 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runJobs = void 0;
+function isThenable(value) {
+    return (typeof value === "object" &&
+        value != null &&
+        // @ts-ignore accessing .then
+        typeof value.then === "function");
+}
+const NOTHING = Symbol("NOTHING");
 async function runJobs(inputs, mapper, { 
 /**
  * How many jobs are allowed to run at once.
@@ -9,22 +16,42 @@ concurrency = 8, } = {}) {
     if (concurrency < 1) {
         throw new Error("Concurrency can't be less than one; that doesn't make any sense.");
     }
-    if (inputs.length === 0) {
-        return Promise.resolve([]);
+    const inputsArray = [];
+    const inputIteratorFactory = inputs[Symbol.asyncIterator || NOTHING] || inputs[Symbol.iterator];
+    const inputIterator = inputIteratorFactory.call(inputs);
+    const maybeLength = Array.isArray(inputs) ? inputs.length : null;
+    let iteratorDone = false;
+    async function readInput() {
+        let nextResult = inputIterator.next();
+        if (isThenable(nextResult)) {
+            nextResult = await nextResult;
+        }
+        if (nextResult.done) {
+            iteratorDone = true;
+            return false;
+        }
+        else {
+            let value = nextResult.value;
+            if (isThenable(value)) {
+                value = await value;
+            }
+            inputsArray.push(value);
+            return true;
+        }
     }
-    concurrency = Math.min(concurrency, inputs.length);
     let unstartedIndex = 0;
-    const results = new Array(inputs.length);
+    const results = new Array(maybeLength || 0);
     const runningPromises = new Set();
     let error = null;
-    function takeInput() {
+    async function takeInput() {
+        const read = await readInput();
+        if (!read)
+            return;
         const inputIndex = unstartedIndex;
         unstartedIndex++;
-        const input = inputs[inputIndex];
-        const promise = mapper(input, inputIndex, inputs.length);
-        if (typeof promise !== "object" ||
-            promise == null ||
-            typeof promise.then !== "function") {
+        const input = inputsArray[inputIndex];
+        const promise = mapper(input, inputIndex, maybeLength || Infinity);
+        if (!isThenable(promise)) {
             throw new Error("Mapper function passed into runJobs didn't return a Promise. The mapper function should always return a Promise. The easiest way to ensure this is the case is to make your mapper function an async function.");
         }
         const promiseWithMore = promise.then((result) => {
@@ -36,20 +63,18 @@ concurrency = 8, } = {}) {
         });
         runningPromises.add(promiseWithMore);
     }
-    function proceed() {
-        if (unstartedIndex < inputs.length) {
-            while (runningPromises.size < concurrency) {
-                takeInput();
-            }
+    async function proceed() {
+        while (!iteratorDone && runningPromises.size < concurrency) {
+            await takeInput();
         }
     }
-    proceed();
+    await proceed();
     while (runningPromises.size > 0 && !error) {
         await Promise.race(runningPromises.values());
         if (error) {
             throw error;
         }
-        proceed();
+        await proceed();
     }
     if (error) {
         throw error;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parallel-park",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Parallel/concurrent async work, optionally using multiple processes",
   "main": "dist/index.js",
   "types": "src/index.ts",

package/src/run-jobs.ts

@@ -1,5 +1,16 @@
+function isThenable<T>(value: unknown): value is Promise<T> {
+  return (
+    typeof value === "object" &&
+    value != null &&
+    // @ts-ignore accessing .then
+    typeof value.then === "function"
+  );
+}
+
+const NOTHING = Symbol("NOTHING");
+
 export async function runJobs<T, U>(
-  inputs: Array<T>,
+  inputs: Iterable<T | Promise<T>> | AsyncIterable<T | Promise<T>>,
   mapper: (input: T, index: number, length: number) => Promise<U>,
   {
     /**
@@ -19,30 +30,49 @@ export async function runJobs<T, U>(
     );
   }
 
-  if (inputs.length === 0) {
-    return Promise.resolve([]);
-  }
+  const inputsArray: Array<T> = [];
+  const inputIteratorFactory =
+    inputs[Symbol.asyncIterator || NOTHING] || inputs[Symbol.iterator];
+  const inputIterator = inputIteratorFactory.call(inputs);
+  const maybeLength = Array.isArray(inputs) ? inputs.length : null;
 
-  concurrency = Math.min(concurrency, inputs.length);
+  let iteratorDone = false;
+
+  async function readInput(): Promise<boolean> {
+    let nextResult = inputIterator.next();
+    if (isThenable(nextResult)) {
+      nextResult = await nextResult;
+    }
+    if (nextResult.done) {
+      iteratorDone = true;
+      return false;
+    } else {
+      let value = nextResult.value;
+      if (isThenable<T>(value)) {
+        value = await value;
+      }
+      inputsArray.push(value);
+      return true;
+    }
+  }
 
   let unstartedIndex = 0;
 
-  const results = new Array(inputs.length);
+  const results = new Array(maybeLength || 0);
   const runningPromises = new Set();
   let error: Error | null = null;
 
-  function takeInput() {
+  async function takeInput() {
+    const read = await readInput();
+    if (!read) return;
+
     const inputIndex = unstartedIndex;
     unstartedIndex++;
 
-    const input = inputs[inputIndex];
-    const promise = mapper(input, inputIndex, inputs.length);
+    const input = inputsArray[inputIndex];
+    const promise = mapper(input, inputIndex, maybeLength || Infinity);
 
-    if (
-      typeof promise !== "object" ||
-      promise == null ||
-      typeof promise.then !== "function"
-    ) {
+    if (!isThenable(promise)) {
       throw new Error(
         "Mapper function passed into runJobs didn't return a Promise. The mapper function should always return a Promise. The easiest way to ensure this is the case is to make your mapper function an async function."
       );
@@ -61,21 +91,19 @@ export async function runJobs<T, U>(
     runningPromises.add(promiseWithMore);
   }
 
-  function proceed() {
-    if (unstartedIndex < inputs.length) {
-      while (runningPromises.size < concurrency) {
-        takeInput();
-      }
+  async function proceed() {
+    while (!iteratorDone && runningPromises.size < concurrency) {
+      await takeInput();
     }
   }
 
-  proceed();
+  await proceed();
   while (runningPromises.size > 0 && !error) {
     await Promise.race(runningPromises.values());
     if (error) {
       throw error;
     }
-    proceed();
+    await proceed();
   }
 
   if (error) {