stream-demux 9.0.1 → 10.0.0

package/README.md

@@ -98,8 +98,5 @@ function wait(duration) {
 
 ## Goal
 
-The goal of this module is to efficiently distribute data to a large number of named asynchronous streams while facilitating functional programming patterns which decrease the probability of memory leaks.
-
-Each stream returned by this module is responsible for picking up its own data from a shared source stream - This means that the stream-demux module doesn't hold any references to streams which it produces via its `stream()` method; this reduces the likelihood of programming mistakes which would lead to memory leaks because streams don't need to be destroyed or cleaned up explicitly.
-
-The downside to making each stream responsible for consuming its own data is that having a lot of concurrent streams can have a negative impact on performance (especially if there are a lot of idle streams). A goal of stream-demux is to keep that overhead to a minimum.
+The goal of this module is to facilitate functional programming patterns which decrease the probability of memory leaks and race conditions.
+It serves as an alternative to callback-based event handling.

package/demuxed-consumable-stream.js

@@ -3,8 +3,8 @@ const ConsumableStream = require('consumable-stream');
 class DemuxedConsumableStream extends ConsumableStream {
   constructor(streamDemux, name) {
     super();
-    this.name = name;
     this._streamDemux = streamDemux;
+    this.name = name;
   }
 
   createConsumer(timeout) {

package/index.js

@@ -1,135 +1,168 @@
 const WritableConsumableStream = require('writable-consumable-stream');
 const DemuxedConsumableStream = require('./demuxed-consumable-stream');
-const StreamConsumer = require('./stream-consumer');
 
 class StreamDemux {
   constructor() {
-    this._mainStream = new WritableConsumableStream();
+    this.streams = {};
+    this._nextConsumerId = 1;
+    this.generateConsumerId = () => {
+      return this._nextConsumerId++;
+    };
   }
 
   write(streamName, value) {
-    this._mainStream.write({
-      stream: streamName,
-      data: {
-        value,
-        done: false
-      }
-    });
+    if (this.streams[streamName]) {
+      this.streams[streamName].write(value);
+    }
   }
 
   close(streamName, value) {
-    this._mainStream.write({
-      stream: streamName,
-      data: {
-        value,
-        done: true
-      }
-    });
+    if (this.streams[streamName]) {
+      this.streams[streamName].close(value);
+    }
   }
 
   closeAll(value) {
-    this._mainStream.close(value);
+    for (let stream of Object.values(this.streams)) {
+      stream.close(value);
+    }
   }
 
   writeToConsumer(consumerId, value) {
-    this._mainStream.writeToConsumer(consumerId, {
-      consumerId,
-      data: {
-        value,
-        done: false
+    for (let stream of Object.values(this.streams)) {
+      if (stream.hasConsumer(consumerId)) {
+        return stream.writeToConsumer(consumerId, value);
       }
-    });
+    }
   }
 
   closeConsumer(consumerId, value) {
-    this._mainStream.closeConsumer(consumerId, {
-      consumerId,
-      data: {
-        value,
-        done: true
+    for (let stream of Object.values(this.streams)) {
+      if (stream.hasConsumer(consumerId)) {
+        return stream.closeConsumer(consumerId, value);
       }
-    });
+    }
   }
 
   getConsumerStats(consumerId) {
-    return this._mainStream.getConsumerStats(consumerId);
+    for (let [streamName, stream] of Object.entries(this.streams)) {
+      if (stream.hasConsumer(consumerId)) {
+        return {
+          ...stream.getConsumerStats(consumerId),
+          stream: streamName
+        };
+      }
+    }
+    return undefined;
   }
 
   getConsumerStatsList(streamName) {
-    let consumerList = this._mainStream.getConsumerStatsList();
-    return consumerList.filter((consumerStats) => {
-      return consumerStats.stream === streamName;
-    });
+    if (this.streams[streamName]) {
+      return this.streams[streamName]
+        .getConsumerStatsList()
+        .map(
+          (stats) => {
+            return {
+              ...stats,
+              stream: streamName
+            };
+          }
+        );
+    }
+    return [];
   }
 
   getConsumerStatsListAll() {
-    return this._mainStream.getConsumerStatsList();
+    let allStatsList = [];
+    for (let streamName of Object.keys(this.streams)) {
+      let statsList = this.getConsumerStatsList(streamName);
+      for (let stats of statsList) {
+        allStatsList.push(stats);
+      }
+    }
+    return allStatsList;
   }
 
   kill(streamName, value) {
-    let consumerList = this.getConsumerStatsList(streamName);
-    let len = consumerList.length;
-    for (let i = 0; i < len; i++) {
-      this.killConsumer(consumerList[i].id, value);
+    if (this.streams[streamName]) {
+      this.streams[streamName].kill(value);
     }
   }
 
   killAll(value) {
-    this._mainStream.kill(value);
+    for (let stream of Object.values(this.streams)) {
+      stream.kill(value);
+    }
   }
 
   killConsumer(consumerId, value) {
-    this._mainStream.killConsumer(consumerId, value);
+    for (let stream of Object.values(this.streams)) {
+      if (stream.hasConsumer(consumerId)) {
+        return stream.killConsumer(consumerId, value);
+      }
+    }
   }
 
   getBackpressure(streamName) {
-    let consumerList = this.getConsumerStatsList(streamName);
-    let len = consumerList.length;
-
-    let maxBackpressure = 0;
-    for (let i = 0; i < len; i++) {
-      let consumer = consumerList[i];
-      if (consumer.backpressure > maxBackpressure) {
-        maxBackpressure = consumer.backpressure;
-      }
+    if (this.streams[streamName]) {
+      return this.streams[streamName].getBackpressure();
     }
-    return maxBackpressure;
+    return 0;
   }
 
   getBackpressureAll() {
-    return this._mainStream.getBackpressure();
+    return Object.values(this.streams).reduce(
+      (max, stream) => Math.max(max, stream.getBackpressure()),
+      0
+    );
   }
 
   getConsumerBackpressure(consumerId) {
-    return this._mainStream.getConsumerBackpressure(consumerId);
+    for (let stream of Object.values(this.streams)) {
+      if (stream.hasConsumer(consumerId)) {
+        return stream.getConsumerBackpressure(consumerId);
+      }
+    }
+    return 0;
   }
 
   hasConsumer(streamName, consumerId) {
-    let consumerStats = this._mainStream.getConsumerStats(consumerId);
-    return !!consumerStats && consumerStats.stream === streamName;
+    if (this.streams[streamName]) {
+      return this.streams[streamName].hasConsumer(consumerId);
+    }
+    return false;
   }
 
   hasConsumerAll(consumerId) {
-    return this._mainStream.hasConsumer(consumerId);
+    return Object.values(this.streams).some(stream => stream.hasConsumer(consumerId));
   }
 
   getConsumerCount(streamName) {
-    return this.getConsumerStatsList(streamName).length;
+    if (this.streams[streamName]) {
+      return this.streams[streamName].getConsumerCount();
+    }
+    return 0;
   }
 
   getConsumerCountAll() {
-    return this.getConsumerStatsListAll().length;
+    return Object.values(this.streams).reduce(
+      (sum, stream) => sum + stream.getConsumerCount(),
+      0
+    );
   }
 
-  createConsumer(streamName, timeout, usabilityMode) {
-    return new StreamConsumer(
-      this._mainStream,
-      this._mainStream.nextConsumerId++,
-      this._mainStream.tailNode,
-      streamName,
-      timeout,
-      usabilityMode
-    );
+  createConsumer(streamName, timeout) {
+    if (!this.streams[streamName]) {
+      this.streams[streamName] = new WritableConsumableStream({
+        generateConsumerId: this.generateConsumerId,
+        removeConsumerCallback: () => {
+          if (!this.getConsumerCount(streamName)) {
+            delete this.streams[streamName];
+          }
+        }
+      });
+    }
+    return this.streams[streamName].createConsumer(timeout);
   }
 
   // Unlike individual consumers, consumable streams support being iterated
@@ -137,6 +170,10 @@ class StreamDemux {
   stream(streamName) {
     return new DemuxedConsumableStream(this, streamName);
   }
+
+  unstream(streamName) {
+    delete this.streams[streamName];
+  }
 }
 
 module.exports = StreamDemux;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "stream-demux",
-  "version": "9.0.1",
+  "version": "10.0.0",
   "description": "A consumable stream demultiplexer.",
   "main": "index.js",
   "scripts": {

package/test/test.js

@@ -231,6 +231,102 @@ describe('StreamDemux', () => {
     assert.equal(receivedPacketsB.length, 10);
   });
 
+  it('should support writing multiple times within the same call stack and across multiple streams', async () => {
+    (async () => {
+      for (let i = 0; i < 10; i++) {
+        await wait(20);
+        for (let j = 0; j < 10; j++) {
+          demux.write('hello', 'world' + i + '-' + j);
+        }
+      }
+      demux.close('hello');
+    })();
+
+    (async () => {
+      for (let i = 0; i < 10; i++) {
+        await wait(10);
+        for (let j = 0; j < 5; j++) {
+          demux.write('other', 'message' + i + '-' + j);
+        }
+      }
+      demux.close('other');
+    })();
+
+    let substream = demux.stream('hello');
+    let otherSubstream = demux.stream('other');
+
+    let otherReceivedPackets = [];
+    (async () => {
+      for await (let otherPacket of otherSubstream) {
+        await wait(10);
+        otherReceivedPackets.push(otherPacket);
+      }
+    })();
+
+    (async () => {
+      for await (let otherPacket of otherSubstream) {
+        await wait(20);
+        otherReceivedPackets.push(otherPacket);
+      }
+    })();
+
+    let receivedPackets = [];
+    for await (let packet of substream) {
+      await wait(20);
+      receivedPackets.push(packet);
+    }
+
+    assert.equal(receivedPackets.length, 100);
+    assert.equal(otherReceivedPackets.length, 100);
+  });
+
+  it('should support writing an arbitrarily large number of times within the same call stack', async () => {
+    (async () => {
+      await wait(50);
+      for (let i = 0; i < 100; i++) {
+        demux.write('hello', 'world' + i);
+      }
+      demux.close('hello');
+    })();
+
+    let substream = demux.stream('hello');
+
+    let receivedPackets = [];
+    for await (let packet of substream) {
+      receivedPackets.push(packet);
+      await wait(10);
+    }
+
+    assert.equal(receivedPackets.length, 100);
+  });
+
+  it('should support writing an arbitrarily large number of times within the same call stack with multiple concurrent consumers', async () => {
+    (async () => {
+      await wait(50);
+      for (let i = 0; i < 50; i++) {
+        demux.write('other', 'message' + i);
+      }
+      demux.close('other');
+    })();
+
+    let otherSubstream = demux.stream('other');
+
+    let receivedPackets = [];
+    (async () => {
+      for await (let otherPacket of otherSubstream) {
+        await wait(10);
+        receivedPackets.push(otherPacket);
+      }
+    })();
+
+    for await (let otherPacket of otherSubstream) {
+      await wait(20);
+      receivedPackets.push(otherPacket);
+    }
+
+    assert.equal(receivedPackets.length, 100);
+  });
+
   it('should support the stream.once() method', async () => {
     (async () => {
       for (let i = 0; i < 10; i++) {
@@ -248,6 +344,8 @@ describe('StreamDemux', () => {
     packet = await substream.once();
     assert.equal(packet, 'world1');
 
+    await wait(1);
+
     packet = await substream.once();
     assert.equal(packet, 'world2');
   });
@@ -319,6 +417,36 @@ describe('StreamDemux', () => {
     assert.equal(packet, null);
   });
 
+  it('should prevent stream timeout from being reset when writing to other streams when iterating over consumer with timeout', async () => {
+    (async () => {
+      for (let i = 0; i < 10; i++) {
+        await wait(200);
+        demux.write('foo', 123);
+        await wait(200);
+        demux.write('hello', 'test' + i);
+      }
+      demux.kill('hello');
+    })();
+
+    let consumer = demux.stream('hello').createConsumer(300);
+    let error;
+
+    let packet;
+
+    try {
+      while (true) {
+        packet = await consumer.next();
+        if (packet.done) break;
+      }
+    } catch (err) {
+      error = err;
+    }
+
+    assert.notEqual(error, null);
+    assert.equal(error.name, 'TimeoutError');
+    assert.equal(packet, null);
+  });
+
   it('should prevent stream.once() timeout from being reset when killing other streams', async () => {
     (async () => {
       for (let i = 0; i < 15; i++) {

package/stream-consumer.js

@@ -1,172 +0,0 @@
-class StreamConsumer {
-  constructor(mainStream, id, startNode, streamName, timeout, usabilityMode) {
-    this.id = id;
-    this._backpressure = 0;
-    this.currentNode = startNode;
-    this.timeout = timeout;
-    this.isAlive = true;
-    this._mainStream = mainStream;
-    this._mainStream.setConsumer(this.id, this);
-    this.streamName = streamName;
-    this.usabilityMode = !!usabilityMode;
-  }
-
-  getStats() {
-    let stats = {
-      id: this.id,
-      backpressure: this._backpressure,
-      stream: this.streamName
-    };
-    if (this.timeout != null) {
-      stats.timeout = this.timeout;
-    }
-    return stats;
-  }
-
-  _resetBackpressure() {
-    this._backpressure = 0;
-  }
-
-  applyBackpressure(packet) {
-    this._backpressure++;
-  }
-
-  releaseBackpressure(packet) {
-    this._backpressure--;
-  }
-
-  getBackpressure() {
-    return this._backpressure;
-  }
-
-  clearActiveTimeout() {
-    clearTimeout(this._timeoutId);
-    delete this._timeoutId;
-  }
-
-  write(packet) {
-    if (packet.done || packet.value.stream === this.streamName) {
-      if (this._timeoutId !== undefined) {
-        this.clearActiveTimeout(packet);
-      }
-      this.applyBackpressure(packet);
-    }
-    if (this._resolve) {
-      this._resolve();
-      delete this._resolve;
-    }
-  }
-
-  kill(value) {
-    this._killPacket = {value, done: true};
-    if (this._timeoutId !== undefined) {
-      this.clearActiveTimeout(this._killPacket);
-    }
-    this._destroy();
-
-    if (this._resolve) {
-      this._resolve();
-      delete this._resolve;
-    }
-  }
-
-  _destroy() {
-    this.isAlive = false;
-    this._resetBackpressure();
-    this._mainStream.removeConsumer(this.id);
-  }
-
-  async _waitForNextItem(timeout) {
-    return new Promise((resolve, reject) => {
-      this._resolve = resolve;
-      let timeoutId;
-      if (timeout != null) {
-        // Create the error object in the outer scope in order
-        // to get the full stack trace.
-        let error = new Error('Stream consumer iteration timed out');
-        (async () => {
-          let delay = wait(timeout);
-          timeoutId = delay.timeoutId;
-          await delay.promise;
-          error.name = 'TimeoutError';
-          delete this._resolve;
-          reject(error);
-        })();
-      }
-      this._timeoutId = timeoutId;
-    });
-  }
-
-  async next() {
-    this._mainStream.setConsumer(this.id, this);
-
-    while (true) {
-      if (!this.currentNode.next) {
-        try {
-          await this._waitForNextItem(this.timeout);
-        } catch (error) {
-          this._destroy();
-          throw error;
-        }
-      }
-      if (this._killPacket) {
-        this._destroy();
-        let killPacket = this._killPacket;
-        delete this._killPacket;
-
-        return killPacket;
-      }
-
-      // Skip over nodes which belong to different streams.
-      while (
-        this.currentNode.next?.data?.value &&
-        this.currentNode.next.data.value.stream !== this.streamName &&
-        this.currentNode.next.consumerId !== this.id &&
-        !this.currentNode.next.data.done
-      ) {
-        this.currentNode = this.currentNode.next;
-        this.usabilityMode && await wait(0);
-      }
-
-      if (!this.currentNode.next) {
-        continue;
-      }
-
-      this.currentNode = this.currentNode.next;
-
-      this.releaseBackpressure(this.currentNode.data);
-
-      if (this.currentNode.data?.value?.data?.done) {
-        this._destroy();
-        return this.currentNode.data.value.data;
-      }
-
-      if (this.currentNode.data.done) {
-        this._destroy();
-        return this.currentNode.data;
-      }
-
-      return this.currentNode.data.value.data;
-    }
-  }
-
-  return() {
-    delete this.currentNode;
-    this._destroy();
-    return {};
-  }
-
-  [Symbol.asyncIterator]() {
-    return this;
-  }
-}
-
-function wait(timeout) {
-  let timeoutId;
-  let promise = new Promise((resolve) => {
-    timeoutId = setTimeout(resolve, timeout);
-  });
-  return {timeoutId, promise};
-}
-
-module.exports = StreamConsumer;