stream-demux 8.1.0 → 9.0.1

package/index.js

@@ -1,5 +1,6 @@
 const WritableConsumableStream = require('writable-consumable-stream');
 const DemuxedConsumableStream = require('./demuxed-consumable-stream');
+const StreamConsumer = require('./stream-consumer');
 
 class StreamDemux {
   constructor() {
@@ -120,74 +121,19 @@ class StreamDemux {
     return this.getConsumerStatsListAll().length;
   }
 
-  createConsumer(streamName, timeout) {
-    let mainStreamConsumer = this._mainStream.createConsumer(timeout);
-
-    let consumerNext = mainStreamConsumer.next;
-    mainStreamConsumer.next = async function () {
-      while (true) {
-        let packet = await consumerNext.apply(this, arguments);
-        if (packet.value) {
-          if (
-            packet.value.stream === streamName ||
-            packet.value.consumerId === this.id
-          ) {
-            if (packet.value.data.done) {
-              this.return();
-            }
-            return packet.value.data;
-          }
-        }
-        if (packet.done) {
-          return packet;
-        }
-      }
-    };
-
-    let consumerGetStats = mainStreamConsumer.getStats;
-    mainStreamConsumer.getStats = function () {
-      let stats = consumerGetStats.apply(this, arguments);
-      stats.stream = streamName;
-      return stats;
-    };
-
-    let consumerApplyBackpressure = mainStreamConsumer.applyBackpressure;
-    mainStreamConsumer.applyBackpressure = function (packet) {
-      if (packet.value) {
-        if (
-          packet.value.stream === streamName ||
-          packet.value.consumerId === this.id
-        ) {
-          consumerApplyBackpressure.apply(this, arguments);
-
-          return;
-        }
-      }
-      if (packet.done) {
-        consumerApplyBackpressure.apply(this, arguments);
-      }
-    };
-
-    let consumerReleaseBackpressure = mainStreamConsumer.releaseBackpressure;
-    mainStreamConsumer.releaseBackpressure = function (packet) {
-      if (packet.value) {
-        if (
-          packet.value.stream === streamName ||
-          packet.value.consumerId === this.id
-        ) {
-          consumerReleaseBackpressure.apply(this, arguments);
-
-          return;
-        }
-      }
-      if (packet.done) {
-        consumerReleaseBackpressure.apply(this, arguments);
-      }
-    };
-
-    return mainStreamConsumer;
+  createConsumer(streamName, timeout, usabilityMode) {
+    return new StreamConsumer(
+      this._mainStream,
+      this._mainStream.nextConsumerId++,
+      this._mainStream.tailNode,
+      streamName,
+      timeout,
+      usabilityMode
+    );
   }
 
+  // Unlike individual consumers, consumable streams support being iterated
+  // over by multiple for-await-of loops in parallel.
   stream(streamName) {
     return new DemuxedConsumableStream(this, streamName);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "stream-demux",
-  "version": "8.1.0",
+  "version": "9.0.1",
   "description": "A consumable stream demultiplexer.",
   "main": "index.js",
   "scripts": {
@@ -25,10 +25,10 @@
   },
   "homepage": "https://github.com/SocketCluster/stream-demux#readme",
   "devDependencies": {
-    "mocha": "^5.0.5"
+    "mocha": "^10.2.0"
   },
   "dependencies": {
-    "consumable-stream": "^2.0.0",
-    "writable-consumable-stream": "^3.0.1"
+    "consumable-stream": "^3.0.0",
+    "writable-consumable-stream": "^4.0.1"
   }
 }

package/stream-consumer.js

@@ -0,0 +1,172 @@
+class StreamConsumer {
+  constructor(mainStream, id, startNode, streamName, timeout, usabilityMode) {
+    this.id = id;
+    this._backpressure = 0;
+    this.currentNode = startNode;
+    this.timeout = timeout;
+    this.isAlive = true;
+    this._mainStream = mainStream;
+    this._mainStream.setConsumer(this.id, this);
+    this.streamName = streamName;
+    this.usabilityMode = !!usabilityMode;
+  }
+
+  getStats() {
+    let stats = {
+      id: this.id,
+      backpressure: this._backpressure,
+      stream: this.streamName
+    };
+    if (this.timeout != null) {
+      stats.timeout = this.timeout;
+    }
+    return stats;
+  }
+
+  _resetBackpressure() {
+    this._backpressure = 0;
+  }
+
+  applyBackpressure(packet) {
+    this._backpressure++;
+  }
+
+  releaseBackpressure(packet) {
+    this._backpressure--;
+  }
+
+  getBackpressure() {
+    return this._backpressure;
+  }
+
+  clearActiveTimeout() {
+    clearTimeout(this._timeoutId);
+    delete this._timeoutId;
+  }
+
+  write(packet) {
+    if (packet.done || packet.value.stream === this.streamName) {
+      if (this._timeoutId !== undefined) {
+        this.clearActiveTimeout(packet);
+      }
+      this.applyBackpressure(packet);
+    }
+    if (this._resolve) {
+      this._resolve();
+      delete this._resolve;
+    }
+  }
+
+  kill(value) {
+    this._killPacket = {value, done: true};
+    if (this._timeoutId !== undefined) {
+      this.clearActiveTimeout(this._killPacket);
+    }
+    this._destroy();
+
+    if (this._resolve) {
+      this._resolve();
+      delete this._resolve;
+    }
+  }
+
+  _destroy() {
+    this.isAlive = false;
+    this._resetBackpressure();
+    this._mainStream.removeConsumer(this.id);
+  }
+
+  async _waitForNextItem(timeout) {
+    return new Promise((resolve, reject) => {
+      this._resolve = resolve;
+      let timeoutId;
+      if (timeout != null) {
+        // Create the error object in the outer scope in order
+        // to get the full stack trace.
+        let error = new Error('Stream consumer iteration timed out');
+        (async () => {
+          let delay = wait(timeout);
+          timeoutId = delay.timeoutId;
+          await delay.promise;
+          error.name = 'TimeoutError';
+          delete this._resolve;
+          reject(error);
+        })();
+      }
+      this._timeoutId = timeoutId;
+    });
+  }
+
+  async next() {
+    this._mainStream.setConsumer(this.id, this);
+
+    while (true) {
+      if (!this.currentNode.next) {
+        try {
+          await this._waitForNextItem(this.timeout);
+        } catch (error) {
+          this._destroy();
+          throw error;
+        }
+      }
+      if (this._killPacket) {
+        this._destroy();
+        let killPacket = this._killPacket;
+        delete this._killPacket;
+
+        return killPacket;
+      }
+
+      // Skip over nodes which belong to different streams.
+      while (
+        this.currentNode.next?.data?.value &&
+        this.currentNode.next.data.value.stream !== this.streamName &&
+        this.currentNode.next.consumerId !== this.id &&
+        !this.currentNode.next.data.done
+      ) {
+        this.currentNode = this.currentNode.next;
+        this.usabilityMode && await wait(0);
+      }
+
+      if (!this.currentNode.next) {
+        continue;
+      }
+
+      this.currentNode = this.currentNode.next;
+
+      this.releaseBackpressure(this.currentNode.data);
+
+      if (this.currentNode.data?.value?.data?.done) {
+        this._destroy();
+        return this.currentNode.data.value.data;
+      }
+
+      if (this.currentNode.data.done) {
+        this._destroy();
+        return this.currentNode.data;
+      }
+
+      return this.currentNode.data.value.data;
+    }
+  }
+
+  return() {
+    delete this.currentNode;
+    this._destroy();
+    return {};
+  }
+
+  [Symbol.asyncIterator]() {
+    return this;
+  }
+}
+
+function wait(timeout) {
+  let timeoutId;
+  let promise = new Promise((resolve) => {
+    timeoutId = setTimeout(resolve, timeout);
+  });
+  return {timeoutId, promise};
+}
+
+module.exports = StreamConsumer;

package/test/test.js

@@ -296,6 +296,74 @@ describe('StreamDemux', () => {
     assert.equal(packet, null);
   });
 
+  it('should prevent stream.once() timeout from being reset when writing to other streams', async () => {
+    (async () => {
+      for (let i = 0; i < 15; i++) {
+        await wait(20);
+        demux.write('hi', 'test');
+      }
+    })();
+
+    let substream = demux.stream('hello');
+
+    let packet;
+    let error;
+    try {
+      packet = await substream.once(200);
+    } catch (err) {
+      error = err;
+    }
+
+    assert.notEqual(error, null);
+    assert.equal(error.name, 'TimeoutError');
+    assert.equal(packet, null);
+  });
+
+  it('should prevent stream.once() timeout from being reset when killing other streams', async () => {
+    (async () => {
+      for (let i = 0; i < 15; i++) {
+        await wait(20);
+        demux.kill('hi' + i, 'test');
+      }
+    })();
+
+    let substream = demux.stream('hello');
+
+    let packet;
+    let error;
+    try {
+      packet = await substream.once(200);
+    } catch (err) {
+      error = err;
+    }
+
+    assert.notEqual(error, null);
+    assert.equal(error.name, 'TimeoutError');
+    assert.equal(packet, null);
+  });
+
+  it('should support stream.once() timeout after killing the stream', async () => {
+    (async () => {
+      await wait(20);
+      demux.kill('hello', 'test');
+    })();
+
+    let substream = demux.stream('hello');
+
+    let start = Date.now();
+
+    let packet;
+    let error;
+    try {
+      packet = await substream.once(200);
+    } catch (err) {
+      error = err;
+    }
+    assert.notEqual(error, null);
+    assert.equal(error.name, 'TimeoutError');
+    assert.equal(packet, null);
+  });
+
   it('should support stream.next() method with close command', async () => {
     (async () => {
       for (let i = 0; i < 3; i++) {
@@ -343,6 +411,7 @@ describe('StreamDemux', () => {
     let consumer = demux.stream('hello').createConsumer();
 
     (async () => {
+      await wait(50);
       for (let i = 0; i < 10; i++) {
         await wait(10);
         demux.writeToConsumer(consumer.id, 'world' + i);