thread-stream 3.1.0 → 4.1.0

package/.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
       contents: read
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           persist-credentials: false
 
@@ -38,20 +38,17 @@ jobs:
       contents: read
     strategy:
       matrix:
-        node-version: [18, 20, 22]
+        node-version: [20, 22, 24, 26]
         os: [macos-latest, ubuntu-latest, windows-latest]
-        exclude:
-          - os: windows-latest
-            node-version: 22
 
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           persist-credentials: false
 
       - name: Setup Node ${{ matrix.node-version }}
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: ${{ matrix.node-version }}
 
@@ -61,23 +58,6 @@ jobs:
       - name: Run tests
         run: npm run test:ci
 
-      - name: Coveralls Parallel
-        uses: coverallsapp/github-action@v2.3.0
-        with:
-          github-token: ${{ secrets.github_token }}
-          parallel: true
-          flag-name: run-${{ matrix.node-version }}-${{ matrix.os }}
-
-  coverage:
-    needs: test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Coveralls Finished
-        uses: coverallsapp/github-action@v2.3.0
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          parallel-finished: true
-
   automerge:
     name: Automerge Dependabot PRs
     if: >

package/README.md

@@ -39,6 +39,8 @@ stream.flush(function () {
 })
 ```
 
+`flush(cb)` waits for the worker destination flush when supported (`flush`, `flushSync`, or pending `drain`).
+
 In `worker.js`:
 
 ```js
@@ -87,7 +89,8 @@ This module works with `yarn` in PnP (plug'n play) mode too!
 ### Emit events
 
 You can emit events on the ThreadStream from your worker using [`worker.parentPort.postMessage()`](https://nodejs.org/api/worker_threads.html#workerparentport).
-The message (JSON object) must have the following data structure:
+Messages that do not carry a thread-stream protocol `code` are ignored.
+For custom events, the message (JSON object) must have the following data structure:
 
 ```js
 parentPort.postMessage({

package/eslint.config.js

@@ -0,0 +1,10 @@
+'use strict'
+
+const neostandard = require('neostandard')
+
+module.exports = neostandard({
+  ignores: [
+    'test/ts/**/*',
+    'test/syntax-error.mjs'
+  ]
+})

package/index.d.ts

@@ -59,6 +59,13 @@ declare class ThreadStream extends EventEmitter {
    * Calling the {@link write()} method after calling {@link end()} will emit an error.
    */
   end(): void
+  /**
+   * Flush the stream asynchronously.
+   *
+   * The callback is invoked once data has been consumed by the worker and the
+   * worker destination has acknowledged the flush.
+   */
+  flush(cb?: (err?: Error) => void): void
   /**
    * Flush the stream synchronously.
    * This method should be called in the shutdown phase to make sure that all data has been flushed.

package/index.js

@@ -18,6 +18,8 @@ const kImpl = Symbol('kImpl')
 // V8 limit for string size
 const MAX_STRING = buffer.constants.MAX_STRING_LENGTH
 
+function noop () {}
+
 class FakeWeakRef {
   constructor (value) {
     this._value = value
@@ -54,6 +56,7 @@ function createWorker (stream, opts) {
 
   const worker = new Worker(toExecute, {
     ...opts.workerOpts,
+    name: opts.workerOpts?.name || 'thread-stream',
     trackUnmanagedFds: false,
     workerData: {
       filename: filename.indexOf('file://') === 0
@@ -114,14 +117,15 @@ function nextFlush (stream) {
       write(stream, toWrite, nextFlush.bind(null, stream))
     } else {
       // multi-byte utf-8
-      stream.flush(() => {
-        // err is already handled in flush()
+      waitForRead(stream, () => {
+        // err is already handled in waitForRead()
         if (stream.destroyed) {
           return
         }
 
         Atomics.store(stream[kImpl].state, READ_INDEX, 0)
         Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+        Atomics.notify(stream[kImpl].state, READ_INDEX)
 
         // Find a toWrite length that fits the buffer
         // it must exists as the buffer is at least 4 bytes length
@@ -140,9 +144,10 @@ function nextFlush (stream) {
       // we had a flushSync in the meanwhile
       return
     }
-    stream.flush(() => {
+    waitForRead(stream, () => {
       Atomics.store(stream[kImpl].state, READ_INDEX, 0)
       Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      Atomics.notify(stream[kImpl].state, READ_INDEX)
       nextFlush(stream)
     })
   } else {
@@ -160,13 +165,19 @@ function onWorkerMessage (msg) {
     return
   }
 
+  // Node.js watch mode may send internal worker messages that do not
+  // participate in thread-stream's worker protocol.
+  if (msg?.code == null) {
+    return
+  }
+
   switch (msg.code) {
     case 'READY':
       // Replace the FakeWeakRef with a
       // proper one.
       this.stream = new WeakRef(stream)
 
-      stream.flush(() => {
+      waitForRead(stream, () => {
         stream[kImpl].ready = true
         stream.emit('ready')
       })
@@ -181,6 +192,19 @@ function onWorkerMessage (msg) {
         stream.emit(msg.name, msg.args)
       }
       break
+    case 'FLUSHED': {
+      if (msg.context !== 'thread-stream') {
+        destroy(stream, new Error('this should not happen: ' + msg.code))
+        break
+      }
+
+      const cb = stream[kImpl].flushCallbacks.get(msg.id)
+      if (cb) {
+        stream[kImpl].flushCallbacks.delete(msg.id)
+        process.nextTick(cb)
+      }
+      break
+    }
     case 'WARNING':
       process.emitWarning(msg.err)
       break
@@ -225,6 +249,8 @@ class ThreadStream extends EventEmitter {
     this[kImpl].errored = null
     this[kImpl].closed = false
     this[kImpl].buf = ''
+    this[kImpl].flushCallbacks = new Map()
+    this[kImpl].nextFlushId = 0
 
     // TODO (fix): Make private?
     this.worker = createWorker(this, opts) // TODO (fix): make private
@@ -285,28 +311,15 @@ class ThreadStream extends EventEmitter {
   }
 
   flush (cb) {
-    if (this[kImpl].destroyed) {
-      if (typeof cb === 'function') {
-        process.nextTick(cb, new Error('the worker has exited'))
-      }
-      return
-    }
+    cb = typeof cb === 'function' ? cb : noop
 
-    // TODO write all .buf
-    const writeIndex = Atomics.load(this[kImpl].state, WRITE_INDEX)
-    // process._rawDebug(`(flush) readIndex (${Atomics.load(this.state, READ_INDEX)}) writeIndex (${Atomics.load(this.state, WRITE_INDEX)})`)
-    wait(this[kImpl].state, READ_INDEX, writeIndex, Infinity, (err, res) => {
+    flushBuffer(this, (err) => {
       if (err) {
-        destroy(this, err)
         process.nextTick(cb, err)
         return
       }
-      if (res === 'not-equal') {
-        // TODO handle deadlock
-        this.flush(cb)
-        return
-      }
-      process.nextTick(cb)
+
+      requestWorkerFlush(this, cb)
     })
   }
 
@@ -364,6 +377,93 @@ class ThreadStream extends EventEmitter {
   }
 }
 
+function flushBuffer (stream, cb) {
+  if (stream[kImpl].destroyed) {
+    process.nextTick(cb, new Error('the worker has exited'))
+    return
+  }
+
+  if (!stream[kImpl].sync && (stream[kImpl].flushing || stream[kImpl].buf.length > 0)) {
+    setImmediate(flushBuffer, stream, cb)
+    return
+  }
+
+  waitForRead(stream, cb)
+}
+
+function waitForRead (stream, cb) {
+  const writeIndex = Atomics.load(stream[kImpl].state, WRITE_INDEX)
+  wait(stream[kImpl].state, READ_INDEX, writeIndex, Infinity, (err, res) => {
+    if (err) {
+      destroy(stream, err)
+      cb(err)
+      return
+    }
+
+    if (res !== 'ok') {
+      waitForRead(stream, cb)
+      return
+    }
+
+    cb()
+  })
+}
+
+function requestWorkerFlush (stream, cb) {
+  if (stream[kImpl].destroyed) {
+    process.nextTick(cb, new Error('the worker has exited'))
+    return
+  }
+
+  if (!stream[kImpl].ready) {
+    const onReady = () => {
+      cleanup()
+      requestWorkerFlush(stream, cb)
+    }
+    const onClose = () => {
+      cleanup()
+      process.nextTick(cb, new Error('the worker has exited'))
+    }
+    const cleanup = () => {
+      stream.off('ready', onReady)
+      stream.off('close', onClose)
+    }
+
+    stream.once('ready', onReady)
+    stream.once('close', onClose)
+    return
+  }
+
+  const id = ++stream[kImpl].nextFlushId
+  stream[kImpl].flushCallbacks.set(id, cb)
+
+  try {
+    stream.worker.postMessage({
+      code: 'FLUSH',
+      context: 'thread-stream',
+      id
+    })
+  } catch (err) {
+    stream[kImpl].flushCallbacks.delete(id)
+    destroy(stream, err)
+    process.nextTick(cb, err)
+  }
+}
+
+function failPendingFlushCallbacks (stream, err) {
+  const callbacks = stream[kImpl].flushCallbacks
+  if (callbacks.size === 0) {
+    return
+  }
+
+  const flushErr = err || new Error('the worker has exited')
+
+  for (const cb of callbacks.values()) {
+    process.nextTick(cb, flushErr)
+  }
+  callbacks.clear()
+}
+
 function error (stream, err) {
   setImmediate(() => {
     stream.emit('error', err)
@@ -375,6 +475,7 @@ function destroy (stream, err) {
     return
   }
   stream[kImpl].destroyed = true
+  failPendingFlushCallbacks(stream, err)
 
   if (err) {
     stream[kImpl].errored = err
@@ -468,6 +569,7 @@ function writeSync (stream) {
       flushSync(stream)
       Atomics.store(stream[kImpl].state, READ_INDEX, 0)
       Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      Atomics.notify(stream[kImpl].state, READ_INDEX)
       continue
     } else if (leftover < 0) {
       // stream should never happen
@@ -485,6 +587,7 @@ function writeSync (stream) {
       flushSync(stream)
       Atomics.store(stream[kImpl].state, READ_INDEX, 0)
       Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      Atomics.notify(stream[kImpl].state, READ_INDEX)
 
       // Find a toWrite length that fits the buffer
       // it must exists as the buffer is at least 4 bytes length

package/lib/wait.js

@@ -1,61 +1,68 @@
 'use strict'
 
-const MAX_TIMEOUT = 1000
+// Maximum wait time for a single waitAsync call
+// Used as a fallback poll interval in case notifications are missed
+// Keep this low enough for good throughput but high enough to not busy-loop
+const WAIT_MS = 10000
 
 function wait (state, index, expected, timeout, done) {
-  const max = Date.now() + timeout
-  let current = Atomics.load(state, index)
-  if (current === expected) {
-    done(null, 'ok')
-    return
-  }
-  let prior = current
-  const check = (backoff) => {
-    if (Date.now() > max) {
+  const max = timeout === Infinity ? Infinity : Date.now() + timeout
+
+  const check = () => {
+    const current = Atomics.load(state, index)
+    if (current === expected) {
+      done(null, 'ok')
+      return
+    }
+
+    if (max !== Infinity && Date.now() > max) {
       done(null, 'timed-out')
+      return
+    }
+
+    // Wait for any change from current value
+    const remaining = max === Infinity ? WAIT_MS : Math.min(WAIT_MS, Math.max(1, max - Date.now()))
+    const result = Atomics.waitAsync(state, index, current, remaining)
+
+    if (result.async) {
+      result.value.then(check)
     } else {
-      setTimeout(() => {
-        prior = current
-        current = Atomics.load(state, index)
-        if (current === prior) {
-          check(backoff >= MAX_TIMEOUT ? MAX_TIMEOUT : backoff * 2)
-        } else {
-          if (current === expected) done(null, 'ok')
-          else done(null, 'not-equal')
-        }
-      }, backoff)
+      // Value already changed (not-equal) - recheck on next tick
+      setImmediate(check)
     }
   }
-  check(1)
+
+  check()
 }
 
-// let waitDiffCount = 0
 function waitDiff (state, index, expected, timeout, done) {
-  // const id = waitDiffCount++
-  // process._rawDebug(`>>> waitDiff ${id}`)
-  const max = Date.now() + timeout
-  let current = Atomics.load(state, index)
-  if (current !== expected) {
-    done(null, 'ok')
-    return
-  }
-  const check = (backoff) => {
-    // process._rawDebug(`${id} ${index} current ${current} expected ${expected}`)
-    // process._rawDebug('' + backoff)
-    if (Date.now() > max) {
+  const max = timeout === Infinity ? Infinity : Date.now() + timeout
+
+  const check = () => {
+    const current = Atomics.load(state, index)
+    if (current !== expected) {
+      done(null, 'ok')
+      return
+    }
+
+    if (max !== Infinity && Date.now() > max) {
       done(null, 'timed-out')
+      return
+    }
+
+    // Wait for value to change from expected
+    const remaining = max === Infinity ? WAIT_MS : Math.min(WAIT_MS, Math.max(1, max - Date.now()))
+    const result = Atomics.waitAsync(state, index, expected, remaining)
+
+    if (result.async) {
+      result.value.then(check)
     } else {
-      setTimeout(() => {
-        current = Atomics.load(state, index)
-        if (current !== expected) {
-          done(null, 'ok')
-        } else {
-          check(backoff >= MAX_TIMEOUT ? MAX_TIMEOUT : backoff * 2)
-        }
-      }, backoff)
+      // Value already changed (not-equal) - recheck on next tick
+      setImmediate(check)
     }
   }
-  check(1)
+
+  check()
 }
 
 module.exports = { wait, waitDiff }

package/lib/worker.js

@@ -12,26 +12,114 @@ const {
 } = workerData
 
 let destination
+const flushQueue = []
+let flushing = false
 
 const state = new Int32Array(stateBuf)
 const data = Buffer.from(dataBuf)
 
+// Keep the event loop alive - Atomics.waitAsync promises don't prevent worker exit
+const keepAlive = setInterval(() => {}, 60 * 60 * 1000)
+
+function onParentPortMessage (msg) {
+  if (!msg || msg.code !== 'FLUSH' || msg.context !== 'thread-stream') {
+    return
+  }
+
+  flushQueue.push(msg.id)
+  processFlushQueue()
+}
+
+function processFlushQueue () {
+  if (flushing || !destination) {
+    return
+  }
+
+  const id = flushQueue.shift()
+  if (id === undefined) {
+    return
+  }
+
+  flushing = true
+  flushDestination((err) => {
+    flushing = false
+
+    if (err) {
+      parentPort.postMessage({
+        code: 'ERROR',
+        err
+      })
+      return
+    }
+
+    parentPort.postMessage({
+      code: 'FLUSHED',
+      context: 'thread-stream',
+      id
+    })
+
+    processFlushQueue()
+  })
+}
+
+function flushDestination (cb) {
+  if (typeof destination?.flush === 'function') {
+    if (destination.flush.length === 0) {
+      try {
+        const result = destination.flush()
+        if (result && typeof result.then === 'function') {
+          result.then(() => cb(), cb)
+        } else {
+          cb()
+        }
+      } catch (err) {
+        cb(err)
+      }
+      return
+    }
+
+    let done = false
+    const onDone = (err) => {
+      if (done) {
+        return
+      }
+      done = true
+      cb(err)
+    }
+
+    try {
+      const result = destination.flush(onDone)
+      if (result && typeof result.then === 'function') {
+        result.then(() => onDone(), onDone)
+      }
+    } catch (err) {
+      onDone(err)
+    }
+    return
+  }
+
+  if (typeof destination?.flushSync === 'function') {
+    try {
+      destination.flushSync()
+      cb()
+    } catch (err) {
+      cb(err)
+    }
+    return
+  }
+
+  if (destination?.writableNeedDrain && !destination?.writableEnded) {
+    destination.once('drain', cb)
+    return
+  }
+
+  cb()
+}
+
 async function start () {
   let worker
   try {
-    if (filename.endsWith('.ts') || filename.endsWith('.cts')) {
-      // TODO: add support for the TSM modules loader ( https://github.com/lukeed/tsm ).
-      if (!process[Symbol.for('ts-node.register.instance')]) {
-        realRequire('ts-node/register')
-      } else if (process.env.TS_NODE_DEV) {
-        realRequire('ts-node-dev')
-      }
-      // TODO: Support ES imports once tsc, tap & ts-node provide better compatibility guarantees.
-      // Remove extra forwardslash on Windows
-      worker = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
-    } else {
-      worker = (await realImport(filename))
-    }
+    worker = (await realImport(filename))
   } catch (error) {
     // A yarn user that tries to start a ThreadStream for an external module
     // provides a filename pointing to a zip file.
@@ -53,6 +141,19 @@ async function start () {
       } catch {
         throw error
       }
+    } else if (filename.endsWith('.ts') || filename.endsWith('.cts')) {
+      // Native TypeScript import failed (type stripping not enabled).
+      // Fall back to ts-node for TypeScript files.
+      try {
+        if (!process[Symbol.for('ts-node.register.instance')]) {
+          realRequire('ts-node/register')
+        } else if (process.env.TS_NODE_DEV) {
+          realRequire('ts-node-dev')
+        }
+        worker = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
+      } catch {
+        throw error
+      }
     } else {
       throw error
     }
@@ -84,16 +185,21 @@ async function start () {
     const end = Atomics.load(state, WRITE_INDEX)
     Atomics.store(state, READ_INDEX, end)
     Atomics.notify(state, READ_INDEX)
+    clearInterval(keepAlive)
     setImmediate(() => {
       process.exit(0)
     })
   })
+
+  processFlushQueue()
 }
 
 // No .catch() handler,
 // in case there is an error it goes
 // to unhandledRejection
 start().then(function () {
+  parentPort.on('message', onParentPortMessage)
+
   parentPort.postMessage({
     code: 'READY'
   })

package/package.json

@@ -1,42 +1,35 @@
 {
   "name": "thread-stream",
-  "version": "3.1.0",
+  "version": "4.1.0",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
   "types": "index.d.ts",
+  "engines": {
+    "node": ">=20"
+  },
   "dependencies": {
-    "real-require": "^0.2.0"
+    "real-require": "^1.0.0"
   },
   "devDependencies": {
-    "@types/node": "^20.1.0",
-    "@types/tap": "^15.0.0",
-    "@yao-pkg/pkg": "^5.11.5",
+    "@types/node": "^25.0.2",
+    "@yao-pkg/pkg": "^6.0.0",
+    "borp": "^1.0.0",
     "desm": "^1.3.0",
+    "eslint": "^9.39.1",
     "fastbench": "^1.0.1",
-    "husky": "^9.0.6",
+    "neostandard": "^0.13.0",
     "pino-elasticsearch": "^8.0.0",
-    "sonic-boom": "^4.0.1",
-    "standard": "^17.0.0",
-    "tap": "^16.2.0",
+    "sonic-boom": "^5.0.0",
     "ts-node": "^10.8.0",
-    "typescript": "^5.3.2",
-    "why-is-node-running": "^2.2.2"
+    "typescript": "~5.7.3"
   },
   "scripts": {
     "build": "tsc --noEmit",
-    "test": "standard && npm run build && npm run transpile && tap \"test/**/*.test.*js\" && tap --ts test/*.test.*ts",
-    "test:ci": "standard && npm run transpile && npm run test:ci:js && npm run test:ci:ts",
-    "test:ci:js": "tap --no-check-coverage --timeout=120 --coverage-report=lcovonly \"test/**/*.test.*js\"",
-    "test:ci:ts": "tap --ts --no-check-coverage --coverage-report=lcovonly \"test/**/*.test.*ts\"",
-    "test:yarn": "npm run transpile && tap \"test/**/*.test.js\" --no-check-coverage",
-    "transpile": "sh ./test/ts/transpile.sh",
-    "prepare": "husky install"
-  },
-  "standard": {
-    "ignore": [
-      "test/ts/**/*",
-      "test/syntax-error.mjs"
-    ]
+    "lint": "eslint",
+    "test": "npm run lint && npm run build && npm run transpile && borp --pattern \"test/*.test.{js,mjs}\"",
+    "test:ci": "npm run lint && npm run transpile && borp --pattern \"test/*.test.{js,mjs}\"",
+    "test:yarn": "npm run transpile && borp --pattern \"test/*.test.js\"",
+    "transpile": "sh ./test/ts/transpile.sh"
   },
   "repository": {
     "type": "git",