thread-stream 3.0.2 → 3.2.0

package/.claude/settings.local.json

@@ -0,0 +1,15 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(npx tap:*)",
+      "Bash(node:*)",
+      "Bash(for i in 1 2 3 4 5)",
+      "Bash(do)",
+      "Bash(echo:*)",
+      "Bash(done)",
+      "Bash(npm test:*)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

package/.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
         run: npm run test:ci
 
       - name: Coveralls Parallel
-        uses: coverallsapp/github-action@v2.2.3
+        uses: coverallsapp/github-action@v2.3.0
         with:
           github-token: ${{ secrets.github_token }}
           parallel: true
@@ -73,7 +73,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Coveralls Finished
-        uses: coverallsapp/github-action@v2.2.3
+        uses: coverallsapp/github-action@v2.3.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           parallel-finished: true

package/CLAUDE.md

@@ -0,0 +1,64 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+thread-stream is a library for streaming data to a Node.js Worker Thread. It uses SharedArrayBuffer and Atomics for efficient inter-thread communication, enabling high-performance data streaming to worker threads.
+
+## Build & Test Commands
+
+```bash
+npm test                    # Run linting (standard), type checking, and all tests
+npm run build               # Type check only (tsc --noEmit)
+npm run test:ci             # CI-specific test run
+
+# Run a single test file
+node --test test/<filename>.test.js
+node --test test/<filename>.test.ts  # For TypeScript tests
+
+# Lint
+npx standard
+```
+
+## Architecture
+
+### Core Components
+
+- **index.js**: Main `ThreadStream` class extending EventEmitter. Manages shared memory buffers, worker lifecycle, and provides stream-like write/flush/end API.
+
+- **lib/worker.js**: Runs inside the Worker Thread. Loads the user-provided destination module, reads from shared buffer, and writes to the destination stream.
+
+- **lib/indexes.js**: Defines shared buffer index constants (`WRITE_INDEX`, `READ_INDEX`) used for Atomics-based synchronization.
+
+- **lib/wait.js**: Provides `wait()` and `waitDiff()` utilities for async waiting on Atomics state changes with exponential backoff.
+
+### Shared Memory Communication
+
+The main thread and worker communicate via two SharedArrayBuffers:
+1. **stateBuf**: Int32Array for READ_INDEX and WRITE_INDEX positions
+2. **dataBuf**: Buffer for actual string data (default 4MB)
+
+Write flow: Main thread writes to dataBuf, updates WRITE_INDEX, worker reads data between READ_INDEX and WRITE_INDEX, updates READ_INDEX when consumed.
+
+### Worker Module Interface
+
+User-provided worker modules must export an async function that receives `workerData` and returns a writable stream:
+
+```js
+async function run(opts) {
+  const stream = fs.createWriteStream(opts.dest)
+  await once(stream, 'open')
+  return stream
+}
+module.exports = run
+```
+
+### Sync vs Async Modes
+
+- `sync: true`: Blocking writes using flushSync, waits for worker to consume
+- `sync: false` (default): Non-blocking writes with drain events when buffer fills
+
+## Code Style
+
+Uses [Standard](https://standardjs.com/) for linting. Test files in `test/ts/**/*` and `test/syntax-error.mjs` are excluded from linting.

package/README.md

@@ -87,7 +87,8 @@ This module works with `yarn` in PnP (plug'n play) mode too!
 ### Emit events
 
 You can emit events on the ThreadStream from your worker using [`worker.parentPort.postMessage()`](https://nodejs.org/api/worker_threads.html#workerparentport).
-The message (JSON object) must have the following data structure:
+Messages that do not carry a thread-stream protocol `code` are ignored.
+For custom events, the message (JSON object) must have the following data structure:
 
 ```js
 parentPort.postMessage({

package/index.js

@@ -8,7 +8,8 @@ const { pathToFileURL } = require('url')
 const { wait } = require('./lib/wait')
 const {
   WRITE_INDEX,
-  READ_INDEX
+  READ_INDEX,
+  SEQ_INDEX
 } = require('./lib/indexes')
 const buffer = require('buffer')
 const assert = require('assert')
@@ -18,6 +19,13 @@ const kImpl = Symbol('kImpl')
 // V8 limit for string size
 const MAX_STRING = buffer.constants.MAX_STRING_LENGTH
 
+function updateState (stream, fn) {
+  Atomics.add(stream[kImpl].state, SEQ_INDEX, 1)
+  fn()
+  Atomics.add(stream[kImpl].state, SEQ_INDEX, 1)
+  Atomics.notify(stream[kImpl].state, SEQ_INDEX)
+}
+
 class FakeWeakRef {
   constructor (value) {
     this._value = value
@@ -120,8 +128,11 @@ function nextFlush (stream) {
           return
         }
 
-        Atomics.store(stream[kImpl].state, READ_INDEX, 0)
-        Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+        updateState(stream, () => {
+          Atomics.store(stream[kImpl].state, READ_INDEX, 0)
+          Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+        })
+        Atomics.notify(stream[kImpl].state, READ_INDEX)
 
         // Find a toWrite length that fits the buffer
         // it must exists as the buffer is at least 4 bytes length
@@ -141,8 +152,11 @@ function nextFlush (stream) {
       return
     }
     stream.flush(() => {
-      Atomics.store(stream[kImpl].state, READ_INDEX, 0)
-      Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      updateState(stream, () => {
+        Atomics.store(stream[kImpl].state, READ_INDEX, 0)
+        Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      })
+      Atomics.notify(stream[kImpl].state, READ_INDEX)
       nextFlush(stream)
     })
   } else {
@@ -160,6 +174,12 @@ function onWorkerMessage (msg) {
     return
   }
 
+  // Node.js watch mode may send internal worker messages that do not
+  // participate in thread-stream's worker protocol.
+  if (msg?.code == null) {
+    return
+  }
+
   switch (msg.code) {
     case 'READY':
       // Replace the FakeWeakRef with a
@@ -401,8 +421,9 @@ function write (stream, data, cb) {
   const current = Atomics.load(stream[kImpl].state, WRITE_INDEX)
   const length = Buffer.byteLength(data)
   stream[kImpl].data.write(data, current)
-  Atomics.store(stream[kImpl].state, WRITE_INDEX, current + length)
-  Atomics.notify(stream[kImpl].state, WRITE_INDEX)
+  updateState(stream, () => {
+    Atomics.store(stream[kImpl].state, WRITE_INDEX, current + length)
+  })
   cb()
   return true
 }
@@ -419,9 +440,10 @@ function end (stream) {
     let readIndex = Atomics.load(stream[kImpl].state, READ_INDEX)
 
     // process._rawDebug('writing index')
-    Atomics.store(stream[kImpl].state, WRITE_INDEX, -1)
+    updateState(stream, () => {
+      Atomics.store(stream[kImpl].state, WRITE_INDEX, -1)
+    })
     // process._rawDebug(`(end) readIndex (${Atomics.load(stream.state, READ_INDEX)}) writeIndex (${Atomics.load(stream.state, WRITE_INDEX)})`)
-    Atomics.notify(stream[kImpl].state, WRITE_INDEX)
 
     // Wait for the process to complete
     let spins = 0
@@ -466,8 +488,11 @@ function writeSync (stream) {
     let leftover = stream[kImpl].data.length - writeIndex
     if (leftover === 0) {
       flushSync(stream)
-      Atomics.store(stream[kImpl].state, READ_INDEX, 0)
-      Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      updateState(stream, () => {
+        Atomics.store(stream[kImpl].state, READ_INDEX, 0)
+        Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      })
+      Atomics.notify(stream[kImpl].state, READ_INDEX)
       continue
     } else if (leftover < 0) {
       // stream should never happen
@@ -483,8 +508,11 @@ function writeSync (stream) {
     } else {
       // multi-byte utf-8
       flushSync(stream)
-      Atomics.store(stream[kImpl].state, READ_INDEX, 0)
-      Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      updateState(stream, () => {
+        Atomics.store(stream[kImpl].state, READ_INDEX, 0)
+        Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      })
+      Atomics.notify(stream[kImpl].state, READ_INDEX)
 
       // Find a toWrite length that fits the buffer
       // it must exists as the buffer is at least 4 bytes length

package/lib/indexes.js

@@ -1,9 +1,11 @@
 'use strict'
 
+const SEQ_INDEX = 2
 const WRITE_INDEX = 4
 const READ_INDEX = 8
 
 module.exports = {
   WRITE_INDEX,
-  READ_INDEX
+  READ_INDEX,
+  SEQ_INDEX
 }

package/lib/worker.js

@@ -2,7 +2,7 @@
 
 const { realImport, realRequire } = require('real-require')
 const { workerData, parentPort } = require('worker_threads')
-const { WRITE_INDEX, READ_INDEX } = require('./indexes')
+const { WRITE_INDEX, READ_INDEX, SEQ_INDEX } = require('./indexes')
 const { waitDiff } = require('./wait')
 
 const {
@@ -48,7 +48,11 @@ async function start () {
       // When bundled with pkg, an undefined error is thrown when called with realImport
       // When bundled with pkg and using node v20, an ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING error is thrown when called with realImport
       // More info at: https://github.com/pinojs/thread-stream/issues/143
-      worker = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
+      try {
+        worker = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
+      } catch {
+        throw error
+      }
     } else {
       throw error
     }
@@ -97,18 +101,30 @@ start().then(function () {
   process.nextTick(run)
 })
 
+function readState () {
+  while (true) {
+    const seq = Atomics.load(state, SEQ_INDEX)
+
+    if ((seq & 1) !== 0) {
+      continue
+    }
+
+    const current = Atomics.load(state, READ_INDEX)
+    const end = Atomics.load(state, WRITE_INDEX)
+
+    if (seq === Atomics.load(state, SEQ_INDEX)) {
+      return { current, end, seq }
+    }
+  }
+}
+
 function run () {
-  const current = Atomics.load(state, READ_INDEX)
-  const end = Atomics.load(state, WRITE_INDEX)
+  const { current, end, seq } = readState()
 
   // process._rawDebug(`pre state ${current} ${end}`)
 
   if (end === current) {
-    if (end === data.length) {
-      waitDiff(state, READ_INDEX, end, Infinity, run)
-    } else {
-      waitDiff(state, WRITE_INDEX, end, Infinity, run)
-    }
+    waitDiff(state, SEQ_INDEX, seq, Infinity, run)
     return
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "thread-stream",
-  "version": "3.0.2",
+  "version": "3.2.0",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
   "types": "index.d.ts",
@@ -34,7 +34,8 @@
   },
   "standard": {
     "ignore": [
-      "test/ts/**/*"
+      "test/ts/**/*",
+      "test/syntax-error.mjs"
     ]
   },
   "repository": {

package/test/base.test.js

@@ -8,9 +8,19 @@ const ThreadStream = require('..')
 const { MessageChannel } = require('worker_threads')
 const { once } = require('events')
 
-test('base sync=true', function (t) {
-  t.plan(15)
+function readFileAsync (path) {
+  return new Promise((resolve, reject) => {
+    readFile(path, 'utf8', (err, data) => {
+      if (err) {
+        reject(err)
+        return
+      }
+      resolve(data)
+    })
+  })
+}
 
+test('base sync=true', async function (t) {
   const dest = file()
   const stream = new ThreadStream({
     filename: join(__dirname, 'to-file.js'),
@@ -18,24 +28,12 @@ test('base sync=true', function (t) {
     sync: true
   })
 
-  t.same(stream.writableObjectMode, false)
+  const finish = once(stream, 'finish')
+  const close = once(stream, 'close')
 
+  t.same(stream.writableObjectMode, false)
   t.same(stream.writableFinished, false)
-  stream.on('finish', () => {
-    t.same(stream.writableFinished, true)
-    readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data, 'hello world\nsomething else\n')
-    })
-  })
-
   t.same(stream.closed, false)
-  stream.on('close', () => {
-    t.same(stream.closed, true)
-    t.notOk(stream.writable)
-    t.pass('close emitted')
-  })
-
   t.same(stream.writableNeedDrain, false)
   t.ok(stream.write('hello world\n'))
   t.ok(stream.write('something else\n'))
@@ -44,11 +42,19 @@ test('base sync=true', function (t) {
   t.same(stream.writableEnded, false)
   stream.end()
   t.same(stream.writableEnded, true)
-})
 
-test('overflow sync=true', function (t) {
-  t.plan(3)
+  await finish
+  t.same(stream.writableFinished, true)
+
+  await close
+  t.same(stream.closed, true)
+  t.notOk(stream.writable)
+
+  const data = await readFileAsync(dest)
+  t.equal(data, 'hello world\nsomething else\n')
+})
 
+test('overflow sync=true', async function (t) {
   const dest = file()
   const stream = new ThreadStream({
     bufferSize: 128,
@@ -57,9 +63,9 @@ test('overflow sync=true', function (t) {
     sync: true
   })
 
+  const close = once(stream, 'close')
   let count = 0
 
-  // Write 10 chars, 20 times
   function write () {
     if (count++ === 20) {
       stream.end()
@@ -67,25 +73,17 @@ test('overflow sync=true', function (t) {
     }
 
     stream.write('aaaaaaaaaa')
-    // do not wait for drain event
     setImmediate(write)
   }
 
   write()
 
-  stream.on('finish', () => {
-    t.pass('finish emitted')
-  })
-
-  stream.on('close', () => {
-    readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data.length, 200)
-    })
-  })
+  await close
+  const data = await readFileAsync(dest)
+  t.equal(data.length, 200)
 })
 
-test('overflow sync=false', function (t) {
+test('overflow sync=false', async function (t) {
   const dest = file()
   const stream = new ThreadStream({
     bufferSize: 128,
@@ -94,14 +92,13 @@ test('overflow sync=false', function (t) {
     sync: false
   })
 
+  const close = once(stream, 'close')
   let count = 0
 
   t.same(stream.writableNeedDrain, false)
 
-  // Write 10 chars, 20 times
   function write () {
     if (count++ === 20) {
-      t.pass('end sent')
       stream.end()
       return
     }
@@ -109,7 +106,6 @@ test('overflow sync=false', function (t) {
     if (!stream.write('aaaaaaaaaa')) {
       t.same(stream.writableNeedDrain, true)
     }
-    // do not wait for drain event
     setImmediate(write)
   }
 
@@ -117,24 +113,15 @@ test('overflow sync=false', function (t) {
 
   stream.on('drain', () => {
     t.same(stream.writableNeedDrain, false)
-    t.pass('drain')
-  })
-
-  stream.on('finish', () => {
-    t.pass('finish emitted')
   })
 
-  stream.on('close', () => {
-    readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data.length, 200)
-      t.end()
-    })
-  })
+  await close
+  const data = await readFileAsync(dest)
+  t.equal(data.length, 200)
 })
 
 test('over the bufferSize at startup', function (t) {
-  t.plan(6)
+  t.plan(5)
 
   const dest = file()
   const stream = new ThreadStream({
@@ -151,10 +138,6 @@ test('over the bufferSize at startup', function (t) {
     })
   })
 
-  stream.on('close', () => {
-    t.pass('close emitted')
-  })
-
   t.ok(stream.write('hello'))
   t.ok(stream.write(' world\n'))
   t.ok(stream.write('something else\n'))
@@ -163,7 +146,7 @@ test('over the bufferSize at startup', function (t) {
 })
 
 test('over the bufferSize at startup (async)', function (t) {
-  t.plan(6)
+  t.plan(5)
 
   const dest = file()
   const stream = new ThreadStream({
@@ -185,13 +168,11 @@ test('over the bufferSize at startup (async)', function (t) {
       t.equal(data, 'hello world\nsomething else\n')
     })
   })
-
-  stream.on('close', () => {
-    t.pass('close emitted')
-  })
 })
 
 test('flushSync sync=false', function (t) {
+  t.plan(2)
+
   const dest = file()
   const stream = new ThreadStream({
     bufferSize: 128,
@@ -200,32 +181,26 @@ test('flushSync sync=false', function (t) {
     sync: false
   })
 
-  stream.on('drain', () => {
-    t.pass('drain')
-    stream.end()
-  })
+  stream.on('ready', () => {
+    for (let count = 0; count < 20; count++) {
+      stream.write('aaaaaaaaaa')
+    }
 
-  stream.on('finish', () => {
-    t.pass('finish emitted')
+    stream.flushSync()
+    setImmediate(() => {
+      stream.end()
+    })
   })
 
-  stream.on('close', () => {
+  stream.on('finish', () => {
     readFile(dest, 'utf8', (err, data) => {
       t.error(err)
       t.equal(data.length, 200)
-      t.end()
     })
   })
-
-  for (let count = 0; count < 20; count++) {
-    stream.write('aaaaaaaaaa')
-  }
-  stream.flushSync()
 })
 
 test('pass down MessagePorts', async function (t) {
-  t.plan(3)
-
   const { port1, port2 } = new MessageChannel()
   const stream = new ThreadStream({
     filename: join(__dirname, 'port.js'),
@@ -235,6 +210,7 @@ test('pass down MessagePorts', async function (t) {
     },
     sync: false
   })
+
   t.teardown(() => {
     stream.end()
   })
@@ -243,13 +219,10 @@ test('pass down MessagePorts', async function (t) {
   t.ok(stream.write('something else\n'))
 
   const [strings] = await once(port2, 'message')
-
   t.equal(strings, 'hello world\nsomething else\n')
 })
 
-test('destroy does not error', function (t) {
-  t.plan(5)
-
+test('destroy does not error', async function (t) {
   const dest = file()
   const stream = new ThreadStream({
     filename: join(__dirname, 'to-file.js'),
@@ -258,16 +231,28 @@ test('destroy does not error', function (t) {
   })
 
   stream.on('ready', () => {
-    t.pass('ready emitted')
     stream.worker.terminate()
   })
 
-  stream.on('error', (err) => {
-    t.equal(err.message, 'the worker thread exited')
+  const [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker thread exited')
+
+  await new Promise((resolve) => {
     stream.flush((err) => {
       t.equal(err.message, 'the worker has exited')
+      resolve()
     })
-    t.doesNotThrow(() => stream.flushSync())
-    t.doesNotThrow(() => stream.end())
   })
+
+  t.doesNotThrow(() => stream.flushSync())
+  t.doesNotThrow(() => stream.end())
+})
+
+test('syntax error', async function (t) {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'syntax-error.mjs')
+  })
+
+  const [err] = await once(stream, 'error')
+  t.equal(err.message, 'Unexpected end of input')
 })

package/test/message-without-code.js

@@ -0,0 +1,19 @@
+'use strict'
+
+const { Writable } = require('stream')
+const { parentPort } = require('worker_threads')
+
+async function run () {
+  parentPort.postMessage({
+    internal: 'watch-mode'
+  })
+
+  return new Writable({
+    autoDestroy: true,
+    write (chunk, enc, cb) {
+      cb()
+    }
+  })
+}
+
+module.exports = run

package/test/syntax-error.mjs

@@ -0,0 +1,2 @@
+// this is a syntax error
+import

package/test/watch-mode.test.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const { test } = require('tap')
+const { join } = require('path')
+const ThreadStream = require('..')
+
+test('ignores worker messages without a protocol code', function (t) {
+  t.plan(2)
+
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'message-without-code.js'),
+    sync: false
+  })
+
+  const errors = []
+  stream.on('error', err => {
+    errors.push(err)
+  })
+
+  stream.on('ready', () => {
+    t.ok(stream.write('hello world\n'))
+    stream.end()
+  })
+
+  stream.on('finish', () => {
+    t.same(errors, [])
+  })
+})