thread-stream 3.1.0 → 4.0.0

package/.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
       contents: read
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
 
@@ -38,15 +38,12 @@ jobs:
       contents: read
     strategy:
       matrix:
-        node-version: [18, 20, 22]
+        node-version: [20, 22, 24]
         os: [macos-latest, ubuntu-latest, windows-latest]
-        exclude:
-          - os: windows-latest
-            node-version: 22
 
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
 
@@ -61,23 +58,6 @@ jobs:
       - name: Run tests
         run: npm run test:ci
 
-      - name: Coveralls Parallel
-        uses: coverallsapp/github-action@v2.3.0
-        with:
-          github-token: ${{ secrets.github_token }}
-          parallel: true
-          flag-name: run-${{ matrix.node-version }}-${{ matrix.os }}
-
-  coverage:
-    needs: test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Coveralls Finished
-        uses: coverallsapp/github-action@v2.3.0
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          parallel-finished: true
-
   automerge:
     name: Automerge Dependabot PRs
     if: >

package/CLAUDE.md

@@ -0,0 +1,53 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Build & Test Commands
+
+```sh
+npm test                    # Run linter (standard), type check, transpile, and all tests
+npm run build               # Type check only (tsc --noEmit)
+npm run transpile           # Generate transpiled test files for various ES targets
+
+# Run a single test file
+npx tap test/base.test.js
+
+# Run only JavaScript tests (faster)
+npx tap "test/**/*.test.*js"
+
+# Run only TypeScript tests
+npx tap --ts test/*.test.*ts
+```
+
+## Code Style
+
+This project uses [Standard](https://standardjs.com/) for linting. No semicolons, 2-space indentation.
+
+## Architecture
+
+thread-stream is a library for streaming data to a Node.js Worker Thread using SharedArrayBuffer for high-performance inter-thread communication.
+
+### Core Components
+
+- **index.js** - Main `ThreadStream` class extending EventEmitter. Manages the worker lifecycle, handles writes via SharedArrayBuffer, and coordinates synchronization using Atomics.
+
+- **lib/worker.js** - Runs in the Worker Thread. Loads the user-provided destination module, reads data from SharedArrayBuffer, and writes to the destination stream. Handles both ESM and CommonJS modules, including yarn PnP compatibility.
+
+- **lib/indexes.js** - Defines SharedArrayBuffer index constants (`WRITE_INDEX`, `READ_INDEX`) used for Atomics coordination.
+
+- **lib/wait.js** - Async polling utilities (`wait`, `waitDiff`) for cross-thread synchronization without blocking the main thread.
+
+### Data Flow
+
+1. Main thread writes strings to an internal buffer, then copies to SharedArrayBuffer
+2. Atomics.notify signals the worker that data is available
+3. Worker reads from SharedArrayBuffer via Atomics.load and writes to destination stream
+4. Worker updates READ_INDEX and notifies main thread when done
+5. Special index values (-1 for end, -2 for error) signal stream termination
+
+### Key Features
+
+- **Sync/async modes** - `sync: true` blocks until data is written; async mode uses `setImmediate` batching
+- **Backpressure** - Handles `drain` events from destination streams
+- **GC cleanup** - Uses FinalizationRegistry to terminate orphaned workers
+- **TypeScript support** - Workers can be `.ts` files (requires ts-node)

package/eslint.config.js

@@ -0,0 +1,10 @@
+'use strict'
+
+const neostandard = require('neostandard')
+
+module.exports = neostandard({
+  ignores: [
+    'test/ts/**/*',
+    'test/syntax-error.mjs'
+  ]
+})

package/index.js

@@ -122,6 +122,7 @@ function nextFlush (stream) {
 
         Atomics.store(stream[kImpl].state, READ_INDEX, 0)
         Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+        Atomics.notify(stream[kImpl].state, READ_INDEX)
 
         // Find a toWrite length that fits the buffer
         // it must exists as the buffer is at least 4 bytes length
@@ -143,6 +144,7 @@ function nextFlush (stream) {
     stream.flush(() => {
       Atomics.store(stream[kImpl].state, READ_INDEX, 0)
       Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      Atomics.notify(stream[kImpl].state, READ_INDEX)
       nextFlush(stream)
     })
   } else {
@@ -468,6 +470,7 @@ function writeSync (stream) {
       flushSync(stream)
       Atomics.store(stream[kImpl].state, READ_INDEX, 0)
       Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      Atomics.notify(stream[kImpl].state, READ_INDEX)
       continue
     } else if (leftover < 0) {
       // stream should never happen
@@ -485,6 +488,7 @@ function writeSync (stream) {
       flushSync(stream)
       Atomics.store(stream[kImpl].state, READ_INDEX, 0)
       Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      Atomics.notify(stream[kImpl].state, READ_INDEX)
 
       // Find a toWrite length that fits the buffer
       // it must exists as the buffer is at least 4 bytes length

package/lib/wait.js

@@ -1,61 +1,68 @@
 'use strict'
 
-const MAX_TIMEOUT = 1000
+// Maximum wait time for a single waitAsync call
+// Used as a fallback poll interval in case notifications are missed
+// Keep this low enough for good throughput but high enough to not busy-loop
+const WAIT_MS = 10000
 
 function wait (state, index, expected, timeout, done) {
-  const max = Date.now() + timeout
-  let current = Atomics.load(state, index)
-  if (current === expected) {
-    done(null, 'ok')
-    return
-  }
-  let prior = current
-  const check = (backoff) => {
-    if (Date.now() > max) {
+  const max = timeout === Infinity ? Infinity : Date.now() + timeout
+
+  const check = () => {
+    const current = Atomics.load(state, index)
+    if (current === expected) {
+      done(null, 'ok')
+      return
+    }
+
+    if (max !== Infinity && Date.now() > max) {
       done(null, 'timed-out')
+      return
+    }
+
+    // Wait for any change from current value
+    const remaining = max === Infinity ? WAIT_MS : Math.min(WAIT_MS, Math.max(1, max - Date.now()))
+    const result = Atomics.waitAsync(state, index, current, remaining)
+
+    if (result.async) {
+      result.value.then(check)
     } else {
-      setTimeout(() => {
-        prior = current
-        current = Atomics.load(state, index)
-        if (current === prior) {
-          check(backoff >= MAX_TIMEOUT ? MAX_TIMEOUT : backoff * 2)
-        } else {
-          if (current === expected) done(null, 'ok')
-          else done(null, 'not-equal')
-        }
-      }, backoff)
+      // Value already changed (not-equal) - recheck on next tick
+      setImmediate(check)
     }
   }
-  check(1)
+
+  check()
 }
 
-// let waitDiffCount = 0
 function waitDiff (state, index, expected, timeout, done) {
-  // const id = waitDiffCount++
-  // process._rawDebug(`>>> waitDiff ${id}`)
-  const max = Date.now() + timeout
-  let current = Atomics.load(state, index)
-  if (current !== expected) {
-    done(null, 'ok')
-    return
-  }
-  const check = (backoff) => {
-    // process._rawDebug(`${id} ${index} current ${current} expected ${expected}`)
-    // process._rawDebug('' + backoff)
-    if (Date.now() > max) {
+  const max = timeout === Infinity ? Infinity : Date.now() + timeout
+
+  const check = () => {
+    const current = Atomics.load(state, index)
+    if (current !== expected) {
+      done(null, 'ok')
+      return
+    }
+
+    if (max !== Infinity && Date.now() > max) {
       done(null, 'timed-out')
+      return
+    }
+
+    // Wait for value to change from expected
+    const remaining = max === Infinity ? WAIT_MS : Math.min(WAIT_MS, Math.max(1, max - Date.now()))
+    const result = Atomics.waitAsync(state, index, expected, remaining)
+
+    if (result.async) {
+      result.value.then(check)
     } else {
-      setTimeout(() => {
-        current = Atomics.load(state, index)
-        if (current !== expected) {
-          done(null, 'ok')
-        } else {
-          check(backoff >= MAX_TIMEOUT ? MAX_TIMEOUT : backoff * 2)
-        }
-      }, backoff)
+      // Value already changed (not-equal) - recheck on next tick
+      setImmediate(check)
     }
   }
-  check(1)
+
+  check()
 }
 
 module.exports = { wait, waitDiff }

package/lib/worker.js

@@ -16,22 +16,13 @@ let destination
 const state = new Int32Array(stateBuf)
 const data = Buffer.from(dataBuf)
 
+// Keep the event loop alive - Atomics.waitAsync promises don't prevent worker exit
+const keepAlive = setInterval(() => {}, 60 * 60 * 1000)
+
 async function start () {
   let worker
   try {
-    if (filename.endsWith('.ts') || filename.endsWith('.cts')) {
-      // TODO: add support for the TSM modules loader ( https://github.com/lukeed/tsm ).
-      if (!process[Symbol.for('ts-node.register.instance')]) {
-        realRequire('ts-node/register')
-      } else if (process.env.TS_NODE_DEV) {
-        realRequire('ts-node-dev')
-      }
-      // TODO: Support ES imports once tsc, tap & ts-node provide better compatibility guarantees.
-      // Remove extra forwardslash on Windows
-      worker = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
-    } else {
-      worker = (await realImport(filename))
-    }
+    worker = (await realImport(filename))
   } catch (error) {
     // A yarn user that tries to start a ThreadStream for an external module
     // provides a filename pointing to a zip file.
@@ -53,6 +44,19 @@ async function start () {
       } catch {
         throw error
       }
+    } else if (filename.endsWith('.ts') || filename.endsWith('.cts')) {
+      // Native TypeScript import failed (type stripping not enabled).
+      // Fall back to ts-node for TypeScript files.
+      try {
+        if (!process[Symbol.for('ts-node.register.instance')]) {
+          realRequire('ts-node/register')
+        } else if (process.env.TS_NODE_DEV) {
+          realRequire('ts-node-dev')
+        }
+        worker = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
+      } catch {
+        throw error
+      }
     } else {
       throw error
     }
@@ -84,6 +88,7 @@ async function start () {
     const end = Atomics.load(state, WRITE_INDEX)
     Atomics.store(state, READ_INDEX, end)
     Atomics.notify(state, READ_INDEX)
+    clearInterval(keepAlive)
     setImmediate(() => {
       process.exit(0)
     })

package/package.json

@@ -1,43 +1,38 @@
 {
   "name": "thread-stream",
-  "version": "3.1.0",
+  "version": "4.0.0",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
   "types": "index.d.ts",
+  "engines": {
+    "node": ">=20"
+  },
   "dependencies": {
     "real-require": "^0.2.0"
   },
   "devDependencies": {
-    "@types/node": "^20.1.0",
-    "@types/tap": "^15.0.0",
-    "@yao-pkg/pkg": "^5.11.5",
+    "@types/node": "^22.0.0",
+    "@yao-pkg/pkg": "^6.0.0",
+    "borp": "^0.21.0",
     "desm": "^1.3.0",
+    "eslint": "^9.39.1",
     "fastbench": "^1.0.1",
     "husky": "^9.0.6",
+    "neostandard": "^0.12.2",
     "pino-elasticsearch": "^8.0.0",
     "sonic-boom": "^4.0.1",
-    "standard": "^17.0.0",
-    "tap": "^16.2.0",
     "ts-node": "^10.8.0",
-    "typescript": "^5.3.2",
-    "why-is-node-running": "^2.2.2"
+    "typescript": "~5.7.3"
   },
   "scripts": {
     "build": "tsc --noEmit",
-    "test": "standard && npm run build && npm run transpile && tap \"test/**/*.test.*js\" && tap --ts test/*.test.*ts",
-    "test:ci": "standard && npm run transpile && npm run test:ci:js && npm run test:ci:ts",
-    "test:ci:js": "tap --no-check-coverage --timeout=120 --coverage-report=lcovonly \"test/**/*.test.*js\"",
-    "test:ci:ts": "tap --ts --no-check-coverage --coverage-report=lcovonly \"test/**/*.test.*ts\"",
-    "test:yarn": "npm run transpile && tap \"test/**/*.test.js\" --no-check-coverage",
+    "lint": "eslint",
+    "test": "npm run lint && npm run build && npm run transpile && borp --pattern 'test/*.test.{js,mjs}'",
+    "test:ci": "npm run lint && npm run transpile && borp --pattern 'test/*.test.{js,mjs}'",
+    "test:yarn": "npm run transpile && borp --pattern 'test/*.test.js'",
     "transpile": "sh ./test/ts/transpile.sh",
     "prepare": "husky install"
   },
-  "standard": {
-    "ignore": [
-      "test/ts/**/*",
-      "test/syntax-error.mjs"
-    ]
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/mcollina/thread-stream.git"

package/test/base.test.js

@@ -1,6 +1,7 @@
 'use strict'
 
-const { test } = require('tap')
+const { test } = require('node:test')
+const assert = require('node:assert')
 const { join } = require('path')
 const { readFile } = require('fs')
 const { file } = require('./helper')
@@ -8,9 +9,7 @@ const ThreadStream = require('..')
 const { MessageChannel } = require('worker_threads')
 const { once } = require('events')
 
-test('base sync=true', function (t) {
-  t.plan(15)
-
+test('base sync=true', function (t, done) {
   const dest = file()
   const stream = new ThreadStream({
     filename: join(__dirname, 'to-file.js'),
@@ -18,37 +17,35 @@ test('base sync=true', function (t) {
     sync: true
   })
 
-  t.same(stream.writableObjectMode, false)
+  assert.deepStrictEqual(stream.writableObjectMode, false)
 
-  t.same(stream.writableFinished, false)
+  assert.deepStrictEqual(stream.writableFinished, false)
   stream.on('finish', () => {
-    t.same(stream.writableFinished, true)
+    assert.deepStrictEqual(stream.writableFinished, true)
     readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data, 'hello world\nsomething else\n')
+      assert.ifError(err)
+      assert.strictEqual(data, 'hello world\nsomething else\n')
     })
   })
 
-  t.same(stream.closed, false)
+  assert.deepStrictEqual(stream.closed, false)
   stream.on('close', () => {
-    t.same(stream.closed, true)
-    t.notOk(stream.writable)
-    t.pass('close emitted')
+    assert.deepStrictEqual(stream.closed, true)
+    assert.ok(!stream.writable)
+    done()
   })
 
-  t.same(stream.writableNeedDrain, false)
-  t.ok(stream.write('hello world\n'))
-  t.ok(stream.write('something else\n'))
-  t.ok(stream.writable)
+  assert.deepStrictEqual(stream.writableNeedDrain, false)
+  assert.ok(stream.write('hello world\n'))
+  assert.ok(stream.write('something else\n'))
+  assert.ok(stream.writable)
 
-  t.same(stream.writableEnded, false)
+  assert.deepStrictEqual(stream.writableEnded, false)
   stream.end()
-  t.same(stream.writableEnded, true)
+  assert.deepStrictEqual(stream.writableEnded, true)
 })
 
-test('overflow sync=true', function (t) {
-  t.plan(3)
-
+test('overflow sync=true', function (t, done) {
   const dest = file()
   const stream = new ThreadStream({
     bufferSize: 128,
@@ -73,19 +70,16 @@ test('overflow sync=true', function (t) {
 
   write()
 
-  stream.on('finish', () => {
-    t.pass('finish emitted')
-  })
-
   stream.on('close', () => {
     readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data.length, 200)
+      assert.ifError(err)
+      assert.strictEqual(data.length, 200)
+      done()
     })
   })
 })
 
-test('overflow sync=false', function (t) {
+test('overflow sync=false', function (t, done) {
   const dest = file()
   const stream = new ThreadStream({
     bufferSize: 128,
@@ -96,18 +90,17 @@ test('overflow sync=false', function (t) {
 
   let count = 0
 
-  t.same(stream.writableNeedDrain, false)
+  assert.deepStrictEqual(stream.writableNeedDrain, false)
 
   // Write 10 chars, 20 times
   function write () {
     if (count++ === 20) {
-      t.pass('end sent')
       stream.end()
       return
     }
 
     if (!stream.write('aaaaaaaaaa')) {
-      t.same(stream.writableNeedDrain, true)
+      assert.deepStrictEqual(stream.writableNeedDrain, true)
     }
     // do not wait for drain event
     setImmediate(write)
@@ -116,26 +109,19 @@ test('overflow sync=false', function (t) {
   write()
 
   stream.on('drain', () => {
-    t.same(stream.writableNeedDrain, false)
-    t.pass('drain')
-  })
-
-  stream.on('finish', () => {
-    t.pass('finish emitted')
+    assert.deepStrictEqual(stream.writableNeedDrain, false)
   })
 
   stream.on('close', () => {
     readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data.length, 200)
-      t.end()
+      assert.ifError(err)
+      assert.strictEqual(data.length, 200)
+      done()
     })
   })
 })
 
-test('over the bufferSize at startup', function (t) {
-  t.plan(6)
-
+test('over the bufferSize at startup', function (t, done) {
   const dest = file()
   const stream = new ThreadStream({
     bufferSize: 10,
@@ -146,25 +132,23 @@ test('over the bufferSize at startup', function (t) {
 
   stream.on('finish', () => {
     readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data, 'hello world\nsomething else\n')
+      assert.ifError(err)
+      assert.strictEqual(data, 'hello world\nsomething else\n')
     })
   })
 
   stream.on('close', () => {
-    t.pass('close emitted')
+    done()
   })
 
-  t.ok(stream.write('hello'))
-  t.ok(stream.write(' world\n'))
-  t.ok(stream.write('something else\n'))
+  assert.ok(stream.write('hello'))
+  assert.ok(stream.write(' world\n'))
+  assert.ok(stream.write('something else\n'))
 
   stream.end()
 })
 
-test('over the bufferSize at startup (async)', function (t) {
-  t.plan(6)
-
+test('over the bufferSize at startup (async)', function (t, done) {
   const dest = file()
   const stream = new ThreadStream({
     bufferSize: 10,
@@ -173,25 +157,25 @@ test('over the bufferSize at startup (async)', function (t) {
     sync: false
   })
 
-  t.ok(stream.write('hello'))
-  t.notOk(stream.write(' world\n'))
-  t.notOk(stream.write('something else\n'))
+  assert.ok(stream.write('hello'))
+  assert.ok(!stream.write(' world\n'))
+  assert.ok(!stream.write('something else\n'))
 
   stream.end()
 
   stream.on('finish', () => {
     readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data, 'hello world\nsomething else\n')
+      assert.ifError(err)
+      assert.strictEqual(data, 'hello world\nsomething else\n')
     })
   })
 
   stream.on('close', () => {
-    t.pass('close emitted')
+    done()
   })
 })
 
-test('flushSync sync=false', function (t) {
+test('flushSync sync=false', function (t, done) {
   const dest = file()
   const stream = new ThreadStream({
     bufferSize: 128,
@@ -201,19 +185,14 @@ test('flushSync sync=false', function (t) {
   })
 
   stream.on('drain', () => {
-    t.pass('drain')
     stream.end()
   })
 
-  stream.on('finish', () => {
-    t.pass('finish emitted')
-  })
-
   stream.on('close', () => {
     readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data.length, 200)
-      t.end()
+      assert.ifError(err)
+      assert.strictEqual(data.length, 200)
+      done()
     })
   })
 
@@ -224,8 +203,6 @@ test('flushSync sync=false', function (t) {
 })
 
 test('pass down MessagePorts', async function (t) {
-  t.plan(3)
-
   const { port1, port2 } = new MessageChannel()
   const stream = new ThreadStream({
     filename: join(__dirname, 'port.js'),
@@ -235,21 +212,19 @@ test('pass down MessagePorts', async function (t) {
     },
     sync: false
   })
-  t.teardown(() => {
+  t.after(() => {
     stream.end()
   })
 
-  t.ok(stream.write('hello world\n'))
-  t.ok(stream.write('something else\n'))
+  assert.ok(stream.write('hello world\n'))
+  assert.ok(stream.write('something else\n'))
 
   const [strings] = await once(port2, 'message')
 
-  t.equal(strings, 'hello world\nsomething else\n')
+  assert.strictEqual(strings, 'hello world\nsomething else\n')
 })
 
-test('destroy does not error', function (t) {
-  t.plan(5)
-
+test('destroy does not error', function (t, done) {
   const dest = file()
   const stream = new ThreadStream({
     filename: join(__dirname, 'to-file.js'),
@@ -258,28 +233,27 @@ test('destroy does not error', function (t) {
   })
 
   stream.on('ready', () => {
-    t.pass('ready emitted')
     stream.worker.terminate()
   })
 
   stream.on('error', (err) => {
-    t.equal(err.message, 'the worker thread exited')
+    assert.strictEqual(err.message, 'the worker thread exited')
     stream.flush((err) => {
-      t.equal(err.message, 'the worker has exited')
+      assert.strictEqual(err.message, 'the worker has exited')
     })
-    t.doesNotThrow(() => stream.flushSync())
-    t.doesNotThrow(() => stream.end())
+    assert.doesNotThrow(() => stream.flushSync())
+    assert.doesNotThrow(() => stream.end())
+    done()
   })
 })
 
-test('syntax error', function (t) {
-  t.plan(1)
-
+test('syntax error', function (t, done) {
   const stream = new ThreadStream({
     filename: join(__dirname, 'syntax-error.mjs')
   })
 
   stream.on('error', (err) => {
-    t.equal(err.message, 'Unexpected end of input')
+    assert.strictEqual(err.message, 'Unexpected end of input')
+    done()
   })
 })