thread-stream 1.0.1 → 2.1.0

package/.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
           persist-credentials: false
 
       - name: Dependency review
-        uses: actions/dependency-review-action@v1
+        uses: actions/dependency-review-action@v2
 
   test:
     name: Test

package/.github/workflows/package-manager-ci.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macOS-latest, windows-latest]
+        os: [ubuntu-latest]
         node-version: [14, 16, 18]
     steps:
       - uses: actions/checkout@v3
@@ -37,7 +37,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macOS-latest]
+        os: [ubuntu-latest]
         node-version: [14, 16, 18]
     steps:
       - uses: actions/checkout@v3

package/README.md

@@ -84,6 +84,31 @@ stream.end()
 
 This module works with `yarn` in PnP (plug'n play) mode too!
 
+### Emit events
+
+You can emit events on the ThreadStream from your worker using [`worker.parentPort.postMessage()`](https://nodejs.org/api/worker_threads.html#workerparentport).
+The message (JSON object) must have the following data structure:
+
+```js
+parentPort.postMessage({
+  code: 'EVENT',
+  name: 'eventName',
+  args: ['list', 'of', 'args', 123, new Error('Boom')]
+})
+```
+
+On your ThreadStream, you can add a listener function for this event name:
+
+```js
+const stream = new ThreadStream({
+  filename: join(__dirname, 'worker.js'),
+  workerData: {},
+})
+stream.on('eventName', function (a, b, c, n, err) {
+  console.log('received:', a, b, c, n, err) // received: list of args 123 Error: Boom
+})
+```
+
 ## License
 
 MIT

package/index.d.ts

@@ -1,9 +1,73 @@
 import { EventEmitter } from 'events'
+import workerThreads from 'worker_threads'
+
+interface ThreadStreamOptions {
+  /**
+   * The size (in bytes) of the buffer.
+   * Must be greater than 4 (i.e. it must at least fit a 4-byte utf-8 char).
+   * 
+   * Default: `4 * 1024 * 1024` = `4194304`
+   */
+  bufferSize?: number,
+  /**
+   * The path to the Worker's main script or module.
+   * Must be either an absolute path or a relative path (i.e. relative to the current working directory) starting with ./ or ../, or a WHATWG URL object using file: or data: protocol.
+   * When using a data: URL, the data is interpreted based on MIME type using the ECMAScript module loader.
+   * 
+   * {@link workerThreads.Worker()}
+   */
+  filename: string | URL,
+  /**
+   * If `true`, write data synchronously; otherwise write data asynchronously.
+   * 
+   * Default: `false`.
+   */
+  sync?: boolean,
+  /**
+   * {@link workerThreads.WorkerOptions.workerData}
+   * 
+   * Default: `{}`
+   */
+  workerData?: any,
+  /**
+   * {@link workerThreads.WorkerOptions}
+   * 
+   * Default: `{}`
+   */
+  workerOpts?: workerThreads.WorkerOptions
+}
+
 
 declare class ThreadStream extends EventEmitter {
-  constructor(opts: {})
-  write (data: string): boolean
-  end (): void
+  /**
+   * @param {ThreadStreamOptions} opts 
+   */
+  constructor(opts: ThreadStreamOptions)
+  /**
+   * Write some data to the stream.
+   * 
+   * **Please note that this method should not throw an {Error} if something goes wrong but emit an error event.**
+   * @param {string} data data to write.
+   * @returns {boolean} false if the stream wishes for the calling code to wait for the 'drain' event to be emitted before continuing to write additional data or if it fails to write; otherwise true.
+   */
+  write(data: string): boolean
+  /**
+   * Signal that no more data will be written.
+   * 
+   * **Please note that this method should not throw an {Error} if something goes wrong but emit an error event.**
+   * 
+   * Calling the {@link write()} method after calling {@link end()} will emit an error.
+   */
+  end(): void
+  /**
+   * Flush the stream synchronously.
+   * This method should be called in the shutdown phase to make sure that all data has been flushed.
+   * 
+   * **Please note that this method will throw an {Error} if something goes wrong.**
+   * 
+   * @throws {Error} if the stream is already flushing, if it fails to flush or if it takes more than 10 seconds to flush.
+   */
+  flushSync(): void
 }
 
 export = ThreadStream;

package/index.js

@@ -136,7 +136,7 @@ function nextFlush (stream) {
     })
   } else {
     // This should never happen
-    throw new Error('overwritten')
+    destroy(stream, new Error('overwritten'))
   }
 }
 
@@ -163,8 +163,15 @@ function onWorkerMessage (msg) {
     case 'ERROR':
       destroy(stream, msg.err)
       break
+    case 'EVENT':
+      if (Array.isArray(msg.args)) {
+        stream.emit(msg.name, ...msg.args)
+      } else {
+        stream.emit(msg.name, msg.args)
+      }
+      break
     default:
-      throw new Error('this should not happen: ' + msg.code)
+      destroy(stream, new Error('this should not happen: ' + msg.code))
   }
 }
 
@@ -177,7 +184,7 @@ function onWorkerExit (code) {
   registry.unregister(stream)
   stream.worker.exited = true
   stream.worker.off('exit', onWorkerExit)
-  destroy(stream, code !== 0 ? new Error('The worker thread exited') : null)
+  destroy(stream, code !== 0 ? new Error('the worker thread exited') : null)
 }
 
 class ThreadStream extends EventEmitter {
@@ -211,11 +218,13 @@ class ThreadStream extends EventEmitter {
 
   write (data) {
     if (this[kImpl].destroyed) {
-      throw new Error('the worker has exited')
+      error(this, new Error('the worker has exited'))
+      return false
     }
 
     if (this[kImpl].ending) {
-      throw new Error('the worker is ending')
+      error(this, new Error('the worker is ending'))
+      return false
     }
 
     if (this[kImpl].flushing && this[kImpl].buf.length + data.length >= MAX_STRING) {
@@ -338,6 +347,12 @@ class ThreadStream extends EventEmitter {
   }
 }
 
+function error (stream, err) {
+  setImmediate(() => {
+    stream.emit('error', err)
+  })
+}
+
 function destroy (stream, err) {
   if (stream[kImpl].destroyed) {
     return
@@ -346,7 +361,7 @@ function destroy (stream, err) {
 
   if (err) {
     stream[kImpl].errored = err
-    stream.emit('error', err)
+    error(stream, err)
   }
 
   if (!stream.worker.exited) {
@@ -399,11 +414,13 @@ function end (stream) {
       readIndex = Atomics.load(stream[kImpl].state, READ_INDEX)
 
       if (readIndex === -2) {
-        throw new Error('end() failed')
+        destroy(stream, new Error('end() failed'))
+        return
       }
 
       if (++spins === 10) {
-        throw new Error('end() took too long (10s)')
+        destroy(stream, new Error('end() took too long (10s)'))
+        return
       }
     }
 
@@ -482,7 +499,7 @@ function flushSync (stream) {
     const readIndex = Atomics.load(stream[kImpl].state, READ_INDEX)
 
     if (readIndex === -2) {
-      throw new Error('_flushSync failed')
+      throw Error('_flushSync failed')
     }
 
     // process._rawDebug(`(flushSync) readIndex (${readIndex}) writeIndex (${writeIndex})`)

package/lib/worker.js

@@ -44,6 +44,9 @@ async function start () {
     if ((error.code === 'ENOTDIR' || error.code === 'ERR_MODULE_NOT_FOUND') &&
      filename.startsWith('file://')) {
       fn = realRequire(decodeURIComponent(filename.replace('file://', '')))
+    } else if (error.code === undefined) {
+      // When bundled with pkg, an undefined error is thrown when called with realImport
+      fn = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
     } else {
       throw error
     }

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "thread-stream",
-  "version": "1.0.1",
+  "version": "2.1.0",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
   "types": "index.d.ts",
   "dependencies": {
-    "real-require": "^0.1.0"
+    "real-require": "^0.2.0"
   },
   "devDependencies": {
     "@types/node": "^18.0.0",

package/test/base.test.js

@@ -263,7 +263,7 @@ test('destroy does not error', function (t) {
   })
 
   stream.on('error', (err) => {
-    t.equal(err.message, 'The worker thread exited')
+    t.equal(err.message, 'the worker thread exited')
     stream.flush((err) => {
       t.equal(err.message, 'the worker has exited')
     })

package/test/bundlers.test.js

@@ -5,7 +5,7 @@ const { join } = require('path')
 const { file } = require('./helper')
 const ThreadStream = require('..')
 
-test('bundlers support', function (t) {
+test('bundlers support with .js file', function (t) {
   t.plan(1)
 
   globalThis.__bundlerPathsOverrides = {
@@ -31,3 +31,30 @@ test('bundlers support', function (t) {
 
   stream.end()
 })
+
+test('bundlers support with .mjs file', function (t) {
+  t.plan(1)
+
+  globalThis.__bundlerPathsOverrides = {
+    'thread-stream-worker': join(__dirname, 'custom-worker.js')
+  }
+
+  const dest = file()
+
+  process.on('uncaughtException', error => {
+    console.log(error)
+  })
+
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.mjs'),
+    workerData: { dest },
+    sync: true
+  })
+
+  stream.worker.removeAllListeners('message')
+  stream.worker.once('message', message => {
+    t.equal(message.code, 'CUSTOM-WORKER-CALLED')
+  })
+
+  stream.end()
+})

package/test/emit-event.js

@@ -0,0 +1,22 @@
+'use strict'
+
+const { Writable } = require('stream')
+const parentPort = require('worker_threads').parentPort
+
+async function run () {
+  return new Writable({
+    autoDestroy: true,
+    write (chunk, enc, cb) {
+      if (parentPort) {
+        parentPort.postMessage({
+          code: 'EVENT',
+          name: 'socketError',
+          args: ['list', 'of', 'args', 123, new Error('unable to write data to the TCP socket')]
+        })
+      }
+      cb()
+    }
+  })
+}
+
+module.exports = run

package/test/event.test.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const { test } = require('tap')
+const { join } = require('path')
+const ThreadStream = require('..')
+
+test('event propagate', function (t) {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'emit-event.js'),
+    workerData: {},
+    sync: true
+  })
+  stream.on('socketError', function (a, b, c, n, error) {
+    t.same(a, 'list')
+    t.same(b, 'of')
+    t.same(c, 'args')
+    t.same(n, 123)
+    t.same(error, new Error('unable to write data to the TCP socket'))
+    t.end()
+  })
+  stream.write('hello')
+  stream.end()
+})

package/test/string-limit-2.test.js

@@ -1,6 +1,12 @@
 'use strict'
 
 const t = require('tap')
+
+if (process.env.CI) {
+  t.skip('skip on CI')
+  process.exit(0)
+}
+
 const { join } = require('path')
 const { file } = require('./helper')
 const { createReadStream } = require('fs')

package/test/string-limit.test.js

@@ -1,6 +1,12 @@
 'use strict'
 
 const t = require('tap')
+
+if (process.env.CI) {
+  t.skip('skip on CI')
+  process.exit(0)
+}
+
 const { join } = require('path')
 const { file } = require('./helper')
 const { stat } = require('fs')

package/test/thread-management.test.js

@@ -29,50 +29,16 @@ test('emit error if thread exits', async function (t) {
     stream.write('hello world\n')
   })
 
-  const [err] = await once(stream, 'error')
-  t.equal(err.message, 'The worker thread exited')
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-})
+  let [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker thread exited')
 
-test('emit error if thread exits', async function (t) {
-  const stream = new ThreadStream({
-    filename: join(__dirname, 'exit.js'),
-    sync: true
-  })
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 
-  stream.on('ready', () => {
-    stream.write('hello world\n')
-  })
-
-  const [err] = await once(stream, 'error')
-  t.equal(err.message, 'The worker thread exited')
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 })
 
 test('emit error if thread have unhandledRejection', async function (t) {
@@ -85,22 +51,16 @@ test('emit error if thread have unhandledRejection', async function (t) {
     stream.write('hello world\n')
   })
 
-  const [err] = await once(stream, 'error')
+  let [err] = await once(stream, 'error')
   t.equal(err.message, 'kaboom')
 
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
+
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 })
 
 test('emit error if worker stream emit error', async function (t) {
@@ -113,22 +73,16 @@ test('emit error if worker stream emit error', async function (t) {
     stream.write('hello world\n')
   })
 
-  const [err] = await once(stream, 'error')
+  let [err] = await once(stream, 'error')
   t.equal(err.message, 'kaboom')
 
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
+
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 })
 
 test('emit error if thread have uncaughtException', async function (t) {
@@ -141,22 +95,16 @@ test('emit error if thread have uncaughtException', async function (t) {
     stream.write('hello world\n')
   })
 
-  const [err] = await once(stream, 'error')
+  let [err] = await once(stream, 'error')
   t.equal(err.message, 'kaboom')
 
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
+
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 })
 
 test('close the work if out of scope on gc', { skip: !global.WeakRef }, async function (t) {