thread-stream 1.0.0 → 2.0.1

package/.github/workflows/ci.yml

@@ -1,4 +1,5 @@
 name: CI
+
 on:
   push:
     paths-ignore:
@@ -8,35 +9,53 @@ on:
     paths-ignore:
         - 'docs/**'
         - '*.md'
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}"
+  cancel-in-progress: true
+
 jobs:
+  dependency-review:
+    name: Dependency Review
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      - name: Dependency review
+        uses: actions/dependency-review-action@v2
+
   test:
+    name: Test
     runs-on: ${{ matrix.os }}
-
     permissions:
       contents: read
-
     strategy:
       matrix:
         node-version: [14, 16, 18]
         os: [macos-latest, ubuntu-latest, windows-latest]
-
     steps:
+      - name: Check out repo
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
 
-      - uses: actions/checkout@v3
-
-      - name: Use Node.js
+      - name: Setup Node ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Install
-        run: |
-          npm install --ignore-scripts
+      - name: Install dependencies
+        run: npm i --ignore-scripts
 
       - name: Run tests
-        shell: bash
-        run: |
-          npm run test:ci
+        run: npm run test:ci
 
       - name: Coveralls Parallel
         uses: coverallsapp/github-action@1.1.3
@@ -56,11 +75,15 @@ jobs:
           parallel-finished: true
 
   automerge:
+    name: Automerge Dependabot PRs
+    if: >
+        github.event_name == 'pull_request' &&
+        github.event.pull_request.user.login == 'dependabot[bot]'
     needs: test
-    runs-on: ubuntu-latest
     permissions:
       pull-requests: write
       contents: write
+    runs-on: ubuntu-latest
     steps:
       - uses: fastify/github-action-merge-dependabot@v3
         with:

package/.github/workflows/package-manager-ci.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macOS-latest, windows-latest]
+        os: [ubuntu-latest]
         node-version: [14, 16, 18]
     steps:
       - uses: actions/checkout@v3
@@ -23,7 +23,7 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - name: Use pnpm
-        uses: pnpm/action-setup@v2.2.1
+        uses: pnpm/action-setup@v2.2.2
         with:
           version: ^6.0.0
       - name: Install dependancies
@@ -37,7 +37,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macOS-latest]
+        os: [ubuntu-latest]
         node-version: [14, 16, 18]
     steps:
       - uses: actions/checkout@v3
@@ -51,6 +51,7 @@ jobs:
           yarn set version berry
           echo "nodeLinker: pnp" >> .yarnrc.yml
           echo "pnpMode: loose" >> .yarnrc.yml
+          echo "pnpEnableEsmLoader: false" >> .yarnrc.yml
           yarn add -D pino-elasticsearch@^6.0.0
           yarn install
         env:

package/README.md

@@ -1,7 +1,6 @@
 # thread-stream
 [![npm version](https://img.shields.io/npm/v/thread-stream)](https://www.npmjs.com/package/thread-stream)
 [![Build Status](https://img.shields.io/github/workflow/status/pinojs/thread-stream/CI)](https://github.com/pinojs/thread-stream/actions)
-[![Known Vulnerabilities](https://snyk.io/test/github/pinojs/thread-stream/badge.svg)](https://snyk.io/test/github/pinojs/thread-stream)
 [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](https://standardjs.com/)
 
 A streaming way to send data to a Node.js Worker Thread.

package/index.d.ts

@@ -1,9 +1,73 @@
 import { EventEmitter } from 'events'
+import workerThreads from 'worker_threads'
+
+interface ThreadStreamOptions {
+  /**
+   * The size (in bytes) of the buffer.
+   * Must be greater than 4 (i.e. it must at least fit a 4-byte utf-8 char).
+   * 
+   * Default: `4 * 1024 * 1024` = `4194304`
+   */
+  bufferSize?: number,
+  /**
+   * The path to the Worker's main script or module.
+   * Must be either an absolute path or a relative path (i.e. relative to the current working directory) starting with ./ or ../, or a WHATWG URL object using file: or data: protocol.
+   * When using a data: URL, the data is interpreted based on MIME type using the ECMAScript module loader.
+   * 
+   * {@link workerThreads.Worker()}
+   */
+  filename: string | URL,
+  /**
+   * If `true`, write data synchronously; otherwise write data asynchronously.
+   * 
+   * Default: `false`.
+   */
+  sync?: boolean,
+  /**
+   * {@link workerThreads.WorkerOptions.workerData}
+   * 
+   * Default: `{}`
+   */
+  workerData?: any,
+  /**
+   * {@link workerThreads.WorkerOptions}
+   * 
+   * Default: `{}`
+   */
+  workerOpts?: workerThreads.WorkerOptions
+}
+
 
 declare class ThreadStream extends EventEmitter {
-  constructor(opts: {})
-  write (data: string): boolean
-  end (): void
+  /**
+   * @param {ThreadStreamOptions} opts 
+   */
+  constructor(opts: ThreadStreamOptions)
+  /**
+   * Write some data to the stream.
+   * 
+   * **Please note that this method should not throw an {Error} if something goes wrong but emit an error event.**
+   * @param {string} data data to write.
+   * @returns {boolean} false if the stream wishes for the calling code to wait for the 'drain' event to be emitted before continuing to write additional data or if it fails to write; otherwise true.
+   */
+  write(data: string): boolean
+  /**
+   * Signal that no more data will be written.
+   * 
+   * **Please note that this method should not throw an {Error} if something goes wrong but emit an error event.**
+   * 
+   * Calling the {@link write()} method after calling {@link end()} will emit an error.
+   */
+  end(): void
+  /**
+   * Flush the stream synchronously.
+   * This method should be called in the shutdown phase to make sure that all data has been flushed.
+   * 
+   * **Please note that this method will throw an {Error} if something goes wrong.**
+   * 
+   * @throws {Error} if the stream is already flushing, if it fails to flush or if it takes more than 10 seconds to flush.
+   */
+  flushSync(): void
 }
 
 export = ThreadStream;

package/index.js

@@ -136,7 +136,7 @@ function nextFlush (stream) {
     })
   } else {
     // This should never happen
-    throw new Error('overwritten')
+    destroy(stream, new Error('overwritten'))
   }
 }
 
@@ -164,7 +164,7 @@ function onWorkerMessage (msg) {
       destroy(stream, msg.err)
       break
     default:
-      throw new Error('this should not happen: ' + msg.code)
+      destroy(stream, new Error('this should not happen: ' + msg.code))
   }
 }
 
@@ -177,7 +177,7 @@ function onWorkerExit (code) {
   registry.unregister(stream)
   stream.worker.exited = true
   stream.worker.off('exit', onWorkerExit)
-  destroy(stream, code !== 0 ? new Error('The worker thread exited') : null)
+  destroy(stream, code !== 0 ? new Error('the worker thread exited') : null)
 }
 
 class ThreadStream extends EventEmitter {
@@ -211,11 +211,13 @@ class ThreadStream extends EventEmitter {
 
   write (data) {
     if (this[kImpl].destroyed) {
-      throw new Error('the worker has exited')
+      error(this, new Error('the worker has exited'))
+      return false
     }
 
     if (this[kImpl].ending) {
-      throw new Error('the worker is ending')
+      error(this, new Error('the worker is ending'))
+      return false
     }
 
     if (this[kImpl].flushing && this[kImpl].buf.length + data.length >= MAX_STRING) {
@@ -338,6 +340,12 @@ class ThreadStream extends EventEmitter {
   }
 }
 
+function error (stream, err) {
+  setImmediate(() => {
+    stream.emit('error', err)
+  })
+}
+
 function destroy (stream, err) {
   if (stream[kImpl].destroyed) {
     return
@@ -346,7 +354,7 @@ function destroy (stream, err) {
 
   if (err) {
     stream[kImpl].errored = err
-    stream.emit('error', err)
+    error(stream, err)
   }
 
   if (!stream.worker.exited) {
@@ -399,11 +407,13 @@ function end (stream) {
       readIndex = Atomics.load(stream[kImpl].state, READ_INDEX)
 
       if (readIndex === -2) {
-        throw new Error('end() failed')
+        destroy(stream, new Error('end() failed'))
+        return
       }
 
       if (++spins === 10) {
-        throw new Error('end() took too long (10s)')
+        destroy(stream, new Error('end() took too long (10s)'))
+        return
       }
     }
 
@@ -482,7 +492,7 @@ function flushSync (stream) {
     const readIndex = Atomics.load(stream[kImpl].state, READ_INDEX)
 
     if (readIndex === -2) {
-      throw new Error('_flushSync failed')
+      throw Error('_flushSync failed')
     }
 
     // process._rawDebug(`(flushSync) readIndex (${readIndex}) writeIndex (${writeIndex})`)

package/lib/worker.js

@@ -32,12 +32,6 @@ async function start () {
     } else {
       fn = (await realImport(filename))
     }
-
-    // Depending on how the default export is performed, and on how the code is
-    // transpiled, we may find cases of two nested "default" objects.
-    // See https://github.com/pinojs/pino/issues/1243#issuecomment-982774762
-    if (typeof fn === 'object') fn = fn.default
-    if (typeof fn === 'object') fn = fn.default
   } catch (error) {
     // A yarn user that tries to start a ThreadStream for an external module
     // provides a filename pointing to a zip file.
@@ -50,10 +44,20 @@ async function start () {
     if ((error.code === 'ENOTDIR' || error.code === 'ERR_MODULE_NOT_FOUND') &&
      filename.startsWith('file://')) {
       fn = realRequire(decodeURIComponent(filename.replace('file://', '')))
+    } else if (error.code === undefined) {
+      // When bundled with pkg, an undefined error is thrown when called with realImport
+      fn = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
     } else {
       throw error
     }
   }
+
+  // Depending on how the default export is performed, and on how the code is
+  // transpiled, we may find cases of two nested "default" objects.
+  // See https://github.com/pinojs/pino/issues/1243#issuecomment-982774762
+  if (typeof fn === 'object') fn = fn.default
+  if (typeof fn === 'object') fn = fn.default
+
   destination = await fn(workerData.workerData)
 
   destination.on('error', function (err) {

package/package.json

@@ -1,19 +1,19 @@
 {
   "name": "thread-stream",
-  "version": "1.0.0",
+  "version": "2.0.1",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
   "types": "index.d.ts",
   "dependencies": {
-    "real-require": "^0.1.0"
+    "real-require": "^0.2.0"
   },
   "devDependencies": {
-    "@types/node": "^17.0.37",
+    "@types/node": "^18.0.0",
     "@types/tap": "^15.0.0",
     "desm": "^1.3.0",
     "fastbench": "^1.0.1",
     "husky": "^8.0.1",
-    "sonic-boom": "^2.8.0",
+    "sonic-boom": "^3.0.0",
     "standard": "^17.0.0",
     "tap": "^16.2.0",
     "ts-node": "^10.8.0",

package/test/base.test.js

@@ -263,7 +263,7 @@ test('destroy does not error', function (t) {
   })
 
   stream.on('error', (err) => {
-    t.equal(err.message, 'The worker thread exited')
+    t.equal(err.message, 'the worker thread exited')
     stream.flush((err) => {
       t.equal(err.message, 'the worker has exited')
     })

package/test/bundlers.test.js

@@ -5,7 +5,7 @@ const { join } = require('path')
 const { file } = require('./helper')
 const ThreadStream = require('..')
 
-test('bundlers support', function (t) {
+test('bundlers support with .js file', function (t) {
   t.plan(1)
 
   globalThis.__bundlerPathsOverrides = {
@@ -31,3 +31,30 @@ test('bundlers support', function (t) {
 
   stream.end()
 })
+
+test('bundlers support with .mjs file', function (t) {
+  t.plan(1)
+
+  globalThis.__bundlerPathsOverrides = {
+    'thread-stream-worker': join(__dirname, 'custom-worker.js')
+  }
+
+  const dest = file()
+
+  process.on('uncaughtException', error => {
+    console.log(error)
+  })
+
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.mjs'),
+    workerData: { dest },
+    sync: true
+  })
+
+  stream.worker.removeAllListeners('message')
+  stream.worker.once('message', message => {
+    t.equal(message.code, 'CUSTOM-WORKER-CALLED')
+  })
+
+  stream.end()
+})

package/test/commonjs-fallback.test.js

@@ -54,3 +54,27 @@ test('yarn module resolution for directories with special characters', { skip: !
 
   t.equal(strings, 'hello world\nsomething else\n')
 })
+
+test('yarn module resolution for typescript commonjs modules', { skip: !isYarnPnp }, async t => {
+  t.plan(3)
+
+  const { port1, port2 } = new MessageChannel()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'ts-commonjs-default-export.zip', 'worker.js'),
+    workerData: { port: port1 },
+    workerOpts: {
+      transferList: [port1]
+    },
+    sync: false
+  })
+  t.teardown(() => {
+    stream.end()
+  })
+
+  t.ok(stream.write('hello world\n'))
+  t.ok(stream.write('something else\n'))
+
+  const [strings] = await once(port2, 'message')
+
+  t.equal(strings, 'hello world\nsomething else\n')
+})

package/test/string-limit-2.test.js

@@ -1,6 +1,12 @@
 'use strict'
 
 const t = require('tap')
+
+if (process.env.CI) {
+  t.skip('skip on CI')
+  process.exit(0)
+}
+
 const { join } = require('path')
 const { file } = require('./helper')
 const { createReadStream } = require('fs')

package/test/string-limit.test.js

@@ -1,6 +1,12 @@
 'use strict'
 
 const t = require('tap')
+
+if (process.env.CI) {
+  t.skip('skip on CI')
+  process.exit(0)
+}
+
 const { join } = require('path')
 const { file } = require('./helper')
 const { stat } = require('fs')

package/test/thread-management.test.js

@@ -29,50 +29,16 @@ test('emit error if thread exits', async function (t) {
     stream.write('hello world\n')
   })
 
-  const [err] = await once(stream, 'error')
-  t.equal(err.message, 'The worker thread exited')
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-})
+  let [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker thread exited')
 
-test('emit error if thread exits', async function (t) {
-  const stream = new ThreadStream({
-    filename: join(__dirname, 'exit.js'),
-    sync: true
-  })
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 
-  stream.on('ready', () => {
-    stream.write('hello world\n')
-  })
-
-  const [err] = await once(stream, 'error')
-  t.equal(err.message, 'The worker thread exited')
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 })
 
 test('emit error if thread have unhandledRejection', async function (t) {
@@ -85,22 +51,16 @@ test('emit error if thread have unhandledRejection', async function (t) {
     stream.write('hello world\n')
   })
 
-  const [err] = await once(stream, 'error')
+  let [err] = await once(stream, 'error')
   t.equal(err.message, 'kaboom')
 
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
+
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 })
 
 test('emit error if worker stream emit error', async function (t) {
@@ -113,22 +73,16 @@ test('emit error if worker stream emit error', async function (t) {
     stream.write('hello world\n')
   })
 
-  const [err] = await once(stream, 'error')
+  let [err] = await once(stream, 'error')
   t.equal(err.message, 'kaboom')
 
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
+
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 })
 
 test('emit error if thread have uncaughtException', async function (t) {
@@ -141,22 +95,16 @@ test('emit error if thread have uncaughtException', async function (t) {
     stream.write('hello world\n')
   })
 
-  const [err] = await once(stream, 'error')
+  let [err] = await once(stream, 'error')
   t.equal(err.message, 'kaboom')
 
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
-
-  try {
-    stream.write('noop')
-    t.fail('unreacheable')
-  } catch (err) {
-    t.equal(err.message, 'the worker has exited')
-  }
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
+
+  stream.write('noop');
+  [err] = await once(stream, 'error')
+  t.equal(err.message, 'the worker has exited')
 })
 
 test('close the work if out of scope on gc', { skip: !global.WeakRef }, async function (t) {