thread-stream 0.8.1 → 0.11.1

package/.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+- package-ecosystem: github-actions
+  directory: '/'
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10
+- package-ecosystem: npm
+  directory: '/'
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10

package/.github/workflows/ci.yml

@@ -0,0 +1,60 @@
+name: CI
+on:
+  push:
+    paths-ignore:
+        - 'docs/**'
+        - '*.md'
+  pull_request:
+    paths-ignore:
+        - 'docs/**'
+        - '*.md'
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        node-version: [12, 14, 16]
+        os: [macos-latest, ubuntu-latest, windows-latest]
+
+    steps:
+
+      - uses: actions/checkout@v2.3.4
+
+      - name: Use Node.js
+        uses: actions/setup-node@v2.4.0
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Install
+        run: |
+          npm install --ignore-scripts
+
+      - name: Run tests
+        run: |
+          npm run test:ci
+
+      - name: Coveralls Parallel
+        uses: coverallsapp/github-action@1.1.3
+        with:
+          github-token: ${{ secrets.github_token }}
+          parallel: true
+          flag-name: run-${{ matrix.node-version }}-${{ matrix.os }}
+
+  coverage:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@1.1.3
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          parallel-finished: true
+
+  automerge:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: fastify/github-action-merge-dependabot@v2.4.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

package/.github/workflows/package-manager-ci.yml

@@ -0,0 +1,59 @@
+name: package-manager-ci
+on:
+  push:
+    paths-ignore:
+        - 'docs/**'
+        - '*.md'
+  pull_request:
+    paths-ignore:
+        - 'docs/**'
+        - '*.md'
+jobs:
+  pnpm:
+    name: pnpm package manager on ${{ matrix.node-version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macOS-latest, windows-latest]
+        node-version: [12, 14, 16]
+    steps:
+      - uses: actions/checkout@v2.3.4
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2.4.0
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Use pnpm
+        uses: pnpm/action-setup@v2.0.1
+        with:
+          version: ^6.0.0
+      - name: Install dependancies
+        run: pnpm install
+      - name: Tests
+        run: pnpm run test:ci
+
+  yarn-pnp:
+    name: yarn-pnp package manager on ${{ matrix.node-version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macOS-latest]
+        node-version: [12, 14, 16]
+    steps:
+      - uses: actions/checkout@v2.3.4
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2.4.0
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Use yarn
+        run: |
+          npm install -g yarn
+          yarn set version berry
+          echo "nodeLinker: pnp" >> .yarnrc.yml
+          echo "pnpMode: loose" >> .yarnrc.yml
+          yarn add -D pino-elasticsearch@^6.0.0
+          yarn install
+        env:
+          # needed due the yarn.lock file in repository's .gitignore
+          YARN_ENABLE_IMMUTABLE_INSTALLS: false
+      - name: Tests
+        run: yarn run test:yarn

package/README.md

@@ -1,4 +1,9 @@
 # thread-stream
+[![npm version](https://img.shields.io/npm/v/thread-stream)](https://www.npmjs.com/package/thread-stream)
+[![Build Status](https://img.shields.io/github/workflow/status/pinojs/thread-stream/CI)](https://github.com/pinojs/thread-stream/actions)
+[![Known Vulnerabilities](https://snyk.io/test/github/pinojs/thread-stream/badge.svg)](https://snyk.io/test/github/pinojs/thread-stream)
+[![Coverage Status](https://coveralls.io/repos/github/pinojs/thread-stream/badge.svg?branch=master)](https://coveralls.io/github/pinojs/thread-stream?branch=master)
+[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](https://standardjs.com/)
 
 A streaming way to send data to a Node.js Worker Thread.
 
@@ -53,6 +58,34 @@ async function run (opts) {
 module.exports = run
 ```
 
+Make sure that the stream emits `'close'` when the stream completes.
+This can usually be achieved by passing the [`autoDestroy: true`](https://nodejs.org/api/stream.html#stream_new_stream_writable_options)
+flag your stream classes.
+
+The underlining worker is automatically closed if the stream is garbage collected.
+
+
+### External modules
+
+You may use this module within compatible external modules, that exports the `worker.js` interface.
+
+```js
+const ThreadStream = require('thread-stream')
+
+const modulePath = require.resolve('pino-elasticsearch')
+
+const stream = new ThreadStream({
+  filename: modulePath,
+  workerData: { node: 'http://localhost:9200' }
+})
+
+stream.write('log to elasticsearch!')
+stream.flushSync()
+stream.end()
+```
+
+This module works with `yarn` in PnP (plug'n play) mode too!
+
 ## License
 
 MIT

package/index.js

@@ -10,6 +10,27 @@ const {
   READ_INDEX
 } = require('./lib/indexes')
 
+class FakeWeakRef {
+  constructor (value) {
+    this._value = value
+  }
+
+  deref () {
+    return this._value
+  }
+}
+
+const FinalizationRegistry = global.FinalizationRegistry || class FakeFinalizationRegistry {
+  register () {}
+  unregister () {}
+}
+
+const WeakRef = global.WeakRef || FakeWeakRef
+
+const registry = new FinalizationRegistry((worker) => {
+  worker.terminate()
+})
+
 function createWorker (stream, opts) {
   const { filename, workerData } = opts
 
@@ -27,6 +48,14 @@ function createWorker (stream, opts) {
     }
   })
 
+  // We keep a strong reference for now,
+  // we need to start writing first
+  worker.stream = new FakeWeakRef(stream)
+
+  worker.on('message', onWorkerMessage)
+  worker.on('exit', onWorkerExit)
+  registry.register(stream, worker)
+
   return worker
 }
 
@@ -90,6 +119,63 @@ function nextFlush (stream) {
   }
 }
 
+function onWorkerMessage (msg) {
+  const stream = this.stream.deref()
+  if (stream === undefined) {
+    // Terminate the worker.
+    this.terminate()
+    return
+  }
+
+  switch (msg.code) {
+    case 'READY':
+      // Replace the FakeWeakRef with a
+      // proper one.
+      this.stream = new WeakRef(stream)
+      if (stream._sync) {
+        stream.ready = true
+        stream.flushSync()
+        stream.emit('ready')
+      } else {
+        stream.once('drain', function () {
+          stream.flush(() => {
+            stream.ready = true
+            stream.emit('ready')
+          })
+        })
+        nextFlush(stream)
+      }
+      break
+    case 'ERROR':
+      stream.closed = true
+      // TODO only remove our own
+      stream.worker.removeAllListeners('exit')
+      stream.worker.terminate().then(null, () => {})
+      process.nextTick(() => {
+        stream.emit('error', msg.err)
+      })
+      break
+    default:
+      throw new Error('this should not happen: ' + msg.code)
+  }
+}
+
+function onWorkerExit (code) {
+  const stream = this.stream.deref()
+  if (stream === undefined) {
+    // Nothing to do, the worker already exit
+    return
+  }
+  registry.unregister(stream)
+  stream.closed = true
+  setImmediate(function () {
+    if (code !== 0) {
+      stream.emit('error', new Error('The worker thread exited'))
+    }
+    stream.emit('close')
+  })
+}
+
 class ThreadStream extends EventEmitter {
   constructor (opts = {}) {
     super()
@@ -110,42 +196,6 @@ class ThreadStream extends EventEmitter {
     this.closed = false
 
     this.buf = ''
-
-    this.worker.on('message', (msg) => {
-      switch (msg.code) {
-        case 'READY':
-          if (this._sync) {
-            this.ready = true
-            this.flushSync()
-            this.emit('ready')
-          } else {
-            this.once('drain', function () {
-              this.flush(() => {
-                this.ready = true
-                this.emit('ready')
-              })
-            })
-            nextFlush(this)
-          }
-          break
-        case 'FINISH':
-          this.emit('finish')
-          break
-        default:
-          throw new Error('this should not happen: ' + msg.code)
-      }
-    })
-
-    this.worker.on('exit', (code) => {
-      this.closed = true
-      setImmediate(() => {
-        if (code === 0) {
-          this.emit('close')
-        } else {
-          this.emit('error', new Error('The worker thread exited'))
-        }
-      })
-    })
   }
 
   _write (data, cb) {
@@ -210,10 +260,28 @@ class ThreadStream extends EventEmitter {
 
     this.flushSync()
 
+    let read = Atomics.load(this._state, READ_INDEX)
+
     // process._rawDebug('writing index')
     Atomics.store(this._state, WRITE_INDEX, -1)
     // process._rawDebug(`(end) readIndex (${Atomics.load(this._state, READ_INDEX)}) writeIndex (${Atomics.load(this._state, WRITE_INDEX)})`)
     Atomics.notify(this._state, WRITE_INDEX)
+
+    // Wait for the process to complete
+    let spins = 0
+    while (read !== -1) {
+      // process._rawDebug(`read = ${read}`)
+      Atomics.wait(this._state, READ_INDEX, read, 1000)
+      read = Atomics.load(this._state, READ_INDEX)
+
+      if (++spins === 10) {
+        throw new Error('end() took too long (10s)')
+      }
+    }
+
+    process.nextTick(() => {
+      this.emit('finish')
+    })
     // process._rawDebug('end finished...')
   }
 

package/lib/worker.js

@@ -15,16 +15,39 @@ const state = new Int32Array(stateBuf)
 const data = Buffer.from(dataBuf)
 
 async function start () {
-  const fn = (await import(workerData.filename)).default
+  let fn
+  try {
+    fn = (await import(workerData.filename)).default
+  } catch (error) {
+    // A yarn user that tries to start a ThreadStream for an external module
+    // provides a filename pointing to a zip file.
+    // eg. require.resolve('pino-elasticsearch') // returns /foo/pino-elasticsearch-npm-6.1.0-0c03079478-6915435172.zip/bar.js
+    // The `import` will fail to try to load it.
+    // This catch block executes the `require` fallback to load the module correctly.
+    // In fact, yarn modifies the `require` function to manage the zipped path.
+    // More details at https://github.com/pinojs/pino/pull/1113
+    // The error codes may change based on the node.js version (ENOTDIR > 12, ERR_MODULE_NOT_FOUND <= 12 )
+    if ((error.code === 'ENOTDIR' || error.code === 'ERR_MODULE_NOT_FOUND') &&
+     workerData.filename.startsWith('file://')) {
+      fn = require(workerData.filename.replace('file://', ''))
+    } else {
+      throw error
+    }
+  }
   destination = await fn(workerData.workerData)
 
-  destination.on('finish', function () {
+  destination.on('error', function (err) {
     parentPort.postMessage({
-      code: 'FINISH'
+      code: 'ERROR',
+      err
     })
   })
 
   destination.on('close', function () {
+    // process._rawDebug('worker close emitted')
+    const end = Atomics.load(state, WRITE_INDEX)
+    Atomics.store(state, READ_INDEX, end)
+    Atomics.notify(state, READ_INDEX)
     setImmediate(() => {
       process.exit(0)
     })
@@ -60,8 +83,6 @@ function run () {
   // process._rawDebug(`post state ${current} ${end}`)
 
   if (end === -1) {
-    Atomics.store(state, READ_INDEX, end)
-    Atomics.notify(state, READ_INDEX)
     // process._rawDebug('end')
     destination.end()
     return
@@ -86,7 +107,17 @@ function run () {
 }
 
 process.on('unhandledRejection', function (err) {
-  // TODO transfer this to main
-  console.error(err)
+  parentPort.postMessage({
+    code: 'ERROR',
+    err
+  })
+  process.exit(1)
+})
+
+process.on('uncaughtException', function (err) {
+  parentPort.postMessage({
+    code: 'ERROR',
+    err
+  })
   process.exit(1)
 })

package/package.json

@@ -1,18 +1,20 @@
 {
   "name": "thread-stream",
-  "version": "0.8.1",
+  "version": "0.11.1",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
   "devDependencies": {
     "desm": "^1.1.0",
     "fastbench": "^1.0.1",
-    "husky": "^5.1.3",
-    "sonic-boom": "^1.3.2",
+    "husky": "^7.0.0",
+    "sonic-boom": "^2.0.1",
     "standard": "^16.0.3",
     "tap": "^15.0.0"
   },
   "scripts": {
     "test": "standard && tap --no-check-coverage test/*.test.*js",
+    "test:ci": "standard && tap \"test/**/*.test.*js\" --no-check-coverage --coverage-report=lcovonly",
+    "test:yarn": "tap \"test/**/*.test.js\" --no-check-coverage",
     "prepare": "husky install"
   },
   "repository": {

package/test/close-on-gc.js

@@ -0,0 +1,39 @@
+'use strict'
+
+const { join } = require('path')
+const ThreadStream = require('..')
+const assert = require('assert')
+
+let worker = null
+
+function setup () {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest: process.argv[2] },
+    sync: true
+  })
+
+  worker = stream.worker
+
+  stream.on('ready', function () {
+    stream.write('hello')
+    stream.write(' ')
+    stream.write('world\n')
+    stream.flushSync()
+    stream.unref()
+
+    // the stream object goes out of scope here
+    setImmediate(gc) // eslint-disable-line
+  })
+}
+
+setup()
+
+let exitEmitted = false
+worker.on('exit', function () {
+  exitEmitted = true
+})
+
+process.on('exit', function () {
+  assert.strictEqual(exitEmitted, true)
+})

package/test/commonjs-fallback.test.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const { test } = require('tap')
+const ThreadStream = require('..')
+
+const isYarnPnp = process.versions.pnp !== undefined
+
+test('yarn module resolution', { skip: !isYarnPnp }, t => {
+  t.plan(5)
+
+  const modulePath = require.resolve('pino-elasticsearch')
+  t.match(modulePath, /.*\.zip.*/)
+
+  const stream = new ThreadStream({
+    filename: modulePath,
+    workerData: { node: null },
+    sync: true
+  })
+
+  stream.on('error', (err) => {
+    t.pass('error emitted')
+    t.equal(err.message, 'Missing node(s) option', 'module custom error')
+  })
+
+  t.ok(stream.write('hello world\n'))
+  t.ok(stream.writable)
+
+  stream.end()
+})

package/test/end.test.js

@@ -0,0 +1,77 @@
+'use strict'
+
+const { test } = require('tap')
+const { join } = require('path')
+const { readFile } = require('fs')
+const { file } = require('./helper')
+const ThreadStream = require('..')
+
+test('destroy support', function (t) {
+  t.plan(9)
+
+  const dest = file()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file-on-destroy.js'),
+    workerData: { dest },
+    sync: true
+  })
+
+  stream.on('drain', () => {
+    t.pass('drain')
+  })
+
+  stream.on('ready', () => {
+    t.pass('ready emitted')
+
+    t.ok(stream.write('hello world\n'))
+    t.ok(stream.write('something else\n'))
+    t.ok(stream.writable)
+
+    stream.end()
+
+    readFile(dest, 'utf8', (err, data) => {
+      t.error(err)
+      t.equal(data, 'hello world\nsomething else\n')
+    })
+  })
+
+  stream.on('close', () => {
+    t.notOk(stream.writable)
+    t.pass('close emitted')
+  })
+})
+
+test('synchronous _final support', function (t) {
+  t.plan(9)
+
+  const dest = file()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file-on-final.js'),
+    workerData: { dest },
+    sync: true
+  })
+
+  stream.on('drain', () => {
+    t.pass('drain')
+  })
+
+  stream.on('ready', () => {
+    t.pass('ready emitted')
+
+    t.ok(stream.write('hello world\n'))
+    t.ok(stream.write('something else\n'))
+    t.ok(stream.writable)
+
+    stream.end()
+
+    readFile(dest, 'utf8', (err, data) => {
+      t.error(err)
+      t.equal(data, 'hello world\nsomething else\n')
+    })
+  })
+
+  stream.on('close', () => {
+    t.notOk(stream.writable)
+    t.pass('close emitted')
+  })
+})

package/test/error.js

@@ -0,0 +1,14 @@
+'use strict'
+
+const { Writable } = require('stream')
+
+async function run (opts) {
+  const stream = new Writable({
+    write (chunk, enc, cb) {
+      cb(new Error('kaboom'))
+    }
+  })
+  return stream
+}
+
+module.exports = run

package/test/port.js

@@ -5,6 +5,7 @@ const { Writable } = require('stream')
 function run (opts) {
   const { port } = opts
   return new Writable({
+    autoDestroy: true,
     write (chunk, enc, cb) {
       port.postMessage(chunk.toString())
       cb()

package/test/thread-management.test.js

@@ -86,7 +86,7 @@ test('emit error if thread have unhandledRejection', async function (t) {
   })
 
   const [err] = await once(stream, 'error')
-  t.equal(err.message, 'The worker thread exited')
+  t.equal(err.message, 'kaboom')
 
   try {
     stream.write('noop')
@@ -102,3 +102,72 @@ test('emit error if thread have unhandledRejection', async function (t) {
     t.equal(err.message, 'the worker has exited')
   }
 })
+
+test('emit error if worker stream emit error', async function (t) {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'error.js'),
+    sync: true
+  })
+
+  stream.on('ready', function () {
+    stream.write('hello world\n')
+  })
+
+  const [err] = await once(stream, 'error')
+  t.equal(err.message, 'kaboom')
+
+  try {
+    stream.write('noop')
+    t.fail('unreacheable')
+  } catch (err) {
+    t.equal(err.message, 'the worker has exited')
+  }
+
+  try {
+    stream.write('noop')
+    t.fail('unreacheable')
+  } catch (err) {
+    t.equal(err.message, 'the worker has exited')
+  }
+})
+
+test('emit error if thread have uncaughtException', async function (t) {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'uncaughtException.js'),
+    sync: true
+  })
+
+  stream.on('ready', function () {
+    stream.write('hello world\n')
+  })
+
+  const [err] = await once(stream, 'error')
+  t.equal(err.message, 'kaboom')
+
+  try {
+    stream.write('noop')
+    t.fail('unreacheable')
+  } catch (err) {
+    t.equal(err.message, 'the worker has exited')
+  }
+
+  try {
+    stream.write('noop')
+    t.fail('unreacheable')
+  } catch (err) {
+    t.equal(err.message, 'the worker has exited')
+  }
+})
+
+test('close the work if out of scope on gc', { skip: !global.WeakRef }, async function (t) {
+  const dest = file()
+  const child = fork(join(__dirname, 'close-on-gc.js'), [dest], {
+    execArgv: ['--expose-gc']
+  })
+
+  const [code] = await once(child, 'exit')
+  t.equal(code, 0)
+
+  const data = await readFile(dest, 'utf8')
+  t.equal(data, 'hello world\n')
+})

package/test/to-file-on-destroy.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const fs = require('fs')
+const { Writable } = require('stream')
+
+function run (opts) {
+  let data = ''
+  return new Writable({
+    autoDestroy: true,
+    write (chunk, enc, cb) {
+      data += chunk.toString()
+      cb()
+    },
+    destroy (err, cb) {
+      // process._rawDebug('destroy called')
+      fs.writeFile(opts.dest, data, function (err2) {
+        cb(err2 || err)
+      })
+    }
+  })
+}
+
+module.exports = run

package/test/to-file-on-final.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const fs = require('fs')
+const { Writable } = require('stream')
+
+function run (opts) {
+  let data = ''
+  return new Writable({
+    autoDestroy: true,
+    write (chunk, enc, cb) {
+      data += chunk.toString()
+      cb()
+    },
+    final (cb) {
+      setTimeout(function () {
+        fs.writeFile(opts.dest, data, function (err) {
+          cb(err)
+        })
+      }, 100)
+    }
+  })
+}
+
+module.exports = run

package/test/uncaughtException.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const { Writable } = require('stream')
+
+// Nop console.error to avoid printing things out
+console.error = () => {}
+
+setImmediate(function () {
+  throw new Error('kaboom')
+})
+
+async function run (opts) {
+  const stream = new Writable({
+    write (chunk, enc, cb) {
+      cb()
+    }
+  })
+  return stream
+}
+
+module.exports = run

package/.github/workflows/nodejs.yml

@@ -1,24 +0,0 @@
-name: Node.js CI
-
-on: [ push, pull_request ]
-
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        node-version: [12.x, 14.x, 15.x]
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - run: node --version
-    - run: npm --version
-    - run: npm install
-    - run: npm test
-      env:
-        CI: true