thread-stream 0.9.0 → 0.11.2

package/.github/workflows/ci.yml

@@ -19,10 +19,10 @@ jobs:
 
     steps:
 
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v2.3.5
 
       - name: Use Node.js
-        uses: actions/setup-node@v2.1.5
+        uses: actions/setup-node@v2.4.1
         with:
           node-version: ${{ matrix.node-version }}
 
@@ -35,7 +35,7 @@ jobs:
           npm run test:ci
 
       - name: Coveralls Parallel
-        uses: coverallsapp/github-action@v1.1.2
+        uses: coverallsapp/github-action@1.1.3
         with:
           github-token: ${{ secrets.github_token }}
           parallel: true
@@ -46,7 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Coveralls Finished
-        uses: coverallsapp/github-action@master
+        uses: coverallsapp/github-action@1.1.3
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           parallel-finished: true
@@ -55,6 +55,6 @@ jobs:
     needs: test
     runs-on: ubuntu-latest
     steps:
-      - uses: fastify/github-action-merge-dependabot@v2.0.0
+      - uses: fastify/github-action-merge-dependabot@v2.4.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

package/.github/workflows/package-manager-ci.yml

@@ -0,0 +1,59 @@
+name: package-manager-ci
+on:
+  push:
+    paths-ignore:
+        - 'docs/**'
+        - '*.md'
+  pull_request:
+    paths-ignore:
+        - 'docs/**'
+        - '*.md'
+jobs:
+  pnpm:
+    name: pnpm package manager on ${{ matrix.node-version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macOS-latest, windows-latest]
+        node-version: [12, 14, 16]
+    steps:
+      - uses: actions/checkout@v2.3.5
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2.4.1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Use pnpm
+        uses: pnpm/action-setup@v2.0.1
+        with:
+          version: ^6.0.0
+      - name: Install dependancies
+        run: pnpm install
+      - name: Tests
+        run: pnpm run test:ci
+
+  yarn-pnp:
+    name: yarn-pnp package manager on ${{ matrix.node-version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macOS-latest]
+        node-version: [12, 14, 16]
+    steps:
+      - uses: actions/checkout@v2.3.5
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2.4.1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Use yarn
+        run: |
+          npm install -g yarn
+          yarn set version berry
+          echo "nodeLinker: pnp" >> .yarnrc.yml
+          echo "pnpMode: loose" >> .yarnrc.yml
+          yarn add -D pino-elasticsearch@^6.0.0
+          yarn install
+        env:
+          # needed due the yarn.lock file in repository's .gitignore
+          YARN_ENABLE_IMMUTABLE_INSTALLS: false
+      - name: Tests
+        run: yarn run test:yarn

package/.taprc

@@ -0,0 +1,2 @@
+jobs: 1
+check-coverage: false

package/README.md

@@ -62,6 +62,30 @@ Make sure that the stream emits `'close'` when the stream completes.
 This can usually be achieved by passing the [`autoDestroy: true`](https://nodejs.org/api/stream.html#stream_new_stream_writable_options)
 flag your stream classes.
 
+The underlining worker is automatically closed if the stream is garbage collected.
+
+
+### External modules
+
+You may use this module within compatible external modules, that exports the `worker.js` interface.
+
+```js
+const ThreadStream = require('thread-stream')
+
+const modulePath = require.resolve('pino-elasticsearch')
+
+const stream = new ThreadStream({
+  filename: modulePath,
+  workerData: { node: 'http://localhost:9200' }
+})
+
+stream.write('log to elasticsearch!')
+stream.flushSync()
+stream.end()
+```
+
+This module works with `yarn` in PnP (plug'n play) mode too!
+
 ## License
 
 MIT

package/index.js

@@ -9,6 +9,34 @@ const {
   WRITE_INDEX,
   READ_INDEX
 } = require('./lib/indexes')
+const buffer = require('buffer')
+
+// V8 limit for string size
+const MAX_STRING = buffer.constants.MAX_STRING_LENGTH
+
+class FakeWeakRef {
+  constructor (value) {
+    this._value = value
+  }
+
+  deref () {
+    return this._value
+  }
+}
+
+const FinalizationRegistry = global.FinalizationRegistry || class FakeFinalizationRegistry {
+  register () {}
+  unregister () {}
+}
+
+const WeakRef = global.WeakRef || FakeWeakRef
+
+const registry = new FinalizationRegistry((worker) => {
+  if (worker.exited) {
+    return
+  }
+  worker.terminate()
+})
 
 function createWorker (stream, opts) {
   const { filename, workerData } = opts
@@ -27,6 +55,14 @@ function createWorker (stream, opts) {
     }
   })
 
+  // We keep a strong reference for now,
+  // we need to start writing first
+  worker.stream = new FakeWeakRef(stream)
+
+  worker.on('message', onWorkerMessage)
+  worker.on('exit', onWorkerExit)
+  registry.register(stream, worker)
+
   return worker
 }
 
@@ -90,6 +126,66 @@ function nextFlush (stream) {
   }
 }
 
+function onWorkerMessage (msg) {
+  const stream = this.stream.deref()
+  if (stream === undefined) {
+    this.exited = true
+    // Terminate the worker.
+    this.terminate()
+    return
+  }
+
+  switch (msg.code) {
+    case 'READY':
+      // Replace the FakeWeakRef with a
+      // proper one.
+      this.stream = new WeakRef(stream)
+      if (stream._sync) {
+        stream.ready = true
+        stream.flushSync()
+        stream.emit('ready')
+      } else {
+        stream.once('drain', function () {
+          stream.flush(() => {
+            stream.ready = true
+            stream.emit('ready')
+          })
+        })
+        nextFlush(stream)
+      }
+      break
+    case 'ERROR':
+      stream.closed = true
+      stream.worker.exited = true
+      // TODO only remove our own
+      stream.worker.removeAllListeners('exit')
+      stream.worker.terminate().then(null, () => {})
+      process.nextTick(() => {
+        stream.emit('error', msg.err)
+      })
+      break
+    default:
+      throw new Error('this should not happen: ' + msg.code)
+  }
+}
+
+function onWorkerExit (code) {
+  const stream = this.stream.deref()
+  if (stream === undefined) {
+    // Nothing to do, the worker already exit
+    return
+  }
+  registry.unregister(stream)
+  stream.closed = true
+  stream.worker.exited = true
+  setImmediate(function () {
+    if (code !== 0) {
+      stream.emit('error', new Error('The worker thread exited'))
+    }
+    stream.emit('close')
+  })
+}
+
 class ThreadStream extends EventEmitter {
   constructor (opts = {}) {
     super()
@@ -110,39 +206,6 @@ class ThreadStream extends EventEmitter {
     this.closed = false
 
     this.buf = ''
-
-    this.worker.on('message', (msg) => {
-      switch (msg.code) {
-        case 'READY':
-          if (this._sync) {
-            this.ready = true
-            this.flushSync()
-            this.emit('ready')
-          } else {
-            this.once('drain', function () {
-              this.flush(() => {
-                this.ready = true
-                this.emit('ready')
-              })
-            })
-            nextFlush(this)
-          }
-          break
-        default:
-          throw new Error('this should not happen: ' + msg.code)
-      }
-    })
-
-    this.worker.on('exit', (code) => {
-      this.closed = true
-      setImmediate(() => {
-        if (code === 0) {
-          this.emit('close')
-        } else {
-          this.emit('error', new Error('The worker thread exited'))
-        }
-      })
-    })
   }
 
   _write (data, cb) {
@@ -166,6 +229,12 @@ class ThreadStream extends EventEmitter {
       throw new Error('the worker has exited')
     }
 
+    if (this.flushing && this.buf.length + data.length >= MAX_STRING) {
+      // process._rawDebug('write: flushing')
+      this._writeSync()
+      this.flushing = true // we are still flushing
+    }
+
     if (!this.ready || this.flushing) {
       this.buf += data
       return this._hasSpace()

package/lib/worker.js

@@ -15,9 +15,34 @@ const state = new Int32Array(stateBuf)
 const data = Buffer.from(dataBuf)
 
 async function start () {
-  const fn = (await import(workerData.filename)).default
+  let fn
+  try {
+    fn = (await import(workerData.filename)).default
+  } catch (error) {
+    // A yarn user that tries to start a ThreadStream for an external module
+    // provides a filename pointing to a zip file.
+    // eg. require.resolve('pino-elasticsearch') // returns /foo/pino-elasticsearch-npm-6.1.0-0c03079478-6915435172.zip/bar.js
+    // The `import` will fail to try to load it.
+    // This catch block executes the `require` fallback to load the module correctly.
+    // In fact, yarn modifies the `require` function to manage the zipped path.
+    // More details at https://github.com/pinojs/pino/pull/1113
+    // The error codes may change based on the node.js version (ENOTDIR > 12, ERR_MODULE_NOT_FOUND <= 12 )
+    if ((error.code === 'ENOTDIR' || error.code === 'ERR_MODULE_NOT_FOUND') &&
+     workerData.filename.startsWith('file://')) {
+      fn = require(workerData.filename.replace('file://', ''))
+    } else {
+      throw error
+    }
+  }
   destination = await fn(workerData.workerData)
 
+  destination.on('error', function (err) {
+    parentPort.postMessage({
+      code: 'ERROR',
+      err
+    })
+  })
+
   destination.on('close', function () {
     // process._rawDebug('worker close emitted')
     const end = Atomics.load(state, WRITE_INDEX)
@@ -82,7 +107,17 @@ function run () {
 }
 
 process.on('unhandledRejection', function (err) {
-  // TODO transfer this to main
-  console.error(err)
+  parentPort.postMessage({
+    code: 'ERROR',
+    err
+  })
+  process.exit(1)
+})
+
+process.on('uncaughtException', function (err) {
+  parentPort.postMessage({
+    code: 'ERROR',
+    err
+  })
   process.exit(1)
 })

package/package.json

@@ -1,19 +1,21 @@
 {
   "name": "thread-stream",
-  "version": "0.9.0",
+  "version": "0.11.2",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
   "devDependencies": {
     "desm": "^1.1.0",
     "fastbench": "^1.0.1",
-    "husky": "^6.0.0",
-    "sonic-boom": "^1.3.2",
+    "husky": "^7.0.0",
+    "sonic-boom": "^2.0.1",
     "standard": "^16.0.3",
-    "tap": "^15.0.0"
+    "tap": "^15.0.0",
+    "why-is-node-running": "^2.2.0"
   },
   "scripts": {
-    "test": "standard && tap --no-check-coverage test/*.test.*js",
+    "test": "standard && tap test/*.test.*js",
     "test:ci": "standard && tap \"test/**/*.test.*js\" --no-check-coverage --coverage-report=lcovonly",
+    "test:yarn": "tap \"test/**/*.test.js\" --no-check-coverage",
     "prepare": "husky install"
   },
   "repository": {

package/test/close-on-gc.js

@@ -0,0 +1,39 @@
+'use strict'
+
+const { join } = require('path')
+const ThreadStream = require('..')
+const assert = require('assert')
+
+let worker = null
+
+function setup () {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest: process.argv[2] },
+    sync: true
+  })
+
+  worker = stream.worker
+
+  stream.on('ready', function () {
+    stream.write('hello')
+    stream.write(' ')
+    stream.write('world\n')
+    stream.flushSync()
+    stream.unref()
+
+    // the stream object goes out of scope here
+    setImmediate(gc) // eslint-disable-line
+  })
+}
+
+setup()
+
+let exitEmitted = false
+worker.on('exit', function () {
+  exitEmitted = true
+})
+
+process.on('exit', function () {
+  assert.strictEqual(exitEmitted, true)
+})

package/test/commonjs-fallback.test.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const { test } = require('tap')
+const ThreadStream = require('..')
+
+const isYarnPnp = process.versions.pnp !== undefined
+
+test('yarn module resolution', { skip: !isYarnPnp }, t => {
+  t.plan(5)
+
+  const modulePath = require.resolve('pino-elasticsearch')
+  t.match(modulePath, /.*\.zip.*/)
+
+  const stream = new ThreadStream({
+    filename: modulePath,
+    workerData: { node: null },
+    sync: true
+  })
+
+  stream.on('error', (err) => {
+    t.pass('error emitted')
+    t.equal(err.message, 'Missing node(s) option', 'module custom error')
+  })
+
+  t.ok(stream.write('hello world\n'))
+  t.ok(stream.writable)
+
+  stream.end()
+})

package/test/error.js

@@ -0,0 +1,14 @@
+'use strict'
+
+const { Writable } = require('stream')
+
+async function run (opts) {
+  const stream = new Writable({
+    write (chunk, enc, cb) {
+      cb(new Error('kaboom'))
+    }
+  })
+  return stream
+}
+
+module.exports = run

package/test/esm.test.mjs

@@ -1,29 +1,9 @@
 import { test } from 'tap'
-import { tmpdir } from 'os'
-import { unlinkSync, readFile } from 'fs'
+import { readFile } from 'fs'
 import ThreadStream from '../index.js'
 import { join } from 'desm'
-import path from 'path'
 import { pathToFileURL } from 'url'
-
-const files = []
-let count = 0
-
-function file () {
-  const file = path.join(tmpdir(), `thread-stream-${process.pid}-${process.hrtime().toString()}-${count++}`)
-  files.push(file)
-  return file
-}
-
-process.on('beforeExit', () => {
-  for (const file of files) {
-    try {
-      unlinkSync(file)
-    } catch (e) {
-      console.log(e)
-    }
-  }
-})
+import { file } from './helper.js'
 
 function basic (text, filename) {
   test(text, function (t) {

package/test/helper.js

@@ -3,24 +3,31 @@
 const { join } = require('path')
 const { tmpdir } = require('os')
 const { unlinkSync } = require('fs')
+const why = require('why-is-node-running')
+const t = require('tap')
 
 const files = []
 let count = 0
 
 function file () {
-  const file = join(tmpdir(), `thread-stream-${process.pid}-${process.hrtime().toString()}-${count++}`)
+  const file = join(tmpdir(), `thread-stream-${process.pid}-${count++}`)
   files.push(file)
   return file
 }
 
 process.on('beforeExit', () => {
+  t.comment('unlink files')
   for (const file of files) {
     try {
+      t.comment(`unliking ${file}`)
       unlinkSync(file)
     } catch (e) {
       console.log(e)
     }
   }
+  t.comment('unlink completed')
 })
 
 module.exports.file = file
+
+setInterval(why, 10000).unref()

package/test/string-limit.test.js

@@ -0,0 +1,40 @@
+'use strict'
+
+const t = require('tap')
+const { join } = require('path')
+const { file } = require('./helper')
+const { stat } = require('fs')
+const ThreadStream = require('..')
+
+t.setTimeout(30000)
+
+const dest = file()
+const stream = new ThreadStream({
+  filename: join(__dirname, 'to-file.js'),
+  workerData: { dest },
+  sync: false
+})
+
+let length = 0
+
+stream.on('ready', () => {
+  t.pass('ready emitted')
+
+  const buf = Buffer.alloc(1024).fill('x').toString() // 1 KB
+
+  // This writes 1 GB of data
+  for (let i = 0; i < 1024 * 1024; i++) {
+    length += buf.length
+    stream.write(buf)
+  }
+
+  stream.end()
+})
+
+stream.on('close', () => {
+  stat(dest, (err, f) => {
+    t.error(err)
+    t.equal(f.size, length)
+    t.end()
+  })
+})

package/test/thread-management.test.js

@@ -86,7 +86,7 @@ test('emit error if thread have unhandledRejection', async function (t) {
   })
 
   const [err] = await once(stream, 'error')
-  t.equal(err.message, 'The worker thread exited')
+  t.equal(err.message, 'kaboom')
 
   try {
     stream.write('noop')
@@ -102,3 +102,72 @@ test('emit error if thread have unhandledRejection', async function (t) {
     t.equal(err.message, 'the worker has exited')
   }
 })
+
+test('emit error if worker stream emit error', async function (t) {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'error.js'),
+    sync: true
+  })
+
+  stream.on('ready', function () {
+    stream.write('hello world\n')
+  })
+
+  const [err] = await once(stream, 'error')
+  t.equal(err.message, 'kaboom')
+
+  try {
+    stream.write('noop')
+    t.fail('unreacheable')
+  } catch (err) {
+    t.equal(err.message, 'the worker has exited')
+  }
+
+  try {
+    stream.write('noop')
+    t.fail('unreacheable')
+  } catch (err) {
+    t.equal(err.message, 'the worker has exited')
+  }
+})
+
+test('emit error if thread have uncaughtException', async function (t) {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'uncaughtException.js'),
+    sync: true
+  })
+
+  stream.on('ready', function () {
+    stream.write('hello world\n')
+  })
+
+  const [err] = await once(stream, 'error')
+  t.equal(err.message, 'kaboom')
+
+  try {
+    stream.write('noop')
+    t.fail('unreacheable')
+  } catch (err) {
+    t.equal(err.message, 'the worker has exited')
+  }
+
+  try {
+    stream.write('noop')
+    t.fail('unreacheable')
+  } catch (err) {
+    t.equal(err.message, 'the worker has exited')
+  }
+})
+
+test('close the work if out of scope on gc', { skip: !global.WeakRef }, async function (t) {
+  const dest = file()
+  const child = fork(join(__dirname, 'close-on-gc.js'), [dest], {
+    execArgv: ['--expose-gc']
+  })
+
+  const [code] = await once(child, 'exit')
+  t.equal(code, 0)
+
+  const data = await readFile(dest, 'utf8')
+  t.equal(data, 'hello world\n')
+})

package/test/uncaughtException.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const { Writable } = require('stream')
+
+// Nop console.error to avoid printing things out
+console.error = () => {}
+
+setImmediate(function () {
+  throw new Error('kaboom')
+})
+
+async function run (opts) {
+  const stream = new Writable({
+    write (chunk, enc, cb) {
+      cb()
+    }
+  })
+  return stream
+}
+
+module.exports = run