thread-stream 0.11.1 → 0.13.0

package/package.json

@@ -1,18 +1,22 @@
 {
   "name": "thread-stream",
-  "version": "0.11.1",
+  "version": "0.13.0",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
+  "dependencies": {
+    "real-require": "^0.1.0"
+  },
   "devDependencies": {
     "desm": "^1.1.0",
     "fastbench": "^1.0.1",
     "husky": "^7.0.0",
     "sonic-boom": "^2.0.1",
     "standard": "^16.0.3",
-    "tap": "^15.0.0"
+    "tap": "^15.0.0",
+    "why-is-node-running": "^2.2.0"
   },
   "scripts": {
-    "test": "standard && tap --no-check-coverage test/*.test.*js",
+    "test": "standard && tap test/*.test.*js",
     "test:ci": "standard && tap \"test/**/*.test.*js\" --no-check-coverage --coverage-report=lcovonly",
     "test:yarn": "tap \"test/**/*.test.js\" --no-check-coverage",
     "prepare": "husky install"

package/test/base.test.js

@@ -9,7 +9,7 @@ const { MessageChannel } = require('worker_threads')
 const { once } = require('events')
 
 test('base sync=true', function (t) {
-  t.plan(9)
+  t.plan(15)
 
   const dest = file()
   const stream = new ThreadStream({
@@ -18,35 +18,36 @@ test('base sync=true', function (t) {
     sync: true
   })
 
-  stream.on('drain', () => {
-    t.pass('drain')
-  })
-
-  stream.on('ready', () => {
-    t.pass('ready emitted')
-  })
-
-  t.ok(stream.write('hello world\n'))
-  t.ok(stream.write('something else\n'))
-  t.ok(stream.writable)
-
-  stream.end()
+  t.same(stream.writableObjectMode, false)
 
+  t.same(stream.writableFinished, false)
   stream.on('finish', () => {
+    t.same(stream.writableFinished, true)
     readFile(dest, 'utf8', (err, data) => {
       t.error(err)
       t.equal(data, 'hello world\nsomething else\n')
     })
   })
 
+  t.same(stream.closed, false)
   stream.on('close', () => {
+    t.same(stream.closed, true)
     t.notOk(stream.writable)
     t.pass('close emitted')
   })
+
+  t.same(stream.writableNeedDrain, false)
+  t.ok(stream.write('hello world\n'))
+  t.ok(stream.write('something else\n'))
+  t.ok(stream.writable)
+
+  t.same(stream.writableEnded, false)
+  stream.end()
+  t.same(stream.writableEnded, true)
 })
 
 test('overflow sync=true', function (t) {
-  t.plan(4)
+  t.plan(3)
 
   const dest = file()
   const stream = new ThreadStream({
@@ -56,11 +57,6 @@ test('overflow sync=true', function (t) {
     sync: true
   })
 
-  stream.on('ready', () => {
-    t.pass('ready emitted')
-    write()
-  })
-
   let count = 0
 
   // Write 10 chars, 20 times
@@ -75,6 +71,8 @@ test('overflow sync=true', function (t) {
     setImmediate(write)
   }
 
+  write()
+
   stream.on('finish', () => {
     t.pass('finish emitted')
   })
@@ -96,13 +94,10 @@ test('overflow sync=false', function (t) {
     sync: false
   })
 
-  stream.on('ready', () => {
-    t.pass('ready emitted')
-    write()
-  })
-
   let count = 0
 
+  t.same(stream.writableNeedDrain, false)
+
   // Write 10 chars, 20 times
   function write () {
     if (count++ === 20) {
@@ -111,12 +106,17 @@ test('overflow sync=false', function (t) {
       return
     }
 
-    stream.write('aaaaaaaaaa')
+    if (!stream.write('aaaaaaaaaa')) {
+      t.same(stream.writableNeedDrain, true)
+    }
     // do not wait for drain event
     setImmediate(write)
   }
 
+  write()
+
   stream.on('drain', () => {
+    t.same(stream.writableNeedDrain, false)
     t.pass('drain')
   })
 
@@ -134,7 +134,7 @@ test('overflow sync=false', function (t) {
 })
 
 test('over the bufferSize at startup', function (t) {
-  t.plan(8)
+  t.plan(6)
 
   const dest = file()
   const stream = new ThreadStream({
@@ -144,20 +144,6 @@ test('over the bufferSize at startup', function (t) {
     sync: true
   })
 
-  stream.on('drain', () => {
-    t.pass('drain')
-  })
-
-  stream.on('ready', () => {
-    t.pass('ready emitted')
-  })
-
-  t.ok(stream.write('hello'))
-  t.notOk(stream.write(' world\n'))
-  t.notOk(stream.write('something else\n'))
-
-  stream.end()
-
   stream.on('finish', () => {
     readFile(dest, 'utf8', (err, data) => {
       t.error(err)
@@ -168,10 +154,16 @@ test('over the bufferSize at startup', function (t) {
   stream.on('close', () => {
     t.pass('close emitted')
   })
+
+  t.ok(stream.write('hello'))
+  t.ok(stream.write(' world\n'))
+  t.ok(stream.write('something else\n'))
+
+  stream.end()
 })
 
 test('over the bufferSize at startup (async)', function (t) {
-  t.plan(8)
+  t.plan(6)
 
   const dest = file()
   const stream = new ThreadStream({
@@ -181,14 +173,6 @@ test('over the bufferSize at startup (async)', function (t) {
     sync: false
   })
 
-  stream.on('drain', () => {
-    t.pass('drain')
-  })
-
-  stream.on('ready', () => {
-    t.pass('ready emitted')
-  })
-
   t.ok(stream.write('hello'))
   t.notOk(stream.write(' world\n'))
   t.notOk(stream.write('something else\n'))
@@ -216,15 +200,6 @@ test('flushSync sync=false', function (t) {
     sync: false
   })
 
-  stream.on('ready', () => {
-    t.pass('ready emitted')
-
-    for (let count = 0; count < 20; count++) {
-      stream.write('aaaaaaaaaa')
-    }
-    stream.flushSync()
-  })
-
   stream.on('drain', () => {
     t.pass('drain')
     stream.end()
@@ -241,50 +216,11 @@ test('flushSync sync=false', function (t) {
       t.end()
     })
   })
-})
-
-test('flushSync on ready if sync=true', function (t) {
-  t.plan(4)
-
-  const dest = file()
-  const stream = new ThreadStream({
-    filename: join(__dirname, 'to-file.js'),
-    workerData: { dest },
-    sync: true
-  })
-
-  stream.on('ready', () => {
-    readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data, 'hello world\nsomething else\n')
-      stream.end()
-    })
-  })
 
-  t.ok(stream.write('hello world\n'))
-  t.ok(stream.write('something else\n'))
-})
-
-test('flush on ready if sync=false', function (t) {
-  t.plan(4)
-
-  const dest = file()
-  const stream = new ThreadStream({
-    filename: join(__dirname, 'to-file.js'),
-    workerData: { dest },
-    sync: false
-  })
-
-  stream.on('ready', () => {
-    readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data, 'hello world\nsomething else\n')
-      stream.end()
-    })
-  })
-
-  t.ok(stream.write('hello world\n'))
-  t.ok(stream.write('something else\n'))
+  for (let count = 0; count < 20; count++) {
+    stream.write('aaaaaaaaaa')
+  }
+  stream.flushSync()
 })
 
 test('pass down MessagePorts', async function (t) {

package/test/bundlers.test.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const { test } = require('tap')
+const { join } = require('path')
+const { file } = require('./helper')
+const ThreadStream = require('..')
+
+test('bundlers support', function (t) {
+  t.plan(1)
+
+  globalThis.__bundlerPathsOverrides = {
+    'thread-stream-worker': join(__dirname, 'custom-worker.js')
+  }
+
+  const dest = file()
+
+  process.on('uncaughtException', error => {
+    console.log(error)
+  })
+
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: true
+  })
+
+  stream.worker.removeAllListeners('message')
+  stream.worker.once('message', message => {
+    t.equal(message.code, 'CUSTOM-WORKER-CALLED')
+  })
+
+  stream.end()
+})

package/test/close-on-gc.js

@@ -15,16 +15,14 @@ function setup () {
 
   worker = stream.worker
 
-  stream.on('ready', function () {
-    stream.write('hello')
-    stream.write(' ')
-    stream.write('world\n')
-    stream.flushSync()
-    stream.unref()
-
-    // the stream object goes out of scope here
-    setImmediate(gc) // eslint-disable-line
-  })
+  stream.write('hello')
+  stream.write(' ')
+  stream.write('world\n')
+  stream.flushSync()
+  stream.unref()
+
+  // the stream object goes out of scope here
+  setImmediate(gc) // eslint-disable-line
 }
 
 setup()

package/test/commonjs-fallback.test.js

@@ -6,7 +6,7 @@ const ThreadStream = require('..')
 const isYarnPnp = process.versions.pnp !== undefined
 
 test('yarn module resolution', { skip: !isYarnPnp }, t => {
-  t.plan(5)
+  t.plan(6)
 
   const modulePath = require.resolve('pino-elasticsearch')
   t.match(modulePath, /.*\.zip.*/)
@@ -17,13 +17,13 @@ test('yarn module resolution', { skip: !isYarnPnp }, t => {
     sync: true
   })
 
+  t.same(stream.writableErrored, null)
   stream.on('error', (err) => {
+    t.same(stream.writableErrored, err)
     t.pass('error emitted')
-    t.equal(err.message, 'Missing node(s) option', 'module custom error')
   })
 
   t.ok(stream.write('hello world\n'))
   t.ok(stream.writable)
-
   stream.end()
 })

package/test/create-and-exit.js

@@ -9,10 +9,8 @@ const stream = new ThreadStream({
   sync: true
 })
 
-stream.on('ready', function () {
-  stream.write('hello')
-  stream.write(' ')
-  stream.write('world\n')
-  stream.flushSync()
-  stream.unref()
-})
+stream.write('hello')
+stream.write(' ')
+stream.write('world\n')
+stream.flushSync()
+stream.unref()

package/test/custom-worker.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const { parentPort } = require('worker_threads')
+
+parentPort.postMessage({
+  code: 'CUSTOM-WORKER-CALLED'
+})
+
+require('../lib/worker')

package/test/end.test.js

@@ -7,7 +7,7 @@ const { file } = require('./helper')
 const ThreadStream = require('..')
 
 test('destroy support', function (t) {
-  t.plan(9)
+  t.plan(7)
 
   const dest = file()
   const stream = new ThreadStream({
@@ -16,33 +16,25 @@ test('destroy support', function (t) {
     sync: true
   })
 
-  stream.on('drain', () => {
-    t.pass('drain')
+  stream.on('close', () => {
+    t.notOk(stream.writable)
+    t.pass('close emitted')
   })
 
-  stream.on('ready', () => {
-    t.pass('ready emitted')
+  t.ok(stream.write('hello world\n'))
+  t.ok(stream.write('something else\n'))
+  t.ok(stream.writable)
 
-    t.ok(stream.write('hello world\n'))
-    t.ok(stream.write('something else\n'))
-    t.ok(stream.writable)
+  stream.end()
 
-    stream.end()
-
-    readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data, 'hello world\nsomething else\n')
-    })
-  })
-
-  stream.on('close', () => {
-    t.notOk(stream.writable)
-    t.pass('close emitted')
+  readFile(dest, 'utf8', (err, data) => {
+    t.error(err)
+    t.equal(data, 'hello world\nsomething else\n')
   })
 })
 
 test('synchronous _final support', function (t) {
-  t.plan(9)
+  t.plan(7)
 
   const dest = file()
   const stream = new ThreadStream({
@@ -51,27 +43,19 @@ test('synchronous _final support', function (t) {
     sync: true
   })
 
-  stream.on('drain', () => {
-    t.pass('drain')
+  stream.on('close', () => {
+    t.notOk(stream.writable)
+    t.pass('close emitted')
   })
 
-  stream.on('ready', () => {
-    t.pass('ready emitted')
+  t.ok(stream.write('hello world\n'))
+  t.ok(stream.write('something else\n'))
+  t.ok(stream.writable)
 
-    t.ok(stream.write('hello world\n'))
-    t.ok(stream.write('something else\n'))
-    t.ok(stream.writable)
+  stream.end()
 
-    stream.end()
-
-    readFile(dest, 'utf8', (err, data) => {
-      t.error(err)
-      t.equal(data, 'hello world\nsomething else\n')
-    })
-  })
-
-  stream.on('close', () => {
-    t.notOk(stream.writable)
-    t.pass('close emitted')
+  readFile(dest, 'utf8', (err, data) => {
+    t.error(err)
+    t.equal(data, 'hello world\nsomething else\n')
   })
 })

package/test/esm.test.mjs

@@ -1,33 +1,13 @@
 import { test } from 'tap'
-import { tmpdir } from 'os'
-import { unlinkSync, readFile } from 'fs'
+import { readFile } from 'fs'
 import ThreadStream from '../index.js'
 import { join } from 'desm'
-import path from 'path'
 import { pathToFileURL } from 'url'
-
-const files = []
-let count = 0
-
-function file () {
-  const file = path.join(tmpdir(), `thread-stream-${process.pid}-${process.hrtime().toString()}-${count++}`)
-  files.push(file)
-  return file
-}
-
-process.on('beforeExit', () => {
-  for (const file of files) {
-    try {
-      unlinkSync(file)
-    } catch (e) {
-      console.log(e)
-    }
-  }
-})
+import { file } from './helper.js'
 
 function basic (text, filename) {
   test(text, function (t) {
-    t.plan(7)
+    t.plan(5)
 
     const dest = file()
     const stream = new ThreadStream({
@@ -36,19 +16,6 @@ function basic (text, filename) {
       sync: true
     })
 
-    stream.on('drain', () => {
-      t.pass('drain')
-    })
-
-    stream.on('ready', () => {
-      t.pass('ready emitted')
-    })
-
-    t.ok(stream.write('hello world\n'))
-    t.ok(stream.write('something else\n'))
-
-    stream.end()
-
     stream.on('finish', () => {
       readFile(dest, 'utf8', (err, data) => {
         t.error(err)
@@ -59,6 +26,11 @@ function basic (text, filename) {
     stream.on('close', () => {
       t.pass('close emitted')
     })
+
+    t.ok(stream.write('hello world\n'))
+    t.ok(stream.write('something else\n'))
+
+    stream.end()
   })
 }
 

package/test/helper.js

@@ -3,24 +3,31 @@
 const { join } = require('path')
 const { tmpdir } = require('os')
 const { unlinkSync } = require('fs')
+const why = require('why-is-node-running')
+const t = require('tap')
 
 const files = []
 let count = 0
 
 function file () {
-  const file = join(tmpdir(), `thread-stream-${process.pid}-${process.hrtime().toString()}-${count++}`)
+  const file = join(tmpdir(), `thread-stream-${process.pid}-${count++}`)
   files.push(file)
   return file
 }
 
 process.on('beforeExit', () => {
+  t.comment('unlink files')
   for (const file of files) {
     try {
+      t.comment(`unliking ${file}`)
       unlinkSync(file)
     } catch (e) {
       console.log(e)
     }
   }
+  t.comment('unlink completed')
 })
 
 module.exports.file = file
+
+setInterval(why, 10000).unref()

package/test/multibyte-chars.test.mjs

@@ -16,17 +16,15 @@ test('break up utf8 multibyte (sync)', (t) => {
     sync: true
   })
 
-  stream.on('ready', function () {
-    stream.write(longString)
-    stream.end()
-  })
-
   stream.on('finish', () => {
     readFile(dest, 'utf8', (err, data) => {
       t.error(err)
       t.equal(data, longString)
     })
   })
+
+  stream.write(longString)
+  stream.end()
 })
 
 test('break up utf8 multibyte (async)', (t) => {
@@ -41,15 +39,13 @@ test('break up utf8 multibyte (async)', (t) => {
     sync: false
   })
 
-  stream.on('ready', function () {
-    stream.write(longString)
-    stream.end()
-  })
-
   stream.on('finish', () => {
     readFile(dest, 'utf8', (err, data) => {
       t.error(err)
       t.equal(data, longString)
     })
   })
+
+  stream.write(longString)
+  stream.end()
 })

package/test/string-limit.test.js

@@ -0,0 +1,36 @@
+'use strict'
+
+const t = require('tap')
+const { join } = require('path')
+const { file } = require('./helper')
+const { stat } = require('fs')
+const ThreadStream = require('..')
+
+t.setTimeout(30000)
+
+const dest = file()
+const stream = new ThreadStream({
+  filename: join(__dirname, 'to-file.js'),
+  workerData: { dest },
+  sync: false
+})
+
+let length = 0
+
+stream.on('close', () => {
+  stat(dest, (err, f) => {
+    t.error(err)
+    t.equal(f.size, length)
+    t.end()
+  })
+})
+
+const buf = Buffer.alloc(1024).fill('x').toString() // 1 KB
+
+// This writes 1 GB of data
+for (let i = 0; i < 1024 * 1024; i++) {
+  length += buf.length
+  stream.write(buf)
+}
+
+stream.end()

package/test/thread-management.test.js

@@ -25,7 +25,7 @@ test('emit error if thread exits', async function (t) {
     sync: true
   })
 
-  stream.on('ready', function () {
+  stream.on('ready', () => {
     stream.write('hello world\n')
   })
 
@@ -53,7 +53,7 @@ test('emit error if thread exits', async function (t) {
     sync: true
   })
 
-  stream.on('ready', function () {
+  stream.on('ready', () => {
     stream.write('hello world\n')
   })
 
@@ -81,7 +81,7 @@ test('emit error if thread have unhandledRejection', async function (t) {
     sync: true
   })
 
-  stream.on('ready', function () {
+  stream.on('ready', () => {
     stream.write('hello world\n')
   })
 
@@ -109,7 +109,7 @@ test('emit error if worker stream emit error', async function (t) {
     sync: true
   })
 
-  stream.on('ready', function () {
+  stream.on('ready', () => {
     stream.write('hello world\n')
   })
 
@@ -137,7 +137,7 @@ test('emit error if thread have uncaughtException', async function (t) {
     sync: true
   })
 
-  stream.on('ready', function () {
+  stream.on('ready', () => {
     stream.write('hello world\n')
   })