thread-stream 0.11.1 → 0.11.2

package/.github/workflows/ci.yml

@@ -19,10 +19,10 @@ jobs:
 
     steps:
 
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v2.3.5
 
       - name: Use Node.js
-        uses: actions/setup-node@v2.4.0
+        uses: actions/setup-node@v2.4.1
         with:
           node-version: ${{ matrix.node-version }}
 

package/.github/workflows/package-manager-ci.yml

@@ -17,9 +17,9 @@ jobs:
         os: [macOS-latest, windows-latest]
         node-version: [12, 14, 16]
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v2.3.5
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2.4.0
+        uses: actions/setup-node@v2.4.1
         with:
           node-version: ${{ matrix.node-version }}
       - name: Use pnpm
@@ -39,9 +39,9 @@ jobs:
         os: [macOS-latest]
         node-version: [12, 14, 16]
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v2.3.5
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2.4.0
+        uses: actions/setup-node@v2.4.1
         with:
           node-version: ${{ matrix.node-version }}
       - name: Use yarn

package/.taprc

@@ -0,0 +1,2 @@
+jobs: 1
+check-coverage: false

package/index.js

@@ -9,6 +9,10 @@ const {
   WRITE_INDEX,
   READ_INDEX
 } = require('./lib/indexes')
+const buffer = require('buffer')
+
+// V8 limit for string size
+const MAX_STRING = buffer.constants.MAX_STRING_LENGTH
 
 class FakeWeakRef {
   constructor (value) {
@@ -28,6 +32,9 @@ const FinalizationRegistry = global.FinalizationRegistry || class FakeFinalizati
 const WeakRef = global.WeakRef || FakeWeakRef
 
 const registry = new FinalizationRegistry((worker) => {
+  if (worker.exited) {
+    return
+  }
   worker.terminate()
 })
 
@@ -122,6 +129,7 @@ function nextFlush (stream) {
 function onWorkerMessage (msg) {
   const stream = this.stream.deref()
   if (stream === undefined) {
+    this.exited = true
     // Terminate the worker.
     this.terminate()
     return
@@ -148,6 +156,7 @@ function onWorkerMessage (msg) {
       break
     case 'ERROR':
       stream.closed = true
+      stream.worker.exited = true
       // TODO only remove our own
       stream.worker.removeAllListeners('exit')
       stream.worker.terminate().then(null, () => {})
@@ -168,6 +177,7 @@ function onWorkerExit (code) {
   }
   registry.unregister(stream)
   stream.closed = true
+  stream.worker.exited = true
   setImmediate(function () {
     if (code !== 0) {
       stream.emit('error', new Error('The worker thread exited'))
@@ -219,6 +229,12 @@ class ThreadStream extends EventEmitter {
       throw new Error('the worker has exited')
     }
 
+    if (this.flushing && this.buf.length + data.length >= MAX_STRING) {
+      // process._rawDebug('write: flushing')
+      this._writeSync()
+      this.flushing = true // we are still flushing
+    }
+
     if (!this.ready || this.flushing) {
       this.buf += data
       return this._hasSpace()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "thread-stream",
-  "version": "0.11.1",
+  "version": "0.11.2",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
   "devDependencies": {
@@ -9,10 +9,11 @@
     "husky": "^7.0.0",
     "sonic-boom": "^2.0.1",
     "standard": "^16.0.3",
-    "tap": "^15.0.0"
+    "tap": "^15.0.0",
+    "why-is-node-running": "^2.2.0"
   },
   "scripts": {
-    "test": "standard && tap --no-check-coverage test/*.test.*js",
+    "test": "standard && tap test/*.test.*js",
     "test:ci": "standard && tap \"test/**/*.test.*js\" --no-check-coverage --coverage-report=lcovonly",
     "test:yarn": "tap \"test/**/*.test.js\" --no-check-coverage",
     "prepare": "husky install"

package/test/esm.test.mjs

@@ -1,29 +1,9 @@
 import { test } from 'tap'
-import { tmpdir } from 'os'
-import { unlinkSync, readFile } from 'fs'
+import { readFile } from 'fs'
 import ThreadStream from '../index.js'
 import { join } from 'desm'
-import path from 'path'
 import { pathToFileURL } from 'url'
-
-const files = []
-let count = 0
-
-function file () {
-  const file = path.join(tmpdir(), `thread-stream-${process.pid}-${process.hrtime().toString()}-${count++}`)
-  files.push(file)
-  return file
-}
-
-process.on('beforeExit', () => {
-  for (const file of files) {
-    try {
-      unlinkSync(file)
-    } catch (e) {
-      console.log(e)
-    }
-  }
-})
+import { file } from './helper.js'
 
 function basic (text, filename) {
   test(text, function (t) {

package/test/helper.js

@@ -3,24 +3,31 @@
 const { join } = require('path')
 const { tmpdir } = require('os')
 const { unlinkSync } = require('fs')
+const why = require('why-is-node-running')
+const t = require('tap')
 
 const files = []
 let count = 0
 
 function file () {
-  const file = join(tmpdir(), `thread-stream-${process.pid}-${process.hrtime().toString()}-${count++}`)
+  const file = join(tmpdir(), `thread-stream-${process.pid}-${count++}`)
   files.push(file)
   return file
 }
 
 process.on('beforeExit', () => {
+  t.comment('unlink files')
   for (const file of files) {
     try {
+      t.comment(`unliking ${file}`)
       unlinkSync(file)
     } catch (e) {
       console.log(e)
     }
   }
+  t.comment('unlink completed')
 })
 
 module.exports.file = file
+
+setInterval(why, 10000).unref()

package/test/string-limit.test.js

@@ -0,0 +1,40 @@
+'use strict'
+
+const t = require('tap')
+const { join } = require('path')
+const { file } = require('./helper')
+const { stat } = require('fs')
+const ThreadStream = require('..')
+
+t.setTimeout(30000)
+
+const dest = file()
+const stream = new ThreadStream({
+  filename: join(__dirname, 'to-file.js'),
+  workerData: { dest },
+  sync: false
+})
+
+let length = 0
+
+stream.on('ready', () => {
+  t.pass('ready emitted')
+
+  const buf = Buffer.alloc(1024).fill('x').toString() // 1 KB
+
+  // This writes 1 GB of data
+  for (let i = 0; i < 1024 * 1024; i++) {
+    length += buf.length
+    stream.write(buf)
+  }
+
+  stream.end()
+})
+
+stream.on('close', () => {
+  stat(dest, (err, f) => {
+    t.error(err)
+    t.equal(f.size, length)
+    t.end()
+  })
+})