thread-stream 0.12.1 → 0.13.2

package/.github/dependabot.yml

@@ -2,6 +2,10 @@ version: 2
 updates:
 - package-ecosystem: github-actions
   directory: '/'
+  ignore:
+      - dependency-name: 'actions/*'
+        update-types:
+            ['version-update:semver-minor', 'version-update:semver-patch']
   schedule:
     interval: daily
   open-pull-requests-limit: 10

package/.github/workflows/ci.yml

@@ -19,10 +19,10 @@ jobs:
 
     steps:
 
-      - uses: actions/checkout@v2.3.5
+      - uses: actions/checkout@v2
 
       - name: Use Node.js
-        uses: actions/setup-node@v2.4.1
+        uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
 
@@ -55,6 +55,6 @@ jobs:
     needs: test
     runs-on: ubuntu-latest
     steps:
-      - uses: fastify/github-action-merge-dependabot@v2.4.0
+      - uses: fastify/github-action-merge-dependabot@v2.7.1
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

package/.github/workflows/package-manager-ci.yml

@@ -17,13 +17,13 @@ jobs:
         os: [macOS-latest, windows-latest]
         node-version: [12, 14, 16]
     steps:
-      - uses: actions/checkout@v2.3.5
+      - uses: actions/checkout@v2
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2.4.1
+        uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
       - name: Use pnpm
-        uses: pnpm/action-setup@v2.0.1
+        uses: pnpm/action-setup@v2.2.1
         with:
           version: ^6.0.0
       - name: Install dependancies
@@ -39,9 +39,9 @@ jobs:
         os: [macOS-latest]
         node-version: [12, 14, 16]
     steps:
-      - uses: actions/checkout@v2.3.5
+      - uses: actions/checkout@v2
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2.4.1
+        uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
       - name: Use yarn

package/index.js

@@ -44,7 +44,8 @@ const registry = new FinalizationRegistry((worker) => {
 function createWorker (stream, opts) {
   const { filename, workerData } = opts
 
-  const toExecute = join(__dirname, 'lib', 'worker.js')
+  const bundlerOverrides = '__bundlerPathsOverrides' in globalThis ? globalThis.__bundlerPathsOverrides : {}
+  const toExecute = bundlerOverrides['thread-stream-worker'] || join(__dirname, 'lib', 'worker.js')
 
   const worker = new Worker(toExecute, {
     ...opts.workerOpts,
@@ -109,7 +110,7 @@ function nextFlush (stream) {
         // Find a toWrite length that fits the buffer
         // it must exists as the buffer is at least 4 bytes length
         // and the max utf-8 length for a char is 4 bytes.
-        while (toWriteBytes > stream[kImpl].buf.length) {
+        while (toWriteBytes > stream[kImpl].data.length) {
           leftover = leftover / 2
           toWrite = stream[kImpl].buf.slice(0, leftover)
           toWriteBytes = Buffer.byteLength(toWrite)

package/lib/worker.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const { realImport, realRequire } = require('real-require')
 const { workerData, parentPort } = require('worker_threads')
 const { WRITE_INDEX, READ_INDEX } = require('./indexes')
 const { waitDiff } = require('./wait')
@@ -17,7 +18,7 @@ const data = Buffer.from(dataBuf)
 async function start () {
   let fn
   try {
-    fn = (await import(workerData.filename)).default
+    fn = (await realImport(workerData.filename)).default
   } catch (error) {
     // A yarn user that tries to start a ThreadStream for an external module
     // provides a filename pointing to a zip file.
@@ -29,7 +30,7 @@ async function start () {
     // The error codes may change based on the node.js version (ENOTDIR > 12, ERR_MODULE_NOT_FOUND <= 12 )
     if ((error.code === 'ENOTDIR' || error.code === 'ERR_MODULE_NOT_FOUND') &&
      workerData.filename.startsWith('file://')) {
-      fn = require(workerData.filename.replace('file://', ''))
+      fn = realRequire(decodeURIComponent(workerData.filename.replace('file://', '')))
     } else {
       throw error
     }

package/package.json

@@ -1,8 +1,11 @@
 {
   "name": "thread-stream",
-  "version": "0.12.1",
+  "version": "0.13.2",
   "description": "A streaming way to send data to a Node.js Worker Thread",
   "main": "index.js",
+  "dependencies": {
+    "real-require": "^0.1.0"
+  },
   "devDependencies": {
     "desm": "^1.1.0",
     "fastbench": "^1.0.1",

package/test/bundlers.test.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const { test } = require('tap')
+const { join } = require('path')
+const { file } = require('./helper')
+const ThreadStream = require('..')
+
+test('bundlers support', function (t) {
+  t.plan(1)
+
+  globalThis.__bundlerPathsOverrides = {
+    'thread-stream-worker': join(__dirname, 'custom-worker.js')
+  }
+
+  const dest = file()
+
+  process.on('uncaughtException', error => {
+    console.log(error)
+  })
+
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: true
+  })
+
+  stream.worker.removeAllListeners('message')
+  stream.worker.once('message', message => {
+    t.equal(message.code, 'CUSTOM-WORKER-CALLED')
+  })
+
+  stream.end()
+})

package/test/commonjs-fallback.test.js

@@ -1,6 +1,9 @@
 'use strict'
 
 const { test } = require('tap')
+const { join } = require('path')
+const { MessageChannel } = require('worker_threads')
+const { once } = require('events')
 const ThreadStream = require('..')
 
 const isYarnPnp = process.versions.pnp !== undefined
@@ -27,3 +30,27 @@ test('yarn module resolution', { skip: !isYarnPnp }, t => {
   t.ok(stream.writable)
   stream.end()
 })
+
+test('yarn module resolution for directories with special characters', { skip: !isYarnPnp }, async t => {
+  t.plan(3)
+
+  const { port1, port2 } = new MessageChannel()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'dir with spaces', 'test-package.zip', 'worker.js'),
+    workerData: { port: port1 },
+    workerOpts: {
+      transferList: [port1]
+    },
+    sync: false
+  })
+  t.teardown(() => {
+    stream.end()
+  })
+
+  t.ok(stream.write('hello world\n'))
+  t.ok(stream.write('something else\n'))
+
+  const [strings] = await once(port2, 'message')
+
+  t.equal(strings, 'hello world\nsomething else\n')
+})

package/test/custom-worker.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const { parentPort } = require('worker_threads')
+
+parentPort.postMessage({
+  code: 'CUSTOM-WORKER-CALLED'
+})
+
+require('../lib/worker')

package/test/multibyte-chars.test.mjs

@@ -49,3 +49,26 @@ test('break up utf8 multibyte (async)', (t) => {
   stream.write(longString)
   stream.end()
 })
+
+test('break up utf8 multibyte several times bigger than write buffer', (t) => {
+  t.plan(2)
+  const longString = '\u03A3'.repeat(32)
+
+  const dest = file()
+  const stream = new ThreadStream({
+    bufferSize: 15, // this must be odd
+    filename: join(import.meta.url, 'to-file.js'),
+    workerData: { dest },
+    sync: false
+  })
+
+  stream.on('finish', () => {
+    readFile(dest, 'utf8', (err, data) => {
+      t.error(err)
+      t.equal(data, longString)
+    })
+  })
+
+  stream.write(longString)
+  stream.end()
+})

package/test/string-limit-2.test.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const t = require('tap')
+const { join } = require('path')
+const { file } = require('./helper')
+const { createReadStream } = require('fs')
+const ThreadStream = require('..')
+const buffer = require('buffer')
+
+const MAX_STRING = buffer.constants.MAX_STRING_LENGTH
+
+t.plan(1)
+
+const dest = file()
+const stream = new ThreadStream({
+  filename: join(__dirname, 'to-file.js'),
+  workerData: { dest },
+  sync: false
+})
+
+stream.on('close', async () => {
+  t.comment('close emitted')
+  let buf
+  for await (const chunk of createReadStream(dest)) {
+    buf = chunk
+  }
+  t.equal('asd', buf.toString().slice(-3))
+})
+
+stream.on('ready', () => {
+  t.comment('open emitted')
+  stream.write('a'.repeat(MAX_STRING - 2))
+  stream.write('asd')
+  stream.end()
+})