thread-stream 0.11.1 → 0.13.0

package/index.js

@@ -9,6 +9,13 @@ const {
   WRITE_INDEX,
   READ_INDEX
 } = require('./lib/indexes')
+const buffer = require('buffer')
+const assert = require('assert')
+
+const kImpl = Symbol('kImpl')
+
+// V8 limit for string size
+const MAX_STRING = buffer.constants.MAX_STRING_LENGTH
 
 class FakeWeakRef {
   constructor (value) {
@@ -28,13 +35,17 @@ const FinalizationRegistry = global.FinalizationRegistry || class FakeFinalizati
 const WeakRef = global.WeakRef || FakeWeakRef
 
 const registry = new FinalizationRegistry((worker) => {
+  if (worker.exited) {
+    return
+  }
   worker.terminate()
 })
 
 function createWorker (stream, opts) {
   const { filename, workerData } = opts
 
-  const toExecute = join(__dirname, 'lib', 'worker.js')
+  const bundlerOverrides = '__bundlerPathsOverrides' in globalThis ? globalThis.__bundlerPathsOverrides : {}
+  const toExecute = bundlerOverrides['thread-stream-worker'] || join(__dirname, 'lib', 'worker.js')
 
   const worker = new Worker(toExecute, {
     ...opts.workerOpts,
@@ -42,8 +53,8 @@ function createWorker (stream, opts) {
       filename: filename.indexOf('file://') === 0
         ? filename
         : pathToFileURL(filename).href,
-      dataBuf: stream._dataBuf,
-      stateBuf: stream._stateBuf,
+      dataBuf: stream[kImpl].dataBuf,
+      stateBuf: stream[kImpl].stateBuf,
       workerData
     }
   })
@@ -60,57 +71,62 @@ function createWorker (stream, opts) {
 }
 
 function drain (stream) {
-  stream.needDrain = false
-  stream.emit('drain')
+  assert(!stream[kImpl].sync)
+  if (stream[kImpl].needDrain) {
+    stream[kImpl].needDrain = false
+    stream.emit('drain')
+  }
 }
 
 function nextFlush (stream) {
-  const writeIndex = Atomics.load(stream._state, WRITE_INDEX)
-  let leftover = stream._data.length - writeIndex
+  const writeIndex = Atomics.load(stream[kImpl].state, WRITE_INDEX)
+  let leftover = stream[kImpl].data.length - writeIndex
 
   if (leftover > 0) {
-    if (stream.buf.length === 0) {
-      stream.flushing = false
-      if (!stream.needDrain) {
-        // process._rawDebug('emitting drain')
-        stream.needDrain = true
+    if (stream[kImpl].buf.length === 0) {
+      stream[kImpl].flushing = false
+
+      if (stream[kImpl].ending) {
+        end(stream)
+      } else if (stream[kImpl].needDrain) {
         process.nextTick(drain, stream)
       }
+
       return
     }
 
-    let toWrite = stream.buf.slice(0, leftover)
+    let toWrite = stream[kImpl].buf.slice(0, leftover)
     let toWriteBytes = Buffer.byteLength(toWrite)
     if (toWriteBytes <= leftover) {
-      stream.buf = stream.buf.slice(leftover)
+      stream[kImpl].buf = stream[kImpl].buf.slice(leftover)
       // process._rawDebug('writing ' + toWrite.length)
-      stream._write(toWrite, nextFlush.bind(null, stream))
+      write(stream, toWrite, nextFlush.bind(null, stream))
     } else {
       // multi-byte utf-8
       stream.flush(() => {
-        Atomics.store(stream._state, READ_INDEX, 0)
-        Atomics.store(stream._state, WRITE_INDEX, 0)
+        Atomics.store(stream[kImpl].state, READ_INDEX, 0)
+        Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
 
         // Find a toWrite length that fits the buffer
         // it must exists as the buffer is at least 4 bytes length
         // and the max utf-8 length for a char is 4 bytes.
-        while (toWriteBytes > stream.buf.length) {
+        while (toWriteBytes > stream[kImpl].buf.length) {
           leftover = leftover / 2
-          toWrite = stream.buf.slice(0, leftover)
+          toWrite = stream[kImpl].buf.slice(0, leftover)
           toWriteBytes = Buffer.byteLength(toWrite)
         }
-        stream.buf = stream.buf.slice(leftover)
-        stream._write(toWrite, nextFlush.bind(null, stream))
+        stream[kImpl].buf = stream[kImpl].buf.slice(leftover)
+        write(stream, toWrite, nextFlush.bind(null, stream))
       })
     }
   } else if (leftover === 0) {
-    if (writeIndex === 0 && stream.buf.length === 0) {
+    if (writeIndex === 0 && stream[kImpl].buf.length === 0) {
       // we had a flushSync in the meanwhile
       return
     }
     stream.flush(() => {
-      Atomics.store(stream._state, READ_INDEX, 0)
-      Atomics.store(stream._state, WRITE_INDEX, 0)
+      Atomics.store(stream[kImpl].state, READ_INDEX, 0)
+      Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
       nextFlush(stream)
     })
   } else {
@@ -122,6 +138,7 @@ function nextFlush (stream) {
 function onWorkerMessage (msg) {
   const stream = this.stream.deref()
   if (stream === undefined) {
+    this.exited = true
     // Terminate the worker.
     this.terminate()
     return
@@ -132,28 +149,14 @@ function onWorkerMessage (msg) {
       // Replace the FakeWeakRef with a
       // proper one.
       this.stream = new WeakRef(stream)
-      if (stream._sync) {
-        stream.ready = true
-        stream.flushSync()
+
+      stream.flush(() => {
+        stream[kImpl].ready = true
         stream.emit('ready')
-      } else {
-        stream.once('drain', function () {
-          stream.flush(() => {
-            stream.ready = true
-            stream.emit('ready')
-          })
-        })
-        nextFlush(stream)
-      }
+      })
       break
     case 'ERROR':
-      stream.closed = true
-      // TODO only remove our own
-      stream.worker.removeAllListeners('exit')
-      stream.worker.terminate().then(null, () => {})
-      process.nextTick(() => {
-        stream.emit('error', msg.err)
-      })
+      destroy(stream, msg.err)
       break
     default:
       throw new Error('this should not happen: ' + msg.code)
@@ -167,13 +170,9 @@ function onWorkerExit (code) {
     return
   }
   registry.unregister(stream)
-  stream.closed = true
-  setImmediate(function () {
-    if (code !== 0) {
-      stream.emit('error', new Error('The worker thread exited'))
-    }
-    stream.emit('close')
-  })
+  stream.worker.exited = true
+  stream.worker.off('exit', onWorkerExit)
+  destroy(stream, code !== 0 ? new Error('The worker thread exited') : null)
 }
 
 class ThreadStream extends EventEmitter {
@@ -184,119 +183,88 @@ class ThreadStream extends EventEmitter {
       throw new Error('bufferSize must at least fit a 4-byte utf-8 char')
     }
 
-    this._stateBuf = new SharedArrayBuffer(128)
-    this._state = new Int32Array(this._stateBuf)
-    this._dataBuf = new SharedArrayBuffer(opts.bufferSize || 4 * 1024 * 1024)
-    this._data = Buffer.from(this._dataBuf)
-    this._sync = opts.sync || false
-    this.worker = createWorker(this, opts)
-    this.ready = false
-    this.ending = false
-    this.needDrain = false
-    this.closed = false
-
-    this.buf = ''
-  }
-
-  _write (data, cb) {
-    // data is smaller than the shared buffer length
-    const current = Atomics.load(this._state, WRITE_INDEX)
-    const length = Buffer.byteLength(data)
-    this._data.write(data, current)
-    Atomics.store(this._state, WRITE_INDEX, current + length)
-    Atomics.notify(this._state, WRITE_INDEX)
-    cb()
-    return true
-  }
-
-  _hasSpace () {
-    const current = Atomics.load(this._state, WRITE_INDEX)
-    return this._data.length - this.buf.length - current > 0
+    this[kImpl] = {}
+    this[kImpl].stateBuf = new SharedArrayBuffer(128)
+    this[kImpl].state = new Int32Array(this[kImpl].stateBuf)
+    this[kImpl].dataBuf = new SharedArrayBuffer(opts.bufferSize || 4 * 1024 * 1024)
+    this[kImpl].data = Buffer.from(this[kImpl].dataBuf)
+    this[kImpl].sync = opts.sync || false
+    this[kImpl].ending = false
+    this[kImpl].ended = false
+    this[kImpl].needDrain = false
+    this[kImpl].destroyed = false
+    this[kImpl].flushing = false
+    this[kImpl].ready = false
+    this[kImpl].finished = false
+    this[kImpl].errored = null
+    this[kImpl].closed = false
+    this[kImpl].buf = ''
+
+    // TODO (fix): Make private?
+    this.worker = createWorker(this, opts) // TODO (fix): make private
   }
 
   write (data) {
-    if (this.closed) {
+    if (this[kImpl].destroyed) {
       throw new Error('the worker has exited')
     }
 
-    if (!this.ready || this.flushing) {
-      this.buf += data
-      return this._hasSpace()
+    if (this[kImpl].ending) {
+      throw new Error('the worker is ending')
     }
 
-    if (this._sync) {
-      this.buf += data
-      this._writeSync()
-
-      return true
-    }
-
-    this.buf = data
-    this.flushing = true
-    setImmediate(nextFlush, this)
-
-    return this._hasSpace()
-  }
-
-  end () {
-    if (this.closed) {
-      throw new Error('the worker has exited')
+    if (this[kImpl].flushing && this[kImpl].buf.length + data.length >= MAX_STRING) {
+      try {
+        writeSync(this)
+        this[kImpl].flushing = true
+      } catch (err) {
+        destroy(this, err)
+        return false
+      }
     }
 
-    if (!this.ready) {
-      this.once('ready', this.end.bind(this))
-      return
-    }
+    this[kImpl].buf += data
 
-    if (this.flushing) {
-      this.once('drain', this.end.bind(this))
-      return
+    if (this[kImpl].sync) {
+      try {
+        writeSync(this)
+        return true
+      } catch (err) {
+        destroy(this, err)
+        return false
+      }
     }
 
-    if (this.ending) {
-      return
+    if (!this[kImpl].flushing) {
+      this[kImpl].flushing = true
+      setImmediate(nextFlush, this)
     }
-    this.ending = true
-
-    this.flushSync()
-
-    let read = Atomics.load(this._state, READ_INDEX)
 
-    // process._rawDebug('writing index')
-    Atomics.store(this._state, WRITE_INDEX, -1)
-    // process._rawDebug(`(end) readIndex (${Atomics.load(this._state, READ_INDEX)}) writeIndex (${Atomics.load(this._state, WRITE_INDEX)})`)
-    Atomics.notify(this._state, WRITE_INDEX)
-
-    // Wait for the process to complete
-    let spins = 0
-    while (read !== -1) {
-      // process._rawDebug(`read = ${read}`)
-      Atomics.wait(this._state, READ_INDEX, read, 1000)
-      read = Atomics.load(this._state, READ_INDEX)
+    this[kImpl].needDrain = this[kImpl].data.length - this[kImpl].buf.length - Atomics.load(this[kImpl].state, WRITE_INDEX) <= 0
+    return !this[kImpl].needDrain
+  }
 
-      if (++spins === 10) {
-        throw new Error('end() took too long (10s)')
-      }
+  end () {
+    if (this[kImpl].destroyed) {
+      throw new Error('the worker has exited')
     }
 
-    process.nextTick(() => {
-      this.emit('finish')
-    })
-    // process._rawDebug('end finished...')
+    this[kImpl].ending = true
+    end(this)
   }
 
   flush (cb) {
-    if (this.closed) {
+    if (this[kImpl].destroyed) {
       throw new Error('the worker has exited')
     }
 
     // TODO write all .buf
-    const writeIndex = Atomics.load(this._state, WRITE_INDEX)
-    // process._rawDebug(`(flush) readIndex (${Atomics.load(this._state, READ_INDEX)}) writeIndex (${Atomics.load(this._state, WRITE_INDEX)})`)
-    wait(this._state, READ_INDEX, writeIndex, Infinity, (err, res) => {
+    const writeIndex = Atomics.load(this[kImpl].state, WRITE_INDEX)
+    // process._rawDebug(`(flush) readIndex (${Atomics.load(this.state, READ_INDEX)}) writeIndex (${Atomics.load(this.state, WRITE_INDEX)})`)
+    wait(this[kImpl].state, READ_INDEX, writeIndex, Infinity, (err, res) => {
       if (err) {
-        this.emit('error', err)
-        cb(err)
+        destroy(this, err)
+        process.nextTick(cb, err)
         return
       }
       if (res === 'not-equal') {
@@ -304,108 +272,224 @@ class ThreadStream extends EventEmitter {
         this.flush(cb)
         return
       }
-      cb()
+      process.nextTick(cb)
     })
   }
 
-  _writeSync () {
-    const cb = () => {
-      if (!this.needDrain) {
-        // process._rawDebug('emitting drain')
-        this.needDrain = true
-        process.nextTick(drain, this)
-      }
+  flushSync () {
+    if (this[kImpl].destroyed) {
+      throw new Error('the worker has exited')
     }
-    this.flushing = false
-
-    while (this.buf.length !== 0) {
-      const writeIndex = Atomics.load(this._state, WRITE_INDEX)
-      let leftover = this._data.length - writeIndex
-      if (leftover === 0) {
-        this._flushSync()
-        Atomics.store(this._state, READ_INDEX, 0)
-        Atomics.store(this._state, WRITE_INDEX, 0)
-        continue
-      } else if (leftover < 0) {
-        // This should never happen
-        throw new Error('overwritten')
-      }
 
-      let toWrite = this.buf.slice(0, leftover)
-      let toWriteBytes = Buffer.byteLength(toWrite)
-      if (toWriteBytes <= leftover) {
-        this.buf = this.buf.slice(leftover)
-        // process._rawDebug('writing ' + toWrite.length)
-        this._write(toWrite, cb)
-      } else {
-        // multi-byte utf-8
-        this._flushSync()
-        Atomics.store(this._state, READ_INDEX, 0)
-        Atomics.store(this._state, WRITE_INDEX, 0)
+    writeSync(this)
+    flushSync(this)
+  }
 
-        // Find a toWrite length that fits the buffer
-        // it must exists as the buffer is at least 4 bytes length
-        // and the max utf-8 length for a char is 4 bytes.
-        while (toWriteBytes > this.buf.length) {
-          leftover = leftover / 2
-          toWrite = this.buf.slice(0, leftover)
-          toWriteBytes = Buffer.byteLength(toWrite)
-        }
-        this.buf = this.buf.slice(leftover)
-        this._write(toWrite, cb)
-      }
-    }
+  unref () {
+    this.worker.unref()
   }
 
-  flushSync () {
-    if (this.closed) {
-      throw new Error('the worker has exited')
-    }
+  ref () {
+    this.worker.ref()
+  }
 
-    this._writeSync()
-    this._flushSync()
+  get ready () {
+    return this[kImpl].ready
   }
 
-  _flushSync () {
-    if (this.flushing) {
-      throw new Error('unable to flush while flushing')
-    }
+  get destroyed () {
+    return this[kImpl].destroyed
+  }
+
+  get closed () {
+    return this[kImpl].closed
+  }
+
+  get writable () {
+    return !this[kImpl].destroyed && !this[kImpl].ending
+  }
+
+  get writableEnded () {
+    return this[kImpl].ending
+  }
+
+  get writableFinished () {
+    return this[kImpl].finished
+  }
 
-    // process._rawDebug('flushSync started')
+  get writableNeedDrain () {
+    return this[kImpl].needDrain
+  }
 
-    const writeIndex = Atomics.load(this._state, WRITE_INDEX)
+  get writableObjectMode () {
+    return false
+  }
+
+  get writableErrored () {
+    return this[kImpl].errored
+  }
+}
+
+function destroy (stream, err) {
+  if (stream[kImpl].destroyed) {
+    return
+  }
+  stream[kImpl].destroyed = true
+
+  if (err) {
+    stream[kImpl].errored = err
+    stream.emit('error', err)
+  }
+
+  if (!stream.worker.exited) {
+    stream.worker.terminate()
+      .catch(() => {})
+      .then(() => {
+        stream[kImpl].closed = true
+        stream.emit('close')
+      })
+  } else {
+    setImmediate(() => {
+      stream[kImpl].closed = true
+      stream.emit('close')
+    })
+  }
+}
+
+function write (stream, data, cb) {
+  // data is smaller than the shared buffer length
+  const current = Atomics.load(stream[kImpl].state, WRITE_INDEX)
+  const length = Buffer.byteLength(data)
+  stream[kImpl].data.write(data, current)
+  Atomics.store(stream[kImpl].state, WRITE_INDEX, current + length)
+  Atomics.notify(stream[kImpl].state, WRITE_INDEX)
+  cb()
+  return true
+}
+
+function end (stream) {
+  if (stream[kImpl].ended || !stream[kImpl].ending || stream[kImpl].flushing) {
+    return
+  }
+  stream[kImpl].ended = true
+
+  try {
+    stream.flushSync()
+
+    let readIndex = Atomics.load(stream[kImpl].state, READ_INDEX)
+
+    // process._rawDebug('writing index')
+    Atomics.store(stream[kImpl].state, WRITE_INDEX, -1)
+    // process._rawDebug(`(end) readIndex (${Atomics.load(stream.state, READ_INDEX)}) writeIndex (${Atomics.load(stream.state, WRITE_INDEX)})`)
+    Atomics.notify(stream[kImpl].state, WRITE_INDEX)
 
+    // Wait for the process to complete
     let spins = 0
+    while (readIndex !== -1) {
+      // process._rawDebug(`read = ${read}`)
+      Atomics.wait(stream[kImpl].state, READ_INDEX, readIndex, 1000)
+      readIndex = Atomics.load(stream[kImpl].state, READ_INDEX)
 
-    // TODO handle deadlock
-    while (true) {
-      const readIndex = Atomics.load(this._state, READ_INDEX)
-      // process._rawDebug(`(flushSync) readIndex (${readIndex}) writeIndex (${writeIndex})`)
-      if (readIndex !== writeIndex) {
-        // TODO this timeouts for some reason.
-        Atomics.wait(this._state, READ_INDEX, readIndex, 1000)
-      } else {
-        break
+      if (readIndex === -2) {
+        throw new Error('end() failed')
       }
 
       if (++spins === 10) {
-        throw new Error('_flushSync took too long (10s)')
+        throw new Error('end() took too long (10s)')
       }
     }
-    // process._rawDebug('flushSync finished')
+
+    process.nextTick(() => {
+      stream[kImpl].finished = true
+      stream.emit('finish')
+    })
+  } catch (err) {
+    destroy(stream, err)
   }
+  // process._rawDebug('end finished...')
+}
 
-  unref () {
-    this.worker.unref()
+function writeSync (stream) {
+  const cb = () => {
+    if (stream[kImpl].ending) {
+      end(stream)
+    } else if (stream[kImpl].needDrain) {
+      process.nextTick(drain, stream)
+    }
   }
+  stream[kImpl].flushing = false
+
+  while (stream[kImpl].buf.length !== 0) {
+    const writeIndex = Atomics.load(stream[kImpl].state, WRITE_INDEX)
+    let leftover = stream[kImpl].data.length - writeIndex
+    if (leftover === 0) {
+      flushSync(stream)
+      Atomics.store(stream[kImpl].state, READ_INDEX, 0)
+      Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+      continue
+    } else if (leftover < 0) {
+      // stream should never happen
+      throw new Error('overwritten')
+    }
 
-  ref () {
-    this.worker.ref()
+    let toWrite = stream[kImpl].buf.slice(0, leftover)
+    let toWriteBytes = Buffer.byteLength(toWrite)
+    if (toWriteBytes <= leftover) {
+      stream[kImpl].buf = stream[kImpl].buf.slice(leftover)
+      // process._rawDebug('writing ' + toWrite.length)
+      write(stream, toWrite, cb)
+    } else {
+      // multi-byte utf-8
+      flushSync(stream)
+      Atomics.store(stream[kImpl].state, READ_INDEX, 0)
+      Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+
+      // Find a toWrite length that fits the buffer
+      // it must exists as the buffer is at least 4 bytes length
+      // and the max utf-8 length for a char is 4 bytes.
+      while (toWriteBytes > stream[kImpl].buf.length) {
+        leftover = leftover / 2
+        toWrite = stream[kImpl].buf.slice(0, leftover)
+        toWriteBytes = Buffer.byteLength(toWrite)
+      }
+      stream[kImpl].buf = stream[kImpl].buf.slice(leftover)
+      write(stream, toWrite, cb)
+    }
   }
+}
 
-  get writable () {
-    return !this.closed
+function flushSync (stream) {
+  if (stream[kImpl].flushing) {
+    throw new Error('unable to flush while flushing')
+  }
+
+  // process._rawDebug('flushSync started')
+
+  const writeIndex = Atomics.load(stream[kImpl].state, WRITE_INDEX)
+
+  let spins = 0
+
+  // TODO handle deadlock
+  while (true) {
+    const readIndex = Atomics.load(stream[kImpl].state, READ_INDEX)
+
+    if (readIndex === -2) {
+      throw new Error('_flushSync failed')
+    }
+
+    // process._rawDebug(`(flushSync) readIndex (${readIndex}) writeIndex (${writeIndex})`)
+    if (readIndex !== writeIndex) {
+      // TODO stream timeouts for some reason.
+      Atomics.wait(stream[kImpl].state, READ_INDEX, readIndex, 1000)
+    } else {
+      break
+    }
+
+    if (++spins === 10) {
+      throw new Error('_flushSync took too long (10s)')
+    }
   }
+  // process._rawDebug('flushSync finished')
 }
 
 module.exports = ThreadStream

package/lib/worker.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const { realImport, realRequire } = require('real-require')
 const { workerData, parentPort } = require('worker_threads')
 const { WRITE_INDEX, READ_INDEX } = require('./indexes')
 const { waitDiff } = require('./wait')
@@ -17,7 +18,7 @@ const data = Buffer.from(dataBuf)
 async function start () {
   let fn
   try {
-    fn = (await import(workerData.filename)).default
+    fn = (await realImport(workerData.filename)).default
   } catch (error) {
     // A yarn user that tries to start a ThreadStream for an external module
     // provides a filename pointing to a zip file.
@@ -29,7 +30,7 @@ async function start () {
     // The error codes may change based on the node.js version (ENOTDIR > 12, ERR_MODULE_NOT_FOUND <= 12 )
     if ((error.code === 'ENOTDIR' || error.code === 'ERR_MODULE_NOT_FOUND') &&
      workerData.filename.startsWith('file://')) {
-      fn = require(workerData.filename.replace('file://', ''))
+      fn = realRequire(workerData.filename.replace('file://', ''))
     } else {
       throw error
     }
@@ -37,6 +38,12 @@ async function start () {
   destination = await fn(workerData.workerData)
 
   destination.on('error', function (err) {
+    Atomics.store(state, WRITE_INDEX, -2)
+    Atomics.notify(state, WRITE_INDEX)
+
+    Atomics.store(state, READ_INDEX, -2)
+    Atomics.notify(state, READ_INDEX)
+
     parentPort.postMessage({
       code: 'ERROR',
       err