teyyare 0.2.7 → 0.2.9

package/src/index.js

@@ -45,32 +45,45 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
   if (socket._writableState) socket._writableState.highWaterMark = 4 * 1024 * 1024;
 
   const chunks = [];
+  let chunkHead = 0;
+  let chunkOffset = 0;
   let bufferedLength = 0;
+  const headerScratch = Buffer.allocUnsafe(HEADER_SIZE);
 
   socket.on('data', (chunk) => {
     chunks.push(chunk);
     bufferedLength += chunk.length;
 
     while (bufferedLength >= HEADER_SIZE) {
-      let headerBuf = chunks[0];
-      if (headerBuf.length < HEADER_SIZE) {
-        const tempHeader = Buffer.allocUnsafe(HEADER_SIZE);
+      const head = chunks[chunkHead];
+      const headAvail = head.length - chunkOffset;
+      let header;
+
+      if (headAvail >= HEADER_SIZE) {
+        try {
+          header = readHeader(head, chunkOffset);
+        } catch (err) {
+          socket.destroy(err);
+          return;
+        }
+      } else {
         let off = 0;
-        for (let i = 0; off < HEADER_SIZE && i < chunks.length; i++) {
-          const c = chunks[i];
-          const toCopy = Math.min(c.length, HEADER_SIZE - off);
-          tempHeader.set(c.subarray(0, toCopy), off);
+        let cIdx = chunkHead;
+        let cOff = chunkOffset;
+        while (off < HEADER_SIZE && cIdx < chunks.length) {
+          const c = chunks[cIdx];
+          const toCopy = Math.min(c.length - cOff, HEADER_SIZE - off);
+          headerScratch.set(c.subarray(cOff, cOff + toCopy), off);
           off += toCopy;
+          cIdx++;
+          cOff = 0;
+        }
+        try {
+          header = readHeader(headerScratch, 0);
+        } catch (err) {
+          socket.destroy(err);
+          return;
         }
-        headerBuf = tempHeader;
-      }
-
-      let header;
-      try {
-        header = readHeader(headerBuf, 0);
-      } catch (err) {
-        socket.destroy(err);
-        return;
       }
 
       const totalLength = HEADER_SIZE + header.payloadLength;
@@ -79,29 +92,35 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
       }
 
       let rawFrame;
-      if (chunks[0].length >= totalLength) {
-        rawFrame = chunks[0].subarray(0, totalLength);
-        if (chunks[0].length === totalLength) {
-          chunks.shift();
-        } else {
-          chunks[0] = chunks[0].subarray(totalLength);
+      if (headAvail >= totalLength) {
+        // Fast-path: single chunk zero-copy slice
+        rawFrame = head.subarray(chunkOffset, chunkOffset + totalLength);
+        chunkOffset += totalLength;
+        if (chunkOffset === head.length) {
+          chunks[chunkHead] = null;
+          chunkHead++;
+          chunkOffset = 0;
         }
         bufferedLength -= totalLength;
       } else {
+        // Multi-chunk frame assembly
         rawFrame = Buffer.allocUnsafe(totalLength);
         let dstOff = 0;
         let remaining = totalLength;
         while (remaining > 0) {
-          const head = chunks[0];
-          if (head.length <= remaining) {
-            rawFrame.set(head, dstOff);
-            dstOff += head.length;
-            remaining -= head.length;
-            chunks.shift();
+          const curr = chunks[chunkHead];
+          const avail = curr.length - chunkOffset;
+          if (avail <= remaining) {
+            rawFrame.set(curr.subarray(chunkOffset), dstOff);
+            dstOff += avail;
+            remaining -= avail;
+            chunks[chunkHead] = null;
+            chunkHead++;
+            chunkOffset = 0;
           } else {
-            rawFrame.set(head.subarray(0, remaining), dstOff);
+            rawFrame.set(curr.subarray(chunkOffset, chunkOffset + remaining), dstOff);
             dstOff += remaining;
-            chunks[0] = head.subarray(remaining);
+            chunkOffset += remaining;
             remaining = 0;
           }
         }
@@ -126,6 +145,16 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
         console.error(`[TCPTransport ${this.id}] Error decoding frame:`, err.message);
       }
     }
+
+    // Amortized array compaction to prevent unbounded array growth without O(N) shift()
+    if (bufferedLength === 0) {
+      chunks.length = 0;
+      chunkHead = 0;
+      chunkOffset = 0;
+    } else if (chunkHead >= 128 && chunkHead > (chunks.length >> 1)) {
+      chunks.splice(0, chunkHead);
+      chunkHead = 0;
+    }
   });
 
   socket.on('close', () => {
@@ -219,18 +248,19 @@ TCPTransport.prototype.sendv = function(peer, buffers) {
 
   const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
   if (pool && pool.length > 0) {
-    const active = pool.filter((s) => !s.destroyed && s.writable);
-    if (active.length > 0) {
-      let idx = this._channelIndex.get(peer) || 0;
-      const socket = active[idx % active.length];
-      this._channelIndex.set(peer, idx + 1);
-
-      socket.cork();
-      for (let i = 0; i < buffers.length; i++) {
-        socket.write(buffers[i]);
+    let idx = this._channelIndex.get(peer) || 0;
+    const poolLen = pool.length;
+    for (let attempts = 0; attempts < poolLen; attempts++) {
+      const s = pool[(idx + attempts) % poolLen];
+      if (!s.destroyed && s.writable) {
+        this._channelIndex.set(peer, idx + attempts + 1);
+        s.cork();
+        for (let i = 0; i < buffers.length; i++) {
+          s.write(buffers[i]);
+        }
+        s.uncork();
+        return;
       }
-      socket.uncork();
-      return;
     }
   }
 
@@ -252,13 +282,15 @@ TCPTransport.prototype.send = function(peer, buffer) {
 
   const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
   if (pool && pool.length > 0) {
-    const active = pool.filter((s) => !s.destroyed && s.writable);
-    if (active.length > 0) {
-      let idx = this._channelIndex.get(peer) || 0;
-      const socket = active[idx % active.length];
-      this._channelIndex.set(peer, idx + 1);
-      socket.write(buffer);
-      return;
+    let idx = this._channelIndex.get(peer) || 0;
+    const poolLen = pool.length;
+    for (let attempts = 0; attempts < poolLen; attempts++) {
+      const s = pool[(idx + attempts) % poolLen];
+      if (!s.destroyed && s.writable) {
+        this._channelIndex.set(peer, idx + attempts + 1);
+        s.write(buffer);
+        return;
+      }
     }
   }
 
@@ -450,8 +482,8 @@ export async function serve({
     transport,
     localNode: 2,
     remoteNode: 1,
-    maxInflightBatches: 16,
-    maxInflightBytes: 64 * 1024 * 1024
+    maxInflightBatches: 32,
+    maxInflightBytes: 128 * 1024 * 1024
   });
 
   const receiver = new ReceiverPipeline({
@@ -478,7 +510,13 @@ export async function serve({
     const [metaBuf, payloadBuf] = msg.buffers;
     const entries = unpackGeneration(metaBuf, payloadBuf);
     if (entries.length > 0 && entries[0].type === Opcodes.PULL_REQUEST) {
-      const { remotePath, verifiedChunks = [] } = unpackPullRequest(entries[0].key);
+      const {
+        remotePath,
+        verifiedChunks = [],
+        chunkSize,
+        maxEntries,
+        maxBytes
+      } = unpackPullRequest(entries[0].key);
       let resolved = path.isAbsolute(remotePath) ? remotePath : path.resolve(destinationDir, remotePath);
       if (!existsSync(resolved)) {
         const alt = path.resolve(destinationDir, remotePath);
@@ -513,9 +551,18 @@ export async function serve({
         }));
       }
 
+      const senderOpts = {
+        maxImmutableBuffers: 32,
+        maxConcurrentFlushes: 32
+      };
+      if (chunkSize) senderOpts.chunkSize = chunkSize;
+      if (maxEntries) senderOpts.maxEntries = maxEntries;
+      if (maxBytes) senderOpts.maxBytes = maxBytes;
+
       const sender = new SenderPipeline({
         replication,
         manifest,
+        options: senderOpts,
         knownResumeState,
         metrics
       });
@@ -604,8 +651,8 @@ export async function send({
     transport,
     localNode: 1,
     remoteNode: 2,
-    maxInflightBatches: pipelineOptions.maxInflightBatches || 16,
-    maxInflightBytes: pipelineOptions.maxInflightBytes || 64 * 1024 * 1024
+    maxInflightBatches: pipelineOptions.maxInflightBatches || 32,
+    maxInflightBytes: pipelineOptions.maxInflightBytes || 128 * 1024 * 1024
   });
 
   const manifest = await TransferManifest.build(sourcePath);
@@ -651,6 +698,15 @@ export async function pull({
   pipelineOptions = {},
   showProgress = true
 }) {
+  const isRemote = host !== '127.0.0.1' && host !== 'localhost';
+  const defaultChunkSize = isRemote ? 2 * 1024 * 1024 : 1024 * 1024;
+  const defaultMaxEntries = 8;
+  const defaultMaxBytes = defaultChunkSize * defaultMaxEntries;
+
+  const effectiveChunkSize = pipelineOptions.chunkSize || defaultChunkSize;
+  const effectiveMaxEntries = pipelineOptions.maxEntries || defaultMaxEntries;
+  const effectiveMaxBytes = pipelineOptions.maxBytes || defaultMaxBytes;
+
   const metrics = new MetricsCollector();
   const progress = showProgress ? new ProgressReporter({ role: 'receiver' }) : null;
   if (progress) {
@@ -675,8 +731,8 @@ export async function pull({
     transport,
     localNode: 1,
     remoteNode: 2,
-    maxInflightBatches: pipelineOptions.maxInflightBatches || 16,
-    maxInflightBytes: pipelineOptions.maxInflightBytes || 64 * 1024 * 1024
+    maxInflightBatches: pipelineOptions.maxInflightBatches || 32,
+    maxInflightBytes: pipelineOptions.maxInflightBytes || 128 * 1024 * 1024
   });
 
   const receiver = new ReceiverPipeline({
@@ -699,11 +755,17 @@ export async function pull({
       }
     } catch {}
 
-    // Submit PULL_REQUEST batch with verified chunks for resume
+    // Submit PULL_REQUEST batch with verified chunks and WAN tuning
     await submitSingleMutation(
       replication,
       Opcodes.PULL_REQUEST,
-      packPullRequest({ remotePath: sourceRemotePath, verifiedChunks })
+      packPullRequest({
+        remotePath: sourceRemotePath,
+        verifiedChunks,
+        chunkSize: effectiveChunkSize,
+        maxEntries: effectiveMaxEntries,
+        maxBytes: effectiveMaxBytes
+      })
     );
 
     // Wait for receiver pipeline to receive and write all files