teyyare 0.2.7 → 0.2.8

package/bin/teyyare.js

@@ -19,6 +19,8 @@ function printUsage() {
 
 \x1b[1mOPTIONS:\x1b[0m
   -c, --channels <n>    Parallel TCP connections (default: 4 for remote, 1 for local)
+  --chunk-size <size>   Chunk size, e.g. 512K, 1M, 2M, 4M (default: 2M remote, 1M local)
+  --batch-size <size>   Generation batch size, e.g. 8M, 16M (default: 8x chunk size)
   --port <port>         Port number (default: 7421)
   --dir <path>          Serve root directory (default: current dir)
 
@@ -30,6 +32,15 @@ function printUsage() {
 `);
 }
 
+function parseByteSize(str) {
+  if (!str) return undefined;
+  const s = String(str).trim().toLowerCase();
+  if (s.endsWith('k') || s.endsWith('kb')) return Math.round(parseFloat(s) * 1024);
+  if (s.endsWith('m') || s.endsWith('mb')) return Math.round(parseFloat(s) * 1024 * 1024);
+  if (s.endsWith('g') || s.endsWith('gb')) return Math.round(parseFloat(s) * 1024 * 1024 * 1024);
+  return Number(str);
+}
+
 function parseDestination(target) {
   // Matches host:port:path or host:path or local path
   if (!target.includes(':')) {
@@ -88,10 +99,16 @@ async function main() {
     await new Promise(() => {});
   } else if (command === 'send') {
     let channels = undefined;
+    let chunkSize = undefined;
+    let batchSize = undefined;
     const positional = [];
     for (let i = 1; i < args.length; i++) {
       if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
         channels = Number(args[++i]);
+      } else if ((args[i] === '--chunk-size' || args[i] === '--chunk') && args[i + 1]) {
+        chunkSize = parseByteSize(args[++i]);
+      } else if ((args[i] === '--batch-size' || args[i] === '--gen-size') && args[i + 1]) {
+        batchSize = parseByteSize(args[++i]);
       } else {
         positional.push(args[i]);
       }
@@ -106,12 +123,17 @@ async function main() {
       process.exit(1);
     }
 
+    const pipelineOptions = {};
+    if (chunkSize) pipelineOptions.chunkSize = chunkSize;
+    if (batchSize) pipelineOptions.maxBytes = batchSize;
+
     const { host, port } = parseDestination(target);
     const metrics = await send({
       sourcePath: source,
       host,
       port,
       channels,
+      pipelineOptions,
       showProgress: true
     });
 
@@ -129,10 +151,16 @@ Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
     process.exit(0);
   } else if (command === 'pull') {
     let channels = undefined;
+    let chunkSize = undefined;
+    let batchSize = undefined;
     const positional = [];
     for (let i = 1; i < args.length; i++) {
       if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
         channels = Number(args[++i]);
+      } else if ((args[i] === '--chunk-size' || args[i] === '--chunk') && args[i + 1]) {
+        chunkSize = parseByteSize(args[++i]);
+      } else if ((args[i] === '--batch-size' || args[i] === '--gen-size') && args[i + 1]) {
+        batchSize = parseByteSize(args[++i]);
       } else {
         positional.push(args[i]);
       }
@@ -147,6 +175,10 @@ Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
       process.exit(1);
     }
 
+    const pipelineOptions = {};
+    if (chunkSize) pipelineOptions.chunkSize = chunkSize;
+    if (batchSize) pipelineOptions.maxBytes = batchSize;
+
     const { host, port, destPath: remotePath } = parseDestination(source);
     const metrics = await pull({
       sourceRemotePath: remotePath,
@@ -154,6 +186,7 @@ Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
       port,
       channels,
       destinationDir: target,
+      pipelineOptions,
       showProgress: true
     });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "teyyare",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "description": "High-performance resumable file transfer engine and CLI built on muttafa and raptiye",
   "main": "src/index.js",
   "type": "module",
@@ -9,7 +9,11 @@
   },
   "scripts": {
     "test": "node --test test/**/*.test.js",
-    "bench": "node bench/run-all.js"
+    "bench": "node bench/run-all.js",
+    "e2e": "node test/e2e/e2e.js",
+    "e2e:profile": "softscope test/e2e/e2e.js",
+    "e2e:live": "softscope --live test/e2e/e2e.js",
+    "e2e:md": "softscope --md test/e2e/e2e.js"
   },
   "keywords": [
     "file-transfer",

package/src/core/protocol.js

@@ -76,10 +76,10 @@ export function unpackChunkMeta(buf, offset = 0) {
  * 14..45: sha256 hex (32 bytes raw or 64 hex chars)
  * 46..:   path utf8
  */
-export function packFileBegin({ fileId, fileSize, path, sha256 = '' }) {
+export function packFileBegin({ fileId, fileSize, path, sha256 = '', chunkSize = 1048576 }) {
   const pathBytes = encoder.encode(path);
   const shaBytes = encoder.encode(sha256.padEnd(64, '0').slice(0, 64));
-  const total = 14 + 64 + pathBytes.byteLength;
+  const total = 14 + 64 + pathBytes.byteLength + 4;
   const buf = new Uint8Array(total);
   const view = new DataView(buf.buffer, buf.byteOffset, total);
 
@@ -88,6 +88,7 @@ export function packFileBegin({ fileId, fileSize, path, sha256 = '' }) {
   view.setUint16(12, pathBytes.byteLength, false);
   buf.set(shaBytes, 14);
   buf.set(pathBytes, 14 + 64);
+  view.setUint32(14 + 64 + pathBytes.byteLength, chunkSize >>> 0, false);
   return buf;
 }
 
@@ -98,7 +99,12 @@ export function unpackFileBegin(buf) {
   const pathLen = view.getUint16(12, false);
   const sha256 = decoder.decode(buf.subarray(14, 14 + 64)).replace(/0+$/, '');
   const path = decoder.decode(buf.subarray(14 + 64, 14 + 64 + pathLen));
-  return { fileId, fileSize, path, sha256 };
+  let chunkSize = 1048576;
+  if (buf.byteLength >= 14 + 64 + pathLen + 4) {
+    const cs = view.getUint32(14 + 64 + pathLen, false);
+    if (cs > 0) chunkSize = cs;
+  }
+  return { fileId, fileSize, path, sha256, chunkSize };
 }
 
 /**
@@ -167,10 +173,18 @@ export function unpackTransferEnd(buf) {
 /**
  * Pack PULL_REQUEST metadata with optional verifiedChunks bitmap/list for resume.
  */
-export function packPullRequest({ remotePath, verifiedChunks = [] }) {
+export function packPullRequest({
+  remotePath,
+  verifiedChunks = [],
+  chunkSize = 0,
+  maxEntries = 0,
+  maxBytes = 0
+}) {
   const pathBytes = encoder.encode(remotePath);
   const count = verifiedChunks.length;
-  const buf = new Uint8Array(2 + pathBytes.byteLength + 4 + count * 4);
+  const hasTuning = (chunkSize > 0 || maxEntries > 0 || maxBytes > 0);
+  const tuningLen = hasTuning ? 10 : 0;
+  const buf = new Uint8Array(2 + pathBytes.byteLength + 4 + count * 4 + tuningLen);
   const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
   view.setUint16(0, pathBytes.byteLength, false);
   buf.set(pathBytes, 2);
@@ -182,6 +196,13 @@ export function packPullRequest({ remotePath, verifiedChunks = [] }) {
     view.setUint32(off, verifiedChunks[i], false);
     off += 4;
   }
+
+  if (hasTuning) {
+    view.setUint32(off, chunkSize >>> 0, false);
+    view.setUint16(off + 4, maxEntries & 0xffff, false);
+    view.setUint32(off + 6, maxBytes >>> 0, false);
+  }
+
   return buf;
 }
 
@@ -201,7 +222,19 @@ export function unpackPullRequest(buf) {
     }
   }
 
-  return { remotePath, verifiedChunks };
+  let chunkSize = undefined;
+  let maxEntries = undefined;
+  let maxBytes = undefined;
+  if (buf.byteLength >= off + 10) {
+    const cs = view.getUint32(off, false);
+    const me = view.getUint16(off + 4, false);
+    const mb = view.getUint32(off + 6, false);
+    if (cs > 0) chunkSize = cs;
+    if (me > 0) maxEntries = me;
+    if (mb > 0) maxBytes = mb;
+  }
+
+  return { remotePath, verifiedChunks, chunkSize, maxEntries, maxBytes };
 }
 
 /**

package/src/index.js

@@ -45,32 +45,45 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
   if (socket._writableState) socket._writableState.highWaterMark = 4 * 1024 * 1024;
 
   const chunks = [];
+  let chunkHead = 0;
+  let chunkOffset = 0;
   let bufferedLength = 0;
+  const headerScratch = Buffer.allocUnsafe(HEADER_SIZE);
 
   socket.on('data', (chunk) => {
     chunks.push(chunk);
     bufferedLength += chunk.length;
 
     while (bufferedLength >= HEADER_SIZE) {
-      let headerBuf = chunks[0];
-      if (headerBuf.length < HEADER_SIZE) {
-        const tempHeader = Buffer.allocUnsafe(HEADER_SIZE);
+      const head = chunks[chunkHead];
+      const headAvail = head.length - chunkOffset;
+      let header;
+
+      if (headAvail >= HEADER_SIZE) {
+        try {
+          header = readHeader(head, chunkOffset);
+        } catch (err) {
+          socket.destroy(err);
+          return;
+        }
+      } else {
         let off = 0;
-        for (let i = 0; off < HEADER_SIZE && i < chunks.length; i++) {
-          const c = chunks[i];
-          const toCopy = Math.min(c.length, HEADER_SIZE - off);
-          tempHeader.set(c.subarray(0, toCopy), off);
+        let cIdx = chunkHead;
+        let cOff = chunkOffset;
+        while (off < HEADER_SIZE && cIdx < chunks.length) {
+          const c = chunks[cIdx];
+          const toCopy = Math.min(c.length - cOff, HEADER_SIZE - off);
+          headerScratch.set(c.subarray(cOff, cOff + toCopy), off);
           off += toCopy;
+          cIdx++;
+          cOff = 0;
+        }
+        try {
+          header = readHeader(headerScratch, 0);
+        } catch (err) {
+          socket.destroy(err);
+          return;
         }
-        headerBuf = tempHeader;
-      }
-
-      let header;
-      try {
-        header = readHeader(headerBuf, 0);
-      } catch (err) {
-        socket.destroy(err);
-        return;
       }
 
       const totalLength = HEADER_SIZE + header.payloadLength;
@@ -79,29 +92,35 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
       }
 
       let rawFrame;
-      if (chunks[0].length >= totalLength) {
-        rawFrame = chunks[0].subarray(0, totalLength);
-        if (chunks[0].length === totalLength) {
-          chunks.shift();
-        } else {
-          chunks[0] = chunks[0].subarray(totalLength);
+      if (headAvail >= totalLength) {
+        // Fast-path: single chunk zero-copy slice
+        rawFrame = head.subarray(chunkOffset, chunkOffset + totalLength);
+        chunkOffset += totalLength;
+        if (chunkOffset === head.length) {
+          chunks[chunkHead] = null;
+          chunkHead++;
+          chunkOffset = 0;
         }
         bufferedLength -= totalLength;
       } else {
+        // Multi-chunk frame assembly
         rawFrame = Buffer.allocUnsafe(totalLength);
         let dstOff = 0;
         let remaining = totalLength;
         while (remaining > 0) {
-          const head = chunks[0];
-          if (head.length <= remaining) {
-            rawFrame.set(head, dstOff);
-            dstOff += head.length;
-            remaining -= head.length;
-            chunks.shift();
+          const curr = chunks[chunkHead];
+          const avail = curr.length - chunkOffset;
+          if (avail <= remaining) {
+            rawFrame.set(curr.subarray(chunkOffset), dstOff);
+            dstOff += avail;
+            remaining -= avail;
+            chunks[chunkHead] = null;
+            chunkHead++;
+            chunkOffset = 0;
           } else {
-            rawFrame.set(head.subarray(0, remaining), dstOff);
+            rawFrame.set(curr.subarray(chunkOffset, chunkOffset + remaining), dstOff);
             dstOff += remaining;
-            chunks[0] = head.subarray(remaining);
+            chunkOffset += remaining;
             remaining = 0;
           }
         }
@@ -126,6 +145,16 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
         console.error(`[TCPTransport ${this.id}] Error decoding frame:`, err.message);
       }
     }
+
+    // Amortized array compaction to prevent unbounded array growth without O(N) shift()
+    if (bufferedLength === 0) {
+      chunks.length = 0;
+      chunkHead = 0;
+      chunkOffset = 0;
+    } else if (chunkHead >= 128 && chunkHead > (chunks.length >> 1)) {
+      chunks.splice(0, chunkHead);
+      chunkHead = 0;
+    }
   });
 
   socket.on('close', () => {
@@ -219,18 +248,19 @@ TCPTransport.prototype.sendv = function(peer, buffers) {
 
   const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
   if (pool && pool.length > 0) {
-    const active = pool.filter((s) => !s.destroyed && s.writable);
-    if (active.length > 0) {
-      let idx = this._channelIndex.get(peer) || 0;
-      const socket = active[idx % active.length];
-      this._channelIndex.set(peer, idx + 1);
-
-      socket.cork();
-      for (let i = 0; i < buffers.length; i++) {
-        socket.write(buffers[i]);
+    let idx = this._channelIndex.get(peer) || 0;
+    const poolLen = pool.length;
+    for (let attempts = 0; attempts < poolLen; attempts++) {
+      const s = pool[(idx + attempts) % poolLen];
+      if (!s.destroyed && s.writable) {
+        this._channelIndex.set(peer, idx + attempts + 1);
+        s.cork();
+        for (let i = 0; i < buffers.length; i++) {
+          s.write(buffers[i]);
+        }
+        s.uncork();
+        return;
       }
-      socket.uncork();
-      return;
     }
   }
 
@@ -252,13 +282,15 @@ TCPTransport.prototype.send = function(peer, buffer) {
 
   const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
   if (pool && pool.length > 0) {
-    const active = pool.filter((s) => !s.destroyed && s.writable);
-    if (active.length > 0) {
-      let idx = this._channelIndex.get(peer) || 0;
-      const socket = active[idx % active.length];
-      this._channelIndex.set(peer, idx + 1);
-      socket.write(buffer);
-      return;
+    let idx = this._channelIndex.get(peer) || 0;
+    const poolLen = pool.length;
+    for (let attempts = 0; attempts < poolLen; attempts++) {
+      const s = pool[(idx + attempts) % poolLen];
+      if (!s.destroyed && s.writable) {
+        this._channelIndex.set(peer, idx + attempts + 1);
+        s.write(buffer);
+        return;
+      }
     }
   }
 
@@ -450,8 +482,8 @@ export async function serve({
     transport,
     localNode: 2,
     remoteNode: 1,
-    maxInflightBatches: 16,
-    maxInflightBytes: 64 * 1024 * 1024
+    maxInflightBatches: 32,
+    maxInflightBytes: 128 * 1024 * 1024
   });
 
   const receiver = new ReceiverPipeline({
@@ -478,7 +510,13 @@ export async function serve({
     const [metaBuf, payloadBuf] = msg.buffers;
     const entries = unpackGeneration(metaBuf, payloadBuf);
     if (entries.length > 0 && entries[0].type === Opcodes.PULL_REQUEST) {
-      const { remotePath, verifiedChunks = [] } = unpackPullRequest(entries[0].key);
+      const {
+        remotePath,
+        verifiedChunks = [],
+        chunkSize,
+        maxEntries,
+        maxBytes
+      } = unpackPullRequest(entries[0].key);
       let resolved = path.isAbsolute(remotePath) ? remotePath : path.resolve(destinationDir, remotePath);
       if (!existsSync(resolved)) {
         const alt = path.resolve(destinationDir, remotePath);
@@ -513,9 +551,18 @@ export async function serve({
         }));
       }
 
+      const senderOpts = {
+        maxImmutableBuffers: 32,
+        maxConcurrentFlushes: 32
+      };
+      if (chunkSize) senderOpts.chunkSize = chunkSize;
+      if (maxEntries) senderOpts.maxEntries = maxEntries;
+      if (maxBytes) senderOpts.maxBytes = maxBytes;
+
       const sender = new SenderPipeline({
         replication,
         manifest,
+        options: senderOpts,
         knownResumeState,
         metrics
       });
@@ -604,8 +651,8 @@ export async function send({
     transport,
     localNode: 1,
     remoteNode: 2,
-    maxInflightBatches: pipelineOptions.maxInflightBatches || 16,
-    maxInflightBytes: pipelineOptions.maxInflightBytes || 64 * 1024 * 1024
+    maxInflightBatches: pipelineOptions.maxInflightBatches || 32,
+    maxInflightBytes: pipelineOptions.maxInflightBytes || 128 * 1024 * 1024
   });
 
   const manifest = await TransferManifest.build(sourcePath);
@@ -651,6 +698,15 @@ export async function pull({
   pipelineOptions = {},
   showProgress = true
 }) {
+  const isRemote = host !== '127.0.0.1' && host !== 'localhost';
+  const defaultChunkSize = isRemote ? 2 * 1024 * 1024 : 1024 * 1024;
+  const defaultMaxEntries = 8;
+  const defaultMaxBytes = defaultChunkSize * defaultMaxEntries;
+
+  const effectiveChunkSize = pipelineOptions.chunkSize || defaultChunkSize;
+  const effectiveMaxEntries = pipelineOptions.maxEntries || defaultMaxEntries;
+  const effectiveMaxBytes = pipelineOptions.maxBytes || defaultMaxBytes;
+
   const metrics = new MetricsCollector();
   const progress = showProgress ? new ProgressReporter({ role: 'receiver' }) : null;
   if (progress) {
@@ -675,8 +731,8 @@ export async function pull({
     transport,
     localNode: 1,
     remoteNode: 2,
-    maxInflightBatches: pipelineOptions.maxInflightBatches || 16,
-    maxInflightBytes: pipelineOptions.maxInflightBytes || 64 * 1024 * 1024
+    maxInflightBatches: pipelineOptions.maxInflightBatches || 32,
+    maxInflightBytes: pipelineOptions.maxInflightBytes || 128 * 1024 * 1024
   });
 
   const receiver = new ReceiverPipeline({
@@ -699,11 +755,17 @@ export async function pull({
       }
     } catch {}
 
-    // Submit PULL_REQUEST batch with verified chunks for resume
+    // Submit PULL_REQUEST batch with verified chunks and WAN tuning
     await submitSingleMutation(
       replication,
       Opcodes.PULL_REQUEST,
-      packPullRequest({ remotePath: sourceRemotePath, verifiedChunks })
+      packPullRequest({
+        remotePath: sourceRemotePath,
+        verifiedChunks,
+        chunkSize: effectiveChunkSize,
+        maxEntries: effectiveMaxEntries,
+        maxBytes: effectiveMaxBytes
+      })
     );
 
     // Wait for receiver pipeline to receive and write all files