teyyare 0.2.1 → 0.2.3

package/bin/teyyare.js

@@ -50,6 +50,11 @@ async function main() {
     process.exit(0);
   }
 
+  if (command === '--version' || command === '-v' || command === 'version') {
+    console.log('0.2.3');
+    process.exit(0);
+  }
+
   if (command === 'serve') {
     let port = 7421;
     let dir = process.cwd();
@@ -105,6 +110,7 @@ Backpressure:     ${snap.backpressureCount}
 Protocol overhead:${snap.protocolOverheadPercent.toFixed(2)}%
 Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
 `);
+    process.exit(0);
   } else if (command === 'pull') {
     const source = args[1];
     const target = args[2] || './';
@@ -135,8 +141,9 @@ Backpressure:     ${snap.backpressureCount}
 Protocol overhead:${snap.protocolOverheadPercent.toFixed(2)}%
 Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
 `);
+    process.exit(0);
   } else if (command === 'status') {
-    console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.1.0 status: ready');
+    console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.3 status: ready');
   } else {
     console.error(`Unknown command: ${command}`);
     printUsage();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "teyyare",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "description": "High-performance resumable file transfer engine and CLI built on muttafa and raptiye",
   "main": "src/index.js",
   "type": "module",

package/src/cli/progress.js

@@ -1,5 +1,6 @@
 /**
  * Throttled terminal progress renderer.
+ * Intelligent, accurate metrics for both Upload (send) and Download (pull) modes.
  */
 
 export class ProgressReporter {
@@ -7,10 +8,12 @@ export class ProgressReporter {
    * @param {object} [options]
    * @param {number} [options.intervalMs=150]
    * @param {boolean} [options.enabled=true]
+   * @param {'sender'|'receiver'} [options.role='sender']
    */
-  constructor({ intervalMs = 150, enabled = true } = {}) {
+  constructor({ intervalMs = 150, enabled = true, role = 'sender' } = {}) {
     this.intervalMs = intervalMs;
     this.enabled = enabled && Boolean(process.stdout.isTTY || process.env.CI !== 'true');
+    this.role = role;
 
     this.fileName = '';
     this.bytesDone = 0;
@@ -23,16 +26,20 @@ export class ProgressReporter {
     this.muttafaQueue = 0;
     this.raptiyeRTT = 0;
     this.tcpPressure = 0;
+    this.resumeHits = 0;
+    this.diskSpeedMBs = 0;
 
     this.startTime = Date.now();
     this.lastRender = 0;
     this._timer = null;
     this._linesRendered = 0;
+    this._speedHistory = [];
   }
 
   start() {
     if (!this.enabled) return;
     this.startTime = Date.now();
+    this._speedHistory = [{ time: this.startTime, bytes: this.bytesDone }];
     this._timer = setInterval(() => this.render(), this.intervalMs);
   }
 
@@ -52,13 +59,15 @@ export class ProgressReporter {
     if (state.bytesDone !== undefined) this.bytesDone = state.bytesDone;
     if (state.totalBytes !== undefined) this.totalBytes = state.totalBytes;
     if (state.chunksDone !== undefined) this.chunksDone = state.chunksDone;
-    if (state.totalChunks !== undefined) this.totalChunks = state.totalChunks;
+    if (state.totalChunks !== undefined && state.totalChunks > 0) this.totalChunks = state.totalChunks;
     if (state.generationId !== undefined) this.generationId = state.generationId;
     if (state.inflightGen !== undefined) this.inflightGen = state.inflightGen;
     if (state.inflightBytes !== undefined) this.inflightBytes = state.inflightBytes;
     if (state.muttafaQueue !== undefined) this.muttafaQueue = state.muttafaQueue;
     if (state.raptiyeRTT !== undefined) this.raptiyeRTT = state.raptiyeRTT;
     if (state.tcpPressure !== undefined) this.tcpPressure = state.tcpPressure;
+    if (state.resumeHits !== undefined) this.resumeHits = state.resumeHits;
+    if (state.diskSpeedMBs !== undefined) this.diskSpeedMBs = state.diskSpeedMBs;
   }
 
   formatBytes(bytes) {
@@ -77,39 +86,75 @@ export class ProgressReporter {
   render() {
     if (!this.enabled) return;
 
-    const elapsedSec = Math.max(0.001, (Date.now() - this.startTime) / 1000);
-    const throughputMBs = (this.bytesDone / (1024 * 1024)) / elapsedSec;
+    const now = Date.now();
+    const elapsedSec = Math.max(0.001, (now - this.startTime) / 1000);
+
+    // Track sliding window (last 2 seconds) for smooth, responsive current speed
+    this._speedHistory.push({ time: now, bytes: this.bytesDone });
+    while (this._speedHistory.length > 2 && (now - this._speedHistory[0].time) > 2000) {
+      this._speedHistory.shift();
+    }
+    const oldest = this._speedHistory[0];
+    const windowSec = Math.max(0.001, (now - oldest.time) / 1000);
+    const windowBytes = Math.max(0, this.bytesDone - oldest.bytes);
+
+    const avgSpeedMBs = (this.bytesDone / (1024 * 1024)) / elapsedSec;
+    const currentSpeedMBs = windowSec >= 0.4 ? (windowBytes / (1024 * 1024)) / windowSec : avgSpeedMBs;
+    const effectiveSpeed = currentSpeedMBs > 0.05 ? currentSpeedMBs : avgSpeedMBs;
+
     const percent = this.totalBytes > 0 ? ((this.bytesDone / this.totalBytes) * 100).toFixed(1) : '0.0';
 
     let etaStr = '--:--';
-    if (throughputMBs > 0 && this.totalBytes > this.bytesDone) {
+    if (effectiveSpeed > 0 && this.totalBytes > this.bytesDone) {
       const remainingBytes = this.totalBytes - this.bytesDone;
-      const remainingSec = Math.round(remainingBytes / (throughputMBs * 1024 * 1024));
+      const remainingSec = Math.round(remainingBytes / (effectiveSpeed * 1024 * 1024));
       const mm = String(Math.floor(remainingSec / 60)).padStart(2, '0');
       const ss = String(remainingSec % 60).padStart(2, '0');
       etaStr = `${mm}:${ss}`;
     }
 
-    const lines = [
-      '\x1b[1m\x1b[36mTeyyare\x1b[0m',
-      '',
-      `\x1b[1m${this.fileName || 'Transferring...'}\x1b[0m`,
-      '',
-      `${this.formatBytes(this.bytesDone)} / ${this.formatBytes(this.totalBytes)}`,
-      `\x1b[32m${percent}%\x1b[0m`,
-      '',
-      `throughput       \x1b[1m${throughputMBs.toFixed(1)} MB/s\x1b[0m`,
-      `ETA              ${etaStr}`,
-      '',
-      `chunks           ${this.chunksDone} / ${this.totalChunks}`,
-      `generation       ${this.generationId}`,
-      `inflight gen     ${this.inflightGen}`,
-      `inflight bytes   ${this.formatBytes(this.inflightBytes)}`,
-      '',
-      `muttafa queue    ${this.muttafaQueue}`,
-      `raptiye RTT      ${this.raptiyeRTT} ms`,
-      `TCP pressure     ${this.tcpPressure}`
-    ];
+    let lines;
+    if (this.role === 'receiver') {
+      const diskStr = this.diskSpeedMBs > 0 ? `${this.diskSpeedMBs.toFixed(1)} MB/s` : `${effectiveSpeed.toFixed(1)} MB/s`;
+      lines = [
+        '\x1b[1m\x1b[36mTeyyare [Download]\x1b[0m',
+        '',
+        `\x1b[1m${this.fileName || 'Transferring...'}\x1b[0m`,
+        '',
+        `${this.formatBytes(this.bytesDone)} / ${this.formatBytes(this.totalBytes)}`,
+        `\x1b[32m${percent}%\x1b[0m`,
+        '',
+        `network speed    \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
+        `disk speed       \x1b[1m${diskStr}\x1b[0m`,
+        `ETA              ${etaStr}`,
+        '',
+        `chunks           ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,
+        `generation       ${this.generationId}`,
+        `verified CRC32   ${this.chunksDone}`,
+        `resume hits      ${this.resumeHits}`
+      ];
+    } else {
+      lines = [
+        '\x1b[1m\x1b[36mTeyyare [Upload]\x1b[0m',
+        '',
+        `\x1b[1m${this.fileName || 'Transferring...'}\x1b[0m`,
+        '',
+        `${this.formatBytes(this.bytesDone)} / ${this.formatBytes(this.totalBytes)}`,
+        `\x1b[32m${percent}%\x1b[0m`,
+        '',
+        `throughput       \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
+        `ETA              ${etaStr}`,
+        '',
+        `chunks           ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,
+        `generation       ${this.generationId}`,
+        `inflight gen     ${this.inflightGen}`,
+        `inflight bytes   ${this.formatBytes(this.inflightBytes)}`,
+        '',
+        `muttafa queue    ${this.muttafaQueue}`,
+        `raptiye RTT      ${this.raptiyeRTT} ms`,
+        `TCP pressure     ${this.tcpPressure}`
+      ];
+    }
 
     if (process.stdout.isTTY) {
       if (this._linesRendered > 0) {
@@ -118,8 +163,7 @@ export class ProgressReporter {
       process.stdout.write(lines.join('\n') + '\n');
       this._linesRendered = lines.length;
     } else {
-      // Non-interactive log
-      process.stdout.write(`[Teyyare] ${percent}% | ${throughputMBs.toFixed(1)} MB/s | Chunks: ${this.chunksDone}/${this.totalChunks} | ETA: ${etaStr}\n`);
+      process.stdout.write(`[Teyyare] ${percent}% | ${effectiveSpeed.toFixed(1)} MB/s | Chunks: ${this.chunksDone}/${this.totalChunks || this.chunksDone} | ETA: ${etaStr}\n`);
     }
   }
 }

package/src/index.js

@@ -5,6 +5,7 @@
 import path from 'node:path';
 import { existsSync } from 'node:fs';
 import { TCPTransport, ReplicationPipeline } from 'raptiye';
+import { readHeader, decodeMessage, HEADER_SIZE } from 'raptiye/src/protocol/wire.js';
 import { SenderPipeline } from './sender/pipeline.js';
 import { ReceiverPipeline } from './receiver/pipeline.js';
 import { TransferManifest } from './manifest/manifest.js';
@@ -22,6 +23,116 @@ import {
   unpackPullError
 } from './core/protocol.js';
 
+// High-performance streaming TCP framer patch for TCPTransport
+// Eliminates O(N^2) Buffer.concat and ensures clean socket lifecycle management
+TCPTransport.prototype._attachSocketReader = function(socket) {
+  socket.setNoDelay(true);
+  const chunks = [];
+  let bufferedLength = 0;
+
+  socket.on('data', (chunk) => {
+    chunks.push(chunk);
+    bufferedLength += chunk.length;
+
+    while (bufferedLength >= HEADER_SIZE) {
+      let headerBuf = chunks[0];
+      if (headerBuf.length < HEADER_SIZE) {
+        const tempHeader = Buffer.allocUnsafe(HEADER_SIZE);
+        let off = 0;
+        for (let i = 0; off < HEADER_SIZE && i < chunks.length; i++) {
+          const c = chunks[i];
+          const toCopy = Math.min(c.length, HEADER_SIZE - off);
+          tempHeader.set(c.subarray(0, toCopy), off);
+          off += toCopy;
+        }
+        headerBuf = tempHeader;
+      }
+
+      let header;
+      try {
+        header = readHeader(headerBuf, 0);
+      } catch (err) {
+        socket.destroy(err);
+        return;
+      }
+
+      const totalLength = HEADER_SIZE + header.payloadLength;
+      if (bufferedLength < totalLength) {
+        break;
+      }
+
+      let rawFrame;
+      if (chunks[0].length >= totalLength) {
+        rawFrame = chunks[0].subarray(0, totalLength);
+        if (chunks[0].length === totalLength) {
+          chunks.shift();
+        } else {
+          chunks[0] = chunks[0].subarray(totalLength);
+        }
+        bufferedLength -= totalLength;
+      } else {
+        rawFrame = Buffer.allocUnsafe(totalLength);
+        let dstOff = 0;
+        let remaining = totalLength;
+        while (remaining > 0) {
+          const head = chunks[0];
+          if (head.length <= remaining) {
+            rawFrame.set(head, dstOff);
+            dstOff += head.length;
+            remaining -= head.length;
+            chunks.shift();
+          } else {
+            rawFrame.set(head.subarray(0, remaining), dstOff);
+            dstOff += remaining;
+            chunks[0] = head.subarray(remaining);
+            remaining = 0;
+          }
+        }
+        bufferedLength -= totalLength;
+      }
+
+      try {
+        const frame = new Uint8Array(rawFrame.buffer, rawFrame.byteOffset, rawFrame.byteLength);
+        const msg = decodeMessage(frame, 0, true);
+        if (msg.sourceNode) {
+          this._outboundSockets.set(msg.sourceNode, socket);
+        }
+        if (this._messageHandler) {
+          this._messageHandler(msg.sourceNode, msg);
+        }
+      } catch (err) {
+        console.error(`[TCPTransport ${this.id}] Error decoding frame:`, err.message);
+      }
+    }
+  });
+
+  socket.on('close', () => {
+    this._inboundSockets.delete(socket);
+    for (const [peer, s] of this._outboundSockets.entries()) {
+      if (s === socket) {
+        this._outboundSockets.delete(peer);
+      }
+    }
+  });
+
+  socket.on('error', () => {
+    socket.destroy();
+  });
+};
+
+// Reset stalled inflight receipts on connection drop or before starting new transfer
+ReplicationPipeline.prototype.reset = function(err = new Error('Replication pipeline reset')) {
+  for (const pending of this._pendingReceipts.values()) {
+    pending.reject(err);
+  }
+  this._pendingReceipts.clear();
+  this.inflightBatches = 0;
+  this.inflightBytes = 0;
+  this.stats.inflightBatches = 0;
+  this.stats.inflightBytes = 0;
+  this._notifyCapacity();
+};
+
 export {
   SenderPipeline,
   ReceiverPipeline,
@@ -108,16 +219,27 @@ export async function serve({
       }
 
       const manifest = await TransferManifest.build(resolved);
+      replication.reset();
       const sender = new SenderPipeline({
         replication,
         manifest,
         metrics
       });
-      await sender.transfer();
+      setImmediate(async () => {
+        try {
+          await sender.transfer();
+        } catch (err) {
+          await submitSingleMutation(
+            replication,
+            Opcodes.PULL_ERROR,
+            packPullError({ errorCode: 500, message: err.message })
+          );
+        }
+      });
       return;
     }
 
-    await receiver._processGenerationBatch(msg);
+    await receiver.enqueueBatch(msg);
   });
 
   return {
@@ -152,7 +274,7 @@ export async function send({
   knownResumeState = new Map()
 }) {
   const metrics = new MetricsCollector();
-  const progress = showProgress ? new ProgressReporter() : null;
+  const progress = showProgress ? new ProgressReporter({ role: 'sender' }) : null;
 
   // Ephemeral sender port
   const senderPort = 10000 + Math.floor(Math.random() * 20000);
@@ -214,7 +336,7 @@ export async function pull({
   showProgress = true
 }) {
   const metrics = new MetricsCollector();
-  const progress = showProgress ? new ProgressReporter() : null;
+  const progress = showProgress ? new ProgressReporter({ role: 'receiver' }) : null;
   if (progress) progress.start();
 
   const clientPort = 10000 + Math.floor(Math.random() * 20000);

package/src/receiver/pipeline.js

@@ -42,14 +42,58 @@ export class ReceiverPipeline {
     this._finishResolvers = [];
     this._errorRejecters = [];
 
+    // Processing queue to ensure strictly ordered, serialized generation processing
+    this._batchQueue = [];
+    this._processingQueue = false;
+
     // Hook up Raptiye batch handler if autoHookBatch is true
     if (this.replication && autoHookBatch) {
       this.replication.onBatch(async (msg) => {
-        await this._processGenerationBatch(msg);
+        return this.enqueueBatch(msg);
       });
     }
   }
 
+  /**
+   * Enqueue incoming generation batch to guarantee strict FIFO sequential processing.
+   * @param {object} msg
+   * @returns {Promise<void>}
+   */
+  enqueueBatch(msg) {
+    return new Promise((resolve, reject) => {
+      this._batchQueue.push({ msg, resolve, reject });
+      this._drainQueue();
+    });
+  }
+
+  async _drainQueue() {
+    if (this._processingQueue) return;
+    this._processingQueue = true;
+
+    try {
+      while (this._batchQueue.length > 0) {
+        const item = this._batchQueue.shift();
+        try {
+          await this._processGenerationBatch(item.msg);
+          item.resolve();
+        } catch (err) {
+          this.transferError = err;
+          for (const reject of this._errorRejecters) {
+            reject(err);
+          }
+          this._finishResolvers = [];
+          this._errorRejecters = [];
+          item.reject(err);
+        }
+      }
+    } finally {
+      this._processingQueue = false;
+      if (this._batchQueue.length > 0) {
+        this._drainQueue();
+      }
+    }
+  }
+
   async waitForCompletion() {
     if (this.transferError) throw this.transferError;
     if (this.transferFinished) return;
@@ -70,6 +114,12 @@ export class ReceiverPipeline {
       this.metrics.generationBytes += (metaBuf.byteLength + payloadBuf.byteLength);
     }
 
+    if (this.progress) {
+      this.progress.update({
+        generationId: msg.generationId || (this.metrics ? this.metrics.generations : 0)
+      });
+    }
+
     // Zero-copy unpacking of generation entries
     const entries = unpackGeneration(metaBuf, payloadBuf);
     const count = entries.length;
@@ -164,9 +214,12 @@ export class ReceiverPipeline {
     });
 
     if (this.progress) {
+      const fileSize = Number(info.fileSize);
+      const estimatedChunks = Math.max(1, Math.ceil(fileSize / (1024 * 1024)));
       this.progress.update({
         fileName: info.path,
-        totalBytes: Number(info.fileSize)
+        totalBytes: fileSize,
+        totalChunks: estimatedChunks
       });
     }
   }
@@ -216,10 +269,18 @@ export class ReceiverPipeline {
     }
 
     if (this.progress) {
+      const diskTimeSec = Math.max(0.001, (this.metrics ? this.metrics.diskWriteTimeMs : 1) / 1000);
+      const writtenMB = (this.metrics ? this.metrics.fileBytesWritten : 0) / (1024 * 1024);
+      const diskMBs = Number((writtenMB / diskTimeSec).toFixed(1));
+      file.maxChunkSize = Math.max(file.maxChunkSize || 0, meta.length);
+      const realTotalChunks = file.maxChunkSize > 0 && file.fileSize > 0 ? Math.ceil(file.fileSize / file.maxChunkSize) : 0;
       this.progress.update({
         fileName: file.relativePath,
         bytesDone: this.metrics ? this.metrics.fileBytesWritten : 0,
-        chunksDone: this.metrics ? this.metrics.chunksVerified : 0
+        chunksDone: this.metrics ? this.metrics.chunksVerified : 0,
+        totalChunks: realTotalChunks > 0 ? realTotalChunks : undefined,
+        resumeHits: this.metrics ? this.metrics.resumeHits : 0,
+        diskSpeedMBs: diskMBs
       });
     }
 

package/src/sender/pipeline.js

@@ -102,9 +102,10 @@ export class SenderPipeline {
     this.replication = replication;
     this.manifest = manifest;
     this.chunkSize = options.chunkSize || 1024 * 1024;
-    this.maxEntries = options.maxEntries || 8;
-    this.maxBytes = options.maxBytes || 8 * 1024 * 1024;
-    this.maxImmutableBuffers = options.maxImmutableBuffers || 4;
+    this.maxEntries = options.maxEntries || 4;
+    this.maxBytes = options.maxBytes || 4 * 1024 * 1024;
+    this.maxImmutableBuffers = options.maxImmutableBuffers || 8;
+    this.maxConcurrentFlushes = options.maxConcurrentFlushes || Math.min(this.maxImmutableBuffers, 4);
     this.metrics = metrics;
     this.progress = progress;
     this.knownResumeState = knownResumeState;
@@ -125,6 +126,8 @@ export class SenderPipeline {
       maxEntries: this.maxEntries,
       maxBytes: this.maxBytes,
       maxImmutableBuffers: this.maxImmutableBuffers,
+      maxConcurrentFlushes: this.maxConcurrentFlushes,
+      ordered: true,
       backpressurePolicy: BackpressurePolicy.WAIT,
       onBackpressure: (active) => {
         if (this.metrics && active) {