teyyare 0.2.5 → 0.2.6

package/bin/teyyare.js

@@ -13,15 +13,20 @@ function printUsage() {
 
 \x1b[1mUSAGE:\x1b[0m
   teyyare serve [--port 7421] [--dir <path>]
-  teyyare send <source> <destination>
-  teyyare pull <source> <destination>
+  teyyare send <source> <destination> [-c <channels>]
+  teyyare pull <source> [destination] [-c <channels>]
   teyyare status
 
+\x1b[1mOPTIONS:\x1b[0m
+  -c, --channels <n>    Parallel TCP connections (default: 4 for remote, 1 for local)
+  --port <port>         Port number (default: 7421)
+  --dir <path>          Serve root directory (default: current dir)
+
 \x1b[1mEXAMPLES:\x1b[0m
   teyyare serve --port 7421
   teyyare send ./10gb.bin localhost:/tmp/10gb.bin
-  teyyare send ./dist 127.0.0.1:7421:/var/www/app
-  teyyare pull server:7421:/backup/db.tar.zst ./
+  teyyare send ./dist 127.0.0.1:7421:/var/www/app -c 4
+  teyyare pull server:7421:/backup/db.tar.zst ./ -c 4
 `);
 }
 
@@ -51,7 +56,7 @@ async function main() {
   }
 
   if (command === '--version' || command === '-v' || command === 'version') {
-    console.log('0.2.5');
+    console.log('0.2.6');
     process.exit(0);
   }
 
@@ -82,8 +87,18 @@ async function main() {
     // Keep alive
     await new Promise(() => {});
   } else if (command === 'send') {
-    const source = args[1];
-    const target = args[2];
+    let channels = undefined;
+    const positional = [];
+    for (let i = 1; i < args.length; i++) {
+      if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
+        channels = Number(args[++i]);
+      } else {
+        positional.push(args[i]);
+      }
+    }
+
+    const source = positional[0];
+    const target = positional[1];
 
     if (!source || !target) {
       console.error('Error: teyyare send requires <source> and <destination>');
@@ -96,6 +111,7 @@ async function main() {
       sourcePath: source,
       host,
       port,
+      channels,
       showProgress: true
     });
 
@@ -112,8 +128,18 @@ Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
 `);
     process.exit(0);
   } else if (command === 'pull') {
-    const source = args[1];
-    const target = args[2] || './';
+    let channels = undefined;
+    const positional = [];
+    for (let i = 1; i < args.length; i++) {
+      if ((args[i] === '-c' || args[i] === '--channels') && args[i + 1]) {
+        channels = Number(args[++i]);
+      } else {
+        positional.push(args[i]);
+      }
+    }
+
+    const source = positional[0];
+    const target = positional[1] || './';
 
     if (!source) {
       console.error('Error: teyyare pull requires <source> and [destination]');
@@ -126,6 +152,7 @@ Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
       sourceRemotePath: remotePath,
       host,
       port,
+      channels,
       destinationDir: target,
       showProgress: true
     });
@@ -143,7 +170,7 @@ Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
 `);
     process.exit(0);
   } else if (command === 'status') {
-    console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.5 status: ready');
+    console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.2.6 status: ready');
   } else {
     console.error(`Unknown command: ${command}`);
     printUsage();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "teyyare",
-  "version": "0.2.5",
+  "version": "0.2.6",
   "description": "High-performance resumable file transfer engine and CLI built on muttafa and raptiye",
   "main": "src/index.js",
   "type": "module",

package/src/cli/progress.js

@@ -28,6 +28,7 @@ export class ProgressReporter {
     this.tcpPressure = 0;
     this.resumeHits = 0;
     this.diskSpeedMBs = 0;
+    this.channels = 1;
 
     this.startTime = Date.now();
     this.lastRender = 0;
@@ -68,6 +69,7 @@ export class ProgressReporter {
     if (state.tcpPressure !== undefined) this.tcpPressure = state.tcpPressure;
     if (state.resumeHits !== undefined) this.resumeHits = state.resumeHits;
     if (state.diskSpeedMBs !== undefined) this.diskSpeedMBs = state.diskSpeedMBs;
+    if (state.channels !== undefined) this.channels = state.channels;
   }
 
   formatBytes(bytes) {
@@ -126,6 +128,7 @@ export class ProgressReporter {
         '',
         `network speed    \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
         `disk speed       \x1b[1m${diskStr}\x1b[0m`,
+        `channels         ${this.channels}`,
         `ETA              ${etaStr}`,
         '',
         `chunks           ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,
@@ -143,6 +146,7 @@ export class ProgressReporter {
         `\x1b[32m${percent}%\x1b[0m`,
         '',
         `throughput       \x1b[1m${effectiveSpeed.toFixed(1)} MB/s\x1b[0m`,
+        `channels         ${this.channels}`,
         `ETA              ${etaStr}`,
         '',
         `chunks           ${this.chunksDone} / ${this.totalChunks || this.chunksDone}`,

package/src/index.js

@@ -3,9 +3,10 @@
  */
 
 import path from 'node:path';
+import net from 'node:net';
 import { existsSync } from 'node:fs';
 import { TCPTransport, ReplicationPipeline, BatchReceipt } from 'raptiye';
-import { readHeader, decodeMessage, HEADER_SIZE, writeHeader } from 'raptiye/src/protocol/wire.js';
+import { readHeader, decodeMessage, encodeMessage, HEADER_SIZE, writeHeader } from 'raptiye/src/protocol/wire.js';
 import { MessageType } from 'raptiye/src/types.js';
 import { crc32Buffers } from './core/crc32.js';
 import { SenderPipeline } from './sender/pipeline.js';
@@ -25,8 +26,19 @@ import {
   unpackPullError
 } from './core/protocol.js';
 
-// High-performance streaming TCP framer patch for TCPTransport
-// Eliminates O(N^2) Buffer.concat and ensures clean socket lifecycle management
+// High-performance streaming TCP framer & Multi-Channel socket pool patch for TCPTransport
+TCPTransport.prototype._getPeerPool = function(peer) {
+  if (!this._peerSocketPool) {
+    this._peerSocketPool = new Map();
+  }
+  let pool = this._peerSocketPool.get(peer);
+  if (!pool) {
+    pool = [];
+    this._peerSocketPool.set(peer, pool);
+  }
+  return pool;
+};
+
 TCPTransport.prototype._attachSocketReader = function(socket) {
   socket.setNoDelay(true);
   if (socket._readableState) socket._readableState.highWaterMark = 4 * 1024 * 1024;
@@ -102,6 +114,10 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
         const msg = decodeMessage(frame, 0, false);
         if (msg.sourceNode) {
           this._outboundSockets.set(msg.sourceNode, socket);
+          const pool = this._getPeerPool(msg.sourceNode);
+          if (!pool.includes(socket)) {
+            pool.push(socket);
+          }
         }
         if (this._messageHandler) {
           this._messageHandler(msg.sourceNode, msg);
@@ -119,6 +135,14 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
         this._outboundSockets.delete(peer);
       }
     }
+    if (this._peerSocketPool) {
+      for (const [peer, pool] of this._peerSocketPool.entries()) {
+        const idx = pool.indexOf(socket);
+        if (idx !== -1) {
+          pool.splice(idx, 1);
+        }
+      }
+    }
   });
 
   socket.on('error', () => {
@@ -126,6 +150,136 @@ TCPTransport.prototype._attachSocketReader = function(socket) {
   });
 };
 
+TCPTransport.prototype.connectChannels = async function(peer, channelCount = 4) {
+  if (channelCount <= 1) return;
+  const addr = this.peerAddresses.get(peer);
+  if (!addr) {
+    throw new Error(`[TCPTransport ${this.id}] No address registered for peer ${peer}`);
+  }
+
+  const pool = this._getPeerPool(peer);
+  const activeSockets = pool.filter((s) => !s.destroyed && s.writable);
+  const needed = channelCount - activeSockets.length;
+  if (needed <= 0) return;
+
+  const connectPromises = [];
+  for (let c = 0; c < needed; c++) {
+    const p = new Promise((resolve, reject) => {
+      const socket = net.createConnection({ host: addr.host, port: addr.port }, () => {
+        this._attachSocketReader(socket);
+        if (!this._outboundSockets.has(peer)) {
+          this._outboundSockets.set(peer, socket);
+        }
+        const currentPool = this._getPeerPool(peer);
+        if (!currentPool.includes(socket)) {
+          currentPool.push(socket);
+        }
+
+        // Send registration HEARTBEAT frame so remote peer associates this socket with this node
+        try {
+          const hbChunks = encodeMessage({
+            type: MessageType.HEARTBEAT,
+            sourceNode: this.id,
+            destNode: peer,
+            leaderId: this.id,
+            leaderCommit: 0n
+          });
+          socket.cork();
+          for (let i = 0; i < hbChunks.length; i++) {
+            socket.write(hbChunks[i]);
+          }
+          socket.uncork();
+        } catch (err) {
+          // Non-fatal
+        }
+
+        resolve(socket);
+      });
+
+      socket.on('error', (err) => {
+        reject(err);
+      });
+    });
+    connectPromises.push(p);
+  }
+
+  await Promise.all(connectPromises);
+  // Short pause to allow remote peer to process initial registration frames
+  await new Promise((r) => setTimeout(r, 20));
+};
+
+TCPTransport.prototype.sendv = function(peer, buffers) {
+  if (!this._channelIndex) this._channelIndex = new Map();
+
+  const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
+  if (pool && pool.length > 0) {
+    const active = pool.filter((s) => !s.destroyed && s.writable);
+    if (active.length > 0) {
+      let idx = this._channelIndex.get(peer) || 0;
+      const socket = active[idx % active.length];
+      this._channelIndex.set(peer, idx + 1);
+
+      socket.cork();
+      for (let i = 0; i < buffers.length; i++) {
+        socket.write(buffers[i]);
+      }
+      socket.uncork();
+      return;
+    }
+  }
+
+  this._getOrCreateSocket(peer)
+    .then((socket) => {
+      if (!socket.destroyed && socket.writable) {
+        socket.cork();
+        for (let i = 0; i < buffers.length; i++) {
+          socket.write(buffers[i]);
+        }
+        socket.uncork();
+      }
+    })
+    .catch(() => {});
+};
+
+TCPTransport.prototype.send = function(peer, buffer) {
+  if (!this._channelIndex) this._channelIndex = new Map();
+
+  const pool = this._peerSocketPool ? this._peerSocketPool.get(peer) : null;
+  if (pool && pool.length > 0) {
+    const active = pool.filter((s) => !s.destroyed && s.writable);
+    if (active.length > 0) {
+      let idx = this._channelIndex.get(peer) || 0;
+      const socket = active[idx % active.length];
+      this._channelIndex.set(peer, idx + 1);
+      socket.write(buffer);
+      return;
+    }
+  }
+
+  this._getOrCreateSocket(peer)
+    .then((socket) => {
+      if (!socket.destroyed && socket.writable) {
+        socket.write(buffer);
+      }
+    })
+    .catch(() => {});
+};
+
+const origClose = TCPTransport.prototype.close;
+TCPTransport.prototype.close = async function() {
+  if (this._peerSocketPool) {
+    for (const pool of this._peerSocketPool.values()) {
+      for (const socket of pool) {
+        if (!socket.destroyed) {
+          socket.destroy();
+        }
+      }
+    }
+    this._peerSocketPool.clear();
+  }
+  return origClose.call(this);
+};
+
 // Fast SIMD-accelerated BATCH_DATA encoder
 function fastEncodeBatchData(msg) {
   const bufs = msg.buffers || [];
@@ -401,12 +555,14 @@ export async function send({
   sourcePath,
   host = '127.0.0.1',
   port = 7421,
+  channels = (host === '127.0.0.1' || host === 'localhost' ? 1 : 4),
   pipelineOptions = {},
   showProgress = true,
   knownResumeState = new Map()
 }) {
   const metrics = new MetricsCollector();
   const progress = showProgress ? new ProgressReporter({ role: 'sender' }) : null;
+  if (progress) progress.update({ channels });
 
   // Ephemeral sender port
   const senderPort = 10000 + Math.floor(Math.random() * 20000);
@@ -418,6 +574,9 @@ export async function send({
   });
 
   await transport.start();
+  if (channels > 1) {
+    await transport.connectChannels(2, channels);
+  }
 
   const replication = new ReplicationPipeline({
     transport,
@@ -454,6 +613,7 @@ export async function send({
  * @param {string} options.sourceRemotePath
  * @param {string} [options.host='127.0.0.1']
  * @param {number} [options.port=7421]
+ * @param {number} [options.channels]
  * @param {string} [options.destinationDir='./']
  * @param {object} [options.pipelineOptions]
  * @param {boolean} [options.showProgress=true]
@@ -463,13 +623,17 @@ export async function pull({
   sourceRemotePath,
   host = '127.0.0.1',
   port = 7421,
+  channels = (host === '127.0.0.1' || host === 'localhost' ? 1 : 4),
   destinationDir = './',
   pipelineOptions = {},
   showProgress = true
 }) {
   const metrics = new MetricsCollector();
   const progress = showProgress ? new ProgressReporter({ role: 'receiver' }) : null;
-  if (progress) progress.start();
+  if (progress) {
+    progress.update({ channels });
+    progress.start();
+  }
 
   const clientPort = 10000 + Math.floor(Math.random() * 20000);
   const transport = new TCPTransport({
@@ -480,6 +644,9 @@ export async function pull({
   });
 
   await transport.start();
+  if (channels > 1) {
+    await transport.connectChannels(2, channels);
+  }
 
   const replication = new ReplicationPipeline({
     transport,

package/src/receiver/pipeline.js

@@ -37,6 +37,8 @@ export class ReceiverPipeline {
 
     this.activeFiles = new Map(); // fileId -> { destPath, partPath, statePath, fileHandle, resumeState, ... }
     this.completedFiles = new Set();
+    this._pendingChunks = new Map(); // fileId -> Array<{ key, val }>
+    this._pendingTransferEnd = false;
     this.transferFinished = false;
     this.transferError = null;
     this._finishResolvers = [];
@@ -135,6 +137,7 @@ export class ReceiverPipeline {
         case Opcodes.TRANSFER_BEGIN: {
           this.transferFinished = false;
           this.transferError = null;
+          this._pendingTransferEnd = false;
           const tinfo = unpackTransferBegin(key);
           break;
         }
@@ -155,12 +158,11 @@ export class ReceiverPipeline {
         }
 
         case Opcodes.TRANSFER_END: {
-          this.transferFinished = true;
-          for (const resolve of this._finishResolvers) {
-            resolve();
+          if (this.activeFiles.size === 0) {
+            this._finishTransfer();
+          } else {
+            this._pendingTransferEnd = true;
           }
-          this._finishResolvers = [];
-          this._errorRejecters = [];
           break;
         }
 
@@ -179,6 +181,15 @@ export class ReceiverPipeline {
     }
   }
 
+  _finishTransfer() {
+    this.transferFinished = true;
+    for (const resolve of this._finishResolvers) {
+      resolve();
+    }
+    this._finishResolvers = [];
+    this._errorRejecters = [];
+  }
+
   async _handleFileBegin(key) {
     const info = unpackFileBegin(key);
     const destPath = path.resolve(this.destinationDir, info.path);
@@ -200,7 +211,7 @@ export class ReceiverPipeline {
 
     const fileHandle = await fs.open(partPath, 'a+');
 
-    this.activeFiles.set(info.fileId, {
+    const file = {
       fileId: info.fileId,
       relativePath: info.path,
       destPath,
@@ -210,8 +221,12 @@ export class ReceiverPipeline {
       resumeState,
       fileSize: Number(info.fileSize),
       sha256: info.sha256,
-      chunksWrittenSinceSave: 0
-    });
+      chunksWrittenSinceSave: 0,
+      pendingEnd: false,
+      endMeta: null
+    };
+
+    this.activeFiles.set(info.fileId, file);
 
     if (this.progress) {
       const fileSize = Number(info.fileSize);
@@ -226,16 +241,38 @@ export class ReceiverPipeline {
         resumeHits: resumeState.verifiedCount
       });
     }
+
+    // Flush any pending chunks that arrived out-of-order before FILE_BEGIN
+    const pending = this._pendingChunks.get(info.fileId);
+    if (pending && pending.length > 0) {
+      this._pendingChunks.delete(info.fileId);
+      for (const item of pending) {
+        const meta = unpackChunkMeta(item.key);
+        await this._writeChunk(file, meta, item.val);
+      }
+    }
   }
 
   async _handleChunk(key, val) {
     const meta = unpackChunkMeta(key);
     const file = this.activeFiles.get(meta.fileId);
     if (!file) {
-      // Stale or out-of-order chunk from an interrupted previous session - ignore safely
+      if (this.completedFiles.has(meta.fileId)) {
+        return;
+      }
+      let pending = this._pendingChunks.get(meta.fileId);
+      if (!pending) {
+        pending = [];
+        this._pendingChunks.set(meta.fileId, pending);
+      }
+      pending.push({ key, val: new Uint8Array(val) });
       return;
     }
 
+    await this._writeChunk(file, meta, val);
+  }
+
+  async _writeChunk(file, meta, val) {
     // Idempotency / resume hit check
     if (file.resumeState.isVerified(meta.chunkIndex)) {
       if (this.metrics) {
@@ -296,6 +333,14 @@ export class ReceiverPipeline {
       file.chunksWrittenSinceSave = 0;
       await file.resumeState.save(file.statePath);
     }
+
+    // If FILE_END arrived earlier while chunks were still arriving
+    if (file.pendingEnd && (file.fileSize === 0 || file.resumeState.isComplete())) {
+      await this._finalizeFile(file);
+      if (this._pendingTransferEnd && this.activeFiles.size === 0) {
+        this._finishTransfer();
+      }
+    }
   }
 
   async _handleFileEnd(key) {
@@ -303,6 +348,22 @@ export class ReceiverPipeline {
     const file = this.activeFiles.get(meta.fileId);
     if (!file) return;
 
+    if (meta.sha256) {
+      file.sha256 = meta.sha256;
+    }
+
+    if (file.fileSize === 0 || file.resumeState.isComplete()) {
+      await this._finalizeFile(file);
+      if (this._pendingTransferEnd && this.activeFiles.size === 0) {
+        this._finishTransfer();
+      }
+    } else {
+      file.pendingEnd = true;
+      file.endMeta = meta;
+    }
+  }
+
+  async _finalizeFile(file) {
     // Save final state before rename
     await file.resumeState.save(file.statePath);
 
@@ -311,7 +372,7 @@ export class ReceiverPipeline {
     await file.fileHandle.close();
 
     // Verify whole-file checksum if available
-    const expectedSha = meta.sha256 || file.sha256;
+    const expectedSha = file.sha256;
     if (expectedSha) {
       const actualSha = await this._computeFileSha256(file.partPath);
       if (actualSha !== expectedSha) {
@@ -323,7 +384,8 @@ export class ReceiverPipeline {
     await fs.rename(file.partPath, file.destPath);
     await FileResumeState.cleanup(file.statePath);
 
-    this.activeFiles.delete(meta.fileId);
+    this.activeFiles.delete(file.fileId);
+    this.completedFiles.add(file.fileId);
     this.completedFiles.add(file.destPath);
   }
 

package/test/integration/multichannel.test.js

@@ -0,0 +1,108 @@
+import test from 'node:test';
+import assert from 'node:assert';
+import fs from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { serve, send, pull } from '../../src/index.js';
+
+async function computeSha256(filePath) {
+  const content = await fs.readFile(filePath);
+  return crypto.createHash('sha256').update(content).digest('hex');
+}
+
+test('Integration - multi-channel parallel pull transfer (4 channels) with SHA-256 verification', async () => {
+  const testDir = path.resolve('tmp/multichannel-pull-test');
+  const serverDir = path.join(testDir, 'server');
+  const clientDir = path.join(testDir, 'client');
+
+  await fs.rm(testDir, { recursive: true, force: true });
+  await fs.mkdir(serverDir, { recursive: true });
+  await fs.mkdir(clientDir, { recursive: true });
+
+  // Generate 16 MB test file
+  const testFile = path.join(serverDir, 'parallel-16mb.bin');
+  const size = 16 * 1024 * 1024;
+  const data = crypto.randomBytes(size);
+  await fs.writeFile(testFile, data);
+  const sourceSha = await computeSha256(testFile);
+
+  const port = 8921;
+  const server = await serve({ port, destinationDir: serverDir });
+
+  try {
+    const metrics = await pull({
+      sourceRemotePath: 'parallel-16mb.bin',
+      host: '127.0.0.1',
+      port,
+      channels: 4,
+      destinationDir: clientDir,
+      showProgress: false
+    });
+
+    const pulledFile = path.join(clientDir, 'parallel-16mb.bin');
+    assert.ok(existsSync(pulledFile), 'Pulled file must exist');
+    assert.strictEqual(await fs.stat(pulledFile).then((s) => s.size), size, 'Size must match');
+
+    const pulledSha = await computeSha256(pulledFile);
+    assert.strictEqual(pulledSha, sourceSha, 'SHA-256 checksum must match source exactly');
+
+    // Verify atomic rename and state cleanup
+    assert.strictEqual(existsSync(`${pulledFile}.teyyare-part`), false, 'Part file must be cleaned up');
+    assert.strictEqual(existsSync(`${pulledFile}.teyyare-state`), false, 'State file must be cleaned up');
+
+    const snap = metrics.snapshot();
+    assert.strictEqual(snap.fileBytesWritten, size);
+    assert.strictEqual(snap.chunksVerified, 16);
+    assert.ok(snap.throughputMBs > 0, 'Throughput must be positive');
+  } finally {
+    await server.close();
+    await fs.rm(testDir, { recursive: true, force: true });
+  }
+});
+
+test('Integration - multi-channel parallel send transfer (4 channels) with SHA-256 verification', async () => {
+  const testDir = path.resolve('tmp/multichannel-send-test');
+  const sourceDir = path.join(testDir, 'source');
+  const destDir = path.join(testDir, 'dest');
+
+  await fs.rm(testDir, { recursive: true, force: true });
+  await fs.mkdir(sourceDir, { recursive: true });
+  await fs.mkdir(destDir, { recursive: true });
+
+  // Generate 12 MB test file
+  const testFile = path.join(sourceDir, 'upload-12mb.bin');
+  const size = 12 * 1024 * 1024;
+  const data = crypto.randomBytes(size);
+  await fs.writeFile(testFile, data);
+  const sourceSha = await computeSha256(testFile);
+
+  const port = 8922;
+  const server = await serve({ port, destinationDir: destDir });
+
+  try {
+    const metrics = await send({
+      sourcePath: testFile,
+      host: '127.0.0.1',
+      port,
+      channels: 4,
+      showProgress: false
+    });
+
+    await server.receiver.waitForCompletion();
+
+    const receivedFile = path.join(destDir, 'upload-12mb.bin');
+    assert.ok(existsSync(receivedFile), 'Received file must exist');
+    assert.strictEqual(await fs.stat(receivedFile).then((s) => s.size), size, 'Size must match');
+
+    const receivedSha = await computeSha256(receivedFile);
+    assert.strictEqual(receivedSha, sourceSha, 'SHA-256 checksum must match source');
+
+    const snap = metrics.snapshot();
+    assert.strictEqual(snap.fileBytesRead, size);
+    assert.strictEqual(snap.chunksCreated, 12);
+  } finally {
+    await server.close();
+    await fs.rm(testDir, { recursive: true, force: true });
+  }
+});