teyyare 0.1.0

package/src/cli/progress.js

@@ -0,0 +1,125 @@
+/**
+ * Throttled terminal progress renderer.
+ */
+
+export class ProgressReporter {
+  /**
+   * @param {object} [options]
+   * @param {number} [options.intervalMs=150]
+   * @param {boolean} [options.enabled=true]
+   */
+  constructor({ intervalMs = 150, enabled = true } = {}) {
+    this.intervalMs = intervalMs;
+    this.enabled = enabled && Boolean(process.stdout.isTTY || process.env.CI !== 'true');
+
+    this.fileName = '';
+    this.bytesDone = 0;
+    this.totalBytes = 0;
+    this.chunksDone = 0;
+    this.totalChunks = 0;
+    this.generationId = 0;
+    this.inflightGen = 0;
+    this.inflightBytes = 0;
+    this.muttafaQueue = 0;
+    this.raptiyeRTT = 0;
+    this.tcpPressure = 0;
+
+    this.startTime = Date.now();
+    this.lastRender = 0;
+    this._timer = null;
+    this._linesRendered = 0;
+  }
+
+  start() {
+    if (!this.enabled) return;
+    this.startTime = Date.now();
+    this._timer = setInterval(() => this.render(), this.intervalMs);
+  }
+
+  stop() {
+    if (this._timer) {
+      clearInterval(this._timer);
+      this._timer = null;
+    }
+    if (this.enabled) {
+      this.render();
+      process.stdout.write('\n');
+    }
+  }
+
+  update(state = {}) {
+    if (state.fileName !== undefined) this.fileName = state.fileName;
+    if (state.bytesDone !== undefined) this.bytesDone = state.bytesDone;
+    if (state.totalBytes !== undefined) this.totalBytes = state.totalBytes;
+    if (state.chunksDone !== undefined) this.chunksDone = state.chunksDone;
+    if (state.totalChunks !== undefined) this.totalChunks = state.totalChunks;
+    if (state.generationId !== undefined) this.generationId = state.generationId;
+    if (state.inflightGen !== undefined) this.inflightGen = state.inflightGen;
+    if (state.inflightBytes !== undefined) this.inflightBytes = state.inflightBytes;
+    if (state.muttafaQueue !== undefined) this.muttafaQueue = state.muttafaQueue;
+    if (state.raptiyeRTT !== undefined) this.raptiyeRTT = state.raptiyeRTT;
+    if (state.tcpPressure !== undefined) this.tcpPressure = state.tcpPressure;
+  }
+
+  formatBytes(bytes) {
+    if (bytes >= 1024 * 1024 * 1024) {
+      return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
+    }
+    if (bytes >= 1024 * 1024) {
+      return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
+    }
+    if (bytes >= 1024) {
+      return `${(bytes / 1024).toFixed(2)} KB`;
+    }
+    return `${bytes} B`;
+  }
+
+  render() {
+    if (!this.enabled) return;
+
+    const elapsedSec = Math.max(0.001, (Date.now() - this.startTime) / 1000);
+    const throughputMBs = (this.bytesDone / (1024 * 1024)) / elapsedSec;
+    const percent = this.totalBytes > 0 ? ((this.bytesDone / this.totalBytes) * 100).toFixed(1) : '0.0';
+
+    let etaStr = '--:--';
+    if (throughputMBs > 0 && this.totalBytes > this.bytesDone) {
+      const remainingBytes = this.totalBytes - this.bytesDone;
+      const remainingSec = Math.round(remainingBytes / (throughputMBs * 1024 * 1024));
+      const mm = String(Math.floor(remainingSec / 60)).padStart(2, '0');
+      const ss = String(remainingSec % 60).padStart(2, '0');
+      etaStr = `${mm}:${ss}`;
+    }
+
+    const lines = [
+      '\x1b[1m\x1b[36mTeyyare\x1b[0m',
+      '',
+      `\x1b[1m${this.fileName || 'Transferring...'}\x1b[0m`,
+      '',
+      `${this.formatBytes(this.bytesDone)} / ${this.formatBytes(this.totalBytes)}`,
+      `\x1b[32m${percent}%\x1b[0m`,
+      '',
+      `throughput       \x1b[1m${throughputMBs.toFixed(1)} MB/s\x1b[0m`,
+      `ETA              ${etaStr}`,
+      '',
+      `chunks           ${this.chunksDone} / ${this.totalChunks}`,
+      `generation       ${this.generationId}`,
+      `inflight gen     ${this.inflightGen}`,
+      `inflight bytes   ${this.formatBytes(this.inflightBytes)}`,
+      '',
+      `muttafa queue    ${this.muttafaQueue}`,
+      `raptiye RTT      ${this.raptiyeRTT} ms`,
+      `TCP pressure     ${this.tcpPressure}`
+    ];
+
+    if (process.stdout.isTTY) {
+      if (this._linesRendered > 0) {
+        process.stdout.write(`\x1b[${this._linesRendered}A\x1b[0J`);
+      }
+      process.stdout.write(lines.join('\n') + '\n');
+      this._linesRendered = lines.length;
+    } else {
+      // Non-interactive log
+      process.stdout.write(`[Teyyare] ${percent}% | ${throughputMBs.toFixed(1)} MB/s | Chunks: ${this.chunksDone}/${this.totalChunks} | ETA: ${etaStr}\n`);
+    }
+  }
+}

package/src/core/protocol.js

@@ -0,0 +1,163 @@
+/**
+ * Packed binary mutation protocol for Teyyare.
+ * Zero JSON on chunk hot-path.
+ */
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+export const Opcodes = Object.freeze({
+  TRANSFER_BEGIN: 10,
+  FILE_BEGIN: 11,
+  CHUNK: 12,
+  FILE_END: 13,
+  TRANSFER_END: 14
+});
+
+export const CHUNK_META_SIZE = 25;
+
+/**
+ * Pack binary chunk metadata into a 25-byte buffer.
+ * Layout:
+ * 0..3:   fileId (u32 BE)
+ * 4..7:   chunkIndex (u32 BE)
+ * 8..15:  offset (u64 BigInt BE)
+ * 16..19: length (u32 BE)
+ * 20..23: checksum (u32 BE, CRC32)
+ * 24:     flags (u8)
+ * 
+ * @param {object} meta
+ * @param {number} meta.fileId
+ * @param {number} meta.chunkIndex
+ * @param {number|bigint} meta.offset
+ * @param {number} meta.length
+ * @param {number} meta.checksum
+ * @param {number} [meta.flags=0]
+ * @returns {Uint8Array}
+ */
+export function packChunkMeta({ fileId, chunkIndex, offset, length, checksum, flags = 0 }) {
+  const buf = new Uint8Array(CHUNK_META_SIZE);
+  const view = new DataView(buf.buffer, buf.byteOffset, CHUNK_META_SIZE);
+  view.setUint32(0, fileId, false);
+  view.setUint32(4, chunkIndex, false);
+  view.setBigUint64(8, BigInt(offset), false);
+  view.setUint32(16, length, false);
+  view.setUint32(20, checksum >>> 0, false);
+  view.setUint8(24, flags);
+  return buf;
+}
+
+/**
+ * Unpack binary chunk metadata from a buffer slice.
+ * @param {Uint8Array} buf
+ * @param {number} [offset=0]
+ * @returns {object}
+ */
+export function unpackChunkMeta(buf, offset = 0) {
+  const view = new DataView(buf.buffer, buf.byteOffset + offset, CHUNK_META_SIZE);
+  return {
+    fileId: view.getUint32(0, false),
+    chunkIndex: view.getUint32(4, false),
+    offset: view.getBigUint64(8, false),
+    length: view.getUint32(16, false),
+    checksum: view.getUint32(20, false),
+    flags: view.getUint8(24)
+  };
+}
+
+/**
+ * Pack FILE_BEGIN metadata.
+ * Layout:
+ * 0..3:   fileId (u32)
+ * 4..11:  fileSize (u64)
+ * 12..13: pathLen (u16)
+ * 14..45: sha256 hex (32 bytes raw or 64 hex chars)
+ * 46..:   path utf8
+ */
+export function packFileBegin({ fileId, fileSize, path, sha256 = '' }) {
+  const pathBytes = encoder.encode(path);
+  const shaBytes = encoder.encode(sha256.padEnd(64, '0').slice(0, 64));
+  const total = 14 + 64 + pathBytes.byteLength;
+  const buf = new Uint8Array(total);
+  const view = new DataView(buf.buffer, buf.byteOffset, total);
+
+  view.setUint32(0, fileId, false);
+  view.setBigUint64(4, BigInt(fileSize), false);
+  view.setUint16(12, pathBytes.byteLength, false);
+  buf.set(shaBytes, 14);
+  buf.set(pathBytes, 14 + 64);
+  return buf;
+}
+
+export function unpackFileBegin(buf) {
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  const fileId = view.getUint32(0, false);
+  const fileSize = view.getBigUint64(4, false);
+  const pathLen = view.getUint16(12, false);
+  const sha256 = decoder.decode(buf.subarray(14, 14 + 64)).replace(/0+$/, '');
+  const path = decoder.decode(buf.subarray(14 + 64, 14 + 64 + pathLen));
+  return { fileId, fileSize, path, sha256 };
+}
+
+/**
+ * Pack TRANSFER_BEGIN metadata.
+ */
+export function packTransferBegin({ transferId, fileCount, totalBytes }) {
+  const idBytes = encoder.encode(transferId);
+  const buf = new Uint8Array(14 + idBytes.byteLength);
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  view.setUint32(0, fileCount, false);
+  view.setBigUint64(4, BigInt(totalBytes), false);
+  view.setUint16(12, idBytes.byteLength, false);
+  buf.set(idBytes, 14);
+  return buf;
+}
+
+export function unpackTransferBegin(buf) {
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  const fileCount = view.getUint32(0, false);
+  const totalBytes = view.getBigUint64(4, false);
+  const idLen = view.getUint16(12, false);
+  const transferId = decoder.decode(buf.subarray(14, 14 + idLen));
+  return { transferId, fileCount, totalBytes };
+}
+
+/**
+ * Pack FILE_END metadata.
+ */
+export function packFileEnd({ fileId, sha256 = '' }) {
+  const shaBytes = encoder.encode(sha256.padEnd(64, '0').slice(0, 64));
+  const buf = new Uint8Array(4 + 64);
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  view.setUint32(0, fileId, false);
+  buf.set(shaBytes, 4);
+  return buf;
+}
+
+export function unpackFileEnd(buf) {
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  const fileId = view.getUint32(0, false);
+  const sha256 = decoder.decode(buf.subarray(4, 4 + 64)).replace(/0+$/, '');
+  return { fileId, sha256 };
+}
+
+/**
+ * Pack TRANSFER_END metadata.
+ */
+export function packTransferEnd({ transferId, status = 0 }) {
+  const idBytes = encoder.encode(transferId);
+  const buf = new Uint8Array(3 + idBytes.byteLength);
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  view.setUint8(0, status);
+  view.setUint16(1, idBytes.byteLength, false);
+  buf.set(idBytes, 3);
+  return buf;
+}
+
+export function unpackTransferEnd(buf) {
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  const status = view.getUint8(0);
+  const idLen = view.getUint16(1, false);
+  const transferId = decoder.decode(buf.subarray(3, 3 + idLen));
+  return { transferId, status };
+}

package/src/index.js

@@ -0,0 +1,139 @@
+/**
+ * Teyyare: High-performance resumable file transfer engine built on Muttafa and Raptiye.
+ */
+
+import path from 'node:path';
+import { TCPTransport, ReplicationPipeline } from 'raptiye';
+import { SenderPipeline } from './sender/pipeline.js';
+import { ReceiverPipeline } from './receiver/pipeline.js';
+import { TransferManifest } from './manifest/manifest.js';
+import { FileResumeState } from './resume/state.js';
+import { MetricsCollector } from './metrics/collector.js';
+import { ProgressReporter } from './cli/progress.js';
+import { Opcodes, packChunkMeta, unpackChunkMeta } from './core/protocol.js';
+
+export {
+  SenderPipeline,
+  ReceiverPipeline,
+  TransferManifest,
+  FileResumeState,
+  MetricsCollector,
+  ProgressReporter,
+  Opcodes,
+  packChunkMeta,
+  unpackChunkMeta
+};
+
+/**
+ * Start a Teyyare receiver server.
+ * @param {object} options
+ * @param {number} [options.port=7421]
+ * @param {string} [options.host='0.0.0.0']
+ * @param {string} [options.destinationDir='./']
+ * @param {number} [options.writeDelayMs=0]
+ * @returns {Promise<{ close: () => Promise<void>, receiver: ReceiverPipeline, transport: TCPTransport, metrics: MetricsCollector }>}
+ */
+export async function serve({
+  port = 7421,
+  host = '0.0.0.0',
+  destinationDir = './',
+  writeDelayMs = 0
+} = {}) {
+  const metrics = new MetricsCollector();
+  const transport = new TCPTransport({
+    id: 2,
+    port,
+    host,
+    peerAddresses: new Map()
+  });
+
+  await transport.start();
+
+  const replication = new ReplicationPipeline({
+    transport,
+    localNode: 2,
+    remoteNode: 1,
+    maxInflightBatches: 16,
+    maxInflightBytes: 64 * 1024 * 1024
+  });
+
+  const receiver = new ReceiverPipeline({
+    replication,
+    destinationDir,
+    metrics,
+    writeDelayMs
+  });
+
+  return {
+    transport,
+    replication,
+    receiver,
+    metrics,
+    async close() {
+      metrics.stop();
+      await transport.close();
+    }
+  };
+}
+
+/**
+ * Send a file or directory to a remote Teyyare server.
+ * @param {object} options
+ * @param {string} options.sourcePath
+ * @param {string} [options.host='127.0.0.1']
+ * @param {number} [options.port=7421]
+ * @param {object} [options.pipelineOptions]
+ * @param {boolean} [options.showProgress=true]
+ * @param {Map<number, import('./resume/state.js').FileResumeState>} [options.knownResumeState]
+ * @returns {Promise<MetricsCollector>}
+ */
+export async function send({
+  sourcePath,
+  host = '127.0.0.1',
+  port = 7421,
+  pipelineOptions = {},
+  showProgress = true,
+  knownResumeState = new Map()
+}) {
+  const metrics = new MetricsCollector();
+  const progress = showProgress ? new ProgressReporter() : null;
+
+  // Ephemeral sender port
+  const senderPort = 10000 + Math.floor(Math.random() * 20000);
+  const transport = new TCPTransport({
+    id: 1,
+    port: senderPort,
+    host: '127.0.0.1',
+    peerAddresses: new Map([[2, { host, port }]])
+  });
+
+  await transport.start();
+
+  const replication = new ReplicationPipeline({
+    transport,
+    localNode: 1,
+    remoteNode: 2,
+    maxInflightBatches: pipelineOptions.maxInflightBatches || 8,
+    maxInflightBytes: pipelineOptions.maxInflightBytes || 32 * 1024 * 1024
+  });
+
+  const manifest = await TransferManifest.build(sourcePath);
+
+  const sender = new SenderPipeline({
+    replication,
+    manifest,
+    options: pipelineOptions,
+    metrics,
+    progress,
+    knownResumeState
+  });
+
+  try {
+    await sender.transfer();
+  } finally {
+    metrics.stop();
+    await transport.close();
+  }
+
+  return metrics;
+}

package/src/manifest/manifest.js

@@ -0,0 +1,84 @@
+/**
+ * Manifest generator for files and directories.
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+export class TransferManifest {
+  /**
+   * @param {object} params
+   * @param {string} params.rootPath
+   * @param {boolean} params.isDirectory
+   * @param {Array<{ fileId: number, relativePath: string, size: number, mode?: number }>} params.files
+   */
+  constructor({ rootPath, isDirectory, files = [] }) {
+    this.rootPath = rootPath;
+    this.isDirectory = isDirectory;
+    this.files = files;
+
+    this.totalFiles = files.length;
+    this.totalBytes = files.reduce((acc, f) => acc + f.size, 0);
+  }
+
+  /**
+   * Build a manifest for a single file or directory.
+   * @param {string} sourcePath
+   * @returns {Promise<TransferManifest>}
+   */
+  static async build(sourcePath) {
+    const absPath = path.resolve(sourcePath);
+    const stat = await fs.stat(absPath);
+
+    if (stat.isFile()) {
+      const base = path.basename(absPath);
+      return new TransferManifest({
+        rootPath: absPath,
+        isDirectory: false,
+        files: [
+          {
+            fileId: 1,
+            relativePath: base,
+            size: stat.size,
+            mode: stat.mode
+          }
+        ]
+      });
+    }
+
+    if (stat.isDirectory()) {
+      const files = [];
+      let fileId = 1;
+
+      async function walk(currentDir, relPrefix = '') {
+        const entries = await fs.readdir(currentDir, { withFileTypes: true });
+        for (const entry of entries) {
+          const entryPath = path.join(currentDir, entry.name);
+          const entryRel = relPrefix ? path.join(relPrefix, entry.name) : entry.name;
+
+          if (entry.isDirectory()) {
+            await walk(entryPath, entryRel);
+          } else if (entry.isFile()) {
+            const fstat = await fs.stat(entryPath);
+            files.push({
+              fileId: fileId++,
+              relativePath: entryRel,
+              size: fstat.size,
+              mode: fstat.mode
+            });
+          }
+        }
+      }
+
+      await walk(absPath);
+
+      return new TransferManifest({
+        rootPath: absPath,
+        isDirectory: true,
+        files
+      });
+    }
+
+    throw new Error(`Source ${sourcePath} is neither a regular file nor a directory`);
+  }
+}

package/src/metrics/collector.js

@@ -0,0 +1,119 @@
+/**
+ * Unified Metrics Collector across Teyyare, Muttafa, and Raptiye layers.
+ */
+
+export class MetricsCollector {
+  constructor() {
+    this.startTime = Date.now();
+    this.endTime = 0;
+
+    // Teyyare
+    this.fileBytesRead = 0;
+    this.fileBytesWritten = 0;
+    this.chunksCreated = 0;
+    this.chunksVerified = 0;
+    this.resumeHits = 0;
+    this.checksumTimeMs = 0;
+    this.diskReadTimeMs = 0;
+    this.diskWriteTimeMs = 0;
+
+    // Muttafa
+    this.mutations = 0;
+    this.generations = 0;
+    this.generationBytes = 0;
+    this.freezeTimeMs = 0;
+    this.queueDepth = 0;
+    this.backpressureCount = 0;
+    this.arenaBytes = 0;
+
+    // Raptiye
+    this.payloadBytes = 0;
+    this.wireBytes = 0;
+    this.frames = 0;
+    this.tcpWrites = 0;
+    this.tcpDrains = 0;
+    this.inflightBytes = 0;
+    this.inflightBatches = 0;
+    this.rttMs = 0;
+    this.ackLatencyMs = 0;
+    this.reconnects = 0;
+
+    // Resource tracking
+    this.peakRSS = process.memoryUsage().rss;
+    this._rssInterval = setInterval(() => {
+      const current = process.memoryUsage().rss;
+      if (current > this.peakRSS) {
+        this.peakRSS = current;
+      }
+    }, 100);
+  }
+
+  stop() {
+    if (this._rssInterval) {
+      clearInterval(this._rssInterval);
+      this._rssInterval = null;
+    }
+    this.endTime = Date.now();
+  }
+
+  get elapsedMs() {
+    return (this.endTime || Date.now()) - this.startTime;
+  }
+
+  get throughputMBs() {
+    const sec = this.elapsedMs / 1000;
+    if (sec <= 0) return 0;
+    const bytes = Math.max(this.fileBytesRead, this.fileBytesWritten);
+    return (bytes / (1024 * 1024)) / sec;
+  }
+
+  get protocolOverheadPercent() {
+    if (this.wireBytes <= 0 || this.payloadBytes <= 0) return 0;
+    const diff = Math.max(0, this.wireBytes - this.payloadBytes);
+    return (diff / this.wireBytes) * 100;
+  }
+
+  snapshot() {
+    const mem = process.memoryUsage();
+    return {
+      elapsedMs: this.elapsedMs,
+      throughputMBs: this.throughputMBs,
+      protocolOverheadPercent: this.protocolOverheadPercent,
+      peakRSS: this.peakRSS,
+      currentRSS: mem.rss,
+      heapUsed: mem.heapUsed,
+      external: mem.external,
+
+      // Teyyare
+      fileBytesRead: this.fileBytesRead,
+      fileBytesWritten: this.fileBytesWritten,
+      chunksCreated: this.chunksCreated,
+      chunksVerified: this.chunksVerified,
+      resumeHits: this.resumeHits,
+      checksumTimeMs: this.checksumTimeMs,
+      diskReadTimeMs: this.diskReadTimeMs,
+      diskWriteTimeMs: this.diskWriteTimeMs,
+
+      // Muttafa
+      mutations: this.mutations,
+      generations: this.generations,
+      generationBytes: this.generationBytes,
+      freezeTimeMs: this.freezeTimeMs,
+      queueDepth: this.queueDepth,
+      backpressureCount: this.backpressureCount,
+      arenaBytes: this.arenaBytes,
+
+      // Raptiye
+      payloadBytes: this.payloadBytes,
+      wireBytes: this.wireBytes,
+      frames: this.frames,
+      tcpWrites: this.tcpWrites,
+      tcpDrains: this.tcpDrains,
+      inflightBytes: this.inflightBytes,
+      inflightBatches: this.inflightBatches,
+      rttMs: this.rttMs,
+      ackLatencyMs: this.ackLatencyMs,
+      reconnects: this.reconnects
+    };
+  }
+}