teyyare 0.1.0

package/src/receiver/pipeline.js

@@ -0,0 +1,234 @@
+/**
+ * Teyyare Receiver Engine.
+ * Consumes delivered Muttafa generations from Raptiye, executes direct random-access writes,
+ * manages resume state, performs whole-file verification, and atomically finalizes files.
+ */
+
+import fs from 'node:fs/promises';
+import { existsSync, createReadStream } from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { FrozenArenaStore } from 'muttafa';
+import { crc32 } from 'raptiye';
+import { Opcodes, unpackChunkMeta, unpackFileBegin, unpackTransferBegin, unpackFileEnd } from '../core/protocol.js';
+import { FileResumeState } from '../resume/state.js';
+
+export class ReceiverPipeline {
+  /**
+   * @param {object} config
+   * @param {import('raptiye').ReplicationPipeline} config.replication
+   * @param {string} config.destinationDir
+   * @param {import('../metrics/collector.js').MetricsCollector} [config.metrics]
+   * @param {number} [config.writeDelayMs=0] For artificial slow-receiver backpressure benchmarking
+   */
+  constructor({
+    replication,
+    destinationDir,
+    metrics = null,
+    writeDelayMs = 0
+  }) {
+    this.replication = replication;
+    this.destinationDir = path.resolve(destinationDir);
+    this.metrics = metrics;
+    this.writeDelayMs = writeDelayMs;
+
+    this.activeFiles = new Map(); // fileId -> { destPath, partPath, statePath, fileHandle, resumeState, ... }
+    this.completedFiles = new Set();
+    this.transferFinished = false;
+    this._finishResolvers = [];
+
+    // Hook up Raptiye batch handler
+    this.replication.onBatch(async (msg) => {
+      await this._processGenerationBatch(msg);
+    });
+  }
+
+  async waitForCompletion() {
+    if (this.transferFinished) return;
+    return new Promise((resolve) => this._finishResolvers.push(resolve));
+  }
+
+  async _processGenerationBatch(msg) {
+    if (!msg.buffers || msg.buffers.length < 2) {
+      return;
+    }
+
+    const [metaBuf, payloadBuf] = msg.buffers;
+    if (this.metrics) {
+      this.metrics.generations++;
+      this.metrics.generationBytes += (metaBuf.byteLength + payloadBuf.byteLength);
+    }
+
+    // Zero-copy reconstruction of Muttafa generation
+    const store = FrozenArenaStore.fromBuffers(metaBuf, payloadBuf);
+    const count = store.length;
+
+    for (let i = 0; i < count; i++) {
+      const type = store.getType(i);
+      const key = store.getKey(i);
+      const val = store.getValue(i);
+
+      if (this.metrics) {
+        this.metrics.mutations++;
+      }
+
+      switch (type) {
+        case Opcodes.TRANSFER_BEGIN: {
+          const tinfo = unpackTransferBegin(key);
+          break;
+        }
+
+        case Opcodes.FILE_BEGIN: {
+          await this._handleFileBegin(key);
+          break;
+        }
+
+        case Opcodes.CHUNK: {
+          await this._handleChunk(key, val);
+          break;
+        }
+
+        case Opcodes.FILE_END: {
+          await this._handleFileEnd(key);
+          break;
+        }
+
+        case Opcodes.TRANSFER_END: {
+          this.transferFinished = true;
+          for (const resolve of this._finishResolvers) {
+            resolve();
+          }
+          this._finishResolvers = [];
+          break;
+        }
+      }
+    }
+  }
+
+  async _handleFileBegin(key) {
+    const info = unpackFileBegin(key);
+    const destPath = path.resolve(this.destinationDir, info.path);
+    const parentDir = path.dirname(destPath);
+    await fs.mkdir(parentDir, { recursive: true });
+
+    const partPath = `${destPath}.teyyare-part`;
+    const statePath = `${destPath}.teyyare-state`;
+
+    let resumeState = await FileResumeState.load(statePath);
+    if (!resumeState) {
+      resumeState = new FileResumeState({
+        fileId: info.fileId,
+        fileSize: Number(info.fileSize),
+        sha256: info.sha256
+      });
+      await resumeState.save(statePath);
+    }
+
+    const fileHandle = await fs.open(partPath, 'a+');
+
+    this.activeFiles.set(info.fileId, {
+      fileId: info.fileId,
+      relativePath: info.path,
+      destPath,
+      partPath,
+      statePath,
+      fileHandle,
+      resumeState,
+      fileSize: Number(info.fileSize),
+      sha256: info.sha256,
+      chunksWrittenSinceSave: 0
+    });
+  }
+
+  async _handleChunk(key, val) {
+    const meta = unpackChunkMeta(key);
+    const file = this.activeFiles.get(meta.fileId);
+    if (!file) {
+      throw new Error(`Unknown fileId ${meta.fileId} in CHUNK mutation`);
+    }
+
+    // Idempotency / resume hit check
+    if (file.resumeState.isVerified(meta.chunkIndex)) {
+      if (this.metrics) {
+        this.metrics.resumeHits++;
+        this.metrics.chunksVerified++;
+      }
+      return;
+    }
+
+    // Per-chunk corruption detection
+    const t0 = Date.now();
+    const computedCrc = crc32(val);
+    if (this.metrics) {
+      this.metrics.checksumTimeMs += (Date.now() - t0);
+    }
+
+    if (computedCrc !== meta.checksum) {
+      throw new Error(
+        `Chunk ${meta.chunkIndex} corruption: CRC 0x${computedCrc.toString(16)} !== 0x${meta.checksum.toString(16)}`
+      );
+    }
+
+    // Artificial write delay for slow-receiver backpressure tests
+    if (this.writeDelayMs > 0) {
+      await new Promise((r) => setTimeout(r, this.writeDelayMs));
+    }
+
+    // Direct random-access write at specific byte offset
+    const writeStart = Date.now();
+    await file.fileHandle.write(val, 0, meta.length, Number(meta.offset));
+    if (this.metrics) {
+      this.metrics.diskWriteTimeMs += (Date.now() - writeStart);
+      this.metrics.fileBytesWritten += meta.length;
+      this.metrics.chunksCreated++;
+      this.metrics.chunksVerified++;
+    }
+
+    file.resumeState.markVerified(meta.chunkIndex);
+    file.chunksWrittenSinceSave++;
+
+    if (file.chunksWrittenSinceSave >= 32) {
+      file.chunksWrittenSinceSave = 0;
+      await file.resumeState.save(file.statePath);
+    }
+  }
+
+  async _handleFileEnd(key) {
+    const meta = unpackFileEnd(key);
+    const file = this.activeFiles.get(meta.fileId);
+    if (!file) return;
+
+    // Save final state before rename
+    await file.resumeState.save(file.statePath);
+
+    // Sync disk buffers
+    await file.fileHandle.sync();
+    await file.fileHandle.close();
+
+    // Verify whole-file checksum if available
+    const expectedSha = meta.sha256 || file.sha256;
+    if (expectedSha) {
+      const actualSha = await this._computeFileSha256(file.partPath);
+      if (actualSha !== expectedSha) {
+        throw new Error(`Whole-file SHA-256 mismatch for ${file.relativePath}: actual ${actualSha} !== expected ${expectedSha}`);
+      }
+    }
+
+    // Atomic finalization
+    await fs.rename(file.partPath, file.destPath);
+    await FileResumeState.cleanup(file.statePath);
+
+    this.activeFiles.delete(meta.fileId);
+    this.completedFiles.add(file.destPath);
+  }
+
+  async _computeFileSha256(filePath) {
+    return new Promise((resolve, reject) => {
+      const hash = crypto.createHash('sha256');
+      const stream = createReadStream(filePath);
+      stream.on('data', (d) => hash.update(d));
+      stream.on('end', () => resolve(hash.digest('hex')));
+      stream.on('error', reject);
+    });
+  }
+}

package/src/resume/state.js

@@ -0,0 +1,125 @@
+/**
+ * Resumable transfer state management.
+ * Persists verified chunk bitmaps/sets with atomic file replacement.
+ */
+
+import fs from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+
+export class FileResumeState {
+  /**
+   * @param {object} params
+   * @param {string} [params.transferId='']
+   * @param {number} [params.fileId=1]
+   * @param {number} [params.fileSize=0]
+   * @param {number} [params.chunkSize=1048576]
+   * @param {string} [params.sha256='']
+   * @param {Set<number>} [params.verifiedChunks=new Set()]
+   */
+  constructor({
+    transferId = '',
+    fileId = 1,
+    fileSize = 0,
+    chunkSize = 1048576,
+    sha256 = '',
+    verifiedChunks = new Set()
+  } = {}) {
+    this.transferId = transferId;
+    this.fileId = fileId;
+    this.fileSize = fileSize;
+    this.chunkSize = chunkSize;
+    this.sha256 = sha256;
+    this.verifiedChunks = verifiedChunks;
+  }
+
+  get verifiedCount() {
+    return this.verifiedChunks.size;
+  }
+
+  isVerified(chunkIndex) {
+    return this.verifiedChunks.has(chunkIndex);
+  }
+
+  markVerified(chunkIndex) {
+    this.verifiedChunks.add(chunkIndex);
+  }
+
+  get totalChunks() {
+    if (this.fileSize === 0) return 0;
+    return Math.ceil(this.fileSize / this.chunkSize);
+  }
+
+  isComplete() {
+    return this.totalChunks > 0 && this.verifiedChunks.size >= this.totalChunks;
+  }
+
+  /**
+   * Return array of missing chunk indices.
+   * @returns {number[]}
+   */
+  getMissingChunks() {
+    const missing = [];
+    const total = this.totalChunks;
+    for (let i = 0; i < total; i++) {
+      if (!this.verifiedChunks.has(i)) {
+        missing.push(i);
+      }
+    }
+    return missing;
+  }
+
+  /**
+   * Atomically save state to disk.
+   * @param {string} stateFilePath
+   */
+  async save(stateFilePath) {
+    const data = {
+      transferId: this.transferId,
+      fileId: this.fileId,
+      fileSize: this.fileSize,
+      chunkSize: this.chunkSize,
+      sha256: this.sha256,
+      verifiedChunks: Array.from(this.verifiedChunks)
+    };
+
+    const tmpPath = `${stateFilePath}.tmp`;
+    await fs.writeFile(tmpPath, JSON.stringify(data), 'utf8');
+    await fs.rename(tmpPath, stateFilePath);
+  }
+
+  /**
+   * Load state from disk if exists.
+   * @param {string} stateFilePath
+   * @returns {Promise<FileResumeState|null>}
+   */
+  static async load(stateFilePath) {
+    if (!existsSync(stateFilePath)) {
+      return null;
+    }
+
+    try {
+      const raw = await fs.readFile(stateFilePath, 'utf8');
+      const data = JSON.parse(raw);
+      return new FileResumeState({
+        transferId: data.transferId,
+        fileId: data.fileId,
+        fileSize: data.fileSize,
+        chunkSize: data.chunkSize,
+        sha256: data.sha256,
+        verifiedChunks: new Set(data.verifiedChunks || [])
+      });
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Clean up state file.
+   * @param {string} stateFilePath
+   */
+  static async cleanup(stateFilePath) {
+    try {
+      await fs.unlink(stateFilePath);
+    } catch {}
+  }
+}

package/src/sender/pipeline.js

@@ -0,0 +1,288 @@
+/**
+ * Teyyare Sender Engine.
+ * Reads source files, computes per-chunk checksums, appends mutations into Muttafa
+ * MutableGeneration, manages generation freeze, submits scatter/gather FrozenGenerations
+ * to Raptiye, and propagates end-to-end backpressure from Raptiye/TCP to disk reads.
+ */
+
+import fs from 'node:fs/promises';
+import crypto from 'node:crypto';
+import { MutationEngine, ArenaStore, BumpAllocator, Sink, BackpressurePolicy } from 'muttafa';
+import { crc32 } from 'raptiye';
+import {
+  Opcodes,
+  packChunkMeta,
+  packFileBegin,
+  packFileEnd,
+  packTransferBegin,
+  packTransferEnd
+} from '../core/protocol.js';
+
+class RaptiyeSink extends Sink {
+  /**
+   * @param {import('raptiye').ReplicationPipeline} replication
+   * @param {import('../metrics/collector.js').MetricsCollector} [metrics]
+   * @param {import('../cli/progress.js').ProgressReporter} [progress]
+   */
+  constructor(replication, metrics = null, progress = null) {
+    super();
+    this.replication = replication;
+    this.metrics = metrics;
+    this.progress = progress;
+  }
+
+  /**
+   * Flush frozen generation to Raptiye replication pipeline.
+   * @param {import('muttafa').ImmutableBuffer} batch
+   */
+  async flush(batch) {
+    const buffers = batch.buffers();
+    const t0 = Date.now();
+
+    if (this.metrics) {
+      this.metrics.generations++;
+      this.metrics.generationBytes += batch.byteLength;
+    }
+
+    const receipt = await this.replication.submitBatch({
+      generationId: batch.generationId,
+      firstSequence: batch.firstSequence,
+      lastSequence: batch.lastSequence,
+      entryCount: batch.entryCount,
+      byteLength: batch.byteLength,
+      buffers
+    });
+
+    if (this.progress) {
+      const stats = this.replication.getStats();
+      this.progress.update({
+        generationId: batch.generationId,
+        inflightGen: stats.inflightBatches,
+        inflightBytes: stats.inflightBytes,
+        raptiyeRTT: stats.rttMs
+      });
+    }
+
+    // Wait until remote peer ACKs receipt
+    await receipt.delivered();
+
+    if (this.metrics) {
+      this.metrics.freezeTimeMs += (Date.now() - t0);
+    }
+  }
+}
+
+export class SenderPipeline {
+  /**
+   * @param {object} config
+   * @param {import('raptiye').ReplicationPipeline} config.replication
+   * @param {import('../manifest/manifest.js').TransferManifest} config.manifest
+   * @param {object} [config.options]
+   * @param {number} [config.options.chunkSize=1048576] 1 MiB chunks default
+   * @param {number} [config.options.maxEntries=8] 8 chunks per generation
+   * @param {number} [config.options.maxBytes=8388608] 8 MiB per generation
+   * @param {number} [config.options.maxImmutableBuffers=4] Bounded queue depth
+   * @param {import('../metrics/collector.js').MetricsCollector} [config.metrics]
+   * @param {import('../cli/progress.js').ProgressReporter} [config.progress]
+   * @param {Map<number, import('../resume/state.js').FileResumeState>} [config.knownResumeState]
+   */
+  constructor({
+    replication,
+    manifest,
+    options = {},
+    metrics = null,
+    progress = null,
+    knownResumeState = new Map()
+  }) {
+    this.replication = replication;
+    this.manifest = manifest;
+    this.chunkSize = options.chunkSize || 1024 * 1024;
+    this.maxEntries = options.maxEntries || 8;
+    this.maxBytes = options.maxBytes || 8 * 1024 * 1024;
+    this.maxImmutableBuffers = options.maxImmutableBuffers || 4;
+    this.metrics = metrics;
+    this.progress = progress;
+    this.knownResumeState = knownResumeState;
+
+    this.transferId = crypto.randomUUID();
+
+    // Dynamically size arena capacity according to generation maxBytes
+    const arenaCapacity = Math.max(16 * 1024 * 1024, this.maxBytes + (this.maxEntries * 256) + 1024 * 1024);
+
+    // Muttafa low-level mutation buffering engine
+    this.sink = new RaptiyeSink(this.replication, this.metrics, this.progress);
+    this.engine = new MutationEngine({
+      sink: this.sink,
+      createStore: () => new ArenaStore({
+        chunkSize: Math.max(64, this.maxEntries),
+        allocator: new BumpAllocator(arenaCapacity)
+      }),
+      maxEntries: this.maxEntries,
+      maxBytes: this.maxBytes,
+      maxImmutableBuffers: this.maxImmutableBuffers,
+      backpressurePolicy: BackpressurePolicy.WAIT,
+      onBackpressure: (active) => {
+        if (this.metrics && active) {
+          this.metrics.backpressureCount++;
+        }
+        if (this.progress) {
+          this.progress.update({
+            muttafaQueue: this.engine._queue.length,
+            tcpPressure: active ? 1 : 0
+          });
+        }
+      }
+    });
+  }
+
+  async _append(opcode, key, value = undefined, keyBytes = 0, valueBytes = 0) {
+    while (this.engine.isBackpressured()) {
+      await this.engine.ready();
+      await new Promise((r) => setImmediate(r));
+    }
+    const kLen = keyBytes || (key ? key.byteLength : 0);
+    const vLen = valueBytes || (value ? value.byteLength : 0);
+    return this.engine.custom(opcode, key, value, kLen, vLen);
+  }
+
+  /**
+   * Execute the full transfer pipeline with streaming reads, checksums,
+   * Muttafa mutation buffering, and end-to-end backpressure.
+   */
+  async transfer() {
+    if (this.progress) {
+      this.progress.start();
+      this.progress.update({
+        totalBytes: this.manifest.totalBytes,
+        totalChunks: Math.ceil(this.manifest.totalBytes / this.chunkSize)
+      });
+    }
+
+    // 1. OP_TRANSFER_BEGIN
+    const tbegin = packTransferBegin({
+      transferId: this.transferId,
+      fileCount: this.manifest.totalFiles,
+      totalBytes: this.manifest.totalBytes
+    });
+    await this._append(Opcodes.TRANSFER_BEGIN, tbegin);
+
+    let bytesSent = 0;
+    let chunksSent = 0;
+
+    // 2. Iterate files
+    for (const file of this.manifest.files) {
+      const filePath = this.manifest.isDirectory
+        ? `${this.manifest.rootPath}/${file.relativePath}`
+        : this.manifest.rootPath;
+
+      if (this.progress) {
+        this.progress.update({ fileName: file.relativePath });
+      }
+
+      // Check if resume state exists for this file
+      const resume = this.knownResumeState.get(file.fileId);
+
+      // OP_FILE_BEGIN
+      const fbegin = packFileBegin({
+        fileId: file.fileId,
+        fileSize: file.size,
+        path: file.relativePath,
+        sha256: file.sha256 || ''
+      });
+      await this._append(Opcodes.FILE_BEGIN, fbegin);
+
+      const totalFileChunks = file.size === 0 ? 0 : Math.ceil(file.size / this.chunkSize);
+
+      if (file.size > 0) {
+        const handle = await fs.open(filePath, 'r');
+        try {
+          for (let i = 0; i < totalFileChunks; i++) {
+            const offset = BigInt(i * this.chunkSize);
+            const remaining = file.size - (i * this.chunkSize);
+            const chunkLen = Math.min(this.chunkSize, remaining);
+
+            // Resume check: if receiver already has verified chunk, skip reading and sending!
+            if (resume && resume.isVerified(i)) {
+              if (this.metrics) {
+                this.metrics.resumeHits++;
+                this.metrics.chunksVerified++;
+              }
+              bytesSent += chunkLen;
+              chunksSent++;
+              if (this.progress) {
+                this.progress.update({ bytesDone: bytesSent, chunksDone: chunksSent });
+              }
+              continue;
+            }
+
+            // Direct read into fresh chunk buffer
+            const chunkBuf = new Uint8Array(chunkLen);
+            const readStart = Date.now();
+            await handle.read(chunkBuf, 0, chunkLen, Number(offset));
+            if (this.metrics) {
+              this.metrics.diskReadTimeMs += (Date.now() - readStart);
+              this.metrics.fileBytesRead += chunkLen;
+            }
+
+            // Checksum
+            const crcStart = Date.now();
+            const checksum = crc32(chunkBuf);
+            if (this.metrics) {
+              this.metrics.checksumTimeMs += (Date.now() - crcStart);
+              this.metrics.chunksCreated++;
+            }
+
+            // Pack compact binary metadata
+            const meta = packChunkMeta({
+              fileId: file.fileId,
+              chunkIndex: i,
+              offset,
+              length: chunkLen,
+              checksum
+            });
+
+            // Fast append to Muttafa MutableGeneration with opcode
+            await this._append(Opcodes.CHUNK, meta, chunkBuf, meta.byteLength, chunkBuf.byteLength);
+
+            bytesSent += chunkLen;
+            chunksSent++;
+
+            if (this.progress) {
+              const rStats = this.replication.getStats();
+              this.progress.update({
+                bytesDone: bytesSent,
+                chunksDone: chunksSent,
+                muttafaQueue: this.engine._queue.length,
+                inflightGen: rStats.inflightBatches,
+                inflightBytes: rStats.inflightBytes,
+                raptiyeRTT: rStats.rttMs
+              });
+            }
+          }
+        } finally {
+          await handle.close();
+        }
+      }
+
+      // OP_FILE_END
+      const fend = packFileEnd({ fileId: file.fileId, sha256: file.sha256 || '' });
+      await this._append(Opcodes.FILE_END, fend);
+    }
+
+    // 3. OP_TRANSFER_END
+    const tend = packTransferEnd({ transferId: this.transferId, status: 0 });
+    await this._append(Opcodes.TRANSFER_END, tend);
+
+    // 4. Flush remaining mutations in Muttafa and wait for all deliveries
+    await this.engine.flush();
+    await this.replication.drain();
+
+    if (this.progress) {
+      this.progress.update({
+        bytesDone: this.manifest.totalBytes,
+        chunksDone: Math.ceil(this.manifest.totalBytes / this.chunkSize)
+      });
+      this.progress.stop();
+    }
+  }
+}

package/test/integration/backpressure.test.js

@@ -0,0 +1,63 @@
+import test from 'node:test';
+import assert from 'node:assert';
+import fs from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import crypto from 'node:crypto';
+import { serve, send } from '../../src/index.js';
+
+test('Integration - slow receiver backpressure: disk throttle triggers Muttafa backpressure and memory plateaus', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'teyyare-backpressure-test-'));
+  const sourceFile = path.join(tmpDir, 'throttle.bin');
+  const destDir = path.join(tmpDir, 'dest');
+  await fs.mkdir(destDir, { recursive: true });
+
+  // 6 MB file
+  const size = 6 * 1024 * 1024;
+  const data = crypto.randomBytes(size);
+  await fs.writeFile(sourceFile, data);
+  const sourceHash = crypto.createHash('sha256').update(data).digest('hex');
+
+  const port = 7525;
+  // Artificially delay receiver disk write by 30ms per chunk
+  const server = await serve({
+    port,
+    destinationDir: destDir,
+    writeDelayMs: 30
+  });
+
+  const senderMetrics = await send({
+    sourcePath: sourceFile,
+    host: '127.0.0.1',
+    port,
+    pipelineOptions: {
+      chunkSize: 512 * 1024,   // 512 KB chunks (12 chunks)
+      maxEntries: 2,           // 1 MB generations
+      maxBytes: 1024 * 1024,
+      maxImmutableBuffers: 2,  // Queue capacity of only 2 generations
+      maxInflightBatches: 2    // Raptiye inflight capacity of 2
+    },
+    showProgress: false
+  });
+
+  await server.receiver.waitForCompletion();
+  await server.close();
+
+  const destFile = path.join(destDir, 'throttle.bin');
+  assert.ok(existsSync(destFile), 'Destination file must exist');
+
+  const destData = await fs.readFile(destFile);
+  const destHash = crypto.createHash('sha256').update(destData).digest('hex');
+  assert.strictEqual(destHash, sourceHash, 'SHA-256 must match despite backpressure throttling');
+
+  const snap = senderMetrics.snapshot();
+  console.log(`Backpressure activations: ${snap.backpressureCount}, Peak RSS: ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB`);
+
+  // Verify backpressure was triggered
+  assert.ok(snap.backpressureCount > 0, 'Muttafa backpressure must be triggered by slow receiver');
+  // Verify peak RSS did not explode
+  assert.ok(snap.peakRSS < 200 * 1024 * 1024, 'RSS memory must remain bounded under backpressure');
+
+  await fs.rm(tmpDir, { recursive: true });
+});