teyyare 0.2.7 → 0.2.9

package/test/e2e/e2e.js

@@ -0,0 +1,276 @@
+/**
+ * Teyyare End-to-End (E2E) Test Suite
+ *
+ * Runs comprehensive end-to-end scenarios:
+ * 1. Single File Upload (send) with Multi-Channel TCP (32 MB)
+ * 2. Directory Transfer with Nested Hierarchy and Small Files
+ * 3. Single File Download (pull) with Multi-Channel TCP (32 MB)
+ * 4. Interruption and Resumption with Bounded Retransmission
+ *
+ * Designed to be monitored and profiled cleanly with:
+ * - node test/e2e/e2e.js
+ * - softscope test/e2e/e2e.js
+ * - softscope --live test/e2e/e2e.js
+ * - softscope --md test/e2e/e2e.js
+ */
+
+import fs from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import net from 'node:net';
+import crypto from 'node:crypto';
+import { serve, send, pull, FileResumeState } from '../../src/index.js';
+
+function getAvailablePort() {
+  return new Promise((resolve, reject) => {
+    const s = net.createServer();
+    s.listen(0, '127.0.0.1', () => {
+      const port = s.address().port;
+      s.close(() => resolve(port));
+    });
+    s.on('error', reject);
+  });
+}
+
+async function computeSha256(filePath) {
+  const data = await fs.readFile(filePath);
+  return crypto.createHash('sha256').update(data).digest('hex');
+}
+
+async function runScenario1_SingleFileUpload(workDir) {
+  console.log('\n[1/4] Running Single File Upload (send) - 32 MB across 4 channels...');
+  const srcDir = path.join(workDir, 's1-src');
+  const dstDir = path.join(workDir, 's1-dst');
+  await fs.mkdir(srcDir, { recursive: true });
+  await fs.mkdir(dstDir, { recursive: true });
+
+  const size = 32 * 1024 * 1024;
+  const srcFile = path.join(srcDir, 'data-32mb.bin');
+  const data = crypto.randomBytes(size);
+  await fs.writeFile(srcFile, data);
+  const srcSha = crypto.createHash('sha256').update(data).digest('hex');
+
+  const port = await getAvailablePort();
+  const server = await serve({ port, destinationDir: dstDir });
+
+  const t0 = Date.now();
+  const metrics = await send({
+    sourcePath: srcFile,
+    host: '127.0.0.1',
+    port,
+    channels: 4,
+    showProgress: false
+  });
+
+  await server.receiver.waitForCompletion();
+  const durationMs = Math.max(1, Date.now() - t0);
+  await server.close();
+
+  const dstFile = path.join(dstDir, 'data-32mb.bin');
+  if (!existsSync(dstFile)) throw new Error('Destination file does not exist');
+  const dstSha = await computeSha256(dstFile);
+  if (dstSha !== srcSha) throw new Error('SHA-256 mismatch in Scenario 1');
+
+  const mb = size / (1024 * 1024);
+  const throughput = mb / (durationMs / 1000);
+  return { name: 'Single File Upload (32 MB, 4 channels)', mb, durationMs, throughput };
+}
+
+async function runScenario2_DirectoryUpload(workDir) {
+  console.log('\n[2/4] Running Directory Upload (send) with Nested Files & Small-File Packing...');
+  const srcDir = path.join(workDir, 's2-src');
+  const dstDir = path.join(workDir, 's2-dst');
+  await fs.mkdir(path.join(srcDir, 'nested', 'deep'), { recursive: true });
+  await fs.mkdir(dstDir, { recursive: true });
+
+  // Create varied files
+  await fs.writeFile(path.join(srcDir, 'empty.txt'), '');
+  await fs.writeFile(path.join(srcDir, 'small.txt'), 'Hello Teyyare ' + 'x'.repeat(2048));
+  await fs.writeFile(path.join(srcDir, 'nested', 'medium.bin'), crypto.randomBytes(4 * 1024 * 1024));
+  await fs.writeFile(path.join(srcDir, 'nested', 'deep', 'large.bin'), crypto.randomBytes(12 * 1024 * 1024));
+
+  const port = await getAvailablePort();
+  const server = await serve({ port, destinationDir: dstDir });
+
+  const t0 = Date.now();
+  const metrics = await send({
+    sourcePath: srcDir,
+    host: '127.0.0.1',
+    port,
+    channels: 4,
+    showProgress: false
+  });
+
+  await server.receiver.waitForCompletion();
+  const durationMs = Math.max(1, Date.now() - t0);
+  await server.close();
+
+  // Verify all files exist and match
+  const checkFiles = [
+    'empty.txt',
+    'small.txt',
+    path.join('nested', 'medium.bin'),
+    path.join('nested', 'deep', 'large.bin')
+  ];
+
+  let totalBytes = 0;
+  for (const rel of checkFiles) {
+    const f1 = path.join(srcDir, rel);
+    const f2 = path.join(dstDir, rel);
+    if (!existsSync(f2)) throw new Error(`Missing received file: ${rel}`);
+    const sha1 = await computeSha256(f1);
+    const sha2 = await computeSha256(f2);
+    if (sha1 !== sha2) throw new Error(`SHA mismatch for ${rel}`);
+    totalBytes += (await fs.stat(f1)).size;
+  }
+
+  const mb = totalBytes / (1024 * 1024);
+  const throughput = mb / (durationMs / 1000);
+  return { name: 'Directory Upload (Nested tree, 16+ MB)', mb, durationMs, throughput };
+}
+
+async function runScenario3_SingleFilePull(workDir) {
+  console.log('\n[3/4] Running Single File Pull (download) - 32 MB across 4 channels...');
+  const serverDir = path.join(workDir, 's3-srv');
+  const clientDir = path.join(workDir, 's3-cli');
+  await fs.mkdir(serverDir, { recursive: true });
+  await fs.mkdir(clientDir, { recursive: true });
+
+  const size = 32 * 1024 * 1024;
+  const srvFile = path.join(serverDir, 'pull-32mb.bin');
+  const data = crypto.randomBytes(size);
+  await fs.writeFile(srvFile, data);
+  const srvSha = crypto.createHash('sha256').update(data).digest('hex');
+
+  const port = await getAvailablePort();
+  const server = await serve({ port, destinationDir: serverDir });
+
+  const t0 = Date.now();
+  const metrics = await pull({
+    sourceRemotePath: 'pull-32mb.bin',
+    host: '127.0.0.1',
+    port,
+    channels: 4,
+    destinationDir: clientDir,
+    showProgress: false
+  });
+
+  const durationMs = Math.max(1, Date.now() - t0);
+  await server.close();
+
+  const clientFile = path.join(clientDir, 'pull-32mb.bin');
+  if (!existsSync(clientFile)) throw new Error('Pulled file does not exist');
+  const clientSha = await computeSha256(clientFile);
+  if (clientSha !== srvSha) throw new Error('SHA-256 mismatch in Scenario 3 (pull)');
+
+  const mb = size / (1024 * 1024);
+  const throughput = mb / (durationMs / 1000);
+  return { name: 'Single File Pull (32 MB, 4 channels)', mb, durationMs, throughput };
+}
+
+async function runScenario4_InterruptedResume(workDir) {
+  console.log('\n[4/4] Running Interrupted Transfer & Resumption with Bounded Retransmission...');
+  const srcDir = path.join(workDir, 's4-src');
+  const dstDir = path.join(workDir, 's4-dst');
+  await fs.mkdir(srcDir, { recursive: true });
+  await fs.mkdir(dstDir, { recursive: true });
+
+  const size = 16 * 1024 * 1024; // 16 MB = 16 chunks
+  const srcFile = path.join(srcDir, 'resume-16mb.bin');
+  const data = crypto.randomBytes(size);
+  await fs.writeFile(srcFile, data);
+  const srcSha = crypto.createHash('sha256').update(data).digest('hex');
+
+  const dstFile = path.join(dstDir, 'resume-16mb.bin');
+  const partPath = `${dstFile}.teyyare-part`;
+  const statePath = `${dstFile}.teyyare-state`;
+
+  // Pre-seed 8 chunks (50%) in destination
+  const partialState = new FileResumeState({
+    transferId: 'e2e-resume-test',
+    fileId: 1,
+    fileSize: size,
+    chunkSize: 1024 * 1024,
+    sha256: srcSha,
+    verifiedChunks: new Set([0, 1, 2, 3, 4, 5, 6, 7])
+  });
+  await partialState.save(statePath);
+
+  const handle = await fs.open(partPath, 'w+');
+  await handle.write(data.subarray(0, 8 * 1024 * 1024), 0, 8 * 1024 * 1024, 0);
+  await handle.sync();
+  await handle.close();
+
+  // Start server and resume
+  const port = await getAvailablePort();
+  const server = await serve({ port, destinationDir: dstDir });
+
+  const knownResume = new Map([[1, partialState]]);
+  const t0 = Date.now();
+  const metrics = await send({
+    sourcePath: srcFile,
+    host: '127.0.0.1',
+    port,
+    channels: 4,
+    knownResumeState: knownResume,
+    showProgress: false
+  });
+
+  await server.receiver.waitForCompletion();
+  const durationMs = Math.max(1, Date.now() - t0);
+  await server.close();
+
+  const finalSha = await computeSha256(dstFile);
+  if (finalSha !== srcSha) throw new Error('SHA mismatch after resume');
+
+  const snap = metrics.snapshot();
+  if (snap.resumeHits !== 8) throw new Error(`Expected 8 resume hits, got ${snap.resumeHits}`);
+  if (snap.chunksCreated !== 8) throw new Error(`Expected 8 chunks created, got ${snap.chunksCreated}`);
+
+  const mb = 8; // Only 8 MB actually re-transferred
+  const throughput = mb / (durationMs / 1000);
+  return { name: 'Resumed Transfer (50% skipped, 8 MB sent)', mb, durationMs, throughput };
+}
+
+async function main() {
+  console.log('====================================================');
+  console.log('         TEYYARE END-TO-END (E2E) TEST SUITE         ');
+  console.log('====================================================');
+
+  const workDir = await fs.mkdtemp(path.join(os.tmpdir(), 'teyyare-e2e-'));
+  const results = [];
+
+  try {
+    results.push(await runScenario1_SingleFileUpload(workDir));
+    results.push(await runScenario2_DirectoryUpload(workDir));
+    results.push(await runScenario3_SingleFilePull(workDir));
+    results.push(await runScenario4_InterruptedResume(workDir));
+
+    console.log('\n' + '='.repeat(80));
+    console.log('                        E2E EXECUTION SUMMARY                        ');
+    console.log('='.repeat(80));
+    console.log('| Scenario                                      | Data (MB) | Time (ms) | Throughput  | Status |');
+    console.log('|-----------------------------------------------|-----------|-----------|-------------|--------|');
+    for (const r of results) {
+      const name = r.name.padEnd(45);
+      const mb = r.mb.toFixed(1).padStart(9);
+      const ms = String(r.durationMs).padStart(9);
+      const speed = `${r.throughput.toFixed(1)} MB/s`.padStart(11);
+      console.log(`| ${name} | ${mb} | ${ms} | ${speed} |  PASS  |`);
+    }
+    console.log('='.repeat(80));
+    console.log('ALL E2E SCENARIOS PASSED SUCCESSFULLY!\n');
+  } finally {
+    await fs.rm(workDir, { recursive: true, force: true });
+  }
+}
+
+main()
+  .then(() => {
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error('\nE2E Test Failed:', err);
+    process.exit(1);
+  });

package/test/integration/pull.test.js

@@ -129,3 +129,54 @@ test('Integration - pull non-existent path throws remote error', async () => {
     await fs.rm(testDir, { recursive: true, force: true });
   }
 });
+
+test('Integration - pull with custom WAN tuning parameters (2MB chunk size)', async () => {
+  const testDir = path.resolve('tmp/pull-test-wan-tuning');
+  const serverDir = path.join(testDir, 'server');
+  const clientDir = path.join(testDir, 'client');
+
+  await fs.rm(testDir, { recursive: true, force: true });
+  await fs.mkdir(serverDir, { recursive: true });
+  await fs.mkdir(clientDir, { recursive: true });
+
+  // Create 6 MB test file
+  const testFile = path.join(serverDir, 'wan-archive.bin');
+  const size = 6 * 1024 * 1024;
+  const data = crypto.randomBytes(size);
+  await fs.writeFile(testFile, data);
+  const sourceSha = crypto.createHash('sha256').update(data).digest('hex');
+
+  const port = 8824;
+  const server = await serve({ port, destinationDir: serverDir });
+
+  try {
+    // Request 2 MB chunks (should result in exactly 3 chunks for 6 MB instead of 6 chunks of 1 MB)
+    const metrics = await pull({
+      sourceRemotePath: 'wan-archive.bin',
+      host: '127.0.0.1',
+      port,
+      destinationDir: clientDir,
+      pipelineOptions: {
+        chunkSize: 2 * 1024 * 1024,
+        maxEntries: 4,
+        maxBytes: 8 * 1024 * 1024
+      },
+      showProgress: false
+    });
+
+    const pulledFile = path.join(clientDir, 'wan-archive.bin');
+    assert.strictEqual(await fs.stat(pulledFile).then(s => s.size), size);
+
+    const pulledSha = await computeSha256(pulledFile);
+    assert.strictEqual(pulledSha, sourceSha);
+
+    const snap = metrics.snapshot();
+    assert.strictEqual(snap.fileBytesWritten, size);
+    // With 2 MB chunks, 6 MB file has exactly 3 chunks!
+    assert.strictEqual(snap.chunksCreated, 3, 'Should create exactly 3 chunks of 2 MB each');
+    assert.strictEqual(snap.chunksVerified, 3, 'Should verify exactly 3 chunks of 2 MB each');
+  } finally {
+    await server.close();
+    await fs.rm(testDir, { recursive: true, force: true });
+  }
+});

package/test/integration/resume.test.js

@@ -95,3 +95,65 @@ test('Integration - resume after interruption at 50%, bounded retransmission, SH
 
   await fs.rm(tmpDir, { recursive: true });
 });
+
+test('Integration - debounced asynchronous resume state saving drains cleanly', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'teyyare-debounce-test-'));
+  const destDir = path.join(tmpDir, 'dest');
+  await fs.mkdir(destDir, { recursive: true });
+
+  const { ReceiverPipeline } = await import('../../src/receiver/pipeline.js');
+  const { packFileBegin, packChunkMeta, packFileEnd, Opcodes } = await import('../../src/core/protocol.js');
+
+  const receiver = new ReceiverPipeline({
+    destinationDir: destDir
+  });
+
+  // Begin a 64 MB file
+  const fileBeginBuf = packFileBegin({
+    fileId: 10,
+    fileSize: 64 * 1024 * 1024,
+    path: 'debounced.bin',
+    sha256: ''
+  });
+  await receiver._handleFileBegin(fileBeginBuf);
+
+  const file = receiver.activeFiles.get(10);
+  assert.ok(file, 'Active file must be registered');
+
+  // Simulate writing 40 chunks rapidly
+  for (let c = 0; c < 40; c++) {
+    const metaBuf = packChunkMeta({
+      fileId: 10,
+      chunkIndex: c,
+      offset: BigInt(c * 1024),
+      length: 1024,
+      checksum: 0 // dummy, won't check if we directly call _writeChunk
+    });
+    // mock CRC match
+    const chunkData = new Uint8Array(1024);
+    const { crc32 } = await import('../../src/core/crc32.js');
+    const crc = crc32(chunkData);
+    const validMeta = packChunkMeta({
+      fileId: 10,
+      chunkIndex: c,
+      offset: BigInt(c * 1024),
+      length: 1024,
+      checksum: crc
+    });
+    await receiver._writeChunk(file, (await import('../../src/core/protocol.js')).unpackChunkMeta(validMeta), chunkData);
+  }
+
+  assert.strictEqual(file.resumeState.verifiedChunks.size, 40);
+
+  // Close receiver - should drain any state save cleanly without errors
+  await receiver.close();
+
+  const statePath = path.join(destDir, 'debounced.bin.teyyare-state');
+  // State must be readable and valid
+  const loaded = await FileResumeState.load(statePath);
+  if (loaded) {
+    assert.ok(loaded.verifiedChunks.size > 0, 'Saved state must contain verified chunks');
+  }
+
+  await fs.rm(tmpDir, { recursive: true });
+});

package/test/integration/single-file.test.js

@@ -62,3 +62,88 @@ test('Integration - end-to-end 10 MB single file transfer with SHA-256 verificat
 
   await fs.rm(tmpDir, { recursive: true });
 });
+
+test('Integration - file pre-allocation allocates full file size upon FILE_BEGIN', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'teyyare-prealloc-'));
+  const destDir = path.join(tmpDir, 'dest');
+  await fs.mkdir(destDir, { recursive: true });
+
+  const receiver = new (await import('../../src/receiver/pipeline.js')).ReceiverPipeline({
+    destinationDir: destDir
+  });
+
+  const { packFileBegin } = await import('../../src/core/protocol.js');
+  const fileBeginBuf = packFileBegin({
+    fileId: 1,
+    fileSize: 50 * 1024 * 1024, // 50 MB
+    path: 'preallocated.bin',
+    sha256: ''
+  });
+
+  await receiver._handleFileBegin(fileBeginBuf);
+
+  const partFile = path.join(destDir, 'preallocated.bin.teyyare-part');
+  assert.ok(existsSync(partFile), 'Part file must exist immediately after FILE_BEGIN');
+  const stat = await fs.stat(partFile);
+  assert.strictEqual(stat.size, 50 * 1024 * 1024, 'Part file must be pre-allocated to full size (50 MB)');
+
+  await receiver.close();
+  await fs.rm(tmpDir, { recursive: true });
+});
+
+test('Integration - chunk coalescing writes contiguous runs using writev and verifies exact content', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'teyyare-coalesce-'));
+  const destDir = path.join(tmpDir, 'dest');
+  await fs.mkdir(destDir, { recursive: true });
+
+  const { ReceiverPipeline } = await import('../../src/receiver/pipeline.js');
+  const { packFileBegin, packChunkMeta } = await import('../../src/core/protocol.js');
+  const { crc32 } = await import('../../src/core/crc32.js');
+
+  const receiver = new ReceiverPipeline({ destinationDir: destDir });
+
+  const fileBeginBuf = packFileBegin({
+    fileId: 42,
+    fileSize: 4 * 1024 * 1024,
+    path: 'coalesced.bin',
+    sha256: ''
+  });
+  await receiver._handleFileBegin(fileBeginBuf);
+
+  // Spy on writev
+  const file = receiver.activeFiles.get(42);
+  let writevCallCount = 0;
+  const originalWritev = file.fileHandle.writev.bind(file.fileHandle);
+  file.fileHandle.writev = async (...args) => {
+    writevCallCount++;
+    return originalWritev(...args);
+  };
+
+  // Build a batch of 4 contiguous 1MB chunks
+  const chunkData = crypto.randomBytes(4 * 1024 * 1024);
+  const items = [];
+  for (let c = 0; c < 4; c++) {
+    const slice = chunkData.subarray(c * 1024 * 1024, (c + 1) * 1024 * 1024);
+    const key = packChunkMeta({
+      fileId: 42,
+      chunkIndex: c,
+      offset: BigInt(c * 1024 * 1024),
+      length: 1024 * 1024,
+      checksum: crc32(slice)
+    });
+    items.push({ key, val: slice });
+  }
+
+  // Handle all 4 chunks together
+  await receiver._handleChunks(items);
+
+  // All 4 contiguous chunks must be coalesced into a single writev syscall!
+  assert.strictEqual(writevCallCount, 1, '4 contiguous chunks must be coalesced into exactly 1 writev syscall');
+
+  // Verify file content matches on disk
+  const written = await fs.readFile(file.partPath);
+  assert.ok(written.equals(chunkData), 'Written file content on disk must match original random bytes exactly');
+
+  await receiver.close();
+  await fs.rm(tmpDir, { recursive: true });
+});

package/test/unit/protocol.test.js

@@ -77,9 +77,29 @@ test('Protocol - pack and unpack transfer begin', () => {
 });
 
 test('Protocol - pack and unpack pull request', () => {
+  // 1. Basic pull request
   const req = packPullRequest({ remotePath: '/root/cdn/app.zip' });
   const unpacked = unpackPullRequest(req);
   assert.strictEqual(unpacked.remotePath, '/root/cdn/app.zip');
+  assert.deepStrictEqual(unpacked.verifiedChunks, []);
+  assert.strictEqual(unpacked.chunkSize, undefined);
+  assert.strictEqual(unpacked.maxEntries, undefined);
+  assert.strictEqual(unpacked.maxBytes, undefined);
+
+  // 2. Pull request with WAN tuning parameters and verified chunks
+  const reqTuned = packPullRequest({
+    remotePath: '/var/data/bigfile.bin',
+    verifiedChunks: [0, 1, 4],
+    chunkSize: 2 * 1024 * 1024,
+    maxEntries: 16,
+    maxBytes: 32 * 1024 * 1024
+  });
+  const unpackedTuned = unpackPullRequest(reqTuned);
+  assert.strictEqual(unpackedTuned.remotePath, '/var/data/bigfile.bin');
+  assert.deepStrictEqual(unpackedTuned.verifiedChunks, [0, 1, 4]);
+  assert.strictEqual(unpackedTuned.chunkSize, 2097152);
+  assert.strictEqual(unpackedTuned.maxEntries, 16);
+  assert.strictEqual(unpackedTuned.maxBytes, 33554432);
 });
 
 test('Protocol - pack and unpack pull error', () => {