teyyare 0.1.0 → 0.2.0

package/README.md

@@ -2,8 +2,8 @@
 
 **Teyyare** is a high-performance, resumable file transfer engine and CLI built directly on top of:
 
-- [`muttafa`](https://github.com/ahmet/muttafa): low-level mutation buffering, generation freeze & lifecycle engine
-- [`raptiye`](https://github.com/ahmet/raptiye): high-performance byte-first replicated log & TCP binary transport
+- [`muttafa`](https://github.com/litepacks/muttafa): low-level mutation buffering, generation freeze & lifecycle engine
+- [`raptiye`](https://github.com/litepacks/raptiye): high-performance byte-first replicated log & TCP binary transport
 
 Teyyare is the first real integration workload for both projects, proving that a mutation lifecycle engine and a binary replication/transport engine compose into a fast, bounded-memory, resumable file transfer system without introducing redundant batching layers or memory copies.
 

package/bin/teyyare.js

@@ -5,7 +5,7 @@
  */
 
 import path from 'node:path';
-import { serve, send } from '../src/index.js';
+import { serve, send, pull } from '../src/index.js';
 
 function printUsage() {
   console.log(`
@@ -106,9 +106,35 @@ Protocol overhead:${snap.protocolOverheadPercent.toFixed(2)}%
 Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
 `);
   } else if (command === 'pull') {
-    console.log('Pull command: inverting peer roles to request remote transfer.');
-    // In point-to-point architecture, pull is receiver-initiated transfer
-    console.error('Pulling is handled via remote agent or server command.');
+    const source = args[1];
+    const target = args[2] || './';
+
+    if (!source) {
+      console.error('Error: teyyare pull requires <source> and [destination]');
+      printUsage();
+      process.exit(1);
+    }
+
+    const { host, port, destPath: remotePath } = parseDestination(source);
+    const metrics = await pull({
+      sourceRemotePath: remotePath,
+      host,
+      port,
+      destinationDir: target,
+      showProgress: true
+    });
+
+    const snap = metrics.snapshot();
+    console.log(`
+\x1b[1m\x1b[32mPull completed successfully!\x1b[0m
+Throughput:       \x1b[1m${snap.throughputMBs.toFixed(2)} MB/s\x1b[0m
+Duration:         ${(snap.elapsedMs / 1000).toFixed(2)}s
+Bytes:            ${(snap.fileBytesWritten / (1024 * 1024)).toFixed(2)} MB
+Generations:      ${snap.generations}
+Backpressure:     ${snap.backpressureCount}
+Protocol overhead:${snap.protocolOverheadPercent.toFixed(2)}%
+Peak RSS:         ${(snap.peakRSS / (1024 * 1024)).toFixed(2)} MB
+`);
   } else if (command === 'status') {
     console.log('\x1b[1m\x1b[36mTeyyare\x1b[0m v0.1.0 status: ready');
   } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "teyyare",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "High-performance resumable file transfer engine and CLI built on muttafa and raptiye",
   "main": "src/index.js",
   "type": "module",

package/src/core/protocol.js

@@ -11,7 +11,9 @@ export const Opcodes = Object.freeze({
   FILE_BEGIN: 11,
   CHUNK: 12,
   FILE_END: 13,
-  TRANSFER_END: 14
+  TRANSFER_END: 14,
+  PULL_REQUEST: 15,
+  PULL_ERROR: 16
 });
 
 export const CHUNK_META_SIZE = 25;
@@ -161,3 +163,44 @@ export function unpackTransferEnd(buf) {
   const transferId = decoder.decode(buf.subarray(3, 3 + idLen));
   return { transferId, status };
 }
+
+/**
+ * Pack PULL_REQUEST metadata.
+ */
+export function packPullRequest({ remotePath }) {
+  const pathBytes = encoder.encode(remotePath);
+  const buf = new Uint8Array(2 + pathBytes.byteLength);
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  view.setUint16(0, pathBytes.byteLength, false);
+  buf.set(pathBytes, 2);
+  return buf;
+}
+
+export function unpackPullRequest(buf) {
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  const pathLen = view.getUint16(0, false);
+  const remotePath = decoder.decode(buf.subarray(2, 2 + pathLen));
+  return { remotePath };
+}
+
+/**
+ * Pack PULL_ERROR metadata.
+ */
+export function packPullError({ errorCode = 1, message = '' }) {
+  const msgBytes = encoder.encode(message);
+  const buf = new Uint8Array(6 + msgBytes.byteLength);
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  view.setUint32(0, errorCode, false);
+  view.setUint16(4, msgBytes.byteLength, false);
+  buf.set(msgBytes, 6);
+  return buf;
+}
+
+export function unpackPullError(buf) {
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  const errorCode = view.getUint32(0, false);
+  const msgLen = view.getUint16(4, false);
+  const message = decoder.decode(buf.subarray(6, 6 + msgLen));
+  return { errorCode, message };
+}
+

package/src/index.js

@@ -3,6 +3,8 @@
  */
 
 import path from 'node:path';
+import { existsSync } from 'node:fs';
+import { MutableBuffer, ArenaStore, BumpAllocator, FrozenArenaStore } from 'muttafa';
 import { TCPTransport, ReplicationPipeline } from 'raptiye';
 import { SenderPipeline } from './sender/pipeline.js';
 import { ReceiverPipeline } from './receiver/pipeline.js';
@@ -10,7 +12,15 @@ import { TransferManifest } from './manifest/manifest.js';
 import { FileResumeState } from './resume/state.js';
 import { MetricsCollector } from './metrics/collector.js';
 import { ProgressReporter } from './cli/progress.js';
-import { Opcodes, packChunkMeta, unpackChunkMeta } from './core/protocol.js';
+import {
+  Opcodes,
+  packChunkMeta,
+  unpackChunkMeta,
+  packPullRequest,
+  unpackPullRequest,
+  packPullError,
+  unpackPullError
+} from './core/protocol.js';
 
 export {
   SenderPipeline,
@@ -21,11 +31,35 @@ export {
   ProgressReporter,
   Opcodes,
   packChunkMeta,
-  unpackChunkMeta
+  unpackChunkMeta,
+  packPullRequest,
+  unpackPullRequest,
+  packPullError,
+  unpackPullError
 };
 
+async function submitSingleMutation(replication, opcode, key, value = new Uint8Array(0)) {
+  const store = new ArenaStore({ allocator: new BumpAllocator(16 * 1024) });
+  const mutable = new MutableBuffer(Date.now() & 0x7fffffff, store);
+  mutable.append(1, opcode, key, value);
+  const frozen = mutable.freeze();
+  frozen.markFlushing();
+  const receipt = await replication.submitBatch({
+    generationId: frozen.generation,
+    firstSequence: frozen.firstSequence,
+    lastSequence: frozen.lastSequence,
+    entryCount: frozen.entryCount,
+    byteLength: frozen.byteLength,
+    buffers: frozen.buffers()
+  });
+  await receipt.delivered();
+  frozen.markFlushed();
+  frozen.release();
+}
+
 /**
- * Start a Teyyare receiver server.
+ * Start a Teyyare receiver/file server.
+ * Handles both incoming push transfers (send) and remote pull requests (pull).
  * @param {object} options
  * @param {number} [options.port=7421]
  * @param {string} [options.host='0.0.0.0']
@@ -61,7 +95,42 @@ export async function serve({
     replication,
     destinationDir,
     metrics,
-    writeDelayMs
+    writeDelayMs,
+    autoHookBatch: false
+  });
+
+  replication.onBatch(async (msg) => {
+    if (!msg.buffers || msg.buffers.length < 2) return;
+    const [metaBuf, payloadBuf] = msg.buffers;
+    const store = FrozenArenaStore.fromBuffers(metaBuf, payloadBuf);
+    if (store.length > 0 && store.getType(0) === Opcodes.PULL_REQUEST) {
+      const { remotePath } = unpackPullRequest(store.getKey(0));
+      let resolved = path.isAbsolute(remotePath) ? remotePath : path.resolve(destinationDir, remotePath);
+      if (!existsSync(resolved)) {
+        const alt = path.resolve(destinationDir, remotePath);
+        if (existsSync(alt)) resolved = alt;
+      }
+
+      if (!existsSync(resolved)) {
+        await submitSingleMutation(
+          replication,
+          Opcodes.PULL_ERROR,
+          packPullError({ errorCode: 404, message: `Path not found on remote: ${remotePath}` })
+        );
+        return;
+      }
+
+      const manifest = await TransferManifest.build(resolved);
+      const sender = new SenderPipeline({
+        replication,
+        manifest,
+        metrics
+      });
+      await sender.transfer();
+      return;
+    }
+
+    await receiver._processGenerationBatch(msg);
   });
 
   return {
@@ -137,3 +206,71 @@ export async function send({
 
   return metrics;
 }
+
+/**
+ * Pull a file or directory from a remote Teyyare server.
+ * @param {object} options
+ * @param {string} options.sourceRemotePath
+ * @param {string} [options.host='127.0.0.1']
+ * @param {number} [options.port=7421]
+ * @param {string} [options.destinationDir='./']
+ * @param {object} [options.pipelineOptions]
+ * @param {boolean} [options.showProgress=true]
+ * @returns {Promise<MetricsCollector>}
+ */
+export async function pull({
+  sourceRemotePath,
+  host = '127.0.0.1',
+  port = 7421,
+  destinationDir = './',
+  pipelineOptions = {},
+  showProgress = true
+}) {
+  const metrics = new MetricsCollector();
+  const progress = showProgress ? new ProgressReporter() : null;
+  if (progress) progress.start();
+
+  const clientPort = 10000 + Math.floor(Math.random() * 20000);
+  const transport = new TCPTransport({
+    id: 1,
+    port: clientPort,
+    host: '127.0.0.1',
+    peerAddresses: new Map([[2, { host, port }]])
+  });
+
+  await transport.start();
+
+  const replication = new ReplicationPipeline({
+    transport,
+    localNode: 1,
+    remoteNode: 2,
+    maxInflightBatches: pipelineOptions.maxInflightBatches || 16,
+    maxInflightBytes: pipelineOptions.maxInflightBytes || 64 * 1024 * 1024
+  });
+
+  const receiver = new ReceiverPipeline({
+    replication,
+    destinationDir,
+    metrics,
+    progress
+  });
+
+  try {
+    // Submit PULL_REQUEST batch
+    await submitSingleMutation(
+      replication,
+      Opcodes.PULL_REQUEST,
+      packPullRequest({ remotePath: sourceRemotePath })
+    );
+
+    // Wait for receiver pipeline to receive and write all files
+    await receiver.waitForCompletion();
+  } finally {
+    if (progress) progress.stop();
+    metrics.stop();
+    await transport.close();
+  }
+
+  return metrics;
+}
+

package/src/receiver/pipeline.js

@@ -10,7 +10,7 @@ import path from 'node:path';
 import crypto from 'node:crypto';
 import { FrozenArenaStore } from 'muttafa';
 import { crc32 } from 'raptiye';
-import { Opcodes, unpackChunkMeta, unpackFileBegin, unpackTransferBegin, unpackFileEnd } from '../core/protocol.js';
+import { Opcodes, unpackChunkMeta, unpackFileBegin, unpackTransferBegin, unpackFileEnd, unpackPullError } from '../core/protocol.js';
 import { FileResumeState } from '../resume/state.js';
 
 export class ReceiverPipeline {
@@ -25,27 +25,38 @@ export class ReceiverPipeline {
     replication,
     destinationDir,
     metrics = null,
-    writeDelayMs = 0
+    progress = null,
+    writeDelayMs = 0,
+    autoHookBatch = true
   }) {
     this.replication = replication;
     this.destinationDir = path.resolve(destinationDir);
     this.metrics = metrics;
+    this.progress = progress;
     this.writeDelayMs = writeDelayMs;
 
     this.activeFiles = new Map(); // fileId -> { destPath, partPath, statePath, fileHandle, resumeState, ... }
     this.completedFiles = new Set();
     this.transferFinished = false;
+    this.transferError = null;
     this._finishResolvers = [];
+    this._errorRejecters = [];
 
-    // Hook up Raptiye batch handler
-    this.replication.onBatch(async (msg) => {
-      await this._processGenerationBatch(msg);
-    });
+    // Hook up Raptiye batch handler if autoHookBatch is true
+    if (this.replication && autoHookBatch) {
+      this.replication.onBatch(async (msg) => {
+        await this._processGenerationBatch(msg);
+      });
+    }
   }
 
   async waitForCompletion() {
+    if (this.transferError) throw this.transferError;
     if (this.transferFinished) return;
-    return new Promise((resolve) => this._finishResolvers.push(resolve));
+    return new Promise((resolve, reject) => {
+      this._finishResolvers.push(resolve);
+      this._errorRejecters.push(reject);
+    });
   }
 
   async _processGenerationBatch(msg) {
@@ -74,6 +85,8 @@ export class ReceiverPipeline {
 
       switch (type) {
         case Opcodes.TRANSFER_BEGIN: {
+          this.transferFinished = false;
+          this.transferError = null;
           const tinfo = unpackTransferBegin(key);
           break;
         }
@@ -99,8 +112,21 @@ export class ReceiverPipeline {
             resolve();
           }
           this._finishResolvers = [];
+          this._errorRejecters = [];
           break;
         }
+
+        case Opcodes.PULL_ERROR: {
+          const { errorCode, message } = unpackPullError(key);
+          const err = new Error(`Remote pull error [${errorCode}]: ${message}`);
+          this.transferError = err;
+          for (const reject of this._errorRejecters) {
+            reject(err);
+          }
+          this._finishResolvers = [];
+          this._errorRejecters = [];
+          return;
+        }
       }
     }
   }
@@ -138,6 +164,13 @@ export class ReceiverPipeline {
       sha256: info.sha256,
       chunksWrittenSinceSave: 0
     });
+
+    if (this.progress) {
+      this.progress.update({
+        fileName: info.path,
+        totalBytes: Number(info.fileSize)
+      });
+    }
   }
 
   async _handleChunk(key, val) {
@@ -184,6 +217,14 @@ export class ReceiverPipeline {
       this.metrics.chunksVerified++;
     }
 
+    if (this.progress) {
+      this.progress.update({
+        fileName: file.relativePath,
+        bytesDone: this.metrics ? this.metrics.fileBytesWritten : 0,
+        chunksDone: this.metrics ? this.metrics.chunksVerified : 0
+      });
+    }
+
     file.resumeState.markVerified(meta.chunkIndex);
     file.chunksWrittenSinceSave++;
 

package/test/integration/pull.test.js

@@ -0,0 +1,131 @@
+import test from 'node:test';
+import assert from 'node:assert';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { serve, pull } from '../../src/index.js';
+
+async function computeSha256(filePath) {
+  const content = await fs.readFile(filePath);
+  return crypto.createHash('sha256').update(content).digest('hex');
+}
+
+test('Integration - end-to-end file pull with SHA-256 verification', async () => {
+  const testDir = path.resolve('tmp/pull-test-single');
+  const serverDir = path.join(testDir, 'server');
+  const clientDir = path.join(testDir, 'client');
+
+  await fs.rm(testDir, { recursive: true, force: true });
+  await fs.mkdir(serverDir, { recursive: true });
+  await fs.mkdir(clientDir, { recursive: true });
+
+  // Create 5 MB test file
+  const testFile = path.join(serverDir, 'archive.bin');
+  const size = 5 * 1024 * 1024;
+  const buf = Buffer.alloc(size);
+  for (let i = 0; i < size; i += 4096) {
+    buf.writeUInt32LE((i * 17) >>> 0, i);
+  }
+  await fs.writeFile(testFile, buf);
+  const sourceSha = await computeSha256(testFile);
+
+  const port = 8821;
+  const server = await serve({ port, destinationDir: serverDir });
+
+  try {
+    const metrics = await pull({
+      sourceRemotePath: 'archive.bin',
+      host: '127.0.0.1',
+      port,
+      destinationDir: clientDir,
+      showProgress: false
+    });
+
+    const pulledFile = path.join(clientDir, 'archive.bin');
+    assert.strictEqual(await fs.stat(pulledFile).then(s => s.size), size);
+
+    const pulledSha = await computeSha256(pulledFile);
+    assert.strictEqual(pulledSha, sourceSha);
+
+    const snap = metrics.snapshot();
+    assert.strictEqual(snap.fileBytesWritten, size);
+  } finally {
+    await server.close();
+    await fs.rm(testDir, { recursive: true, force: true });
+  }
+});
+
+test('Integration - directory pull with nested structure', async () => {
+  const testDir = path.resolve('tmp/pull-test-dir');
+  const serverDir = path.join(testDir, 'server');
+  const clientDir = path.join(testDir, 'client');
+
+  await fs.rm(testDir, { recursive: true, force: true });
+  await fs.mkdir(serverDir, { recursive: true });
+  await fs.mkdir(clientDir, { recursive: true });
+
+  const tree = {
+    'docs/readme.txt': 'Hello world documentation',
+    'src/index.js': 'console.log("hello");',
+    'src/nested/deep.json': JSON.stringify({ ok: true, nested: [1, 2, 3] })
+  };
+
+  for (const [relPath, content] of Object.entries(tree)) {
+    const fullPath = path.join(serverDir, 'my-pkg', relPath);
+    await fs.mkdir(path.dirname(fullPath), { recursive: true });
+    await fs.writeFile(fullPath, content);
+  }
+
+  const port = 8822;
+  const server = await serve({ port, destinationDir: serverDir });
+
+  try {
+    await pull({
+      sourceRemotePath: 'my-pkg',
+      host: '127.0.0.1',
+      port,
+      destinationDir: clientDir,
+      showProgress: false
+    });
+
+    for (const [relPath, content] of Object.entries(tree)) {
+      const destFile = path.join(clientDir, relPath);
+      const read = await fs.readFile(destFile, 'utf8');
+      assert.strictEqual(read, content);
+    }
+  } finally {
+    await server.close();
+    await fs.rm(testDir, { recursive: true, force: true });
+  }
+});
+
+test('Integration - pull non-existent path throws remote error', async () => {
+  const testDir = path.resolve('tmp/pull-test-err');
+  const serverDir = path.join(testDir, 'server');
+  const clientDir = path.join(testDir, 'client');
+
+  await fs.rm(testDir, { recursive: true, force: true });
+  await fs.mkdir(serverDir, { recursive: true });
+  await fs.mkdir(clientDir, { recursive: true });
+
+  const port = 8823;
+  const server = await serve({ port, destinationDir: serverDir });
+
+  try {
+    await assert.rejects(
+      async () => {
+        await pull({
+          sourceRemotePath: 'does-not-exist.tar',
+          host: '127.0.0.1',
+          port,
+          destinationDir: clientDir,
+          showProgress: false
+        });
+      },
+      /Remote pull error \[404\]: Path not found on remote/
+    );
+  } finally {
+    await server.close();
+    await fs.rm(testDir, { recursive: true, force: true });
+  }
+});

package/test/unit/protocol.test.js

@@ -10,7 +10,11 @@ import {
   packTransferBegin,
   unpackTransferBegin,
   packFileEnd,
-  unpackFileEnd
+  unpackFileEnd,
+  packPullRequest,
+  unpackPullRequest,
+  packPullError,
+  unpackPullError
 } from '../../src/core/protocol.js';
 
 test('Protocol - pack and unpack chunk metadata', () => {
@@ -71,3 +75,17 @@ test('Protocol - pack and unpack transfer begin', () => {
   assert.strictEqual(unpacked.fileCount, 250);
   assert.strictEqual(unpacked.totalBytes, 53687091200n);
 });
+
+test('Protocol - pack and unpack pull request', () => {
+  const req = packPullRequest({ remotePath: '/root/cdn/app.zip' });
+  const unpacked = unpackPullRequest(req);
+  assert.strictEqual(unpacked.remotePath, '/root/cdn/app.zip');
+});
+
+test('Protocol - pack and unpack pull error', () => {
+  const err = packPullError({ errorCode: 404, message: 'File not found: /nonexistent' });
+  const unpacked = unpackPullError(err);
+  assert.strictEqual(unpacked.errorCode, 404);
+  assert.strictEqual(unpacked.message, 'File not found: /nonexistent');
+});
+