teyyare 0.2.0 → 0.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "teyyare",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "High-performance resumable file transfer engine and CLI built on muttafa and raptiye",
   "main": "src/index.js",
   "type": "module",

package/src/core/generation.js

@@ -0,0 +1,107 @@
+/**
+ * Zero-copy generation framing and buffer extraction utilities.
+ * Ensures complete compatibility across all versions of Muttafa and Raptiye.
+ */
+
+/**
+ * Extract scatter/gather buffers [metaBuf, payloadBuf] from any Muttafa generation.
+ * @param {object} batch
+ * @returns {Uint8Array[]}
+ */
+export function extractGenerationBuffers(batch) {
+  if (typeof batch.buffers === 'function') {
+    return batch.buffers();
+  }
+
+  const store = batch.store || batch;
+  if (typeof store.buffers === 'function') {
+    return store.buffers();
+  }
+
+  // Direct extraction from ArenaStore
+  const rawBuf = store._allocator?.rawBuffer;
+  const payload = rawBuf ? rawBuf.subarray(0, store._allocator.usedBytes) : new Uint8Array(0);
+  const metaBuf = new Uint8Array(store.length * 25);
+  const view = new DataView(metaBuf.buffer, metaBuf.byteOffset, metaBuf.byteLength);
+
+  let off = 0;
+  const chunks = store._chunks || [];
+  for (let c = 0; c < chunks.length; c++) {
+    const chunk = chunks[c];
+    for (let i = 0; i < chunk.count; i++) {
+      view.setFloat64(off, chunk.seqs[i], true);
+      view.setUint8(off + 8, chunk.types[i]);
+      view.setUint32(off + 9, chunk.keyOffsets[i], true);
+      view.setUint32(off + 13, chunk.keyLengths[i], true);
+      view.setUint32(off + 17, chunk.valOffsets[i], true);
+      view.setUint32(off + 21, chunk.valLengths[i], true);
+      off += 25;
+    }
+  }
+
+  return [metaBuf, payload];
+}
+
+/**
+ * Fast zero-copy generator / unpacker for received generation buffers.
+ * @param {Uint8Array} metaBuf
+ * @param {Uint8Array} payloadBuf
+ * @returns {Array<{ seq: number, type: number, key: Uint8Array, val: Uint8Array }>}
+ */
+export function unpackGeneration(metaBuf, payloadBuf) {
+  const entryCount = (metaBuf.byteLength / 25) | 0;
+  const view = new DataView(metaBuf.buffer, metaBuf.byteOffset, metaBuf.byteLength);
+  const entries = new Array(entryCount);
+
+  let off = 0;
+  for (let i = 0; i < entryCount; i++) {
+    const seq = view.getFloat64(off, true);
+    const type = view.getUint8(off + 8);
+    const keyOffset = view.getUint32(off + 9, true);
+    const keyLength = view.getUint32(off + 13, true);
+    const valOffset = view.getUint32(off + 17, true);
+    const valLength = view.getUint32(off + 21, true);
+    off += 25;
+
+    entries[i] = {
+      seq,
+      type,
+      key: payloadBuf.subarray(keyOffset, keyOffset + keyLength),
+      val: payloadBuf.subarray(valOffset, valOffset + valLength)
+    };
+  }
+
+  return entries;
+}
+
+/**
+ * Pack a single-mutation batch directly into scatter/gather wire buffers.
+ * @param {number} generationId
+ * @param {number} opcode
+ * @param {Uint8Array} key
+ * @param {Uint8Array} [val]
+ * @returns {object}
+ */
+export function packSingleMutationBatch(generationId, opcode, key, val = new Uint8Array(0)) {
+  const metaBuf = new Uint8Array(25);
+  const view = new DataView(metaBuf.buffer, metaBuf.byteOffset, 25);
+  view.setFloat64(0, 1, true); // seq
+  view.setUint8(8, opcode);
+  view.setUint32(9, 0, true); // keyOffset
+  view.setUint32(13, key.byteLength, true); // keyLength
+  view.setUint32(17, key.byteLength, true); // valOffset
+  view.setUint32(21, val.byteLength, true); // valLength
+
+  const payloadBuf = new Uint8Array(key.byteLength + val.byteLength);
+  payloadBuf.set(key, 0);
+  payloadBuf.set(val, key.byteLength);
+
+  return {
+    generationId,
+    firstSequence: 1,
+    lastSequence: 1,
+    entryCount: 1,
+    byteLength: metaBuf.byteLength + payloadBuf.byteLength,
+    buffers: [metaBuf, payloadBuf]
+  };
+}

package/src/index.js

@@ -4,7 +4,6 @@
 
 import path from 'node:path';
 import { existsSync } from 'node:fs';
-import { MutableBuffer, ArenaStore, BumpAllocator, FrozenArenaStore } from 'muttafa';
 import { TCPTransport, ReplicationPipeline } from 'raptiye';
 import { SenderPipeline } from './sender/pipeline.js';
 import { ReceiverPipeline } from './receiver/pipeline.js';
@@ -12,6 +11,7 @@ import { TransferManifest } from './manifest/manifest.js';
 import { FileResumeState } from './resume/state.js';
 import { MetricsCollector } from './metrics/collector.js';
 import { ProgressReporter } from './cli/progress.js';
+import { packSingleMutationBatch, unpackGeneration } from './core/generation.js';
 import {
   Opcodes,
   packChunkMeta,
@@ -39,22 +39,9 @@ export {
 };
 
 async function submitSingleMutation(replication, opcode, key, value = new Uint8Array(0)) {
-  const store = new ArenaStore({ allocator: new BumpAllocator(16 * 1024) });
-  const mutable = new MutableBuffer(Date.now() & 0x7fffffff, store);
-  mutable.append(1, opcode, key, value);
-  const frozen = mutable.freeze();
-  frozen.markFlushing();
-  const receipt = await replication.submitBatch({
-    generationId: frozen.generation,
-    firstSequence: frozen.firstSequence,
-    lastSequence: frozen.lastSequence,
-    entryCount: frozen.entryCount,
-    byteLength: frozen.byteLength,
-    buffers: frozen.buffers()
-  });
+  const batch = packSingleMutationBatch(Date.now() & 0x7fffffff, opcode, key, value);
+  const receipt = await replication.submitBatch(batch);
   await receipt.delivered();
-  frozen.markFlushed();
-  frozen.release();
 }
 
 /**
@@ -102,9 +89,9 @@ export async function serve({
   replication.onBatch(async (msg) => {
     if (!msg.buffers || msg.buffers.length < 2) return;
     const [metaBuf, payloadBuf] = msg.buffers;
-    const store = FrozenArenaStore.fromBuffers(metaBuf, payloadBuf);
-    if (store.length > 0 && store.getType(0) === Opcodes.PULL_REQUEST) {
-      const { remotePath } = unpackPullRequest(store.getKey(0));
+    const entries = unpackGeneration(metaBuf, payloadBuf);
+    if (entries.length > 0 && entries[0].type === Opcodes.PULL_REQUEST) {
+      const { remotePath } = unpackPullRequest(entries[0].key);
       let resolved = path.isAbsolute(remotePath) ? remotePath : path.resolve(destinationDir, remotePath);
       if (!existsSync(resolved)) {
         const alt = path.resolve(destinationDir, remotePath);

package/src/receiver/pipeline.js

@@ -8,9 +8,9 @@ import fs from 'node:fs/promises';
 import { existsSync, createReadStream } from 'node:fs';
 import path from 'node:path';
 import crypto from 'node:crypto';
-import { FrozenArenaStore } from 'muttafa';
 import { crc32 } from 'raptiye';
 import { Opcodes, unpackChunkMeta, unpackFileBegin, unpackTransferBegin, unpackFileEnd, unpackPullError } from '../core/protocol.js';
+import { unpackGeneration } from '../core/generation.js';
 import { FileResumeState } from '../resume/state.js';
 
 export class ReceiverPipeline {
@@ -70,14 +70,12 @@ export class ReceiverPipeline {
       this.metrics.generationBytes += (metaBuf.byteLength + payloadBuf.byteLength);
     }
 
-    // Zero-copy reconstruction of Muttafa generation
-    const store = FrozenArenaStore.fromBuffers(metaBuf, payloadBuf);
-    const count = store.length;
+    // Zero-copy unpacking of generation entries
+    const entries = unpackGeneration(metaBuf, payloadBuf);
+    const count = entries.length;
 
     for (let i = 0; i < count; i++) {
-      const type = store.getType(i);
-      const key = store.getKey(i);
-      const val = store.getValue(i);
+      const { type, key, val } = entries[i];
 
       if (this.metrics) {
         this.metrics.mutations++;

package/src/sender/pipeline.js

@@ -17,6 +17,8 @@ import {
   packTransferBegin,
   packTransferEnd
 } from '../core/protocol.js';
+import { extractGenerationBuffers } from '../core/generation.js';
+import { FileResumeState } from '../resume/state.js';
 
 class RaptiyeSink extends Sink {
   /**
@@ -36,20 +38,23 @@ class RaptiyeSink extends Sink {
    * @param {import('muttafa').ImmutableBuffer} batch
    */
   async flush(batch) {
-    const buffers = batch.buffers();
+    const buffers = extractGenerationBuffers(batch);
     const t0 = Date.now();
+    const generationId = batch.generationId || batch.generation || 1;
+    const entryCount = batch.entryCount || batch.store?.length || 1;
+    const byteLength = batch.byteLength || (buffers[0].byteLength + buffers[1].byteLength);
 
     if (this.metrics) {
       this.metrics.generations++;
-      this.metrics.generationBytes += batch.byteLength;
+      this.metrics.generationBytes += byteLength;
     }
 
     const receipt = await this.replication.submitBatch({
-      generationId: batch.generationId,
-      firstSequence: batch.firstSequence,
-      lastSequence: batch.lastSequence,
-      entryCount: batch.entryCount,
-      byteLength: batch.byteLength,
+      generationId,
+      firstSequence: batch.firstSequence || 1,
+      lastSequence: batch.lastSequence || 1,
+      entryCount,
+      byteLength,
       buffers
     });