raptiye 0.1.0

package/src/replication/pipeline.js

@@ -0,0 +1,235 @@
+/**
+ * Generic point-to-point batch replication pipeline.
+ * Manages reliable delivery, in-flight windowing, flow control, and ACKs.
+ */
+
+import { MessageType } from '../types.js';
+import { encodeMessage } from '../protocol/wire.js';
+
+export class BatchReceipt {
+  /**
+   * @param {number} generationId
+   * @param {Promise<object>} promise
+   */
+  constructor(generationId, promise) {
+    this.generationId = generationId;
+    this._promise = promise;
+  }
+
+  delivered() {
+    return this._promise;
+  }
+}
+
+export class ReplicationPipeline {
+  /**
+   * @param {object} config
+   * @param {import('../transport/interface.js').Transport} config.transport
+   * @param {number} [config.localNode=1]
+   * @param {number} [config.remoteNode=2]
+   * @param {number} [config.maxInflightBatches=8]
+   * @param {number} [config.maxInflightBytes=32 * 1024 * 1024]
+   */
+  constructor({
+    transport,
+    localNode = 1,
+    remoteNode = 2,
+    maxInflightBatches = 8,
+    maxInflightBytes = 32 * 1024 * 1024
+  }) {
+    this.transport = transport;
+    this.localNode = Number(localNode);
+    this.remoteNode = Number(remoteNode);
+    this.maxInflightBatches = maxInflightBatches;
+    this.maxInflightBytes = maxInflightBytes;
+
+    this.inflightBatches = 0;
+    this.inflightBytes = 0;
+
+    // generationId -> { resolve, reject, sentAt, bytes }
+    this._pendingReceipts = new Map();
+    // Capacity waiters
+    this._capacityWaiters = [];
+    // Receiver callback
+    this._batchHandler = null;
+
+    // Metrics
+    this.stats = {
+      payloadBytes: 0,
+      wireBytes: 0,
+      frames: 0,
+      tcpWrites: 0,
+      tcpDrains: 0,
+      inflightBytes: 0,
+      inflightBatches: 0,
+      rttMs: 0,
+      ackLatencyMs: 0,
+      reconnects: 0
+    };
+
+    // Listen to transport messages
+    this.transport.onMessage((from, msg) => {
+      this._handleMessage(from, msg);
+    });
+  }
+
+  onBatch(handler) {
+    this._batchHandler = handler;
+  }
+
+  canSubmit(bytes = 0) {
+    return (
+      this.inflightBatches < this.maxInflightBatches &&
+      (this.inflightBytes + bytes) <= this.maxInflightBytes
+    );
+  }
+
+  async waitForCapacity(bytes = 0) {
+    while (!this.canSubmit(bytes)) {
+      await new Promise((resolve) => this._capacityWaiters.push(resolve));
+    }
+  }
+
+  _notifyCapacity() {
+    while (this._capacityWaiters.length > 0) {
+      const waiter = this._capacityWaiters.shift();
+      waiter();
+    }
+  }
+
+  /**
+   * Submit a batch to be replicated to the remote peer.
+   * @param {object} batch
+   * @param {number} batch.generationId
+   * @param {number|bigint} batch.firstSequence
+   * @param {number|bigint} batch.lastSequence
+   * @param {number} batch.entryCount
+   * @param {number} [batch.byteLength]
+   * @param {Uint8Array[]} batch.buffers
+   * @returns {Promise<BatchReceipt>}
+   */
+  async submitBatch({
+    generationId,
+    firstSequence,
+    lastSequence,
+    entryCount,
+    byteLength = 0,
+    buffers = []
+  }) {
+    let totalPayload = byteLength;
+    if (totalPayload === 0) {
+      for (let i = 0; i < buffers.length; i++) {
+        totalPayload += buffers[i].byteLength;
+      }
+    }
+
+    // Await inflight capacity (backpressure from slow network / slow receiver)
+    await this.waitForCapacity(totalPayload);
+
+    this.inflightBatches++;
+    this.inflightBytes += totalPayload;
+    this.stats.inflightBatches = this.inflightBatches;
+    this.stats.inflightBytes = this.inflightBytes;
+
+    const deliveryPromise = new Promise((resolve, reject) => {
+      this._pendingReceipts.set(generationId, {
+        resolve,
+        reject,
+        sentAt: Date.now(),
+        bytes: totalPayload
+      });
+    });
+
+    const chunks = encodeMessage({
+      type: MessageType.BATCH_DATA,
+      sourceNode: this.localNode,
+      destNode: this.remoteNode,
+      generationId,
+      firstSequence: BigInt(firstSequence),
+      lastSequence: BigInt(lastSequence),
+      entryCount,
+      buffers
+    });
+
+    this.stats.frames++;
+    this.stats.tcpWrites++;
+    this.stats.payloadBytes += totalPayload;
+
+    // Approximate wire bytes from chunks
+    for (let i = 0; i < chunks.length; i++) {
+      this.stats.wireBytes += chunks[i].byteLength;
+    }
+
+    this.transport.sendv(this.remoteNode, chunks);
+
+    return new BatchReceipt(generationId, deliveryPromise);
+  }
+
+  async _handleMessage(from, msg) {
+    if (msg.type === MessageType.BATCH_DATA) {
+      this.stats.frames++;
+      if (this._batchHandler) {
+        try {
+          await this._batchHandler(msg);
+          // Send ACK
+          const ackChunks = encodeMessage({
+            type: MessageType.BATCH_ACK,
+            sourceNode: this.localNode,
+            destNode: from,
+            generationId: msg.generationId,
+            sequence: msg.lastSequence,
+            bytesReceived: BigInt(msg.buffers ? msg.buffers.reduce((a, b) => a + b.byteLength, 0) : 0),
+            status: 0
+          });
+          this.transport.sendv(from, ackChunks);
+        } catch (err) {
+          const nackChunks = encodeMessage({
+            type: MessageType.BATCH_ACK,
+            sourceNode: this.localNode,
+            destNode: from,
+            generationId: msg.generationId,
+            sequence: msg.lastSequence,
+            bytesReceived: 0n,
+            status: 1
+          });
+          this.transport.sendv(from, nackChunks);
+        }
+      }
+    } else if (msg.type === MessageType.BATCH_ACK) {
+      const pending = this._pendingReceipts.get(msg.generationId);
+      if (pending) {
+        this._pendingReceipts.delete(msg.generationId);
+
+        const rtt = Math.max(0, Date.now() - pending.sentAt);
+        this.stats.rttMs = rtt;
+        this.stats.ackLatencyMs = rtt;
+
+        this.inflightBatches = Math.max(0, this.inflightBatches - 1);
+        this.inflightBytes = Math.max(0, this.inflightBytes - pending.bytes);
+        this.stats.inflightBatches = this.inflightBatches;
+        this.stats.inflightBytes = this.inflightBytes;
+
+        if (msg.status === 0) {
+          pending.resolve(msg);
+        } else {
+          pending.reject(new Error(`Remote node rejected batch ${msg.generationId}`));
+        }
+
+        this._notifyCapacity();
+      }
+    }
+  }
+
+  /**
+   * Wait until all submitted batches are delivered.
+   */
+  async drain() {
+    while (this._pendingReceipts.size > 0) {
+      await new Promise((resolve) => setTimeout(resolve, 10));
+    }
+  }
+
+  getStats() {
+    return { ...this.stats };
+  }
+}

package/src/sim/cluster.js

@@ -0,0 +1,316 @@
+/**
+ * Deterministic Cluster Simulator for Raptiye.
+ * Fully reproducible from random seed with comprehensive fault injection.
+ */
+
+import { PRNG } from './prng.js';
+import { VirtualClock } from './virtual-clock.js';
+import { ConsensusEngine } from '../core/engine.js';
+import { MemoryLog } from '../storage/memory-log.js';
+import { ClusterInvariants } from '../core/invariants.js';
+import { Role, EffectType, EventType } from '../types.js';
+
+export class SimulatedNode {
+  constructor({ id, peers, storage, clock, prng, cluster, minElection = 150, maxElection = 300, heartbeatInterval = 50 }) {
+    this.id = Number(id);
+    this.peers = peers;
+    this.storage = storage;
+    this.clock = clock;
+    this.prng = prng;
+    this.cluster = cluster;
+
+    this.minElection = minElection;
+    this.maxElection = maxElection;
+    this.heartbeatInterval = heartbeatInterval;
+
+    this.alive = true;
+    this.electionTimerId = null;
+    this.heartbeatTimerId = null;
+
+    this.appliedEntries = [];
+    this.committedNotifications = [];
+
+    this.engine = new ConsensusEngine({
+      id: this.id,
+      peers: this.peers,
+      storage: this.storage
+    });
+  }
+
+  start() {
+    this.resetElectionTimer();
+  }
+
+  stop() {
+    this.alive = false;
+    if (this.electionTimerId) {
+      this.clock.clearTimeout(this.electionTimerId);
+      this.electionTimerId = null;
+    }
+    if (this.heartbeatTimerId) {
+      this.clock.clearTimeout(this.heartbeatTimerId);
+      this.heartbeatTimerId = null;
+    }
+  }
+
+  restart() {
+    this.alive = true;
+    // Recreate engine from stored state
+    this.engine = new ConsensusEngine({
+      id: this.id,
+      peers: this.peers,
+      storage: this.storage
+    });
+    this.start();
+  }
+
+  resetElectionTimer() {
+    if (!this.alive) return;
+    if (this.electionTimerId) {
+      this.clock.clearTimeout(this.electionTimerId);
+    }
+    const timeout = this.prng.nextInt(this.minElection, this.maxElection);
+    this.electionTimerId = this.clock.setTimeout(() => {
+      this.electionTimerId = null;
+      this.step({ type: EventType.ELECTION_TIMEOUT });
+    }, timeout);
+  }
+
+  resetHeartbeatTimer() {
+    if (!this.alive) return;
+    if (this.heartbeatTimerId) {
+      this.clock.clearTimeout(this.heartbeatTimerId);
+    }
+    this.heartbeatTimerId = this.clock.setTimeout(() => {
+      this.heartbeatTimerId = null;
+      this.step({ type: EventType.HEARTBEAT_TIMEOUT });
+    }, this.heartbeatInterval);
+  }
+
+  step(event) {
+    if (!this.alive) return [];
+    const effects = this.engine.step(event);
+    this.processEffects(effects);
+    return effects;
+  }
+
+  processEffects(effects) {
+    for (const effect of effects) {
+      switch (effect.type) {
+        case EffectType.SEND:
+          this.cluster.deliverMessage(this.id, effect.to, effect.message);
+          break;
+
+        case EffectType.RESET_ELECTION_TIMER:
+          this.resetElectionTimer();
+          break;
+
+        case EffectType.RESET_HEARTBEAT_TIMER:
+          this.resetHeartbeatTimer();
+          break;
+
+        case EffectType.BECOME_LEADER:
+          if (this.electionTimerId) {
+            this.clock.clearTimeout(this.electionTimerId);
+            this.electionTimerId = null;
+          }
+          this.resetHeartbeatTimer();
+          break;
+
+        case EffectType.BECOME_FOLLOWER:
+          if (this.heartbeatTimerId) {
+            this.clock.clearTimeout(this.heartbeatTimerId);
+            this.heartbeatTimerId = null;
+          }
+          this.resetElectionTimer();
+          break;
+
+        case EffectType.APPLY:
+          this.appliedEntries.push(effect.entry);
+          break;
+
+        case EffectType.NOTIFY_COMMITTED:
+          this.committedNotifications.push(effect.index);
+          break;
+      }
+    }
+  }
+}
+
+export class ClusterSimulator {
+  constructor({
+    nodeCount = 3,
+    seed = 184822,
+    minElection = 150,
+    maxElection = 300,
+    heartbeatInterval = 50,
+    packetDelayMin = 1,
+    packetDelayMax = 10,
+    packetDropRate = 0,
+    packetDuplicateRate = 0
+  } = {}) {
+    this.seed = seed;
+    this.prng = new PRNG(seed);
+    this.clock = new VirtualClock();
+    this.invariants = new ClusterInvariants();
+
+    this.packetDelayMin = packetDelayMin;
+    this.packetDelayMax = packetDelayMax;
+    this.packetDropRate = packetDropRate;
+    this.packetDuplicateRate = packetDuplicateRate;
+
+    this.partitions = new Set(); // Strings of "from:to"
+    this.nodes = new Map();
+
+    const nodeIds = [];
+    for (let i = 1; i <= nodeCount; i++) {
+      nodeIds.push(i);
+    }
+
+    for (const id of nodeIds) {
+      const peers = nodeIds.filter(nid => nid !== id);
+      const storage = new MemoryLog();
+      const node = new SimulatedNode({
+        id,
+        peers,
+        storage,
+        clock: this.clock,
+        prng: this.prng,
+        cluster: this,
+        minElection,
+        maxElection,
+        heartbeatInterval
+      });
+      this.nodes.set(id, node);
+    }
+  }
+
+  start() {
+    for (const node of this.nodes.values()) {
+      node.start();
+    }
+  }
+
+  getNode(id) {
+    return this.nodes.get(Number(id));
+  }
+
+  activeNodes() {
+    return Array.from(this.nodes.values()).filter(n => n.alive);
+  }
+
+  getLeader() {
+    let bestLeader = null;
+    let maxTerm = -1n;
+    for (const node of this.activeNodes()) {
+      if (node.engine.role === Role.LEADER) {
+        if (node.engine.currentTerm > maxTerm) {
+          maxTerm = node.engine.currentTerm;
+          bestLeader = node;
+        }
+      }
+    }
+    return bestLeader;
+  }
+
+  deliverMessage(from, to, message) {
+    const sender = this.getNode(from);
+    const receiver = this.getNode(to);
+
+    if (!sender || !receiver || !sender.alive || !receiver.alive) {
+      return;
+    }
+
+    // Check partition
+    if (this.isPartitioned(from, to)) {
+      return;
+    }
+
+    // Check packet drop
+    if (this.packetDropRate > 0 && this.prng.boolean(this.packetDropRate)) {
+      return;
+    }
+
+    const sendPacket = () => {
+      const delay = this.prng.nextInt(this.packetDelayMin, this.packetDelayMax);
+      this.clock.setTimeout(() => {
+        if (receiver.alive && !this.isPartitioned(from, to)) {
+          receiver.step({
+            type: EventType.MESSAGE,
+            message
+          });
+          this.verify();
+        }
+      }, delay);
+    };
+
+    sendPacket();
+
+    // Check packet duplication
+    if (this.packetDuplicateRate > 0 && this.prng.boolean(this.packetDuplicateRate)) {
+      sendPacket();
+    }
+  }
+
+  isPartitioned(from, to) {
+    return this.partitions.has(`${from}:${to}`) || this.partitions.has(`${to}:${from}`);
+  }
+
+  partition(groupA, groupB) {
+    for (const a of groupA) {
+      for (const b of groupB) {
+        this.partitions.add(`${a}:${b}`);
+        this.partitions.add(`${b}:${a}`);
+      }
+    }
+  }
+
+  heal() {
+    this.partitions.clear();
+  }
+
+  killNode(id) {
+    const node = this.getNode(id);
+    if (node) {
+      node.stop();
+    }
+  }
+
+  restartNode(id) {
+    const node = this.getNode(id);
+    if (node) {
+      node.restart();
+    }
+  }
+
+  submit(payload) {
+    const leader = this.getLeader();
+    if (!leader) {
+      return { success: false, error: 'NO_LEADER' };
+    }
+    leader.step({
+      type: EventType.SUBMIT,
+      payload
+    });
+    this.verify();
+    return { success: true, leaderId: leader.id };
+  }
+
+  advance(ms) {
+    this.clock.advance(ms);
+    this.verify();
+  }
+
+  runUntilQuiet(maxSteps = 1000) {
+    let steps = 0;
+    while (this.clock.pendingCount() > 0 && steps < maxSteps) {
+      this.clock.step();
+      this.verify();
+      steps++;
+    }
+  }
+
+  verify() {
+    this.invariants.check(this.activeNodes());
+  }
+}

package/src/sim/prng.js

@@ -0,0 +1,52 @@
+/**
+ * Seeded pseudo-random number generator for deterministic cluster simulations.
+ * Uses 32-bit Mulberry32 algorithm.
+ */
+
+export class PRNG {
+  constructor(seed = 184822) {
+    this.initialSeed = seed;
+    this.s = seed >>> 0;
+  }
+
+  /**
+   * Generates a float in [0, 1).
+   * @returns {number}
+   */
+  nextFloat() {
+    let t = (this.s += 0x6D2B79F5) >>> 0;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  }
+
+  /**
+   * Generates an integer in [min, max] inclusive.
+   * @param {number} min
+   * @param {number} max
+   * @returns {number}
+   */
+  nextInt(min, max) {
+    return Math.floor(this.nextFloat() * (max - min + 1)) + min;
+  }
+
+  /**
+   * Returns true with given probability [0, 1].
+   * @param {number} probability
+   * @returns {boolean}
+   */
+  boolean(probability = 0.5) {
+    return this.nextFloat() < probability;
+  }
+
+  /**
+   * Picks a random element from an array.
+   * @template T
+   * @param {T[]} array
+   * @returns {T}
+   */
+  pick(array) {
+    if (array.length === 0) return null;
+    return array[this.nextInt(0, array.length - 1)];
+  }
+}

package/src/sim/virtual-clock.js

@@ -0,0 +1,82 @@
+/**
+ * Virtual discrete-event clock for deterministic simulation.
+ */
+
+export class VirtualClock {
+  constructor() {
+    this.now = 0;
+    this._timers = []; // Array of { id, dueTime, callback, period }
+    this._nextId = 1;
+  }
+
+  _insertTimer(timer) {
+    let low = 0;
+    let high = this._timers.length;
+    while (low < high) {
+      const mid = (low + high) >>> 1;
+      if (this._timers[mid].dueTime <= timer.dueTime) {
+        low = mid + 1;
+      } else {
+        high = mid;
+      }
+    }
+    this._timers.splice(low, 0, timer);
+  }
+
+  setTimeout(callback, delayMs) {
+    const id = this._nextId++;
+    const dueTime = this.now + Math.max(0, delayMs);
+    this._insertTimer({ id, dueTime, callback, period: null });
+    return id;
+  }
+
+  setInterval(callback, periodMs) {
+    const id = this._nextId++;
+    const dueTime = this.now + Math.max(0, periodMs);
+    this._insertTimer({ id, dueTime, callback, period: periodMs });
+    return id;
+  }
+
+  clearTimeout(id) {
+    const idx = this._timers.findIndex(t => t.id === id);
+    if (idx !== -1) {
+      this._timers.splice(idx, 1);
+    }
+  }
+
+  clearInterval(id) {
+    this.clearTimeout(id);
+  }
+
+  advance(ms) {
+    const target = this.now + ms;
+    while (this._timers.length > 0 && this._timers[0].dueTime <= target) {
+      const timer = this._timers.shift();
+      this.now = timer.dueTime;
+      timer.callback();
+
+      if (timer.period !== null) {
+        timer.dueTime = this.now + timer.period;
+        this._insertTimer(timer);
+      }
+    }
+    this.now = target;
+  }
+
+  step() {
+    if (this._timers.length === 0) return false;
+    const timer = this._timers.shift();
+    this.now = timer.dueTime;
+    timer.callback();
+
+    if (timer.period !== null) {
+      timer.dueTime = this.now + timer.period;
+      this._insertTimer(timer);
+    }
+    return true;
+  }
+
+  pendingCount() {
+    return this._timers.length;
+  }
+}