raptiye 0.1.0

package/src/core/invariants.js

@@ -0,0 +1,146 @@
+/**
+ * Safety invariant assertions for Raptiye clusters.
+ * Enforces the 8 core Raft safety properties across the cluster state.
+ */
+
+export class ClusterInvariants {
+  constructor() {
+    // History of committed entries by index: index -> { term, checksum }
+    this.committedEntries = new Map();
+  }
+
+  /**
+   * Run all invariant checks across all nodes in the cluster.
+   * @param {Array<{ id: number, engine: object, storage: object }>} nodes
+   */
+  check(nodes) {
+    this.checkSingleLeaderPerTerm(nodes);
+    this.checkCommitIndexMonotonicity(nodes);
+    this.checkLastAppliedLeqCommitIndex(nodes);
+    this.checkLogMatching(nodes);
+    this.checkCommittedEntriesImmutability(nodes);
+  }
+
+  /**
+   * Invariant 1: Election Safety
+   * At most one leader can be elected in a given term.
+   */
+  checkSingleLeaderPerTerm(nodes) {
+    const termLeaders = new Map();
+    for (const node of nodes) {
+      if (node.engine.role === 4 /* LEADER */) {
+        const term = node.engine.currentTerm;
+        if (termLeaders.has(term)) {
+          const existing = termLeaders.get(term);
+          if (existing !== node.id) {
+            throw new Error(
+              `Invariant Violation [Election Safety]: Multiple leaders in term ${term}: node ${existing} and node ${node.id}`
+            );
+          }
+        } else {
+          termLeaders.set(term, node.id);
+        }
+      }
+    }
+  }
+
+  /**
+   * Invariant 4: CommitIndex Monotonicity
+   * A node's commitIndex never decreases.
+   */
+  checkCommitIndexMonotonicity(nodes) {
+    for (const node of nodes) {
+      if (node.engine.commitIndex < node.engine._prevCheckedCommitIndex) {
+        throw new Error(
+          `Invariant Violation [Commit Index]: Node ${node.id} commitIndex decreased: ${node.engine._prevCheckedCommitIndex} -> ${node.engine.commitIndex}`
+        );
+      }
+      node.engine._prevCheckedCommitIndex = node.engine.commitIndex;
+    }
+  }
+
+  /**
+   * Invariant 5: LastApplied <= CommitIndex
+   */
+  checkLastAppliedLeqCommitIndex(nodes) {
+    for (const node of nodes) {
+      if (node.engine.lastApplied > node.engine.commitIndex) {
+        throw new Error(
+          `Invariant Violation [Applied Leq Commit]: Node ${node.id} lastApplied (${node.engine.lastApplied}) > commitIndex (${node.engine.commitIndex})`
+        );
+      }
+    }
+  }
+
+  /**
+   * Invariant 7: Log Matching Property
+   * If two logs contain an entry with the same index and term,
+   * then the logs are identical in all entries up through the given index.
+   */
+  checkLogMatching(nodes) {
+    for (let i = 0; i < nodes.length; i++) {
+      for (let j = i + 1; j < nodes.length; j++) {
+        const n1 = nodes[i];
+        const n2 = nodes[j];
+        const minLast = n1.storage.lastIndex() < n2.storage.lastIndex()
+          ? n1.storage.lastIndex()
+          : n2.storage.lastIndex();
+
+        const firstCommon = n1.storage.firstIndex() > n2.storage.firstIndex()
+          ? n1.storage.firstIndex()
+          : n2.storage.firstIndex();
+
+        let matchedAtHigher = false;
+        // Scan backwards from minLast
+        for (let idx = minLast; idx >= firstCommon; idx--) {
+          const t1 = n1.storage.term(idx);
+          const t2 = n2.storage.term(idx);
+          if (t1 !== null && t2 !== null) {
+            if (t1 === t2) {
+              matchedAtHigher = true;
+            } else if (matchedAtHigher) {
+              // Violated: matched at a higher index, but differed at prior index!
+              throw new Error(
+                `Invariant Violation [Log Matching]: Node ${n1.id} and Node ${n2.id} matched at higher index but differ at prior index ${idx} (${t1} vs ${t2})`
+              );
+            }
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Invariants 2 & 3: Committed Entries Durability and State Machine Safety
+   * Once an entry is committed by any leader, it must never disappear
+   * and no node may commit a different entry for that index.
+   */
+  checkCommittedEntriesImmutability(nodes) {
+    for (const node of nodes) {
+      const commitIndex = node.engine.commitIndex;
+      if (commitIndex === 0n) continue;
+
+      const first = node.storage.firstIndex();
+      const checkFrom = first > 1n ? first : 1n;
+
+      if (commitIndex >= checkFrom) {
+        const entries = node.storage.entries(checkFrom, commitIndex);
+        for (const entry of entries) {
+          const existing = this.committedEntries.get(entry.index);
+          if (existing) {
+            if (existing.term !== entry.term || existing.checksum !== entry.checksum) {
+              throw new Error(
+                `Invariant Violation [State Machine Safety]: Committed index ${entry.index} has conflicting entries! Existing: term ${existing.term}, crc ${existing.checksum}; Node ${node.id}: term ${entry.term}, crc ${entry.checksum}`
+              );
+            }
+          } else {
+            this.committedEntries.set(entry.index, {
+              term: entry.term,
+              checksum: entry.checksum
+            });
+          }
+        }
+      }
+    }
+  }
+}

package/src/node/raptiye.js

@@ -0,0 +1,296 @@
+/**
+ * High-level Raptiye Node coordinating consensus engine, storage, transport, and public API.
+ */
+
+import { ConsensusEngine } from '../core/engine.js';
+import { Role, RoleName, EffectType, EventType, MessageType } from '../types.js';
+import { encodeMessage } from '../protocol/wire.js';
+import { NodeStats } from './stats.js';
+
+export class Raptiye {
+  /**
+   * @param {object} config
+   * @param {number} config.id
+   * @param {number[]} [config.peers=[]]
+   * @param {import('../storage/interface.js').LogStorage} config.storage
+   * @param {import('../transport/interface.js').Transport} config.transport
+   * @param {(entry: { term: bigint, index: bigint, payload: Uint8Array }) => void} [config.apply]
+   * @param {object} [config.election]
+   * @param {number} [config.election.minTimeout=150]
+   * @param {number} [config.election.maxTimeout=300]
+   * @param {number} [config.heartbeatInterval=50]
+   * @param {object} [config.replication]
+   * @param {number} [config.replication.maxBatchBytes=64*1024]
+   * @param {number} [config.replication.maxBatchEntries=256]
+   * @param {number} [config.replication.maxInflightBatches=16]
+   * @param {number} [config.replication.maxInflightBytes=4*1024*1024]
+   */
+  constructor({
+    id,
+    peers = [],
+    storage,
+    transport,
+    apply = null,
+    election = {},
+    heartbeatInterval = 50,
+    replication = {}
+  }) {
+    this.id = Number(id);
+    this.peers = peers.map(Number).filter(p => p !== this.id);
+    this.storage = storage;
+    this.transport = transport;
+    this.applyCallback = apply;
+
+    this.minElectionTimeout = election.minTimeout || 150;
+    this.maxElectionTimeout = election.maxTimeout || 300;
+    this.heartbeatInterval = heartbeatInterval;
+
+    this.engine = new ConsensusEngine({
+      id: this.id,
+      peers: this.peers,
+      storage: this.storage,
+      maxBatchBytes: replication.maxBatchBytes,
+      maxBatchEntries: replication.maxBatchEntries,
+      maxInflightBatches: replication.maxInflightBatches,
+      maxInflightBytes: replication.maxInflightBytes
+    });
+
+    this.statsCollector = new NodeStats();
+    this.started = false;
+    this._electionTimer = null;
+    this._heartbeatTimer = null;
+
+    // Fast sequence -> promise resolvers for committed entries
+    this._pendingCommits = new Map();
+
+    // Wire up transport incoming message handler
+    this.transport.onMessage((from, msg) => {
+      if (!this.started) return;
+      this._step({
+        type: EventType.MESSAGE,
+        message: msg
+      });
+    });
+  }
+
+  async start() {
+    if (this.started) return;
+    this.started = true;
+    await this.transport.start();
+    this._resetElectionTimer();
+  }
+
+  role() {
+    return RoleName[this.engine.role];
+  }
+
+  leader() {
+    return this.engine.leaderId;
+  }
+
+  stats() {
+    return this.statsCollector.snapshot(this);
+  }
+
+  /**
+   * Submit an opaque binary payload for consensus replication.
+   * @param {Uint8Array} payload
+   * @returns {bigint} Log index assigned to this command
+   */
+  submit(payload) {
+    if (this.engine.role !== Role.LEADER) {
+      throw new Error(`Not leader (current leader is ${this.engine.leaderId})`);
+    }
+
+    this.statsCollector.recordSubmit(payload.byteLength);
+
+    const targetIndex = this.storage.lastIndex() + 1n;
+
+    this._step({
+      type: EventType.SUBMIT,
+      payload
+    });
+
+    return targetIndex;
+  }
+
+  /**
+   * Wait for a submitted sequence/log index to be committed by consensus quorum.
+   * @param {bigint} index
+   * @returns {Promise<void>}
+   */
+  committed(index) {
+    const idx = BigInt(index);
+    if (this.engine.commitIndex >= idx) {
+      return Promise.resolve();
+    }
+
+    return new Promise((resolve, reject) => {
+      let bucket = this._pendingCommits.get(idx);
+      if (!bucket) {
+        bucket = [];
+        this._pendingCommits.set(idx, bucket);
+      }
+      bucket.push({ resolve, reject });
+    });
+  }
+
+  /**
+   * Transfer leadership to a specific peer.
+   * @param {number} targetPeer
+   * @returns {Promise<void>}
+   */
+  async transferLeadership(targetPeer) {
+    if (this.engine.role !== Role.LEADER) {
+      throw new Error(`Not leader`);
+    }
+
+    this._step({
+      type: EventType.TRANSFER_LEADERSHIP,
+      targetPeer: Number(targetPeer)
+    });
+
+    // Wait until leadership leaves this node or times out
+    const start = Date.now();
+    while (this.engine.role === Role.LEADER && Date.now() - start < 3000) {
+      await new Promise(resolve => setTimeout(resolve, 20));
+    }
+  }
+
+  /**
+   * Graceful shutdown with optional leadership transfer.
+   * @param {object} [options]
+   * @param {boolean} [options.transferLeadership=false]
+   */
+  async shutdown({ transferLeadership = false } = {}) {
+    if (transferLeadership && this.engine.role === Role.LEADER && this.peers.length > 0) {
+      const candidate = this.peers[0];
+      try {
+        await this.transferLeadership(candidate);
+      } catch {
+        // Continue shutdown
+      }
+    }
+
+    this.started = false;
+    this._clearTimers();
+
+    // Reject any pending uncommitted submissions
+    for (const bucket of this._pendingCommits.values()) {
+      for (const p of bucket) {
+        p.reject(new Error('Node shutting down'));
+      }
+    }
+    this._pendingCommits.clear();
+
+    await this.transport.close();
+    this.storage.close();
+  }
+
+  _step(event) {
+    const prevRole = this.engine.role;
+    const effects = this.engine.step(event);
+    this._processEffects(effects);
+
+    if (this.engine.role !== prevRole) {
+      this.statsCollector.recordLeaderChange();
+      if (this.engine.role === Role.LEADER) {
+        this.statsCollector.recordElection();
+      }
+    }
+  }
+
+  _processEffects(effects) {
+    for (let i = 0; i < effects.length; i++) {
+      const effect = effects[i];
+      switch (effect.type) {
+        case EffectType.SEND: {
+          const chunks = encodeMessage(effect.message);
+          this.transport.sendv(effect.to, chunks);
+          break;
+        }
+
+        case EffectType.RESET_ELECTION_TIMER:
+          this._resetElectionTimer();
+          break;
+
+        case EffectType.RESET_HEARTBEAT_TIMER:
+          this._resetHeartbeatTimer();
+          break;
+
+        case EffectType.BECOME_LEADER:
+          this._clearElectionTimer();
+          this._resetHeartbeatTimer();
+          break;
+
+        case EffectType.BECOME_FOLLOWER:
+          this._clearHeartbeatTimer();
+          this._resetElectionTimer();
+          break;
+
+        case EffectType.APPLY:
+          this.statsCollector.recordApply();
+          if (this.applyCallback) {
+            this.applyCallback(effect.entry);
+          }
+          break;
+
+        case EffectType.NOTIFY_COMMITTED: {
+          this.statsCollector.recordCommit();
+          const idx = effect.index;
+          const bucket = this._pendingCommits.get(idx);
+          if (bucket) {
+            for (let j = 0; j < bucket.length; j++) {
+              bucket[j].resolve();
+            }
+            this._pendingCommits.delete(idx);
+          }
+          break;
+        }
+      }
+    }
+  }
+
+  _resetElectionTimer() {
+    if (!this.started || this.engine.role === Role.LEADER) return;
+    this._clearElectionTimer();
+
+    // Jittered random election timeout
+    const range = this.maxElectionTimeout - this.minElectionTimeout;
+    const timeout = this.minElectionTimeout + Math.floor(Math.random() * (range + 1));
+
+    this._electionTimer = setTimeout(() => {
+      this._electionTimer = null;
+      this._step({ type: EventType.ELECTION_TIMEOUT });
+    }, timeout);
+  }
+
+  _clearElectionTimer() {
+    if (this._electionTimer) {
+      clearTimeout(this._electionTimer);
+      this._electionTimer = null;
+    }
+  }
+
+  _resetHeartbeatTimer() {
+    if (!this.started || this.engine.role !== Role.LEADER) return;
+    this._clearHeartbeatTimer();
+
+    this._heartbeatTimer = setTimeout(() => {
+      this._heartbeatTimer = null;
+      this._step({ type: EventType.HEARTBEAT_TIMEOUT });
+    }, this.heartbeatInterval);
+  }
+
+  _clearHeartbeatTimer() {
+    if (this._heartbeatTimer) {
+      clearTimeout(this._heartbeatTimer);
+      this._heartbeatTimer = null;
+    }
+  }
+
+  _clearTimers() {
+    this._clearElectionTimer();
+    this._clearHeartbeatTimer();
+  }
+}

package/src/node/stats.js

@@ -0,0 +1,82 @@
+/**
+ * Zero-allocation aggregated counters and telemetry for Raptiye.
+ */
+
+export class NodeStats {
+  constructor() {
+    this.submitted = 0;
+    this.replicated = 0;
+    this.committed = 0;
+    this.applied = 0;
+
+    this.wireBytes = 0;
+    this.payloadBytes = 0;
+    this.protocolOverheadBytes = 0;
+    this.copiedBytes = 0;
+    this.allocatedBytes = 0;
+
+    this.elections = 0;
+    this.failedElections = 0;
+    this.leaderChanges = 0;
+
+    this.snapshotsSent = 0;
+    this.snapshotsReceived = 0;
+    this.catchupEntries = 0;
+  }
+
+  recordSubmit(byteLength) {
+    this.submitted++;
+    this.payloadBytes += byteLength;
+  }
+
+  recordCommit() {
+    this.committed++;
+  }
+
+  recordApply() {
+    this.applied++;
+  }
+
+  recordElection() {
+    this.elections++;
+  }
+
+  recordLeaderChange() {
+    this.leaderChanges++;
+  }
+
+  recordWireBytes(wire, payload, overhead, copied, allocated) {
+    this.wireBytes += wire;
+    this.protocolOverheadBytes += overhead;
+    this.copiedBytes += copied;
+    this.allocatedBytes += allocated;
+  }
+
+  snapshot(node) {
+    const engine = node ? node.engine : null;
+    return {
+      role: node ? node.role() : null,
+      term: engine ? engine.currentTerm : 0n,
+      leaderId: engine ? engine.leaderId : null,
+      commitIndex: engine ? engine.commitIndex : 0n,
+      lastApplied: engine ? engine.lastApplied : 0n,
+      lastLogIndex: node && node.storage ? node.storage.lastIndex() : 0n,
+
+      submitted: this.submitted,
+      committed: this.committed,
+      applied: this.applied,
+
+      wireBytes: this.wireBytes,
+      payloadBytes: this.payloadBytes,
+      protocolOverheadBytes: this.protocolOverheadBytes,
+      copiedBytes: this.copiedBytes,
+      allocatedBytes: this.allocatedBytes,
+      copiesPerReplicatedByte: this.payloadBytes > 0 ? (this.copiedBytes / this.payloadBytes) : 0,
+
+      elections: this.elections,
+      leaderChanges: this.leaderChanges,
+      snapshotsSent: this.snapshotsSent,
+      snapshotsReceived: this.snapshotsReceived
+    };
+  }
+}

package/src/protocol/checksum.js

@@ -0,0 +1,66 @@
+/**
+ * High-performance, zero-dependency CRC32 implementation using Uint32Array lookup table.
+ */
+
+const CRC_TABLE = new Uint32Array(256);
+
+// Precompute CRC-32 lookup table (polynomial 0xEDB88320)
+for (let i = 0; i < 256; i++) {
+  let c = i;
+  for (let j = 0; j < 8; j++) {
+    c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
+  }
+  CRC_TABLE[i] = c >>> 0;
+}
+
+/**
+ * Calculate CRC32 of a Uint8Array buffer or slice.
+ * @param {Uint8Array} buf
+ * @param {number} [offset=0]
+ * @param {number} [length=buf.length - offset]
+ * @param {number} [prevCrc=0]
+ * @returns {number} 32-bit unsigned integer
+ */
+export function crc32(buf, offset = 0, length = buf.length - offset, prevCrc = 0) {
+  let crc = (prevCrc ^ -1) >>> 0;
+  const end = offset + length;
+  
+  // Unroll loop for speed
+  let i = offset;
+  const unrolledEnd = offset + ((length >>> 3) << 3);
+  
+  while (i < unrolledEnd) {
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ buf[i]) & 0xFF];
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ buf[i + 1]) & 0xFF];
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ buf[i + 2]) & 0xFF];
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ buf[i + 3]) & 0xFF];
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ buf[i + 4]) & 0xFF];
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ buf[i + 5]) & 0xFF];
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ buf[i + 6]) & 0xFF];
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ buf[i + 7]) & 0xFF];
+    i += 8;
+  }
+
+  while (i < end) {
+    crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ buf[i]) & 0xFF];
+    i++;
+  }
+
+  return (crc ^ -1) >>> 0;
+}
+
+/**
+ * Update CRC32 across multiple buffers (scatter/gather).
+ * @param {Uint8Array[]} buffers
+ * @returns {number}
+ */
+export function crc32Buffers(buffers) {
+  let crc = 0;
+  for (let i = 0; i < buffers.length; i++) {
+    const buf = buffers[i];
+    if (buf && buf.byteLength > 0) {
+      crc = crc32(buf, 0, buf.byteLength, crc);
+    }
+  }
+  return crc;
+}