raptiye 0.1.0

package/src/transport/interface.js

@@ -0,0 +1,47 @@
+/**
+ * Abstract Transport interface for Raptiye.
+ */
+
+export class Transport {
+  /**
+   * Send a single buffer to a peer.
+   * @param {number} peer
+   * @param {Uint8Array} buffer
+   */
+  send(peer, buffer) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Send multiple buffers (scatter/gather) to a peer without concatenation.
+   * @param {number} peer
+   * @param {Uint8Array[]} buffers
+   */
+  sendv(peer, buffers) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Register incoming message callback.
+   * @param {(from: number, message: object) => void} handler
+   */
+  onMessage(handler) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Start transport listening and connections.
+   * @returns {Promise<void>}
+   */
+  async start() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Close transport and all connections.
+   * @returns {Promise<void>}
+   */
+  async close() {
+    throw new Error('Not implemented');
+  }
+}

package/src/transport/memory-transport.js

@@ -0,0 +1,67 @@
+/**
+ * In-memory zero-copy transport for local clusters and deterministic microbenchmarks.
+ */
+
+import { Transport } from './interface.js';
+import { decodeMessage, flattenChunks } from '../protocol/wire.js';
+
+export class MemoryNetwork {
+  constructor() {
+    this.transports = new Map();
+  }
+
+  register(id, transport) {
+    this.transports.set(Number(id), transport);
+  }
+
+  unregister(id) {
+    this.transports.delete(Number(id));
+  }
+
+  deliver(from, to, buffers) {
+    const target = this.transports.get(Number(to));
+    if (target && target.started && target._messageHandler) {
+      // Decode message directly from scatter/gather buffers without extra copy if single buffer
+      let flat = null;
+      if (buffers.length === 1) {
+        flat = buffers[0];
+      } else {
+        flat = flattenChunks(buffers);
+      }
+      const msg = decodeMessage(flat);
+      target._messageHandler(from, msg);
+    }
+  }
+}
+
+export class MemoryTransport extends Transport {
+  constructor(id, network) {
+    super();
+    this.id = Number(id);
+    this.network = network;
+    this.started = false;
+    this._messageHandler = null;
+    this.network.register(this.id, this);
+  }
+
+  send(peer, buffer) {
+    this.network.deliver(this.id, peer, [buffer]);
+  }
+
+  sendv(peer, buffers) {
+    this.network.deliver(this.id, peer, buffers);
+  }
+
+  onMessage(handler) {
+    this._messageHandler = handler;
+  }
+
+  async start() {
+    this.started = true;
+  }
+
+  async close() {
+    this.started = false;
+    this.network.unregister(this.id);
+  }
+}

package/src/transport/tcp-transport.js

@@ -0,0 +1,200 @@
+/**
+ * High-performance TCP Transport for Raptiye using Node.js net.Socket and writev.
+ * Zero external dependencies.
+ */
+
+import net from 'node:net';
+import { Transport } from './interface.js';
+import { HEADER_SIZE, MAGIC, readHeader, decodeMessage, flattenChunks } from '../protocol/wire.js';
+
+export class TCPTransport extends Transport {
+  /**
+   * @param {object} config
+   * @param {number} config.id
+   * @param {number} config.port
+   * @param {string} [config.host='127.0.0.1']
+   * @param {Map<number, { host: string, port: number }>} config.peerAddresses
+   */
+  constructor({ id, port, host = '127.0.0.1', peerAddresses = new Map() }) {
+    super();
+    this.id = Number(id);
+    this.port = port;
+    this.host = host;
+    this.peerAddresses = peerAddresses;
+
+    this.server = null;
+    this.started = false;
+    this._messageHandler = null;
+
+    // Outbound peer sockets: peerId -> Socket
+    this._outboundSockets = new Map();
+    // Inbound sockets: Set<Socket>
+    this._inboundSockets = new Set();
+    // Connecting promises: peerId -> Promise<Socket>
+    this._connecting = new Map();
+  }
+
+  registerPeer(peerId, host, port) {
+    this.peerAddresses.set(Number(peerId), { host, port });
+  }
+
+  onMessage(handler) {
+    this._messageHandler = handler;
+  }
+
+  async start() {
+    if (this.started) return;
+
+    await new Promise((resolve, reject) => {
+      this.server = net.createServer((socket) => {
+        this._handleIncomingConnection(socket);
+      });
+
+      this.server.on('error', reject);
+      this.server.listen(this.port, this.host, () => {
+        this.started = true;
+        resolve();
+      });
+    });
+  }
+
+  _handleIncomingConnection(socket) {
+    this._inboundSockets.add(socket);
+    socket.on('close', () => this._inboundSockets.delete(socket));
+    this._attachSocketReader(socket);
+  }
+
+  _attachSocketReader(socket) {
+    socket.setNoDelay(true);
+    let buffer = null;
+
+    socket.on('data', (chunk) => {
+      if (!buffer) {
+        buffer = chunk;
+      } else {
+        buffer = Buffer.concat([buffer, chunk]);
+      }
+
+      while (buffer && buffer.length >= HEADER_SIZE) {
+        let header;
+        try {
+          header = readHeader(buffer, 0);
+        } catch (err) {
+          socket.destroy(err);
+          return;
+        }
+
+        const totalLength = HEADER_SIZE + header.payloadLength;
+        if (buffer.length < totalLength) {
+          break;
+        }
+
+        const rawFrame = buffer.subarray(0, totalLength);
+        buffer = buffer.length > totalLength ? buffer.subarray(totalLength) : null;
+
+        try {
+          const frame = new Uint8Array(rawFrame.buffer, rawFrame.byteOffset, rawFrame.byteLength);
+          const msg = decodeMessage(frame, 0, true);
+          if (msg.sourceNode) {
+            this._outboundSockets.set(msg.sourceNode, socket);
+          }
+          if (this._messageHandler) {
+            this._messageHandler(msg.sourceNode, msg);
+          }
+        } catch (err) {
+          console.error(`[TCPTransport ${this.id}] Error decoding frame:`, err.message);
+        }
+      }
+    });
+
+    socket.on('error', () => {
+      socket.destroy();
+    });
+  }
+
+  async _getOrCreateSocket(peer) {
+    const existing = this._outboundSockets.get(peer);
+    if (existing && !existing.destroyed && existing.writable) {
+      return existing;
+    }
+
+    if (this._connecting.has(peer)) {
+      return this._connecting.get(peer);
+    }
+
+    const addr = this.peerAddresses.get(peer);
+    if (!addr) {
+      throw new Error(`[TCPTransport ${this.id}] No address registered for peer ${peer}`);
+    }
+
+    const connectPromise = new Promise((resolve, reject) => {
+      const socket = net.createConnection({ host: addr.host, port: addr.port }, () => {
+        this._attachSocketReader(socket);
+        this._outboundSockets.set(peer, socket);
+        this._connecting.delete(peer);
+        resolve(socket);
+      });
+
+      socket.on('error', (err) => {
+        this._outboundSockets.delete(peer);
+        this._connecting.delete(peer);
+        reject(err);
+      });
+
+      socket.on('close', () => {
+        this._outboundSockets.delete(peer);
+        this._connecting.delete(peer);
+      });
+    });
+
+    this._connecting.set(peer, connectPromise);
+    return connectPromise;
+  }
+
+  send(peer, buffer) {
+    this._getOrCreateSocket(peer)
+      .then((socket) => {
+        if (!socket.destroyed && socket.writable) {
+          socket.write(buffer);
+        }
+      })
+      .catch(() => {
+        // Connection error handled by retry on next message
+      });
+  }
+
+  sendv(peer, buffers) {
+    this._getOrCreateSocket(peer)
+      .then((socket) => {
+        if (!socket.destroyed && socket.writable) {
+          socket.cork();
+          for (let i = 0; i < buffers.length; i++) {
+            socket.write(buffers[i]);
+          }
+          socket.uncork();
+        }
+      })
+      .catch(() => {
+        // Retry on next send
+      });
+  }
+
+  async close() {
+    this.started = false;
+    for (const socket of this._outboundSockets.values()) {
+      socket.destroy();
+    }
+    this._outboundSockets.clear();
+    this._connecting.clear();
+
+    for (const socket of this._inboundSockets) {
+      socket.destroy();
+    }
+    this._inboundSockets.clear();
+
+    if (this.server) {
+      await new Promise((resolve) => this.server.close(resolve));
+      this.server = null;
+    }
+  }
+}

package/src/types.js

@@ -0,0 +1,87 @@
+/**
+ * Core protocol and consensus definitions for Raptiye.
+ */
+
+export const Role = Object.freeze({
+  FOLLOWER: 1,
+  PRE_CANDIDATE: 2,
+  CANDIDATE: 3,
+  LEADER: 4
+});
+
+export const RoleName = Object.freeze({
+  [Role.FOLLOWER]: 'FOLLOWER',
+  [Role.PRE_CANDIDATE]: 'PRE_CANDIDATE',
+  [Role.CANDIDATE]: 'CANDIDATE',
+  [Role.LEADER]: 'LEADER'
+});
+
+export const MessageType = Object.freeze({
+  PRE_VOTE_REQUEST: 1,
+  PRE_VOTE_RESPONSE: 2,
+  VOTE_REQUEST: 3,
+  VOTE_RESPONSE: 4,
+  APPEND_REQUEST: 5,
+  APPEND_RESPONSE: 6,
+  HEARTBEAT: 7,
+  SNAPSHOT_BEGIN: 8,
+  SNAPSHOT_CHUNK: 9,
+  SNAPSHOT_END: 10,
+  SNAPSHOT_RESPONSE: 11,
+  TIMEOUT_NOW: 12,
+  BATCH_DATA: 13,
+  BATCH_ACK: 14,
+  SESSION_HANDSHAKE: 15,
+  SESSION_ACK: 16
+});
+
+export const MessageTypeName = Object.freeze({
+  [MessageType.PRE_VOTE_REQUEST]: 'PRE_VOTE_REQUEST',
+  [MessageType.PRE_VOTE_RESPONSE]: 'PRE_VOTE_RESPONSE',
+  [MessageType.VOTE_REQUEST]: 'VOTE_REQUEST',
+  [MessageType.VOTE_RESPONSE]: 'VOTE_RESPONSE',
+  [MessageType.APPEND_REQUEST]: 'APPEND_REQUEST',
+  [MessageType.APPEND_RESPONSE]: 'APPEND_RESPONSE',
+  [MessageType.HEARTBEAT]: 'HEARTBEAT',
+  [MessageType.SNAPSHOT_BEGIN]: 'SNAPSHOT_BEGIN',
+  [MessageType.SNAPSHOT_CHUNK]: 'SNAPSHOT_CHUNK',
+  [MessageType.SNAPSHOT_END]: 'SNAPSHOT_END',
+  [MessageType.SNAPSHOT_RESPONSE]: 'SNAPSHOT_RESPONSE',
+  [MessageType.TIMEOUT_NOW]: 'TIMEOUT_NOW',
+  [MessageType.BATCH_DATA]: 'BATCH_DATA',
+  [MessageType.BATCH_ACK]: 'BATCH_ACK',
+  [MessageType.SESSION_HANDSHAKE]: 'SESSION_HANDSHAKE',
+  [MessageType.SESSION_ACK]: 'SESSION_ACK'
+});
+
+export const EntryType = Object.freeze({
+  NORMAL: 1,
+  CONFIG: 2,
+  NOOP: 3
+});
+
+export const EffectType = Object.freeze({
+  SEND: 'SEND',
+  PERSIST_HARD_STATE: 'PERSIST_HARD_STATE',
+  PERSIST_ENTRIES: 'PERSIST_ENTRIES',
+  APPLY: 'APPLY',
+  RESET_ELECTION_TIMER: 'RESET_ELECTION_TIMER',
+  RESET_HEARTBEAT_TIMER: 'RESET_HEARTBEAT_TIMER',
+  BECOME_LEADER: 'BECOME_LEADER',
+  BECOME_FOLLOWER: 'BECOME_FOLLOWER',
+  NOTIFY_COMMITTED: 'NOTIFY_COMMITTED'
+});
+
+export const EventType = Object.freeze({
+  ELECTION_TIMEOUT: 'ELECTION_TIMEOUT',
+  HEARTBEAT_TIMEOUT: 'HEARTBEAT_TIMEOUT',
+  SUBMIT: 'SUBMIT',
+  MESSAGE: 'MESSAGE',
+  TRANSFER_LEADERSHIP: 'TRANSFER_LEADERSHIP'
+});
+
+export const ProtocolConstants = Object.freeze({
+  MAGIC: 0x5250, // 'RP'
+  VERSION: 1,
+  HEADER_SIZE: 28
+});

package/test/chaos/chaos.test.js

@@ -0,0 +1,112 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { ClusterSimulator } from '../../src/sim/cluster.js';
+
+test('Deterministic Chaos Test - 500 randomized faults with continuous invariant checking', () => {
+  const seed = 20260919;
+  const cluster = new ClusterSimulator({
+    nodeCount: 3,
+    seed,
+    packetDelayMin: 1,
+    packetDelayMax: 15,
+    packetDropRate: 0.05,
+    packetDuplicateRate: 0.02
+  });
+
+  cluster.start();
+  cluster.advance(500);
+
+  let submittedCount = 0;
+
+  for (let iteration = 0; iteration < 200; iteration++) {
+    const action = cluster.prng.nextInt(1, 6);
+
+    switch (action) {
+      case 1: {
+        // Submit command
+        submittedCount++;
+        const payload = new Uint8Array([iteration & 0xFF, (iteration * 3) & 0xFF]);
+        cluster.submit(payload);
+        break;
+      }
+
+      case 2: {
+        // Kill a random node if at least 2 are alive
+        const alive = cluster.activeNodes();
+        if (alive.length > 2) {
+          const victim = cluster.prng.pick(alive);
+          cluster.killNode(victim.id);
+        }
+        break;
+      }
+
+      case 3: {
+        // Restart a dead node
+        const dead = Array.from(cluster.nodes.values()).filter(n => !n.alive);
+        if (dead.length > 0) {
+          const reviver = cluster.prng.pick(dead);
+          cluster.restartNode(reviver.id);
+        }
+        break;
+      }
+
+      case 4: {
+        // Create random partition
+        if (cluster.partitions.size === 0) {
+          cluster.partition([1], [2, 3]);
+        }
+        break;
+      }
+
+      case 5: {
+        // Heal partition
+        if (cluster.partitions.size > 0) {
+          cluster.heal();
+        }
+        break;
+      }
+
+      case 6: {
+        // Advance time by small increment
+        const delta = cluster.prng.nextInt(10, 100);
+        cluster.advance(delta);
+        break;
+      }
+    }
+
+    // Advance clock to let network events progress
+    cluster.advance(cluster.prng.nextInt(10, 50));
+    // Continuous invariant assertion!
+    cluster.verify();
+  }
+
+  // Heal everything and revive all nodes to verify final consistency
+  cluster.heal();
+  for (const node of cluster.nodes.values()) {
+    if (!node.alive) {
+      cluster.restartNode(node.id);
+    }
+  }
+
+  // Allow cluster to reach steady state
+  cluster.advance(1500);
+  cluster.verify();
+
+  const leader = cluster.getLeader();
+  assert.ok(leader !== null, 'Cluster must reach stable leader after healing faults');
+
+  // Verify that all caught-up nodes have identical committed logs
+  const leaderCommit = leader.engine.commitIndex;
+  for (const node of cluster.activeNodes()) {
+    if (node.engine.commitIndex === leaderCommit) {
+      const leaderEntries = leader.storage.entries(1n, leaderCommit);
+      const nodeEntries = node.storage.entries(1n, leaderCommit);
+      assert.equal(nodeEntries.length, leaderEntries.length);
+      for (let i = 0; i < leaderEntries.length; i++) {
+        assert.equal(nodeEntries[i].index, leaderEntries[i].index);
+        assert.equal(nodeEntries[i].term, leaderEntries[i].term);
+        assert.deepEqual(nodeEntries[i].payload, leaderEntries[i].payload);
+      }
+    }
+  }
+});

package/test/integration/cluster.test.js

@@ -0,0 +1,192 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { ClusterSimulator } from '../../src/sim/cluster.js';
+import { Role } from '../../src/types.js';
+
+test('ClusterSimulator - 3-node automatic leader election', () => {
+  const cluster = new ClusterSimulator({
+    nodeCount: 3,
+    seed: 42
+  });
+
+  cluster.start();
+  // Initially no leader
+  assert.equal(cluster.getLeader(), null);
+
+  // Advance time past election timeout (e.g. 500ms)
+  cluster.advance(500);
+
+  const leader = cluster.getLeader();
+  assert.ok(leader !== null, 'A leader must be elected');
+  assert.ok([1, 2, 3].includes(leader.id));
+  assert.equal(leader.engine.role, Role.LEADER);
+
+  // Verify invariants
+  cluster.verify();
+});
+
+test('ClusterSimulator - 3-node replication and quorum commit', () => {
+  const cluster = new ClusterSimulator({
+    nodeCount: 3,
+    seed: 12345
+  });
+  cluster.start();
+  cluster.advance(500);
+
+  const leader = cluster.getLeader();
+  assert.ok(leader);
+
+  // Submit 5 commands
+  for (let i = 1; i <= 5; i++) {
+    const payload = new Uint8Array([i, i * 2, i * 3]);
+    const res = cluster.submit(payload);
+    assert.equal(res.success, true);
+    // Allow replication roundtrips
+    cluster.advance(50);
+  }
+
+  // After replication, all active nodes should have committed entries
+  assert.ok(leader.engine.commitIndex >= 5n);
+
+  for (const node of cluster.activeNodes()) {
+    assert.ok(node.engine.commitIndex >= 5n, `Node ${node.id} should have committed index >= 5`);
+    assert.equal(node.storage.lastIndex(), leader.storage.lastIndex());
+  }
+
+  cluster.verify();
+});
+
+test('ClusterSimulator - automatic failover when leader dies', () => {
+  const cluster = new ClusterSimulator({
+    nodeCount: 3,
+    seed: 999
+  });
+  cluster.start();
+  cluster.advance(500);
+
+  const initialLeader = cluster.getLeader();
+  assert.ok(initialLeader);
+  const initialLeaderId = initialLeader.id;
+
+  // Submit 2 commands
+  cluster.submit(new Uint8Array([10, 20]));
+  cluster.advance(50);
+  cluster.submit(new Uint8Array([30, 40]));
+  cluster.advance(50);
+
+  const committedBeforeKill = initialLeader.engine.commitIndex;
+
+  // Kill the leader!
+  cluster.killNode(initialLeaderId);
+  assert.equal(cluster.getLeader(), null);
+
+  // Advance time for remaining 2 nodes to detect missing leader, run pre-vote and election
+  cluster.advance(600);
+
+  const newLeader = cluster.getLeader();
+  assert.ok(newLeader !== null, 'A new leader must be elected after leader failure');
+  assert.notEqual(newLeader.id, initialLeaderId);
+  assert.ok(newLeader.engine.currentTerm > initialLeader.engine.currentTerm);
+
+  // Submit a new command to the new leader!
+  const res = cluster.submit(new Uint8Array([99, 100]));
+  assert.equal(res.success, true);
+  cluster.advance(100);
+
+  // Newly submitted command must successfully commit!
+  assert.ok(newLeader.engine.commitIndex > committedBeforeKill);
+
+  cluster.verify();
+});
+
+test('ClusterSimulator - split-brain prevention during partition', () => {
+  const cluster = new ClusterSimulator({
+    nodeCount: 3,
+    seed: 5555
+  });
+  cluster.start();
+  cluster.advance(500);
+
+  const leader = cluster.getLeader();
+  assert.ok(leader);
+  const leaderId = leader.id;
+  const followers = [1, 2, 3].filter(id => id !== leaderId);
+
+  // Partition: Leader is isolated from followers [B, C]
+  // Group A: [Leader]
+  // Group B: [Follower1, Follower2]
+  cluster.partition([leaderId], followers);
+
+  // Isolated leader tries to submit a command
+  const initialCommit = leader.engine.commitIndex;
+  leader.step({
+    type: 'SUBMIT',
+    payload: new Uint8Array([111, 222])
+  });
+
+  // Advance time
+  cluster.advance(600);
+
+  // The isolated leader MUST NOT be able to commit! (Cannot achieve quorum without majority)
+  assert.equal(leader.engine.commitIndex, initialCommit, 'Isolated leader must not advance commitIndex');
+
+  // Followers in majority group [B, C] must elect a new leader
+  const newLeader = cluster.getLeader();
+  assert.ok(newLeader !== null);
+  assert.notEqual(newLeader.id, leaderId);
+
+  // Majority can commit new commands
+  newLeader.step({
+    type: 'SUBMIT',
+    payload: new Uint8Array([333, 444])
+  });
+  cluster.advance(100);
+  assert.ok(newLeader.engine.commitIndex > initialCommit);
+
+  // Heal partition
+  cluster.heal();
+  cluster.advance(600);
+
+  // Old leader must step down and accept history of new leader
+  assert.equal(leader.engine.role, Role.FOLLOWER);
+  assert.equal(leader.engine.currentTerm, newLeader.engine.currentTerm);
+  assert.equal(leader.storage.lastIndex(), newLeader.storage.lastIndex());
+
+  cluster.verify();
+});
+
+test('ClusterSimulator - automatic follower catch-up after rejoin', () => {
+  const cluster = new ClusterSimulator({
+    nodeCount: 3,
+    seed: 777
+  });
+  cluster.start();
+  cluster.advance(500);
+
+  const leader = cluster.getLeader();
+  assert.ok(leader);
+
+  // Kill node 3
+  const victim = [1, 2, 3].find(id => id !== leader.id);
+  cluster.killNode(victim);
+
+  // Commit 10 commands with the remaining 2 nodes
+  for (let i = 0; i < 10; i++) {
+    cluster.submit(new Uint8Array([i]));
+    cluster.advance(30);
+  }
+
+  const leaderCommit = leader.engine.commitIndex;
+
+  // Restart node 3
+  cluster.restartNode(victim);
+
+  // Advance time to allow leader to detect victim and stream catch-up entries
+  cluster.advance(500);
+
+  const restartedNode = cluster.getNode(victim);
+  assert.equal(restartedNode.storage.lastIndex(), leader.storage.lastIndex());
+  assert.equal(restartedNode.engine.commitIndex, leaderCommit);
+
+  cluster.verify();
+});