raptiye 0.1.0

package/src/protocol/wire.js

@@ -0,0 +1,551 @@
+/**
+ * Binary wire protocol encoder and decoder for Raptiye.
+ * Zero-copy scatter/gather design using Uint8Array and DataView.
+ */
+
+import { MessageType, EntryType, ProtocolConstants } from '../types.js';
+import { crc32, crc32Buffers } from './checksum.js';
+
+export const HEADER_SIZE = ProtocolConstants.HEADER_SIZE; // 28 bytes
+export const MAGIC = ProtocolConstants.MAGIC; // 0x5250 ('RP')
+export const VERSION = ProtocolConstants.VERSION; // 1
+
+export class WireMetrics {
+  constructor() {
+    this.wireBytes = 0;
+    this.payloadBytes = 0;
+    this.protocolOverheadBytes = 0;
+    this.allocatedBytes = 0;
+    this.copiedBytes = 0;
+  }
+
+  reset() {
+    this.wireBytes = 0;
+    this.payloadBytes = 0;
+    this.protocolOverheadBytes = 0;
+    this.allocatedBytes = 0;
+    this.copiedBytes = 0;
+  }
+}
+
+export const globalWireMetrics = new WireMetrics();
+
+/**
+ * Write common header into a buffer.
+ * Header layout (28 bytes):
+ * 0..1:   Magic (0x5250) [uint16 BE]
+ * 2:      Version (1) [uint8]
+ * 3:      MessageType [uint8]
+ * 4..5:   Flags [uint16 BE]
+ * 6..13:  Term [uint64 BigInt BE]
+ * 14..15: SourceNode [uint16 BE]
+ * 16..17: DestNode [uint16 BE]
+ * 18..21: PayloadLength [uint32 BE]
+ * 22..25: Checksum [uint32 BE]
+ * 26..27: Reserved [uint16 BE]
+ */
+export function writeHeader(buf, offset, {
+  messageType,
+  flags = 0,
+  term = 0n,
+  sourceNode = 0,
+  destNode = 0,
+  payloadLength = 0,
+  checksum = 0
+}) {
+  const view = new DataView(buf.buffer, buf.byteOffset + offset, HEADER_SIZE);
+  view.setUint16(0, MAGIC, false);
+  view.setUint8(2, VERSION);
+  view.setUint8(3, messageType);
+  view.setUint16(4, flags, false);
+  view.setBigUint64(6, BigInt(term), false);
+  view.setUint16(14, sourceNode, false);
+  view.setUint16(16, destNode, false);
+  view.setUint32(18, payloadLength, false);
+  view.setUint32(22, checksum, false);
+  view.setUint16(26, 0, false); // reserved
+}
+
+/**
+ * Read common header from a buffer.
+ */
+export function readHeader(buf, offset = 0) {
+  if (buf.byteLength < offset + HEADER_SIZE) {
+    throw new Error(`Buffer too short for header: ${buf.byteLength} < ${offset + HEADER_SIZE}`);
+  }
+  const view = new DataView(buf.buffer, buf.byteOffset + offset, HEADER_SIZE);
+  const magic = view.getUint16(0, false);
+  if (magic !== MAGIC) {
+    throw new Error(`Invalid protocol magic: 0x${magic.toString(16)} (expected 0x${MAGIC.toString(16)})`);
+  }
+  const version = view.getUint8(2);
+  const messageType = view.getUint8(3);
+  const flags = view.getUint16(4, false);
+  const term = view.getBigUint64(6, false);
+  const sourceNode = view.getUint16(14, false);
+  const destNode = view.getUint16(16, false);
+  const payloadLength = view.getUint32(18, false);
+  const checksum = view.getUint32(22, false);
+
+  return {
+    magic,
+    version,
+    messageType,
+    flags,
+    term,
+    sourceNode,
+    destNode,
+    payloadLength,
+    checksum
+  };
+}
+
+/**
+ * Encodes a message into scatter/gather chunks [header, metadata, ...payloads].
+ * Does NOT copy application payloads!
+ * 
+ * @param {object} msg
+ * @param {WireMetrics} [metrics=globalWireMetrics]
+ * @returns {Uint8Array[]} Array of Uint8Array buffers ready for sendv
+ */
+export function encodeMessage(msg, metrics = globalWireMetrics) {
+  const headerBuf = new Uint8Array(HEADER_SIZE);
+  let metadataBuf = null;
+  const payloadBuffers = [];
+  let payloadBytes = 0;
+
+  switch (msg.type) {
+    case MessageType.PRE_VOTE_REQUEST:
+    case MessageType.VOTE_REQUEST: {
+      // Body: candidateId (2), lastLogIndex (8), lastLogTerm (8) = 18 bytes
+      metadataBuf = new Uint8Array(18);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 18);
+      view.setUint16(0, msg.candidateId, false);
+      view.setBigUint64(2, BigInt(msg.lastLogIndex), false);
+      view.setBigUint64(10, BigInt(msg.lastLogTerm), false);
+      break;
+    }
+
+    case MessageType.PRE_VOTE_RESPONSE:
+    case MessageType.VOTE_RESPONSE: {
+      // Body: voteGranted (1) = 1 byte
+      metadataBuf = new Uint8Array(1);
+      metadataBuf[0] = msg.voteGranted ? 1 : 0;
+      break;
+    }
+
+    case MessageType.HEARTBEAT: {
+      // Body: leaderId (2), leaderCommit (8) = 10 bytes
+      metadataBuf = new Uint8Array(10);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 10);
+      view.setUint16(0, msg.leaderId, false);
+      view.setBigUint64(2, BigInt(msg.leaderCommit), false);
+      break;
+    }
+
+    case MessageType.APPEND_REQUEST: {
+      // Body: leaderId (2), prevLogIndex (8), prevLogTerm (8), leaderCommit (8), entryCount (4) = 30 bytes
+      // For each entry: term (8), index (8), type (1), length (4), checksum (4) = 25 bytes per entry
+      const entries = msg.entries || [];
+      const entryCount = entries.length;
+      const metaLen = 30 + entryCount * 25;
+      metadataBuf = new Uint8Array(metaLen);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, metaLen);
+
+      view.setUint16(0, msg.leaderId, false);
+      view.setBigUint64(2, BigInt(msg.prevLogIndex), false);
+      view.setBigUint64(10, BigInt(msg.prevLogTerm), false);
+      view.setBigUint64(18, BigInt(msg.leaderCommit), false);
+      view.setUint32(26, entryCount, false);
+
+      let offset = 30;
+      for (let i = 0; i < entryCount; i++) {
+        const e = entries[i];
+        const payload = e.payload || new Uint8Array(0);
+        const pLen = payload.byteLength;
+        const pCrc = e.checksum !== undefined ? e.checksum : crc32(payload);
+
+        view.setBigUint64(offset, BigInt(e.term), false);
+        view.setBigUint64(offset + 8, BigInt(e.index), false);
+        view.setUint8(offset + 16, e.type || EntryType.NORMAL);
+        view.setUint32(offset + 17, pLen, false);
+        view.setUint32(offset + 21, pCrc, false);
+        offset += 25;
+
+        if (pLen > 0) {
+          payloadBuffers.push(payload);
+          payloadBytes += pLen;
+        }
+      }
+      break;
+    }
+
+    case MessageType.APPEND_RESPONSE: {
+      // Body: followerId (2), success (1), matchIndex (8), lastLogIndex (8) = 19 bytes
+      metadataBuf = new Uint8Array(19);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 19);
+      view.setUint16(0, msg.followerId, false);
+      view.setUint8(2, msg.success ? 1 : 0);
+      view.setBigUint64(3, BigInt(msg.matchIndex), false);
+      view.setBigUint64(11, BigInt(msg.lastLogIndex), false);
+      break;
+    }
+
+    case MessageType.SNAPSHOT_BEGIN: {
+      // Body: leaderId (2), snapshotIndex (8), snapshotTerm (8), totalBytes (8) = 26 bytes
+      metadataBuf = new Uint8Array(26);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 26);
+      view.setUint16(0, msg.leaderId, false);
+      view.setBigUint64(2, BigInt(msg.snapshotIndex), false);
+      view.setBigUint64(10, BigInt(msg.snapshotTerm), false);
+      view.setBigUint64(18, BigInt(msg.totalBytes), false);
+      break;
+    }
+
+    case MessageType.SNAPSHOT_CHUNK: {
+      // Body: snapshotIndex (8), offset (8), chunkLength (4) = 20 bytes
+      const chunk = msg.chunkData || new Uint8Array(0);
+      payloadBytes = chunk.byteLength;
+      metadataBuf = new Uint8Array(20);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 20);
+      view.setBigUint64(0, BigInt(msg.snapshotIndex), false);
+      view.setBigUint64(8, BigInt(msg.offset), false);
+      view.setUint32(16, payloadBytes, false);
+      if (payloadBytes > 0) {
+        payloadBuffers.push(chunk);
+      }
+      break;
+    }
+
+    case MessageType.SNAPSHOT_END: {
+      // Body: snapshotIndex (8), totalChecksum (4) = 12 bytes
+      metadataBuf = new Uint8Array(12);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 12);
+      view.setBigUint64(0, BigInt(msg.snapshotIndex), false);
+      view.setUint32(8, msg.totalChecksum >>> 0, false);
+      break;
+    }
+
+    case MessageType.SNAPSHOT_RESPONSE: {
+      // Body: followerId (2), success (1), lastIndex (8) = 11 bytes
+      metadataBuf = new Uint8Array(11);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 11);
+      view.setUint16(0, msg.followerId, false);
+      view.setUint8(2, msg.success ? 1 : 0);
+      view.setBigUint64(3, BigInt(msg.lastIndex), false);
+      break;
+    }
+
+
+    case MessageType.BATCH_DATA: {
+      const bufs = msg.buffers || [];
+      const numBuffers = bufs.length;
+      const metaLen = 26 + numBuffers * 4;
+      metadataBuf = new Uint8Array(metaLen);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, metaLen);
+      view.setUint32(0, msg.generationId || 0, false);
+      view.setBigUint64(4, BigInt(msg.firstSequence || 0), false);
+      view.setBigUint64(12, BigInt(msg.lastSequence || 0), false);
+      view.setUint32(20, msg.entryCount || 0, false);
+      view.setUint16(24, numBuffers, false);
+
+      let off = 26;
+      for (let i = 0; i < numBuffers; i++) {
+        const b = bufs[i] || new Uint8Array(0);
+        view.setUint32(off, b.byteLength, false);
+        off += 4;
+        if (b.byteLength > 0) {
+          payloadBuffers.push(b);
+          payloadBytes += b.byteLength;
+        }
+      }
+      break;
+    }
+
+    case MessageType.BATCH_ACK: {
+      metadataBuf = new Uint8Array(21);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 21);
+      view.setUint32(0, msg.generationId || 0, false);
+      view.setBigUint64(4, BigInt(msg.sequence || 0), false);
+      view.setBigUint64(12, BigInt(msg.bytesReceived || 0), false);
+      view.setUint8(20, msg.status || 0);
+      break;
+    }
+
+    case MessageType.SESSION_HANDSHAKE:
+    case MessageType.SESSION_ACK: {
+      const payload = msg.payload || new Uint8Array(0);
+      payloadBytes = payload.byteLength;
+      metadataBuf = new Uint8Array(8);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 8);
+      view.setUint32(0, msg.sessionId || 0, false);
+      view.setUint32(4, payloadBytes, false);
+      if (payloadBytes > 0) {
+        payloadBuffers.push(payload);
+      }
+      break;
+    }
+
+
+    case MessageType.TIMEOUT_NOW: {
+      // Body: leaderId (2) = 2 bytes
+      metadataBuf = new Uint8Array(2);
+      const view = new DataView(metadataBuf.buffer, metadataBuf.byteOffset, 2);
+      view.setUint16(0, msg.leaderId, false);
+      break;
+    }
+
+    default:
+      metadataBuf = new Uint8Array(0);
+      break;
+  }
+
+  const metaLength = metadataBuf ? metadataBuf.byteLength : 0;
+  const totalPayloadLength = metaLength + payloadBytes;
+
+  // Calculate checksum of body (metadata + payloads)
+  const bodyChunks = [];
+  if (metadataBuf && metadataBuf.byteLength > 0) bodyChunks.push(metadataBuf);
+  for (let i = 0; i < payloadBuffers.length; i++) bodyChunks.push(payloadBuffers[i]);
+  const checksum = crc32Buffers(bodyChunks);
+
+  writeHeader(headerBuf, 0, {
+    messageType: msg.type,
+    flags: msg.flags || 0,
+    term: msg.term || 0n,
+    sourceNode: msg.sourceNode || 0,
+    destNode: msg.destNode || 0,
+    payloadLength: totalPayloadLength,
+    checksum
+  });
+
+  const chunks = [headerBuf];
+  if (metadataBuf && metadataBuf.byteLength > 0) {
+    chunks.push(metadataBuf);
+  }
+  for (let i = 0; i < payloadBuffers.length; i++) {
+    chunks.push(payloadBuffers[i]);
+  }
+
+  // Update wire metrics
+  const totalWireBytes = HEADER_SIZE + totalPayloadLength;
+  metrics.wireBytes += totalWireBytes;
+  metrics.payloadBytes += payloadBytes;
+  metrics.protocolOverheadBytes += (HEADER_SIZE + metaLength);
+  metrics.allocatedBytes += (HEADER_SIZE + metaLength); // Only headers/metadata allocated, payloads untouched!
+  // copiedBytes += 0 (Zero payload copies!)
+
+  return chunks;
+}
+
+/**
+ * Decodes a contiguous buffer or slice containing header + payload.
+ * Slices are returned as zero-copy views on the input buffer.
+ * 
+ * @param {Uint8Array} buf
+ * @param {number} [offset=0]
+ * @param {boolean} [verifyChecksum=true]
+ * @returns {object} Decoded message object
+ */
+export function decodeMessage(buf, offset = 0, verifyChecksum = true) {
+  const header = readHeader(buf, offset);
+  const bodyOffset = offset + HEADER_SIZE;
+  const bodyLength = header.payloadLength;
+
+  if (buf.byteLength < bodyOffset + bodyLength) {
+    throw new Error(`Incomplete message payload: have ${buf.byteLength - bodyOffset} bytes, needed ${bodyLength}`);
+  }
+
+  if (verifyChecksum && bodyLength > 0) {
+    const computedCrc = crc32(buf, bodyOffset, bodyLength);
+    if (computedCrc !== header.checksum) {
+      throw new Error(`Checksum mismatch: computed 0x${computedCrc.toString(16)}, header 0x${header.checksum.toString(16)}`);
+    }
+  }
+
+  const msg = {
+    type: header.messageType,
+    flags: header.flags,
+    term: header.term,
+    sourceNode: header.sourceNode,
+    destNode: header.destNode
+  };
+
+  const view = new DataView(buf.buffer, buf.byteOffset + bodyOffset, bodyLength);
+
+  switch (header.messageType) {
+    case MessageType.PRE_VOTE_REQUEST:
+    case MessageType.VOTE_REQUEST: {
+      msg.candidateId = view.getUint16(0, false);
+      msg.lastLogIndex = view.getBigUint64(2, false);
+      msg.lastLogTerm = view.getBigUint64(10, false);
+      break;
+    }
+
+    case MessageType.PRE_VOTE_RESPONSE:
+    case MessageType.VOTE_RESPONSE: {
+      msg.voteGranted = view.getUint8(0) === 1;
+      break;
+    }
+
+    case MessageType.HEARTBEAT: {
+      msg.leaderId = view.getUint16(0, false);
+      msg.leaderCommit = view.getBigUint64(2, false);
+      break;
+    }
+
+    case MessageType.APPEND_REQUEST: {
+      msg.leaderId = view.getUint16(0, false);
+      msg.prevLogIndex = view.getBigUint64(2, false);
+      msg.prevLogTerm = view.getBigUint64(10, false);
+      msg.leaderCommit = view.getBigUint64(18, false);
+      const entryCount = view.getUint32(26, false);
+      const entries = new Array(entryCount);
+
+      let curMetaOffset = 30;
+      // Calculate total metadata size for all entries
+      const totalMetaSize = 30 + entryCount * 25;
+      let curPayloadOffset = bodyOffset + totalMetaSize;
+
+      for (let i = 0; i < entryCount; i++) {
+        const term = view.getBigUint64(curMetaOffset, false);
+        const index = view.getBigUint64(curMetaOffset + 8, false);
+        const type = view.getUint8(curMetaOffset + 16);
+        const pLen = view.getUint32(curMetaOffset + 17, false);
+        const pCrc = view.getUint32(curMetaOffset + 21, false);
+        curMetaOffset += 25;
+
+        // Zero-copy view into underlying buffer
+        let payload = null;
+        if (pLen > 0) {
+          payload = buf.subarray(curPayloadOffset, curPayloadOffset + pLen);
+          curPayloadOffset += pLen;
+        } else {
+          payload = new Uint8Array(0);
+        }
+
+        entries[i] = {
+          term,
+          index,
+          type,
+          checksum: pCrc,
+          payload
+        };
+      }
+      msg.entries = entries;
+      break;
+    }
+
+    case MessageType.APPEND_RESPONSE: {
+      msg.followerId = view.getUint16(0, false);
+      msg.success = view.getUint8(2) === 1;
+      msg.matchIndex = view.getBigUint64(3, false);
+      msg.lastLogIndex = view.getBigUint64(11, false);
+      break;
+    }
+
+    case MessageType.SNAPSHOT_BEGIN: {
+      msg.leaderId = view.getUint16(0, false);
+      msg.snapshotIndex = view.getBigUint64(2, false);
+      msg.snapshotTerm = view.getBigUint64(10, false);
+      msg.totalBytes = view.getBigUint64(18, false);
+      break;
+    }
+
+    case MessageType.SNAPSHOT_CHUNK: {
+      msg.snapshotIndex = view.getBigUint64(0, false);
+      msg.offset = view.getBigUint64(8, false);
+      const chunkLen = view.getUint32(16, false);
+      msg.chunkData = buf.subarray(bodyOffset + 20, bodyOffset + 20 + chunkLen);
+      break;
+    }
+
+    case MessageType.SNAPSHOT_END: {
+      msg.snapshotIndex = view.getBigUint64(0, false);
+      msg.totalChecksum = view.getUint32(8, false);
+      break;
+    }
+
+    case MessageType.SNAPSHOT_RESPONSE: {
+      msg.followerId = view.getUint16(0, false);
+      msg.success = view.getUint8(2) === 1;
+      msg.lastIndex = view.getBigUint64(3, false);
+      break;
+    }
+
+    case MessageType.TIMEOUT_NOW: {
+      msg.leaderId = view.getUint16(0, false);
+      break;
+    }
+
+    case MessageType.BATCH_DATA: {
+      msg.generationId = view.getUint32(0, false);
+      msg.firstSequence = view.getBigUint64(4, false);
+      msg.lastSequence = view.getBigUint64(12, false);
+      msg.entryCount = view.getUint32(20, false);
+      const numBuffers = view.getUint16(24, false);
+
+      const bufferLens = new Array(numBuffers);
+      let off = 26;
+      for (let i = 0; i < numBuffers; i++) {
+        bufferLens[i] = view.getUint32(off, false);
+        off += 4;
+      }
+
+      const totalMetaLen = 26 + numBuffers * 4;
+      let payloadOff = bodyOffset + totalMetaLen;
+      const bufs = new Array(numBuffers);
+      for (let i = 0; i < numBuffers; i++) {
+        const len = bufferLens[i];
+        bufs[i] = buf.subarray(payloadOff, payloadOff + len);
+        payloadOff += len;
+      }
+      msg.buffers = bufs;
+      break;
+    }
+
+    case MessageType.BATCH_ACK: {
+      msg.generationId = view.getUint32(0, false);
+      msg.sequence = view.getBigUint64(4, false);
+      msg.bytesReceived = view.getBigUint64(12, false);
+      msg.status = view.getUint8(20);
+      break;
+    }
+
+    case MessageType.SESSION_HANDSHAKE:
+    case MessageType.SESSION_ACK: {
+      msg.sessionId = view.getUint32(0, false);
+      const dataLen = view.getUint32(4, false);
+      msg.payload = buf.subarray(bodyOffset + 8, bodyOffset + 8 + dataLen);
+      break;
+    }
+
+  }
+
+  return msg;
+}
+
+/**
+ * Flattens scatter/gather chunks into a single contiguous Uint8Array if needed
+ * (e.g. for streams that cannot accept scatter/gather buffers).
+ * 
+ * @param {Uint8Array[]} chunks
+ * @param {WireMetrics} [metrics=globalWireMetrics]
+ * @returns {Uint8Array}
+ */
+export function flattenChunks(chunks, metrics = globalWireMetrics) {
+  let totalLength = 0;
+  for (let i = 0; i < chunks.length; i++) {
+    totalLength += chunks[i].byteLength;
+  }
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (let i = 0; i < chunks.length; i++) {
+    const chunk = chunks[i];
+    result.set(chunk, offset);
+    offset += chunk.byteLength;
+    metrics.copiedBytes += chunk.byteLength;
+  }
+  metrics.allocatedBytes += totalLength;
+  return result;
+}