raptiye 0.1.0

package/src/storage/file-log.js

@@ -0,0 +1,394 @@
+/**
+ * Durable, crash-safe Write-Ahead Log (WAL) storage for Raptiye.
+ * Uses append-only log format with CRC32 integrity checks and atomic state barriers.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { LogStorage } from './interface.js';
+import { crc32 } from '../protocol/checksum.js';
+
+const WAL_MAGIC = 0x574C; // 'WL'
+const WAL_RECORD_HEADER_SIZE = 27; // 2 magic + 4 crc + 8 term + 8 index + 1 type + 4 payloadLen
+
+export class FileLog extends LogStorage {
+  constructor(dataDir) {
+    super();
+    this.dataDir = dataDir;
+    this.walPath = path.join(dataDir, 'wal.log');
+    this.statePath = path.join(dataDir, 'hard_state.bin');
+    this.snapPath = path.join(dataDir, 'snapshot.bin');
+
+    this._fd = null;
+    this._entries = [];
+    this._snapshot = null;
+    this._snapshotIndex = 0n;
+    this._snapshotTerm = 0n;
+
+    this._hardState = {
+      currentTerm: 0n,
+      votedFor: null
+    };
+
+    this._init();
+  }
+
+  _init() {
+    if (!fs.existsSync(this.dataDir)) {
+      fs.mkdirSync(this.dataDir, { recursive: true });
+    }
+
+    this._loadHardState();
+    this._loadSnapshot();
+    this._openAndRecoverWal();
+  }
+
+  _loadHardState() {
+    if (fs.existsSync(this.statePath)) {
+      const buf = fs.readFileSync(this.statePath);
+      if (buf.length >= 10) {
+        const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+        this._hardState.currentTerm = view.getBigUint64(0, false);
+        const votedForRaw = view.getUint16(8, false);
+        this._hardState.votedFor = votedForRaw === 0xFFFF ? null : votedForRaw;
+      }
+    }
+  }
+
+  _saveHardState() {
+    const buf = new Uint8Array(10);
+    const view = new DataView(buf.buffer, buf.byteOffset, 10);
+    view.setBigUint64(0, this._hardState.currentTerm, false);
+    view.setUint16(8, this._hardState.votedFor === null ? 0xFFFF : this._hardState.votedFor, false);
+
+    const tmpPath = `${this.statePath}.tmp`;
+    const tmpFd = fs.openSync(tmpPath, 'w');
+    fs.writeSync(tmpFd, buf);
+    fs.fsyncSync(tmpFd);
+    fs.closeSync(tmpFd);
+
+    fs.renameSync(tmpPath, this.statePath);
+  }
+
+  _loadSnapshot() {
+    if (fs.existsSync(this.snapPath)) {
+      const buf = fs.readFileSync(this.snapPath);
+      if (buf.length >= 16) {
+        const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+        const index = view.getBigUint64(0, false);
+        const term = view.getBigUint64(8, false);
+        const data = buf.subarray(16);
+        this._snapshot = { index, term, data };
+        this._snapshotIndex = index;
+        this._snapshotTerm = term;
+      }
+    }
+  }
+
+  _openAndRecoverWal() {
+    const exists = fs.existsSync(this.walPath);
+    this._fd = fs.openSync(this.walPath, exists ? 'r+' : 'w+');
+
+    if (!exists) return;
+
+    const stats = fs.fstatSync(this._fd);
+    const fileSize = stats.size;
+    if (fileSize === 0) return;
+
+    const fileBuf = fs.readFileSync(this.walPath);
+    let offset = 0;
+    let validEnd = 0;
+
+    while (offset + WAL_RECORD_HEADER_SIZE <= fileSize) {
+      const view = new DataView(fileBuf.buffer, fileBuf.byteOffset + offset, fileBuf.byteLength - offset);
+      const magic = view.getUint16(0, false);
+      if (magic !== WAL_MAGIC) break;
+
+      const recordCrc = view.getUint32(2, false);
+      const term = view.getBigUint64(6, false);
+      const index = view.getBigUint64(14, false);
+      const type = view.getUint8(22);
+      const payloadLen = view.getUint32(23, false);
+
+      if (offset + WAL_RECORD_HEADER_SIZE + payloadLen > fileSize) {
+        // Partial record at EOF
+        break;
+      }
+
+      const payloadOffset = offset + WAL_RECORD_HEADER_SIZE;
+      // Verify CRC over [term, index, type, payloadLen, payload]
+      const checkedSlice = fileBuf.subarray(offset + 6, payloadOffset + payloadLen);
+      const computedCrc = crc32(checkedSlice);
+
+      if (computedCrc !== recordCrc) {
+        // Corrupt record, stop recovery here
+        break;
+      }
+
+      const payload = new Uint8Array(payloadLen);
+      payload.set(fileBuf.subarray(payloadOffset, payloadOffset + payloadLen));
+
+      this._entries.push({
+        term,
+        index,
+        type,
+        payload,
+        checksum: recordCrc,
+        fileOffset: offset,
+        totalRecordSize: WAL_RECORD_HEADER_SIZE + payloadLen
+      });
+
+      offset += WAL_RECORD_HEADER_SIZE + payloadLen;
+      validEnd = offset;
+    }
+
+    if (validEnd < fileSize) {
+      // Truncate partial or corrupt tail
+      fs.ftruncateSync(this._fd, validEnd);
+    }
+  }
+
+  append(entry) {
+    const term = BigInt(entry.term);
+    const index = BigInt(entry.index);
+    const payload = entry.payload || new Uint8Array(0);
+    const payloadLen = payload.byteLength;
+
+    const recordSize = WAL_RECORD_HEADER_SIZE + payloadLen;
+    const buf = new Uint8Array(recordSize);
+    const view = new DataView(buf.buffer, buf.byteOffset, recordSize);
+
+    view.setUint16(0, WAL_MAGIC, false);
+    view.setBigUint64(6, term, false);
+    view.setBigUint64(14, index, false);
+    view.setUint8(22, entry.type || 1);
+    view.setUint32(23, payloadLen, false);
+
+    if (payloadLen > 0) {
+      buf.set(payload, WAL_RECORD_HEADER_SIZE);
+    }
+
+    const checkedSlice = buf.subarray(6, recordSize);
+    const recordCrc = crc32(checkedSlice);
+    view.setUint32(2, recordCrc, false);
+
+    // Synchronously write to WAL file
+    const stats = fs.fstatSync(this._fd);
+    const fileOffset = stats.size;
+    fs.writeSync(this._fd, buf, 0, recordSize, fileOffset);
+
+    this._entries.push({
+      term,
+      index,
+      type: entry.type || 1,
+      payload,
+      checksum: recordCrc,
+      fileOffset,
+      totalRecordSize: recordSize
+    });
+  }
+
+  appendMany(entries) {
+    for (let i = 0; i < entries.length; i++) {
+      this.append(entries[i]);
+    }
+  }
+
+  sync() {
+    if (this._fd !== null) {
+      fs.fsyncSync(this._fd);
+    }
+  }
+
+  entries(fromIndex, toIndex, maxBytes = Infinity) {
+    const from = BigInt(fromIndex);
+    const to = BigInt(toIndex);
+    const first = this.firstIndex();
+    const last = this.lastIndex();
+
+    if (from > to || from > last || to < first) {
+      return [];
+    }
+
+    const startIdx = from < first ? 0 : Number(from - first);
+    const endIdx = to > last ? this._entries.length - 1 : Number(to - first);
+
+    const result = [];
+    let accBytes = 0;
+
+    for (let i = startIdx; i <= endIdx; i++) {
+      const e = this._entries[i];
+      const entryBytes = e.payload.byteLength + 25;
+      if (result.length > 0 && accBytes + entryBytes > maxBytes) {
+        break;
+      }
+      result.push(e);
+      accBytes += entryBytes;
+    }
+
+    return result;
+  }
+
+  term(index) {
+    const idx = BigInt(index);
+    if (idx === 0n) return 0n;
+    if (idx === this._snapshotIndex) return this._snapshotTerm;
+    if (idx < this.firstIndex()) return null;
+    if (idx > this.lastIndex()) return null;
+
+    const offset = Number(idx - this.firstIndex());
+    return this._entries[offset].term;
+  }
+
+  lastIndex() {
+    if (this._entries.length === 0) {
+      return this._snapshotIndex;
+    }
+    return this._entries[this._entries.length - 1].index;
+  }
+
+  lastTerm() {
+    if (this._entries.length === 0) {
+      return this._snapshotTerm;
+    }
+    return this._entries[this._entries.length - 1].term;
+  }
+
+  firstIndex() {
+    return this._snapshotIndex + 1n;
+  }
+
+  truncateFrom(index) {
+    const idx = BigInt(index);
+    const first = this.firstIndex();
+
+    if (idx < first) {
+      throw new Error(`Cannot truncate log prior to first retained index: ${idx} < ${first}`);
+    }
+    if (idx > this.lastIndex()) {
+      return;
+    }
+
+    const keepCount = Number(idx - first);
+    let truncateOffset = 0;
+    if (keepCount < this._entries.length) {
+      truncateOffset = this._entries[keepCount].fileOffset;
+      this._entries.length = keepCount;
+      fs.ftruncateSync(this._fd, truncateOffset);
+    }
+  }
+
+  compactThrough(index) {
+    const idx = BigInt(index);
+    if (idx <= this._snapshotIndex) {
+      return;
+    }
+
+    const last = this.lastIndex();
+    if (idx > last) {
+      this._snapshotIndex = idx;
+      this._entries.length = 0;
+      fs.ftruncateSync(this._fd, 0);
+      return;
+    }
+
+    const discardCount = Number(idx - this._snapshotIndex);
+    const lastDiscarded = this._entries[discardCount - 1];
+    this._snapshotTerm = lastDiscarded.term;
+    this._snapshotIndex = idx;
+    this._entries.splice(0, discardCount);
+
+    // Rewrite WAL with remaining entries
+    this._rewriteWal();
+  }
+
+  _rewriteWal() {
+    const tmpWal = `${this.walPath}.tmp`;
+    const tmpFd = fs.openSync(tmpWal, 'w');
+
+    for (let i = 0; i < this._entries.length; i++) {
+      const e = this._entries[i];
+      const payloadLen = e.payload.byteLength;
+      const recordSize = WAL_RECORD_HEADER_SIZE + payloadLen;
+      const buf = new Uint8Array(recordSize);
+      const view = new DataView(buf.buffer, buf.byteOffset, recordSize);
+
+      view.setUint16(0, WAL_MAGIC, false);
+      view.setBigUint64(6, e.term, false);
+      view.setBigUint64(14, e.index, false);
+      view.setUint8(22, e.type || 1);
+      view.setUint32(23, payloadLen, false);
+      if (payloadLen > 0) {
+        buf.set(e.payload, WAL_RECORD_HEADER_SIZE);
+      }
+      const checkedSlice = buf.subarray(6, recordSize);
+      const recordCrc = crc32(checkedSlice);
+      view.setUint32(2, recordCrc, false);
+
+      fs.writeSync(tmpFd, buf);
+    }
+
+    fs.fsyncSync(tmpFd);
+    fs.closeSync(tmpFd);
+    fs.closeSync(this._fd);
+    fs.renameSync(tmpWal, this.walPath);
+    this._fd = fs.openSync(this.walPath, 'r+');
+
+    // Update in-memory offsets
+    let offset = 0;
+    for (let i = 0; i < this._entries.length; i++) {
+      this._entries[i].fileOffset = offset;
+      offset += this._entries[i].totalRecordSize;
+    }
+  }
+
+  getHardState() {
+    return {
+      currentTerm: this._hardState.currentTerm,
+      votedFor: this._hardState.votedFor
+    };
+  }
+
+  setHardState({ currentTerm, votedFor }) {
+    this._hardState.currentTerm = BigInt(currentTerm);
+    this._hardState.votedFor = votedFor !== null && votedFor !== undefined ? Number(votedFor) : null;
+    this._saveHardState();
+  }
+
+  getSnapshot() {
+    return this._snapshot;
+  }
+
+  saveSnapshot(snapshot) {
+    const index = BigInt(snapshot.index);
+    const term = BigInt(snapshot.term);
+    const data = snapshot.data || new Uint8Array(0);
+
+    const buf = new Uint8Array(16 + data.byteLength);
+    const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+    view.setBigUint64(0, index, false);
+    view.setBigUint64(8, term, false);
+    if (data.byteLength > 0) {
+      buf.set(data, 16);
+    }
+
+    const tmpSnap = `${this.snapPath}.tmp`;
+    const tmpFd = fs.openSync(tmpSnap, 'w');
+    fs.writeSync(tmpFd, buf);
+    fs.fsyncSync(tmpFd);
+    fs.closeSync(tmpFd);
+    fs.renameSync(tmpSnap, this.snapPath);
+
+    this._snapshot = { index, term, data };
+    this.compactThrough(index);
+    this._snapshotIndex = index;
+    this._snapshotTerm = term;
+  }
+
+  close() {
+    if (this._fd !== null) {
+      fs.closeSync(this._fd);
+      this._fd = null;
+    }
+  }
+}

package/src/storage/interface.js

@@ -0,0 +1,113 @@
+/**
+ * Abstract storage interface for Raptiye replicated log and hard state.
+ */
+
+export class LogStorage {
+  /**
+   * Append a single entry.
+   * @param {{ term: bigint, index: bigint, type: number, payload: Uint8Array, checksum?: number }} entry
+   */
+  append(entry) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Append multiple entries.
+   * @param {Array<{ term: bigint, index: bigint, type: number, payload: Uint8Array, checksum?: number }>} entries
+   */
+  appendMany(entries) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Retrieve entries in [fromIndex, toIndex] range, optionally bounded by maxBytes.
+   * @param {bigint} fromIndex
+   * @param {bigint} toIndex
+   * @param {number} [maxBytes=Infinity]
+   * @returns {Array<{ term: bigint, index: bigint, type: number, payload: Uint8Array, checksum?: number }>}
+   */
+  entries(fromIndex, toIndex, maxBytes = Infinity) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Get the term for a specific log index.
+   * @param {bigint} index
+   * @returns {bigint|null}
+   */
+  term(index) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * @returns {bigint} Last log index (0n if empty)
+   */
+  lastIndex() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * @returns {bigint} Last log term (0n if empty)
+   */
+  lastTerm() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * @returns {bigint} First retained log index (1n initially, or snapshotIndex + 1n)
+   */
+  firstIndex() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Truncate log starting from index (deletes all entries >= index).
+   * @param {bigint} index
+   */
+  truncateFrom(index) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Compact log up through index (discards entries <= index).
+   * @param {bigint} index
+   */
+  compactThrough(index) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * @returns {{ currentTerm: bigint, votedFor: number|null }}
+   */
+  getHardState() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Persist hard state.
+   * @param {{ currentTerm: bigint, votedFor: number|null }} state
+   */
+  setHardState(state) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * @returns {{ index: bigint, term: bigint, data: Uint8Array }|null}
+   */
+  getSnapshot() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Save snapshot and record metadata.
+   * @param {{ index: bigint, term: bigint, data: Uint8Array }} snapshot
+   */
+  saveSnapshot(snapshot) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Close storage.
+   */
+  close() {}
+}

package/src/storage/memory-log.js

@@ -0,0 +1,183 @@
+/**
+ * High-performance, byte-first in-memory replicated log storage for Raptiye.
+ */
+
+import { LogStorage } from './interface.js';
+import { crc32 } from '../protocol/checksum.js';
+
+export class MemoryLog extends LogStorage {
+  constructor() {
+    super();
+    this._entries = [];
+    this._snapshot = null;
+    this._snapshotIndex = 0n;
+    this._snapshotTerm = 0n;
+    this._hardState = {
+      currentTerm: 0n,
+      votedFor: null
+    };
+    this._byteSize = 0;
+  }
+
+  append(entry) {
+    const term = BigInt(entry.term);
+    const index = BigInt(entry.index);
+    const payload = entry.payload || new Uint8Array(0);
+    const checksum = entry.checksum !== undefined ? entry.checksum : crc32(payload);
+
+    const stored = {
+      term,
+      index,
+      type: entry.type || 1,
+      payload,
+      checksum
+    };
+
+    this._entries.push(stored);
+    this._byteSize += payload.byteLength + 25; // 25 bytes metadata
+  }
+
+  appendMany(entries) {
+    for (let i = 0; i < entries.length; i++) {
+      this.append(entries[i]);
+    }
+  }
+
+  entries(fromIndex, toIndex, maxBytes = Infinity) {
+    const from = BigInt(fromIndex);
+    const to = BigInt(toIndex);
+    const first = this.firstIndex();
+    const last = this.lastIndex();
+
+    if (from > to || from > last || to < first) {
+      return [];
+    }
+
+    const startIdx = from < first ? 0 : Number(from - first);
+    const endIdx = to > last ? this._entries.length - 1 : Number(to - first);
+
+    const result = [];
+    let accBytes = 0;
+
+    for (let i = startIdx; i <= endIdx; i++) {
+      const e = this._entries[i];
+      const entryBytes = e.payload.byteLength + 25;
+      if (result.length > 0 && accBytes + entryBytes > maxBytes) {
+        break;
+      }
+      result.push(e);
+      accBytes += entryBytes;
+    }
+
+    return result;
+  }
+
+  term(index) {
+    const idx = BigInt(index);
+    if (idx === 0n) return 0n;
+    if (idx === this._snapshotIndex) return this._snapshotTerm;
+    if (idx < this.firstIndex()) return null; // Compacted away
+    if (idx > this.lastIndex()) return null;
+
+    const offset = Number(idx - this.firstIndex());
+    return this._entries[offset].term;
+  }
+
+  lastIndex() {
+    if (this._entries.length === 0) {
+      return this._snapshotIndex;
+    }
+    return this._entries[this._entries.length - 1].index;
+  }
+
+  lastTerm() {
+    if (this._entries.length === 0) {
+      return this._snapshotTerm;
+    }
+    return this._entries[this._entries.length - 1].term;
+  }
+
+  firstIndex() {
+    return this._snapshotIndex + 1n;
+  }
+
+  truncateFrom(index) {
+    const idx = BigInt(index);
+    const first = this.firstIndex();
+
+    if (idx < first) {
+      throw new Error(`Cannot truncate log prior to first retained index: ${idx} < ${first}`);
+    }
+    if (idx > this.lastIndex()) {
+      return;
+    }
+
+    const keepCount = Number(idx - first);
+    for (let i = keepCount; i < this._entries.length; i++) {
+      this._byteSize -= (this._entries[i].payload.byteLength + 25);
+    }
+    this._entries.length = keepCount;
+  }
+
+  compactThrough(index) {
+    const idx = BigInt(index);
+    if (idx <= this._snapshotIndex) {
+      return;
+    }
+
+    const last = this.lastIndex();
+    if (idx > last) {
+      // Discard entire log
+      this._snapshotIndex = idx;
+      this._entries.length = 0;
+      this._byteSize = 0;
+      return;
+    }
+
+    const discardCount = Number(idx - this._snapshotIndex);
+    for (let i = 0; i < discardCount; i++) {
+      this._byteSize -= (this._entries[i].payload.byteLength + 25);
+    }
+    const lastDiscarded = this._entries[discardCount - 1];
+    this._snapshotTerm = lastDiscarded.term;
+    this._snapshotIndex = idx;
+    this._entries.splice(0, discardCount);
+  }
+
+  getHardState() {
+    return {
+      currentTerm: this._hardState.currentTerm,
+      votedFor: this._hardState.votedFor
+    };
+  }
+
+  setHardState({ currentTerm, votedFor }) {
+    this._hardState.currentTerm = BigInt(currentTerm);
+    this._hardState.votedFor = votedFor !== null && votedFor !== undefined ? Number(votedFor) : null;
+  }
+
+  getSnapshot() {
+    return this._snapshot;
+  }
+
+  saveSnapshot(snapshot) {
+    const index = BigInt(snapshot.index);
+    const term = BigInt(snapshot.term);
+    this._snapshot = {
+      index,
+      term,
+      data: snapshot.data
+    };
+    this.compactThrough(index);
+    this._snapshotIndex = index;
+    this._snapshotTerm = term;
+  }
+
+  totalBytes() {
+    return this._byteSize;
+  }
+
+  entryCount() {
+    return this._entries.length;
+  }
+}