procmesh-js 0.1.0

package/src/codec.js

@@ -0,0 +1,54 @@
+'use strict';
+
+/**
+ * A codec serializes/deserializes message objects to/from Buffers.
+ * Shape: { encode(obj) -> Buffer, decode(Buffer) -> obj }.
+ *
+ * The default JSON codec is zero-dependency and debuggable. `msgpackr` is
+ * offered as an optional faster binary codec (graceful fallback if not installed).
+ */
+
+const jsonCodec = {
+  name: 'json',
+  encode(obj) {
+    return Buffer.from(JSON.stringify(obj));
+  },
+  decode(buf) {
+    return JSON.parse(buf.toString('utf8'));
+  },
+};
+
+let msgpackCodec = null;
+function loadMsgpack() {
+  if (msgpackCodec) return msgpackCodec;
+  let mod;
+  try {
+    mod = require('msgpackr');
+  } catch (err) {
+    throw new Error(
+      'codec "msgpack" requires the optional dependency "msgpackr". Run `npm install msgpackr`, or use the default "json" codec.'
+    );
+  }
+  const packer = new mod.Packr({ structuredClone: true });
+  msgpackCodec = {
+    name: 'msgpack',
+    encode: (obj) => packer.pack(obj),
+    decode: (buf) => packer.unpack(buf),
+  };
+  return msgpackCodec;
+}
+
+/**
+ * Normalize a codec option into a concrete codec object.
+ * Accepts: undefined | 'json' | 'msgpack' | a custom { encode, decode } object.
+ */
+function resolveCodec(codec) {
+  if (!codec || codec === 'json') return jsonCodec;
+  if (codec === 'msgpack' || codec === 'msgpackr') return loadMsgpack();
+  if (typeof codec === 'object' && typeof codec.encode === 'function' && typeof codec.decode === 'function') {
+    return codec;
+  }
+  throw new Error(`invalid codec: ${String(codec)} (expected "json", "msgpack", or a { encode, decode } object)`);
+}
+
+module.exports = { jsonCodec, resolveCodec };

package/src/errors.js

@@ -0,0 +1,43 @@
+'use strict';
+
+class ProcMeshError extends Error {
+  constructor(message, code) {
+    super(message);
+    this.name = this.constructor.name;
+    this.code = code;
+  }
+}
+
+class Disconnected extends ProcMeshError {
+  constructor(message = 'not connected to broker') {
+    super(message, 'EDISCONNECTED');
+  }
+}
+
+class CallTimeout extends ProcMeshError {
+  constructor(message = 'request timed out') {
+    super(message, 'ETIMEOUT');
+  }
+}
+
+class LockTimeout extends ProcMeshError {
+  constructor(message = 'could not acquire lock') {
+    super(message, 'ELOCKTIMEOUT');
+  }
+}
+
+/** A fenced write was rejected: the caller's lock was superseded (stale fencing token). */
+class Fenced extends ProcMeshError {
+  constructor(message = 'fenced: stale lock token') {
+    super(message, 'EFENCED');
+  }
+}
+
+/** An error raised on the broker or a remote RPC handler, relayed to the caller. */
+class RemoteError extends ProcMeshError {
+  constructor(message, code) {
+    super(message, code || 'EREMOTE');
+  }
+}
+
+module.exports = { ProcMeshError, Disconnected, CallTimeout, LockTimeout, Fenced, RemoteError };

package/src/hashring.js

@@ -0,0 +1,27 @@
+'use strict';
+
+/**
+ * Keyspace hashing for sharded deployments. Every process that joins a sharded mesh
+ * must hash each key the same way, so that cache/atomic/locks for a given key always
+ * land on the single broker that owns it (preserving correctness). This is the one
+ * shared implementation used by both the library (src/sharded-client.js) and the
+ * benchmark harness (bench/shard.js re-exports it) — they can never drift apart.
+ */
+
+/** FNV-1a 32-bit hash of a string — fast, dependency-free, good enough for sharding. */
+function fnv1a(str) {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return h >>> 0;
+}
+
+/** Pick the shard index for a key given the number of shards. */
+function shardIndex(key, n) {
+  if (n <= 1) return 0;
+  return fnv1a(String(key)) % n;
+}
+
+module.exports = { fnv1a, shardIndex };

package/src/index.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const Client = require('./client');
+const ShardedClient = require('./sharded-client');
+const Broker = require('./broker');
+const errors = require('./errors');
+const { resolveAddress } = require('./transport');
+
+/** True when opts asks for more than one shard (a count > 1 or a multi-element array). */
+function isSharded(opts) {
+  const s = opts.shards;
+  return (typeof s === 'number' && s > 1) || (Array.isArray(s) && s.length > 1);
+}
+
+/**
+ * Connect to (and, by default, auto-spawn) the shared broker.
+ * Returns a connected Client — or, when `opts.shards` requests more than one shard, a
+ * ShardedClient that spreads work across N broker processes behind the identical API.
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.name='default']   logical mesh name (maps to a socket)
+ * @param {string} [opts.address]          explicit socket path / pipe name
+ * @param {string|object} [opts.codec]     'json' (default) | 'msgpack' | custom
+ * @param {boolean} [opts.autoSpawn=true]  spawn a broker if none is running
+ * @param {boolean} [opts.reconnect=true]  auto-reconnect on connection loss
+ * @param {object} [opts.cache]            broker cache config { max, ttl, maxSize }
+ * @param {number|Array<string|object>} [opts.shards]  shard across N brokers: a count N
+ *        (auto-spawns brokers named `${name}#0..#N-1`) or an array of names/{name,address} specs
+ */
+async function createClient(opts = {}) {
+  const client = isSharded(opts) ? new ShardedClient(opts) : new Client(opts);
+  await client.connect();
+  return client;
+}
+
+/** Create (but do not start) a Broker. Call `.start()` to begin listening. */
+function createBroker(opts = {}) {
+  return new Broker(opts);
+}
+
+module.exports = {
+  createClient,
+  createBroker,
+  Client,
+  ShardedClient,
+  Broker,
+  resolveAddress,
+  errors,
+};

package/src/locks.js

@@ -0,0 +1,129 @@
+'use strict';
+
+/**
+ * Cross-process mutex manager, held in the broker. Each lock has at most one
+ * owner; contenders queue FIFO. A lock auto-releases after its TTL (so a crashed
+ * owner can't deadlock the system) and all of a connection's locks are released
+ * when it disconnects.
+ *
+ * Connections are identified by an opaque `connId`. acquire() resolves with a
+ * boolean: true if the lock was granted, false if it timed out / was unavailable.
+ */
+class LockManager {
+  /**
+   * @param {object} [opts]
+   * @param {function} [opts.mintToken]  returns the next monotonic fencing token; the broker
+   *   injects one backed by a (persisted) global counter. Defaults to a private counter so the
+   *   manager is usable standalone in tests.
+   */
+  constructor({ mintToken } = {}) {
+    this.locks = new Map(); // key -> { owner, ttl, timer, token, waiters: [{ connId, ttl, resolve, timer }] }
+    this.fenceHigh = new Map(); // lockKey -> highest fencing token ever issued (never deleted)
+    let n = 0;
+    this.mintToken = mintToken || (() => (n += 1));
+  }
+
+  acquire(key, connId, { ttl = 30000, wait = 0 } = {}) {
+    return new Promise((resolve) => {
+      let lock = this.locks.get(key);
+      if (!lock) {
+        lock = { owner: null, ttl, timer: null, waiters: [] };
+        this.locks.set(key, lock);
+      }
+      if (lock.owner === null) {
+        this._grant(key, lock, connId, ttl);
+        return resolve({ acquired: true, token: lock.token });
+      }
+      if (wait <= 0) {
+        this._gcIfEmpty(key, lock);
+        return resolve({ acquired: false, token: null });
+      }
+      const waiter = { connId, ttl, resolve, timer: null };
+      waiter.timer = setTimeout(() => {
+        lock.waiters = lock.waiters.filter((w) => w !== waiter);
+        this._gcIfEmpty(key, lock);
+        resolve({ acquired: false, token: null });
+      }, wait);
+      if (waiter.timer.unref) waiter.timer.unref();
+      lock.waiters.push(waiter);
+    });
+  }
+
+  release(key, connId) {
+    const lock = this.locks.get(key);
+    if (!lock || lock.owner !== connId) return false;
+    this._release(key);
+    return true;
+  }
+
+  /** Release everything held or awaited by a (disconnected) connection. */
+  releaseAll(connId) {
+    for (const [key, lock] of this.locks) {
+      lock.waiters = lock.waiters.filter((w) => {
+        if (w.connId === connId) {
+          clearTimeout(w.timer);
+          return false;
+        }
+        return true;
+      });
+      if (lock.owner === connId) this._release(key);
+      else this._gcIfEmpty(key, lock);
+    }
+  }
+
+  _grant(key, lock, connId, ttl) {
+    lock.owner = connId;
+    lock.ttl = ttl;
+    // Each grant gets a fresh, strictly-larger token (its epoch). Raising the fence bar
+    // here means the instant a lock is (re-)granted — including a TTL-expiry hand-off to a
+    // waiter — any prior holder's token is already below the bar and will be fenced off.
+    lock.token = this.mintToken();
+    this.bumpFence(key, lock.token);
+    clearTimeout(lock.timer);
+    lock.timer = setTimeout(() => this._release(key), ttl);
+    if (lock.timer.unref) lock.timer.unref();
+  }
+
+  _release(key) {
+    const lock = this.locks.get(key);
+    if (!lock) return;
+    clearTimeout(lock.timer);
+    lock.owner = null;
+    lock.timer = null;
+    const next = lock.waiters.shift();
+    if (next) {
+      clearTimeout(next.timer);
+      this._grant(key, lock, next.connId, next.ttl);
+      next.resolve({ acquired: true, token: lock.token });
+    } else {
+      this._gcIfEmpty(key, lock);
+    }
+  }
+
+  _gcIfEmpty(key, lock) {
+    if (lock.owner === null && lock.waiters.length === 0) {
+      clearTimeout(lock.timer);
+      this.locks.delete(key);
+    }
+  }
+
+  /** Raise the per-key fence bar. Never lowered, never deleted (a freed key may be re-locked). */
+  bumpFence(key, token) {
+    const cur = this.fenceHigh.get(key) || 0;
+    if (token > cur) this.fenceHigh.set(key, token);
+  }
+
+  /** Highest fencing token ever issued for a key (0 if never locked). */
+  getFenceHigh(key) {
+    return this.fenceHigh.get(key) || 0;
+  }
+
+  /** Lightweight observability snapshot: held lock count and total queued waiters. */
+  stats() {
+    let waiters = 0;
+    for (const lock of this.locks.values()) waiters += lock.waiters.length;
+    return { locks: this.locks.size, waiters };
+  }
+}
+
+module.exports = LockManager;

package/src/persistence.js

@@ -0,0 +1,327 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { encodeFrame, FrameDecoder } = require('./protocol');
+
+/**
+ * Crash-survival persistence for the broker: a periodic snapshot + an append-only log (AOF)
+ * of mutation EFFECTS (absolute set / delete / clear), so replay is idempotent and order-
+ * independent. Connection-scoped state (locks, subscriptions, RPC regs, in-flight calls) is
+ * intentionally NOT persisted — see the production-hardening plan. Node built-ins only.
+ *
+ * Durability modes (fsync policy):
+ *   'no'       — best-effort async writes; OS decides when to flush.
+ *   'everysec' — async writes + ~1s periodic fdatasync (default); ≤1s power-loss window.
+ *   'always'   — synchronous write + fsync before returning; durable, blocks the loop.
+ *
+ * The fencing-token counter is kept monotonic across restarts by reserving token blocks: each
+ * time issuance crosses a block boundary we log a 'token' record for the new ceiling, so the
+ * restored seed is always ≥ any token ever issued.
+ */
+
+const SNAPSHOT_VERSION = 1;
+const TOKEN_BLOCK = 1024; // reserve fencing tokens in blocks; ~1 AOF record per 1024 grants
+const DEFAULT_AOF_REWRITE_OPS = 100000; // compact (snapshot + truncate) after this many appends
+
+/** No-op persistence used when the feature is disabled (zero-config default). */
+class NullPersistence {
+  constructor() {
+    this.enabled = false;
+    this.loadedToken = 0;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  async load() {}
+
+  // eslint-disable-next-line class-methods-use-this
+  logMutation() {}
+
+  // eslint-disable-next-line class-methods-use-this
+  noteToken() {}
+
+  // eslint-disable-next-line class-methods-use-this
+  start() {}
+
+  // eslint-disable-next-line class-methods-use-this
+  async flushAndClose() {}
+}
+
+class Persistence {
+  constructor({ dir, mode = 'everysec', codec, snapshotInterval = 0, aofRewriteOps = DEFAULT_AOF_REWRITE_OPS } = {}) {
+    this.enabled = true;
+    this.dir = dir;
+    this.mode = mode; // 'no' | 'everysec' | 'always'
+    this.codec = codec;
+    this.snapshotInterval = snapshotInterval;
+    this.aofRewriteOps = aofRewriteOps;
+
+    this.snapshotPath = path.join(dir, 'snapshot.bin');
+    this.aofPath = path.join(dir, 'aof.bin');
+    this.lockPath = path.join(dir, 'broker.lock');
+
+    this.fd = null; // AOF file descriptor (append)
+    this.queue = []; // pending frames (async modes)
+    this.writing = false;
+    this.dirty = false; // unsynced bytes present
+    this.opsSinceSnapshot = 0;
+
+    this.loadedToken = 0; // highest reserved token recovered on load
+    this.tokenReserved = 0; // current reserved ceiling
+    this._store = null; // set in load(), used for compaction snapshots
+
+    this._fsyncTimer = null;
+    this._snapshotTimer = null;
+    this._closed = false;
+  }
+
+  // --------------------------------------------------------------------- recovery
+
+  async load(store) {
+    this._store = store;
+    fs.mkdirSync(this.dir, { recursive: true });
+    this._acquireLock();
+
+    // 1. Snapshot (compaction base).
+    if (fs.existsSync(this.snapshotPath)) {
+      try {
+        const snap = this.codec.decode(fs.readFileSync(this.snapshotPath));
+        if (snap && snap.version === SNAPSHOT_VERSION) {
+          store.load(snap.entries || []);
+          this.loadedToken = Math.max(this.loadedToken, snap.fenceToken || 0);
+        }
+      } catch {
+        // Corrupt/foreign snapshot — recover from the AOF alone rather than refuse to start.
+      }
+    }
+
+    // 2. AOF tail. FrameDecoder yields only complete frames, so a torn final record (kill -9
+    //    mid-write) is silently dropped — the log self-truncates at the last good frame.
+    if (fs.existsSync(this.aofPath)) {
+      const decoder = new FrameDecoder(this.codec);
+      const buf = fs.readFileSync(this.aofPath);
+      try {
+        decoder.push(buf, (rec) => this._apply(store, rec));
+      } catch {
+        // Decode error mid-stream — stop at the corruption; valid prefix is already applied.
+      }
+    }
+
+    // 3. Compact: fold what we just recovered into a fresh snapshot, then start a clean AOF.
+    this.tokenReserved = this.loadedToken;
+    this._writeSnapshot(store);
+    this.fd = fs.openSync(this.aofPath, 'a');
+  }
+
+  _apply(store, rec) {
+    switch (rec.op) {
+      case 'set':
+        store.load([{ k: rec.k, v: rec.v, e: rec.e || 0 }]);
+        break;
+      case 'del':
+        store.del(rec.k);
+        break;
+      case 'clear':
+        store.clear();
+        break;
+      case 'token':
+        this.loadedToken = Math.max(this.loadedToken, rec.n || 0);
+        break;
+      default:
+        break;
+    }
+  }
+
+  // --------------------------------------------------------------------- logging
+
+  /** Record a mutation effect. `rec` is { op:'set',k,v,e } | { op:'del',k } | { op:'clear' }. */
+  logMutation(rec) {
+    this._append(rec);
+    this.opsSinceSnapshot += 1;
+    if (this.opsSinceSnapshot >= this.aofRewriteOps && this._store) this._compact();
+  }
+
+  /** Called by the broker on every fencing-token mint; reserves a block when crossed. */
+  noteToken(n) {
+    if (n > this.tokenReserved) {
+      this.tokenReserved = Math.ceil((n + 1) / TOKEN_BLOCK) * TOKEN_BLOCK;
+      this._append({ op: 'token', n: this.tokenReserved });
+    }
+  }
+
+  _append(rec) {
+    if (this._closed || this.fd == null) {
+      // Before the AOF fd is open (during load), records are already in the snapshot/store.
+      return;
+    }
+    const frame = encodeFrame(this.codec, rec);
+    if (this.mode === 'always') {
+      try {
+        fs.writeSync(this.fd, frame);
+        fs.fsyncSync(this.fd);
+      } catch (err) {
+        this._onWriteError(err);
+      }
+      return;
+    }
+    this.queue.push(frame);
+    this._drain();
+  }
+
+  _drain() {
+    if (this.writing || this.queue.length === 0 || this.fd == null) return;
+    const batch = this.queue.length === 1 ? this.queue[0] : Buffer.concat(this.queue);
+    this.queue = [];
+    this.writing = true;
+    fs.write(this.fd, batch, (err) => {
+      this.writing = false;
+      if (err) {
+        this._onWriteError(err);
+        return;
+      }
+      this.dirty = true;
+      if (this.queue.length) this._drain();
+    });
+  }
+
+  _onWriteError(err) {
+    // Disk full / read-only / etc.: keep serving from memory, surface the failure, stop trying.
+    this.writeError = err;
+    if (this.onError) this.onError(err);
+  }
+
+  // ------------------------------------------------------------------- snapshots
+
+  _writeSnapshot(store) {
+    const payload = this.codec.encode({
+      version: SNAPSHOT_VERSION,
+      createdAt: Date.now(),
+      fenceToken: this.tokenReserved,
+      entries: store.dump(),
+    });
+    const tmp = `${this.snapshotPath}.tmp`;
+    const fd = fs.openSync(tmp, 'w');
+    try {
+      fs.writeSync(fd, payload);
+      fs.fsyncSync(fd);
+    } finally {
+      fs.closeSync(fd);
+    }
+    fs.renameSync(tmp, this.snapshotPath); // atomic replace
+  }
+
+  /** Compaction: snapshot current state, then truncate the AOF (snapshot now subsumes it). */
+  _compact() {
+    if (!this._store) return;
+    this._writeSnapshot(this._store);
+    this.opsSinceSnapshot = 0;
+    // Truncate and reopen the AOF so future appends start from empty.
+    if (this.fd != null) fs.closeSync(this.fd);
+    this.fd = fs.openSync(this.aofPath, 'w'); // 'w' truncates
+    fs.closeSync(this.fd);
+    this.fd = fs.openSync(this.aofPath, 'a');
+  }
+
+  // ------------------------------------------------------------------- lifecycle
+
+  start() {
+    if (this.mode === 'everysec') {
+      this._fsyncTimer = setInterval(() => {
+        if (this.dirty && this.fd != null) {
+          fs.fdatasync(this.fd, () => {});
+          this.dirty = false;
+        }
+      }, 1000);
+      if (this._fsyncTimer.unref) this._fsyncTimer.unref();
+    }
+    if (this.snapshotInterval > 0) {
+      this._snapshotTimer = setInterval(() => {
+        if (this._store) this._compact();
+      }, this.snapshotInterval);
+      if (this._snapshotTimer.unref) this._snapshotTimer.unref();
+    }
+  }
+
+  async flushAndClose() {
+    this._closed = true;
+    if (this._fsyncTimer) clearInterval(this._fsyncTimer);
+    if (this._snapshotTimer) clearInterval(this._snapshotTimer);
+    // Final compaction makes a planned shutdown lossless.
+    try {
+      if (this._store) this._writeSnapshot(this._store);
+    } catch (err) {
+      this._onWriteError(err);
+    }
+    if (this.fd != null) {
+      try {
+        fs.fsyncSync(this.fd);
+        fs.closeSync(this.fd);
+      } catch {
+        /* ignore */
+      }
+      this.fd = null;
+    }
+    this._releaseLock();
+  }
+
+  // ------------------------------------------------------------------- dir lock
+
+  _acquireLock() {
+    for (let attempt = 0; attempt < 2; attempt++) {
+      try {
+        const fd = fs.openSync(this.lockPath, 'wx'); // fail if exists
+        fs.writeSync(fd, String(process.pid));
+        fs.closeSync(fd);
+        return;
+      } catch (err) {
+        if (err.code !== 'EEXIST') throw err;
+        const stale = this._lockIsStale();
+        if (!stale) {
+          const e = new Error(`persist dir ${this.dir} is locked by a live broker`);
+          e.code = 'EPERSISTLOCKED';
+          throw e;
+        }
+        try {
+          fs.unlinkSync(this.lockPath);
+        } catch {
+          /* race: another cleared it; retry */
+        }
+      }
+    }
+  }
+
+  _lockIsStale() {
+    try {
+      const pid = parseInt(fs.readFileSync(this.lockPath, 'utf8').trim(), 10);
+      if (!pid) return true;
+      process.kill(pid, 0); // throws ESRCH if the pid is gone
+      return false; // pid alive → not stale
+    } catch (err) {
+      return err.code === 'ESRCH' || err.code === 'ENOENT'; // gone → stale
+    }
+  }
+
+  _releaseLock() {
+    try {
+      fs.unlinkSync(this.lockPath);
+    } catch {
+      /* ignore */
+    }
+  }
+}
+
+/**
+ * Build a Persistence (or a no-op) from broker options.
+ * Disabled unless `opts` is set or PROCMESH_PERSIST_DIR is present (zero-config stays in-memory).
+ */
+function createPersistence(opts, name, codec) {
+  const envDir = process.env.PROCMESH_PERSIST_DIR;
+  if (!opts && !envDir) return new NullPersistence();
+  const cfg = opts === true ? {} : opts || {};
+  if (cfg.mode === 'off') return new NullPersistence();
+  const dir = cfg.dir || envDir || path.join(os.tmpdir(), `procmesh-${name || 'default'}`);
+  return new Persistence({ ...cfg, dir, codec });
+}
+
+module.exports = { Persistence, NullPersistence, createPersistence };