threadforge 0.1.0

package/src/core/DirectMessageBus.js

@@ -0,0 +1,364 @@
+import { EventEmitter } from "node:events";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const EXIT_SOCKET_DIRS = new Set();
+let EXIT_CLEANUP_INSTALLED = false;
+
+function cleanupSocketDir(socketDir) {
+  try {
+    if (fs.existsSync(socketDir)) {
+      for (const f of fs.readdirSync(socketDir)) {
+        try { fs.unlinkSync(path.join(socketDir, f)); } catch {}
+      }
+      try { fs.rmdirSync(socketDir); } catch {}
+    }
+  } catch {}
+}
+
+function installExitCleanupHook() {
+  if (EXIT_CLEANUP_INSTALLED) return;
+  EXIT_CLEANUP_INSTALLED = true;
+  process.once("exit", () => {
+    for (const socketDir of EXIT_SOCKET_DIRS) {
+      cleanupSocketDir(socketDir);
+    }
+    EXIT_SOCKET_DIRS.clear();
+  });
+}
+
+function isExpectedChannelCloseError(err) {
+  if (!err) return false;
+  const code = err.code;
+  if (
+    code === "EPIPE" ||
+    code === "ECONNRESET" ||
+    code === "ERR_IPC_CHANNEL_CLOSED" ||
+    code === "ERR_IPC_DISCONNECTED"
+  ) {
+    return true;
+  }
+  const msg = String(err.message ?? "").toLowerCase();
+  return (
+    msg.includes("channel closed") ||
+    msg.includes("ipc channel is already disconnected") ||
+    msg.includes("broken pipe")
+  );
+}
+
+function isWorkerSendable(worker) {
+  if (!worker) return false;
+  if (typeof worker.isDead === "function" && worker.isDead()) return false;
+  if (typeof worker.isConnected === "function" && !worker.isConnected()) return false;
+  if (worker.process?.connected === false) return false;
+  return true;
+}
+
+/**
+ * DirectMessageBus — Unix Domain Socket Mesh (Supervisor Side)
+ *
+ * The fundamental problem: cluster.Worker.send() can only transfer
+ * sockets/servers, not MessagePort objects. So we can't use MessageChannel
+ * for direct worker-to-worker communication in cluster mode.
+ *
+ * Solution: Each worker opens a Unix domain socket server. Workers connect
+ * directly to each other via these sockets. The supervisor only tells
+ * workers WHERE to connect (socket paths), then gets out of the way.
+ *
+ * Architecture:
+ *
+ *   SETUP (supervisor involved briefly):
+ *     1. Each worker starts a UDS server at /tmp/forge-{pid}/{service}-{worker}.sock
+ *     2. Worker reports its socket path to supervisor via IPC
+ *     3. Supervisor broadcasts the full socket registry to all workers
+ *     4. Workers establish direct connections to each other
+ *
+ *   RUNTIME (supervisor NOT involved in message routing):
+ *     Worker A ──UDS──► Worker B   (direct, length-prefixed JSON)
+ *     Worker B ──UDS──► Worker A
+ *
+ *   Supervisor only handles:
+ *     - Socket path registry distribution (one-time + on new workers)
+ *     - Health checks (periodic pull, not per-message)
+ *     - Worker lifecycle (restart, scale)
+ */
+export class DirectMessageBus extends EventEmitter {
+  constructor() {
+    super();
+
+    /** @type {Map<string, Array<{id: number, worker: object, mode: string, socketPath?: string}>>} */
+    this.workers = new Map();
+
+    /** @type {Map<string, string>} "serviceName:workerId" → socket path */
+    this.socketRegistry = new Map();
+
+    /** @type {Set<number>} Track registered worker IDs to prevent duplicate listeners */
+    this._registeredWorkerIds = new Set();
+
+    /** @type {WeakMap<object, Function>} Shared per-worker error handlers */
+    this._workerErrorHandlers = new WeakMap();
+
+    /** @type {Map<string, Set<string>>} CR-IPC-9: worker key → set of connected peer keys */
+    this._connections = new Map();
+
+    // S-IPC-2: Random suffix prevents socket path prediction
+    this._socketDir = path.join(os.tmpdir(), `forge-${process.pid}-${crypto.randomBytes(4).toString('hex')}`);
+    this._broadcastTimer = null;
+    try {
+      fs.mkdirSync(this._socketDir, { recursive: true });
+    } catch (e) {
+      if (e.code !== "EEXIST") console.error("[DirectMessageBus] mkdirSync failed:", e.message);
+    }
+    EXIT_SOCKET_DIRS.add(this._socketDir);
+    installExitCleanupHook();
+  }
+
+  get socketDir() {
+    return this._socketDir;
+  }
+
+  /**
+   * Register a worker. Tell it to start a UDS server, then
+   * broadcast the updated registry to all workers.
+   */
+  registerWorker(serviceName, worker, mode = "cluster") {
+    if (!this.workers.has(serviceName)) {
+      this.workers.set(serviceName, []);
+    }
+
+    const workerId = mode === "cluster" ? worker.id : worker.threadId;
+
+    const entry = {
+      id: workerId,
+      worker,
+      mode,
+      socketPath: null,
+    };
+
+    this.workers.get(serviceName).push(entry);
+
+    // Attach one shared error listener per worker so late IPC channel errors
+    // during churn don't surface as unhandled EventEmitter errors.
+    if (!this._workerErrorHandlers.has(worker)) {
+      const errorHandler = (err) => {
+        if (!isExpectedChannelCloseError(err)) {
+          console.error("[DirectMessageBus] Worker error:", err?.message ?? err);
+        }
+      };
+      this._workerErrorHandlers.set(worker, errorHandler);
+      worker.on("error", errorHandler);
+    }
+    entry._errorHandler = this._workerErrorHandlers.get(worker);
+
+    // Prevent duplicate message listener registration
+    // If workerId is already registered (e.g., crashed worker's ID reused), clean up stale listeners
+    if (this._registeredWorkerIds.has(workerId)) {
+      // Find and clean up stale entry with this workerId across all services
+      for (const [svcName, entries] of this.workers) {
+        const staleIdx = entries.findIndex((e) => e.id === workerId && e.worker !== worker);
+        if (staleIdx !== -1) {
+          const stale = entries[staleIdx];
+          if (stale._messageHandler) stale.worker.off("message", stale._messageHandler);
+          if (stale.socketPath) {
+            try { fs.unlinkSync(stale.socketPath); } catch {}
+            this.socketRegistry.delete(`${svcName}:${workerId}`);
+          }
+          entries.splice(staleIdx, 1);
+          if (entries.length === 0) this.workers.delete(svcName);
+        }
+      }
+      this._registeredWorkerIds.delete(workerId);
+    }
+    this._registeredWorkerIds.add(workerId);
+
+    // Listen for supervisor-level IPC from this worker
+    const messageHandler = (msg) => {
+      if (!msg || !msg.type) return;
+
+      switch (msg.type) {
+        case "forge:socket-ready":
+          // In colocated groups a worker may host multiple services, each with its
+          // own socket. Ignore ready messages for sibling services on this entry.
+          if (msg.serviceName && msg.serviceName !== serviceName) break;
+          entry.socketPath = msg.socketPath;
+          this.socketRegistry.set(`${serviceName}:${msg.workerId}`, msg.socketPath);
+          this._scheduleBroadcast();
+          break;
+
+        case "forge:worker-ready":
+          this._sendInitSocket(worker, serviceName, entry.id);
+          this._sendRegistryTo(worker);
+          break;
+
+        case "forge:channel-ready": {
+          // CR-IPC-9: Track established connections
+          const workerKey = `${serviceName}:${entry.id}`;
+          if (!this._connections.has(workerKey)) {
+            this._connections.set(workerKey, new Set());
+          }
+          if (msg.peerKey) {
+            this._connections.get(workerKey).add(msg.peerKey);
+          }
+          break;
+        }
+
+        case "forge:metric":
+        case "forge:log":
+        case "forge:health-response": {
+          const eventName = msg.type.replace("forge:", "");
+          this.emit(eventName, { service: serviceName, ...msg });
+          break;
+        }
+      }
+    };
+    entry._messageHandler = messageHandler;
+    worker.on("message", messageHandler);
+
+    // Tell the new worker to start its UDS server
+    this._sendInitSocket(worker, serviceName, entry.id);
+    this._sendRegistryTo(worker);
+  }
+
+  _scheduleBroadcast() {
+    // CR-IPC-5: Don't cancel existing timer — let the first registration broadcast on schedule
+    if (this._broadcastTimer) return;
+    this._broadcastTimer = setTimeout(() => {
+      this._broadcastTimer = null;
+      this._broadcastRegistry();
+    }, 50 + Math.floor(Math.random() * 50));
+  }
+
+  _sendInitSocket(worker, serviceName, workerId) {
+    if (!isWorkerSendable(worker)) return;
+    try {
+      worker.send({
+        type: "forge:init-socket",
+        socketDir: this._socketDir,
+        serviceName,
+        workerId,
+      });
+    } catch (e) {
+      if (!isExpectedChannelCloseError(e)) {
+        console.error("[DirectMessageBus] Failed to send init-socket:", e.message);
+      }
+    }
+  }
+
+  _sendRegistryTo(worker) {
+    if (!isWorkerSendable(worker)) return;
+    try {
+      const msg = {
+        type: "forge:socket-registry",
+        registry: Object.fromEntries(this.socketRegistry),
+      };
+      // S-IPC-1: Include cluster secret for handshake authentication
+      const secret = process.env.FORGE_CLUSTER_SECRET;
+      if (secret) {
+        msg.secret = secret;
+      }
+      worker.send(msg);
+    } catch (e) {
+      if (!isExpectedChannelCloseError(e)) {
+        console.error("[DirectMessageBus] Failed to send registry:", e.message);
+      }
+    }
+  }
+
+  _broadcastRegistry() {
+    for (const [, workers] of this.workers) {
+      for (const entry of workers) {
+        this._sendRegistryTo(entry.worker);
+      }
+    }
+  }
+
+  unregisterWorker(serviceName, workerId, options = {}) {
+    const workers = this.workers.get(serviceName);
+    if (!workers) return;
+
+    const idx = workers.findIndex((w) => w.id === workerId);
+    if (idx === -1) return;
+
+    const entry = workers[idx];
+    if (entry._messageHandler) {
+      entry.worker.off("message", entry._messageHandler);
+    }
+    if (entry.socketPath) {
+      try {
+        fs.unlinkSync(entry.socketPath);
+      } catch (e) {
+        if (e.code !== "ENOENT") console.error("[DirectMessageBus] unlink failed:", e.message);
+      }
+      this.socketRegistry.delete(`${serviceName}:${workerId}`);
+    }
+    workers.splice(idx, 1);
+
+    // Clean up the registered worker ID tracking
+    this._registeredWorkerIds.delete(workerId);
+
+    if (workers.length === 0) {
+      this.workers.delete(serviceName);
+    }
+
+    if (!options.suppressBroadcast && this.workers.size > 0) {
+      this._broadcastRegistry();
+    }
+  }
+
+  unregisterService(serviceName) {
+    const workers = [...(this.workers.get(serviceName) ?? [])];
+    for (const w of workers) {
+      this.unregisterWorker(serviceName, w.id);
+    }
+  }
+
+  requestHealthChecks() {
+    for (const [, workers] of this.workers) {
+      for (const entry of workers) {
+        if (!isWorkerSendable(entry.worker)) continue;
+        try {
+          entry.worker.send({ type: "forge:health-check", timestamp: Date.now() });
+        } catch (err) {
+          if (!isExpectedChannelCloseError(err)) {
+            console.error("[DirectMessageBus] Failed to send health-check:", err.message);
+          }
+        }
+      }
+    }
+  }
+
+  stats() {
+    const result = {};
+    for (const [name, workers] of this.workers) {
+      result[name] = {
+        workerCount: workers.length,
+        ids: workers.map((w) => w.id),
+        directSockets: workers.filter((w) => w.socketPath).length,
+      };
+    }
+    return result;
+  }
+
+  /**
+   * CR-IPC-9: Get the connection matrix for debugging.
+   * @returns {Object<string, string[]>} worker key → array of connected peer keys
+   */
+  getConnectionMatrix() {
+    const result = {};
+    for (const [key, peers] of this._connections) {
+      result[key] = [...peers];
+    }
+    return result;
+  }
+
+  cleanup() {
+    if (this._broadcastTimer) {
+      clearTimeout(this._broadcastTimer);
+      this._broadcastTimer = null;
+    }
+    cleanupSocketDir(this._socketDir);
+    EXIT_SOCKET_DIRS.delete(this._socketDir);
+  }
+}

package/src/core/EndpointResolver.js

@@ -0,0 +1,247 @@
+/**
+ * EndpointResolver
+ *
+ * Resolves service names to { host, port, remote } endpoints.
+ * Supports:
+ *   - Single endpoints (one instance per service)
+ *   - Multi-instance arrays (round-robin selection)
+ *   - Dynamic updates via set/remove (used by ServiceRegistry)
+ *   - Fallback from FORGE_SERVICE_ENDPOINTS to FORGE_SERVICE_PORTS
+ *   - C3: File fallback via FORGE_SERVICE_ENDPOINTS_FILE for large maps
+ */
+import { readFileSync } from "node:fs";
+
+/** Validate that an endpoint has the expected shape */
+function validateEndpoint(ep) {
+  if (!ep || typeof ep !== 'object') return false;
+  if (typeof ep.host !== 'string' || !ep.host) return false;
+  if (ep.port !== undefined && (typeof ep.port !== 'number' || ep.port < 1 || ep.port > 65535)) return false;
+  return true;
+}
+
+export class EndpointResolver {
+  constructor() {
+    /** @type {Map<string, Array<{host: string, port: number, remote: boolean}>>} */
+    this._endpoints = new Map();
+
+    /** @type {Map<string, number>} round-robin counters */
+    this._counters = new Map();
+
+    /** @type {Map<string, Object>} per-service routing strategies (optional override) */
+    this._strategies = new Map();
+  }
+
+  /**
+   * Set a routing strategy for a specific service.
+   * When set, resolve() delegates to the strategy's pick() method
+   * instead of using built-in round-robin.
+   *
+   * The strategy must implement pick(entries, callContext) where entries
+   * are wrapped as { key: "host:port", host, port, remote }.
+   *
+   * @param {string} serviceName
+   * @param {Object} strategy - Must have a pick(workers, callContext) method
+   */
+  setStrategy(serviceName, strategy) {
+    if (!strategy || typeof strategy.pick !== 'function') {
+      throw new Error(`Strategy for "${serviceName}" must have a pick() method`);
+    }
+    this._strategies.set(serviceName, strategy);
+  }
+
+  /**
+   * Create an EndpointResolver from environment variables.
+   *
+   * Parses FORGE_SERVICE_ENDPOINTS first (full endpoint map with host/port/remote).
+   * Falls back to FORGE_SERVICE_PORTS (localhost-only port map) when endpoints
+   * aren't available.
+   *
+   * @returns {EndpointResolver}
+   */
+  static fromEnv() {
+    const resolver = new EndpointResolver();
+
+    // C3: Support file fallback for large endpoint maps (written by Supervisor)
+    let endpointsJson = process.env.FORGE_SERVICE_ENDPOINTS;
+    if (!endpointsJson && process.env.FORGE_SERVICE_ENDPOINTS_FILE) {
+      try {
+        endpointsJson = readFileSync(process.env.FORGE_SERVICE_ENDPOINTS_FILE, "utf8");
+      } catch (e) {
+        console.error("[EndpointResolver] Failed to read FORGE_SERVICE_ENDPOINTS_FILE:", e.message);
+      }
+    }
+    if (endpointsJson) {
+      try {
+        const parsed = JSON.parse(endpointsJson);
+        for (const [name, value] of Object.entries(parsed)) {
+          if (Array.isArray(value)) {
+            const valid = value.filter(ep => validateEndpoint(ep));
+            if (valid.length > 0) resolver._endpoints.set(name, valid);
+          } else if (validateEndpoint(value)) {
+            resolver._endpoints.set(name, [value]);
+          }
+        }
+      } catch (e) {
+        console.error("[EndpointResolver] Failed to parse FORGE_SERVICE_ENDPOINTS:", e.message);
+      }
+    }
+
+    // Merge FORGE_SERVICE_PORTS as fallback for any services not in endpoints
+    const portsJson = process.env.FORGE_SERVICE_PORTS;
+    if (portsJson) {
+      try {
+        const ports = JSON.parse(portsJson);
+        for (const [name, port] of Object.entries(ports)) {
+          const p = Number(port);
+          if (!resolver._endpoints.has(name) && Number.isInteger(p) && p >= 1 && p <= 65535) {
+            resolver._endpoints.set(name, [{ host: "127.0.0.1", port: p, remote: false }]);
+          }
+        }
+      } catch (e) {
+        console.error("[EndpointResolver] Failed to parse FORGE_SERVICE_PORTS:", e.message);
+      }
+    }
+
+    return resolver;
+  }
+
+  /**
+   * Resolve an endpoint for a service.
+   * Delegates to a configured RoutingStrategy when available,
+   * otherwise round-robins across instances.
+   *
+   * @param {string} serviceName
+   * @param {Object} [callContext] - Optional call context passed to strategy.pick()
+   * @returns {{ host: string, port: number, remote: boolean } | null}
+   */
+  resolve(serviceName, callContext) {
+    const endpoints = this._endpoints.get(serviceName);
+    if (!endpoints || endpoints.length === 0) return null;
+
+    if (endpoints.length === 1) return endpoints[0];
+
+    // Delegate to configured strategy if present
+    const strategy = this._strategies.get(serviceName);
+    if (strategy) {
+      // Wrap endpoints as WorkerEntry-compatible objects for RoutingStrategy
+      const entries = endpoints.map(ep => ({
+        key: `${ep.host}:${ep.port}`,
+        host: ep.host,
+        port: ep.port,
+        remote: ep.remote,
+      }));
+      const picked = strategy.pick(entries, callContext);
+      if (picked) {
+        // Handle broadcast (returns array) — return first for resolve(), use all() for broadcast
+        if (Array.isArray(picked)) return picked[0] ?? null;
+        return { host: picked.host, port: picked.port, remote: picked.remote };
+      }
+      // Strategy returned null — fall through to round-robin
+    }
+
+    // Round-robin: read-then-write is atomic in single-threaded Node.js
+    // (no microtask or I/O can interleave between get and set)
+    const idx = (this._counters.get(serviceName) ?? 0);
+    this._counters.set(serviceName, (idx + 1) % 1_000_000_000);
+    const endpoint = endpoints[idx % endpoints.length];
+    return endpoint;
+  }
+
+  /**
+   * Get all endpoints for a service (used for broadcast/event delivery).
+   *
+   * @param {string} serviceName
+   * @returns {Array<{ host: string, port: number, remote: boolean }>}
+   */
+  all(serviceName) {
+    return (this._endpoints.get(serviceName) ?? []).map(e => ({ ...e }));
+  }
+
+  /**
+   * Add or update an endpoint for a service.
+   * Used by dynamic registry discovery.
+   *
+   * @param {string} serviceName
+   * @param {{ host: string, port: number, remote?: boolean }} endpoint
+   */
+  set(serviceName, endpoint) {
+    if (endpoint.port !== undefined && (!Number.isInteger(endpoint.port) || endpoint.port < 1 || endpoint.port > 65535)) {
+      throw new Error(`Invalid port for service "${serviceName}": ${endpoint.port}`);
+    }
+    const ep = { remote: true, ...endpoint };
+    const existing = this._endpoints.get(serviceName) ?? [];
+
+    // Replace if same host:port exists, otherwise append
+    const idx = existing.findIndex((e) => e.host === ep.host && e.port === ep.port);
+    if (idx !== -1) {
+      existing[idx] = ep;
+    } else {
+      existing.push(ep);
+    }
+
+    this._endpoints.set(serviceName, existing);
+  }
+
+  /**
+   * Remove a specific instance of a service.
+   * Used when a remote node goes down.
+   *
+   * @param {string} serviceName
+   * @param {string} host
+   * @param {number} port
+   */
+  remove(serviceName, host, port) {
+    const existing = this._endpoints.get(serviceName);
+    if (!existing) return;
+
+    const filtered = existing.filter((e) => !(e.host === host && e.port === port));
+    if (filtered.length === 0) {
+      this._endpoints.delete(serviceName);
+      // CR-IPC-7: Clean up counter when all endpoints removed
+      this._counters.delete(serviceName);
+    } else {
+      this._endpoints.set(serviceName, filtered);
+      // Let counter naturally wrap via modulo in resolve()
+    }
+  }
+
+  /**
+   * COR-C1: Acquire a pending slot for a resolved endpoint.
+   * Delegates to the service's routing strategy if it supports acquire().
+   * Must be paired with releaseEndpoint() in a finally block.
+   *
+   * @param {string} serviceName
+   * @param {string} endpointKey - "host:port" key from resolve()
+   */
+  acquireEndpoint(serviceName, endpointKey) {
+    const strategy = this._strategies.get(serviceName);
+    if (strategy && typeof strategy.acquire === 'function') {
+      strategy.acquire(endpointKey);
+    }
+  }
+
+  /**
+   * COR-C1: Release a pending slot for a resolved endpoint.
+   * Delegates to the service's routing strategy if it supports release().
+   *
+   * @param {string} serviceName
+   * @param {string} endpointKey - "host:port" key from resolve()
+   */
+  releaseEndpoint(serviceName, endpointKey) {
+    const strategy = this._strategies.get(serviceName);
+    if (strategy && typeof strategy.release === 'function') {
+      strategy.release(endpointKey);
+    }
+  }
+
+  /**
+   * Check if a service has any known endpoints.
+   *
+   * @param {string} serviceName
+   * @returns {boolean}
+   */
+  has(serviceName) {
+    const eps = this._endpoints.get(serviceName);
+    return eps != null && eps.length > 0;
+  }
+}