threadforge 0.1.0

package/src/core/Interceptors.js

@@ -0,0 +1,420 @@
+import { normalizeErrorCode } from "./Prometheus.js";
+
+/**
+ * RPC Interceptors
+ *
+ * Middleware chain that wraps every proxy call. Interceptors run
+ * in order before the call, and in reverse order after.
+ *
+ * Middleware chain for proxy calls (IPC and HTTP).
+ *
+ *   this.users.getUser('123')
+ *     → DeadlineInterceptor  (propagate remaining time)
+ *     → LoggingInterceptor   (log call + duration)
+ *     → CircuitBreaker       (fail fast if service is down)
+ *     → RetryInterceptor     (retry on transient failures)
+ *     → actual IPC call
+ *     ← RetryInterceptor
+ *     ← CircuitBreaker       (track success/failure)
+ *     ← LoggingInterceptor   (log result)
+ *     ← DeadlineInterceptor
+ *
+ * Usage in config:
+ *
+ *   static contract = {
+ *     expose: ['getUser'],
+ *     interceptors: ['deadline', 'logging', 'circuitBreaker'],
+ *   };
+ *
+ * Or per-method:
+ *
+ *   @Expose({ interceptors: ['retry'], retry: { maxAttempts: 3 } })
+ *   async getUser(id) { ... }
+ */
+
+/**
+ * @typedef {Object} CallContext
+ * @property {string} from        - Calling service name
+ * @property {string} target      - Target service name
+ * @property {string} method      - Method name being called
+ * @property {any[]}  args        - Method arguments
+ * @property {number} deadline    - Absolute timestamp when the call must complete
+ * @property {Map}    metadata    - Key-value metadata
+ * @property {number} attempt     - Current retry attempt (0-based)
+ */
+
+/**
+ * Run a call through an interceptor chain.
+ *
+ * Interceptors MAY call next() multiple times (e.g., the retry interceptor
+ * invokes next() on each attempt). Downstream interceptors and the final
+ * handler should therefore be idempotent-safe.
+ *
+ * @param {Function[]} interceptors - Array of interceptor functions
+ * @param {CallContext} callCtx - The call context
+ * @param {Function} finalHandler - The actual IPC call
+ * @returns {Promise<*>}
+ */
+export async function runInterceptorChain(interceptors, callCtx, finalHandler) {
+  let index = 0;
+
+  async function next() {
+    if (index >= interceptors.length) {
+      return finalHandler(callCtx);
+    }
+    const interceptor = interceptors[index++];
+    return interceptor(callCtx, next);
+  }
+
+  return next();
+}
+
+// ─── Deadline Propagation ───────────────────────────────────
+
+/**
+ * Propagates deadlines across service boundaries.
+ *
+ * If gateway sets a 5s timeout, and the users service call takes
+ * 2s, the billing service call from users only gets 3s — not a
+ * fresh 5s.
+ *
+ * This prevents cascading timeouts where each service adds its
+ * own full timeout on top.
+ */
+export function deadlineInterceptor(defaultTimeoutMs = 5000) {
+  return async function deadline(callCtx, next) {
+    // If no deadline set yet, create one
+    if (!callCtx.deadline) {
+      callCtx.deadline = Date.now() + defaultTimeoutMs;
+    }
+
+    // Check if we've already exceeded the deadline
+    const remaining = callCtx.deadline - Date.now();
+    if (remaining <= 0) {
+      const err = new Error(
+        `Deadline exceeded before calling ${callCtx.target}.${callCtx.method}` + ` (from ${callCtx.from})`,
+      );
+      err.code = 'DEADLINE_EXCEEDED';
+      throw err;
+    }
+
+    // Pass the remaining time as the timeout for the actual call
+    callCtx.timeout = remaining;
+
+    // Add deadline to metadata so the receiving service knows
+    callCtx.metadata.deadline = callCtx.deadline;
+
+    return next();
+  };
+}
+
+// ─── Logging ────────────────────────────────────────────────
+
+/**
+ * Logs every RPC call with duration and outcome.
+ * Integrates with the service's structured logger.
+ */
+export function loggingInterceptor(logger) {
+  return async function logging(callCtx, next) {
+    const start = performance.now();
+
+    try {
+      const result = await next();
+      const duration = performance.now() - start;
+
+      if (logger) {
+        logger.debug("RPC call", {
+          target: callCtx.target,
+          method: callCtx.method,
+          duration: `${duration.toFixed(2)}ms`,
+          attempt: callCtx.attempt,
+        });
+      }
+
+      return result;
+    } catch (err) {
+      const duration = performance.now() - start;
+
+      if (logger) {
+        logger.warn("RPC call failed", {
+          target: callCtx.target,
+          method: callCtx.method,
+          duration: `${duration.toFixed(2)}ms`,
+          error: err.message,
+          attempt: callCtx.attempt,
+        });
+      }
+
+      throw err;
+    }
+  };
+}
+
+// ─── Circuit Breaker ────────────────────────────────────────
+
+/**
+ * Implements the circuit breaker pattern.
+ *
+ * States:
+ *   CLOSED  → Normal operation. Tracks failure rate.
+ *   OPEN    → Calls fail immediately. Checked periodically.
+ *   HALF    → Allows one probe call. Success → CLOSED, failure → OPEN.
+ *
+ * Prevents a failing service from consuming resources on the
+ * calling side. Without this, a slow/dead service causes thread
+ * starvation as pending requests pile up.
+ */
+export class CircuitBreaker {
+  /**
+   * @param {Object} options
+   * @param {number} [options.failureThreshold=5]  - Failures before opening
+   * @param {number} [options.resetTimeoutMs=10000] - Time before half-open probe
+   * @param {number} [options.successThreshold=2]  - Successes in half-open before closing
+   */
+  constructor(options = {}) {
+    this.failureThreshold = options.failureThreshold ?? 5;
+    this.resetTimeoutMs = options.resetTimeoutMs ?? 10000;
+    this.successThreshold = options.successThreshold ?? 2;
+    /** H-IPC-1: Sliding window duration for failure counting (default 60s) */
+    this._windowMs = options.windowMs ?? 60_000;
+
+    /** @type {'closed'|'open'|'half-open'} */
+    this.state = "closed";
+    this.failureCount = 0;
+    this.successCount = 0;
+    this.lastFailureTime = 0;
+    this.nextAttemptTime = 0;
+    /** H-IPC-1: Sliding window of failure timestamps */
+    this._failureWindow = [];
+    this._probeInFlight = false;
+  }
+
+  createInterceptor() {
+    const breaker = this;
+
+    return async function circuitBreaker(callCtx, next) {
+      if (breaker.state === "open") {
+        if (Date.now() < breaker.nextAttemptTime) {
+          const err = new Error(
+            `Circuit breaker OPEN for ${callCtx.target}.${callCtx.method}` +
+              ` — service appears unavailable. Retry after ${new Date(breaker.nextAttemptTime).toISOString()}`,
+          );
+          err.code = 'CIRCUIT_OPEN';
+          throw err;
+        }
+        // Only allow one probe request at a time in half-open.
+        // Safe in single-threaded JS: no await between check and set,
+        // so no other microtask can interleave here.
+        if (breaker._probeInFlight) {
+          const err = new Error(
+            `Circuit breaker OPEN for ${callCtx.target}.${callCtx.method}` +
+              ` — probe already in flight. Retry after ${new Date(breaker.nextAttemptTime).toISOString()}`,
+          );
+          err.code = 'CIRCUIT_OPEN';
+          throw err;
+        }
+        breaker._probeInFlight = true;
+        breaker.state = "half-open";
+      }
+
+      try {
+        const result = await next();
+
+        // Success
+        if (breaker.state === "half-open") {
+          breaker.successCount++;
+          // Allow next probe through while still in half-open
+          breaker._probeInFlight = false;
+          if (breaker.successCount >= breaker.successThreshold) {
+            breaker.state = "closed";
+            breaker.failureCount = 0;
+            breaker.successCount = 0;
+            breaker._failureWindow.length = 0;
+          }
+        }
+        // H-IPC-1: In closed state, don't reset failureCount on success.
+        // Use sliding window — old failures expire naturally.
+
+        return result;
+      } catch (err) {
+        // Only count transient/server errors toward circuit breaker
+        const isTransient = !err.statusCode || err.statusCode >= 500 ||
+          err.code === "ECONNREFUSED" || err.code === "ECONNRESET" ||
+          err.message?.includes("timeout") || err.message?.includes("deadline");
+
+        if (isTransient) {
+          // H-IPC-1: Use sliding window for failure counting — prune expired entries
+          const now = Date.now();
+          breaker._failureWindow.push(now);
+          const cutoff = now - breaker._windowMs;
+          while (breaker._failureWindow.length > 0 && breaker._failureWindow[0] < cutoff) {
+            breaker._failureWindow.shift();
+          }
+          breaker.failureCount = breaker._failureWindow.length;
+          breaker.lastFailureTime = now;
+        }
+
+        if (breaker.state === "half-open") {
+          // Reset probe flag so next probe can be attempted
+          breaker._probeInFlight = false;
+          if (isTransient) {
+            // Transient probe failure — back to open
+            breaker.state = "open";
+            breaker.nextAttemptTime = Date.now() + breaker.resetTimeoutMs * (0.75 + Math.random() * 0.5);
+            breaker.successCount = 0;
+          }
+          // Non-transient errors (4xx) in half-open: stay half-open, allow next probe
+        } else if (isTransient && breaker.failureCount >= breaker.failureThreshold) {
+          breaker.state = "open";
+          breaker.nextAttemptTime = Date.now() + breaker.resetTimeoutMs;
+        }
+
+        throw err;
+      }
+    };
+  }
+
+  get stats() {
+    return {
+      state: this.state,
+      failureCount: this.failureCount,
+      successCount: this.successCount,
+      nextAttemptTime: this.state === "open" ? new Date(this.nextAttemptTime).toISOString() : null,
+    };
+  }
+}
+
+// ─── Retry ──────────────────────────────────────────────────
+
+// Errors where the request was never sent (always safe to retry)
+const PRE_SEND_PATTERNS = ["ECONNREFUSED", "no workers"];
+
+// Errors where the request may have been partially sent (only retry if idempotent)
+const POST_SEND_PATTERNS = ["ECONNRESET", "EPIPE", "ETIMEDOUT", "timed out"];
+
+/**
+ * Determine if an error is retryable given an idempotency setting.
+ *
+ * @param {Error} err
+ * @param {boolean|undefined} idempotent - true: always retry transient errors,
+ *   false: never retry, undefined: retry only pre-send errors
+ * @returns {boolean}
+ */
+export function isRetryable(err, idempotent) {
+  if (idempotent === false) return false;
+
+  const msg = err.message?.toLowerCase() ?? "";
+
+  const isPreSend = PRE_SEND_PATTERNS.some((p) => msg.includes(p.toLowerCase()));
+  if (isPreSend) return true;
+
+  const isPostSend = POST_SEND_PATTERNS.some((p) => msg.includes(p.toLowerCase()));
+  if (isPostSend && idempotent === true) return true;
+
+  // When idempotent is undefined, post-send errors are NOT retryable
+  return false;
+}
+
+/**
+ * Retry interceptor with exponential backoff and jitter.
+ *
+ * @param {object} [options]
+ * @param {number} [options.maxAttempts=3]
+ * @param {number} [options.baseDelayMs=100]
+ * @param {number} [options.maxDelayMs=2000]
+ * @param {boolean} [options.idempotent] - passed to isRetryable()
+ * @returns {function} interceptor
+ */
+let _retryDeprecationWarned = false;
+export function retryInterceptor(options = {}) {
+  if (!_retryDeprecationWarned) {
+    _retryDeprecationWarned = true;
+    console.warn('[Interceptors] retryInterceptor is deprecated — retry logic is handled by ServiceProxy.createProxiedMethod(). This interceptor is now a no-op passthrough.');
+  }
+  // Return a passthrough interceptor instead of throwing, so existing
+  // chains that include retryInterceptor don't break at startup.
+  return async function retryNoop(callCtx, next) {
+    return next();
+  };
+}
+
+// ─── Metrics ────────────────────────────────────────────────
+
+/**
+ * Tracks RPC metrics (call count, latency histogram, error rate).
+ */
+export function metricsInterceptor(metrics) {
+  return async function rpcMetrics(callCtx, next) {
+    const start = performance.now();
+    const labels = {
+      target: callCtx.target,
+      method: callCtx.method,
+    };
+
+    try {
+      const result = await next();
+      const duration = performance.now() - start;
+
+      if (metrics) {
+        metrics.counter("rpc_calls_total", 1, { ...labels, status: "success" });
+        metrics.histogram("rpc_duration_ms", duration, labels);
+      }
+
+      return result;
+    } catch (err) {
+      const duration = performance.now() - start;
+
+      if (metrics) {
+        metrics.counter("rpc_calls_total", 1, { ...labels, status: "error" });
+        metrics.histogram("rpc_duration_ms", duration, labels);
+        metrics.counter("rpc_errors_total", 1, { ...labels, error: normalizeErrorCode(err) });
+      }
+
+      throw err;
+    }
+  };
+}
+
+// ─── Bulkhead ────────────────────────────────────────────────
+
+/**
+ * Limits concurrent outgoing calls per service to prevent
+ * cascade failures. Uses a simple counter — no external deps.
+ *
+ * @param {Object} [options]
+ * @param {number} [options.maxConcurrent=100] - Max concurrent calls per service
+ * @returns {function} interceptor
+ */
+export function bulkheadInterceptor(options = {}) {
+  const maxConcurrent = options.maxConcurrent ?? 100;
+  /** @type {Map<string, number>} service name → in-flight count */
+  const inFlight = new Map();
+
+  return async function bulkhead(callCtx, next) {
+    const target = callCtx.target;
+    const current = inFlight.get(target) ?? 0;
+
+    if (current >= maxConcurrent) {
+      const err = new Error(
+        `Bulkhead limit reached for service "${target}": ${current}/${maxConcurrent} concurrent calls. ` +
+          `Rejecting ${callCtx.method}.`,
+      );
+      err.code = 'BULKHEAD_FULL';
+      err.statusCode = 503;
+      throw err;
+    }
+
+    inFlight.set(target, current + 1);
+    try {
+      const result = await next();
+      return result;
+    } finally {
+      const count = inFlight.get(target) ?? 1;
+      if (count <= 1) {
+        inFlight.delete(target);
+      } else {
+        inFlight.set(target, count - 1);
+      }
+    }
+  };
+}