threadforge 0.1.0

package/src/scaling/ScaleAdvisor.js

@@ -0,0 +1,442 @@
+/**
+ * ScaleAdvisor
+ *
+ * Monitors service health across the cluster and produces
+ * actionable scaling recommendations.
+ *
+ * This is the "automatic" part of horizontal scaling. Instead of
+ * a human watching dashboards and deciding "billing needs its own
+ * machine", the ScaleAdvisor detects pressure and recommends:
+ *
+ *   1. SCALE UP   — add more workers on the same machine
+ *   2. SPLIT OUT  — move a service to its own dedicated process
+ *   3. MIGRATE    — move a service to another machine
+ *   4. SCALE DOWN — reduce workers or colocate back
+ *
+ * ═══════════════════════════════════════════════════════════════
+ * SCALING SIGNALS
+ * ═══════════════════════════════════════════════════════════════
+ *
+ * The advisor monitors three signals per service:
+ *
+ *   CPU saturation  — is the service's process(es) maxing out?
+ *   Latency drift   — is P99 latency increasing over time?
+ *   Queue pressure  — are pending requests growing?
+ *
+ * Thresholds:
+ *
+ *   ┌──────────────────┬────────────┬─────────────┬──────────────┐
+ *   │ Signal           │ Normal     │ Warning     │ Critical     │
+ *   ├──────────────────┼────────────┼─────────────┼──────────────┤
+ *   │ CPU per worker   │ < 60%      │ 60-85%      │ > 85%        │
+ *   │ P99 latency      │ < 100ms    │ 100-500ms   │ > 500ms      │
+ *   │ Pending requests │ < 50       │ 50-200      │ > 200        │
+ *   │ Memory per proc  │ < 512MB    │ 512MB-1GB   │ > 1GB        │
+ *   └──────────────────┴────────────┴─────────────┴──────────────┘
+ *
+ * ═══════════════════════════════════════════════════════════════
+ * DECISION TREE
+ * ═══════════════════════════════════════════════════════════════
+ *
+ *   Service under pressure?
+ *   │
+ *   ├─ Is it colocated (sharing a process with others)?
+ *   │   └─ YES → Recommendation: SPLIT OUT to its own process
+ *   │            (colocation was causing resource contention)
+ *   │
+ *   ├─ Does the machine have idle cores?
+ *   │   └─ YES → Recommendation: SCALE UP (add more workers)
+ *   │
+ *   ├─ Is the MACHINE saturated (all cores busy)?
+ *   │   └─ YES → Recommendation: MIGRATE to a new machine
+ *   │            (generate deployment command + config change)
+ *   │
+ *   └─ Is this a latency problem, not a CPU problem?
+ *       └─ YES → Recommendation: check for slow I/O,
+ *                add connection pooling, or add caching
+ *
+ *   Service underutilized?
+ *   │
+ *   ├─ Has it been under 20% CPU for 30+ minutes?
+ *   │   └─ Recommendation: SCALE DOWN (reduce workers)
+ *   │
+ *   └─ Is it running alone and barely used?
+ *       └─ Recommendation: COLOCATE with other light services
+ */
+
+import { EventEmitter } from "node:events";
+import os from "node:os";
+
+/**
+ * @typedef {Object} ScaleRecommendation
+ * @property {string} service
+ * @property {'scale_up'|'split_out'|'migrate'|'scale_down'|'colocate'|'investigate'} action
+ * @property {string} reason
+ * @property {Object} details
+ * @property {number} confidence  - 0 to 1
+ * @property {number} timestamp
+ */
+
+export class ScaleAdvisor extends EventEmitter {
+  /**
+   * @param {import('../registry/ServiceRegistry.js').ServiceRegistry} registry
+   * @param {Object} [options]
+   * @param {number} [options.evaluationIntervalMs=30000]
+   * @param {Object} [options.thresholds]
+   */
+  constructor(registry, options = {}) {
+    super();
+
+    this.registry = registry;
+    this.evaluationIntervalMs = options.evaluationIntervalMs ?? 30000;
+
+    const ot = options.thresholds ?? {};
+    this.thresholds = {
+      cpu: { warning: 60, critical: 85, scaleDown: 40, ...ot.cpu },
+      latencyP99: { warning: 100, critical: 500, ...ot.latencyP99 },
+      pending: { warning: 50, critical: 200, ...ot.pending },
+      memory: { warning: 512, critical: 1024, ...ot.memory },
+      underutilized: { cpu: 20, duration: 30 * 60 * 1000, ...ot.underutilized },
+    };
+
+    this.cooldownMs = options.cooldownMs ?? 120000;
+
+    /** O3: Whether to auto-execute scaling recommendations */
+    this._autoExecute = options.scaling?.autoExecute === true;
+
+    /**
+     * Historical snapshots for trend detection.
+     * Key: service name, Value: array of health snapshots
+     * @type {Map<string, Array<{timestamp: number, health: Object}>>}
+     */
+    this.history = new Map();
+    this.maxHistory = 60; // keep 60 snapshots (30 min at 30s interval)
+
+    /** @type {ScaleRecommendation[]} */
+    this.recommendations = [];
+
+    /**
+     * Tracks the last scaling recommendation timestamp per service.
+     * @type {Map<string, number>}
+     */
+    this._lastScaleAction = new Map();
+
+    /** @type {NodeJS.Timeout|null} */
+    this._timer = null;
+
+    // Delta-based CPU measurement state
+    this._lastCpuMeasurement = null;
+    this._lastCpuUsage = 0;
+
+    /** O3: Executor function for auto-scaling, set via setExecutor() */
+    this._executor = null;
+
+    /** O11: Worker-reported CPU values, keyed by service name */
+    this._workerCpuReports = new Map();
+  }
+
+  /**
+   * O3: Set an executor function that actually performs scaling actions.
+   * Called by the Supervisor to provide a callback for auto-scaling.
+   *
+   * @param {function(ScaleRecommendation): Promise<void>} fn
+   */
+  setExecutor(fn) {
+    if (typeof fn !== 'function') {
+      throw new Error('ScaleAdvisor.setExecutor() requires a function');
+    }
+    this._executor = fn;
+  }
+
+  /**
+   * O11: Receive a CPU measurement reported by a worker via IPC.
+   * Workers should report their own CPU usage periodically.
+   *
+   * @param {string} serviceName
+   * @param {number} cpu - CPU percentage (0-100)
+   */
+  reportWorkerCpu(serviceName, cpu) {
+    this._workerCpuReports.set(serviceName, { cpu, timestamp: Date.now() });
+  }
+
+  start() {
+    this._timer = setInterval(() => {
+      this.evaluate();
+    }, this.evaluationIntervalMs);
+    this._timer.unref();
+
+    // Run initial evaluation
+    this.evaluate();
+    return this;
+  }
+
+  stop() {
+    if (this._timer) {
+      clearInterval(this._timer);
+      this._timer = null;
+    }
+  }
+
+  /**
+   * Evaluate all services and generate recommendations.
+   */
+  evaluate() {
+    const topology = this.registry.topology();
+    const newRecs = [];
+
+    for (const [serviceName, instances] of Object.entries(topology)) {
+      this._recordSnapshot(serviceName, instances);
+
+      for (const instance of instances) {
+        if (instance.nodeId !== this.registry.nodeId) continue;
+
+        const recs = this._evaluateService(serviceName, instance, instances, topology);
+        newRecs.push(...recs);
+      }
+    }
+
+    // Only emit if recommendations changed
+    const changed =
+      JSON.stringify(newRecs.map((r) => r.action + r.service)) !==
+      JSON.stringify(this.recommendations.map((r) => r.action + r.service));
+
+    this.recommendations = newRecs;
+
+    if (changed && newRecs.length > 0) {
+      this.emit("recommendations", newRecs);
+      for (const rec of newRecs) {
+        this.emit("recommendation", rec);
+
+        // O3: Auto-execute if configured and executor is available
+        if (this._executor && this._autoExecute) {
+          try {
+            this._executor(rec);
+          } catch (err) {
+            this.emit('error', err);
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Evaluate a single service instance.
+   */
+  _evaluateService(serviceName, instance, _allInstances, topology) {
+    const recs = [];
+    const health = instance;
+
+    // Check cooldown — suppress recommendations during cooldown period
+    const lastAction = this._lastScaleAction.get(serviceName) ?? 0;
+    if (Date.now() - lastAction < this.cooldownMs) {
+      return recs;
+    }
+
+    // O11: Use worker-reported CPU if available (more accurate than supervisor CPU).
+    // Workers report their own CPU via reportWorkerCpu(), which is aggregated here.
+    // Falls back to registry health data, then to topology data.
+    const workerReport = this._workerCpuReports.get(serviceName);
+    const workerCpuFresh = workerReport && (Date.now() - workerReport.timestamp < this.evaluationIntervalMs * 2);
+    const regHealth = this.registry.localRegistrations.get(serviceName)?.health;
+    const cpu = workerCpuFresh ? workerReport.cpu : (regHealth?.cpu ?? health.cpu);
+
+    // CPU check — use critical threshold for scale-up decisions
+    if (cpu >= this.thresholds.cpu.critical) {
+      const healthWithCpu = { ...health, cpu };
+      const rec = this._cpuRecommendation(serviceName, healthWithCpu, topology);
+      recs.push(rec);
+      this._lastScaleAction.set(serviceName, Date.now());
+    }
+
+    // Underutilization check — use scaleDown threshold (hysteresis)
+    // Only recommend scale_down when CPU is below the lower hysteresis threshold
+    if (cpu < this.thresholds.underutilized.cpu && cpu < this.thresholds.cpu.scaleDown) {
+      const duration = this._lowCpuDuration(serviceName);
+      if (duration >= this.thresholds.underutilized.duration) {
+        recs.push({
+          service: serviceName,
+          action: "scale_down",
+          reason: `CPU below ${this.thresholds.underutilized.cpu}% for ${Math.round(duration / 60000)} minutes`,
+          details: {
+            currentCpu: cpu,
+            currentWorkers: health.workers,
+            suggestedWorkers: Math.max(1, Math.ceil(health.workers / 2)),
+          },
+          confidence: 0.7,
+          timestamp: Date.now(),
+        });
+        this._lastScaleAction.set(serviceName, Date.now());
+      }
+    }
+
+    return recs;
+  }
+
+  _cpuRecommendation(serviceName, health, topology) {
+    const machineCpu = this._getMachineCpu();
+
+    // Is this service colocated (sharing a process group with other services)?
+    const localRegs = [...this.registry.localRegistrations.values()];
+    const thisGroup = localRegs.find((lr) => lr.name === serviceName)?.metadata?.group;
+    // Isolated services (_isolated:name) are not truly colocated
+    const isColocated = thisGroup && !thisGroup.startsWith('_isolated:') &&
+      localRegs.some((r) => r.name !== serviceName && r.metadata?.group === thisGroup);
+
+    if (isColocated) {
+      return {
+        service: serviceName,
+        action: "split_out",
+        reason: `CPU at ${health.cpu}% while colocated — resource contention likely`,
+        details: {
+          currentCpu: health.cpu,
+          suggestion: `Move ${serviceName} to its own process group`,
+          configChange: {
+            [serviceName]: { group: undefined }, // remove from colocation group
+          },
+        },
+        confidence: 0.85,
+        timestamp: Date.now(),
+      };
+    }
+
+    if (machineCpu < 80) {
+      // Machine has headroom — add workers
+      return {
+        service: serviceName,
+        action: "scale_up",
+        reason: `CPU at ${health.cpu}% but machine has idle capacity (${machineCpu}% total)`,
+        details: {
+          currentCpu: health.cpu,
+          machineCpu,
+          currentWorkers: health.workers,
+          suggestedWorkers: health.workers + Math.ceil(health.workers * 0.5),
+          command: `forge scale ${serviceName} ${health.workers + Math.ceil(health.workers * 0.5)}`,
+        },
+        confidence: 0.8,
+        timestamp: Date.now(),
+      };
+    }
+
+    // Machine is saturated — migrate
+    return {
+      service: serviceName,
+      action: "migrate",
+      reason: `CPU at ${health.cpu}% and machine at ${machineCpu}% — needs dedicated hardware`,
+      details: {
+        currentCpu: health.cpu,
+        machineCpu,
+        steps: [
+          `1. Provision a new machine`,
+          `2. Deploy: forge start --http ${this.registry.httpBasePort}`,
+          `3. Update config: ${serviceName}: { type: 'remote', address: 'http://<new-host>:${this.registry.httpBasePort}' }`,
+          `4. Restart this node — traffic routes automatically`,
+        ],
+        generatedConfig: {
+          [serviceName]: {
+            type: "remote",
+            address: `http://<new-host>:${this.registry.httpBasePort}`,
+          },
+        },
+      },
+      confidence: 0.9,
+      timestamp: Date.now(),
+    };
+  }
+
+  _recordSnapshot(serviceName, instances) {
+    if (!this.history.has(serviceName)) {
+      this.history.set(serviceName, []);
+    }
+
+    const snapshots = this.history.get(serviceName);
+    snapshots.push({
+      timestamp: Date.now(),
+      health: instances.map((i) => ({ cpu: i.cpu, status: i.status, workers: i.workers })),
+    });
+
+    // Trim history
+    while (snapshots.length > this.maxHistory) {
+      snapshots.shift();
+    }
+  }
+
+  _lowCpuDuration(serviceName) {
+    const snapshots = this.history.get(serviceName) ?? [];
+    if (snapshots.length === 0) return 0;
+
+    // Walk backwards to find how long CPU has been low
+    let duration = 0;
+    for (let i = snapshots.length - 1; i >= 0; i--) {
+      const allLow = snapshots[i].health.every((h) => h.cpu < this.thresholds.underutilized.cpu);
+      if (!allLow) break;
+      duration = Date.now() - snapshots[i].timestamp;
+    }
+
+    return duration;
+  }
+
+  _getMachineCpu() {
+    const now = Date.now();
+    const currentCpus = os.cpus().map(cpu => ({ ...cpu.times }));
+
+    if (!this._lastCpuMeasurement) {
+      this._lastCpuMeasurement = { timestamp: now, cpus: currentCpus };
+      return this._lastCpuUsage ?? 0;
+    }
+
+    const prev = this._lastCpuMeasurement.cpus;
+    let totalDelta = 0;
+    let idleDelta = 0;
+
+    for (let i = 0; i < currentCpus.length && i < prev.length; i++) {
+      const c = currentCpus[i];
+      const p = prev[i];
+      const delta = (c.user - p.user) + (c.nice - p.nice) + (c.sys - p.sys) +
+                    (c.idle - p.idle) + (c.irq - p.irq) + ((c.steal ?? 0) - (p.steal ?? 0));
+      totalDelta += delta;
+      idleDelta += (c.idle - p.idle);
+    }
+
+    this._lastCpuMeasurement = { timestamp: now, cpus: currentCpus };
+    const usage = totalDelta > 0 ? Math.round(100 - (100 * idleDelta / totalDelta)) : 0;
+    this._lastCpuUsage = usage;
+    return usage;
+  }
+
+  /**
+   * Get current recommendations as a formatted report.
+   */
+  report() {
+    if (this.recommendations.length === 0) {
+      return "  All services nominal. No scaling actions needed.";
+    }
+
+    let report = "";
+    for (const rec of this.recommendations) {
+      const icon =
+        {
+          scale_up: "\u2191",
+          scale_down: "\u2193",
+          split_out: "\u229E",
+          migrate: "\u2192",
+          colocate: "\u229F",
+          investigate: "?",
+        }[rec.action] ?? "\u2022";
+
+      report += `  ${icon} ${rec.service}: ${rec.action.toUpperCase()}\n`;
+      report += `    ${rec.reason}\n`;
+
+      if (rec.details.command) {
+        report += `    Run: ${rec.details.command}\n`;
+      }
+      if (rec.details.steps) {
+        for (const step of rec.details.steps) {
+          report += `    ${step}\n`;
+        }
+      }
+      report += "\n";
+    }
+
+    return report;
+  }
+}

package/src/services/Service.js

@@ -0,0 +1,195 @@
+import { autoRegisterRoutes, autoWireSubscriptions, handleProxyRequest, NOT_HANDLED } from "../decorators/ServiceProxy.js";
+
+/**
+ * Service Base Class v2
+ *
+ * Developers extend this to create ThreadForge services.
+ *
+ * NEW in v2:
+ *   - Auto-generated proxy clients for other services
+ *     (this.users.getUser(id) instead of this.request('users', {...}))
+ *   - @Expose, @Route, @Emit, @On decorators for contracts
+ *   - Plain JS fallback via static `contract` property
+ *   - Automatic HTTP route registration from @Route
+ *   - Automatic event subscription wiring from @On
+ *
+ * @example
+ * ```js
+ * import { Service, Expose, Route, Emit } from 'threadforge';
+ *
+ * export default class UserService extends Service {
+ *
+ *   @Expose()
+ *   async getUser(userId) {
+ *     return this.db.findUser(userId);
+ *   }
+ *
+ *   @Expose()
+ *   @Route('POST', '/users')
+ *   @Emit('user.created')
+ *   async createUser(data) {
+ *     const user = await this.db.insert(data);
+ *     await this.notifications.userCreated(user);  // auto-generated proxy
+ *     return user;
+ *   }
+ * }
+ * ```
+ *
+ * Plain JS equivalent (no decorators):
+ * ```js
+ * export default class UserService extends Service {
+ *   static contract = {
+ *     expose: ['getUser', 'createUser'],
+ *     routes: [
+ *       { method: 'POST', path: '/users', handler: 'createUser' },
+ *     ],
+ *     emits: { createUser: 'user.created' },
+ *   };
+ *
+ *   async getUser(userId) { ... }
+ *   async createUser(data) { ... }
+ * }
+ * ```
+ *
+ * **Route handler signatures — two approaches:**
+ *
+ * 1. Contract routes (`@Route` / `static contract.routes`):
+ *    Handler receives `(body, params, query)` and returns a value that is
+ *    auto-serialized as JSON (201 for POST, 200 otherwise).
+ *
+ * 2. Manual routes (registered in `onStart` via `ctx.router`):
+ *    Handler receives the raw `(req, res)` and must call `res.json()` or
+ *    `res.end()` itself.
+ *
+ * See `autoRegisterRoutes` in `ServiceProxy.js` for full details.
+ */
+export class Service {
+  constructor() {
+    /** @type {import('../core/ForgeContext.js').ForgeContext | null} */
+    this.ctx = null;
+
+    /**
+     * Auto-generated proxy clients for connected services.
+     * Populated by the framework during initialization.
+     *
+     * Usage: this.users.getUser('123')
+     *        this.notifications.sendAlert({ ... })
+     *
+     * @type {Object}
+     */
+    // Proxies are set as direct properties on the instance
+    // e.g., this.users, this.billing, this.notifications
+  }
+
+  // ── Lifecycle hooks (override in subclass) ──
+
+  /**
+   * Called when the service starts. Initialize resources here.
+   * Routes from @Route decorators are auto-registered BEFORE this is called,
+   * so you can add additional routes in onStart if needed.
+   */
+  async onStart(_ctx) {}
+
+  /**
+   * Called when a message arrives from another service.
+   * For proxy-style calls, this is handled automatically.
+   * Override for custom fire-and-forget message handling.
+   */
+  async onMessage(_from, _payload) {}
+
+  /**
+   * Called when a request arrives from another service.
+   *
+   * For proxy-style calls (@Expose methods), this is dispatched
+   * automatically — you don't need to implement a switch/case.
+   *
+   * Override only for custom non-proxy request handling.
+   */
+  async onRequest(from, payload) {
+    // Try proxy-style dispatch first
+    const result = handleProxyRequest(this, from, payload);
+    if (result !== NOT_HANDLED) return result;
+
+    // Subclass can handle other request formats
+    return null;
+  }
+
+  /**
+   * Called when the service is shutting down.
+   */
+  async onStop() {}
+
+  // ── Convenience methods ──
+
+  /**
+   * Send a fire-and-forget message to another service.
+   * Prefer using proxy clients (this.serviceName.method()) for
+   * request/response patterns.
+   */
+  async send(target, payload) {
+    if (!this.ctx) throw new Error("Service not initialized");
+    return this.ctx.send(target, payload);
+  }
+
+  /**
+   * Send a request to another service.
+   * Prefer using proxy clients instead of this low-level API.
+   */
+  async request(target, payload, timeoutMs) {
+    if (!this.ctx) throw new Error("Service not initialized");
+    return this.ctx.request(target, payload, timeoutMs);
+  }
+
+  /** Broadcast to all workers of a target service. */
+  async broadcast(target, payload) {
+    if (!this.ctx) throw new Error("Service not initialized");
+    return this.ctx.broadcast(target, payload);
+  }
+
+  // ── Internal lifecycle (called by worker bootstrap) ──
+
+  /** @internal */
+  async _init(ctx) {
+    this.ctx = ctx;
+
+    // Wire IPC handlers
+    ctx._onMessage = (from, payload) => this.onMessage(from, payload);
+    ctx._onRequest = (from, payload) => this.onRequest(from, payload);
+    ctx._wireMessageHandlers();
+
+    // Auto-register HTTP routes from @Route decorators
+    autoRegisterRoutes(this, ctx);
+
+    // Auto-wire event subscriptions from @On decorators
+    autoWireSubscriptions(this, ctx);
+  }
+
+  /**
+   * @internal
+   * Set proxy clients on the service instance.
+   * Called by the worker bootstrap after all services are loaded.
+   */
+  _setProxies(proxies) {
+    for (const [name, proxy] of Object.entries(proxies)) {
+      this[name] = proxy;
+    }
+  }
+
+  /** @internal */
+  async _start() {
+    await this.onStart(this.ctx);
+
+    if (this.ctx._needsHttpServer) {
+      await this.ctx.startServer();
+    }
+  }
+
+  /** @internal */
+  async _stop() {
+    try {
+      await this.onStop();
+    } finally {
+      await this.ctx.stop();
+    }
+  }
+}