wolverine-ai 1.0.0

package/src/monitor/process-monitor.js

@@ -0,0 +1,231 @@
+const { execSync } = require("child_process");
+const chalk = require("chalk");
+
+/**
+ * Process Monitor — heartbeat, memory leak detection, frozen state detection.
+ *
+ * Tracks the child process health:
+ * - Memory usage (RSS, heap) with leak detection
+ * - CPU usage percentage
+ * - Heartbeat (is the process alive?)
+ * - Frozen detection (process alive but not responding)
+ *
+ * Triggers restart when:
+ * - Memory exceeds threshold (configurable, default 512MB)
+ * - Memory growing consistently for N samples (leak detection)
+ * - Process not responding to health checks (frozen)
+ */
+
+class ProcessMonitor {
+  constructor(options = {}) {
+    this.logger = options.logger;
+    this.pid = null;
+
+    // Thresholds
+    this.maxMemoryMB = options.maxMemoryMB || parseInt(process.env.WOLVERINE_MAX_MEMORY_MB, 10) || 512;
+    this.leakSamples = options.leakSamples || 10;   // consecutive growing samples = leak
+    this.sampleIntervalMs = options.sampleIntervalMs || 10000; // sample every 10s
+
+    // State
+    this._samples = [];        // { timestamp, rss, heap, cpu }
+    this._timer = null;
+    this._running = false;
+    this._onRestart = null;
+    this._consecutiveGrowth = 0;
+
+    // Analytics aggregates
+    this._peakMemory = 0;
+    this._avgMemory = 0;
+    this._sampleCount = 0;
+    this._cpuSamples = [];
+    this._lastCpuUsage = null;
+    this._lastCpuTime = null;
+  }
+
+  /**
+   * Start monitoring a process.
+   * @param {number} pid — the child process PID
+   * @param {function} onRestart — callback when restart is needed
+   */
+  start(pid, onRestart) {
+    this.pid = pid;
+    this._onRestart = onRestart;
+    this._running = true;
+    this._consecutiveGrowth = 0;
+    this._samples = [];
+    this._lastCpuUsage = null;
+    this._lastCpuTime = null;
+
+    this._timer = setInterval(() => this._sample(), this.sampleIntervalMs);
+  }
+
+  stop() {
+    this._running = false;
+    if (this._timer) {
+      clearInterval(this._timer);
+      this._timer = null;
+    }
+  }
+
+  /**
+   * Reset for new process (after restart).
+   */
+  reset(newPid) {
+    this.pid = newPid;
+    this._consecutiveGrowth = 0;
+    this._samples = [];
+    this._lastCpuUsage = null;
+    this._lastCpuTime = null;
+  }
+
+  /**
+   * Get current analytics snapshot.
+   */
+  getMetrics() {
+    const latest = this._samples.length > 0 ? this._samples[this._samples.length - 1] : null;
+    const avgCpu = this._cpuSamples.length > 0
+      ? Math.round(this._cpuSamples.reduce((a, b) => a + b, 0) / this._cpuSamples.length)
+      : 0;
+
+    return {
+      pid: this.pid,
+      alive: this._isAlive(),
+      current: latest ? {
+        rss: latest.rss,
+        heap: latest.heap,
+        cpu: latest.cpu,
+      } : null,
+      peak: {
+        memory: this._peakMemory,
+      },
+      average: {
+        memory: this._sampleCount > 0 ? Math.round(this._avgMemory / this._sampleCount) : 0,
+        cpu: avgCpu,
+      },
+      leakDetection: {
+        consecutiveGrowth: this._consecutiveGrowth,
+        threshold: this.leakSamples,
+        warning: this._consecutiveGrowth >= Math.floor(this.leakSamples * 0.7),
+      },
+      samples: this._samples.slice(-30).map(s => ({
+        t: s.timestamp,
+        rss: s.rss,
+        heap: s.heap,
+        cpu: s.cpu,
+      })),
+    };
+  }
+
+  // -- Private --
+
+  _sample() {
+    if (!this._running || !this.pid) return;
+
+    if (!this._isAlive()) {
+      // Process died — runner will handle this via exit event
+      return;
+    }
+
+    try {
+      // Get memory from process
+      const memInfo = this._getMemory();
+      const cpuPercent = this._getCpu();
+
+      const sample = {
+        timestamp: Date.now(),
+        rss: memInfo.rss,
+        heap: memInfo.heap,
+        cpu: cpuPercent,
+      };
+
+      this._samples.push(sample);
+      if (this._samples.length > 100) this._samples.shift();
+
+      // Track peaks/averages
+      if (memInfo.rss > this._peakMemory) this._peakMemory = memInfo.rss;
+      this._avgMemory += memInfo.rss;
+      this._sampleCount++;
+
+      this._cpuSamples.push(cpuPercent);
+      if (this._cpuSamples.length > 60) this._cpuSamples.shift();
+
+      // Check memory threshold
+      if (memInfo.rss > this.maxMemoryMB) {
+        console.log(chalk.red(`  🚨 Memory limit exceeded: ${memInfo.rss}MB > ${this.maxMemoryMB}MB`));
+        if (this.logger) {
+          this.logger.error("process.memory_exceeded", `Memory ${memInfo.rss}MB exceeds ${this.maxMemoryMB}MB`, sample);
+        }
+        if (this._onRestart) this._onRestart("memory_exceeded");
+        return;
+      }
+
+      // Leak detection: memory growing for N consecutive samples
+      if (this._samples.length >= 2) {
+        const prev = this._samples[this._samples.length - 2];
+        if (sample.rss > prev.rss) {
+          this._consecutiveGrowth++;
+        } else {
+          this._consecutiveGrowth = 0;
+        }
+
+        if (this._consecutiveGrowth >= this.leakSamples) {
+          const growth = sample.rss - this._samples[this._samples.length - this.leakSamples].rss;
+          console.log(chalk.yellow(`  ⚠️  Memory leak detected: +${growth}MB over ${this.leakSamples} samples`));
+          if (this.logger) {
+            this.logger.warn("process.memory_leak", `Memory leak: +${growth}MB over ${this.leakSamples} samples`, { growth, rss: sample.rss });
+          }
+          if (this._onRestart) this._onRestart("memory_leak");
+          this._consecutiveGrowth = 0;
+          return;
+        }
+      }
+    } catch {}
+  }
+
+  _getMemory() {
+    try {
+      // On Windows, use tasklist; on Unix, use /proc
+      if (process.platform === "win32") {
+        const output = execSync(`tasklist /FI "PID eq ${this.pid}" /FO CSV /NH`, { encoding: "utf-8", timeout: 3000 });
+        const match = output.match(/"(\d[\d,]+)\s*K"/);
+        if (match) {
+          const kb = parseInt(match[1].replace(/,/g, ""), 10);
+          return { rss: Math.round(kb / 1024), heap: 0 };
+        }
+      } else {
+        const output = execSync(`ps -o rss= -p ${this.pid}`, { encoding: "utf-8", timeout: 3000 });
+        const kb = parseInt(output.trim(), 10);
+        return { rss: Math.round(kb / 1024), heap: 0 };
+      }
+    } catch {}
+    return { rss: 0, heap: 0 };
+  }
+
+  _getCpu() {
+    try {
+      if (process.platform === "win32") {
+        const output = execSync(
+          `wmic path Win32_PerfFormattedData_PerfProc_Process where "IDProcess=${this.pid}" get PercentProcessorTime /VALUE`,
+          { encoding: "utf-8", timeout: 3000 }
+        );
+        const match = output.match(/PercentProcessorTime=(\d+)/);
+        return match ? parseInt(match[1], 10) : 0;
+      } else {
+        const output = execSync(`ps -o %cpu= -p ${this.pid}`, { encoding: "utf-8", timeout: 3000 });
+        return Math.round(parseFloat(output.trim()));
+      }
+    } catch {}
+    return 0;
+  }
+
+  _isAlive() {
+    try {
+      process.kill(this.pid, 0); // signal 0 = check if alive
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}
+
+module.exports = { ProcessMonitor };

package/src/monitor/route-prober.js

@@ -0,0 +1,191 @@
+const http = require("http");
+const chalk = require("chalk");
+
+/**
+ * Route Prober — discovers and tests ALL server routes periodically.
+ *
+ * Instead of only checking /health, probes every route discovered
+ * in the function map. Tracks response times per endpoint over time.
+ *
+ * Adapts automatically: when the function map updates (new routes added),
+ * the prober picks them up on the next cycle.
+ */
+
+class RouteProber {
+  constructor(options = {}) {
+    this.port = options.port || parseInt(process.env.PORT, 10) || 3000;
+    this.logger = options.logger;
+    this.brain = options.brain;
+    this.intervalMs = options.intervalMs || 30000; // probe every 30s
+
+    // Per-route analytics
+    this._routeMetrics = {}; // path → { samples[], avg, min, max, errors, lastStatus }
+    this._timer = null;
+    this._running = false;
+  }
+
+  start() {
+    this._running = true;
+    // First probe after a delay to let server boot
+    setTimeout(() => {
+      this._probe();
+      this._timer = setInterval(() => this._probe(), this.intervalMs);
+    }, 15000);
+  }
+
+  stop() {
+    this._running = false;
+    if (this._timer) {
+      clearInterval(this._timer);
+      this._timer = null;
+    }
+  }
+
+  /**
+   * Get analytics for all probed routes.
+   */
+  getMetrics() {
+    const result = {};
+    for (const [path, m] of Object.entries(this._routeMetrics)) {
+      const samples = m.samples;
+      const avg = samples.length > 0 ? Math.round(samples.reduce((a, b) => a + b, 0) / samples.length) : 0;
+      result[path] = {
+        avgMs: avg,
+        minMs: m.min,
+        maxMs: m.max,
+        samples: samples.length,
+        errors: m.errors,
+        lastStatus: m.lastStatus,
+        lastProbe: m.lastProbe,
+        healthy: m.lastStatus >= 200 && m.lastStatus < 400,
+        // Response time trend (last 5 vs overall)
+        trend: this._calcTrend(samples),
+      };
+    }
+    return result;
+  }
+
+  /**
+   * Get a summary suitable for the dashboard.
+   */
+  getSummary() {
+    const metrics = this.getMetrics();
+    const routes = Object.keys(metrics);
+    const healthy = routes.filter(r => metrics[r].healthy).length;
+    const unhealthy = routes.filter(r => !metrics[r].healthy).length;
+    const slowest = routes.sort((a, b) => (metrics[b].avgMs || 0) - (metrics[a].avgMs || 0))[0];
+
+    return {
+      totalRoutes: routes.length,
+      healthy,
+      unhealthy,
+      slowest: slowest ? { path: slowest, avgMs: metrics[slowest].avgMs } : null,
+    };
+  }
+
+  async _probe() {
+    if (!this._running) return;
+
+    // Get current routes from brain's function map
+    let routes = [];
+    if (this.brain && this.brain.functionMap) {
+      routes = (this.brain.functionMap.routes || [])
+        .filter(r => r.method === "GET" || r.method === "*")
+        .map(r => r.path);
+    }
+
+    // Always include root and health
+    if (!routes.includes("/")) routes.unshift("/");
+    if (!routes.includes("/health")) routes.push("/health");
+
+    // Deduplicate
+    routes = [...new Set(routes)];
+
+    for (const routePath of routes) {
+      await this._probeRoute(routePath);
+    }
+  }
+
+  _probeRoute(routePath) {
+    return new Promise((resolve) => {
+      const startTime = Date.now();
+
+      const req = http.get({
+        hostname: "127.0.0.1",
+        port: this.port,
+        path: routePath,
+        timeout: 5000,
+      }, (res) => {
+        const responseTime = Date.now() - startTime;
+        let body = "";
+        res.on("data", (d) => { body += d; });
+        res.on("end", () => {
+          this._record(routePath, responseTime, res.statusCode);
+          resolve();
+        });
+      });
+
+      req.on("timeout", () => {
+        req.destroy();
+        const responseTime = Date.now() - startTime;
+        this._record(routePath, responseTime, 0);
+        resolve();
+      });
+
+      req.on("error", () => {
+        const responseTime = Date.now() - startTime;
+        this._record(routePath, responseTime, 0);
+        resolve();
+      });
+    });
+  }
+
+  _record(routePath, responseTime, statusCode) {
+    if (!this._routeMetrics[routePath]) {
+      this._routeMetrics[routePath] = {
+        samples: [],
+        min: Infinity,
+        max: 0,
+        errors: 0,
+        lastStatus: 0,
+        lastProbe: 0,
+      };
+    }
+
+    const m = this._routeMetrics[routePath];
+    m.samples.push(responseTime);
+    if (m.samples.length > 60) m.samples.shift(); // keep last 60 samples
+    m.min = Math.min(m.min, responseTime);
+    m.max = Math.max(m.max, responseTime);
+    m.lastStatus = statusCode;
+    m.lastProbe = Date.now();
+
+    if (statusCode === 0 || statusCode >= 500) {
+      m.errors++;
+    }
+
+    // Log slow routes
+    if (responseTime > 2000) {
+      console.log(chalk.yellow(`  ⚡ Slow route: ${routePath} took ${responseTime}ms`));
+      if (this.logger) {
+        this.logger.warn("perf.slow_route", `${routePath}: ${responseTime}ms`, { path: routePath, ms: responseTime, status: statusCode });
+      }
+    }
+  }
+
+  _calcTrend(samples) {
+    if (samples.length < 6) return "stable";
+    const recent = samples.slice(-5);
+    const older = samples.slice(-10, -5);
+    if (older.length === 0) return "stable";
+
+    const recentAvg = recent.reduce((a, b) => a + b, 0) / recent.length;
+    const olderAvg = older.reduce((a, b) => a + b, 0) / older.length;
+
+    if (recentAvg > olderAvg * 1.5) return "degrading";
+    if (recentAvg < olderAvg * 0.7) return "improving";
+    return "stable";
+  }
+}
+
+module.exports = { RouteProber };

package/src/notifications/notifier.js

@@ -0,0 +1,227 @@
+const chalk = require("chalk");
+const { aiCall } = require("../core/ai-client");
+const { getModel } = require("../core/models");
+
+/**
+ * Notification System — alerts for issues the AI cannot fix.
+ *
+ * Some errors require human intervention:
+ * - API keys expired/rotated/revoked
+ * - Billing/quota exceeded
+ * - External service down (database, third-party API)
+ * - Permission denied (file system, network)
+ * - Certificate errors
+ * - Environment misconfiguration
+ *
+ * The notifier:
+ * 1. Classifies errors as AI-fixable vs human-required
+ * 2. Summarizes human-required issues into short, clear sentences
+ * 3. Sanitizes through the redactor (no secrets in notifications)
+ * 4. Delivers via console, dashboard events, and optional webhook
+ */
+
+// Patterns that indicate human-required intervention
+const HUMAN_REQUIRED_PATTERNS = [
+  // API/Auth failures
+  { pattern: /401\s*unauthorized/i, category: "auth", hint: "API key or credentials may be invalid or expired" },
+  { pattern: /403\s*forbidden/i, category: "auth", hint: "Access denied — check permissions or API key scope" },
+  { pattern: /invalid.*(api|auth|token|key|credential)/i, category: "auth", hint: "Authentication credential is invalid" },
+  { pattern: /(api|auth|token|key|credential).*(expired|revoked|rotated|invalid)/i, category: "auth", hint: "Credential has expired or been revoked" },
+  { pattern: /authentication\s+failed/i, category: "auth", hint: "Authentication failed — check credentials" },
+
+  // Billing/Quota
+  { pattern: /429\s*(too many|rate limit)/i, category: "billing", hint: "Rate limit hit — may need to upgrade plan or wait" },
+  { pattern: /(quota|limit|credits?)\s*(exceeded|exhausted|depleted)/i, category: "billing", hint: "Usage quota or credits exhausted" },
+  { pattern: /billing.*(?:issue|error|failed|inactive)/i, category: "billing", hint: "Billing issue on the account" },
+  { pattern: /insufficient.*(funds|credits|quota)/i, category: "billing", hint: "Insufficient credits or funds" },
+
+  // External service failures
+  { pattern: /ECONNREFUSED/i, category: "service", hint: "External service connection refused — is it running?" },
+  { pattern: /ENOTFOUND/i, category: "service", hint: "DNS lookup failed — check hostname or network" },
+  { pattern: /ETIMEDOUT/i, category: "service", hint: "Connection timed out — service may be down" },
+  { pattern: /ECONNRESET/i, category: "service", hint: "Connection reset by remote server" },
+  { pattern: /503\s*service\s*unavailable/i, category: "service", hint: "External service is temporarily unavailable" },
+  { pattern: /502\s*bad\s*gateway/i, category: "service", hint: "Bad gateway — upstream service may be down" },
+
+  // Certificates
+  { pattern: /CERT_|certificate|SSL|TLS/i, category: "cert", hint: "SSL/TLS certificate issue — may need renewal" },
+  { pattern: /self.signed/i, category: "cert", hint: "Self-signed certificate rejected" },
+
+  // Permissions
+  { pattern: /EACCES/i, category: "permission", hint: "Permission denied on file system" },
+  { pattern: /EPERM/i, category: "permission", hint: "Operation not permitted — check file/process permissions" },
+
+  // Environment
+  { pattern: /not\s+set|undefined.*env|missing.*env/i, category: "env", hint: "Environment variable not configured" },
+  { pattern: /missing.*config/i, category: "env", hint: "Configuration file or value missing" },
+
+  // Disk
+  { pattern: /ENOSPC/i, category: "disk", hint: "Disk space full" },
+  { pattern: /ENOMEM/i, category: "disk", hint: "Out of memory" },
+];
+
+const CATEGORY_ICONS = {
+  auth: "🔑",
+  billing: "💳",
+  service: "🌐",
+  cert: "📜",
+  permission: "🔒",
+  env: "⚙️",
+  disk: "💾",
+};
+
+class Notifier {
+  constructor(options = {}) {
+    this.logger = options.logger;
+    this.redactor = options.redactor;
+    this.webhookUrl = options.webhookUrl || process.env.WOLVERINE_WEBHOOK_URL || null;
+
+    // Dedup: don't spam the same notification repeatedly
+    this._sentNotifications = new Map(); // key → timestamp
+    this._dedupeWindowMs = 300000; // 5 minutes
+  }
+
+  /**
+   * Classify an error: can AI fix it, or does it need human intervention?
+   * Returns { humanRequired: boolean, category?, hint?, matches? }
+   */
+  classify(errorMessage, stackTrace) {
+    const combined = `${errorMessage}\n${stackTrace || ""}`;
+    const matches = [];
+
+    for (const { pattern, category, hint } of HUMAN_REQUIRED_PATTERNS) {
+      if (pattern.test(combined)) {
+        matches.push({ category, hint });
+      }
+    }
+
+    if (matches.length === 0) {
+      return { humanRequired: false };
+    }
+
+    // Use the most specific match (first one)
+    return {
+      humanRequired: true,
+      category: matches[0].category,
+      hint: matches[0].hint,
+      matches,
+    };
+  }
+
+  /**
+   * Process an error — classify it, and if human-required, generate and send a notification.
+   * Returns the notification object if sent, null if AI-fixable.
+   */
+  async notify(errorMessage, stackTrace) {
+    const classification = this.classify(errorMessage, stackTrace);
+    if (!classification.humanRequired) return null;
+
+    // Dedup check
+    const dedupeKey = `${classification.category}:${classification.hint}`;
+    const lastSent = this._sentNotifications.get(dedupeKey);
+    if (lastSent && Date.now() - lastSent < this._dedupeWindowMs) {
+      return null; // already notified recently
+    }
+
+    // Sanitize through redactor
+    const safeError = this.redactor ? this.redactor.redact(errorMessage) : errorMessage;
+    const safeStack = this.redactor ? this.redactor.redact(stackTrace || "") : (stackTrace || "");
+
+    // Generate short summary using AI
+    let summary;
+    try {
+      summary = await this._summarize(safeError, safeStack, classification);
+    } catch {
+      // If AI call fails (maybe THAT's the auth error), use the pattern hint
+      summary = classification.hint;
+    }
+
+    const notification = {
+      category: classification.category,
+      icon: CATEGORY_ICONS[classification.category] || "⚠️",
+      summary,
+      hint: classification.hint,
+      timestamp: Date.now(),
+      iso: new Date().toISOString(),
+    };
+
+    // Mark as sent for dedup
+    this._sentNotifications.set(dedupeKey, Date.now());
+
+    // Deliver through all channels
+    this._deliverConsole(notification);
+    this._deliverLogger(notification);
+    this._deliverWebhook(notification);
+
+    return notification;
+  }
+
+  /**
+   * Use CHAT_MODEL to summarize the issue in 1-2 short sentences.
+   */
+  async _summarize(safeError, safeStack, classification) {
+    let model;
+    try {
+      model = getModel("chat");
+    } catch {
+      return classification.hint;
+    }
+
+    const result = await aiCall({
+      model,
+      systemPrompt: "You summarize server errors for developers. Write 1-2 short sentences. Be direct and actionable. Do not include any secrets, passwords, or API key values — only refer to them by name (e.g. 'the OPENAI_API_KEY').",
+      userPrompt: `Summarize this error for a developer notification:\n\nCategory: ${classification.category}\nError: ${safeError}\n\nStack (first 300 chars): ${safeStack.slice(0, 300)}`,
+      maxTokens: 100,
+      category: "security",
+    });
+
+    // Double-sanitize the AI response (in case the AI echoes something)
+    const summary = this.redactor ? this.redactor.redact(result.content) : result.content;
+    return summary || classification.hint;
+  }
+
+  _deliverConsole(notification) {
+    console.log(chalk.red.bold(`\n  ${notification.icon} HUMAN ACTION REQUIRED`));
+    console.log(chalk.red(`  Category: ${notification.category}`));
+    console.log(chalk.yellow(`  ${notification.summary}`));
+    console.log(chalk.gray(`  Hint: ${notification.hint}\n`));
+  }
+
+  _deliverLogger(notification) {
+    if (!this.logger) return;
+    this.logger.critical("notify.human_required", notification.summary, {
+      category: notification.category,
+      hint: notification.hint,
+    });
+  }
+
+  async _deliverWebhook(notification) {
+    if (!this.webhookUrl) return;
+
+    try {
+      const http = this.webhookUrl.startsWith("https") ? require("https") : require("http");
+      const url = new URL(this.webhookUrl);
+      const payload = JSON.stringify({
+        text: `${notification.icon} **Wolverine Alert** [${notification.category}]: ${notification.summary}`,
+        ...notification,
+      });
+
+      const req = http.request({
+        hostname: url.hostname,
+        port: url.port,
+        path: url.pathname + url.search,
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(payload) },
+        timeout: 5000,
+      });
+
+      req.on("error", () => {}); // swallow webhook errors
+      req.write(payload);
+      req.end();
+    } catch {
+      // Webhook delivery is best-effort
+    }
+  }
+}
+
+module.exports = { Notifier, HUMAN_REQUIRED_PATTERNS, CATEGORY_ICONS };