total-recall-brain 3.18.2 → 3.18.3

package/src/core/webhook-handlers.mjs

@@ -1,19 +1,128 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { buildTaskEnvelope } from './task-envelope.mjs';
+import { persistTaskToDisk } from './scheduler.mjs';
+import { brainDir } from './config.mjs';
 import { logger } from './logger.mjs';
 
+const QUEUE_DIR = path.join(brainDir, 'scheduler', 'queue');
+
+/** Fixed slug so re-pushes overwrite one card instead of flooding the queue. */
+const DEPLOY_SLUG = 'deploy-auto';
+const SKILL_SYNC_SLUG = 'skillsync-auto';
+const NPM_UPDATE_SLUG = 'npm-update-auto';
+
+/**
+ * True when a queue file already exists for this slug (pending/in-progress).
+ * Avoids N× "Auto-deploy" cards for every GitHub push delivery / multi-ref push.
+ */
+function hasPendingTask(slug) {
+  try {
+    return fs.existsSync(path.join(QUEUE_DIR, `${slug}.md`));
+  } catch {
+    return false;
+  }
+}
+
 /**
- * Verified webhooks are recorded as SSSS events by the route. They do not run
- * shell commands or deployments. Product-specific automation must subscribe to
- * those events through an explicitly reviewed workflow with its own gates.
+ * GitHub may fire multiple push webhooks (multi-repo, multi-ref, redeliveries).
+ * Only production/main refs matter for deploy notices; always coalesce to one task.
  */
-export async function handleWebhook(provider, eventType, _payload) {
-  const known =
-    (provider === 'github' && ['push', 'release'].includes(eventType)) ||
-    (provider === 'stripe' && typeof eventType === 'string' && eventType.includes('.'));
-
-  logger.info('webhooks', `Recorded ${provider}.${eventType} webhook`, {
-    provider,
-    eventType,
-    automation: 'disabled',
-  });
-  return { handled: known, action: known ? 'recorded' : 'ignored' };
+function pushRefIsDeployable(payload = {}) {
+  const ref = String(payload.ref || '');
+  if (!ref) return true; // tests / bare payload still queue once
+  return (
+    ref === 'refs/heads/production' ||
+    ref === 'refs/heads/main' ||
+    ref.endsWith('/production') ||
+    ref.endsWith('/main')
+  );
+}
+
+export async function handleWebhook(provider, eventType, payload = {}) {
+  logger.info({ subsystem: 'webhooks', message: `Handling ${provider}.${eventType} webhook` });
+
+  if (provider === 'github' && eventType === 'push') {
+    if (!pushRefIsDeployable(payload)) {
+      logger.info({
+        subsystem: 'webhooks',
+        message: `Skipping deploy queue for non-deploy ref ${payload.ref || '(none)'}`,
+      });
+      return { queued: false, reason: 'ref_not_deployable' };
+    }
+    if (hasPendingTask(DEPLOY_SLUG)) {
+      logger.info({
+        subsystem: 'webhooks',
+        message: 'Deploy task already pending — not queuing another (coalesce)',
+      });
+      return { queued: false, reason: 'already_pending', slug: DEPLOY_SLUG };
+    }
+    // Notify-only card: real production deploy is the droplet watcher on push to production.
+    // Do NOT run bin/deploy.sh here — that caused multi-deploy spam and wrong deploy path.
+    const env = buildTaskEnvelope({
+      intent: 'Auto-deploy triggered by push',
+      slug: DEPLOY_SLUG,
+      kind: 'system',
+      executor: 'command',
+      category: 'deployment',
+      priority: 80,
+      system: true,
+      origin: { agent: 'daemon', created_by: 'webhook-github' },
+      capabilities: ['vault:read'],
+      payload: {
+        command: 'echo "Deploy notice only — production droplet auto-deploy handles the real ship"',
+        ref: payload.ref || null,
+        after: payload.after || null,
+        repository: payload.repository?.full_name || null,
+      },
+    });
+    persistTaskToDisk(env, QUEUE_DIR);
+    logger.info({ subsystem: 'webhooks', message: 'Queued single coalesced deploy notice task' });
+    return { queued: true, slug: DEPLOY_SLUG };
+  }
+
+  if (provider === 'github' && eventType === 'release') {
+    if (hasPendingTask(SKILL_SYNC_SLUG)) {
+      return { queued: false, reason: 'already_pending', slug: SKILL_SYNC_SLUG };
+    }
+    const env = buildTaskEnvelope({
+      intent: 'Skill sync triggered by release',
+      slug: SKILL_SYNC_SLUG,
+      kind: 'system',
+      executor: 'command',
+      category: 'sync',
+      priority: 70,
+      system: true,
+      origin: { agent: 'daemon', created_by: 'webhook-github' },
+      capabilities: ['vault:read'],
+      payload: { command: 'npx total-recall skill sync' },
+    });
+    persistTaskToDisk(env, QUEUE_DIR);
+    logger.info({ subsystem: 'webhooks', message: 'Queued skill-sync task' });
+    return { queued: true, slug: SKILL_SYNC_SLUG };
+  }
+
+  if (provider === 'npm' && eventType === 'package-publish') {
+    if (hasPendingTask(NPM_UPDATE_SLUG)) {
+      return { queued: false, reason: 'already_pending', slug: NPM_UPDATE_SLUG };
+    }
+    const env = buildTaskEnvelope({
+      intent: 'Package update triggered by npm publish',
+      slug: NPM_UPDATE_SLUG,
+      kind: 'system',
+      executor: 'command',
+      category: 'package',
+      priority: 60,
+      system: true,
+      origin: { agent: 'daemon', created_by: 'webhook-npm' },
+      capabilities: ['vault:read'],
+      payload: { command: 'npm update' },
+    });
+    persistTaskToDisk(env, QUEUE_DIR);
+    logger.info({ subsystem: 'webhooks', message: 'Queued npm update task' });
+    return { queued: true, slug: NPM_UPDATE_SLUG };
+  }
+
+  logger.info({ subsystem: 'webhooks', message: `No specific handler for ${provider}.${eventType}` });
+  return { queued: false, reason: 'no_handler' };
 }

package/src/server/auth.mjs

@@ -246,6 +246,12 @@ export function requireAuthOrLocal(req, res, next) {
   const config = loadSecurityConfig();
   if (config.network?.public_health === true) return next();
   if (isLocalRequest(req)) return next();
+  // Mesh peers probe /health for latency + LAN discovery. Traffic is already on
+  // WireGuard; treat mesh socket addresses like local for this gate only.
+  // Never trust X-Forwarded-For for this exemption (same as requireHttps mesh rule).
+  if (isMeshIp(req.socket?.remoteAddress) && forwardedClientIps(req).length === 0) {
+    return next();
+  }
   return requireAuth(req, res, next);
 }
 

package/src/server/rest.mjs

@@ -120,6 +120,7 @@ import scriptsRouter      from './routes/scripts.mjs';
 import ttsRouter          from './routes/tts.mjs';
 import updateRouter       from './routes/update.mjs';
 import { rulesRouter }    from './routes/rules.mjs';
+import notificationsRouter from './routes/notifications.mjs';
 // ollamaUrl removed — CLI agents replace Ollama
 import {
   AGENT_DIR,
@@ -233,6 +234,7 @@ router.use(ttsRouter);
 router.use(updateRouter);
 router.use(ssssRouter);
 router.use(rulesRouter);
+router.use(notificationsRouter);
 
 export { router as restRouter };
 export default router;

package/src/server/route-manifest.json

@@ -327,6 +327,26 @@
     "method": "GET",
     "path": "/api/network/stats"
   },
+  {
+    "method": "GET",
+    "path": "/api/notifications/history"
+  },
+  {
+    "method": "GET",
+    "path": "/api/notifications/rules"
+  },
+  {
+    "method": "POST",
+    "path": "/api/notifications/rules"
+  },
+  {
+    "method": "DELETE",
+    "path": "/api/notifications/rules/:id"
+  },
+  {
+    "method": "POST",
+    "path": "/api/notifications/test"
+  },
   {
     "method": "GET",
     "path": "/api/openai-models"
@@ -335,6 +355,10 @@
     "method": "GET",
     "path": "/api/openrouter-models"
   },
+  {
+    "method": "GET",
+    "path": "/api/pairing"
+  },
   {
     "method": "GET",
     "path": "/api/research"

package/src/server/routes/config.mjs

@@ -68,7 +68,7 @@ router.get('/api/config', requireAuth, requireScope('config:read'), async (req,
  * GET /api/config-json
  * Returns full config bundle: security, budget, brain, and allowed secrets keys.
  */
-router.get('/api/config-json', requireAuth, requireScope('config:read'), (req, res) => {
+router.get('/api/config-json', requireAuth, requireScope('config:read'), async (req, res) => {
   try {
     const securityPath = path.join(CONFIG_DIR, 'security.yml');
     const budgetPath   = path.join(CONFIG_DIR, 'budget.yml');
@@ -89,8 +89,40 @@ router.get('/api/config-json', requireAuth, requireScope('config:read'), (req, r
     if (fs.existsSync(brainPath)) {
       try { brain = JSON.parse(fs.readFileSync(brainPath, 'utf8')) || {}; } catch {}
     }
-    if (fs.existsSync(secretsPath)) {
-      try { secrets = JSON.parse(fs.readFileSync(secretsPath, 'utf8')) || {}; } catch {}
+    // Prefer brain secrets store (AES). Never return raw secret values to the dashboard.
+    const allowedKeys = [
+      'google_api_key',
+      'anthropic_api_key',
+      'openai_api_key',
+      'openrouter_api_key',
+      'tavily_api_key',
+      'brave_api_key',
+      'exa_api_key',
+      'serper_api_key',
+      'github_token',
+    ];
+    const safeSecrets = {};
+    try {
+      const { listSecretsMeta } = await import('../../core/secrets-store.mjs');
+      const meta = await listSecretsMeta(BRAIN_DIR);
+      for (const row of meta) {
+        if (!allowedKeys.includes(row.key)) continue;
+        // Mask only — Settings/Secrets UIs use presence indicators, not values.
+        safeSecrets[row.key] = row.set ? '••••••••' : '';
+      }
+    } catch {
+      if (fs.existsSync(secretsPath)) {
+        try {
+          secrets = JSON.parse(fs.readFileSync(secretsPath, 'utf8')) || {};
+        } catch {
+          /* ignore AES/binary */
+        }
+        for (const key of allowedKeys) {
+          if (secrets[key] !== undefined && secrets[key] !== null && secrets[key] !== '') {
+            safeSecrets[key] = '••••••••';
+          }
+        }
+      }
     }
 
     const safeBrain = { ...brain };
@@ -99,14 +131,6 @@ router.get('/api/config-json', requireAuth, requireScope('config:read'), (req, r
       delete safeBrain.token;
     }
 
-    const allowedKeys = ['google_api_key', 'anthropic_api_key', 'openai_api_key', 'openrouter_api_key', 'tavily_api_key', 'brave_api_key', 'exa_api_key', 'serper_api_key', 'github_token'];
-    const safeSecrets = {};
-    for (const key of allowedKeys) {
-      if (secrets[key] !== undefined) {
-        safeSecrets[key] = secrets[key];
-      }
-    }
-
     res.json({ security, budget, brain: safeBrain, secrets: safeSecrets });
   } catch (err) { serverError(res, err); }
 });
@@ -146,22 +170,39 @@ router.post('/api/config-json', requireAuth, requireScope('config:write'), async
       fs.writeFileSync(brainPath, JSON.stringify(nextBrain, null, 2), { encoding: 'utf8', mode: 0o600 });
     }
     if (secrets) {
-      let existingSecrets = {};
-      if (fs.existsSync(secretsPath)) {
-        try { existingSecrets = JSON.parse(fs.readFileSync(secretsPath, 'utf8')) || {}; } catch {}
-      }
-      const allowedKeys = ['google_api_key', 'anthropic_api_key', 'openai_api_key', 'openrouter_api_key', 'tavily_api_key', 'brave_api_key', 'exa_api_key', 'serper_api_key', 'github_token'];
+      // Only accept real secret values — ignore masked placeholders from GET /api/config-json.
+      const allowedKeys = [
+        'google_api_key',
+        'anthropic_api_key',
+        'openai_api_key',
+        'openrouter_api_key',
+        'tavily_api_key',
+        'brave_api_key',
+        'exa_api_key',
+        'serper_api_key',
+        'github_token',
+      ];
+      const { setSecret, deleteSecret } = await import('../../core/secrets-store.mjs');
       for (const key of allowedKeys) {
-        if (secrets[key] !== undefined) {
-          if (secrets[key] === '') {
-            delete existingSecrets[key];
-          } else {
-            existingSecrets[key] = secrets[key];
+        if (secrets[key] === undefined) continue;
+        const value = secrets[key];
+        if (value === '' || value == null) {
+          try {
+            await deleteSecret(BRAIN_DIR, key, { action: 'delete', actor: 'settings-config' });
+          } catch {
+            /* key may not exist */
           }
+          continue;
+        }
+        if (typeof value === 'string' && (value.includes('•') || value === '[redacted]')) {
+          // Masked echo from the UI — do not overwrite the store.
+          continue;
         }
+        await setSecret(BRAIN_DIR, key, String(value), {
+          action: 'set',
+          actor: 'settings-config',
+        });
       }
-      const { saveSecrets } = await import('../../core/secrets-store.mjs');
-      await saveSecrets(AGENT_DIR, existingSecrets);
     }
 
     res.json({ success: true });

package/src/server/routes/mesh.mjs

@@ -281,7 +281,8 @@ router.get('/api/mesh/latency', requireAuth, requireScope('config:read'), async
     }
     const start = Date.now();
     try {
-      const response = await throttledFetch(meshPeerUrl(peer.ip, '/health'), {}, 3000);
+      // WAN mesh RTT can exceed 3s under load; gate + Tailscale need headroom.
+      const response = await throttledFetch(meshPeerUrl(peer.ip, '/health'), {}, 10_000);
       const ms = Date.now() - start;
       const ok = response.ok || response.status === 200;
       latency_ms[peer.hostname] = ok ? ms : null;

package/src/server/routes/notifications.mjs

@@ -0,0 +1,103 @@
+/**
+ * /api/notifications/* — alert rules + delivery history for the Notifications page.
+ *
+ * Rules: VFS docs at system/notification-rules/*.md
+ * History: SSSS event workspace `notifications`
+ */
+import { Router } from 'express';
+import { requireAuth, requireScope } from '../auth.mjs';
+import {
+  createNotificationRule,
+  deleteNotificationRule,
+  listNotificationHistory,
+  listNotificationRules,
+  sendTestNotification,
+} from '../../core/notifications.mjs';
+import { logger } from '../../core/logger.mjs';
+
+const router = Router();
+
+router.get(
+  '/api/notifications/rules',
+  requireAuth,
+  requireScope('config:read'),
+  async (_req, res) => {
+    try {
+      const rules = listNotificationRules();
+      res.json(rules);
+    } catch (err) {
+      logger.error('notifications', 'list rules failed', { error: err.message });
+      res.status(500).json({ error: err.message || 'Failed to list notification rules' });
+    }
+  },
+);
+
+router.post(
+  '/api/notifications/rules',
+  requireAuth,
+  requireScope('config:write'),
+  async (req, res) => {
+    try {
+      const rule = await createNotificationRule(req.body || {});
+      res.status(201).json(rule);
+    } catch (err) {
+      const status = err.status || 500;
+      if (status >= 500) {
+        logger.error('notifications', 'create rule failed', { error: err.message });
+      }
+      res.status(status).json({ error: err.message || 'Failed to create notification rule' });
+    }
+  },
+);
+
+router.delete(
+  '/api/notifications/rules/:id',
+  requireAuth,
+  requireScope('config:write'),
+  async (req, res) => {
+    try {
+      const result = await deleteNotificationRule(req.params.id);
+      res.json(result);
+    } catch (err) {
+      const status = err.status || 500;
+      if (status >= 500) {
+        logger.error('notifications', 'delete rule failed', { error: err.message });
+      }
+      res.status(status).json({ error: err.message || 'Failed to delete notification rule' });
+    }
+  },
+);
+
+router.get(
+  '/api/notifications/history',
+  requireAuth,
+  requireScope('config:read'),
+  async (req, res) => {
+    try {
+      const limit = req.query.limit ? Number(req.query.limit) : 100;
+      const history = await listNotificationHistory({ limit });
+      res.json(history);
+    } catch (err) {
+      logger.error('notifications', 'list history failed', { error: err.message });
+      res.status(500).json({ error: err.message || 'Failed to list notification history' });
+    }
+  },
+);
+
+router.post(
+  '/api/notifications/test',
+  requireAuth,
+  requireScope('config:write'),
+  async (_req, res) => {
+    try {
+      const entry = await sendTestNotification();
+      res.json({ success: true, entry });
+    } catch (err) {
+      logger.error('notifications', 'test notification failed', { error: err.message });
+      res.status(500).json({ error: err.message || 'Failed to send test notification' });
+    }
+  },
+);
+
+export default router;
+export { router as notificationsRouter };

package/src/server/routes/rules.mjs

@@ -1,32 +1,87 @@
 import express from 'express';
 import { getBothBrains } from '../../cli/agent-dir.mjs';
 import { getNodes } from '../../core/vault-cache.mjs';
+import { logger } from '../../core/logger.mjs';
 
 import path from 'node:path';
 
 export const rulesRouter = express.Router();
 
-rulesRouter.get('/api/rules', (req, res) => {
-  const { global: globalBrain, project: projectBrain } = getBothBrains();
-  
-  const rules = [];
-
-  const addRules = (brainDir, scope) => {
-    if (!brainDir) return;
-    const vaultDir = path.join(brainDir, 'memory-vault');
-    const nodes = getNodes(vaultDir);
-    const validCategories = new Set(['invariants', 'preferences', 'anti-patterns']);
-    
-    nodes.filter(n => validCategories.has(n.category)).forEach(n => {
-      rules.push({
-        ...n,
-        scope
-      });
-    });
+/** Rule categories that surface on the Agent Rules page / instruction compiler. */
+const RULE_CATEGORIES = new Set(['invariants', 'preferences', 'anti-patterns', 'corrections']);
+
+/**
+ * Resolve the vault root for a brain layer.
+ * getBothBrains() returns `brainDir` (not `brainRoot`).
+ * Project layer also carries `agentDir` — fall back to skills/total-recall under it.
+ */
+function resolveBrainDir(layer) {
+  if (!layer) return null;
+  if (typeof layer.brainDir === 'string' && layer.brainDir) return layer.brainDir;
+  if (typeof layer.brainRoot === 'string' && layer.brainRoot) return layer.brainRoot; // legacy alias
+  if (typeof layer.agentDir === 'string' && layer.agentDir) {
+    return path.join(layer.agentDir, 'skills', 'total-recall');
+  }
+  return null;
+}
+
+function serializeRule(node, scope) {
+  const importanceRaw = Number(node.importance);
+  const importance = Number.isFinite(importanceRaw)
+    ? Math.min(5, Math.max(0, Math.round(importanceRaw)))
+    : 0;
+  const body =
+    typeof node.body === 'string'
+      ? node.body
+      : typeof node.content === 'string'
+        ? node.content
+        : '';
+  return {
+    slug: node.slug,
+    category: node.category === 'corrections' ? 'anti-patterns' : node.category,
+    title: node.title || node.slug || 'Untitled rule',
+    status: node.status || 'active',
+    importance,
+    body,
+    scope,
   };
+}
+
+rulesRouter.get('/api/rules', (req, res) => {
+  try {
+    const { global: globalBrain, project: projectBrain } = getBothBrains();
+    const rules = [];
+    const seenVaults = new Set();
+
+    const addRules = (brainDir, scope) => {
+      if (!brainDir) return;
+      const vaultDir = path.join(brainDir, 'memory-vault');
+      const vaultKey = path.resolve(vaultDir);
+      // Avoid double-listing when project brainDir incorrectly points at global.
+      if (seenVaults.has(vaultKey)) return;
+      seenVaults.add(vaultKey);
+
+      let nodes = [];
+      try {
+        nodes = getNodes(vaultDir) || [];
+      } catch (err) {
+        logger.warn('rules', 'Failed to load vault nodes', { vaultDir, error: err.message });
+        return;
+      }
+
+      for (const n of nodes) {
+        if (!RULE_CATEGORIES.has(n.category)) continue;
+        // Prefer active rules first in natural order; archived still included for restore UI
+        rules.push(serializeRule(n, scope));
+      }
+    };
 
-  addRules(globalBrain?.brainRoot, 'global');
-  addRules(projectBrain?.brainRoot, 'project');
+    addRules(resolveBrainDir(globalBrain), 'global');
+    addRules(resolveBrainDir(projectBrain), 'project');
 
-  res.json({ rules });
+    res.json({ rules, count: rules.length });
+  } catch (err) {
+    logger.error('rules', 'GET /api/rules failed', { error: err.message, stack: err.stack });
+    res.status(500).json({ error: 'Failed to load agent rules', message: err.message });
+  }
 });