total-recall-brain 3.18.2 → 3.18.3

package/src/core/provider-catalog.mjs

@@ -311,15 +311,22 @@ export function getProvider(providerId) {
  * @param {string} keyName
  */
 export function providerForKeyName(keyName) {
-  const k = String(keyName || '').toUpperCase();
+  // Strip packaging prefixes so DEVELOPER_BRAVE_SEARCH_API_KEY → BRAVE_SEARCH_API_KEY
+  let k = String(keyName || '').toUpperCase();
+  k = k.replace(/^(DEVELOPER_|VITE_|NEXT_PUBLIC_|PUBLIC_|NUXT_PUBLIC_|PORTFOLIO_|ULTRACHAT_)+/g, '');
   for (const p of PROVIDER_CATALOG) {
-    if (p.key_patterns.some((pat) => pat.toUpperCase() === k || k.includes(pat.replace(/_API_KEY$/, '')))) {
+    if (p.key_patterns.some((pat) => {
+      const P = pat.toUpperCase();
+      return P === k || k === P || k.endsWith('_' + P) || k.includes(P.replace(/_API_KEY$|_TOKEN$|_KEY$/i, ''));
+    })) {
       return p;
     }
   }
-  // fuzzy
+  // fuzzy product id in the remaining name
+  const compact = k.replace(/-/g, '_');
   for (const p of PROVIDER_CATALOG) {
-    if (k.includes(p.id.toUpperCase().replace('-', '_'))) return p;
+    const id = p.id.toUpperCase().replace(/-/g, '_');
+    if (compact.includes(id) || compact.startsWith(id + '_')) return p;
   }
   return null;
 }

package/src/core/research.mjs

@@ -77,9 +77,10 @@ export async function handleProactiveResearch(task, context = {}) {
   }
 
   // Phase 3: Write and integrate findings into a single consolidated main draft node
+  let draftSlug = null;
   for (const [i, results] of allResults.entries()) {
     if (results.length === 0) continue;
-    writeOrUpdateConsolidatedDraft(task.target, queries[i], results, inboxDir);
+    draftSlug = writeOrUpdateConsolidatedDraft(task.target, queries[i], results, inboxDir);
   }
 
   // Phase 4: Synthesize using local runtime (which consists of high-powered CLI agents)
@@ -88,7 +89,7 @@ export async function handleProactiveResearch(task, context = {}) {
 
   // Save the beautiful synthesized executive summary at the top of our main consolidated document
   if (finalReport) {
-    saveSynthesizedReportToDraft(task.target, finalReport, inboxDir);
+    draftSlug = saveSynthesizedReportToDraft(task.target, finalReport, inboxDir) || draftSlug;
   }
 
   // Phase 5: Also add topic to the Research Agenda for ongoing tracking
@@ -100,7 +101,16 @@ export async function handleProactiveResearch(task, context = {}) {
     tags: ['deep-research'],
   });
 
-  return finalReport;
+  // Always return structured result so executors can advance the research queue with a real node slug
+  if (!draftSlug) {
+    draftSlug = `research-report-${slugify(task.target)}`;
+  }
+  return {
+    report: finalReport || null,
+    factSlug: draftSlug,
+    sources: flatResults.length,
+    queries,
+  };
 }
 
 // ─── Planning ───────────────────────────────────────────────────────────────────
@@ -146,43 +156,105 @@ Generate 3 targeted search queries targeting active, timely information post-${c
 
 // ─── Real Source Gathering ───────────────────────────────────────────────────────
 
+/**
+ * Decide source mix from the *meaning* of the query (not only crude keyword lists).
+ * Combines semantic hints with optional suggested_sources from the planner.
+ */
+function planSourcesForQuery(query, availability, suggested = []) {
+  const q = String(query || '').toLowerCase();
+  const suggestedSet = new Set((suggested || []).map((s) => String(s).toLowerCase()));
+  const avail = new Set(availability?.available || []);
+
+  const want = {
+    web: true, // always try web when possible
+    wikipedia: true,
+    arxiv: false,
+    npm: false,
+    github: false,
+  };
+
+  if (
+    suggestedSet.has('arxiv') ||
+    /\b(paper|arxiv|neural|transformer|llm|benchmark|dataset|training|model card)\b/.test(q)
+  ) {
+    want.arxiv = true;
+  }
+  if (
+    suggestedSet.has('npm') ||
+    /\b(npm|package\.json|node\.js|typescript package|js library)\b/.test(q)
+  ) {
+    want.npm = true;
+  }
+  if (
+    suggestedSet.has('github') ||
+    /\b(github|open.?source|repository|pull request|commit)\b/.test(q)
+  ) {
+    want.github = true;
+  }
+  // API / integration research: bias web + github docs, not arxiv
+  if (/\b(api|endpoint|oauth|sdk|webhook|authentication|base url)\b/.test(q)) {
+    want.github = true;
+    want.web = true;
+    want.arxiv = false;
+  }
+
+  return {
+    web: want.web && (
+      avail.has('searx') ||
+      avail.has('brave-search') ||
+      avail.has('serper') ||
+      avail.has('tavily') ||
+      avail.has('exa') ||
+      avail.has('duckduckgo')
+    ),
+    wikipedia: want.wikipedia && avail.has('wikipedia'),
+    arxiv: want.arxiv && avail.has('arxiv'),
+    npm: want.npm && avail.has('npm'),
+    github: want.github && avail.has('github'),
+  };
+}
+
 async function gatherForQuery(query, researchConfig, availability) {
   const results = [];
   const fetches = [];
+  const sources = planSourcesForQuery(query, availability);
 
-  // Always try: Wikipedia + DuckDuckGo
-  fetches.push(
-    wikipediaFetch(query, researchConfig).then(r => r && results.push(r)).catch(() => {}),
-  );
+  if (sources.wikipedia) {
+    fetches.push(
+      wikipediaFetch(query, researchConfig).then((r) => r && results.push(r)).catch(() => {}),
+    );
+  }
 
-  // 1. Web search — Brave with Serper fallback
-  if (availability.available.includes('brave-search') || availability.available.includes('serper')) {
+  // Paid chain (Brave→Tavily→Exa→Serper) or DuckDuckGo — always prefer web for product/API research
+  if (sources.web) {
     fetches.push(
-      webSearch(query, researchConfig, 4).then(r => results.push(...r)).catch(err => {
-        logger.info({ subsystem: 'deep-research', message: `Web search failed for "${query}": ${err.message}` });
-      }),
+      webSearch(query, researchConfig, 4)
+        .then((r) => results.push(...r))
+        .catch((err) => {
+          logger.info({
+            subsystem: 'deep-research',
+            message: `Web search failed for "${query}": ${err.message}`,
+          });
+        }),
     );
   }
 
-  // arXiv for academic topics
-  if (/\b(model|llm|neural|ml|ai|research|paper|algorithm|training)\b/i.test(query)) {
+  if (sources.arxiv) {
     fetches.push(
-      arxivSearch(query, researchConfig, 2).then(r => results.push(...r)).catch(() => {}),
+      arxivSearch(query, researchConfig, 2).then((r) => results.push(...r)).catch(() => {}),
     );
   }
 
-  // npm for JS/package topics
-  if (/\b(npm|node|javascript|typescript|package|library|framework)\b/i.test(query)) {
+  if (sources.npm) {
     fetches.push(
-      npmSearch(query, researchConfig, 3).then(r => results.push(...r)).catch(() => {}),
+      npmSearch(query, researchConfig, 3).then((r) => results.push(...r)).catch(() => {}),
     );
   }
 
-  // GitHub for code/tool topics
-  if (/\b(github|repo|open.source|tool|cli|sdk|api|integration)\b/i.test(query)) {
+  if (sources.github) {
     fetches.push(
       githubSearch(query, researchConfig, 'repositories', 3)
-        .then(r => results.push(...r))
+        .then((r) => results.push(...r))
         .catch(() => {}),
     );
   }
@@ -190,7 +262,9 @@ async function gatherForQuery(query, researchConfig, availability) {
   await Promise.all(fetches);
 
   // Deep-crawl top web result using smartFetch (Playwright if installed, plain fetch otherwise)
-  const topWebResult = results.find(r => ['brave-search', 'serper'].includes(r.source) && r.url);
+  const topWebResult = results.find((r) =>
+    ['searx', 'brave-search', 'serper', 'tavily', 'exa'].includes(r.source) && r.url,
+  );
   if (topWebResult) {
     try {
       const crawled = await smartFetch(topWebResult.url, researchConfig);
@@ -343,7 +417,7 @@ export function writeOrUpdateConsolidatedDraft(parentTopic, query, results, inbo
   };
 
   atomicWrite(filePath, safeStringify(bodyLines.join('\n'), frontmatter));
-  return filePath;
+  return slug;
 }
 
 export function saveSynthesizedReportToDraft(parentTopic, finalReport, inboxDir) {
@@ -437,7 +511,7 @@ export function saveSynthesizedReportToDraft(parentTopic, finalReport, inboxDir)
   };
 
   atomicWrite(filePath, safeStringify(bodyLines.join('\n'), frontmatter));
-  return filePath;
+  return slug;
 }
 
 // ─── Synthesis ───────────────────────────────────────────────────────────────────

package/src/core/scheduler.mjs

@@ -510,16 +510,38 @@ export function createScheduler({ queueDir, vaultDir, sessionsDir }) {
     for (const item of allItems) {
       if (item.status === 'done' || item.status === 'failed') {
         const completedTime = new Date(item.completed_at || item.updated_at || 0).getTime();
-        if (Date.now() - completedTime >= COOLDOWN_MS) {
-          try {
-            updateQueueItem(item.id, { status: 'pending' });
-            resetCount++;
-          } catch (updateErr) {
-            logger.error({
-              subsystem: 'scheduler',
-              message: `Failed to reset research queue item ${item.id} to pending: ${updateErr.message}`,
+        if (Date.now() - completedTime < COOLDOWN_MS) continue;
+
+        // Empty "done" (no node, no conclusions) is NOT real research — re-acquire
+        const emptyDone =
+          item.status === 'done' &&
+          !item.node_slug &&
+          !(item.summary?.conclusions?.length > 0);
+
+        // Successful done with real content: re-queue only for monitoring refresh, keep node
+        // Failed or empty: force acquisition from scratch
+        try {
+          if (emptyDone || item.status === 'failed' || !item.node_slug) {
+            updateQueueItem(item.id, {
+              status: 'pending',
+              research_phase: 'acquisition',
+              notes: emptyDone
+                ? 'Auto-reset empty done project → re-run acquisition (previous run produced no memory node)'
+                : item.notes || null,
+            });
+          } else {
+            // Has a real node — optional re-monitor after cooldown (don't wipe conclusions)
+            updateQueueItem(item.id, {
+              status: 'pending',
+              research_phase: 'monitoring',
             });
           }
+          resetCount++;
+        } catch (updateErr) {
+          logger.error({
+            subsystem: 'scheduler',
+            message: `Failed to reset research queue item ${item.id} to pending: ${updateErr.message}`,
+          });
         }
       }
     }
@@ -527,7 +549,7 @@ export function createScheduler({ queueDir, vaultDir, sessionsDir }) {
     if (resetCount > 0) {
       logger.info({
         subsystem: 'scheduler',
-        message: `Auto-reset ${resetCount} completed/failed research items back to pending after cooldown.`,
+        message: `Auto-reset ${resetCount} research items after cooldown (empty→acquisition, failed→acquisition, done+node→monitoring).`,
       });
     }
 

package/src/core/schema.mjs

@@ -908,6 +908,27 @@ export const WebhookConfigSchema = z.object({
   events: z.array(z.string()).default([]),
 }).passthrough();
 
+/** Alert rule for mesh/system events → desktop/webhook/email channels. */
+export const NotificationRuleSchema = z.object({
+  type: z.literal('notification_rule'),
+  title: z.string().min(1),
+  description: z.string().min(1),
+  timestamp: ssssDatetime(),
+  id: z.string().min(1),
+  event: z.enum([
+    'node_offline',
+    'leader_change',
+    'webhook_failed',
+    'secret_sync_failed',
+    'daemon_error',
+    'research_complete',
+  ]),
+  channel: z.enum(['desktop', 'webhook', 'email']),
+  priority: z.enum(['critical', 'high', 'low']).default('high'),
+  enabled: z.boolean().default(true),
+  quietHours: z.boolean().default(false),
+}).passthrough();
+
 const NetworkInterfaceEntrySchema = z.object({
   name: z.string().min(1),
   kind: z.enum(['loopback', 'wifi', 'ethernet', 'bridge', 'vpn_overlay', 'other']),
@@ -990,6 +1011,7 @@ export const DaemonLeaderSchema = z.object({
 export const SSSS_HOST_EXTENSION_TYPES = [
   'network_policy',
   'webhook_config',
+  'notification_rule',
   'mesh_node',
   'daemon_leader',
 ];
@@ -998,6 +1020,7 @@ export const SSSS_HOST_EXTENSION_TYPES = [
 export const SSSS_SCHEMAS = {
   network_policy: NetworkPolicySchema,
   webhook_config: WebhookConfigSchema,
+  notification_rule: NotificationRuleSchema,
   mesh_node: MeshNodeSchema,
   daemon_leader: DaemonLeaderSchema,
   memory: MemoryNodeSchema,

package/src/core/secret-integration-research.mjs

@@ -0,0 +1,325 @@
+/**
+ * AI-only secret → integration research.
+ *
+ * When a new secret is stored, a CLI agent (callLocalRuntime) decides:
+ *   - whether this credential is worth product-API research
+ *   - which product/provider it is
+ *   - what research topic + notes to enqueue
+ *
+ * No hardcoded provider lists, no key-name regex classification, no catalog matching.
+ * Optional code-usage snippets are gathered as *evidence for the model*, not as rules.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { logger } from './logger.mjs';
+
+const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+
+const INFER_SYSTEM = `You are a senior integrations engineer for Total Recall (a personal memory / mesh OS).
+You are given only:
+  (1) an environment/secret KEY NAME (never the secret value)
+  (2) optional code snippets where that key is referenced in THIS repo
+
+Decide whether Total Recall should queue a *research* job to learn an *external third-party* product API that is NOT already part of this product.
+
+Output ONLY valid JSON (no markdown fences):
+{
+  "researchable": boolean,
+  "skip_reason": string|null,
+  "product_name": string|null,
+  "product_slug": string|null,
+  "kind": "api_key"|"password"|"oauth"|"webhook"|"internal"|"self_hosted"|"already_integrated"|"unknown",
+  "topic": string|null,
+  "notes": string|null,
+  "priority": "low"|"medium"|"high",
+  "confidence": number
+}
+
+Rules (strict):
+- researchable=true ONLY for external SaaS/third-party APIs the product might need to learn about from the public internet.
+- researchable=false when code shows the key is for:
+  - infrastructure we already own/run (mesh, Tailscale-compatible control planes, Headscale, local routers, self-hosted mail, etc.)
+  - features already implemented in this repo (routes, clients, adapters present)
+  - passwords, email logins, SSO/client secrets, webhook signing secrets, private keys, session/DB secrets
+  - Total Recall / TR_ / mesh-sync internal credentials
+- If the key is used by existing modules (e.g. headscale routes, mesh auth, mail), set researchable=false with kind "already_integrated" or "self_hosted".
+- Never queue research whose topic is the env var name itself.
+- topic (when researchable) must name the external product and API surface; notes must say use official docs, not the raw key string.
+- Prefer skip when uncertain. Do not invent product paths.`;
+
+/**
+ * Collect a small amount of code-usage context for the AI (not classification rules).
+ * @param {string} key
+ * @param {{ roots?: string[], maxHits?: number }} [opts]
+ * @returns {string}
+ */
+export function gatherCodeUsageContext(key, opts = {}) {
+  const roots = opts.roots || [path.join(REPO_ROOT, 'src')];
+  const maxHits = opts.maxHits ?? 12;
+  const snippets = [];
+
+  // Prefer ripgrep if available; fall back to a tiny walk.
+  const rg = spawnSync(
+    'rg',
+    [
+      '-n',
+      '--no-heading',
+      '-m',
+      '8',
+      '-g',
+      '!**/node_modules/**',
+      '-g',
+      '!**/*.spec.*',
+      '-e',
+      key,
+      ...roots.filter((r) => fs.existsSync(r)),
+    ],
+    { encoding: 'utf8', timeout: 8000, maxBuffer: 512_000 },
+  );
+
+  if (rg.status === 0 && rg.stdout) {
+    for (const line of rg.stdout.split('\n').filter(Boolean).slice(0, maxHits)) {
+      snippets.push(line.slice(0, 400));
+    }
+  } else {
+    // Minimal fallback walk
+    for (const root of roots) {
+      if (!fs.existsSync(root)) continue;
+      walkForKey(root, key, snippets, maxHits, 0);
+      if (snippets.length >= maxHits) break;
+    }
+  }
+
+  if (!snippets.length) {
+    return '(no code references found in local src tree)';
+  }
+  return snippets.join('\n');
+}
+
+function walkForKey(dir, key, out, maxHits, depth) {
+  if (out.length >= maxHits || depth > 6) return;
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const ent of entries) {
+    if (out.length >= maxHits) return;
+    if (ent.name === 'node_modules' || ent.name === '.git' || ent.name.startsWith('.')) continue;
+    const full = path.join(dir, ent.name);
+    if (ent.isDirectory()) {
+      walkForKey(full, key, out, maxHits, depth + 1);
+      continue;
+    }
+    if (!/\.(mjs|js|ts|tsx|yml|yaml|json|md)$/i.test(ent.name)) continue;
+    if (/\.spec\./i.test(ent.name)) continue;
+    let text;
+    try {
+      text = fs.readFileSync(full, 'utf8');
+    } catch {
+      continue;
+    }
+    if (!text.includes(key)) continue;
+    const lines = text.split('\n');
+    for (let i = 0; i < lines.length && out.length < maxHits; i++) {
+      if (lines[i].includes(key)) {
+        const from = Math.max(0, i - 1);
+        const to = Math.min(lines.length, i + 2);
+        const chunk = lines
+          .slice(from, to)
+          .map((l, idx) => `${full}:${from + idx + 1}:${l.trim().slice(0, 200)}`)
+          .join('\n');
+        out.push(chunk);
+      }
+    }
+  }
+}
+
+/**
+ * Parse first JSON object from model output.
+ * @param {string} raw
+ */
+export function parseAiJson(raw) {
+  const text = String(raw || '');
+  const start = text.indexOf('{');
+  const end = text.lastIndexOf('}');
+  if (start < 0 || end <= start) return null;
+  try {
+    return JSON.parse(text.slice(start, end + 1));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Ask a CLI agent to infer provider / research brief from key name + code usage.
+ *
+ * @param {string} key
+ * @param {{
+ *   meta?: object,
+ *   codeContext?: string,
+ *   runtimeConfig?: object,
+ *   callRuntime?: Function,
+ * }} [opts]
+ * @returns {Promise<{
+ *   researchable: boolean,
+ *   skip_reason: string|null,
+ *   product_name: string|null,
+ *   product_slug: string|null,
+ *   kind: string,
+ *   topic: string|null,
+ *   notes: string|null,
+ *   priority: string,
+ *   confidence: number,
+ *   source: 'ai',
+ * }>}
+ */
+export async function inferSecretIntegrationWithAi(key, opts = {}) {
+  const codeContext =
+    opts.codeContext !== undefined
+      ? opts.codeContext
+      : gatherCodeUsageContext(key, { roots: opts.roots });
+
+  const prompt = [
+    `Secret key name: ${key}`,
+    opts.meta?.provider ? `Stored provider tag (may be wrong/empty): ${opts.meta.provider}` : null,
+    opts.meta?.label ? `Label: ${opts.meta.label}` : null,
+    opts.meta?.notes ? `User notes: ${String(opts.meta.notes).slice(0, 500)}` : null,
+    '',
+    'Code references (evidence only — reason about them):',
+    codeContext,
+    '',
+    'Return the JSON decision now.',
+  ]
+    .filter((l) => l !== null)
+    .join('\n');
+
+  const callRuntime =
+    opts.callRuntime ||
+    (async (user, system, cfg) => {
+      const { callLocalRuntime, loadRuntimeConfig } = await import('./runtime.mjs');
+      return callLocalRuntime(user, system, cfg || opts.runtimeConfig || loadRuntimeConfig());
+    });
+
+  let raw;
+  try {
+    raw = await callRuntime(prompt, INFER_SYSTEM, opts.runtimeConfig);
+  } catch (err) {
+    logger.warn('secrets', 'AI secret integration inference failed', {
+      key,
+      error: err.message,
+    });
+    return {
+      researchable: false,
+      skip_reason: `AI inference unavailable: ${err.message}`,
+      product_name: null,
+      product_slug: null,
+      kind: 'unknown',
+      topic: null,
+      notes: null,
+      priority: 'low',
+      confidence: 0,
+      source: 'ai',
+    };
+  }
+
+  const parsed = parseAiJson(raw);
+  if (!parsed || typeof parsed !== 'object') {
+    return {
+      researchable: false,
+      skip_reason: 'AI returned non-JSON inference',
+      product_name: null,
+      product_slug: null,
+      kind: 'unknown',
+      topic: null,
+      notes: null,
+      priority: 'low',
+      confidence: 0,
+      source: 'ai',
+    };
+  }
+
+  const researchable = parsed.researchable === true;
+  return {
+    researchable,
+    skip_reason: researchable ? null : String(parsed.skip_reason || 'AI declined research'),
+    product_name: parsed.product_name ? String(parsed.product_name) : null,
+    product_slug: parsed.product_slug ? String(parsed.product_slug) : null,
+    kind: String(parsed.kind || 'unknown'),
+    topic: researchable && parsed.topic ? String(parsed.topic) : null,
+    notes: researchable && parsed.notes ? String(parsed.notes) : null,
+    priority: ['low', 'medium', 'high'].includes(parsed.priority) ? parsed.priority : 'low',
+    confidence: Number.isFinite(Number(parsed.confidence)) ? Number(parsed.confidence) : 0,
+    source: 'ai',
+  };
+}
+
+/**
+ * Build brief via AI only.
+ * @returns {Promise<{ topic: string, notes: string, priority: string, inference: object } | null>}
+ */
+export async function buildIntegrationResearchBrief(key, meta = {}, opts = {}) {
+  const inference = await inferSecretIntegrationWithAi(key, { meta, ...opts });
+  if (!inference.researchable || !inference.topic) {
+    return null;
+  }
+  return {
+    topic: inference.topic,
+    notes: inference.notes || `AI-inferred product research for secret \`${key}\`.`,
+    priority: inference.priority || 'low',
+    inference,
+  };
+}
+
+/**
+ * Enqueue integration research when the AI says so.
+ * @returns {Promise<{ enqueued: boolean, item?: object, skipped?: string, inference?: object }>}
+ */
+export async function maybeEnqueueIntegrationResearch(brainDir, key, meta = {}, opts = {}) {
+  const inference = await inferSecretIntegrationWithAi(key, { meta, ...opts });
+  if (!inference.researchable || !inference.topic) {
+    return {
+      enqueued: false,
+      skipped: inference.skip_reason || 'AI declined research',
+      inference,
+    };
+  }
+
+  const { addToQueue } = await import('./research-queue.mjs');
+  const item = addToQueue({
+    topic: inference.topic,
+    priority: inference.priority || 'low',
+    notes: inference.notes || `AI-inferred product research for secret \`${key}\`.`,
+    brainDir,
+  });
+  return { enqueued: true, item, inference };
+}
+
+/**
+ * Retire obsolete key-name scrape queue items (old Automated API Integration Build: KEY topics).
+ */
+export function cancelBogusApiIntegrationQueueItems(overrideBrainDir) {
+  return import('./research-queue.mjs').then(({ loadQueue, updateQueueItem }) => {
+    const items = loadQueue(overrideBrainDir);
+    let cancelled = 0;
+    for (const item of items) {
+      if (item.status !== 'pending' && item.status !== 'in_progress') continue;
+      if (!/^Automated API Integration Build:/i.test(String(item.topic || ''))) continue;
+      updateQueueItem(
+        item.id,
+        {
+          status: 'failed',
+          notes: `${item.notes || ''}\n\n[auto-cancelled] Obsolete key-name scrape job; re-add the secret to trigger AI provider inference.`,
+          research_phase: 'cancelled',
+        },
+        overrideBrainDir,
+      );
+      cancelled += 1;
+    }
+    return { cancelled };
+  });
+}