total-recall-brain 3.28.2 → 3.30.0

package/src/core/research-gate.mjs

@@ -0,0 +1,182 @@
+/**
+ * src/core/research-gate.mjs
+ *
+ * The only doors into the research queue.
+ *
+ * Research has exactly two reasons to run (RESEARCH_SYSTEM2):
+ *   1. A human asked — in chat, from the browser extension, the dashboard, the
+ *      CLI or any API client. `requestResearch` records that and skips budgets.
+ *   2. While working on a project, the AI found a knowledge gap that would make
+ *      it better at that project. `proposeAutonomousResearch` admits such gaps
+ *      only within budget and only when existing research doesn't cover them.
+ *
+ * Nothing inside the research pipeline calls either: research never spawns
+ * research. That rule is what stops the fan-out that filled the queue with 89
+ * self-generated tangents in two days.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import yaml from 'yaml';
+import { addToQueue, loadQueue, normalizeTopic, RESEARCH_VIAS } from './research-queue.mjs';
+import { brainDir as defaultBrainDir } from './config.mjs';
+import { logger } from './logger.mjs';
+
+export const AUTONOMOUS_DEFAULTS = Object.freeze({
+  enabled: true,
+  maxPendingPerProject: 3,
+  maxPerDay: 6,
+  maxPerSession: 2,
+  // Existing research at or above this cosine similarity already answers the gap.
+  coverageSimilarity: 0.72,
+});
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const OPEN_STATUSES = new Set(['pending', 'in_progress']);
+
+/**
+ * Merge `autonomous:` from config/research.yml over the defaults.
+ * @param {object} [fileConfig] parsed research.yml
+ */
+export function resolveAutonomousConfig(fileConfig = {}) {
+  const raw = (fileConfig && fileConfig.autonomous) || {};
+  const num = (v, d) => (Number.isFinite(Number(v)) && Number(v) >= 0 ? Number(v) : d);
+  return {
+    enabled: raw.enabled === undefined ? AUTONOMOUS_DEFAULTS.enabled : raw.enabled !== false,
+    maxPendingPerProject: num(raw.max_pending_per_project, AUTONOMOUS_DEFAULTS.maxPendingPerProject),
+    maxPerDay: num(raw.max_per_day, AUTONOMOUS_DEFAULTS.maxPerDay),
+    maxPerSession: num(raw.max_per_session, AUTONOMOUS_DEFAULTS.maxPerSession),
+    coverageSimilarity: num(raw.coverage_similarity, AUTONOMOUS_DEFAULTS.coverageSimilarity),
+  };
+}
+
+export function loadAutonomousConfig(brainDir = defaultBrainDir) {
+  const file = path.join(brainDir, 'config', 'research.yml');
+  try {
+    return resolveAutonomousConfig(fs.existsSync(file) ? yaml.parse(fs.readFileSync(file, 'utf8')) || {} : {});
+  } catch {
+    return resolveAutonomousConfig({});
+  }
+}
+
+function assertVia(via) {
+  if (!RESEARCH_VIAS.includes(via)) throw new Error(`invalid research via: ${via}`);
+}
+
+/**
+ * A human asked for research. No budgets, no coverage check: they asked.
+ *
+ * @param {{ topic: string, notes?: string, via: string, priority?: string, project?: string|null, brainDir?: string }} req
+ */
+export function requestResearch({ topic, notes, via, priority = 'high', project = null, brainDir } = {}) {
+  const clean = String(topic || '').trim();
+  if (!clean) throw new Error('topic is required');
+  assertVia(via);
+  return addToQueue({ topic: clean, notes, priority, origin: 'user', requested_via: via, project, brainDir });
+}
+
+/**
+ * Default coverage check: the best cosine similarity between the gap and any
+ * research-layer node in the vault. Returns 0 when search is unavailable, so a
+ * broken embedding provider can only make the gate *more* permissive by the
+ * budget, never unbounded.
+ */
+export async function defaultCoverageCheck(topic, { vaultDir, derivedDir } = {}) {
+  try {
+    const { semanticSearch } = await import('./search.mjs');
+    const { inferMemoryLayer } = await import('./memory-layers.mjs');
+    const results = await semanticSearch(topic, { vaultDir, derivedDir, top_k: 5, includeSessions: false });
+    let best = 0;
+    for (const r of results) {
+      const isResearch = inferMemoryLayer(r) === 'research' || (Array.isArray(r.tags) && r.tags.includes('research'));
+      if (isResearch && typeof r.similarity === 'number' && r.similarity > best) best = r.similarity;
+    }
+    return best;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * The AI, while working on `project`, thinks these gaps are worth researching.
+ * Admit at most what the budgets allow, skipping anything already covered.
+ *
+ * @param {Array<{topic: string, rationale?: string, priority?: number|string}>} candidates
+ * @param {{ project: string|null, sessionId?: string|null, via?: 'session'|'secret',
+ *           config?: object, now?: number, brainDir?: string,
+ *           coverageCheck?: (topic: string) => Promise<number> }} ctx
+ * @returns {Promise<{ queued: object[], skipped: Array<{topic: string, reason: string}> }>}
+ */
+export async function proposeAutonomousResearch(candidates, {
+  project = null,
+  sessionId = null,
+  via = 'session',
+  config,
+  now = Date.now(),
+  brainDir,
+  coverageCheck,
+} = {}) {
+  assertVia(via);
+  const cfg = config || loadAutonomousConfig(brainDir);
+  const queued = [];
+  const skipped = [];
+  const list = (Array.isArray(candidates) ? candidates : [])
+    .map((c) => ({ ...c, topic: String((c && c.topic) || '').trim() }))
+    .filter((c) => c.topic);
+
+  if (!cfg.enabled) {
+    return { queued, skipped: list.map((c) => ({ topic: c.topic, reason: 'disabled' })) };
+  }
+
+  const check = coverageCheck || ((topic) => defaultCoverageCheck(topic, {
+    vaultDir: path.join(brainDir || defaultBrainDir, 'memory-vault'),
+    derivedDir: path.join(brainDir || defaultBrainDir, 'memory-derived'),
+  }));
+
+  const items = loadQueue(brainDir);
+  const autonomous = items.filter((i) => i.origin === 'autonomous');
+  let pendingForProject = autonomous.filter((i) => OPEN_STATUSES.has(i.status) && (i.project || null) === project).length;
+  let today = autonomous.filter((i) => now - Date.parse(i.created_at || 0) < DAY_MS).length;
+  let thisSession = sessionId ? autonomous.filter((i) => i.session_id === sessionId).length : 0;
+  const known = new Set(items.map((i) => normalizeTopic(i.topic)));
+
+  for (const c of list) {
+    const key = normalizeTopic(c.topic);
+    if (known.has(key)) { skipped.push({ topic: c.topic, reason: 'duplicate' }); continue; }
+    if (today >= cfg.maxPerDay) { skipped.push({ topic: c.topic, reason: 'budget-day' }); continue; }
+    if (pendingForProject >= cfg.maxPendingPerProject) { skipped.push({ topic: c.topic, reason: 'budget-project' }); continue; }
+    if (sessionId && thisSession >= cfg.maxPerSession) { skipped.push({ topic: c.topic, reason: 'budget-session' }); continue; }
+
+    const covered = await check(c.topic);
+    if (covered >= cfg.coverageSimilarity) {
+      skipped.push({ topic: c.topic, reason: 'covered' });
+      continue;
+    }
+
+    const item = addToQueue({
+      topic: c.topic,
+      priority: typeof c.priority === 'string' ? c.priority : (Number(c.priority) >= 70 ? 'high' : 'medium'),
+      notes: c.rationale || null,
+      origin: 'autonomous',
+      requested_via: via,
+      project,
+      rationale: c.rationale || null,
+      session_id: sessionId,
+      brainDir,
+    });
+    queued.push(item);
+    known.add(key);
+    today++;
+    pendingForProject++;
+    thisSession++;
+  }
+
+  if (queued.length || skipped.length) {
+    logger.info({
+      subsystem: 'research-gate',
+      message: `Autonomous research for ${project || 'no project'}: queued ${queued.length}, skipped ${skipped.length}`
+        + (skipped.length ? ` (${skipped.map((s) => s.reason).join(', ')})` : ''),
+    });
+  }
+  return { queued, skipped };
+}

package/src/core/research-queue.mjs

@@ -5,8 +5,13 @@
  * MCP tools and REST endpoints are thin callers.
  *
  * Storage: .agent/research-queue.jsonl (one JSON object per line)
- * Each item: { id, topic, status, priority, notes, node_slug, research_phase, created_at, updated_at, completed_at, summary }
+ * Each item: { id, topic, status, priority, notes, node_slug, research_phase, created_at, updated_at, completed_at, summary,
+ *              origin, requested_via, project, rationale, session_id, attempts }
  * Status values: 'pending' | 'in_progress' | 'done' | 'failed'
+ * Phases: 'acquisition' → 'deliberation' → 'improvement' → done (scheduler.mjs RESEARCH_PHASES)
+ * Provenance: origin 'user' (a human asked) | 'autonomous' (the project gate approved a
+ *   knowledge gap; see research-gate.mjs) | 'legacy' (queued before provenance existed;
+ *   set only by scripts/migrate-research-system2.mjs — never pinned into instructions).
  */
 
 import fs from 'node:fs';
@@ -339,19 +344,46 @@ export function listQueue({ status, query, limit = 100, offset = 0, brainDir: ov
   return { counts, total: items.length, items: items.slice(off, off + lim) };
 }
 
+export const RESEARCH_ORIGINS = Object.freeze(['user', 'autonomous']);
+export const RESEARCH_VIAS = Object.freeze(['chat', 'extension', 'dashboard', 'cli', 'api', 'share', 'session', 'secret']);
+
+export function normalizeTopic(topic) {
+  return String(topic || '').trim().toLowerCase().replace(/\s+/g, ' ');
+}
+
 /**
  * Add a new research topic (status: pending).
  *
- * @param {{ topic: string, priority?: string, notes?: string }} opts
- * @returns {object} The created item
+ * Prefer research-gate.mjs (`requestResearch` / `proposeAutonomousResearch`),
+ * which applies provenance and the autonomous budgets; this is the raw store.
+ *
+ * @param {{ topic: string, priority?: string, notes?: string, origin?: 'user'|'autonomous',
+ *           requested_via?: string, project?: string|null, rationale?: string|null,
+ *           session_id?: string|null, brainDir?: string }} opts
+ * @returns {object} The created item (or the existing one for a duplicate topic)
  */
-export function addToQueue({ topic, priority = 'medium', notes, brainDir: overrideBrainDir } = {}) {
+export function addToQueue({
+  topic, priority = 'medium', notes, brainDir: overrideBrainDir,
+  origin = 'user', requested_via = 'api', project = null, rationale = null, session_id = null,
+} = {}) {
   if (!topic) throw new Error('topic is required');
+  if (!RESEARCH_ORIGINS.includes(origin)) throw new Error(`invalid research origin: ${origin}`);
 
   const items = loadQueue(overrideBrainDir);
-  const normalizedTopic = String(topic).trim().toLowerCase();
-  const existing = items.find(i => i.topic.trim().toLowerCase() === normalizedTopic);
-  if (existing) {
+  const normalizedTopic = normalizeTopic(topic);
+  const existingIdx = items.findIndex(i => normalizeTopic(i.topic) === normalizedTopic);
+  if (existingIdx !== -1) {
+    const existing = items[existingIdx];
+    // A human asking for something the AI queued on its own makes it theirs:
+    // it then runs ahead of autonomous work.
+    if (origin === 'user' && existing.origin !== 'user' && existing.status !== 'done') {
+      items[existingIdx] = {
+        ...existing, origin: 'user', requested_via, priority: priority || existing.priority,
+        updated_at: new Date().toISOString(),
+      };
+      saveQueue(items, overrideBrainDir);
+      return items[existingIdx];
+    }
     return existing;
   }
 
@@ -366,6 +398,12 @@ export function addToQueue({ topic, priority = 'medium', notes, brainDir: overri
     created_at:   new Date().toISOString(),
     updated_at:   new Date().toISOString(),
     completed_at: null,
+    origin,
+    requested_via: requested_via || null,
+    project:      project || null,
+    rationale:    rationale || null,
+    session_id:   session_id || null,
+    attempts:     0,
   };
   item.summary = compileResearchProjectSummary(item, null);
   items.unshift(item);
@@ -400,6 +438,8 @@ export function updateQueueItem(id, patch = {}, overrideBrainDir) {
   if (patch.node_slug !== undefined) item.node_slug = patch.node_slug;
   if (patch.priority  !== undefined) item.priority  = patch.priority;
   if (patch.research_phase !== undefined) item.research_phase = patch.research_phase;
+  if (patch.attempts !== undefined) item.attempts = Number(patch.attempts) || 0;
+  if (patch.completed_at !== undefined) item.completed_at = patch.completed_at;
   
   item.updated_at = new Date().toISOString();
   if ((item.status === 'done' || item.status === 'failed') && !item.completed_at) {

package/src/core/research-surface.mjs

@@ -0,0 +1,211 @@
+/**
+ * src/core/research-surface.mjs
+ *
+ * System 2 → System 1: bring finished background research into the compiled
+ * instructions so the agent can use it mid-conversation instead of only when
+ * someone happens to `recall` the right words.
+ *
+ * Selection is deliberate and small: research finished for the current project,
+ * plus research a human asked for recently. A report only qualifies if it holds
+ * an actual synthesis — the appendix of raw search snippets, or a synthesizer
+ * that answered about itself instead of the topic, is not knowledge.
+ */
+
+const APPENDIX_RE = /^#{1,3}\s*appendix\b/im;
+const USER_RECENT_DAYS = 30;
+
+// A synthesizer that answered about its prompt instead of the topic. All seen in
+// the live vault (2026-09-22): the synthesis agent treated the research prompt as
+// a prompt injection, questioned whether it was test content, or explained why
+// it would not cite the snippets. None of that is knowledge.
+const META_RESPONSE_RE = new RegExp([
+  String.raw`my actual (?:cutoff|role)`,
+  String.raw`training (?:data )?cutoff`,
+  String.raw`\bpersona\b`,
+  String.raw`scratchpad`,
+  String.raw`prompt[- ]injection`,
+  String.raw`system_instructions`,
+  String.raw`deep research synthesizer`,
+  String.raw`i'?m claude\b|as (?:your )?claude code|working in this repo`,
+  String.raw`is this (?:test content|related to work)`,
+  String.raw`(?:genuine|real) (?:research summary|citations|info)`,
+  String.raw`without fabricating`,
+  String.raw`i (?:have no record|don'?t have (?:real-time )?(?:web )?(?:search|access))`,
+  String.raw`i can(?:no|')t (?:actually |help|synthesize|verify|produce|do that)`,
+  String.raw`(?:retrieved|provided|\[searx\]) (?:docs|results|snippets) (?:are|contain|don'?t|do not)`,
+  String.raw`if you want (?:a |me to |real |genuine )`,
+].join('|'), 'i');
+
+/** The report's synthesis: body after the H1, before the raw-sources appendix. */
+export function extractSynthesis(body) {
+  const text = String(body || '');
+  const cut = text.search(APPENDIX_RE);
+  const head = cut === -1 ? text : text.slice(0, cut);
+  return head
+    .replace(/^#\s+.*$/m, '')
+    // Obsidian-style callout banners ("> [!NOTE] Temporal Context: …") are chrome.
+    .replace(/^>\s*\[![A-Z]+\][^\n]*(?:\n>[^\n]*)*/gm, '')
+    .trim();
+}
+
+/** True when the synthesis is real research rather than empty or meta-commentary. */
+export function isUsableSynthesis(synthesis) {
+  const s = String(synthesis || '').trim();
+  if (s.length < 160) return false;
+  return !META_RESPONSE_RE.test(s.slice(0, 2000));
+}
+
+function cleanLine(line) {
+  return line
+    .replace(/^\s*(?:[-*+]|\d+[.)])\s+/, '')
+    .replace(/\*\*([^*]+)\*\*/g, '$1')
+    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
+    .replace(/`([^`]+)`/g, '$1')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function clip(text, max) {
+  if (text.length <= max) return text;
+  const cut = text.slice(0, max);
+  const stop = cut.lastIndexOf('. ');
+  return stop > max * 0.5 ? cut.slice(0, stop + 1) : `${cut.slice(0, cut.lastIndexOf(' '))}…`;
+}
+
+/**
+ * 2–4 findings: prefer bullets under a findings/summary/bottom-line heading, then
+ * any bullets, then the opening sentences.
+ */
+export function extractFindings(synthesis, max = 4) {
+  const lines = String(synthesis || '').split('\n');
+  const bullets = [];
+  let inPreferred = false;
+  const preferred = [];
+  for (const line of lines) {
+    const heading = line.match(/^#{2,4}\s+(.*)$/);
+    if (heading) {
+      inPreferred = /finding|summary|bottom line|takeaway|conclusion|tl;?dr|key/i.test(heading[1]);
+      continue;
+    }
+    if (/^\s*(?:[-*+]|\d+[.)])\s+\S/.test(line)) {
+      const clean = cleanLine(line);
+      if (clean.length < 12) continue;
+      (inPreferred ? preferred : bullets).push(clean);
+    }
+  }
+  let picked = preferred.length ? preferred : bullets;
+  if (!picked.length) {
+    const prose = lines.filter((l) => l.trim() && !/^#/.test(l)).map(cleanLine).join(' ');
+    picked = prose.split(/(?<=[.!?])\s+(?=[A-Z])/).filter((s) => s.length > 20);
+  }
+  return picked.slice(0, max).map((f) => clip(f, 220));
+}
+
+/**
+ * @param {{ queueItems: object[], nodes: object[], project?: string|null, now?: number, limit?: number }} opts
+ * @returns {Array<{ slug, title, project, origin, completedAt, findings: string[] }>}
+ */
+export function selectResearchBriefs({ queueItems = [], nodes = [], project = null, now = Date.now(), limit = 6 } = {}) {
+  const bySlug = new Map(nodes.map((n) => [n.slug, n]));
+  const recentCutoff = now - USER_RECENT_DAYS * 24 * 60 * 60 * 1000;
+  const projectKey = project ? String(project).toLowerCase() : null;
+  const seen = new Set();
+  const briefs = [];
+
+  for (const item of queueItems) {
+    if (item.status !== 'done' || !item.node_slug || seen.has(item.node_slug)) continue;
+    const forProject = projectKey && String(item.project || '').toLowerCase() === projectKey;
+    const completed = Date.parse(item.completed_at || item.updated_at || 0) || 0;
+    // Only research a human explicitly asked for follows the user across projects.
+    // 'legacy' (pre-provenance) items can't prove that, so they reach chat by
+    // relevance but are never pinned into instructions.
+    const userRecent = item.origin === 'user' && completed >= recentCutoff;
+    if (!forProject && !userRecent) continue;
+
+    const node = bySlug.get(item.node_slug);
+    if (!node || node.status === 'archived' || node.status === 'deprecated') continue;
+    const synthesis = extractSynthesis(node.body || node.content);
+    if (!isUsableSynthesis(synthesis)) continue;
+    const findings = extractFindings(synthesis);
+    if (!findings.length) continue;
+
+    seen.add(item.node_slug);
+    briefs.push({
+      slug: node.slug,
+      title: String(node.title || item.topic).replace(/^Consolidated Research Report:\s*/i, ''),
+      project: item.project || null,
+      origin: item.origin,
+      completedAt: completed,
+      forProject: Boolean(forProject),
+      findings,
+    });
+  }
+
+  briefs.sort((a, b) => (b.forProject - a.forProject) || (b.completedAt - a.completedAt));
+  return briefs.slice(0, limit);
+}
+
+/** Markdown section for the instruction surface; '' when there is nothing to show. */
+export function formatResearchBriefs(briefs, { maxChars = 2500 } = {}) {
+  if (!briefs || !briefs.length) return '';
+  const header = `## Background Research (System 2)
+
+Research your brain completed in the background. Use these findings instead of guessing about the tools involved; run \`npx total-recall recall "<slug>"\` for the full cited report.
+`;
+  let out = header;
+  for (const b of briefs) {
+    const date = b.completedAt ? new Date(b.completedAt).toISOString().slice(0, 10) : 'undated';
+    const why = b.origin === 'autonomous' ? `researched for ${b.project || 'a project'}` : 'you asked';
+    const block = `\n**${b.title}** (${date}, ${why}; \`${b.slug}\`)\n${b.findings.map((f) => `- ${f}`).join('\n')}\n`;
+    if (out.length + block.length > maxChars) break;
+    out += block;
+  }
+  return out === header ? '' : out.trimEnd();
+}
+
+// Question-to-report cosine runs lower than title-to-report. Calibrated on the
+// live vault 2026-09-22 (usable reports only): related questions 0.47–0.70;
+// unrelated ones (React tests, Python refactor, SQL, dinner) matched no usable
+// report at all.
+export const CHAT_RESEARCH_MIN_SIMILARITY = 0.45;
+
+/**
+ * Research relevant to a chat message: one semantic search (no LLM call), keeping
+ * research-layer reports that clear the similarity bar and hold a real synthesis.
+ *
+ * @param {string} message latest user message
+ * @param {{ vaultDir: string, derivedDir: string, limit?: number, search?: Function }} opts
+ * @returns {Promise<Array<{ slug, title, similarity, excerpt }>>}
+ */
+export async function findRelevantResearch(message, { vaultDir, derivedDir, limit = 3, search } = {}) {
+  const query = String(message || '').trim().slice(0, 1000);
+  if (query.length < 12) return [];
+  const run = search || (async (q, o) => (await import('./search.mjs')).semanticSearch(q, o));
+  const { inferMemoryLayer } = await import('./memory-layers.mjs');
+  // Wide candidate set: unusable reports and rule nodes must not crowd out the
+  // one real report that answers the question.
+  const results = await run(query, { vaultDir, derivedDir, top_k: 20, includeSessions: false });
+  const out = [];
+  for (const r of results || []) {
+    if (r.type === 'session' || typeof r.similarity !== 'number' || r.similarity < CHAT_RESEARCH_MIN_SIMILARITY) continue;
+    const isResearch = inferMemoryLayer(r) === 'research' || (Array.isArray(r.tags) && r.tags.includes('research'));
+    if (!isResearch) continue;
+    const synthesis = extractSynthesis(r.body || r.content);
+    if (!isUsableSynthesis(synthesis)) continue;
+    out.push({
+      slug: r.slug,
+      title: String(r.title || r.slug).replace(/^Consolidated Research Report:\s*/i, ''),
+      similarity: r.similarity,
+      excerpt: clip(synthesis.replace(/\n{3,}/g, '\n\n'), 1500),
+    });
+    if (out.length >= limit) break;
+  }
+  return out;
+}
+
+/** System-prompt block for chat; '' when nothing relevant was found. */
+export function formatChatResearch(found) {
+  if (!found || !found.length) return '';
+  return `\n\n=== BACKGROUND RESEARCH (System 2) ===\nYour background research already produced findings that bear on this message. Prefer them over guessing, say when you rely on them, and cite the slug.\n`
+    + found.map((f) => `\n--- ${f.title} (slug: ${f.slug}, ${Math.round(f.similarity * 100)}% match) ---\n${f.excerpt}`).join('\n');
+}

package/src/core/research.mjs

@@ -87,8 +87,14 @@ export async function handleProactiveResearch(task, context = {}) {
   logger.info({ subsystem: 'deep-research', message: 'Phase 4: Synthesizing research results...' });
   const finalReport = await synthesizeLocally(task, flatResults, context.runtimeConfig);
 
-  // Save the beautiful synthesized executive summary at the top of our main consolidated document
-  if (finalReport) {
+  // Save the synthesis at the top of the consolidated document — but only if it is
+  // research. A synthesizer that answered about its prompt would otherwise become
+  // the report's "findings" and later be surfaced into the agent's instructions.
+  const { isUsableSynthesis } = await import('./research-surface.mjs');
+  const usableReport = finalReport && isUsableSynthesis(finalReport) ? finalReport : null;
+  if (finalReport && !usableReport) {
+    logger.info({ subsystem: 'deep-research', message: `Discarded unusable synthesis for "${task.target}" (meta-response or too thin); keeping sources only.` });
+  } else if (finalReport) {
     draftSlug = saveSynthesizedReportToDraft(task.target, finalReport, inboxDir) || draftSlug;
   }
 
@@ -103,21 +109,12 @@ export async function handleProactiveResearch(task, context = {}) {
     );
   }
 
-  // Phase 5: Also add topic to the Research Agenda for ongoing tracking
-  addToAgenda({
-    topic: task.target,
-    priority: 75,
-    source: 'deep-research-task',
-    rationale: task.body || '',
-    tags: ['deep-research'],
-  });
-
   // Always return structured result so executors can advance the research queue with a real node slug
   if (!draftSlug) {
     draftSlug = `research-report-${slugify(task.target)}`;
   }
   return {
-    report: finalReport || null,
+    report: usableReport,
     factSlug: draftSlug,
     sources: flatResults.length,
     queries,
@@ -589,34 +586,30 @@ export async function promoteDraftToVault(parentTopic, inboxDir, vaultDir) {
 
 async function synthesizeLocally(task, results, runtimeConfig) {
   const today = getLocalizedDateTime();
-  const cutoff = runtimeConfig?.training_cutoff || 'January 2025';
 
+  // Plain, honest task text. callLocalRuntime hands `system` to CLI agents as part
+  // of the *user* turn, so fake <system_instructions> tags, a persona and a
+  // mandatory scratchpad read as a prompt injection — and in the live vault 88 of
+  // 93 reports were the agent refusing or questioning the prompt instead of
+  // researching. Sources carry their URLs so citations are possible.
   const sourceSummary = results
-    .map(r => `[${r.source}] ${r.title}: ${r.snippet?.slice(0, 300)}`)
+    .map((r, i) => `${i + 1}. ${r.title || 'Untitled'}${r.url ? ` <${r.url}>` : ''} (${r.source})\n   ${String(r.snippet || '').slice(0, 300).replace(/\s+/g, ' ')}`)
     .join('\n');
 
-  const system = `<system_instructions>
-You are a Deep Research Synthesizer.
-- Synthesize research results into a concise report.
-- Cite sources inline [Source: URL].
-- Prioritize fresh, timely information published after the cutoff.
-- ALWAYS use a <scratchpad> block first to outline your synthesis and evaluate source reliability before writing the final report.
-</system_instructions>
+  const system = `This request comes from Total Recall, the user's own background research daemon, and the search results below were fetched by it for the topic. Today is ${today}.`;
 
-<context>
-Today's date and time is ${today}.
-The model's training data cutoff is ${cutoff}.
-</context>`;
+  const prompt = `Summarize what these search results establish about: "${task.target}"
 
-  const prompt = `<user_goal>
-Topic: "${task.target}"
-</user_goal>
+Write Markdown with exactly these sections:
+## Key findings
+3–7 bullets. Each is a concrete, specific fact (versions, limits, dates, API names, required steps) supported by the results, ending with its source URL in parentheses.
+## Caveats
+Bullets on what the results leave uncertain or contradict. Omit the section if there is nothing to say.
 
-<retrieved_docs>
-${sourceSummary.slice(0, 6000)}
-</retrieved_docs>
+Use only what the results support; if they are too thin to support a finding, say so in one line under Key findings instead of guessing.
 
-Please synthesize the research results.`;
+Search results:
+${sourceSummary.slice(0, 6000)}`;
 
   try {
     let report = await callLocalRuntime(prompt, system, runtimeConfig);