pi-research 1.0.0

package/lib/web-research.js

@@ -0,0 +1,718 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+
+import { complete } from "@mariozechner/pi-ai";
+
+import profiles from "./research-profiles.json" with { type: "json" };
+import { createResearchResult } from "./types.js";
+import {
+  buildConfidenceSummary,
+  buildDeepQueries,
+  buildFallbackQueries,
+  buildFastQueries,
+  buildFollowUpQuery,
+  buildJinaReaderUrl,
+  classifySourceType,
+  compactResearchPayload,
+  normalizePaperTitle,
+  detectConflictSignals,
+  evaluateSufficiency,
+  extractCodeBlocks,
+  extractDuckDuckGoLiteResults,
+  extractDuckDuckGoResults,
+  extractJinaSearchResults,
+  extractPageSnapshot,
+  extractPublishDate,
+  factCheckAnswer,
+  formatResearchResponse,
+  normalizeUrl,
+  parseDeepQueryPlan,
+  prioritizeSourceEntries,
+  rankFetchedPages,
+  rankSearchResults,
+  scoreSourceEntry,
+  selectRelevantChunks,
+} from "./research.js";
+import { planResearch } from "./planner.js";
+import {
+  clearResearchMemory,
+  getResearchMemory,
+  hashResearchQuery,
+  readCachedResult,
+  setResearchMemory,
+  writeCachedResult,
+} from "./research-memory.js";
+
+const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36";
+const MIN_PAGE_TEXT = 300;
+const SEARCH_CACHE_TTL_MS = 5 * 60 * 1000;
+const PAGE_CACHE_TTL_MS = 30 * 60 * 1000;
+const searchCache = new Map();
+const pageCache = new Map();
+
+function getCacheValue(cache, key) {
+  const entry = cache.get(key);
+  if (!entry) return null;
+  if (entry.expiresAt <= Date.now()) {
+    cache.delete(key);
+    return null;
+  }
+  return entry.value;
+}
+
+function setCacheValue(cache, key, value, ttlMs) {
+  cache.set(key, { value, expiresAt: Date.now() + ttlMs });
+  return value;
+}
+
+function hashText(text) {
+  return createHash("sha1").update(String(text || "")).digest("hex");
+}
+
+function normalizeResearchOptions(input = "fast") {
+  if (typeof input === "string") return { mode: input };
+  if (input && typeof input === "object") return input;
+  return { mode: "fast" };
+}
+
+export function resolveResearchConfig(input = "fast") {
+  const options = normalizeResearchOptions(input);
+  const base = profiles[options.mode] || profiles.fast;
+  const deep = options.deepResearchConfig || {};
+
+  return {
+    ...base,
+    ...options,
+    mode: base.mode,
+    maxTurns: options.maxTurns ?? (deep.depth ? Math.max(base.maxTurns || 1, deep.depth) : (base.maxTurns || 1)),
+    maxQueries: options.maxQueries ?? (deep.breadth ? Math.max(base.maxQueries || 2, deep.breadth * (deep.depth || 1)) : (base.maxQueries || 2)),
+    maxPages: options.maxSites ?? options.maxPages ?? base.maxPages,
+    allowedSourceTypes: options.allowedSourceTypes ?? base.allowedSourceTypes,
+    searchProvider: options.searchProvider ?? base.searchProvider,
+    concurrentQueries: deep.concurrency ?? options.concurrentQueries ?? 3,
+    depth: deep.depth ?? 1,
+    breadth: deep.breadth ?? 2,
+    pageTextLimit: options.pageTextLimit ?? base.pageTextLimit,
+    minPageText: options.minPageText ?? base.minPageText ?? MIN_PAGE_TEXT,
+    preferRecent: options.preferRecent ?? base.preferRecent ?? false,
+    minYear: options.minYear ?? base.minYear,
+    maxYear: options.maxYear ?? base.maxYear,
+    cacheTtlMs: options.cacheTtlMs ?? base.cacheTtlMs ?? 24 * 60 * 60 * 1000,
+    files: Array.isArray(options.files) ? options.files : [],
+    isolate: Boolean(options.isolate || process.env.RESEARCH_ISOLATE === "1"),
+    force: Boolean(options.force),
+    format: options.format ?? "markdown",
+  };
+}
+
+export function getResearchConfig(mode = "fast") {
+  return resolveResearchConfig(mode);
+}
+
+export function resolveResearchModel(ctx) {
+  return process.env.WEB_RESEARCH_MODEL || ctx?.model || null;
+}
+
+function textFromCompletion(response) {
+  return response.content.filter((part) => part.type === "text").map((part) => part.text).join("\n").trim();
+}
+
+function parseJsonBlock(text) {
+  const trimmed = String(text || "").trim();
+  const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i);
+  const candidate = fenced ? fenced[1].trim() : trimmed;
+  return JSON.parse(candidate);
+}
+
+async function completeWithResearchModel(ctx, signal, prompt, reasoningEffort = "low") {
+  if (typeof ctx?.completeResearch === "function") {
+    return ctx.completeResearch(prompt, { signal, reasoningEffort });
+  }
+
+  const model = resolveResearchModel(ctx);
+  if (!model) return null;
+
+  const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
+  if (!auth.ok || !auth.apiKey) return null;
+
+  const response = await complete(model, {
+    messages: [{ role: "user", content: [{ type: "text", text: prompt }], timestamp: Date.now() }],
+  }, {
+    apiKey: auth.apiKey,
+    headers: auth.headers,
+    signal,
+    reasoningEffort,
+  });
+
+  if (response.stopReason === "aborted") return null;
+  return textFromCompletion(response);
+}
+
+export async function buildQueries(query, mode = "fast", ctx, signal) {
+  const config = getResearchConfig(mode);
+  if (config.mode === "code") {
+    return planResearch(query, "code").subqueries.slice(0, config.maxQueries);
+  }
+  if (config.mode === "deep" || config.mode === "academic") {
+    const prompt = [
+      "Generate web research search queries as JSON only.",
+      'Return shape: {"queries":["..."]}',
+      config.mode === "academic"
+        ? "Use 3-5 focused paper-search queries covering arXiv, DOI, Semantic Scholar, benchmarks, and official references."
+        : "Use 3-5 focused queries covering official docs, examples, source/readme, and recent status when relevant.",
+      `Question: ${query}`,
+    ].join("\n");
+
+    try {
+      const text = await completeWithResearchModel(ctx, signal, prompt, "low");
+      if (text) return parseDeepQueryPlan(text, query, config.maxQueries);
+    } catch {
+      // fall through
+    }
+
+    return buildDeepQueries(query, config.maxQueries);
+  }
+
+  return buildFastQueries(query, config.maxQueries);
+}
+
+function withTimeoutSignal(signal, timeoutMs) {
+  if (!timeoutMs) return signal;
+  const timeoutSignal = AbortSignal.timeout(timeoutMs);
+  return signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
+}
+
+async function fetchTextWithRetry(url, signal, attempts = 2, headers = {
+  "user-agent": USER_AGENT,
+  "accept-language": "en-US,en;q=0.9",
+}, timeoutMs) {
+  let lastError;
+  for (let attempt = 0; attempt < attempts; attempt++) {
+    try {
+      const response = await fetch(url, { headers, redirect: "follow", signal: withTimeoutSignal(signal, timeoutMs) });
+      return response;
+    } catch (error) {
+      lastError = error;
+      if (attempt + 1 < attempts) await new Promise((resolve) => setTimeout(resolve, 100 * (attempt + 1)));
+    }
+  }
+  throw lastError;
+}
+
+function inferAllowedSources(config) {
+  if (!Array.isArray(config.allowedSources) || config.allowedSources.length === 0) return null;
+  return new Set(config.allowedSources.map((value) => String(value).toLowerCase()));
+}
+
+function filterBySourceOptions(result, config) {
+  const allowed = inferAllowedSources(config);
+  if (!allowed) return true;
+  const type = classifySourceType(result.url, result.title);
+  if (allowed.has("official_docs") && type === "official_doc") return true;
+  if (allowed.has("paper") && type === "paper") return true;
+  if (allowed.has(type)) return true;
+  try {
+    const hostname = new URL(result.url).hostname.toLowerCase();
+    if (allowed.has(hostname)) return true;
+  } catch {
+    // ignore
+  }
+  return false;
+}
+
+function filterSearchResults(results, config = getResearchConfig()) {
+  return results.filter((result) => {
+    try {
+      const hostname = new URL(result.url).hostname;
+      if (hostname.includes("duckduckgo.com") || !result.snippet) return false;
+      const sourceType = classifySourceType(result.url, result.title);
+      if (Array.isArray(config.allowedSourceTypes) && !config.allowedSourceTypes.includes(sourceType)) return false;
+      return true;
+    } catch {
+      return false;
+    }
+  });
+}
+
+function sourceFromPaper(title, url, snippet, publishDate) {
+  return { title: normalizePaperTitle(title), url, snippet, publishDate, sourceType: "paper" };
+}
+
+async function searchArxiv(query, signal, config) {
+  try {
+    const response = await fetchTextWithRetry(`https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${config.resultsPerQuery}`, signal, 2, {}, config.pageTimeoutMs);
+    const xml = await response.text();
+    return [...xml.matchAll(/<entry>([\s\S]*?)<\/entry>/g)].map((match) => {
+      const entry = match[1];
+      const url = entry.match(/<id>([^<]+)<\/id>/)?.[1] || "";
+      const title = normalizePaperTitle((entry.match(/<title>([\s\S]*?)<\/title>/)?.[1] || "").replace(/\s+/g, " ").trim());
+      const summary = (entry.match(/<summary>([\s\S]*?)<\/summary>/)?.[1] || "").replace(/\s+/g, " ").trim();
+      const published = entry.match(/<published>([^<]+)<\/published>/)?.[1]?.slice(0, 10);
+      return sourceFromPaper(title, url, summary, published);
+    }).filter((item) => item.url && item.title);
+  } catch {
+    return [];
+  }
+}
+
+async function searchSemanticScholar(query, signal, config) {
+  try {
+    const response = await fetchTextWithRetry(`https://api.semanticscholar.org/graph/v1/paper/search?query=${encodeURIComponent(query)}&limit=${config.resultsPerQuery}&fields=title,abstract,url,year`, signal, 2, {}, config.pageTimeoutMs);
+    const data = await response.json();
+    return (data?.data || []).map((item) => sourceFromPaper(item.title, item.url || `https://www.semanticscholar.org/search?q=${encodeURIComponent(item.title)}`, item.abstract || "", item.year ? `${item.year}-01-01` : null)).filter((item) => item.title);
+  } catch {
+    return [];
+  }
+}
+
+async function searchCrossref(query, signal, config) {
+  try {
+    const response = await fetchTextWithRetry(`https://api.crossref.org/works?query.title=${encodeURIComponent(query)}&rows=${config.resultsPerQuery}`, signal, 2, { "user-agent": USER_AGENT }, config.pageTimeoutMs);
+    const data = await response.json();
+    return (data?.message?.items || []).map((item) => {
+      const doi = item.DOI ? `https://doi.org/${item.DOI}` : "";
+      const dateParts = item.published?.["date-parts"]?.[0] || [];
+      const publishDate = dateParts.length ? `${String(dateParts[0]).padStart(4, "0")}-${String(dateParts[1] || 1).padStart(2, "0")}-${String(dateParts[2] || 1).padStart(2, "0")}` : null;
+      return sourceFromPaper(item.title?.[0] || "", doi, String(item.abstract || "").replace(/<[^>]+>/g, " "), publishDate);
+    }).filter((item) => item.url && item.title);
+  } catch {
+    return [];
+  }
+}
+
+export async function searchDuckDuckGo(query, signal, config = getResearchConfig()) {
+  const cacheKey = `${query}::${config.resultsPerQuery}::${config.searchProvider || "ddg_html"}::${JSON.stringify({
+    allowedSourceTypes: config.allowedSourceTypes || [],
+    allowedSources: config.allowedSources || [],
+    preferRecent: config.preferRecent || false,
+    minYear: config.minYear || "",
+    maxYear: config.maxYear || "",
+  })}`;
+  const cached = config.isolate ? null : getCacheValue(searchCache, cacheKey);
+  if (cached) return cached;
+
+  let results = [];
+  const providerOrder = config.searchProvider === "lite"
+    ? ["lite", "ddg_html", "jina"]
+    : config.searchProvider === "jina"
+      ? ["jina", "ddg_html", "lite"]
+      : ["ddg_html", "lite", "jina"];
+
+  for (const provider of providerOrder) {
+    try {
+      if (provider === "ddg_html") {
+        const htmlResponse = await fetchTextWithRetry(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, signal, 1);
+        results = filterSearchResults(extractDuckDuckGoResults(await htmlResponse.text()), config);
+      } else if (provider === "lite") {
+        const liteResponse = await fetchTextWithRetry(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`, signal);
+        results = filterSearchResults(extractDuckDuckGoLiteResults(await liteResponse.text()), config);
+      } else {
+        const jinaResponse = await fetchTextWithRetry(`https://r.jina.ai/http://duckduckgo.com/html/?q=${encodeURIComponent(query)}`, signal, 2, {});
+        results = filterSearchResults(extractJinaSearchResults(await jinaResponse.text()), config);
+      }
+    } catch {
+      results = [];
+    }
+    if (results.length > 0) break;
+  }
+
+  if (config.mode === "academic") {
+    const academic = [
+      ...(await searchArxiv(query, signal, config)),
+      ...(await searchSemanticScholar(query, signal, config)),
+      ...(await searchCrossref(query, signal, config)),
+    ];
+    results = [...results, ...academic];
+  }
+
+  const ranked = rankSearchResults(results, query, config.resultsPerQuery, config);
+  return config.isolate ? ranked : setCacheValue(searchCache, cacheKey, ranked, SEARCH_CACHE_TTL_MS);
+}
+
+function shouldSkipUrl(url) {
+  return /(\/login|\/signin|\/sign-in|\/account|\/subscribe|\/checkout)/i.test(url);
+}
+
+function shouldUseJinaFirst(url) {
+  return /(^|\.)medium\.com$|(^|\.)dev\.to$|(^|\.)substack\.com$/i.test(new URL(url).hostname);
+}
+
+function pageFromText(title, url, text, config, extra = {}) {
+  const trimmed = String(text || "").slice(0, config.pageTextLimit).trim();
+  if (trimmed.length < config.minPageText) return null;
+  return { title, url, text: trimmed, codeBlocks: extractCodeBlocks(text), ...extra };
+}
+
+async function fetchJinaPageSource(url, signal, config) {
+  if (!config.useJinaFallback || shouldSkipUrl(url)) return null;
+  try {
+    const response = await fetchTextWithRetry(buildJinaReaderUrl(url), signal, 2, {}, config.pageTimeoutMs);
+    const body = await response.text();
+    const firstLine = body.split("\n").find((line) => line.trim().replace(/^#+\s*/, ""));
+    const title = firstLine ? firstLine.trim().replace(/^#+\s*/, "") : url;
+    return pageFromText(title, url, body, config, { sourceType: classifySourceType(url, title) });
+  } catch {
+    return null;
+  }
+}
+
+function withinTimeframe(page, config) {
+  if (!config.minYear && !config.maxYear && !config.preferRecent) return true;
+  const year = page.publishDate ? Number(String(page.publishDate).slice(0, 4)) : null;
+  if (config.minYear && year && year < config.minYear) return false;
+  if (config.maxYear && year && year > config.maxYear) return false;
+  return true;
+}
+
+export async function fetchPageSource(url, signal, config = getResearchConfig()) {
+  if (shouldSkipUrl(url)) return null;
+  const cacheKey = `${normalizeUrl(url)}::${config.pageTextLimit}::${JSON.stringify({
+    preferRecent: config.preferRecent || false,
+    minYear: config.minYear || "",
+    maxYear: config.maxYear || "",
+    useJinaFallback: Boolean(config.useJinaFallback),
+  })}`;
+  const cached = config.isolate ? null : getCacheValue(pageCache, cacheKey);
+  if (cached) return cached;
+  if (shouldUseJinaFirst(url)) {
+    const first = await fetchJinaPageSource(url, signal, config);
+    return config.isolate ? first : setCacheValue(pageCache, cacheKey, first, PAGE_CACHE_TTL_MS);
+  }
+
+  try {
+    const response = await fetchTextWithRetry(url, signal, 2, {
+      "user-agent": USER_AGENT,
+      "accept-language": "en-US,en;q=0.9",
+    }, config.pageTimeoutMs);
+
+    const contentType = response.headers.get("content-type") || "";
+    if (!contentType.includes("text/html") && !contentType.includes("text/plain")) return null;
+
+    const body = await response.text();
+    const snapshot = extractPageSnapshot(body, response.url || url);
+    const page = pageFromText(snapshot.title, snapshot.url, snapshot.text, config, {
+      publishDate: extractPublishDate(body),
+      sourceType: classifySourceType(snapshot.url, snapshot.title),
+      codeBlocks: snapshot.codeBlocks,
+    });
+
+    const resolved = page || await fetchJinaPageSource(url, signal, config);
+    const finalPage = resolved && withinTimeframe(resolved, config) ? resolved : null;
+    return config.isolate ? finalPage : setCacheValue(pageCache, cacheKey, finalPage, PAGE_CACHE_TTL_MS);
+  } catch {
+    const fallback = await fetchJinaPageSource(url, signal, config);
+    return config.isolate ? fallback : setCacheValue(pageCache, cacheKey, fallback, PAGE_CACHE_TTL_MS);
+  }
+}
+
+async function readLocalFiles(paths, config) {
+  const pages = [];
+  for (const path of paths) {
+    try {
+      const text = await readFile(path, "utf8");
+      const page = pageFromText(path.split("/").pop() || path, `file://${path}`, text, config, {
+        sourceType: "file",
+        publishDate: null,
+        local: true,
+      });
+      if (page) pages.push(page);
+    } catch {
+      // ignore unreadable files
+    }
+  }
+  return pages;
+}
+
+function fallbackSynthesis(query, pages) {
+  const sources = prioritizeSourceEntries(pages.slice(0, Math.min(5, pages.length)).map((page, index) => ({
+    number: index + 1,
+    title: page.title,
+    url: page.url,
+    freshness: page.publishDate ? page.publishDate.slice(0, 10) : undefined,
+    sourceType: page.sourceType,
+    score: page.score,
+    authoritative: page.authoritative,
+  })), query);
+
+  const bullets = pages.slice(0, Math.min(5, pages.length)).map((page, index) => `${page.text.replace(/\s+/g, " ").slice(0, 180).trim()} [${index + 1}]`);
+  const answer = pages.length
+    ? `I found ${pages.length} relevant sources for “${query}” [1]. The strongest sources are summarized below.`
+    : `I could not find enough reliable sources for “${query}”.`;
+
+  return { answer, bullets, sources, citations: sources.map((source) => ({ text: source.title, sourceIndex: source.number || 0 })) };
+}
+
+export async function synthesizeResearch(query, pages, ctx, signal) {
+  const prompt = [
+    "You are a concise research synthesizer.",
+    "Answer only from the provided sources.",
+    "Return only JSON with this exact shape:",
+    '{"answer":"...","bullets":["..."],"sourceIds":[1,2],"citations":[{"text":"...","sourceIndex":1}]}',
+    "Rules:",
+    "- answer: one short paragraph with inline citations like [1] [2]",
+    "- bullets: 3-5 short bullet strings, each with inline citations",
+    `Question: ${query}`,
+    "Sources:",
+    ...pages.map((page, index) => [
+      `[${index + 1}] ${page.title}`,
+      `URL: ${page.url}`,
+      `Type: ${page.sourceType || classifySourceType(page.url, page.title)}`,
+      `Score: ${typeof page.score === "number" ? page.score : scoreSourceEntry(page, query).total}`,
+      `Text: ${page.text}`,
+    ].join("\n")),
+  ].join("\n\n");
+
+  try {
+    const text = await completeWithResearchModel(ctx, signal, prompt, "medium");
+    const parsed = text ? parseJsonBlock(text) : null;
+    if (parsed && typeof parsed.answer === "string" && Array.isArray(parsed.bullets) && Array.isArray(parsed.sourceIds)) {
+      const sourceIds = [...new Set(parsed.sourceIds.map((id) => Number(id)).filter((id) => Number.isInteger(id) && id >= 1 && id <= pages.length))];
+      if (sourceIds.length > 0) {
+        const sources = prioritizeSourceEntries(sourceIds.map((id) => ({
+          number: id,
+          title: pages[id - 1].title,
+          url: pages[id - 1].url,
+          freshness: pages[id - 1].publishDate ? pages[id - 1].publishDate.slice(0, 10) : undefined,
+          sourceType: pages[id - 1].sourceType || classifySourceType(pages[id - 1].url, pages[id - 1].title),
+          score: typeof pages[id - 1].score === "number" ? pages[id - 1].score : scoreSourceEntry(pages[id - 1], query).total,
+          authoritative: typeof pages[id - 1].authoritative === "boolean" ? pages[id - 1].authoritative : scoreSourceEntry(pages[id - 1], query).authoritative,
+        })), query);
+        return {
+          answer: parsed.answer.trim(),
+          bullets: parsed.bullets.map((item) => String(item).trim()).filter(Boolean).slice(0, 5),
+          sources,
+          citations: Array.isArray(parsed.citations) ? parsed.citations.slice(0, 8) : sources.map((source) => ({ text: source.title, sourceIndex: source.number || 0 })),
+        };
+      }
+    }
+  } catch {
+    // fall through
+  }
+
+  return fallbackSynthesis(query, pages);
+}
+
+function planSubqueries(rootQuery, currentQuery, config, sufficiency) {
+  const queries = [];
+  if (sufficiency?.openSubQuestions?.length) queries.push(...sufficiency.openSubQuestions);
+  if (queries.length === 0) queries.push(buildFollowUpQuery(currentQuery || rootQuery, []));
+  return [...new Set(queries.filter(Boolean))].slice(0, Math.max(1, config.breadth || 2));
+}
+
+function formatResultText(result) {
+  return formatResearchResponse({ answer: result.answer, bullets: result.bullets, sources: result.sources, confidence: result.confidence });
+}
+
+function modeCacheKey(query, config) {
+  return `${config.mode}:${hashResearchQuery(query)}:${hashText(JSON.stringify({
+    files: config.files || [],
+    allowedSources: config.allowedSources || [],
+    allowedSourceTypes: config.allowedSourceTypes || [],
+    maxPages: config.maxPages,
+    maxTurns: config.maxTurns,
+    maxQueries: config.maxQueries,
+    resultsPerQuery: config.resultsPerQuery,
+    preferRecent: config.preferRecent,
+    minYear: config.minYear || "",
+    maxYear: config.maxYear || "",
+    searchProvider: config.searchProvider || "",
+  }))}`;
+}
+
+export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast") {
+  const config = getResearchConfig(mode);
+  const cacheKey = modeCacheKey(query, config);
+
+  if (!config.isolate && !config.force) {
+    const memoryHit = getResearchMemory(cacheKey);
+    if (memoryHit) return memoryHit;
+    const persistentHit = readCachedResult(cacheKey);
+    if (persistentHit) {
+      setResearchMemory(cacheKey, persistentHit);
+      return persistentHit;
+    }
+  }
+
+  const emit = (stage, text) => onUpdate?.({ content: [{ type: "text", text: `[pipeline:${stage}] ${text}` }] });
+  const startedAt = Date.now();
+  const seenUrls = new Set();
+  const seenContentHashes = new Set();
+  const mergedPages = [];
+  const allCodeBlocks = [];
+  let subqueries = [];
+  let followupRounds = 0;
+  let followupQuery = null;
+  let conflictDetected = false;
+  let conflictSummary = "";
+  let conflictingSourcePairs = [];
+  let sufficiency = { sufficient: false, confidenceScore: 0.1, missingAspects: [], openSubQuestions: [] };
+  let currentQueries = await buildQueries(query, config.mode, ctx, signal);
+  subqueries = [...currentQueries];
+
+  const localPages = await readLocalFiles(config.files || [], config);
+  for (const page of localPages) {
+    const scored = scoreSourceEntry(page, query);
+    const contentHash = hashText(page.text);
+    if (seenContentHashes.has(contentHash)) continue;
+    seenContentHashes.add(contentHash);
+    mergedPages.push({
+      ...page,
+      score: scored.total,
+      authoritative: scored.authoritative,
+      freshness: scored.freshness,
+      sourceType: page.sourceType || scored.sourceType,
+      local: true,
+    });
+    if (Array.isArray(page.codeBlocks)) allCodeBlocks.push(...page.codeBlocks);
+  }
+
+  for (let turn = 0; turn < Math.max(1, config.maxTurns || 1); turn++) {
+    emit(turn === 0 ? "plan" : "followup", `Planning ${config.mode} research... turn=${turn + 1}/${config.maxTurns}`);
+    const queriesThisTurn = currentQueries.slice(0, config.maxQueries);
+    emit("search", `Searching ${queriesThisTurn.length} queries...`);
+
+    const searchGroups = await Promise.all(queriesThisTurn.map((subquery) => searchDuckDuckGo(subquery, signal, config)));
+    const results = rankSearchResults(searchGroups.flat(), query, config.maxPages * 2, config)
+      .filter((result) => {
+        const key = normalizeUrl(result.url);
+        if (seenUrls.has(key)) return false;
+        seenUrls.add(key);
+        return true;
+      })
+      .slice(0, config.maxPages);
+
+    emit("fetch", `Reading ${results.length} sources...`);
+    const pageCandidates = await Promise.all(results.map((result) => fetchPageSource(result.url, signal, config)));
+    const rankedPages = rankFetchedPages(pageCandidates.filter(Boolean).map((page) => {
+      const scored = scoreSourceEntry(page, query);
+      return {
+        ...page,
+        score: typeof page.score === "number" ? page.score : scored.total,
+        authoritative: typeof page.authoritative === "boolean" ? page.authoritative : scored.authoritative,
+        freshness: page.freshness || scored.freshness,
+        sourceType: page.sourceType || scored.sourceType,
+        text: selectRelevantChunks(page.text, query, config.maxChunksPerPage).join("\n\n") || page.text,
+      };
+    }).filter((page) => withinTimeframe(page, config)), query, config.maxPages, config);
+
+    for (const page of prioritizeSourceEntries(rankedPages, query)) {
+      const key = normalizeUrl(page.url);
+      const contentHash = hashText(page.text);
+      if (mergedPages.some((existing) => normalizeUrl(existing.url) === key)) continue;
+      if (seenContentHashes.has(contentHash)) continue;
+      seenContentHashes.add(contentHash);
+      mergedPages.push(page);
+      if (Array.isArray(page.codeBlocks)) allCodeBlocks.push(...page.codeBlocks);
+    }
+
+    const conflict = detectConflictSignals(mergedPages);
+    conflictDetected = conflict.detected;
+    conflictSummary = conflict.conflictSummary || "";
+    conflictingSourcePairs = conflict.conflictingSourcePairs || [];
+
+    const minSources = config.mode === "fast"
+      ? (mergedPages.some((page) => page.authoritative) ? 1 : Math.max(3, config.minSources || 3))
+      : (config.minSources || 3);
+
+    sufficiency = evaluateSufficiency({
+      query,
+      sources: mergedPages,
+      conflictDetected,
+      minSources,
+    });
+
+    if (mergedPages.length >= minSources && sufficiency.confidenceScore >= 0.85 && (!conflictDetected || mergedPages.some((page) => page.authoritative))) {
+      sufficiency = { ...sufficiency, sufficient: true };
+    }
+
+    if (sufficiency.sufficient || turn === (config.maxTurns - 1)) break;
+
+    followupRounds += 1;
+    followupQuery = buildFollowUpQuery(query, mergedPages);
+    currentQueries = planSubqueries(query, followupQuery, config, sufficiency);
+    subqueries = [...new Set([...subqueries, ...currentQueries])];
+  }
+
+  if (mergedPages.length === 0) {
+    return {
+      ok: false,
+      action: "web_research",
+      query,
+      mode: config.mode,
+      subqueries,
+      reason: "No readable web sources were retrieved.",
+      openSubQuestions: buildFallbackQueries(query),
+      error: "No readable web sources were retrieved.",
+    };
+  }
+
+  emit("synthesis", `Synthesizing ${mergedPages.length} sources...`);
+  const synthesis = await synthesizeResearch(query, mergedPages, ctx, signal);
+  const sources = prioritizeSourceEntries(synthesis.sources.map((source) => ({
+    ...source,
+    ...(source.number ? {} : { number: undefined }),
+  })), query);
+  const confidence = buildConfidenceSummary(mergedPages, { conflictDetected, followupRounds });
+  const codeBlocks = [...new Set(allCodeBlocks)].slice(0, 5);
+  const sourceTypes = [...new Set(sources.map((source) => source.sourceType).filter(Boolean))];
+  const openSubQuestions = sufficiency.openSubQuestions.length ? sufficiency.openSubQuestions : (sufficiency.sufficient ? subqueries.slice(0, Math.min(3, subqueries.length)) : []);
+
+  const factCheck = factCheckAnswer(synthesis.answer, mergedPages);
+  const unverifiedRatio = synthesis.answer ? factCheck.unverifiedClaims.length / Math.max(1, factCheck.verifiedClaims.length + factCheck.unverifiedClaims.length) : 0;
+  const normalizedResult = createResearchResult({
+    answer: synthesis.answer,
+    bullets: synthesis.bullets,
+    citations: synthesis.citations || [],
+    sources,
+    codeBlocks,
+    sufficient: sufficiency.sufficient && unverifiedRatio <= 0.2,
+    missingAspects: sufficiency.missingAspects,
+    openSubQuestions,
+    conflictSummary: conflictSummary || sufficiency.conflictSummary || "",
+    confidence: sufficiency.confidenceScore,
+    sourceTypes,
+    unverifiedClaims: factCheck.unverifiedClaims,
+    meta: {
+      turns: followupRounds + 1,
+      sitesVisited: mergedPages.length,
+      totalFetchTimeMs: Date.now() - startedAt,
+      cacheHit: false,
+    },
+  });
+
+  const result = {
+    ok: true,
+    action: "web_research",
+    query,
+    mode: config.mode,
+    subqueries,
+    followupRounds,
+    followupQuery,
+    conflictDetected,
+    conflictSummary: normalizedResult.conflictSummary,
+    conflictingSourcePairs,
+    pagesRead: mergedPages.length,
+    answer: normalizedResult.answer,
+    bullets: normalizedResult.bullets,
+    citations: normalizedResult.citations,
+    sources: normalizedResult.sources,
+    sourceTypes,
+    codeBlocks: normalizedResult.codeBlocks,
+    confidence,
+    meta: normalizedResult.meta,
+    confidenceScore: sufficiency.confidenceScore,
+    authoritativeSourcesFound: sufficiency.authoritativeSourcesFound,
+    sufficient: normalizedResult.sufficient,
+    followupRecommended: !normalizedResult.sufficient,
+    openSubQuestions: normalizedResult.openSubQuestions,
+    missingAspects: normalizedResult.missingAspects,
+    unverifiedClaims: normalizedResult.unverifiedClaims,
+    contentText: formatResultText({ answer: normalizedResult.answer, bullets: normalizedResult.bullets, sources: normalizedResult.sources, confidence }),
+  };
+
+  setResearchMemory(cacheKey, result);
+  writeCachedResult(cacheKey, result, config.cacheTtlMs);
+  return result;
+}
+
+export { compactResearchPayload, clearResearchMemory };