pi-research 1.3.1 → 1.4.0

package/lib/web-research.js

@@ -13,6 +13,7 @@ import {
   buildFallbackQueries,
   buildFastQueries,
   buildFollowUpQuery,
+  buildActionBasedFollowUpQuery,
   buildJinaReaderUrl,
   classifySourceType,
   compactResearchPayload,
@@ -35,8 +36,8 @@ import {
   scoreSourceEntry,
   selectRelevantChunks,
 } from "./research.js";
-import { pageFetchAdapter } from "./page-fetch-adapter.js";
-import { pageQualitySignals } from "./research-policy.js";
+import { getScraplingRuntimeStatus, pageFetchAdapter } from "./page-fetch-adapter.js";
+import { isUsableContent, pageQualitySignals } from "./research-policy.js";
 import { resolveOutputFormat, shouldRequireAuthoritativeSources } from "./research-output.js";
 import { planResearch } from "./planner.js";
 import {
@@ -53,6 +54,7 @@ const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/
 const MIN_PAGE_TEXT = 300;
 const SEARCH_CACHE_TTL_MS = 5 * 60 * 1000;
 const PAGE_CACHE_TTL_MS = 30 * 60 * 1000;
+const EXPENSIVE_PAGE_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
 const searchCache = new Map();
 const pageCache = new Map();
 
@@ -71,6 +73,10 @@ function setCacheValue(cache, key, value, ttlMs) {
   return value;
 }
 
+function pageCacheTtl(page) {
+  return page?.expensive ? EXPENSIVE_PAGE_CACHE_TTL_MS : PAGE_CACHE_TTL_MS;
+}
+
 function hashText(text) {
   return createHash("sha1").update(String(text || "")).digest("hex");
 }
@@ -165,6 +171,7 @@ export async function buildQueries(query, mode = "fast", ctx, signal) {
   const hintedQueries = Array.isArray(config.queryHints) && config.queryHints.length
     ? config.queryHints.map((hint) => `${query} ${hint}`)
     : [];
+
   if (config.mode === "code") {
     return [...new Set([...planResearch(query, "code").subqueries, ...hintedQueries])].slice(0, config.maxQueries);
   }
@@ -208,6 +215,7 @@ async function fetchTextWithRetry(url, signal, attempts = 2, headers = {
       return response;
     } catch (error) {
       lastError = error;
+      if (signal?.aborted || error?.name === "AbortError" || error?.name === "TimeoutError") throw error;
       if (attempt + 1 < attempts) await new Promise((resolve) => setTimeout(resolve, 100 * (attempt + 1)));
     }
   }
@@ -428,7 +436,7 @@ export async function fetchPageSource(url, signal, config = getResearchConfig())
   if (shouldUseJinaFirst(url)) {
     const first = finalizeFetchedPage(await fetchJinaPageSource(url, signal, config), config, { url, contentType: "text/plain" });
     if (first && withinTimeframe(first, config)) {
-      const page = config.isolate ? first : setCacheValue(pageCache, cacheKey, first, PAGE_CACHE_TTL_MS);
+      const page = config.isolate ? first : setCacheValue(pageCache, cacheKey, first, pageCacheTtl(first));
       await logResearchEvent("fetch_end", { url, via: "jina_first", success: Boolean(page), page: page ? { title: page.title, sourceType: page.sourceType, publishDate: page.publishDate, textLength: page.text?.length || 0 } : null });
       return page;
     }
@@ -444,7 +452,7 @@ export async function fetchPageSource(url, signal, config = getResearchConfig())
     if (!contentType.includes("text/html") && !contentType.includes("text/plain")) {
       const fallback = finalizeFetchedPage(await fetchJinaPageSource(url, signal, config), config, { url, contentType });
       if (fallback && withinTimeframe(fallback, config)) {
-        const page = config.isolate ? fallback : setCacheValue(pageCache, cacheKey, fallback, PAGE_CACHE_TTL_MS);
+        const page = config.isolate ? fallback : setCacheValue(pageCache, cacheKey, fallback, pageCacheTtl(fallback));
         await logResearchEvent("fetch_end", { url, via: "unsupported_content_type_fallback", success: Boolean(page), contentType, page: page ? { title: page.title, sourceType: page.sourceType, publishDate: page.publishDate, textLength: page.text?.length || 0 } : null });
         return page;
       }
@@ -479,27 +487,60 @@ export async function fetchPageSource(url, signal, config = getResearchConfig())
           codeBlocks: scraplingSnapshot.codeBlocks,
           fetchStatus: scrapling.status ?? 200,
           contentType: scrapling.contentType || "text/html",
+          expensive: true,
         });
+      } else if (assessment?.blocked || assessment?.dynamic) {
+        await logResearchEvent("fetch_scrapling_unavailable", { url, mode: assessment.mode, runtime: getScraplingRuntimeStatus?.() || null });
       }
     }
 
     const resolved = page || await fetchJinaPageSource(url, signal, config);
     const finalPage = finalizeFetchedPage(resolved, config, { url: response.url || url, status: response.status ?? 200, contentType });
     const stored = finalPage && withinTimeframe(finalPage, config)
-      ? (config.isolate ? finalPage : setCacheValue(pageCache, cacheKey, finalPage, PAGE_CACHE_TTL_MS))
+      ? (config.isolate ? finalPage : setCacheValue(pageCache, cacheKey, finalPage, pageCacheTtl(finalPage)))
       : null;
     await logResearchEvent("fetch_end", { url, success: Boolean(stored), page: stored ? { title: stored.title, sourceType: stored.sourceType, publishDate: stored.publishDate, textLength: stored.text?.length || 0 } : null });
     return stored;
   } catch (error) {
+    if (signal?.aborted || error?.name === "AbortError") {
+      await logResearchEvent("fetch_abort", { url });
+      return null;
+    }
     const fallback = finalizeFetchedPage(await fetchJinaPageSource(url, signal, config), config, { url, contentType: "text/plain" });
     const stored = fallback && withinTimeframe(fallback, config)
-      ? (config.isolate ? fallback : setCacheValue(pageCache, cacheKey, fallback, PAGE_CACHE_TTL_MS))
+      ? (config.isolate ? fallback : setCacheValue(pageCache, cacheKey, fallback, pageCacheTtl(fallback)))
       : null;
     await logResearchEvent("fetch_error", { url, error, fallback: stored ? { title: stored.title, sourceType: stored.sourceType, publishDate: stored.publishDate, textLength: stored.text?.length || 0 } : null });
     return stored;
   }
 }
 
+async function speculativeFetch(results, signal, config, query) {
+  const target = Math.max(1, config.minSources || 1);
+  const controllers = results.map(() => new AbortController());
+  const abortAll = () => controllers.forEach((controller) => controller.abort());
+  if (signal) signal.addEventListener("abort", abortAll, { once: true });
+
+  let usableCount = 0;
+  const pages = await Promise.all(results.map(async (result, index) => {
+    const scopedSignal = signal ? AbortSignal.any([signal, controllers[index].signal]) : controllers[index].signal;
+    const page = await fetchPageSource(result.url, scopedSignal, { ...config, query });
+    if (scopedSignal.aborted || !page) return null;
+    if (isUsableContent(page, { ...config, query })) {
+      usableCount += 1;
+      if (usableCount >= target) {
+        controllers.forEach((controller, controllerIndex) => {
+          if (controllerIndex !== index && !controller.signal.aborted) controller.abort();
+        });
+      }
+    }
+    return page;
+  }));
+
+  if (signal) signal.removeEventListener("abort", abortAll);
+  return pages.filter(Boolean);
+}
+
 async function readLocalFiles(paths, config) {
   const pages = [];
   for (const path of paths) {
@@ -620,8 +661,72 @@ function modeCacheKey(query, config) {
   }))}`;
 }
 
+import {
+  applyConflictTinyRouterDecision,
+  applySufficiencyTinyRouterDecision,
+  chooseTinyRouterDomain,
+  classifyConflictWithTinyRouter,
+  classifyDomainWithTinyRouter,
+  classifyFollowupWithTinyRouter,
+  classifySufficiencyWithTinyRouter,
+} from "./tiny-router.js";
+
+function missingAspectFromStructuredDecision(decision) {
+  if (decision === "need_authority") return "authoritative sources";
+  if (decision === "need_more_sources") return "readable sources";
+  if (decision === "need_recency") return "recent sources";
+  if (decision === "need_version_context") return "version context";
+  if (decision === "need_conflict_resolution") return "conflict resolution";
+  return null;
+}
+
+function withStructuredSufficiencyDecision(sufficiency, decision, query, seenUrls = []) {
+  if (!decision) return sufficiency;
+  if (decision === "sufficient") return sufficiency;
+
+  const missingAspect = missingAspectFromStructuredDecision(decision);
+  const followupQuery = buildActionBasedFollowUpQuery(query, decision, { seenUrls });
+
+  return {
+    ...sufficiency,
+    sufficient: false,
+    missingAspects: missingAspect
+      ? [...new Set([...(sufficiency.missingAspects || []), missingAspect])]
+      : sufficiency.missingAspects,
+    openSubQuestions: followupQuery
+      ? [...new Set([...(sufficiency.openSubQuestions || []), followupQuery])]
+      : sufficiency.openSubQuestions,
+  };
+}
+
+async function resolveQuestionDomain(query, mode, signal) {
+  const fallback = classifyQuestionDomain(query);
+  const normalizedMode = typeof mode === "object" ? mode?.mode || "fast" : mode;
+  try {
+    const tinyStartedAt = Date.now();
+    const tinyDomain = await classifyDomainWithTinyRouter(query, normalizedMode, signal);
+    const tinyLatencyMs = Date.now() - tinyStartedAt;
+    await logResearchEvent("tiny_router_latency", { task: "domain", latencyMs: tinyLatencyMs, accepted: Boolean(tinyDomain) });
+
+    const domain = chooseTinyRouterDomain(fallback, tinyDomain);
+    if (tinyDomain && domain !== fallback) {
+      await logResearchEvent("tiny_router_domain", { query, mode: normalizedMode, heuristicDomain: fallback, predictedDomain: tinyDomain, acceptedDomain: domain });
+      return domain;
+    }
+    if (tinyDomain && domain === fallback && tinyDomain !== fallback) {
+      await logResearchEvent("tiny_router_fallback", { task: "domain", query, mode: normalizedMode, heuristicDomain: fallback, predictedDomain: tinyDomain, reason: "high_risk_not_downgraded" });
+    }
+
+    await logResearchEvent("tiny_router_fallback", { task: "domain", query, mode: normalizedMode, heuristicDomain: fallback, reason: tinyDomain ? "heuristic_kept" : "tiny_router_unavailable_or_low_confidence" });
+    return fallback;
+  } catch (error) {
+    await logResearchEvent("tiny_router_fallback", { task: "domain", query, mode: normalizedMode, heuristicDomain: fallback, reason: "error", error });
+    return fallback;
+  }
+}
+
 export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast") {
-  const domain = classifyQuestionDomain(query);
+  const domain = await resolveQuestionDomain(query, mode, signal);
   const config = getResearchConfig(typeof mode === "object" ? { ...mode, domain } : { mode, domain });
   const cacheKey = modeCacheKey(query, config);
 
@@ -699,6 +804,9 @@ export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast"
     } else {
       lastEmptySearchSignature = null;
     }
+    const fetchWindow = config.mode === "fast"
+      ? Math.max(config.maxPages, Math.min(config.maxPages * 2, (config.minSources || 3) + 2))
+      : config.maxPages;
     const results = rankSearchResults(flatResults, query, config.maxPages * 2, config)
       .filter((result) => {
         const key = normalizeUrl(result.url);
@@ -706,10 +814,12 @@ export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast"
         seenUrls.add(key);
         return true;
       })
-      .slice(0, config.maxPages);
+      .slice(0, fetchWindow);
 
     emit("fetch", `Reading ${results.length} sources...`);
-    const pageCandidates = await Promise.all(results.map((result) => fetchPageSource(result.url, signal, { ...config, query })));
+    const pageCandidates = config.mode === "fast"
+      ? await speculativeFetch(results, signal, { ...config, minSources: config.minSources || 3 }, query)
+      : await Promise.all(results.map((result) => fetchPageSource(result.url, signal, { ...config, query })));
     await logResearchEvent("page_fetch_results", {
       query,
       urls: results.map((result) => result.url),
@@ -742,6 +852,22 @@ export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast"
     conflictSummary = conflict.conflictSummary || "";
     conflictingSourcePairs = conflict.conflictingSourcePairs || [];
 
+    const structuredConflictStartedAt = Date.now();
+    const structuredConflictDecision = await classifyConflictWithTinyRouter(query, mergedPages, signal);
+    if (structuredConflictDecision) {
+      await logResearchEvent("tiny_router_latency", { task: "conflict", latencyMs: Date.now() - structuredConflictStartedAt, accepted: true });
+      const nextConflictDetected = applyConflictTinyRouterDecision(
+        conflictDetected,
+        structuredConflictDecision,
+        { allowClear: process.env.PI_RESEARCH_TINY_ROUTER_CONFLICT_ALLOW_CLEAR === "1" || process.env.PI_RESEARCH_TINY_ROUTER_CONFLICT_ALLOW_CLEAR === "true" },
+      );
+      if (nextConflictDetected !== conflictDetected) {
+        conflictDetected = nextConflictDetected;
+        if (conflictDetected && !conflictSummary) conflictSummary = `Structured router flagged ${query} for conflict review.`;
+      }
+      await logResearchEvent("tiny_router_structured_decision", { task: "conflict", query, decision: structuredConflictDecision, heuristicConflictDetected: conflict.detected, finalConflictDetected: conflictDetected });
+    }
+
     const minSources = config.mode === "fast"
       ? (mergedPages.some((page) => page.authoritative) ? 1 : Math.max(3, config.minSources || 3))
       : (config.minSources || 3);
@@ -757,10 +883,45 @@ export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast"
       sufficiency = { ...sufficiency, sufficient: true };
     }
 
+    const structuredSufficiencyStartedAt = Date.now();
+    const structuredSufficiencyDecision = await classifySufficiencyWithTinyRouter(query, mergedPages, signal);
+    if (structuredSufficiencyDecision) {
+      const heuristicSufficient = sufficiency.sufficient;
+      await logResearchEvent("tiny_router_latency", { task: "sufficiency", latencyMs: Date.now() - structuredSufficiencyStartedAt, accepted: true });
+      const finalSufficient = applySufficiencyTinyRouterDecision(heuristicSufficient, structuredSufficiencyDecision);
+      if (finalSufficient !== heuristicSufficient) {
+        sufficiency = withStructuredSufficiencyDecision(sufficiency, structuredSufficiencyDecision, query, mergedPages.map((page) => page.url));
+      }
+      await logResearchEvent("tiny_router_structured_decision", { task: "sufficiency", query, decision: structuredSufficiencyDecision, heuristicSufficient, finalSufficient });
+      sufficiency = { ...sufficiency, sufficient: finalSufficient };
+    }
+
     if (sufficiency.sufficient || turn === (config.maxTurns - 1)) break;
 
     followupRounds += 1;
-    followupQuery = buildFollowUpQuery(query, mergedPages);
+    
+    const conflictState = conflictDetected ? (mergedPages.some(p => p.authoritative) ? "minor" : "severe") : "none";
+    const sourcesMeta = {
+      has_authority: mergedPages.some(p => p.authoritative),
+      has_forum: mergedPages.some(p => p.sourceType === "forum" || /forum|reddit|stack/i.test(p.url)),
+      has_news: mergedPages.some(p => p.sourceType === "news" || /news|blog|article/i.test(p.url)),
+      has_recent: mergedPages.some(p => p.freshness === "recent" || p.freshness === "current_year"),
+      source_count: mergedPages.length
+    };
+    
+    const action = await classifyFollowupWithTinyRouter(query, config.mode, conflictState, sourcesMeta, signal);
+    
+    if (action === "stop") {
+      await logResearchEvent("tiny_router_stop", { query, reason: "router_suggested_stop" });
+      break;
+    }
+
+    if (!action) {
+      followupQuery = buildFollowUpQuery(query, mergedPages, { seenUrls: mergedPages.map((page) => page.url) });
+    } else {
+      followupQuery = buildActionBasedFollowUpQuery(query, action, { seenUrls: mergedPages.map((page) => page.url) });
+    }
+    
     currentQueries = planSubqueries(query, followupQuery, config, sufficiency);
     subqueries = [...new Set([...subqueries, ...currentQueries])];
   }

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "pi-research",
-  "version": "1.3.1",
+  "version": "1.4.0",
   "private": false,
   "type": "module",
-  "description": "Pi extension for web research.",
+  "description": "Zero-setup grounded web research for AI coding agents.",
   "license": "MIT",
   "main": "./index.js",
   "bin": {