pi-research 1.2.1 → 1.3.1

package/index.js

@@ -63,7 +63,7 @@ export default function webResearchExtension(pi) {
     RESEARCH_STATE.clear();
     clearResearchMemory();
     await logResearchEvent("agent_start", {
-      systemPrompt: event.systemPrompt,
+      systemPromptLength: String(event.systemPrompt || "").length,
       guidance: buildWebResearchGuidance(),
     });
     return { systemPrompt: `${event.systemPrompt}\n\n${buildWebResearchGuidance()}` };

package/lib/eval/case-loader.js

@@ -6,6 +6,7 @@ export function loadEvalCases(domain) {
   try {
     return readdirSync(dir)
       .filter((file) => file.endsWith(".json"))
+      .sort((a, b) => a.localeCompare(b))
       .map((file) => JSON.parse(readFileSync(join(dir, file), "utf8")));
   } catch {
     return [];

package/lib/eval/runner.js

@@ -1,8 +1,186 @@
+import { classifyQuestionDomain } from "../research-intent.js";
+import { buildFollowUpQuery, evaluateSufficiency, scoreSourceEntry } from "../research.js";
+import { pageQualitySignals } from "../research-policy.js";
 import { loadEvalCases } from "./case-loader.js";
 
+function normalizeHost(url = "") {
+  try {
+    return new URL(url).hostname.replace(/^www\./, "").toLowerCase();
+  } catch {
+    return "";
+  }
+}
+
+function runChecks(checks = []) {
+  const passed = checks.filter((check) => check.ok).length;
+  return {
+    passed,
+    total: checks.length,
+    checks,
+    ok: passed === checks.length,
+  };
+}
+
+function evaluateFollowupProbe(probe = {}) {
+  if (!probe || !probe.query) return [];
+  const followup = buildFollowUpQuery(probe.query, probe.pages || []);
+  const checks = [];
+
+  if (probe.expectNoQuestionMark !== false) {
+    checks.push({
+      name: "followup-no-question-mark",
+      ok: !followup.includes("?"),
+      actual: followup,
+    });
+  }
+  if (Array.isArray(probe.expectNotIncludes)) {
+    for (const token of probe.expectNotIncludes) {
+      checks.push({
+        name: `followup-not-includes:${token}`,
+        ok: !followup.toLowerCase().includes(String(token).toLowerCase()),
+        actual: followup,
+      });
+    }
+  }
+  if (Array.isArray(probe.expectAnyIncludes) && probe.expectAnyIncludes.length) {
+    checks.push({
+      name: "followup-includes-one-expected-token",
+      ok: probe.expectAnyIncludes.some((token) => followup.toLowerCase().includes(String(token).toLowerCase())),
+      actual: followup,
+    });
+  }
+
+  return checks;
+}
+
+function evaluatePageProbe(probe = {}) {
+  if (!probe || !probe.page) return [];
+  const quality = pageQualitySignals(probe.page);
+  const checks = [];
+
+  if (typeof probe.expectBlocked === "boolean") {
+    checks.push({ name: "page-blocked", ok: quality.blocked === probe.expectBlocked, actual: quality.blocked });
+  }
+  if (typeof probe.expectWeak === "boolean") {
+    checks.push({ name: "page-weak", ok: quality.weak === probe.expectWeak, actual: quality.weak });
+  }
+  if (Array.isArray(probe.expectSignals)) {
+    for (const signal of probe.expectSignals) {
+      checks.push({
+        name: `page-signal:${signal}`,
+        ok: quality.negativeSignals.includes(signal),
+        actual: quality.negativeSignals,
+      });
+    }
+  }
+
+  return checks;
+}
+
+function evaluateSourceProbe(probe = {}) {
+  if (!probe || !Array.isArray(probe.sources)) return [];
+  const scored = probe.sources.map((source) => ({ source, scored: scoreSourceEntry(source, probe.query || "") }));
+  const checks = [];
+
+  if (Array.isArray(probe.expectAuthoritativeHosts)) {
+    for (const host of probe.expectAuthoritativeHosts) {
+      const match = scored.find(({ source }) => normalizeHost(source.url) === host);
+      checks.push({
+        name: `authoritative-host:${host}`,
+        ok: Boolean(match?.scored.authoritative),
+        actual: match?.scored,
+      });
+    }
+  }
+  if (Array.isArray(probe.expectNonAuthoritativeHosts)) {
+    for (const host of probe.expectNonAuthoritativeHosts) {
+      const match = scored.find(({ source }) => normalizeHost(source.url) === host);
+      checks.push({
+        name: `non-authoritative-host:${host}`,
+        ok: match ? !match.scored.authoritative : false,
+        actual: match?.scored,
+      });
+    }
+  }
+  if (Array.isArray(probe.expectSourceTypes)) {
+    for (const expected of probe.expectSourceTypes) {
+      const match = scored.find(({ source }) => normalizeHost(source.url) === expected.host);
+      checks.push({
+        name: `source-type:${expected.host}`,
+        ok: match?.scored.sourceType === expected.sourceType,
+        actual: match?.scored,
+      });
+    }
+  }
+
+  return checks;
+}
+
+function evaluateSufficiencyProbe(probe = {}) {
+  if (!probe || !probe.query) return [];
+  const result = evaluateSufficiency({
+    query: probe.query,
+    sources: probe.sources || [],
+    conflictDetected: Boolean(probe.conflictDetected),
+    minSources: probe.minSources,
+  });
+  const checks = [];
+
+  if (typeof probe.expectSufficient === "boolean") {
+    checks.push({ name: "sufficient", ok: result.sufficient === probe.expectSufficient, actual: result.sufficient });
+  }
+  if (typeof probe.expectAuthoritativeSourcesFound === "boolean") {
+    checks.push({
+      name: "authoritative-sources-found",
+      ok: result.authoritativeSourcesFound === probe.expectAuthoritativeSourcesFound,
+      actual: result.authoritativeSourcesFound,
+    });
+  }
+  if (probe.expectOpenSubQuestionsNoQuestionMark) {
+    checks.push({
+      name: "open-subquestions-no-question-mark",
+      ok: result.openSubQuestions.every((item) => !String(item).includes("?")),
+      actual: result.openSubQuestions,
+    });
+  }
+
+  return checks;
+}
+
+function evaluateCase(domain, item) {
+  const checks = [];
+  checks.push({
+    name: "domain-match",
+    ok: classifyQuestionDomain(item.question) === (item.expectedDomain || domain),
+    actual: classifyQuestionDomain(item.question),
+  });
+
+  checks.push(...evaluateFollowupProbe(item.followupProbe));
+  checks.push(...evaluatePageProbe(item.pageProbe));
+  checks.push(...evaluateSourceProbe(item.sourceProbe));
+  checks.push(...evaluateSufficiencyProbe(item.sufficiencyProbe));
+
+  return {
+    question: item.question,
+    notes: item.notes || "",
+    ...runChecks(checks),
+  };
+}
+
 export async function runEvalSuite({ domain }) {
   const cases = loadEvalCases(domain);
-  const passed = cases.filter((item) => item.expectedDomain === domain).length;
-  const total = cases.length;
-  return { total, passed, passRate: total ? passed / total : 0 };
+  const details = cases.map((item) => evaluateCase(domain, item));
+  const passedCases = details.filter((item) => item.ok).length;
+  const passedChecks = details.reduce((sum, item) => sum + item.passed, 0);
+  const totalChecks = details.reduce((sum, item) => sum + item.total, 0);
+
+  return {
+    total: cases.length,
+    passed: passedCases,
+    passRate: cases.length ? passedCases / cases.length : 0,
+    checkPassRate: totalChecks ? passedChecks / totalChecks : 0,
+    passedChecks,
+    totalChecks,
+    details,
+  };
 }

package/lib/page-fetch-adapter.js

@@ -2,6 +2,8 @@ import { spawn } from "node:child_process";
 import { fileURLToPath } from "node:url";
 import path from "node:path";
 
+import { WEAK_PAGE_POLICY } from "./research-policy.js";
+
 const SCRAPLING_ROOT = fileURLToPath(new URL("../Scrapling", import.meta.url));
 const BLOCKED_PATTERNS = [
   /cloudflare/i,
@@ -11,6 +13,10 @@ const BLOCKED_PATTERNS = [
   /bot detection/i,
   /verify you are human/i,
   /security check/i,
+  /access denied/i,
+  /temporarily unavailable/i,
+  /attention required/i,
+  /challenge-platform/i,
 ];
 const DYNAMIC_PATTERNS = [
   /__next_data__/i,
@@ -38,9 +44,18 @@ export function assessPageAttempt({ status = 200, body = "", contentType = "", u
   const plain = stripHtml(text);
   const lower = `${text}\n${url}`.toLowerCase();
   const antiBotSignal = BLOCKED_PATTERNS.some((pattern) => pattern.test(lower));
-  const blocked = status === 403 || status === 429 || (antiBotSignal && plain.length < 1000);
-  const dynamic = !blocked && (DYNAMIC_PATTERNS.some((pattern) => pattern.test(lower)) || (text.includes("<script") && plain.length < 400));
-  const weak = blocked || plain.length < 300 || (!/text\/(html|plain)/i.test(contentType) && plain.length < 500);
+  const negativeSignals = [];
+
+  if (plain.length < WEAK_PAGE_POLICY.weakTextLimit) negativeSignals.push("weak_text");
+  else if (plain.length < WEAK_PAGE_POLICY.thinTextLimit) negativeSignals.push("thin_text");
+  if (antiBotSignal) negativeSignals.push("placeholder");
+  if (!/text\/(html|plain)/i.test(contentType) && plain.length < 500) negativeSignals.push("unsupported_content_type");
+
+  const blocked = status === 403
+    || status === 429
+    || (antiBotSignal && plain.length < WEAK_PAGE_POLICY.blockedTextLimit);
+  const dynamic = !blocked && (DYNAMIC_PATTERNS.some((pattern) => pattern.test(lower)) || (text.includes("<script") && plain.length < WEAK_PAGE_POLICY.weakTextLimit));
+  const weak = blocked || negativeSignals.includes("weak_text") || negativeSignals.length >= WEAK_PAGE_POLICY.minNegativeSignals;
 
   return {
     blocked,
@@ -48,6 +63,7 @@ export function assessPageAttempt({ status = 200, body = "", contentType = "", u
     weak,
     mode: blocked ? "stealthy" : dynamic ? "dynamic" : "async",
     plainLength: plain.length,
+    negativeSignals,
   };
 }
 

package/lib/research-policy.js

@@ -0,0 +1,188 @@
+import { classifyQuestionDomain } from "./research-intent.js";
+
+export const WEAK_PAGE_POLICY = {
+  blockedTextLimit: 1200,
+  weakTextLimit: 400,
+  thinTextLimit: 1200,
+  minQueryTermMatches: 2,
+  minNegativeSignals: 2,
+};
+
+export const DOMAIN_AUTHORITY_RULES = {
+  security: {
+    hosts: ["nvd.nist.gov", "cisa.gov", "mitre.org", "github.com", "ubuntu.com", "redhat.com", "debian.org", "suse.com"],
+    type: "official_doc",
+  },
+  "vendor-status": {
+    hosts: ["statuspage.io", "status.github.com"],
+    type: "official_doc",
+  },
+  "package-registry": {
+    hosts: ["npmjs.com", "pypi.org", "crates.io", "mvnrepository.com"],
+    type: "official_doc",
+  },
+  github: {
+    hosts: ["github.com"],
+    type: "github_repo",
+  },
+  papers: {
+    hosts: ["arxiv.org", "semanticscholar.org", "doi.org", "pubmed.ncbi.nlm.nih.gov", "nature.com", "science.org"],
+    type: "paper",
+  },
+  web: {
+    hosts: [],
+    type: "official_doc",
+  },
+};
+
+const PLACEHOLDER_PATTERNS = [
+  /cloudflare/i,
+  /access denied/i,
+  /temporarily unavailable/i,
+  /attention required/i,
+  /verify you are human/i,
+  /security check/i,
+  /captcha/i,
+  /turnstile/i,
+  /challenge-platform/i,
+];
+
+const VENDOR_RESEARCH_HOSTS = [
+  "research.ibm.com",
+  "research.google",
+];
+
+function baseQuery(query = "") {
+  return String(query || "")
+    .trim()
+    .replace(/[?!.]+$/g, "")
+    .replace(/\s+/g, " ");
+}
+
+function meaningfulTerms(text = "") {
+  return [...new Set(String(text || "")
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]+/g, " ")
+    .split(/\s+/)
+    .filter((term) => term.length > 2 && !["the", "and", "for", "with", "from", "that", "this", "what", "which", "best", "official", "docs"].includes(term)))];
+}
+
+export function normalizeHostname(url = "") {
+  try {
+    return new URL(url).hostname.replace(/^www\./, "").toLowerCase();
+  } catch {
+    return "";
+  }
+}
+
+function hostMatches(hostname, candidate) {
+  return hostname === candidate || hostname.endsWith(`.${candidate}`);
+}
+
+function countOverlap(query = "", title = "", text = "") {
+  const terms = meaningfulTerms(query);
+  const haystack = `${title} ${String(text || "").slice(0, 1200)}`.toLowerCase();
+  return terms.filter((term) => haystack.includes(term)).length;
+}
+
+export function resolvePolicyDomain(query = "", explicitDomain = "") {
+  return explicitDomain || classifyQuestionDomain(query || "");
+}
+
+export function pageQualitySignals({ title = "", text = "", status = 200, contentType = "", url = "", query = "" } = {}) {
+  const plain = String(text || "").replace(/\s+/g, " ").trim();
+  const corpus = `${title}\n${plain}\n${url}`;
+  const placeholder = PLACEHOLDER_PATTERNS.some((pattern) => pattern.test(corpus));
+  const queryTermMatches = countOverlap(query, title, plain);
+  const negativeSignals = [];
+
+  if (plain.length < WEAK_PAGE_POLICY.weakTextLimit) negativeSignals.push("weak_text");
+  else if (plain.length < WEAK_PAGE_POLICY.thinTextLimit) negativeSignals.push("thin_text");
+  if (placeholder) negativeSignals.push("placeholder");
+  if (contentType && !/text\/(html|plain)/i.test(contentType)) negativeSignals.push("unsupported_content_type");
+  if (query && queryTermMatches < WEAK_PAGE_POLICY.minQueryTermMatches) negativeSignals.push("query_overlap_low");
+
+  const blocked = status === 403
+    || status === 429
+    || (placeholder && plain.length < WEAK_PAGE_POLICY.blockedTextLimit);
+  const weak = blocked
+    || negativeSignals.includes("weak_text")
+    || negativeSignals.length >= WEAK_PAGE_POLICY.minNegativeSignals;
+
+  return {
+    blocked,
+    weak,
+    placeholder,
+    plainLength: plain.length,
+    queryTermMatches,
+    negativeSignals,
+  };
+}
+
+export function sourceAuthorityProfile({ url = "", title = "", text = "", query = "", domain = "" } = {}) {
+  const hostname = normalizeHostname(url);
+  const resolvedDomain = resolvePolicyDomain(query, domain);
+  const quality = pageQualitySignals({ title, text, url, query });
+
+  if (hostMatches(hostname, "researchgate.net")) {
+    if (quality.blocked || quality.placeholder) {
+      return { sourceType: "other", authoritative: false, domainBoost: -8, reasons: ["researchgate_placeholder"] };
+    }
+    return { sourceType: "other", authoritative: false, domainBoost: resolvedDomain === "papers" ? 2 : 0, reasons: ["researchgate_secondary"] };
+  }
+
+  if (VENDOR_RESEARCH_HOSTS.some((host) => hostMatches(hostname, host))) {
+    return { sourceType: "official_doc", authoritative: true, domainBoost: 8, reasons: ["vendor_research_host"] };
+  }
+
+  const rule = DOMAIN_AUTHORITY_RULES[resolvedDomain] || DOMAIN_AUTHORITY_RULES.web;
+  if (rule.hosts.some((host) => hostMatches(hostname, host))) {
+    const reason = resolvedDomain === "github" && /\/(issues|pull|pulls|discussions)\//.test(url)
+      ? "github_state_page"
+      : "domain_authority_host";
+    const authoritative = !(resolvedDomain === "github" && /\/(issues|pull|pulls|discussions)\//.test(url)) || /#readme|\/releases|\/blob\//.test(url);
+    return { sourceType: rule.type, authoritative, domainBoost: authoritative ? 10 : 4, reasons: [reason] };
+  }
+
+  return { sourceType: null, authoritative: false, domainBoost: 0, reasons: [] };
+}
+
+export function buildAuthorityFollowUpQueries(query = "", explicitDomain = "") {
+  const resolvedDomain = resolvePolicyDomain(query, explicitDomain);
+  const base = baseQuery(query);
+
+  switch (resolvedDomain) {
+    case "security":
+      return [`${base} cve advisory vendor`, `${base} nvd cisa mitre`];
+    case "vendor-status":
+      return [`${base} status page incident`, `${base} official outage status`];
+    case "package-registry":
+      return [`${base} npm pypi crates readme`, `${base} official package docs`];
+    case "github":
+      return [`${base} github readme releases`, `${base} site:github.com readme docs`];
+    case "papers":
+      return [`${base} arxiv doi publisher`, `${base} semanticscholar arxiv doi`];
+    default:
+      return [`${base} official docs`, `${base} documentation reference`];
+  }
+}
+
+export function buildConflictFollowUpQueries(query = "", explicitDomain = "") {
+  const resolvedDomain = resolvePolicyDomain(query, explicitDomain);
+  const base = baseQuery(query);
+
+  switch (resolvedDomain) {
+    case "security":
+      return [`${base} vendor advisory official`, `${base} cve mitigation official`];
+    case "vendor-status":
+      return [`${base} incident status official`, `${base} status page postmortem`];
+    case "package-registry":
+      return [`${base} release notes changelog`, `${base} maintainer docs`];
+    case "github":
+      return [`${base} github releases readme`, `${base} canonical repo docs`];
+    case "papers":
+      return [`${base} arxiv doi compare`, `${base} publisher abstract official`];
+    default:
+      return [`${base} official docs support status`, `${base} official comparison reference`];
+  }
+}

package/lib/research.js

@@ -1,3 +1,10 @@
+import {
+  buildAuthorityFollowUpQueries,
+  buildConflictFollowUpQueries,
+  pageQualitySignals,
+  sourceAuthorityProfile,
+} from "./research-policy.js";
+
 function decodeHtmlEntities(text) {
   return String(text || "")
     .replace(/&/g, "&")
@@ -307,6 +314,7 @@ export function classifySourceType(url, title = "") {
   if (/github\.com\/[^/]+\/[^/]+#readme|github\.com\/[^/]+\/[^/]+\/blob\//.test(lower)) return "github_readme";
   if (/github\.com\/[^/]+\/[^/]+/.test(lower)) return "github_repo";
   if (/arxiv\.org|ieee\.org|springer\.com|pubmed\.ncbi\.nlm\.nih\.gov|doi\.org|semanticscholar\.org|acm\.org|nature\.com|science\.org/.test(lower)) return "paper";
+  if (/research\.ibm\.com|research\.google/.test(lower)) return "official_doc";
   if (/reddit\.com|stackoverflow\.com|forum/.test(lower)) return "forum";
   if (/blog\.|medium\.com|dev\.to|substack\.com/.test(lower)) return "blog";
   if (/\/docs?\b|documentation|developer|reference|official/.test(lower) || /official|documentation|reference|guide/i.test(title) || /\.edu\/|\.ac\.uk\//.test(lower)) return "official_doc";
@@ -315,7 +323,7 @@ export function classifySourceType(url, title = "") {
 
 export function isAuthoritativeUrl(url) {
   const lower = String(url || "").toLowerCase();
-  return /\/docs?\b|documentation|developer|reference|official|github\.com\/[^/]+\/[^/]+(#readme|\/tree\/[^/]+\/docs)?|npmjs\.com\/package\/|arxiv\.org|pubmed\.ncbi\.nlm\.nih\.gov|semanticscholar\.org|doi\.org|\.edu\/|\.ac\.uk\//.test(lower);
+  return /\/docs?\b|documentation|developer|reference|official|github\.com\/[^/]+\/[^/]+(#readme|\/tree\/[^/]+\/docs)?|npmjs\.com\/package\/|arxiv\.org|pubmed\.ncbi\.nlm\.nih\.gov|semanticscholar\.org|doi\.org|research\.ibm\.com|research\.google|\.edu\/|\.ac\.uk\//.test(lower);
 }
 
 export function scoreSearchResult(result, query, config = {}) {
@@ -369,6 +377,18 @@ export function scoreFetchedPage(page, query, config = {}) {
   if (/stackoverflow\.com|reddit\.com|quora\.com/.test(url)) score -= 2;
   if (countTermMatches(firstChunk, terms) > 0) score += 5;
 
+  const quality = page?.quality || pageQualitySignals({
+    title: page?.title || "",
+    text,
+    url: page?.url || "",
+    query,
+  });
+  if (quality.blocked) score -= 20;
+  if (quality.negativeSignals?.includes("placeholder")) score -= 10;
+  if (quality.negativeSignals?.includes("weak_text")) score -= 8;
+  if (quality.negativeSignals?.includes("thin_text")) score -= 4;
+  if (quality.negativeSignals?.includes("query_overlap_low")) score -= 6;
+
   const ageInMonths = monthsSince(page?.publishDate);
   if (config.preferRecent && ageInMonths !== null) {
     if (ageInMonths <= 6) score += 8;
@@ -448,7 +468,7 @@ export function detectResearchGaps(query, pages) {
     return {
       detected: true,
       reason: "Retrieved pages lack an authoritative docs or README source.",
-      followupQuery: `${queryBase(query)} official docs`,
+      followupQuery: buildAuthorityFollowUpQueries(query)[0] || `${queryBase(query)} official docs`,
       missingAspects: ["authoritative sources"],
     };
   }
@@ -458,10 +478,10 @@ export function detectResearchGaps(query, pages) {
 
 export function buildFollowUpQuery(query, pages) {
   const conflict = detectConflictSignals(pages);
-  if (conflict.detected) return `${queryBase(query)} official docs support status`;
+  if (conflict.detected) return buildConflictFollowUpQueries(query)[0] || `${queryBase(query)} official docs support status`;
   const gaps = detectResearchGaps(query, pages);
   if (gaps.detected) return gaps.followupQuery;
-  return `${queryBase(query)} clarification official docs`;
+  return buildAuthorityFollowUpQueries(`${queryBase(query)} clarification`)[0] || `${queryBase(query)} clarification official docs`;
 }
 
 function queryTermsForFactCheck(text) {
@@ -536,12 +556,18 @@ export function buildConfidenceSummary(pages, meta = {}) {
 export function scoreSourceEntry(source, query = "") {
   const url = String(source?.url || "");
   const title = String(source?.title || "");
-  const sourceType = classifySourceType(url, title);
+  const authorityProfile = sourceAuthorityProfile({
+    url,
+    title,
+    text: source?.text || source?.snippet || "",
+    query,
+  });
+  const sourceType = authorityProfile.sourceType || classifySourceType(url, title);
   const freshness = summarizeFreshness(source?.publishDate || source?.freshness);
   let typeScore = 0;
   let freshnessScore = 0;
-  let domainScore = 0;
-  let authoritative = isAuthoritativeUrl(url) || sourceType === "official_doc" || sourceType === "paper" || sourceType === "file";
+  let domainScore = authorityProfile.domainBoost || 0;
+  let authoritative = authorityProfile.authoritative || isAuthoritativeUrl(url) || sourceType === "official_doc" || sourceType === "paper" || sourceType === "file";
 
   if (authoritative) typeScore += 10;
   if (sourceType === "official_doc") typeScore += 8;
@@ -625,9 +651,10 @@ export function evaluateSufficiency(input, legacyPages, legacyConflictDetected =
   if (conflictDetected) missingAspects.push("conflict resolution");
   if (!payload.sources.length) missingAspects.push("readable sources");
 
-  const openSubQuestions = [];
-  if (!authoritativeSourcesFound) openSubQuestions.push(`${queryBase(payload.query)} official docs`);
-  if (conflictDetected) openSubQuestions.push(`Which authoritative source resolves the conflicting claims about ${queryBase(payload.query)}?`);
+  const openSubQuestions = [
+    ...(!authoritativeSourcesFound ? buildAuthorityFollowUpQueries(payload.query) : []),
+    ...(conflictDetected ? buildConflictFollowUpQueries(payload.query) : []),
+  ];
   if (!openSubQuestions.length) openSubQuestions.push(`${queryBase(payload.query)} follow-up`);
 
   const minSources = payload.minSources || 1;

package/lib/web-research.js

@@ -36,6 +36,7 @@ import {
   selectRelevantChunks,
 } from "./research.js";
 import { pageFetchAdapter } from "./page-fetch-adapter.js";
+import { pageQualitySignals } from "./research-policy.js";
 import { resolveOutputFormat, shouldRequireAuthoritativeSources } from "./research-output.js";
 import { planResearch } from "./planner.js";
 import {
@@ -111,7 +112,7 @@ export function resolveResearchConfig(input = "fast") {
     force: Boolean(options.force),
     format: resolveOutputFormat(options, domainConfig.format || "markdown"),
     queryHints: Array.isArray(domainConfig.queryHints) ? domainConfig.queryHints : [],
-    requireAuthoritative: Boolean(options.requireAuthoritative ?? domainConfig.requireAuthoritative),
+    requireAuthoritative: Boolean(options.requireAuthoritative ?? (domainConfig.requireAuthoritative || domainConfig.domain === "github")),
     domain: domainConfig.domain,
   };
 }
@@ -371,7 +372,7 @@ async function fetchJinaPageSource(url, signal, config) {
     const body = await response.text();
     const firstLine = body.split("\n").find((line) => line.trim().replace(/^#+\s*/, ""));
     const title = firstLine ? firstLine.trim().replace(/^#+\s*/, "") : url;
-    return pageFromText(title, url, body, config, { sourceType: classifySourceType(url, title) });
+    return pageFromText(title, url, body, config, { sourceType: classifySourceType(url, title), fetchStatus: 200, contentType: "text/plain" });
   } catch {
     return null;
   }
@@ -385,6 +386,20 @@ function withinTimeframe(page, config) {
   return true;
 }
 
+function finalizeFetchedPage(page, config, meta = {}) {
+  if (!page) return null;
+  const quality = page.quality || pageQualitySignals({
+    title: page.title,
+    text: page.text,
+    url: page.url || meta.url || "",
+    query: config.query || "",
+    status: page.fetchStatus ?? meta.status,
+    contentType: page.contentType || meta.contentType || "text/html",
+  });
+  if (quality.blocked) return null;
+  return { ...page, quality };
+}
+
 export async function fetchPageSource(url, signal, config = getResearchConfig()) {
   if (shouldSkipUrl(url)) {
     await logResearchEvent("fetch_skip", { url, reason: "login_or_account_url" });
@@ -399,14 +414,19 @@ export async function fetchPageSource(url, signal, config = getResearchConfig())
   })}`;
   const cached = config.isolate ? null : getCacheValue(pageCache, cacheKey);
   if (cached) {
-    await logResearchEvent("fetch_cache_hit", { url, cacheKey, title: cached.title, textLength: cached.text?.length || 0 });
-    return cached;
+    const validated = finalizeFetchedPage(cached, config, { url: cached.url || url, contentType: "text/html" });
+    if (!validated) {
+      await logResearchEvent("fetch_skip", { url, cacheKey, reason: "cached_page_blocked_or_placeholder" });
+      return null;
+    }
+    await logResearchEvent("fetch_cache_hit", { url, cacheKey, title: validated.title, textLength: validated.text?.length || 0 });
+    return validated;
   }
 
   await logResearchEvent("fetch_start", { url, cacheKey, config: { isolate: config.isolate, useJinaFallback: Boolean(config.useJinaFallback), pageTextLimit: config.pageTextLimit } });
 
   if (shouldUseJinaFirst(url)) {
-    const first = await fetchJinaPageSource(url, signal, config);
+    const first = finalizeFetchedPage(await fetchJinaPageSource(url, signal, config), config, { url, contentType: "text/plain" });
     if (first && withinTimeframe(first, config)) {
       const page = config.isolate ? first : setCacheValue(pageCache, cacheKey, first, PAGE_CACHE_TTL_MS);
       await logResearchEvent("fetch_end", { url, via: "jina_first", success: Boolean(page), page: page ? { title: page.title, sourceType: page.sourceType, publishDate: page.publishDate, textLength: page.text?.length || 0 } : null });
@@ -422,6 +442,12 @@ export async function fetchPageSource(url, signal, config = getResearchConfig())
 
     const contentType = response.headers.get("content-type") || "";
     if (!contentType.includes("text/html") && !contentType.includes("text/plain")) {
+      const fallback = finalizeFetchedPage(await fetchJinaPageSource(url, signal, config), config, { url, contentType });
+      if (fallback && withinTimeframe(fallback, config)) {
+        const page = config.isolate ? fallback : setCacheValue(pageCache, cacheKey, fallback, PAGE_CACHE_TTL_MS);
+        await logResearchEvent("fetch_end", { url, via: "unsupported_content_type_fallback", success: Boolean(page), contentType, page: page ? { title: page.title, sourceType: page.sourceType, publishDate: page.publishDate, textLength: page.text?.length || 0 } : null });
+        return page;
+      }
       await logResearchEvent("fetch_end", { url, success: false, reason: "unsupported_content_type", contentType });
       return null;
     }
@@ -432,6 +458,8 @@ export async function fetchPageSource(url, signal, config = getResearchConfig())
       publishDate: extractPublishDate(body),
       sourceType: classifySourceType(snapshot.url, snapshot.title),
       codeBlocks: snapshot.codeBlocks,
+      fetchStatus: response.status ?? 200,
+      contentType,
     });
 
     const assessment = adapter.assessPageAttempt?.({
@@ -449,18 +477,24 @@ export async function fetchPageSource(url, signal, config = getResearchConfig())
           publishDate: extractPublishDate(scrapling.body),
           sourceType: classifySourceType(scraplingSnapshot.url, scraplingSnapshot.title),
           codeBlocks: scraplingSnapshot.codeBlocks,
+          fetchStatus: scrapling.status ?? 200,
+          contentType: scrapling.contentType || "text/html",
         });
       }
     }
 
     const resolved = page || await fetchJinaPageSource(url, signal, config);
-    const finalPage = resolved && withinTimeframe(resolved, config) ? resolved : null;
-    const stored = config.isolate ? finalPage : setCacheValue(pageCache, cacheKey, finalPage, PAGE_CACHE_TTL_MS);
+    const finalPage = finalizeFetchedPage(resolved, config, { url: response.url || url, status: response.status ?? 200, contentType });
+    const stored = finalPage && withinTimeframe(finalPage, config)
+      ? (config.isolate ? finalPage : setCacheValue(pageCache, cacheKey, finalPage, PAGE_CACHE_TTL_MS))
+      : null;
     await logResearchEvent("fetch_end", { url, success: Boolean(stored), page: stored ? { title: stored.title, sourceType: stored.sourceType, publishDate: stored.publishDate, textLength: stored.text?.length || 0 } : null });
     return stored;
   } catch (error) {
-    const fallback = await fetchJinaPageSource(url, signal, config);
-    const stored = config.isolate ? fallback : setCacheValue(pageCache, cacheKey, fallback, PAGE_CACHE_TTL_MS);
+    const fallback = finalizeFetchedPage(await fetchJinaPageSource(url, signal, config), config, { url, contentType: "text/plain" });
+    const stored = fallback && withinTimeframe(fallback, config)
+      ? (config.isolate ? fallback : setCacheValue(pageCache, cacheKey, fallback, PAGE_CACHE_TTL_MS))
+      : null;
     await logResearchEvent("fetch_error", { url, error, fallback: stored ? { title: stored.title, sourceType: stored.sourceType, publishDate: stored.publishDate, textLength: stored.text?.length || 0 } : null });
     return stored;
   }
@@ -625,6 +659,7 @@ export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast"
   let conflictSummary = "";
   let conflictingSourcePairs = [];
   let sufficiency = { sufficient: false, confidenceScore: 0.1, missingAspects: [], openSubQuestions: [] };
+  let lastEmptySearchSignature = null;
   let currentQueries = await buildQueries(query, config, ctx, signal);
   subqueries = [...currentQueries];
 
@@ -651,12 +686,20 @@ export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast"
     emit("search", `Searching ${queriesThisTurn.length} queries...`);
 
     const searchGroups = await Promise.all(queriesThisTurn.map((subquery) => searchDuckDuckGo(subquery, signal, config)));
+    const flatResults = searchGroups.flat();
     await logResearchEvent("search_results", {
       query,
       queries: queriesThisTurn,
-      results: searchGroups.flat().map((result) => ({ title: result.title, url: result.url, snippet: result.snippet, sourceType: result.sourceType, publishDate: result.publishDate })),
+      results: flatResults.map((result) => ({ title: result.title, url: result.url, snippet: result.snippet, sourceType: result.sourceType, publishDate: result.publishDate })),
     });
-    const results = rankSearchResults(searchGroups.flat(), query, config.maxPages * 2, config)
+    const searchSignature = queriesThisTurn.join(" || ");
+    if (flatResults.length === 0) {
+      if (lastEmptySearchSignature === searchSignature) break;
+      lastEmptySearchSignature = searchSignature;
+    } else {
+      lastEmptySearchSignature = null;
+    }
+    const results = rankSearchResults(flatResults, query, config.maxPages * 2, config)
       .filter((result) => {
         const key = normalizeUrl(result.url);
         if (seenUrls.has(key)) return false;
@@ -666,7 +709,7 @@ export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast"
       .slice(0, config.maxPages);
 
     emit("fetch", `Reading ${results.length} sources...`);
-    const pageCandidates = await Promise.all(results.map((result) => fetchPageSource(result.url, signal, config)));
+    const pageCandidates = await Promise.all(results.map((result) => fetchPageSource(result.url, signal, { ...config, query })));
     await logResearchEvent("page_fetch_results", {
       query,
       urls: results.map((result) => result.url),
@@ -682,7 +725,7 @@ export async function runWebResearch(query, ctx, signal, onUpdate, mode = "fast"
         sourceType: page.sourceType || scored.sourceType,
         text: selectRelevantChunks(page.text, query, config.maxChunksPerPage).join("\n\n") || page.text,
       };
-    }).filter((page) => withinTimeframe(page, config)), query, config.maxPages, config);
+    }).filter((page) => withinTimeframe(page, config) && !page.quality?.blocked), query, config.maxPages, config);
 
     for (const page of prioritizeSourceEntries(rankedPages, query)) {
       const key = normalizeUrl(page.url);

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "pi-research",
-  "version": "1.2.1",
+  "version": "1.3.1",
   "private": false,
   "type": "module",
   "description": "Pi extension for web research.",
   "license": "MIT",
   "main": "./index.js",
   "bin": {
-    "pi-research": "./pi-research.js",
-    "unblind-mcp": "./unblind-mcp.js"
+    "pi-research": "./bin/pi-research.js",
+    "unblind-mcp": "./bin/unblind-mcp.js"
   },
   "files": [
     "bin",