pi-research 1.3.1 → 1.4.0

package/lib/page-fetch-adapter.js

@@ -1,4 +1,5 @@
-import { spawn } from "node:child_process";
+import { spawn as nodeSpawn, spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import path from "node:path";
 
@@ -28,6 +29,13 @@ const DYNAMIC_PATTERNS = [
   /id=["']root["']/i,
 ];
 
+let spawnProcess = nodeSpawn;
+let daemonState = null;
+let daemonSequence = 0;
+let exitHookInstalled = false;
+let runtimeStatus = null;
+const DAEMON_IDLE_TIMEOUT_MS = 3000;
+
 function stripHtml(value) {
   return String(value || "")
     .replace(/<script[\s\S]*?<\/script>/gi, " ")
@@ -71,35 +79,62 @@ export function chooseScraplingMode(input) {
   return assessPageAttempt(input).mode;
 }
 
-function pythonScript() {
+function pythonDaemonScript() {
   return String.raw`
 import asyncio
+import atexit
 import json
 import os
 import sys
 
 root = sys.argv[1]
-mode = sys.argv[2]
-url = sys.argv[3]
-payload = json.loads(sys.argv[4])
-
 sys.path.insert(0, root)
 
-async def main():
-    from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher
+from scrapling.fetchers import AsyncFetcher, AsyncDynamicSession, AsyncStealthySession, ProxyRotator
 
-    timeout = payload.get("timeout")
-    kwargs = {}
-    if timeout:
-        kwargs["timeout"] = timeout
+sessions = {}
 
-    if mode == "async":
-        response = await AsyncFetcher.get(url, **kwargs)
-    elif mode == "dynamic":
-        response = DynamicFetcher.fetch(url, **kwargs)
-    else:
-        response = StealthyFetcher.fetch(url, **kwargs)
+def session_key(mode, proxy_rotation):
+    if not proxy_rotation:
+        return mode
+    return f"{mode}:{json.dumps(proxy_rotation, sort_keys=True)}"
 
+async def build_session(mode, payload):
+    proxy_rotation = payload.get("proxyRotation") or []
+    key = session_key(mode, proxy_rotation)
+    session = sessions.get(key)
+    if session is not None:
+        return session
+
+    kwargs = {
+        "headless": True,
+        "disable_resources": True,
+        "network_idle": True,
+        "timeout": payload.get("timeout") or 30000,
+    }
+    if proxy_rotation:
+        kwargs["proxy_rotator"] = ProxyRotator(proxy_rotation)
+
+    session = AsyncDynamicSession(**kwargs) if mode == "dynamic" else AsyncStealthySession(**kwargs)
+    await session.start()
+    sessions[key] = session
+    return session
+
+async def cleanup():
+    for session in sessions.values():
+        try:
+            await session.close()
+        except Exception:
+            pass
+    sessions.clear()
+
+def cleanup_sync():
+    asyncio.get_event_loop().run_until_complete(cleanup())
+
+atexit.register(cleanup_sync)
+
+
+def normalize_response(response, fallback_url):
     headers = {}
     raw_headers = getattr(response, "headers", None)
     if hasattr(raw_headers, "items"):
@@ -120,77 +155,289 @@ async def main():
     elif not isinstance(body, str):
         body = str(body or "")
 
-    out = {
+    return {
         "ok": True,
-        "url": getattr(response, "url", url),
+        "url": getattr(response, "url", fallback_url),
         "status": getattr(response, "status", 200),
         "contentType": headers.get("content-type", ""),
         "body": body,
         "headers": headers,
     }
-    print(json.dumps(out))
 
-try:
-    asyncio.run(main())
-except Exception as exc:
-    print(json.dumps({"ok": False, "error": str(exc), "type": exc.__class__.__name__}))
-    sys.exit(1)
+
+async def handle_job(job):
+    mode = job.get("mode")
+    url = job.get("url")
+    payload = job.get("payload") or {}
+    timeout = payload.get("timeout") or 30000
+    proxy = payload.get("proxy")
+
+    kwargs = {"timeout": timeout}
+    if proxy:
+        kwargs["proxy"] = proxy
+
+    if mode == "async":
+        response = await AsyncFetcher.get(url, **kwargs)
+    else:
+        session = await build_session(mode, payload)
+        response = await session.fetch(url, **kwargs)
+
+    out = normalize_response(response, url)
+    out["id"] = job.get("id")
+    return out
+
+
+async def main():
+    print(json.dumps({"type": "ready"}), flush=True)
+
+    for raw_line in sys.stdin:
+        line = raw_line.strip()
+        if not line:
+            continue
+        try:
+            job = json.loads(line)
+        except Exception as exc:
+            print(json.dumps({"type": "error", "ok": False, "error": str(exc)}), flush=True)
+            continue
+
+        if job.get("type") == "shutdown":
+            break
+
+        try:
+            out = await handle_job(job)
+        except Exception as exc:
+            out = {"id": job.get("id"), "ok": False, "error": str(exc), "type": exc.__class__.__name__}
+        print(json.dumps(out), flush=True)
+
+    await cleanup()
+
+
+asyncio.run(main())
 `;
 }
 
+function handleDaemonStdout(state, chunk) {
+  state.stdoutBuffer += String(chunk || "");
+  while (state.stdoutBuffer.includes("\n")) {
+    const newlineIndex = state.stdoutBuffer.indexOf("\n");
+    const line = state.stdoutBuffer.slice(0, newlineIndex).trim();
+    state.stdoutBuffer = state.stdoutBuffer.slice(newlineIndex + 1);
+    if (!line) continue;
+
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      continue;
+    }
+
+    if (parsed.type === "ready") {
+      state.ready = true;
+      state.resolveReady?.(state);
+      continue;
+    }
+
+    const pending = state.pending.get(parsed.id);
+    if (!pending) continue;
+    state.pending.delete(parsed.id);
+    pending.cleanup?.();
+    pending.resolve(parsed.ok ? parsed : null);
+    scheduleDaemonIdleStop(state);
+  }
+}
+
+function failDaemonState(state) {
+  if (state.idleTimer) clearTimeout(state.idleTimer);
+  for (const pending of state.pending.values()) {
+    pending.cleanup?.();
+    pending.resolve(null);
+  }
+  state.pending.clear();
+}
+
+function scheduleDaemonIdleStop(state) {
+  if (state.idleTimer) clearTimeout(state.idleTimer);
+  if (state.pending.size > 0) return;
+  state.idleTimer = setTimeout(() => {
+    if (daemonState === state && state.pending.size === 0) void stopScraplingDaemon();
+  }, DAEMON_IDLE_TIMEOUT_MS);
+  state.idleTimer.unref?.();
+}
+
+function resolvePythonExecutable() {
+  if (process.env.PYTHON) return process.env.PYTHON;
+  const venvPython = path.join(process.cwd(), ".venv-scrapling", "bin", "python");
+  return existsSync(venvPython) ? venvPython : "python3";
+}
+
+function daemonEnv() {
+  return {
+    ...process.env,
+    PYTHONPATH: [SCRAPLING_ROOT, process.env.PYTHONPATH].filter(Boolean).join(path.delimiter),
+  };
+}
+
+function validateScraplingRuntime() {
+  if (runtimeStatus) return runtimeStatus;
+  const python = resolvePythonExecutable();
+  const probe = spawnSync(python, ["-c", "import sys; sys.path.insert(0, sys.argv[1]); import lxml, patchright, playwright, scrapling; print('OK')", SCRAPLING_ROOT], {
+    env: daemonEnv(),
+    encoding: "utf8",
+    timeout: 15000,
+  });
+
+  runtimeStatus = probe.status === 0
+    ? { ok: true, python }
+    : {
+      ok: false,
+      python,
+      error: (probe.stderr || probe.stdout || `scrapling runtime check failed with status ${probe.status ?? "unknown"}`).trim(),
+    };
+  return runtimeStatus;
+}
+
+export function getScraplingRuntimeStatus() {
+  return validateScraplingRuntime();
+}
+
+function ensureExitHook() {
+  if (exitHookInstalled) return;
+  exitHookInstalled = true;
+  process.once("exit", () => {
+    daemonState?.child?.kill?.("SIGKILL");
+  });
+}
+
+async function ensureScraplingDaemon() {
+  if (daemonState?.ready) return daemonState;
+  if (daemonState?.readyPromise) return daemonState.readyPromise;
+
+  ensureExitHook();
+  const runtime = validateScraplingRuntime();
+  if (!runtime.ok) throw new Error(runtime.error || "scrapling runtime unavailable");
+  const child = spawnProcess(runtime.python, ["-c", pythonDaemonScript(), SCRAPLING_ROOT], {
+    env: daemonEnv(),
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+
+  const state = {
+    child,
+    pending: new Map(),
+    stdoutBuffer: "",
+    stderrBuffer: "",
+    ready: false,
+    readyPromise: null,
+    resolveReady: null,
+    rejectReady: null,
+    idleTimer: null,
+  };
+
+  state.readyPromise = new Promise((resolve, reject) => {
+    state.resolveReady = resolve;
+    state.rejectReady = reject;
+  });
+
+  child.stdout.on("data", (chunk) => handleDaemonStdout(state, chunk));
+  child.stderr.on("data", (chunk) => {
+    state.stderrBuffer += String(chunk || "");
+    if (state.stderrBuffer.length > 20_000) state.stderrBuffer = state.stderrBuffer.slice(-20_000);
+  });
+  child.on("error", (error) => {
+    if (!state.ready) state.rejectReady?.(error);
+    failDaemonState(state);
+    if (daemonState === state) daemonState = null;
+  });
+  child.on("close", (code) => {
+    if (!state.ready) state.rejectReady?.(new Error(`scrapling daemon exited before ready (${code ?? "unknown"})`));
+    failDaemonState(state);
+    if (daemonState === state) daemonState = null;
+  });
+
+  daemonState = state;
+  return state.readyPromise;
+}
+
+function requestPayload(mode, config = {}) {
+  return {
+    timeout: mode === "stealthy"
+      ? (config.stealthTimeoutMs || config.pageTimeoutMs || 30_000)
+      : (config.pageTimeoutMs || 30_000),
+    proxy: config.proxy || null,
+    proxyRotation: Array.isArray(config.proxyRotation) && config.proxyRotation.length ? config.proxyRotation : null,
+  };
+}
+
 export async function fetchWithScrapling(url, mode, signal, config = {}) {
   if (!mode) return null;
+  let state;
+  try {
+    state = await ensureScraplingDaemon();
+  } catch {
+    return null;
+  }
+  const id = `job-${++daemonSequence}`;
+  const payload = requestPayload(mode, config);
 
   return await new Promise((resolve) => {
-    const child = spawn(process.env.PYTHON || "python3", ["-c", pythonScript(), SCRAPLING_ROOT, mode, url, JSON.stringify({ timeout: config.pageTimeoutMs || 30000 })], {
-      env: {
-        ...process.env,
-        PYTHONPATH: [SCRAPLING_ROOT, process.env.PYTHONPATH].filter(Boolean).join(path.delimiter),
-      },
-      stdio: ["ignore", "pipe", "pipe"],
-    });
-
-    let stdout = "";
-    let stderr = "";
-    child.stdout.on("data", (chunk) => {
-      stdout += chunk;
-    });
-    child.stderr.on("data", (chunk) => {
-      stderr += chunk;
-    });
-
+    let settled = false;
     const finish = (value) => {
-      if (!signal) return resolve(value);
-      if (signal.aborted) return resolve(null);
-      return resolve(value);
+      if (settled) return;
+      settled = true;
+      resolve(signal?.aborted ? null : value);
+    };
+
+    const cleanup = () => {
+      if (signal && abort) signal.removeEventListener("abort", abort);
+    };
+
+    const abort = () => {
+      state.pending.delete(id);
+      cleanup();
+      scheduleDaemonIdleStop(state);
+      finish(null);
     };
 
-    child.on("error", () => finish(null));
-    child.on("close", (code) => {
-      if (code !== 0) return finish(null);
-      try {
-        const parsed = JSON.parse(stdout.trim() || "{}");
-        if (!parsed.ok) return finish(null);
-        return finish(parsed);
-      } catch {
-        if (stderr) return finish(null);
-        return finish(null);
-      }
-    });
+    if (state.idleTimer) clearTimeout(state.idleTimer);
+    state.pending.set(id, { resolve: finish, cleanup });
 
     if (signal) {
-      const abort = () => {
-        child.kill("SIGKILL");
-        finish(null);
-      };
-      if (signal.aborted) abort();
-      else signal.addEventListener("abort", abort, { once: true });
+      if (signal.aborted) return abort();
+      signal.addEventListener("abort", abort, { once: true });
+    }
+
+    try {
+      state.child.stdin.write(`${JSON.stringify({ id, url, mode, payload })}\n`);
+    } catch {
+      state.pending.delete(id);
+      cleanup();
+      finish(null);
     }
   });
 }
 
+export async function stopScraplingDaemon() {
+  if (!daemonState) return;
+  const state = daemonState;
+  daemonState = null;
+  failDaemonState(state);
+  try {
+    state.child.kill("SIGKILL");
+  } catch {
+    // ignore
+  }
+}
+
+export function setScraplingSpawnForTests(factory) {
+  spawnProcess = factory || nodeSpawn;
+}
+
+export function setScraplingRuntimeStatusForTests(status) {
+  runtimeStatus = status;
+}
+
 export const pageFetchAdapter = {
   assessPageAttempt,
   chooseScraplingMode,
   fetchWithScrapling,
+  stopScraplingDaemon,
 };

package/lib/research-policy.js

@@ -35,7 +35,7 @@ export const DOMAIN_AUTHORITY_RULES = {
   },
 };
 
-const PLACEHOLDER_PATTERNS = [
+export const PLACEHOLDER_PATTERNS = [
   /cloudflare/i,
   /access denied/i,
   /temporarily unavailable/i,
@@ -119,6 +119,20 @@ export function pageQualitySignals({ title = "", text = "", status = 200, conten
   };
 }
 
+export function isUsableContent(page, config = {}) {
+  if (!page || !page.text) return false;
+  const quality = page.quality || pageQualitySignals({
+    title: page.title,
+    text: page.text,
+    url: page.url,
+    query: config.query || "",
+    status: page.fetchStatus ?? 200,
+    contentType: page.contentType || "text/html",
+  });
+  const minPageText = config.minPageText ?? WEAK_PAGE_POLICY.weakTextLimit;
+  return !quality.blocked && !quality.placeholder && !quality.weak && quality.plainLength >= minPageText;
+}
+
 export function sourceAuthorityProfile({ url = "", title = "", text = "", query = "", domain = "" } = {}) {
   const hostname = normalizeHostname(url);
   const resolvedDomain = resolvePolicyDomain(query, domain);
@@ -147,42 +161,49 @@ export function sourceAuthorityProfile({ url = "", title = "", text = "", query
   return { sourceType: null, authoritative: false, domainBoost: 0, reasons: [] };
 }
 
-export function buildAuthorityFollowUpQueries(query = "", explicitDomain = "") {
+function followUpSiteExclusions(seenUrls = []) {
+  const sites = [...new Set(seenUrls.map((url) => normalizeHostname(url)).filter(Boolean))];
+  return sites.length ? ` ${sites.map((site) => `-site:${site}`).join(" ")}` : "";
+}
+
+export function buildAuthorityFollowUpQueries(query = "", explicitDomain = "", options = {}) {
   const resolvedDomain = resolvePolicyDomain(query, explicitDomain);
   const base = baseQuery(query);
+  const exclusions = followUpSiteExclusions(options.seenUrls);
 
   switch (resolvedDomain) {
     case "security":
-      return [`${base} cve advisory vendor`, `${base} nvd cisa mitre`];
+      return [`${base} cve advisory vendor${exclusions}`, `${base} nvd cisa mitre${exclusions}`];
     case "vendor-status":
-      return [`${base} status page incident`, `${base} official outage status`];
+      return [`${base} status page incident${exclusions}`, `${base} official outage status${exclusions}`];
     case "package-registry":
-      return [`${base} npm pypi crates readme`, `${base} official package docs`];
+      return [`${base} npm pypi crates readme${exclusions}`, `${base} official package docs${exclusions}`];
     case "github":
-      return [`${base} github readme releases`, `${base} site:github.com readme docs`];
+      return [`${base} github readme releases${exclusions}`, `${base} site:github.com readme docs${exclusions}`];
     case "papers":
-      return [`${base} arxiv doi publisher`, `${base} semanticscholar arxiv doi`];
+      return [`${base} arxiv doi publisher${exclusions}`, `${base} semanticscholar arxiv doi${exclusions}`];
     default:
-      return [`${base} official docs`, `${base} documentation reference`];
+      return [`${base} official docs${exclusions}`, `${base} documentation reference${exclusions}`];
   }
 }
 
-export function buildConflictFollowUpQueries(query = "", explicitDomain = "") {
+export function buildConflictFollowUpQueries(query = "", explicitDomain = "", options = {}) {
   const resolvedDomain = resolvePolicyDomain(query, explicitDomain);
   const base = baseQuery(query);
+  const exclusions = followUpSiteExclusions(options.seenUrls);
 
   switch (resolvedDomain) {
     case "security":
-      return [`${base} vendor advisory official`, `${base} cve mitigation official`];
+      return [`${base} vendor advisory official${exclusions}`, `${base} cve mitigation official${exclusions}`];
     case "vendor-status":
-      return [`${base} incident status official`, `${base} status page postmortem`];
+      return [`${base} incident status official${exclusions}`, `${base} status page postmortem${exclusions}`];
     case "package-registry":
-      return [`${base} release notes changelog`, `${base} maintainer docs`];
+      return [`${base} release notes changelog${exclusions}`, `${base} maintainer docs${exclusions}`];
     case "github":
-      return [`${base} github releases readme`, `${base} canonical repo docs`];
+      return [`${base} github releases readme${exclusions}`, `${base} canonical repo docs${exclusions}`];
     case "papers":
-      return [`${base} arxiv doi compare`, `${base} publisher abstract official`];
+      return [`${base} arxiv doi compare${exclusions}`, `${base} publisher abstract official${exclusions}`];
     default:
-      return [`${base} official docs support status`, `${base} official comparison reference`];
+      return [`${base} official docs support status${exclusions}`, `${base} official comparison reference${exclusions}`];
   }
 }

package/lib/research-profiles.json

@@ -6,6 +6,7 @@
     "resultsPerQuery": 4,
     "maxPages": 3,
     "pageTimeoutMs": 6000,
+    "stealthTimeoutMs": 30000,
     "pageTextLimit": 4000,
     "minPageText": 300,
     "useJinaFallback": true,
@@ -23,6 +24,7 @@
     "resultsPerQuery": 5,
     "maxPages": 8,
     "pageTimeoutMs": 10000,
+    "stealthTimeoutMs": 40000,
     "pageTextLimit": 8000,
     "minPageText": 300,
     "useJinaFallback": true,
@@ -40,6 +42,7 @@
     "resultsPerQuery": 5,
     "maxPages": 6,
     "pageTimeoutMs": 10000,
+    "stealthTimeoutMs": 40000,
     "pageTextLimit": 8000,
     "minPageText": 300,
     "useJinaFallback": true,
@@ -57,6 +60,7 @@
     "resultsPerQuery": 5,
     "maxPages": 6,
     "pageTimeoutMs": 10000,
+    "stealthTimeoutMs": 40000,
     "pageTextLimit": 8000,
     "minPageText": 300,
     "useJinaFallback": true,

package/lib/research.js

@@ -459,7 +459,7 @@ export function detectConflictSignals(pages) {
   return { detected: false, reason: null, conflictSummary: "", conflictingSourcePairs: [] };
 }
 
-export function detectResearchGaps(query, pages) {
+export function detectResearchGaps(query, pages, options = {}) {
   const hasAuthoritativeSource = pages.some((page) => {
     const scored = scoreSourceEntry(page, query || "");
     return Boolean(page.authoritative || scored.authoritative);
@@ -468,7 +468,7 @@ export function detectResearchGaps(query, pages) {
     return {
       detected: true,
       reason: "Retrieved pages lack an authoritative docs or README source.",
-      followupQuery: buildAuthorityFollowUpQueries(query)[0] || `${queryBase(query)} official docs`,
+      followupQuery: buildAuthorityFollowUpQueries(query, "", options)[0] || `${queryBase(query)} official docs`,
       missingAspects: ["authoritative sources"],
     };
   }
@@ -476,12 +476,21 @@ export function detectResearchGaps(query, pages) {
   return { detected: false, reason: null, followupQuery: null, missingAspects: [] };
 }
 
-export function buildFollowUpQuery(query, pages) {
+export function buildFollowUpQuery(query, pages, options = {}) {
   const conflict = detectConflictSignals(pages);
-  if (conflict.detected) return buildConflictFollowUpQueries(query)[0] || `${queryBase(query)} official docs support status`;
-  const gaps = detectResearchGaps(query, pages);
+  if (conflict.detected) return buildConflictFollowUpQueries(query, "", options)[0] || `${queryBase(query)} official docs support status`;
+  const gaps = detectResearchGaps(query, pages, options);
   if (gaps.detected) return gaps.followupQuery;
-  return buildAuthorityFollowUpQueries(`${queryBase(query)} clarification`)[0] || `${queryBase(query)} clarification official docs`;
+  return buildAuthorityFollowUpQueries(`${queryBase(query)} clarification`, "", options)[0] || `${queryBase(query)} clarification official docs`;
+}
+
+export function buildActionBasedFollowUpQuery(query, action, options = {}) {
+  if (action === "need_conflict_resolution") return buildConflictFollowUpQueries(query, "", options)[0] || `${queryBase(query)} official docs support status`;
+  if (action === "need_authority") return buildAuthorityFollowUpQueries(query, "", options)[0] || `${queryBase(query)} official docs`;
+  if (action === "need_recency") return `${queryBase(query)} latest`;
+  if (action === "need_version_context") return `${queryBase(query)} version diff`;
+  if (action === "need_primary_source") return `${queryBase(query)} source announcement`;
+  return buildAuthorityFollowUpQueries(`${queryBase(query)} clarification`, "", options)[0] || `${queryBase(query)} clarification official docs`;
 }
 
 function queryTermsForFactCheck(text) {