pi-research 1.0.0

package/README.md

@@ -0,0 +1,90 @@
+# pi-research
+
+[![npm version](https://img.shields.io/npm/v/pi-research?color=blue)](https://www.npmjs.com/package/pi-research)
+[![tests](https://img.shields.io/badge/tests-33%2F33-brightgreen)](https://github.com/endgegnerbert-tech/pi-research)
+[![Pi package](https://img.shields.io/badge/pi-package-blueviolet)](https://pi.ai)
+
+`pi-research` is a Pi extension for web research.
+
+## Install
+
+```bash
+pi install npm:pi-research
+```
+
+GitHub repository: https://github.com/endgegnerbert-tech/pi-research
+
+You can also fork the repository and install it from a local path while developing.
+
+## What it is for
+
+Use `pi-research` when you want the agent to search and synthesize the web.
+It is designed for research, not browser navigation.
+Use `browser_action` for clicks, screenshots, DOM inspection, or page interaction.
+
+## Modes
+
+| Mode | Best for |
+| --- | --- |
+| `fast` | quick answers with a quality floor |
+| `deep` | broader retrieval with follow-up rounds |
+| `code` | official docs, READMEs, repos, and code snippets |
+| `academic` | scholarly sources like arXiv, Semantic Scholar, and DOI papers |
+
+## Key features
+
+- query-isolated caching and sufficiency gating
+- source scoring with visible `sourceType`, `authoritative`, `score`, and `freshness`
+- `openSubQuestions`, `missingAspects`, `conflictSummary`
+- inline citations in the final answer
+- `minYear`, `maxYear`, and `preferRecent` support
+- `files[]` for local source input
+- `codeBlocks[]` extraction for code-focused answers
+
+## Output
+
+The tool returns structured data including:
+
+- `answer`
+- `bullets`
+- `sources`
+- `citations`
+- `codeBlocks`
+- `confidence`
+- `confidenceScore`
+- `sufficient`
+- `authoritativeSourcesFound`
+- `followupRounds`
+- `followupQuery`
+- `openSubQuestions`
+- `missingAspects`
+- `conflictSummary`
+- `conflictingSourcePairs`
+- `unverifiedClaims`
+
+## Examples
+
+```text
+What are the trade-offs between B-trees and LSM-trees?
+```
+
+```text
+Show me the best way to add health checks to Docker Compose.
+```
+
+```text
+Compare React Server Components with traditional SSR.
+```
+
+## Package manifest
+
+This repo is a Pi package. The extension entrypoint is:
+
+- `extensions/pi-research.ts`
+
+## Release notes
+
+- Package name: `pi-research`
+- Install command: `pi install npm:pi-research`
+- GitHub: `https://github.com/endgegnerbert-tech/pi-research`
+- Tool name: `pi-research`

package/extensions/pi-research.ts

@@ -0,0 +1 @@
+export { default } from "../index.js";

package/index.js

@@ -0,0 +1,144 @@
+import { Type } from "typebox";
+
+import { compactResearchPayload, classifyQueryIntent, inferOfficialDocsSite } from "./lib/research.js";
+import { clearResearchMemory, hashResearchQuery, setResearchMemory, shouldSkipResearch } from "./lib/research-memory.js";
+import { runWebResearch } from "./lib/web-research.js";
+
+const RESEARCH_STATE = new Map();
+
+function buildWebResearchGuidance() {
+  return "Use pi-research for web search and research. Prefer fast mode for simple questions, deep mode for comparisons or ambiguous cases, and code/academic modes when source type matters.";
+}
+
+function defaultMode(query) {
+  const intent = classifyQueryIntent(query);
+  if (intent === "comparison" || intent === "comparative") return "deep";
+  if (intent === "academic") return "academic";
+  return "fast";
+}
+
+function buildFastRecoveryQuery(query) {
+  const docsSite = inferOfficialDocsSite(query || "");
+  return docsSite ? `site:${docsSite} ${query}` : `${query} official docs`;
+}
+
+function toolResponse(payload, text) {
+  return { content: [{ type: "text", text }], details: payload };
+}
+
+function compactWebResearchToolResult(event) {
+  if (event.isError || event.toolName !== "pi-research") return null;
+  const payload = event.details;
+  if (!payload?.ok || payload.action !== "web_research") return null;
+
+  const compact = compactResearchPayload(payload);
+  const citationLines = Array.isArray(compact.citations)
+    ? compact.citations.map((citation, index) => `${index + 1}. ${citation.text} [source ${citation.sourceIndex}]`)
+    : [];
+  const text = [
+    payload.contentText,
+    "",
+    "## Citations",
+    "",
+    ...(citationLines.length ? citationLines : ["None"]),
+    "",
+    "## Status",
+    "",
+    `sufficient: ${compact.sufficient}`,
+    `authoritativeSourcesFound: ${compact.authoritativeSourcesFound}`,
+    ...(compact.conflictSummary ? [`conflictSummary: ${compact.conflictSummary}`] : []),
+  ].join("\n").trim();
+
+  return { content: [{ type: "text", text }] };
+}
+
+function getState(queryHash) {
+  if (!RESEARCH_STATE.has(queryHash)) RESEARCH_STATE.set(queryHash, { count: 0, lastHash: null, lastSufficient: false, fastRecoveryAllowed: false });
+  return RESEARCH_STATE.get(queryHash);
+}
+
+export default function webResearchExtension(pi) {
+  pi.on("before_agent_start", async (event) => {
+    RESEARCH_STATE.clear();
+    clearResearchMemory();
+    return { systemPrompt: `${event.systemPrompt}\n\n${buildWebResearchGuidance()}` };
+  });
+
+  pi.on("tool_call", async (event) => {
+    if (event.toolName !== "pi-research") return;
+    if (!event.input.mode) event.input.mode = defaultMode(event.input.query || "");
+
+    const queryHash = hashResearchQuery(event.input.query || "");
+    const state = getState(queryHash);
+    const mode = event.input.mode;
+    const isolate = Boolean(event.input.isolate || process.env.RESEARCH_ISOLATE === "1");
+    const force = Boolean(event.input.force);
+
+    if (shouldSkipResearch({ queryHash, lastHash: state.lastHash, lastWasSufficient: state.lastSufficient, force, isolate })) {
+      return { block: true, reason: "Recent pi-research result was already sufficient for this exact query." };
+    }
+
+    if (mode === "fast" && state.count === 1 && state.fastRecoveryAllowed && !force && !isolate) {
+      event.input.query = buildFastRecoveryQuery(event.input.query || "");
+      state.fastRecoveryAllowed = false;
+    }
+
+    state.count += 1;
+    state.lastHash = queryHash;
+  });
+
+  pi.on("tool_result", async (event) => {
+    if (event.toolName === "pi-research" && !event.isError && event.details?.ok) {
+      const queryHash = hashResearchQuery(event.input?.query || "");
+      const state = getState(queryHash);
+      state.lastHash = queryHash;
+      state.lastSufficient = Boolean(event.details.sufficient);
+      const query = event.input?.query || "";
+      state.fastRecoveryAllowed = !event.details.sufficient
+        && !event.details.authoritativeSourcesFound
+        && ["best_practice", "temporal", "definition"].includes(classifyQueryIntent(query || ""));
+      setResearchMemory(`last:${queryHash}`, event.details);
+    }
+    return compactWebResearchToolResult(event) || undefined;
+  });
+
+  pi.registerTool({
+    name: "pi-research",
+    label: "Pi Research",
+    description: "Search and research the web.",
+    promptSnippet: "Use this for web research when needed.",
+    promptGuidelines: ["Use pi-research for search, source ranking, and summarization."],
+    parameters: Type.Object({
+      query: Type.String({ description: "Research question to answer from the web" }),
+      mode: Type.Optional(Type.Union([Type.Literal("fast"), Type.Literal("deep"), Type.Literal("code"), Type.Literal("academic")], { description: "Research mode", default: "fast" })),
+      force: Type.Optional(Type.Boolean({ description: "Bypass sufficiency gating and cached answers for this call" })),
+      isolate: Type.Optional(Type.Boolean({ description: "Run this query in isolation without session/query cache reuse" })),
+      options: Type.Optional(Type.Object({
+        allowedSources: Type.Optional(Type.Array(Type.String())),
+        maxTurns: Type.Optional(Type.Number()),
+        maxSites: Type.Optional(Type.Number()),
+        requireAuthoritative: Type.Optional(Type.Boolean()),
+        minYear: Type.Optional(Type.Number()),
+        maxYear: Type.Optional(Type.Number()),
+        preferRecent: Type.Optional(Type.Boolean()),
+        files: Type.Optional(Type.Array(Type.String())),
+        format: Type.Optional(Type.Union([Type.Literal("markdown"), Type.Literal("json"), Type.Literal("table"), Type.Literal("latex")], { default: "markdown" })),
+        deepResearchConfig: Type.Optional(Type.Object({
+          depth: Type.Optional(Type.Union([Type.Literal(1), Type.Literal(2), Type.Literal(3)])),
+          breadth: Type.Optional(Type.Union([Type.Literal(2), Type.Literal(3), Type.Literal(4)])),
+          concurrency: Type.Optional(Type.Union([Type.Literal(1), Type.Literal(2), Type.Literal(3), Type.Literal(4)])),
+        })),
+      })),
+    }),
+    async execute(_toolCallId, params, signal, onUpdate, ctx) {
+      const mode = params.mode ?? defaultMode(params.query || "");
+      const payload = await runWebResearch(params.query || "", ctx, signal, onUpdate, {
+        mode,
+        force: params.force,
+        isolate: params.isolate,
+        ...(params.options || {}),
+      });
+      return toolResponse(payload, payload.ok ? payload.contentText : JSON.stringify(payload, null, 2));
+    },
+  });
+}

package/lib/planner.js

@@ -0,0 +1,36 @@
+import { buildDeepQueries, buildFastQueries, classifyQueryIntent, inferOfficialDocsSite } from "./research.js";
+
+export function planResearch(query, mode = "fast") {
+  const intent = classifyQueryIntent(query);
+  const docsSite = inferOfficialDocsSite(query || "");
+
+  if (mode === "academic" || intent === "academic") {
+    const subqueries = buildDeepQueries(query, 4);
+    return {
+      subqueries,
+      expectedSources: ["paper", "official_doc"],
+    };
+  }
+
+  if (mode === "code") {
+    const base = String(query || "").trim();
+    const comparisonFallback = intent === "comparison" ? `${base} benchmark comparison` : null;
+    const subqueries = [...new Set([
+      `${base} site:github.com`,
+      `${base} official docs`,
+      docsSite ? `${base} site:${docsSite}` : `${base} README`,
+      comparisonFallback,
+      ...buildFastQueries(query, 2),
+    ])].filter(Boolean).slice(0, 4);
+
+    return {
+      subqueries,
+      expectedSources: ["github_readme", "github_repo", "official_doc"],
+    };
+  }
+
+  return {
+    subqueries: buildFastQueries(query, 2),
+    expectedSources: intent === "comparison" ? ["official_doc", "other"] : ["official_doc"],
+  };
+}

package/lib/research-memory.js

@@ -0,0 +1,87 @@
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+
+const memory = new Map();
+const CACHE_PATH = join(homedir(), ".pi", "agent", "lazy-modules", "web-research", ".cache", "research-cache.json");
+
+function ensureCacheDir() {
+  mkdirSync(dirname(CACHE_PATH), { recursive: true });
+}
+
+function readPersistentCache() {
+  try {
+    const raw = readFileSync(CACHE_PATH, "utf8");
+    return JSON.parse(raw);
+  } catch {
+    return {};
+  }
+}
+
+function writePersistentCache(cache) {
+  ensureCacheDir();
+  writeFileSync(CACHE_PATH, JSON.stringify(cache, null, 2));
+}
+
+export function normalizeResearchQuery(query = "") {
+  return String(query)
+    .toLowerCase()
+    .trim()
+    .replace(/[?!.,:;]+/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+export function hashResearchQuery(query = "") {
+  return createHash("sha1").update(normalizeResearchQuery(query)).digest("hex");
+}
+
+export function shouldSkipResearch({ queryHash, lastHash, lastWasSufficient, force = false, isolate = false }) {
+  if (force || isolate) return false;
+  return Boolean(queryHash && lastHash && queryHash === lastHash && lastWasSufficient);
+}
+
+export function getResearchMemory(key) {
+  return memory.get(key) || null;
+}
+
+export function setResearchMemory(key, value) {
+  memory.set(key, value);
+  return value;
+}
+
+export function clearResearchMemory() {
+  memory.clear();
+}
+
+export function writeCachedResult(key, value, ttlMs) {
+  const payload = {
+    expiresAt: Date.now() + ttlMs,
+    value,
+  };
+  memory.set(`persistent:${key}`, payload);
+  const cache = readPersistentCache();
+  cache[key] = payload;
+  writePersistentCache(cache);
+  return value;
+}
+
+export function readCachedResult(key) {
+  const inMemory = memory.get(`persistent:${key}`);
+  if (inMemory) {
+    if (inMemory.expiresAt > Date.now()) return inMemory.value;
+    memory.delete(`persistent:${key}`);
+  }
+
+  const cache = readPersistentCache();
+  const entry = cache[key];
+  if (!entry) return null;
+  if (entry.expiresAt <= Date.now()) {
+    delete cache[key];
+    writePersistentCache(cache);
+    return null;
+  }
+  memory.set(`persistent:${key}`, entry);
+  return entry.value;
+}

package/lib/research-profiles.json

@@ -0,0 +1,72 @@
+{
+  "fast": {
+    "mode": "fast",
+    "maxTurns": 1,
+    "maxQueries": 2,
+    "resultsPerQuery": 4,
+    "maxPages": 3,
+    "pageTimeoutMs": 6000,
+    "pageTextLimit": 4000,
+    "minPageText": 300,
+    "useJinaFallback": true,
+    "maxChunksPerPage": 3,
+    "searchProvider": "ddg_html",
+    "allowedSourceTypes": ["official_doc", "github_readme", "github_repo", "paper", "blog", "forum", "other", "file"],
+    "minAuthoritativeSources": 1,
+    "minSources": 3,
+    "cacheTtlMs": 86400000
+  },
+  "deep": {
+    "mode": "deep",
+    "maxTurns": 2,
+    "maxQueries": 10,
+    "resultsPerQuery": 5,
+    "maxPages": 8,
+    "pageTimeoutMs": 10000,
+    "pageTextLimit": 8000,
+    "minPageText": 300,
+    "useJinaFallback": true,
+    "maxChunksPerPage": 3,
+    "searchProvider": "ddg_html",
+    "allowedSourceTypes": ["official_doc", "github_readme", "github_repo", "paper", "blog", "forum", "other", "file"],
+    "minAuthoritativeSources": 1,
+    "minSources": 4,
+    "cacheTtlMs": 86400000
+  },
+  "code": {
+    "mode": "code",
+    "maxTurns": 2,
+    "maxQueries": 6,
+    "resultsPerQuery": 5,
+    "maxPages": 6,
+    "pageTimeoutMs": 10000,
+    "pageTextLimit": 8000,
+    "minPageText": 300,
+    "useJinaFallback": true,
+    "maxChunksPerPage": 3,
+    "searchProvider": "ddg_html",
+    "allowedSourceTypes": ["official_doc", "github_readme", "github_repo", "other", "file"],
+    "minAuthoritativeSources": 1,
+    "minSources": 3,
+    "cacheTtlMs": 172800000
+  },
+  "academic": {
+    "mode": "academic",
+    "maxTurns": 2,
+    "maxQueries": 6,
+    "resultsPerQuery": 5,
+    "maxPages": 6,
+    "pageTimeoutMs": 10000,
+    "pageTextLimit": 8000,
+    "minPageText": 300,
+    "useJinaFallback": true,
+    "maxChunksPerPage": 3,
+    "searchProvider": "academic",
+    "allowedSourceTypes": ["paper", "official_doc", "github_readme", "github_repo", "other", "file"],
+    "minAuthoritativeSources": 1,
+    "minSources": 3,
+    "preferRecent": true,
+    "minYear": 2018,
+    "cacheTtlMs": 604800000
+  }
+}