pi-unsloth-webtools 0.3.0 → 0.4.0

package/index.ts

@@ -1,10 +1,23 @@
-import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { defineTool, type ExtensionAPI, type ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
-import { webSearch } from "./web-search.ts";
-import { DEFAULT_FETCH_TIMEOUT_MS, fetchPageText } from "./web-fetch.ts";
+import { webSearch as defaultWebSearch } from "./web-search.ts";
+import { DEFAULT_FETCH_TIMEOUT_MS, fetchPageText as defaultFetchPageText } from "./web-fetch.ts";
+import { loadDefaultFetchSettings } from "./settings.ts";
 
-function positiveMaxChars(value: unknown): number | undefined {
-  return typeof value === "number" && value > 0 ? value : undefined;
+function positiveNumber(value: unknown): number | undefined {
+  if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
+  const n = Math.floor(value);
+  return n > 0 ? n : undefined;
+}
+async function fetchDefaults(cwd: string | undefined, params: { timeoutMs?: unknown; maxChars?: unknown }) {
+  const timeoutParam = positiveNumber(params.timeoutMs);
+  const maxCharsParam = positiveNumber(params.maxChars);
+  if (timeoutParam !== undefined && maxCharsParam !== undefined) return { timeoutMs: timeoutParam, maxChars: maxCharsParam };
+  const defaults = await loadDefaultFetchSettings(cwd);
+  return {
+    timeoutMs: timeoutParam ?? defaults.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS,
+    maxChars: maxCharsParam ?? defaults.maxChars,
+  };
 }
 
 const WebSearchParams = Type.Object({
@@ -23,6 +36,19 @@ const WebSearchParams = Type.Object({
         "Truncate the fetched page to this many characters (only used with the url parameter)",
     }),
   ),
+  maxResults: Type.Optional(
+    Type.Number({
+      minimum: 1,
+      maximum: 20,
+      description: "Maximum number of search results to return (default: 5)",
+    }),
+  ),
+  timeoutMs: Type.Optional(
+    Type.Number({
+      minimum: 1000,
+      description: "Overall timeout in milliseconds for the search or fetch",
+    }),
+  ),
 });
 
 const WebFetchParams = Type.Object({
@@ -32,65 +58,93 @@ const WebFetchParams = Type.Object({
       description: "Truncate the returned content to this many characters (default: no limit)",
     }),
   ),
+  timeoutMs: Type.Optional(
+    Type.Number({
+      minimum: 1000,
+      description: "Overall timeout in milliseconds (default: 60000)",
+    }),
+  ),
 });
 
-export default function (pi: ExtensionAPI) {
-  pi.registerTool({
-    name: "web_search",
-    label: "Web Search",
-    description:
-      "Search the web and fetch page content. Returns snippets for all results. " +
-      "Use the url parameter to fetch full page text from a specific URL.",
-    promptSnippet: "Search the web and fetch page content",
-    promptGuidelines: [
-      "Use web_search with the url parameter (e.g. {\"url\": \"<URL>\"}) to read the full text of a page found in search results.",
-    ],
-    parameters: WebSearchParams,
-    async execute(_toolCallId, params, signal, onUpdate, _ctx) {
-      if (params.url?.trim()) {
-        const url = params.url.trim();
-        onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}...` }], details: {} });
-        return {
-          content: [
-            {
-              type: "text",
-              text: await fetchPageText(url, {
-                timeoutMs: DEFAULT_FETCH_TIMEOUT_MS,
-                signal: signal ?? undefined,
-                maxChars: positiveMaxChars(params.maxChars),
-              }),
-            },
-          ],
-          details: {},
-        };
-      }
-      onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
-      const text = await webSearch(params.query, { signal: signal ?? undefined });
-      return { content: [{ type: "text", text }], details: {} };
-    },
-  });
+export interface WebToolsDeps {
+  fetchPageText?: typeof defaultFetchPageText;
+  webSearch?: typeof defaultWebSearch;
+}
 
-  pi.registerTool({
-    name: "web_fetch",
-    label: "Web Fetch",
-    description:
-      "Fetch a URL and return its readable text. HTML pages are converted to Markdown using a " +
-      "main-content heuristic: article/main scoping plus hidden-element and boilerplate " +
-      "stripping. Non-HTML text is returned as-is. GitHub repo root pages are rewritten to the " +
-      "README API, so the README is returned instead of the repo page's UI chrome. " +
-      "Private/loopback/link-local targets are blocked (SSRF protection), and the download size " +
-      "is capped.",
-    promptSnippet: "Fetch a web page and return readable text content",
-    parameters: WebFetchParams,
-    async execute(_toolCallId, params, signal, onUpdate, _ctx) {
-      const maxChars = positiveMaxChars(params.maxChars);
-      onUpdate?.({ content: [{ type: "text", text: `Fetching ${params.url}...` }], details: {} });
-      const text = await fetchPageText(params.url, {
-        timeoutMs: DEFAULT_FETCH_TIMEOUT_MS,
-        signal: signal ?? undefined,
-        maxChars,
-      });
-      return { content: [{ type: "text", text }], details: {} };
-    },
-  });
+export function createWebTools(deps: WebToolsDeps = {}) {
+  const fetchPageText = deps.fetchPageText ?? defaultFetchPageText;
+  const webSearch = deps.webSearch ?? defaultWebSearch;
+  return {
+    webSearchTool: defineTool({
+      name: "web_search",
+      label: "Web Search",
+      description:
+        "Search the web and fetch page content. Returns snippets for all results. " +
+        "Use the url parameter to fetch full page text from a specific URL.",
+      promptSnippet: "Search the web and fetch page content",
+      promptGuidelines: [
+        'Use web_search with the url parameter (e.g. {"url": "<URL>"}) to read the full text of a page found in search results.',
+      ],
+      parameters: WebSearchParams,
+      async execute(_toolCallId, params, signal, onUpdate, _ctx) {
+        if (params.url?.trim()) {
+          const url = params.url.trim();
+          onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}...` }], details: {} });
+          const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
+          const { timeoutMs, maxChars } = await fetchDefaults(cwd, params);
+          return {
+            content: [
+              {
+                type: "text",
+                text: await fetchPageText(url, {
+                  timeoutMs,
+                  signal: signal ?? undefined,
+                  maxChars,
+                }),
+              },
+            ],
+            details: {},
+          };
+        }
+        onUpdate?.({ content: [{ type: "text", text: "Searching the web..." }], details: {} });
+        const text = await webSearch(params.query, {
+          signal: signal ?? undefined,
+          timeoutMs: positiveNumber(params.timeoutMs),
+          maxResults: positiveNumber(params.maxResults),
+          cwd: (_ctx as ExtensionContext | undefined)?.cwd,
+        });
+        return { content: [{ type: "text", text }], details: {} };
+      },
+    }),
+    webFetchTool: defineTool({
+      name: "web_fetch",
+      label: "Web Fetch",
+      description:
+        "Fetch a URL and return its readable text. HTML pages are converted to Markdown using a " +
+        "main-content heuristic: article/main scoping plus hidden-element and boilerplate " +
+        "stripping. Non-HTML text is returned as-is. GitHub repo root pages are rewritten to the " +
+        "README API, so the README is returned instead of the repo page's UI chrome. " +
+        "Private/loopback/link-local targets are blocked (SSRF protection), and the download size " +
+        "is capped.",
+      promptSnippet: "Fetch a web page and return readable text content",
+      parameters: WebFetchParams,
+      async execute(_toolCallId, params, signal, onUpdate, _ctx) {
+        onUpdate?.({ content: [{ type: "text", text: `Fetching ${params.url}...` }], details: {} });
+        const cwd = (_ctx as ExtensionContext | undefined)?.cwd;
+        const { timeoutMs, maxChars } = await fetchDefaults(cwd, params);
+        const text = await fetchPageText(params.url, {
+          timeoutMs,
+          signal: signal ?? undefined,
+          maxChars,
+        });
+        return { content: [{ type: "text", text }], details: {} };
+      },
+    }),
+  };
+}
+
+export default function (pi: ExtensionAPI) {
+  const { webSearchTool, webFetchTool } = createWebTools();
+  pi.registerTool(webSearchTool);
+  pi.registerTool(webFetchTool);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -31,6 +31,9 @@
     "entities.ts",
     "pdf.ts",
     "user-agents.ts",
+    "cache.ts",
+    "settings.ts",
+    "agent-dir.ts",
     "README.md",
     "LICENSE"
   ],

package/pdf.ts

@@ -594,7 +594,7 @@ function assemblePages(
   pageLimitOverride?: boolean,
 ): string {
   const parts: string[] = [];
-  const pageLimitReached = pageLimitOverride ?? pages.length >= MAX_WEB_PDF_PAGES;
+  const pageLimitReached = pageLimitOverride ?? pages.length > MAX_WEB_PDF_PAGES;
   for (const page of pages) {
     const pageText = page.text.trim();
     if (!pageText) continue;
@@ -697,6 +697,11 @@ function extractTextOps(bytes: Buffer): string {
         }
       }
     }
+    if (c === "'" || c === '"') {
+      if (inText) out.push("\n");
+      i++;
+      continue;
+    }
     const token = text.slice(i, i + 2);
     if (token === "BT") {
       inText = true;

package/settings.ts

@@ -0,0 +1,114 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { agentDir } from "./agent-dir.ts";
+
+async function readJson(file: string): Promise<Record<string, unknown> | undefined> {
+  try {
+    const raw = await readFile(file, "utf-8");
+    const data = JSON.parse(raw);
+    if (data && typeof data === "object" && !Array.isArray(data)) return data as Record<string, unknown>;
+    return undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function toNumber(value: unknown): number | undefined {
+  if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
+  return Math.floor(value);
+}
+
+function pickNumber(data: Record<string, unknown>, paths: string[][]): number | undefined {
+  for (const path of paths) {
+    let cur: unknown = data;
+    for (const key of path) {
+      if (cur && typeof cur === "object" && !Array.isArray(cur)) cur = (cur as Record<string, unknown>)[key];
+      else {
+        cur = undefined;
+        break;
+      }
+    }
+    const n = toNumber(cur);
+    if (n !== undefined) return n;
+  }
+  return undefined;
+}
+
+const MAX_RESULTS = 5;
+
+const MAX_RESULTS_PATHS: string[][] = [
+  ["unslothWebTools", "maxResults"],
+  ["webSearch", "maxResults"],
+  ["smartWebSearch", "resultsPerQuery"],
+];
+
+const FETCH_MAX_CHARS_PATHS: string[][] = [
+  ["unslothWebTools", "maxChars"],
+  ["webFetch", "maxChars"],
+  ["smartFetchDefaultMaxChars"],
+];
+
+const FETCH_TIMEOUT_PATHS: string[][] = [
+  ["unslothWebTools", "timeoutMs"],
+  ["webFetch", "timeoutMs"],
+  ["smartFetchDefaultTimeoutMs"],
+];
+
+function clampMaxResults(value: number): number {
+  return Math.min(20, Math.max(1, value));
+}
+
+function settingsFiles(cwd?: string): string[] {
+  const base = agentDir();
+  const globalFile = base ? join(base, "settings.json") : "";
+  const files = globalFile ? [globalFile] : [];
+  if (cwd) files.push(join(cwd, ".pi", "settings.json"));
+  return files;
+}
+
+async function settingsEntries(cwd?: string): Promise<Record<string, unknown>[]> {
+  const entries: Record<string, unknown>[] = [];
+  for (const file of settingsFiles(cwd)) {
+    const data = await readJson(file);
+    if (data) entries.push(data);
+  }
+  return entries;
+}
+
+export async function loadDefaultMaxResults(cwd?: string): Promise<number> {
+  let result = MAX_RESULTS;
+  for (const data of await settingsEntries(cwd)) {
+    const candidate = pickNumber(data, MAX_RESULTS_PATHS);
+    if (candidate !== undefined) result = clampMaxResults(candidate);
+  }
+  return result;
+}
+
+async function loadFetchSetting(cwd: string | undefined, paths: string[][], valid: (n: number) => boolean): Promise<number | undefined> {
+  let result: number | undefined;
+  for (const data of await settingsEntries(cwd)) {
+    const candidate = pickNumber(data, paths);
+    if (candidate !== undefined && valid(candidate)) result = candidate;
+  }
+  return result;
+}
+
+export async function loadDefaultFetchMaxChars(cwd?: string): Promise<number | undefined> {
+  return loadFetchSetting(cwd, FETCH_MAX_CHARS_PATHS, (n) => n > 0);
+}
+
+export async function loadDefaultFetchTimeoutMs(cwd?: string): Promise<number | undefined> {
+  return loadFetchSetting(cwd, FETCH_TIMEOUT_PATHS, (n) => n >= 1000);
+}
+
+export async function loadDefaultFetchSettings(cwd?: string): Promise<{ maxChars?: number; timeoutMs?: number }> {
+  let maxChars: number | undefined;
+  let timeoutMs: number | undefined;
+  for (const data of await settingsEntries(cwd)) {
+    const c = pickNumber(data, FETCH_MAX_CHARS_PATHS);
+    if (c !== undefined && c > 0) maxChars = c;
+    const t = pickNumber(data, FETCH_TIMEOUT_PATHS);
+    if (t !== undefined && t >= 1000) timeoutMs = t;
+  }
+  return { maxChars, timeoutMs };
+}

package/web-access.ts

@@ -6,7 +6,13 @@ const MAX_CACHEABLE_DOMAIN_LEN = 253;
 const SITE_FILTER_LIMIT = 8;
 const DOTTED_HOST_RE = /^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+$/;
 const PORT_RE = /^[0-9]{1,5}$/;
+export const MAX_SIGNAL_TIMEOUT_MS = 2 ** 31 - 1;
 
+function throwInvalidDomain(value: unknown): never {
+  throw new Error(`Invalid website domain: ${String(value)}`);
+}
+
+const INVALID_HOST_REASON = "Blocked: the URL has an invalid hostname or port.";
 export interface WebsitePolicy {
   allowedDomains: string[];
   blockedDomains: string[];
@@ -19,24 +25,30 @@ export function normalizeDomain(value: unknown): string {
     [...domain].some((char) => char.charCodeAt(0) < 32) ||
     ["\\", "/", "@", "?", "#"].some((char) => domain.includes(char))
   ) {
-    throw new Error(`Invalid website domain: ${String(value)}`);
+    throwInvalidDomain(value);
   }
-  const bracketed = domain.startsWith("[") && domain.endsWith("]");
-  if (domain.startsWith("[") !== domain.endsWith("]")) {
-    throw new Error(`Invalid website domain: ${String(value)}`);
+  const startsBracket = domain.startsWith("[");
+  const endsBracket = domain.endsWith("]");
+  if (startsBracket !== endsBracket) {
+    throwInvalidDomain(value);
   }
-  const stripped = (bracketed ? domain.slice(1, -1) : domain).replace(/\.+$/, "");
+  const stripped = (startsBracket ? domain.slice(1, -1) : domain).replace(/\.+$/, "");
   if (/^[0-9a-fA-F:.]+$/.test(stripped) && stripped.includes(":")) {
     try {
       return compressIpv6(stripped);
     } catch {
-      throw new Error(`Invalid website domain: ${String(value)}`);
+      throwInvalidDomain(value);
     }
   }
   if (stripped.includes(":")) {
     throw new Error("Website limits must contain domains without schemes or ports");
   }
   const numericParts = stripped.split(".");
+  const canonicalIpv4 =
+    numericParts.length === 4 &&
+    numericParts.every((part) => /^(?:0|[1-9][0-9]{0,2})$/.test(part)) &&
+    numericParts.every((part) => Number(part) <= 255);
+  if (canonicalIpv4) return stripped;
   if (
     numericParts.length <= 4 &&
     numericParts.every((part) => /^(?:0x[0-9a-f]+|[0-9]+)$/.test(part))
@@ -47,13 +59,13 @@ export function normalizeDomain(value: unknown): string {
   try {
     asciiDomain = domainToASCII(stripped).toLowerCase();
   } catch {
-    throw new Error(`Invalid website domain: ${String(value)}`);
+    throwInvalidDomain(value);
   }
   if (
     asciiDomain.length > 253 ||
     !asciiDomain.split(".").every((label) => DOMAIN_LABEL_RE.test(label))
   ) {
-    throw new Error(`Invalid website domain: ${String(value)}`);
+    throwInvalidDomain(value);
   }
   return asciiDomain;
 }
@@ -189,7 +201,7 @@ export function checkUrlAccess(
   try {
     parsed = new URL(candidate);
   } catch {
-    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
+    return [false, INVALID_HOST_REASON, ""];
   }
   const scheme = parsed.protocol.replace(/:$/, "").toLowerCase();
   if (scheme !== "http" && scheme !== "https") {
@@ -199,20 +211,16 @@ export function checkUrlAccess(
     return [false, "Blocked: URLs with credentials or encoded hostnames are not allowed.", ""];
   }
   if (!parsed.hostname) {
-    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
+    return [false, INVALID_HOST_REASON, ""];
   }
-  try {
-    if (parsed.port && !(PORT_RE.test(parsed.port) && Number(parsed.port) >= 1 && Number(parsed.port) <= 65535)) {
-      return [false, "Blocked: the URL has an invalid hostname or port.", ""];
-    }
-  } catch {
-    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
+  if (parsed.port && !(PORT_RE.test(parsed.port) && Number(parsed.port) >= 1 && Number(parsed.port) <= 65535)) {
+    return [false, INVALID_HOST_REASON, ""];
   }
   let hostname: string;
   try {
     hostname = normalizeDomain(parsed.hostname);
   } catch {
-    return [false, "Blocked: the URL has an invalid hostname or port.", ""];
+    return [false, INVALID_HOST_REASON, ""];
   }
   if (!policyAllows(hostname, policy)) {
     return [false, `Blocked: the website access policy disallows ${hostname}.`, hostname];
@@ -318,7 +326,7 @@ const GITHUB_NON_OWNER_SEGMENTS = new Set([
 
 const GITHUB_NAME_RE = /^[A-Za-z0-9_.\-]{1,100}$/;
 
-function githubRepoOwnerRepo(url: string): [string, string] | null {
+function parseGithubRepo(url: string): { owner: string; repo: string; rest: string[] } | null {
   let parsed: URL;
   try {
     parsed = new URL(url);
@@ -328,22 +336,40 @@ function githubRepoOwnerRepo(url: string): [string, string] | null {
   const host = (parsed.hostname ?? "").toLowerCase();
   if (host !== "github.com" && host !== "www.github.com") return null;
   const parts = parsed.pathname.split("/").filter((part) => part.length > 0);
-  if (parts.length !== 2) return null;
-  const [owner, repo] = parts;
+  if (parts.length < 2) return null;
+  const owner = parts[0];
+  const repoRaw = parts[1];
   if (GITHUB_NON_OWNER_SEGMENTS.has(owner.toLowerCase())) return null;
-  const cleanRepo = repo.endsWith(".git") ? repo.slice(0, -4) : repo;
-  if (!GITHUB_NAME_RE.test(owner) || !GITHUB_NAME_RE.test(cleanRepo)) return null;
-  return [owner, cleanRepo];
+  const repo = repoRaw.endsWith(".git") ? repoRaw.slice(0, -4) : repoRaw;
+  if (!GITHUB_NAME_RE.test(owner) || !GITHUB_NAME_RE.test(repo)) return null;
+  return { owner, repo, rest: parts.slice(2) };
+}
+function githubRepoRoot(url: string): { owner: string; repo: string; rest: string[] } | null {
+  const parsed = parseGithubRepo(url);
+  if (!parsed || parsed.rest.length !== 0) return null;
+  return parsed;
 }
 
 export function githubRepoReadmeApiUrl(url: string): string | null {
-  const pair = githubRepoOwnerRepo(url);
-  return pair ? `https://api.github.com/repos/${pair[0]}/${pair[1]}/readme` : null;
+  const parsed = githubRepoRoot(url);
+  if (!parsed) return null;
+  return `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/readme`;
 }
 
 export function githubRepoRawReadmeUrl(url: string): string | null {
-  const pair = githubRepoOwnerRepo(url);
-  return pair ? `https://raw.githubusercontent.com/${pair[0]}/${pair[1]}/HEAD/README.md` : null;
+  const parsed = githubRepoRoot(url);
+  if (!parsed) return null;
+  return `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/HEAD/README.md`;
+}
+export function githubRawContentUrl(url: string): string | null {
+  const parsed = parseGithubRepo(url);
+  if (!parsed || parsed.rest.length < 2) return null;
+  const [kind, ref] = parsed.rest;
+  if (kind !== "blob" && kind !== "raw") return null;
+  if (!ref) return null;
+  const filePath = parsed.rest.slice(2).join("/");
+  if (!filePath) return null;
+  return `https://raw.githubusercontent.com/${parsed.owner}/${parsed.repo}/${ref}/${filePath}`;
 }
 
 function ipv4Octets(ip: string): number[] | null {