pi-unsloth-webtools 0.1.0

package/index.ts

@@ -0,0 +1,86 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { webSearch } from "./web-search.ts";
+import { fetchPageText } from "./web-fetch.ts";
+
+const FETCH_TIMEOUT_MS = 60_000;
+
+const WebSearchParams = Type.Object({
+  query: Type.Optional(
+    Type.String({ description: "The search query" }),
+  ),
+  url: Type.Optional(
+    Type.String({
+      description:
+        "A URL to fetch full page content from (instead of searching). Use this to read a page found in search results.",
+    }),
+  ),
+});
+
+const WebFetchParams = Type.Object({
+  url: Type.String({ description: "URL of the page to fetch" }),
+  maxChars: Type.Optional(
+    Type.Number({
+      description: "Truncate the returned content to this many characters (default: no limit)",
+    }),
+  ),
+});
+
+export default function (pi: ExtensionAPI) {
+  pi.registerTool({
+    name: "web_search",
+    label: "Web Search",
+    description:
+      "Search the web and fetch page content. Returns snippets for all results. " +
+      "Use the url parameter to fetch full page text from a specific URL.",
+    promptSnippet: "Search the web and fetch page content",
+    promptGuidelines: [
+      "Use web_search with the url parameter (e.g. {\"url\": \"<URL>\"}) to read the full text of a page found in search results.",
+    ],
+    parameters: WebSearchParams,
+    async execute(_toolCallId, params, signal, _onUpdate, _ctx) {
+      if (params.url?.trim()) {
+        return {
+          content: [
+            {
+              type: "text",
+              text: await fetchPageText(params.url.trim(), {
+                timeoutMs: FETCH_TIMEOUT_MS,
+                signal: signal ?? undefined,
+              }),
+            },
+          ],
+          details: {},
+        };
+      }
+      const text = await webSearch(params.query, { signal: signal ?? undefined });
+      return { content: [{ type: "text", text }], details: {} };
+    },
+  });
+
+  pi.registerTool({
+    name: "web_fetch",
+    label: "Web Fetch",
+    description:
+      "Fetch a URL and return readable text content. HTML responses are converted to Markdown " +
+      "with a main-content heuristic (article/main scoping, hidden-element and boilerplate " +
+      "stripping); non-HTML text responses are returned as-is. GitHub repo root pages are " +
+      "rewritten to the README API so the model reads the README instead of the repo page's UI " +
+      "chrome. Blocks private/loopback/link-local targets (SSRF protection) and caps the " +
+      "download size.",
+    promptSnippet: "Fetch a web page and return readable text content",
+    parameters: WebFetchParams,
+    async execute(_toolCallId, params, signal, _onUpdate, _ctx) {
+      const maxChars =
+        typeof params.maxChars === "number" && params.maxChars > 0
+          ? params.maxChars
+          : undefined;
+      const text = await fetchPageText(params.url, {
+        timeoutMs: FETCH_TIMEOUT_MS,
+        signal: signal ?? undefined,
+        maxChars,
+      });
+      return { content: [{ type: "text", text }], details: {} };
+    },
+  });
+}

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "pi-unsloth-webtools",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Pi extension: web_search and web_fetch tools ported from the Unsloth Studio codebase (DuckDuckGo search, SSRF-safe direct fetching, HTML-to-Markdown extraction)",
+  "main": "index.ts",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/YuGiMob/pi-unsloth-webtools.git"
+  },
+  "author": "YuGiMob",
+  "keywords": [
+    "pi-package",
+    "pi",
+    "coding-agent",
+    "extension",
+    "web-search",
+    "web-fetch",
+    "unsloth",
+    "duckduckgo",
+    "ssrf"
+  ],
+  "license": "AGPL-3.0",
+  "files": [
+    "index.ts",
+    "web-search.ts",
+    "web-fetch.ts",
+    "web-access.ts",
+    "html-to-md.ts",
+    "engines.ts",
+    "entities.ts",
+    "pdf.ts",
+    "README.md",
+    "ROADMAP.md",
+    "LICENSE"
+  ],
+  "pi": {
+    "extensions": [
+      "./index.ts"
+    ]
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "typebox": "*"
+  },
+  "engines": {
+    "node": ">=22.19.0"
+  },
+  "scripts": {
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "npm run typecheck"
+  },
+  "devDependencies": {
+    "@earendil-works/pi-coding-agent": "^0.84.0",
+    "@types/node": "^24.0.0",
+    "typebox": "^1.3.7",
+    "typescript": "~5.9.3",
+    "vitest": "^4.1.9"
+  },
+  "dependencies": {
+    "mupdf": "^1.28.0"
+  }
+}