ultimate-pi 0.4.1 → 0.5.0

package/.agents/skills/harness-decisions/SKILL.md

@@ -19,6 +19,21 @@ description: Structured user decisions via ask_user for harness setup, planning,
 3. If the user **cancels** (Esc), stop with `needs_clarification` / `human_required` — do not assume defaults.
 4. **CI / automation only:** pass `--non-interactive` to `/harness-setup` to skip prompts and use documented defaults.
 
+## Example (harness-setup — search engine)
+
+```json
+{
+  "question": "Which harness-web search backend should this project use?",
+  "context": "Scrapling handles scrape/map/bulk. Search: DDG HTML needs no Docker. SearXNG must be self-hosted — public instances often block JSON and rate-limit API to ~4/hour per IP.",
+  "options": [
+    { "title": "DuckDuckGo HTML (default)", "description": "HARNESS_WEB_SEARCH_ENGINE=ddg_html" },
+    { "title": "Self-host SearXNG here (Docker)", "description": "node harness-searxng-bootstrap.mjs" },
+    { "title": "Use existing SearXNG instance", "description": "Freeform base URL → HARNESS_WEB_SEARXNG_URL" }
+  ],
+  "allowFreeform": true
+}
+```
+
 ## Example (plan — scope)
 
 ```json

package/.agents/skills/scrapling-web/SKILL.md

@@ -1,22 +1,33 @@
 ---
 name: scrapling-web
 description: |
-  Harness web search and scrape via the local harness-web CLI (Scrapling). Use for any
-  non-API web task: search, scrape URLs, map site links, bulk research fetches.
+  Harness web search and scrape via pi tools web_search and web_fetch (harness-web.py).
+  Use for any non-API web task: search, scrape URLs, map site links, bulk research fetches.
   Replaces Firecrawl in ultimate-pi harness agents. Triggers on: search the web,
-  scrape URL, fetch page, research online, harness-web, .web/ artifacts.
-allowed-tools:
-  - Bash(python3 *harness-web.py *)
-  - Bash(python3 .pi/scripts/harness-web.py *)
-  - Bash(scrapling *)
+  scrape URL, fetch page, research online, web_search, web_fetch, .web/ artifacts.
 ---
 
 # scrapling-web (harness-web)
 
-Local web layer for harness agents — **no API keys**, no Docker compose stack.
-Uses [Scrapling](https://scrapling.readthedocs.io/) under `node $UP_PKG/.pi/scripts/harness-web.py`.
+Local web layer for harness agents — **no API keys** for default search/scrape.
+Pi registers **`web_search`** and **`web_fetch`** (wrap `harness-web.py` with Scrapling bootstrap).
+Optional **self-hosted SearXNG** — see `/harness-setup` Step 4.0b.
 
-## Install (once per machine)
+## Agent tools (preferred)
+
+| Task | Tool |
+|------|------|
+| Search (SERP) | `web_search` with `query` |
+| Search + multi-scrape | `web_search` with `bulk: true` |
+| Scrape URL | `web_fetch` with `url` (default mode `scrape`) |
+| Map same-host links | `web_fetch` with `mode: map` |
+| Static / simple page | `web_fetch` with `fast: true` |
+
+**Never before search/fetch:** resolve `UP_PKG`, `ls harness-web.py`, `python3 -c "import scrapling"`, or Firecrawl/curl/wget/scrapling CLI for SERP or page fetch.
+
+Full JSON/markdown lives under **`.web/`** (gitignored). Use `read` on `output` paths after tool calls.
+
+## Install (once per machine — setup/humans only)
 
 ```bash
 command -v uv &>/dev/null || curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -24,25 +35,23 @@ uv tool install "scrapling[fetchers]"
 scrapling install   # browser binaries for default stealth scrape
 ```
 
-Verify: `bash "$UP_PKG/.pi/scripts/harness-cli-verify.sh"`
-
-## Output directory
+Verify: `bash "$UP_PKG/.pi/scripts/harness-cli-verify.sh"`  
+Config diagnostics: `python3 "$UP_PKG/.pi/scripts/harness-web.py" status` (JSON; setup only)
 
-Write artifacts under **`.web/`** (gitignored), not `.firecrawl/`:
+## Bash fallback (if pi tools unavailable)
 
 | Task | Command |
 |------|---------|
 | Search | `python3 "$UP_PKG/.pi/scripts/harness-web.py" search "query" -o .web/search.json --limit 5` |
-| Scrape URL | `python3 "$UP_PKG/.pi/scripts/harness-web.py" scrape "<url>" -o .web/page.md` |
-| Fast/static scrape | add `--fast` (example.com, raw docs, localhost) |
-| Map same-host links | `python3 "$UP_PKG/.pi/scripts/harness-web.py" map "<url>" -o .web/map.json --limit 50` |
-| Bulk | `python3 "$UP_PKG/.pi/scripts/harness-web.py" bulk-scrape "query" -o .web/bulk/ --limit 3` |
+| Scrape | `python3 "$UP_PKG/.pi/scripts/harness-web.py" scrape "<url>" -o .web/page.md` |
+| Fast/static | add `--fast` |
+| Map | `python3 "$UP_PKG/.pi/scripts/harness-web.py" map "<url>" -o .web/map.json` |
+| Bulk | `python3 "$UP_PKG/.pi/scripts/harness-web.py" bulk-scrape "query" -o .web/bulk/` |
 
 ## Search JSON shape (Firecrawl-compatible)
 
 ```bash
 jq -r '.data.web[].url' .web/search.json
-jq -r '.data.web[] | "\(.title): \(.url)"' .web/search.json
 ```
 
 Each entry: `url`, `title`, `description`.
@@ -51,43 +60,39 @@ Each entry: `url`, `title`, `description`.
 
 | Mode | When |
 |------|------|
-| **stealth** (default scrape) | Arbitrary URLs, JS-heavy sites |
-| **fast** (`--fast` or `HARNESS_WEB_FETCH_MODE=fast`) | Static docs, example.com, localhost |
+| **stealth** (default) | Arbitrary URLs, JS-heavy sites |
+| **fast** (`fast: true` / `--fast`) | Static docs, example.com, localhost |
 | **auto** (`HARNESS_WEB_FETCH_MODE=auto`) | fast for known-static hosts, else stealth |
 
-Search always uses lightweight HTTP to `html.duckduckgo.com/html/`; on 403/challenge, **one** stealth retry then fail clearly.
+| Search backend | Behavior |
+|--------------|----------|
+| `ddg_html` (default) | DuckDuckGo HTML SERP |
+| `searxng` | JSON at `HARNESS_WEB_SEARXNG_URL` — bootstrap via `harness-searxng-bootstrap.mjs` |
 
 ## Environment
 
 | Variable | Default | Purpose |
 |----------|---------|---------|
 | `HARNESS_WEB_FETCH_MODE` | `stealth` | `stealth` \| `fast` \| `auto` |
-| `HARNESS_WEB_SEARCH_ENGINE` | `ddg_html` | SERP backend |
-| `HARNESS_WEB_PROXY` | (unset) | Proxy URL for fetch/search |
-| `HARNESS_WEB_RATE_LIMIT_MS` | `2000` | Delay between bulk scrapes |
-| `HARNESS_WEB_TIMEOUT_MS` | `30000` | Per-request timeout |
+| `HARNESS_WEB_SEARCH_ENGINE` | `ddg_html` | `ddg_html` \| `searxng` |
+| `HARNESS_WEB_SEARXNG_URL` | (unset) | Required when `SEARCH_ENGINE=searxng` |
 
 ## Escalation
 
-1. `harness-web search` (HTTP SERP)
-2. `harness-web scrape` (stealth default)
-3. `harness-web scrape --fast` when the target is known static
-4. `scrapling extract …` only when harness-web flags are insufficient
+1. `web_search` / `web_fetch`
+2. `web_fetch` with `fast: true` for static hosts
+3. `web_fetch` with `mode: map` then targeted fetches
+4. Site-specific Scrapling only when tools are insufficient (not for routine SERP/fetch)
 
-## Gaps vs old Firecrawl
+## Gaps vs Firecrawl
 
 | Firecrawl | Harness path |
 |-----------|----------------|
-| `interact` | No 1:1 — rare flows use gstack browse or Scrapling MCP session |
-| `agent` (structured extract) | Agent reasoning + graphify, or site-specific selectors |
-| `parse` (local PDF) | Dedicated doc tools (pypdf, markitdown) |
-| `crawl` (site-wide) | `map` + `bulk-scrape` or future Spiders integration |
+| `interact` | gstack browse or manual browser |
+| `agent` | Agent reasoning + graphify |
+| `parse` (PDF) | pypdf, markitdown |
+| `crawl` | `web_search` bulk or map + multiple `web_fetch` |
 
 ## Ethics
 
 Respect site terms and rate limits. SERP scraping is for dev research, not high-volume harvesting.
-See [Scrapling ethical considerations](https://scrapling.readthedocs.io/en/latest/cli/extract-commands.html#legal-and-ethical-considerations).
-
-## Drawbacks of default stealth scrape
-
-Higher latency and RAM (Chromium per session). Use `--fast` for static docs; reuse one `bulk-scrape` run (single `StealthySession`) instead of many cold starts.

package/.agents/skills/wiki-autoresearch/SKILL.md

@@ -8,7 +8,7 @@ description: >
   Triggers on: "/wiki-autoresearch", "/autoresearch", "wiki-autoresearch", "autoresearch",
   "research [topic]", "deep dive into [topic]", "investigate [topic]",
   "find everything about [topic]", "research and file", "go research", "build a wiki on".
-allowed-tools: Read Write Edit Glob Grep WebFetch WebSearch Bash
+allowed-tools: Read Write Edit Glob Grep web_search web_fetch Bash
 ---
 
 # wiki-autoresearch: Autonomous Research Loop with Graphify
@@ -129,8 +129,8 @@ Input: topic (from Topic Selection, above)
 
 Round 1. Broad search
 1. Decompose topic into 3-5 distinct search angles
-2. For each angle: run 2-3 WebSearch queries
-3. For top 2-3 results per angle: WebFetch the page
+2. For each angle: run 2-3 `web_search` queries
+3. For top 2-3 results per angle: `web_fetch` each URL (or `read` `.web/` artifacts)
 4. Save each fetched page to ./raw/ as a markdown file
 5. Extract from each: key claims, entities, concepts, open questions
 

package/.pi/SYSTEM.md

@@ -23,26 +23,25 @@ You are an enterprise coding agent. Optimize for correctness, minimal diffs, and
 ## Web Policy (Mandatory)
 
 > [!warning] No raw HTTP
-> Route **all** web fetches through [[context7]] (API/library docs) or **harness-web** / [[scrapling-web]] (all other). No `curl`, `wget`, or raw bash HTTP.
+> Route **all** web through [[context7]] (API/library docs) or **`web_search` / `web_fetch`** ([[scrapling-web]]). No `curl`, `wget`, Firecrawl, or scrapling CLI preflight.
 
 ### API / Library Docs — context7 ONLY
 - `ctx7 library <name> <query>` then `ctx7 docs <id> <query>`
 - context7 owns: function signatures, class APIs, config options, stdlib, framework specs.
-- **Never** use quality-sites for API docs.
+- **Never** use quality-sites or web_fetch for API docs.
 
-### All Non-API Web Fetch — harness-web (Scrapling)
-See `.agents/skills/scrapling-web/SKILL.md` for workflow escalation.
+### All Non-API Web — web_search + web_fetch
+See `.agents/skills/scrapling-web/SKILL.md`. **No preflight:** never resolve `UP_PKG`, `ls harness-web.py`, or `python3 -c "import scrapling"` before searching.
 
-| Task | Command |
-|------|---------|
-| Search (no URL) | `python3 "$UP_PKG/.pi/scripts/harness-web.py" search "query" -o .web/search.json --limit 5` |
-| Scrape (have URL) | `python3 "$UP_PKG/.pi/scripts/harness-web.py" scrape "<url>" -o .web/page.md` |
-| Static / known-simple | add `--fast` to scrape |
-| Map same-host links | `python3 "$UP_PKG/.pi/scripts/harness-web.py" map "<url>" -o .web/map.json` |
-| Bulk search + scrape | `python3 "$UP_PKG/.pi/scripts/harness-web.py" bulk-scrape "query" -o .web/bulk/` |
+| Task | Tool |
+|------|------|
+| Search (SERP) | `web_search` (`query`, optional `limit`, `bulk`) |
+| Scrape page | `web_fetch` (`url`, optional `fast: true`) |
+| Map links | `web_fetch` (`url`, `mode: map`) |
 
-- **Artifacts:** always write under `.web/` with `-o` (token discipline).
-- **Default scrape:** stealth browser; opt out with `--fast` or `HARNESS_WEB_FETCH_MODE=fast`.
+- **Artifacts:** default under `.web/`; use `read` for full JSON/markdown.
+- **Fallback** (tools unavailable): `python3 "$UP_PKG/.pi/scripts/harness-web.py" …` per scrapling-web skill.
+- **Setup diagnostics only:** `harness-web.py status` (JSON config).
 - **Quality sites:** check `.agents/skills/wiki-autoresearch/references/quality-sites.md` before citing non-API sources. Prefer Tier 1 (StackOverflow, GitHub issues, engineering blogs, arxiv). Exclude AI content farms, mirrors, stale packages.
 - **Research:** use `/wiki-autoresearch <topic>` for deep research. Results are graphified into `graphify-out/`.
 

package/.pi/agents/pi-pi/agent-expert.md

@@ -187,9 +187,9 @@ Before answering ANY question, search the local codebase for existing agent defi
 find .pi/agents -name "*.md" -type f 2>/dev/null
 ```
 
-Fetch the latest pi-subagents docs:
-```bash
-firecrawl scrape "https://raw.githubusercontent.com/tintinweb/pi-subagents/refs/heads/master/README.md" -o .firecrawl/pi-subagents-readme.md --only-main-content
+Fetch the latest pi-subagents docs (use `web_fetch` with `fast: true` for raw GitHub):
+```
+web_fetch url="https://raw.githubusercontent.com/tintinweb/pi-subagents/refs/heads/master/README.md" fast=true output=.web/pi-subagents-readme.md
 ```
 
 ## How to Respond

package/.pi/extensions/harness-web-guard.ts

@@ -0,0 +1,95 @@
+/**
+ * harness-web-guard — block bash that bypasses web_search / web_fetch tools.
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+
+const BLOCK_REASON =
+	"harness-web-guard: use web_search (SERP) or web_fetch (page content) instead of raw curl/wget/firecrawl/scrapling fetch. " +
+	"Setup may use harness-web.py status directly.";
+
+const ALLOW_PATTERNS = [
+	/harness-web\.py\b/i,
+	/harness-cli-verify\.sh\b/i,
+	/\bgraphify\b/i,
+	/\bctx7\b/i,
+	/\bcontext7\b/i,
+	/\bgit\b/i,
+	/harness-searxng-bootstrap/i,
+];
+
+const BLOCK_PATTERNS: Array<{ re: RegExp; note: string }> = [
+	{ re: /\bfirecrawl\b/i, note: "firecrawl" },
+	{
+		re: /\b(?:curl|wget)\b[^\n|;&]*\s+https?:\/\//i,
+		note: "curl/wget http(s)",
+	},
+	{
+		re: /\bscrapling\s+(?:fetch|extract)\b/i,
+		note: "scrapling fetch/extract",
+	},
+];
+
+function isBootstrapPrompt(prompt: string): boolean {
+	const p = prompt.toLowerCase();
+	return (
+		p.includes("/harness-setup") ||
+		p.includes("harness-setup") ||
+		p.includes("full harness bootstrap")
+	);
+}
+
+function latestUserPrompt(ctx: {
+	sessionManager: { getEntries(): unknown[] };
+}): string {
+	const entries = ctx.sessionManager.getEntries() as Array<{
+		type?: string;
+		message?: { role?: string; content?: unknown };
+	}>;
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const entry = entries[i];
+		if (entry?.message?.role !== "user") continue;
+		const content = entry.message.content;
+		if (typeof content === "string") return content;
+		if (Array.isArray(content)) {
+			return content
+				.map((part) =>
+					typeof part === "object" && part && "text" in part
+						? String((part as { text?: string }).text ?? "")
+						: "",
+				)
+				.join("\n");
+		}
+	}
+	return "";
+}
+
+function isAllowedBash(command: string): boolean {
+	return ALLOW_PATTERNS.some((re) => re.test(command));
+}
+
+function blockedWebBash(command: string): string | null {
+	if (isAllowedBash(command)) return null;
+	for (const { re, note } of BLOCK_PATTERNS) {
+		if (re.test(command)) return note;
+	}
+	return null;
+}
+
+export default function harnessWebGuard(pi: ExtensionAPI) {
+	pi.on("tool_call", async (event, ctx) => {
+		if (event.toolName !== "bash") return undefined;
+
+		const prompt = latestUserPrompt(ctx);
+		if (isBootstrapPrompt(prompt)) return undefined;
+
+		const command = String((event.input as { command?: string }).command ?? "");
+		const hit = blockedWebBash(command);
+		if (!hit) return undefined;
+
+		return {
+			block: true,
+			reason: `${BLOCK_REASON} (matched: ${hit})`,
+		};
+	});
+}

package/.pi/extensions/harness-web-tools.ts

@@ -0,0 +1,209 @@
+/**
+ * harness-web-tools — web_search + web_fetch pi tools wrapping harness-web.py.
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { Type } from "@sinclair/typebox";
+import {
+	harnessWebContextLine,
+	readTextExcerpt,
+	runHarnessWeb,
+	summarizeSearchJson,
+} from "./lib/harness-web/run-cli.js";
+
+// @ts-expect-error pi extensions run as ESM
+const MODULE_URL = import.meta.url;
+
+const WEB_SEARCH_GUIDELINES = [
+	"Use web_search for open-web SERP — never preflight UP_PKG, ls harness-web.py, or python3 -c import scrapling.",
+	"Never use Firecrawl, curl/wget for search, or scrapling CLI for SERP.",
+	"After search, use web_fetch on URLs or read the output JSON under .web/.",
+	"Use bulk:true only when you need search plus multi-page scrape in one step.",
+];
+
+const WEB_FETCH_GUIDELINES = [
+	"Use web_fetch for page markdown or same-host link maps — never curl/wget the URL.",
+	"Never use raw scrapling CLI for fetch; harness-web handles Scrapling bootstrap.",
+	"Library API documentation → context7 only, not web_fetch.",
+	"Set fast:true for static docs (example.com, raw HTML docs, localhost).",
+];
+
+const WebSearchSchema = Type.Object({
+	query: Type.String({ description: "Search query" }),
+	limit: Type.Optional(
+		Type.Number({
+			description: "Max results (default 5)",
+			minimum: 1,
+			maximum: 20,
+		}),
+	),
+	output: Type.Optional(
+		Type.String({
+			description:
+				"Output path (default .web/search.json or .web/bulk for bulk)",
+		}),
+	),
+	bulk: Type.Optional(
+		Type.Boolean({
+			description:
+				"If true, run bulk-scrape (search then scrape top URLs to output directory)",
+			default: false,
+		}),
+	),
+});
+
+const WebFetchSchema = Type.Object({
+	url: Type.String({ description: "URL to fetch" }),
+	mode: Type.Optional(
+		Type.Union([Type.Literal("scrape"), Type.Literal("map")], {
+			description: "scrape (markdown) or map (same-host links JSON)",
+			default: "scrape",
+		}),
+	),
+	output: Type.Optional(
+		Type.String({ description: "Output file path under .web/" }),
+	),
+	fast: Type.Optional(
+		Type.Boolean({
+			description: "Use fast HTTP scrape (static/simple pages)",
+			default: false,
+		}),
+	),
+	limit: Type.Optional(
+		Type.Number({
+			description: "For map mode: max links (default 100)",
+			minimum: 1,
+			maximum: 500,
+		}),
+	),
+});
+
+function failResult(text: string) {
+	return {
+		content: [{ type: "text" as const, text }],
+		details: { ok: false },
+	};
+}
+
+function okResult(text: string, details: Record<string, unknown> = {}) {
+	return {
+		content: [{ type: "text" as const, text }],
+		details: { ok: true, ...details },
+	};
+}
+
+function sessionCwd(ctx: { cwd?: string }): string {
+	return ctx.cwd ?? process.cwd();
+}
+
+export default function harnessWebTools(pi: ExtensionAPI) {
+	pi.on("before_agent_start", async (event) => {
+		return {
+			systemPrompt: `${event.systemPrompt}\n\n${harnessWebContextLine()}`,
+		};
+	});
+
+	pi.registerTool({
+		name: "web_search",
+		label: "Web Search",
+		description:
+			"Search the web via harness-web (DuckDuckGo HTML or self-hosted SearXNG from .env). Returns result summaries and output path.",
+		promptSnippet: "SERP via configured engine (ddg_html or searxng from .env)",
+		promptGuidelines: WEB_SEARCH_GUIDELINES,
+		parameters: WebSearchSchema,
+
+		async execute(_id, params, _signal, _onUpdate, ctx) {
+			const cwd = sessionCwd(ctx);
+			const query = String(params.query ?? "").trim();
+			if (!query) return failResult("web_search: query is required.");
+
+			const limit = typeof params.limit === "number" ? params.limit : 5;
+			const bulk = params.bulk === true;
+			const output = String(
+				params.output ?? (bulk ? ".web/bulk" : ".web/search.json"),
+			);
+
+			const argv = bulk
+				? ["bulk-scrape", query, "-o", output, "--limit", String(limit)]
+				: ["search", query, "-o", output, "--limit", String(limit)];
+
+			const run = runHarnessWeb(MODULE_URL, argv, cwd);
+			if (!run.ok) {
+				const hint =
+					"\n\nHints: run /harness-setup; for searxng set HARNESS_WEB_SEARXNG_URL; " +
+					"enable json in SearXNG search.formats.";
+				return failResult(
+					`web_search failed (exit ${run.exitCode}).\n${run.stderr || run.stdout}${hint}`,
+				);
+			}
+
+			const parts = [run.stdout];
+			if (!bulk) {
+				const summary = summarizeSearchJson(output, cwd);
+				if (summary) {
+					parts.push("", summary);
+				}
+			}
+			parts.push("", `output: ${output}`);
+			parts.push("Use read tool for full JSON, or web_fetch on result URLs.");
+
+			return okResult(parts.join("\n"), {
+				output,
+				query,
+				bulk,
+				engine: process.env.HARNESS_WEB_SEARCH_ENGINE,
+			});
+		},
+	});
+
+	pi.registerTool({
+		name: "web_fetch",
+		label: "Web Fetch",
+		description:
+			"Fetch a URL via harness-web/Scrapling (scrape to markdown or map same-host links).",
+		promptSnippet: "Scrape/map URL via Scrapling (harness-web)",
+		promptGuidelines: WEB_FETCH_GUIDELINES,
+		parameters: WebFetchSchema,
+
+		async execute(_id, params, _signal, _onUpdate, ctx) {
+			const cwd = sessionCwd(ctx);
+			const url = String(params.url ?? "").trim();
+			if (!url) return failResult("web_fetch: url is required.");
+
+			const mode = params.mode === "map" ? "map" : "scrape";
+			const fast = params.fast === true;
+			const limit = typeof params.limit === "number" ? params.limit : 100;
+			const defaultOut = mode === "map" ? ".web/map.json" : ".web/page.md";
+			const output = String(params.output ?? defaultOut);
+
+			const argv =
+				mode === "map"
+					? [
+							"map",
+							url,
+							"-o",
+							output,
+							"--limit",
+							String(limit),
+							...(fast ? ["--fast"] : []),
+						]
+					: ["scrape", url, "-o", output, ...(fast ? ["--fast"] : [])];
+
+			const run = runHarnessWeb(MODULE_URL, argv, cwd);
+			if (!run.ok) {
+				return failResult(
+					`web_fetch failed (exit ${run.exitCode}).\n${run.stderr || run.stdout}\n` +
+						"Try fast:true for static pages, or run harness-cli-verify for Scrapling install.",
+				);
+			}
+
+			const parts = [run.stdout, "", `output: ${output}`];
+			const excerpt = readTextExcerpt(output, cwd);
+			if (excerpt) {
+				parts.push("", "--- excerpt ---", excerpt);
+			}
+
+			return okResult(parts.join("\n"), { output, url, mode });
+		},
+	});
+}

package/.pi/extensions/lib/harness-web/run-cli.ts

@@ -0,0 +1,92 @@
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { resolveHarnessScript } from "../harness-paths.js";
+
+export interface RunHarnessWebResult {
+	ok: boolean;
+	exitCode: number;
+	stdout: string;
+	stderr: string;
+}
+
+export function runHarnessWeb(
+	moduleUrl: string,
+	args: string[],
+	cwd: string,
+): RunHarnessWebResult {
+	const script = resolveHarnessScript(moduleUrl, "harness-web.py");
+	const result = spawnSync("python3", [script, ...args], {
+		cwd,
+		env: process.env,
+		encoding: "utf-8",
+		maxBuffer: 16 * 1024 * 1024,
+	});
+	return {
+		ok: result.status === 0,
+		exitCode: result.status ?? 1,
+		stdout: (result.stdout ?? "").trim(),
+		stderr: (result.stderr ?? "").trim(),
+	};
+}
+
+export function readTextExcerpt(
+	filePath: string,
+	cwd: string,
+	maxChars = 2000,
+): string {
+	const full = resolve(cwd, filePath);
+	if (!existsSync(full)) return "";
+	const text = readFileSync(full, "utf-8");
+	if (text.length <= maxChars) return text;
+	return `${text.slice(0, maxChars)}\n… (truncated; use read tool for full file)`;
+}
+
+export interface SearchHit {
+	url: string;
+	title: string;
+	description: string;
+}
+
+export function summarizeSearchJson(filePath: string, cwd: string): string {
+	const full = resolve(cwd, filePath);
+	if (!existsSync(full)) return "";
+	try {
+		const data = JSON.parse(readFileSync(full, "utf-8")) as {
+			query?: string;
+			engine?: string;
+			data?: { web?: SearchHit[] };
+		};
+		const hits = data.data?.web ?? [];
+		const lines = [
+			`engine: ${data.engine ?? "unknown"}`,
+			`query: ${data.query ?? ""}`,
+			`results: ${hits.length}`,
+			"",
+		];
+		for (const [i, hit] of hits.entries()) {
+			lines.push(`${i + 1}. ${hit.title || "(no title)"}`);
+			lines.push(`   ${hit.url}`);
+			if (hit.description) {
+				const snip =
+					hit.description.length > 120
+						? `${hit.description.slice(0, 120)}…`
+						: hit.description;
+				lines.push(`   ${snip}`);
+			}
+		}
+		return lines.join("\n");
+	} catch {
+		return "";
+	}
+}
+
+export function harnessWebContextLine(): string {
+	const engine = process.env.HARNESS_WEB_SEARCH_ENGINE?.trim() || "ddg_html";
+	const searx = process.env.HARNESS_WEB_SEARXNG_URL?.trim();
+	const searxPart = searx ? ` searxng_url=${searx}` : "";
+	return (
+		`[HarnessWeb] search_engine=${engine}${searxPart} — use web_search / web_fetch tools; ` +
+		"never resolve UP_PKG, ls harness-web.py, or python3 -c import scrapling before searching."
+	);
+}

package/.pi/harness/env.harness.template

@@ -4,9 +4,11 @@
 # Telemetry (set false to disable harness PostHog events)
 HARNESS_TELEMETRY_ENABLED=true
 
-# harness-web (Scrapling) — local fetch/search; no API key
+# harness-web (Scrapling scrape + pluggable search)
 HARNESS_WEB_FETCH_MODE=stealth
 HARNESS_WEB_SEARCH_ENGINE=ddg_html
+# SearXNG (when HARNESS_WEB_SEARCH_ENGINE=searxng):
+# HARNESS_WEB_SEARXNG_URL=http://127.0.0.1:8080
 # HARNESS_WEB_PROXY=
 # HARNESS_WEB_RATE_LIMIT_MS=2000
 # HARNESS_WEB_TIMEOUT_MS=30000