ultimate-pi 0.4.0 → 0.5.0

package/.pi/extensions/lib/harness-web/run-cli.ts

@@ -0,0 +1,92 @@
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { resolveHarnessScript } from "../harness-paths.js";
+
+export interface RunHarnessWebResult {
+	ok: boolean;
+	exitCode: number;
+	stdout: string;
+	stderr: string;
+}
+
+export function runHarnessWeb(
+	moduleUrl: string,
+	args: string[],
+	cwd: string,
+): RunHarnessWebResult {
+	const script = resolveHarnessScript(moduleUrl, "harness-web.py");
+	const result = spawnSync("python3", [script, ...args], {
+		cwd,
+		env: process.env,
+		encoding: "utf-8",
+		maxBuffer: 16 * 1024 * 1024,
+	});
+	return {
+		ok: result.status === 0,
+		exitCode: result.status ?? 1,
+		stdout: (result.stdout ?? "").trim(),
+		stderr: (result.stderr ?? "").trim(),
+	};
+}
+
+export function readTextExcerpt(
+	filePath: string,
+	cwd: string,
+	maxChars = 2000,
+): string {
+	const full = resolve(cwd, filePath);
+	if (!existsSync(full)) return "";
+	const text = readFileSync(full, "utf-8");
+	if (text.length <= maxChars) return text;
+	return `${text.slice(0, maxChars)}\n… (truncated; use read tool for full file)`;
+}
+
+export interface SearchHit {
+	url: string;
+	title: string;
+	description: string;
+}
+
+export function summarizeSearchJson(filePath: string, cwd: string): string {
+	const full = resolve(cwd, filePath);
+	if (!existsSync(full)) return "";
+	try {
+		const data = JSON.parse(readFileSync(full, "utf-8")) as {
+			query?: string;
+			engine?: string;
+			data?: { web?: SearchHit[] };
+		};
+		const hits = data.data?.web ?? [];
+		const lines = [
+			`engine: ${data.engine ?? "unknown"}`,
+			`query: ${data.query ?? ""}`,
+			`results: ${hits.length}`,
+			"",
+		];
+		for (const [i, hit] of hits.entries()) {
+			lines.push(`${i + 1}. ${hit.title || "(no title)"}`);
+			lines.push(`   ${hit.url}`);
+			if (hit.description) {
+				const snip =
+					hit.description.length > 120
+						? `${hit.description.slice(0, 120)}…`
+						: hit.description;
+				lines.push(`   ${snip}`);
+			}
+		}
+		return lines.join("\n");
+	} catch {
+		return "";
+	}
+}
+
+export function harnessWebContextLine(): string {
+	const engine = process.env.HARNESS_WEB_SEARCH_ENGINE?.trim() || "ddg_html";
+	const searx = process.env.HARNESS_WEB_SEARXNG_URL?.trim();
+	const searxPart = searx ? ` searxng_url=${searx}` : "";
+	return (
+		`[HarnessWeb] search_engine=${engine}${searxPart} — use web_search / web_fetch tools; ` +
+		"never resolve UP_PKG, ls harness-web.py, or python3 -c import scrapling before searching."
+	);
+}

package/.pi/extensions/ultimate-pi-vcc.ts

@@ -0,0 +1,17 @@
+/**
+ * In-house VCC compaction for ultimate-pi.
+ *
+ * Vendored compaction core from [pi-vcc](https://github.com/sting8k/pi-vcc),
+ * inspired by [VCC](https://github.com/lllyasviel/VCC).
+ *
+ * Configuration is **env-only** (no JSON config files):
+ * - `HARNESS_VCC_COMPACTION` — default on; set `false` for Pi LLM compaction on /compact + auto-compact
+ * - `HARNESS_VCC_DEBUG` — set `true` to write `/tmp/pi-vcc-debug.json` on compaction
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import registerVcc from "../../vendor/pi-vcc/index.js";
+
+export default function ultimatePiVcc(pi: ExtensionAPI): void {
+	registerVcc(pi);
+}

package/.pi/harness/docs/adrs/0030-inhouse-vcc-compaction.md

@@ -0,0 +1,40 @@
+# ADR 0030: In-house VCC compaction (vendored pi-vcc)
+
+- **Status:** Accepted
+- **Date:** 2026-05-17
+- **Deciders:** ultimate-pi harness team
+
+## Context
+
+ultimate-pi depended on the npm package `@sting8k/pi-vcc` for deterministic, view-oriented session compaction (no LLM summarization call). We need that behavior by default for harness sessions, without an external package coupling, while preserving attribution to upstream [pi-vcc](https://github.com/sting8k/pi-vcc) and the conceptual [VCC](https://github.com/lllyasviel/VCC) work.
+
+## Decision
+
+1. Vendor [sting8k/pi-vcc](https://github.com/sting8k/pi-vcc) under `vendor/pi-vcc/` (refresh via `npm run vendor:sync-vcc`), following the same pattern as `vendor/pi-model-router`.
+2. Load compaction through [`.pi/extensions/ultimate-pi-vcc.ts`](../../../extensions/ultimate-pi-vcc.ts).
+3. Remove `@sting8k/pi-vcc` from `package.json` dependencies and from `.pi/settings*.json` `packages` arrays.
+4. **Configuration is env-only** — no JSON config files (`PI_VCC_CONFIG_PATH` and `.pi/pi-vcc-config.json` are not used).
+5. **Default:** `HARNESS_VCC_COMPACTION` unset → VCC overrides Pi built-in LLM compaction for `/compact`, auto-threshold, and overflow.
+6. **Opt-out:** `HARNESS_VCC_COMPACTION=false` (also `0` / `off`) uses Pi’s LLM compaction for those paths; explicit `/pi-vcc` still uses VCC.
+7. **Debug:** `HARNESS_VCC_DEBUG=true` writes `/tmp/pi-vcc-debug.json` on compaction (default off).
+8. Settings implementation: [`.pi/extensions/lib/harness-vcc-settings.ts`](../../../extensions/lib/harness-vcc-settings.ts).
+9. Compaction telemetry `details.compactor` is `ultimate-pi-vcc`.
+
+## Consequences
+
+### Positive
+
+- No runtime dependency on `@sting8k/pi-vcc` npm; vendored tree is pinned and patchable.
+- Harness-default compaction matches ADR intent (deterministic, recall-friendly summaries).
+- Operators can revert to LLM compaction per project via one env var.
+
+### Negative / trade-offs
+
+- Maintainer must run `vendor:sync-vcc` to pick up upstream pi-vcc fixes.
+- Vendored `loadSettings` re-exports harness env settings from `.pi/extensions/lib/` (couples vendor tree to ultimate-pi layout).
+
+## References
+
+- [THIRD_PARTY_NOTICES.md](../../../../THIRD_PARTY_NOTICES.md)
+- [vendor/pi-vcc/UPSTREAM_PIN.md](../../../../vendor/pi-vcc/UPSTREAM_PIN.md)
+- [`.env.example`](../../../../.env.example) — `HARNESS_VCC_COMPACTION`, `HARNESS_VCC_DEBUG`

package/.pi/harness/docs/adrs/README.md

@@ -15,6 +15,7 @@ Team-shared ADRs for the ultimate-pi harness live under `.pi/harness/docs/adrs/`
 | [0007](0007-interactive-drift-monitor.md) | Interactive drift monitor | Accepted |
 | [0008](0008-harness-posthog-telemetry.md) | Harness PostHog telemetry | Accepted |
 | [0009](0009-sentrux-rules-lifecycle.md) | Sentrux rules.toml lifecycle | Accepted |
+| [0030](0030-inhouse-vcc-compaction.md) | In-house VCC compaction (vendored pi-vcc) | Accepted |
 
 ## Template
 

package/.pi/harness/env.harness.template

@@ -4,9 +4,11 @@
 # Telemetry (set false to disable harness PostHog events)
 HARNESS_TELEMETRY_ENABLED=true
 
-# harness-web (Scrapling) — local fetch/search; no API key
+# harness-web (Scrapling scrape + pluggable search)
 HARNESS_WEB_FETCH_MODE=stealth
 HARNESS_WEB_SEARCH_ENGINE=ddg_html
+# SearXNG (when HARNESS_WEB_SEARCH_ENGINE=searxng):
+# HARNESS_WEB_SEARXNG_URL=http://127.0.0.1:8080
 # HARNESS_WEB_PROXY=
 # HARNESS_WEB_RATE_LIMIT_MS=2000
 # HARNESS_WEB_TIMEOUT_MS=30000

package/.pi/prompts/harness-setup.md

@@ -134,10 +134,13 @@ export PATH="$HOME/.local/bin:$PATH"
 uv tool install "scrapling[fetchers]"
 scrapling install   # Chromium for default stealth scrape; may need sudo for OS libs on Linux
 mkdir -p .web
+python3 "$UP_PKG/.pi/scripts/harness-web.py" status   # JSON config (setup/diagnostics only)
 python3 "$UP_PKG/.pi/scripts/harness-web.py" search "ultimate-pi harness" -o .web/smoke-search.json --limit 3
 python3 "$UP_PKG/.pi/scripts/harness-web.py" scrape "https://example.com" -o .web/smoke-page.md --fast
 ```
 
+After pi loads extensions, agents should smoke **`web_search`** once (not `UP_PKG` / `import scrapling` preflight). Example intent: query `ultimate-pi harness`, `limit` 2.
+
 - **`--skip-tools`:** skip Step 2 (includes Scrapling verify).
 - On Linux/WSL, if stealth scrape fails, install browser libs from `harness-cli-verify.sh` output or use `--fast` for static targets.
 
@@ -343,7 +346,7 @@ Verify each package:
 | `@posthog/pi` | Analytics event capture | F0 |
 | `pi-lean-ctx` | Context runtime (read/bash/find/grep/MCP bridge) | F0 |
 | `harness-subagents` (bundled extension) | L4 sub-agent spawn, blackboard, package agents | P16 |
-| `@sting8k/pi-vcc` | VCC compaction / conversation memory | Shipped |
+| Vendored `pi-vcc` (`vendor/pi-vcc`, `.pi/extensions/ultimate-pi-vcc.ts`) | VCC compaction / `vcc_recall` — env-only: `HARNESS_VCC_COMPACTION` (default on), `HARNESS_VCC_DEBUG` | Shipped |
 | `pi-model-router` | Vendored (`vendor/`); activates after `.pi/model-router.json` exists | F0 |
 
 ## Step 3.5 — Model Router Configuration (Dynamic)
@@ -421,6 +424,47 @@ If **no** `.env` at project root:
 - On **skip** or `--non-interactive`: warn in report (non-interactive skips creation)
 - If `ask_user` cancelled: stop with `needs_clarification`
 
+### 4.0b — harness-web search engine (non-destructive)
+
+Unless `--non-interactive`, **call `ask_user`** after Step 4.0 (harness-decisions skill):
+
+```json
+{
+  "question": "Which harness-web search backend should this project use?",
+  "context": "Scrapling still handles scrape/map/bulk. Search only: DuckDuckGo HTML needs no extra services. SearXNG must be self-hosted for agents — public instances often block JSON (403) and default to ~4 API requests/hour per IP.",
+  "options": [
+    {
+      "title": "DuckDuckGo HTML (default)",
+      "description": "HARNESS_WEB_SEARCH_ENGINE=ddg_html — no Docker"
+    },
+    {
+      "title": "Self-host SearXNG here (Docker)",
+      "description": "Bootstrap .searxng/ with official compose, enable JSON API, set harness env"
+    },
+    {
+      "title": "Use existing SearXNG instance",
+      "description": "You provide base URL; harness writes HARNESS_WEB_SEARXNG_URL"
+    }
+  ],
+  "allowFreeform": true
+}
+```
+
+| User choice | Actions |
+|-------------|---------|
+| **DDG** | Ensure `.env` has `HARNESS_WEB_SEARCH_ENGINE=ddg_html` via `harness-sync-env.mjs` (append only if missing; do not overwrite user values) |
+| **Self-host** | `node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs"` (requires Docker). Script sets `HARNESS_WEB_SEARCH_ENGINE=searxng` and `HARNESS_WEB_SEARXNG_URL` |
+| **Existing instance** | Parse base URL from freeform answer. Run `node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs" --set-url {url}` (health check + upsert `.env`) |
+| **Cancelled** | Stop with `needs_clarification` |
+| **`--non-interactive`** | Skip prompt; leave/default `ddg_html`; do not run Docker bootstrap |
+
+Post-choice smoke (report pass/fail):
+
+```bash
+mkdir -p .web
+python3 "$UP_PKG/.pi/scripts/harness-web.py" search "ultimate-pi harness" -o .web/setup-search.json --limit 2
+```
+
 Rules:
 
 - **Do not** `cp` over an existing `.env`.
@@ -428,7 +472,7 @@ Rules:
 - Re-runs only add keys from `$UP_PKG/.pi/harness/env.harness.template` that are absent (managed block at EOF).
 - Ensure `.env` is gitignored (Step 4.1).
 
-Template keys (placeholders — user fills secrets): `HARNESS_TELEMETRY_ENABLED`, `HARNESS_WEB_*`, `PI_VCC_CONFIG_PATH`, plus commented optional PostHog / Graphify vars.
+Template keys (placeholders — user fills secrets): `HARNESS_TELEMETRY_ENABLED`, `HARNESS_WEB_*`, `HARNESS_VCC_COMPACTION`, `HARNESS_VCC_DEBUG`, plus commented optional PostHog / Graphify vars.
 
 ### 4.1 — .gitignore Entries
 
@@ -436,6 +480,7 @@ Ensure `.gitignore` contains:
 ```
 .env
 .web/
+.searxng/
 .raw/
 .vault-meta/
 .pi/harness/critics/
@@ -646,6 +691,7 @@ Output summary table:
 | .gitignore | ✓/✗ | entries added (incl. `.env`) |
 | ./raw directory | ✓/✗ | Created for graphify source ingestion |
 | harness-web (Scrapling) | ✓/✗ | search + scrape smoke |
+| harness-web search engine | ddg / searxng / — | Step 4.0b choice; SearXNG URL if applicable |
 
 Next steps:
 1. If tools missing: re-run with `--force` or install individually

package/.pi/scripts/harness-cli-verify.sh

@@ -200,10 +200,19 @@ verify_scrapling() {
 		return
 	fi
 	mkdir -p .web
-	if python3 "$_hw" search "ultimate-pi harness" -o .web/verify-search.json --limit 2 2>/dev/null | grep -q wrote; then
-		pass "harness-web search smoke"
+	_search_engine="${HARNESS_WEB_SEARCH_ENGINE:-ddg_html}"
+	if [ "$_search_engine" = "searxng" ]; then
+		if [ -z "${HARNESS_WEB_SEARXNG_URL:-}" ]; then
+			fail "HARNESS_WEB_SEARCH_ENGINE=searxng but HARNESS_WEB_SEARXNG_URL is unset"
+		elif python3 "$_hw" search "ultimate-pi harness" -o .web/verify-search.json --limit 2 2>/dev/null | grep -q wrote; then
+			pass "harness-web search smoke (searxng)"
+		else
+			fail "harness-web search smoke failed (searxng at ${HARNESS_WEB_SEARXNG_URL})"
+		fi
+	elif python3 "$_hw" search "ultimate-pi harness" -o .web/verify-search.json --limit 2 2>/dev/null | grep -q wrote; then
+		pass "harness-web search smoke (ddg_html)"
 	else
-		fail "harness-web search smoke failed"
+		fail "harness-web search smoke failed (ddg_html)"
 	fi
 	if python3 "$_hw" scrape "https://example.com" -o .web/verify-page.md --fast 2>/dev/null | grep -q wrote; then
 		pass "harness-web scrape --fast smoke"

package/.pi/scripts/harness-searxng-bootstrap.mjs

@@ -0,0 +1,270 @@
+#!/usr/bin/env node
+/**
+ * Bootstrap a project-local SearXNG instance for harness-web (Docker Compose).
+ *
+ * - Creates .searxng/ with official upstream compose template
+ * - Writes core-config/settings.yml with json format + limiter off (local dev)
+ * - Starts containers and waits for JSON search health
+ * - Upserts HARNESS_WEB_SEARCH_ENGINE / HARNESS_WEB_SEARXNG_URL in project .env
+ *
+ * Usage:
+ *   node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs" [PROJECT_ROOT] [--url-only]
+ *   node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs" --set-url http://127.0.0.1:8080
+ *
+ * Requires: docker, docker compose, curl
+ */
+
+import {
+	access,
+	copyFile,
+	mkdir,
+	readFile,
+	writeFile,
+} from "node:fs/promises";
+import { constants } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn } from "node:child_process";
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const UP_PKG = join(SCRIPT_DIR, "..", "..");
+
+const SEARXNG_BASE =
+	"https://raw.githubusercontent.com/searxng/searxng/master/container";
+const DEFAULT_PORT = "8080";
+const HEALTH_PATH = "/search?q=harness&format=json";
+
+const MANAGED_START = "# --- harness:env:start ---";
+const MANAGED_END = "# --- harness:env:end ---";
+
+const args = process.argv.slice(2).filter((a) => !a.startsWith("-"));
+const flags = new Set(process.argv.slice(2).filter((a) => a.startsWith("-")));
+const urlOnly = flags.has("--url-only");
+const setUrlIdx = process.argv.indexOf("--set-url");
+const setUrl = setUrlIdx !== -1 ? process.argv[setUrlIdx + 1] : null;
+
+const PROJECT_ROOT = args[0] || process.cwd();
+const SEARXNG_DIR = join(PROJECT_ROOT, ".searxng");
+const CORE_CONFIG = join(SEARXNG_DIR, "core-config");
+const SETTINGS_PATH = join(CORE_CONFIG, "settings.yml");
+const COMPOSE_PATH = join(SEARXNG_DIR, "docker-compose.yml");
+const ENV_COMPOSE = join(SEARXNG_DIR, ".env");
+
+const HARNESS_SETTINGS = `use_default_settings: true
+
+search:
+  formats:
+    - html
+    - json
+
+server:
+  limiter: false
+  public_instance: false
+`;
+
+async function exists(path) {
+	try {
+		await access(path, constants.F_OK);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+function run(cmd, cmdArgs, opts = {}) {
+	return new Promise((resolve, reject) => {
+		const child = spawn(cmd, cmdArgs, {
+			stdio: opts.inherit ? "inherit" : "pipe",
+			cwd: opts.cwd,
+			env: { ...process.env, ...opts.env },
+		});
+		let stdout = "";
+		let stderr = "";
+		if (!opts.inherit) {
+			child.stdout?.on("data", (d) => {
+				stdout += d;
+			});
+			child.stderr?.on("data", (d) => {
+				stderr += d;
+			});
+		}
+		child.on("error", reject);
+		child.on("close", (code) => {
+			if (code === 0) resolve({ stdout, stderr });
+			else
+				reject(
+					new Error(
+						`${cmd} ${cmdArgs.join(" ")} exited ${code}\n${stderr || stdout}`,
+					),
+				);
+		});
+	});
+}
+
+async function requireDocker() {
+	for (const bin of ["docker"]) {
+		try {
+			await run(bin, ["--version"]);
+		} catch {
+			console.error(`✗ ${bin} not found`);
+			console.error(
+				"Install Docker: https://docs.searxng.org/admin/installation-docker.html",
+			);
+			process.exit(1);
+		}
+	}
+	try {
+		await run("docker", ["compose", "version"]);
+	} catch {
+		console.error("✗ docker compose not available");
+		console.error(
+			"Install Docker Compose v2: https://docs.docker.com/compose/install/",
+		);
+		process.exit(1);
+	}
+}
+
+async function curlToFile(url, dest) {
+	await run("curl", ["-fsSL", "-o", dest, url]);
+}
+
+async function readComposePort() {
+	if (!(await exists(ENV_COMPOSE))) return DEFAULT_PORT;
+	const text = await readFile(ENV_COMPOSE, "utf8");
+	for (const line of text.split("\n")) {
+		const m = line.match(/^SEARXNG_PORT=(.+)$/);
+		if (m) return m[1].trim().replace(/^["']|["']$/g, "") || DEFAULT_PORT;
+	}
+	return DEFAULT_PORT;
+}
+
+async function ensureSearxngLayout() {
+	await mkdir(CORE_CONFIG, { recursive: true });
+	if (!(await exists(COMPOSE_PATH))) {
+		console.log("Fetching SearXNG docker-compose.yml …");
+		await curlToFile(`${SEARXNG_BASE}/docker-compose.yml`, COMPOSE_PATH);
+	}
+	if (!(await exists(ENV_COMPOSE))) {
+		const example = join(SEARXNG_DIR, ".env.example");
+		if (!(await exists(example))) {
+			console.log("Fetching SearXNG .env.example …");
+			await curlToFile(`${SEARXNG_BASE}/.env.example`, example);
+		}
+		await copyFile(example, ENV_COMPOSE);
+	}
+	const needsSettings =
+		!(await exists(SETTINGS_PATH)) ||
+		!(await readFile(SETTINGS_PATH, "utf8")).includes("json");
+	if (needsSettings) {
+		await writeFile(SETTINGS_PATH, HARNESS_SETTINGS, "utf8");
+		console.log(`✓ Wrote ${SETTINGS_PATH} (json format, limiter off)`);
+	}
+}
+
+async function composeUp() {
+	console.log("Starting SearXNG (docker compose up -d) …");
+	await run("docker", ["compose", "up", "-d"], { cwd: SEARXNG_DIR, inherit: true });
+}
+
+async function waitForHealth(baseUrl) {
+	const url = `${baseUrl}${HEALTH_PATH}`;
+	const deadline = Date.now() + 90_000;
+	let lastErr = "";
+	while (Date.now() < deadline) {
+		try {
+			const res = await fetch(url, {
+				headers: { Accept: "application/json" },
+				signal: AbortSignal.timeout(10_000),
+			});
+			if (res.status === 403) {
+				throw new Error(
+					"SearXNG returned 403 for format=json — ensure search.formats includes json in .searxng/core-config/settings.yml",
+				);
+			}
+			if (res.ok) {
+				const data = await res.json();
+				if (data && typeof data === "object") {
+					console.log(`✓ SearXNG healthy at ${baseUrl}`);
+					return;
+				}
+			}
+			lastErr = `HTTP ${res.status}`;
+		} catch (err) {
+			lastErr = err instanceof Error ? err.message : String(err);
+		}
+		await new Promise((r) => setTimeout(r, 3000));
+	}
+	throw new Error(`SearXNG health check timed out (${url}): ${lastErr}`);
+}
+
+function upsertEnvKey(content, key, value) {
+	const line = `${key}=${value}`;
+	const re = new RegExp(`^${key}=.*$`, "m");
+	if (re.test(content)) {
+		return content.replace(re, line);
+	}
+	if (content.includes(MANAGED_START) && content.includes(MANAGED_END)) {
+		const end = content.indexOf(MANAGED_END);
+		return `${content.slice(0, end)}${line}\n${content.slice(end)}`;
+	}
+	const sep = content.endsWith("\n") || content.length === 0 ? "" : "\n";
+	return `${content}${sep}${MANAGED_START}\n# harness-web (SearXNG)\n${line}\n${MANAGED_END}\n`;
+}
+
+async function upsertHarnessEnv(baseUrl) {
+	const envPath = join(PROJECT_ROOT, ".env");
+	let content = "";
+	if (await exists(envPath)) {
+		content = await readFile(envPath, "utf8");
+	} else {
+		const template = join(UP_PKG, ".pi", "harness", "env.harness.template");
+		if (await exists(template)) {
+			content = await readFile(template, "utf8");
+		}
+	}
+	content = upsertEnvKey(content, "HARNESS_WEB_SEARCH_ENGINE", "searxng");
+	content = upsertEnvKey(content, "HARNESS_WEB_SEARXNG_URL", baseUrl);
+	await writeFile(envPath, content.endsWith("\n") ? content : `${content}\n`, "utf8");
+	console.log(`✓ Updated .env: HARNESS_WEB_SEARCH_ENGINE=searxng, HARNESS_WEB_SEARXNG_URL=${baseUrl}`);
+}
+
+function normalizeBaseUrl(raw) {
+	const url = raw.trim().replace(/\/+$/, "");
+	if (!/^https?:\/\//i.test(url)) {
+		throw new Error(`Invalid SearXNG URL: ${raw}`);
+	}
+	return url;
+}
+
+async function main() {
+	if (setUrl) {
+		const baseUrl = normalizeBaseUrl(setUrl);
+		await waitForHealth(baseUrl);
+		await upsertHarnessEnv(baseUrl);
+		process.exit(0);
+	}
+
+	if (urlOnly) {
+		const port = (await exists(ENV_COMPOSE)) ? await readComposePort() : DEFAULT_PORT;
+		console.log(`http://127.0.0.1:${port}`);
+		process.exit(0);
+	}
+
+	await requireDocker();
+	await ensureSearxngLayout();
+	const port = await readComposePort();
+	const baseUrl = `http://127.0.0.1:${port}`;
+	await composeUp();
+	await waitForHealth(baseUrl);
+	await upsertHarnessEnv(baseUrl);
+
+	console.log("");
+	console.log("SearXNG is ready for harness-web:");
+	console.log(`  HARNESS_WEB_SEARXNG_URL=${baseUrl}`);
+	console.log(`  Test: python3 "${join(UP_PKG, ".pi/scripts/harness-web.py")}" search "test" -o .web/search.json --limit 2`);
+}
+
+main().catch((err) => {
+	console.error(`✗ ${err.message || err}`);
+	process.exit(1);
+});

package/.pi/scripts/harness-web-search.md

@@ -1,12 +1,21 @@
 # harness-web search (internal)
 
-## Engine
+Routing: `harness_web/search.py` dispatches by `HARNESS_WEB_SEARCH_ENGINE`.
 
-Default: DuckDuckGo static HTML — `GET https://html.duckduckgo.com/html/?q=…`
+## Engines
 
-Implemented in `harness_web/search_ddg.py` via `Fetcher.get` (HTTP, not a browser per query).
+| Value | Module | Notes |
+|-------|--------|-------|
+| `ddg_html` (default) | `search_ddg.py` | DuckDuckGo HTML SERP via Scrapling HTTP (+ one stealth retry on challenge) |
+| `searxng` | `search_searxng.py` | Self-hosted JSON API — requires `HARNESS_WEB_SEARXNG_URL` |
 
-## Selectors
+Bootstrap local SearXNG: `node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs"`
+
+## DuckDuckGo HTML (`ddg_html`)
+
+`GET https://html.duckduckgo.com/html/?q=…`
+
+### Selectors
 
 | Field | CSS |
 |-------|-----|
@@ -16,10 +25,18 @@ Implemented in `harness_web/search_ddg.py` via `Fetcher.get` (HTTP, not a browse
 
 DDG redirect URLs (`//duckduckgo.com/l/?uddg=…`) are unwrapped to the target `uddg` parameter.
 
-## Challenge detection
+### Challenge detection
 
 If status 403 or HTML contains challenge markers (`anomaly-modal`, etc.), retry **once** with `StealthyFetcher`, then exit with a clear “search engine blocked” message.
 
+## SearXNG (`searxng`)
+
+`GET {HARNESS_WEB_SEARXNG_URL}/search?q=…&format=json&pageno=1`
+
+- No client API token (SearXNG has no standard search API key).
+- `search.formats` in instance `settings.yml` must include `json` or the API returns **403**.
+- Public instances are unsuitable (~4 JSON req/hr when limiter on; JSON often disabled). Use self-hosted bootstrap.
+
 ## Output
 
 `.web/search.json` — envelope compatible with legacy Firecrawl skills:
@@ -31,3 +48,5 @@ If status 403 or HTML contains challenge markers (`anomaly-modal`, etc.), retry
   "data": { "web": [{ "url", "title", "description" }] }
 }
 ```
+
+`engine` reflects the active backend (`ddg_html` or `searxng`).