ultimate-pi 0.4.1 → 0.5.0

package/.pi/prompts/harness-setup.md

@@ -134,10 +134,13 @@ export PATH="$HOME/.local/bin:$PATH"
 uv tool install "scrapling[fetchers]"
 scrapling install   # Chromium for default stealth scrape; may need sudo for OS libs on Linux
 mkdir -p .web
+python3 "$UP_PKG/.pi/scripts/harness-web.py" status   # JSON config (setup/diagnostics only)
 python3 "$UP_PKG/.pi/scripts/harness-web.py" search "ultimate-pi harness" -o .web/smoke-search.json --limit 3
 python3 "$UP_PKG/.pi/scripts/harness-web.py" scrape "https://example.com" -o .web/smoke-page.md --fast
 ```
 
+After pi loads extensions, agents should smoke **`web_search`** once (not `UP_PKG` / `import scrapling` preflight). Example intent: query `ultimate-pi harness`, `limit` 2.
+
 - **`--skip-tools`:** skip Step 2 (includes Scrapling verify).
 - On Linux/WSL, if stealth scrape fails, install browser libs from `harness-cli-verify.sh` output or use `--fast` for static targets.
 
@@ -421,6 +424,47 @@ If **no** `.env` at project root:
 - On **skip** or `--non-interactive`: warn in report (non-interactive skips creation)
 - If `ask_user` cancelled: stop with `needs_clarification`
 
+### 4.0b — harness-web search engine (non-destructive)
+
+Unless `--non-interactive`, **call `ask_user`** after Step 4.0 (harness-decisions skill):
+
+```json
+{
+  "question": "Which harness-web search backend should this project use?",
+  "context": "Scrapling still handles scrape/map/bulk. Search only: DuckDuckGo HTML needs no extra services. SearXNG must be self-hosted for agents — public instances often block JSON (403) and default to ~4 API requests/hour per IP.",
+  "options": [
+    {
+      "title": "DuckDuckGo HTML (default)",
+      "description": "HARNESS_WEB_SEARCH_ENGINE=ddg_html — no Docker"
+    },
+    {
+      "title": "Self-host SearXNG here (Docker)",
+      "description": "Bootstrap .searxng/ with official compose, enable JSON API, set harness env"
+    },
+    {
+      "title": "Use existing SearXNG instance",
+      "description": "You provide base URL; harness writes HARNESS_WEB_SEARXNG_URL"
+    }
+  ],
+  "allowFreeform": true
+}
+```
+
+| User choice | Actions |
+|-------------|---------|
+| **DDG** | Ensure `.env` has `HARNESS_WEB_SEARCH_ENGINE=ddg_html` via `harness-sync-env.mjs` (append only if missing; do not overwrite user values) |
+| **Self-host** | `node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs"` (requires Docker). Script sets `HARNESS_WEB_SEARCH_ENGINE=searxng` and `HARNESS_WEB_SEARXNG_URL` |
+| **Existing instance** | Parse base URL from freeform answer. Run `node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs" --set-url {url}` (health check + upsert `.env`) |
+| **Cancelled** | Stop with `needs_clarification` |
+| **`--non-interactive`** | Skip prompt; leave/default `ddg_html`; do not run Docker bootstrap |
+
+Post-choice smoke (report pass/fail):
+
+```bash
+mkdir -p .web
+python3 "$UP_PKG/.pi/scripts/harness-web.py" search "ultimate-pi harness" -o .web/setup-search.json --limit 2
+```
+
 Rules:
 
 - **Do not** `cp` over an existing `.env`.
@@ -436,6 +480,7 @@ Ensure `.gitignore` contains:
 ```
 .env
 .web/
+.searxng/
 .raw/
 .vault-meta/
 .pi/harness/critics/
@@ -646,6 +691,7 @@ Output summary table:
 | .gitignore | ✓/✗ | entries added (incl. `.env`) |
 | ./raw directory | ✓/✗ | Created for graphify source ingestion |
 | harness-web (Scrapling) | ✓/✗ | search + scrape smoke |
+| harness-web search engine | ddg / searxng / — | Step 4.0b choice; SearXNG URL if applicable |
 
 Next steps:
 1. If tools missing: re-run with `--force` or install individually

package/.pi/scripts/harness-cli-verify.sh

@@ -200,10 +200,19 @@ verify_scrapling() {
 		return
 	fi
 	mkdir -p .web
-	if python3 "$_hw" search "ultimate-pi harness" -o .web/verify-search.json --limit 2 2>/dev/null | grep -q wrote; then
-		pass "harness-web search smoke"
+	_search_engine="${HARNESS_WEB_SEARCH_ENGINE:-ddg_html}"
+	if [ "$_search_engine" = "searxng" ]; then
+		if [ -z "${HARNESS_WEB_SEARXNG_URL:-}" ]; then
+			fail "HARNESS_WEB_SEARCH_ENGINE=searxng but HARNESS_WEB_SEARXNG_URL is unset"
+		elif python3 "$_hw" search "ultimate-pi harness" -o .web/verify-search.json --limit 2 2>/dev/null | grep -q wrote; then
+			pass "harness-web search smoke (searxng)"
+		else
+			fail "harness-web search smoke failed (searxng at ${HARNESS_WEB_SEARXNG_URL})"
+		fi
+	elif python3 "$_hw" search "ultimate-pi harness" -o .web/verify-search.json --limit 2 2>/dev/null | grep -q wrote; then
+		pass "harness-web search smoke (ddg_html)"
 	else
-		fail "harness-web search smoke failed"
+		fail "harness-web search smoke failed (ddg_html)"
 	fi
 	if python3 "$_hw" scrape "https://example.com" -o .web/verify-page.md --fast 2>/dev/null | grep -q wrote; then
 		pass "harness-web scrape --fast smoke"

package/.pi/scripts/harness-searxng-bootstrap.mjs

@@ -0,0 +1,270 @@
+#!/usr/bin/env node
+/**
+ * Bootstrap a project-local SearXNG instance for harness-web (Docker Compose).
+ *
+ * - Creates .searxng/ with official upstream compose template
+ * - Writes core-config/settings.yml with json format + limiter off (local dev)
+ * - Starts containers and waits for JSON search health
+ * - Upserts HARNESS_WEB_SEARCH_ENGINE / HARNESS_WEB_SEARXNG_URL in project .env
+ *
+ * Usage:
+ *   node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs" [PROJECT_ROOT] [--url-only]
+ *   node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs" --set-url http://127.0.0.1:8080
+ *
+ * Requires: docker, docker compose, curl
+ */
+
+import {
+	access,
+	copyFile,
+	mkdir,
+	readFile,
+	writeFile,
+} from "node:fs/promises";
+import { constants } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn } from "node:child_process";
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const UP_PKG = join(SCRIPT_DIR, "..", "..");
+
+const SEARXNG_BASE =
+	"https://raw.githubusercontent.com/searxng/searxng/master/container";
+const DEFAULT_PORT = "8080";
+const HEALTH_PATH = "/search?q=harness&format=json";
+
+const MANAGED_START = "# --- harness:env:start ---";
+const MANAGED_END = "# --- harness:env:end ---";
+
+const args = process.argv.slice(2).filter((a) => !a.startsWith("-"));
+const flags = new Set(process.argv.slice(2).filter((a) => a.startsWith("-")));
+const urlOnly = flags.has("--url-only");
+const setUrlIdx = process.argv.indexOf("--set-url");
+const setUrl = setUrlIdx !== -1 ? process.argv[setUrlIdx + 1] : null;
+
+const PROJECT_ROOT = args[0] || process.cwd();
+const SEARXNG_DIR = join(PROJECT_ROOT, ".searxng");
+const CORE_CONFIG = join(SEARXNG_DIR, "core-config");
+const SETTINGS_PATH = join(CORE_CONFIG, "settings.yml");
+const COMPOSE_PATH = join(SEARXNG_DIR, "docker-compose.yml");
+const ENV_COMPOSE = join(SEARXNG_DIR, ".env");
+
+const HARNESS_SETTINGS = `use_default_settings: true
+
+search:
+  formats:
+    - html
+    - json
+
+server:
+  limiter: false
+  public_instance: false
+`;
+
+async function exists(path) {
+	try {
+		await access(path, constants.F_OK);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+function run(cmd, cmdArgs, opts = {}) {
+	return new Promise((resolve, reject) => {
+		const child = spawn(cmd, cmdArgs, {
+			stdio: opts.inherit ? "inherit" : "pipe",
+			cwd: opts.cwd,
+			env: { ...process.env, ...opts.env },
+		});
+		let stdout = "";
+		let stderr = "";
+		if (!opts.inherit) {
+			child.stdout?.on("data", (d) => {
+				stdout += d;
+			});
+			child.stderr?.on("data", (d) => {
+				stderr += d;
+			});
+		}
+		child.on("error", reject);
+		child.on("close", (code) => {
+			if (code === 0) resolve({ stdout, stderr });
+			else
+				reject(
+					new Error(
+						`${cmd} ${cmdArgs.join(" ")} exited ${code}\n${stderr || stdout}`,
+					),
+				);
+		});
+	});
+}
+
+async function requireDocker() {
+	for (const bin of ["docker"]) {
+		try {
+			await run(bin, ["--version"]);
+		} catch {
+			console.error(`✗ ${bin} not found`);
+			console.error(
+				"Install Docker: https://docs.searxng.org/admin/installation-docker.html",
+			);
+			process.exit(1);
+		}
+	}
+	try {
+		await run("docker", ["compose", "version"]);
+	} catch {
+		console.error("✗ docker compose not available");
+		console.error(
+			"Install Docker Compose v2: https://docs.docker.com/compose/install/",
+		);
+		process.exit(1);
+	}
+}
+
+async function curlToFile(url, dest) {
+	await run("curl", ["-fsSL", "-o", dest, url]);
+}
+
+async function readComposePort() {
+	if (!(await exists(ENV_COMPOSE))) return DEFAULT_PORT;
+	const text = await readFile(ENV_COMPOSE, "utf8");
+	for (const line of text.split("\n")) {
+		const m = line.match(/^SEARXNG_PORT=(.+)$/);
+		if (m) return m[1].trim().replace(/^["']|["']$/g, "") || DEFAULT_PORT;
+	}
+	return DEFAULT_PORT;
+}
+
+async function ensureSearxngLayout() {
+	await mkdir(CORE_CONFIG, { recursive: true });
+	if (!(await exists(COMPOSE_PATH))) {
+		console.log("Fetching SearXNG docker-compose.yml …");
+		await curlToFile(`${SEARXNG_BASE}/docker-compose.yml`, COMPOSE_PATH);
+	}
+	if (!(await exists(ENV_COMPOSE))) {
+		const example = join(SEARXNG_DIR, ".env.example");
+		if (!(await exists(example))) {
+			console.log("Fetching SearXNG .env.example …");
+			await curlToFile(`${SEARXNG_BASE}/.env.example`, example);
+		}
+		await copyFile(example, ENV_COMPOSE);
+	}
+	const needsSettings =
+		!(await exists(SETTINGS_PATH)) ||
+		!(await readFile(SETTINGS_PATH, "utf8")).includes("json");
+	if (needsSettings) {
+		await writeFile(SETTINGS_PATH, HARNESS_SETTINGS, "utf8");
+		console.log(`✓ Wrote ${SETTINGS_PATH} (json format, limiter off)`);
+	}
+}
+
+async function composeUp() {
+	console.log("Starting SearXNG (docker compose up -d) …");
+	await run("docker", ["compose", "up", "-d"], { cwd: SEARXNG_DIR, inherit: true });
+}
+
+async function waitForHealth(baseUrl) {
+	const url = `${baseUrl}${HEALTH_PATH}`;
+	const deadline = Date.now() + 90_000;
+	let lastErr = "";
+	while (Date.now() < deadline) {
+		try {
+			const res = await fetch(url, {
+				headers: { Accept: "application/json" },
+				signal: AbortSignal.timeout(10_000),
+			});
+			if (res.status === 403) {
+				throw new Error(
+					"SearXNG returned 403 for format=json — ensure search.formats includes json in .searxng/core-config/settings.yml",
+				);
+			}
+			if (res.ok) {
+				const data = await res.json();
+				if (data && typeof data === "object") {
+					console.log(`✓ SearXNG healthy at ${baseUrl}`);
+					return;
+				}
+			}
+			lastErr = `HTTP ${res.status}`;
+		} catch (err) {
+			lastErr = err instanceof Error ? err.message : String(err);
+		}
+		await new Promise((r) => setTimeout(r, 3000));
+	}
+	throw new Error(`SearXNG health check timed out (${url}): ${lastErr}`);
+}
+
+function upsertEnvKey(content, key, value) {
+	const line = `${key}=${value}`;
+	const re = new RegExp(`^${key}=.*$`, "m");
+	if (re.test(content)) {
+		return content.replace(re, line);
+	}
+	if (content.includes(MANAGED_START) && content.includes(MANAGED_END)) {
+		const end = content.indexOf(MANAGED_END);
+		return `${content.slice(0, end)}${line}\n${content.slice(end)}`;
+	}
+	const sep = content.endsWith("\n") || content.length === 0 ? "" : "\n";
+	return `${content}${sep}${MANAGED_START}\n# harness-web (SearXNG)\n${line}\n${MANAGED_END}\n`;
+}
+
+async function upsertHarnessEnv(baseUrl) {
+	const envPath = join(PROJECT_ROOT, ".env");
+	let content = "";
+	if (await exists(envPath)) {
+		content = await readFile(envPath, "utf8");
+	} else {
+		const template = join(UP_PKG, ".pi", "harness", "env.harness.template");
+		if (await exists(template)) {
+			content = await readFile(template, "utf8");
+		}
+	}
+	content = upsertEnvKey(content, "HARNESS_WEB_SEARCH_ENGINE", "searxng");
+	content = upsertEnvKey(content, "HARNESS_WEB_SEARXNG_URL", baseUrl);
+	await writeFile(envPath, content.endsWith("\n") ? content : `${content}\n`, "utf8");
+	console.log(`✓ Updated .env: HARNESS_WEB_SEARCH_ENGINE=searxng, HARNESS_WEB_SEARXNG_URL=${baseUrl}`);
+}
+
+function normalizeBaseUrl(raw) {
+	const url = raw.trim().replace(/\/+$/, "");
+	if (!/^https?:\/\//i.test(url)) {
+		throw new Error(`Invalid SearXNG URL: ${raw}`);
+	}
+	return url;
+}
+
+async function main() {
+	if (setUrl) {
+		const baseUrl = normalizeBaseUrl(setUrl);
+		await waitForHealth(baseUrl);
+		await upsertHarnessEnv(baseUrl);
+		process.exit(0);
+	}
+
+	if (urlOnly) {
+		const port = (await exists(ENV_COMPOSE)) ? await readComposePort() : DEFAULT_PORT;
+		console.log(`http://127.0.0.1:${port}`);
+		process.exit(0);
+	}
+
+	await requireDocker();
+	await ensureSearxngLayout();
+	const port = await readComposePort();
+	const baseUrl = `http://127.0.0.1:${port}`;
+	await composeUp();
+	await waitForHealth(baseUrl);
+	await upsertHarnessEnv(baseUrl);
+
+	console.log("");
+	console.log("SearXNG is ready for harness-web:");
+	console.log(`  HARNESS_WEB_SEARXNG_URL=${baseUrl}`);
+	console.log(`  Test: python3 "${join(UP_PKG, ".pi/scripts/harness-web.py")}" search "test" -o .web/search.json --limit 2`);
+}
+
+main().catch((err) => {
+	console.error(`✗ ${err.message || err}`);
+	process.exit(1);
+});

package/.pi/scripts/harness-web-search.md

@@ -1,12 +1,21 @@
 # harness-web search (internal)
 
-## Engine
+Routing: `harness_web/search.py` dispatches by `HARNESS_WEB_SEARCH_ENGINE`.
 
-Default: DuckDuckGo static HTML — `GET https://html.duckduckgo.com/html/?q=…`
+## Engines
 
-Implemented in `harness_web/search_ddg.py` via `Fetcher.get` (HTTP, not a browser per query).
+| Value | Module | Notes |
+|-------|--------|-------|
+| `ddg_html` (default) | `search_ddg.py` | DuckDuckGo HTML SERP via Scrapling HTTP (+ one stealth retry on challenge) |
+| `searxng` | `search_searxng.py` | Self-hosted JSON API — requires `HARNESS_WEB_SEARXNG_URL` |
 
-## Selectors
+Bootstrap local SearXNG: `node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs"`
+
+## DuckDuckGo HTML (`ddg_html`)
+
+`GET https://html.duckduckgo.com/html/?q=…`
+
+### Selectors
 
 | Field | CSS |
 |-------|-----|
@@ -16,10 +25,18 @@ Implemented in `harness_web/search_ddg.py` via `Fetcher.get` (HTTP, not a browse
 
 DDG redirect URLs (`//duckduckgo.com/l/?uddg=…`) are unwrapped to the target `uddg` parameter.
 
-## Challenge detection
+### Challenge detection
 
 If status 403 or HTML contains challenge markers (`anomaly-modal`, etc.), retry **once** with `StealthyFetcher`, then exit with a clear “search engine blocked” message.
 
+## SearXNG (`searxng`)
+
+`GET {HARNESS_WEB_SEARXNG_URL}/search?q=…&format=json&pageno=1`
+
+- No client API token (SearXNG has no standard search API key).
+- `search.formats` in instance `settings.yml` must include `json` or the API returns **403**.
+- Public instances are unsuitable (~4 JSON req/hr when limiter on; JSON often disabled). Use self-hosted bootstrap.
+
 ## Output
 
 `.web/search.json` — envelope compatible with legacy Firecrawl skills:
@@ -31,3 +48,5 @@ If status 403 or HTML contains challenge markers (`anomaly-modal`, etc.), retry
   "data": { "web": [{ "url", "title", "description" }] }
 }
 ```
+
+`engine` reflects the active backend (`ddg_html` or `searxng`).

package/.pi/scripts/harness-web.py

@@ -36,7 +36,7 @@ if str(SCRIPT_DIR) not in sys.path:
 from harness_web.config import HarnessWebConfig, load_config  # noqa: E402
 from harness_web.output import write_search_results  # noqa: E402
 from harness_web.scrape import bulk_scrape, map_url, scrape_url  # noqa: E402
-from harness_web.search_ddg import search_ddg  # noqa: E402
+from harness_web.search import search  # noqa: E402
 
 DEFAULT_WEB_DIR = ".web"
 
@@ -47,8 +47,8 @@ def _default_out(sub: str) -> Path:
 
 def cmd_search(args: argparse.Namespace, config: HarnessWebConfig) -> int:
     out = Path(args.output or _default_out("search.json"))
-    results = search_ddg(args.query, limit=args.limit, config=config)
-    write_search_results(out, results, args.query)
+    results = search(args.query, limit=args.limit, config=config)
+    write_search_results(out, results, args.query, engine=config.search_engine)
     print(f"wrote {out} ({len(results)} results)")
     return 0
 
@@ -76,6 +76,20 @@ def cmd_map(args: argparse.Namespace, config: HarnessWebConfig) -> int:
     return 0
 
 
+def cmd_status(_args: argparse.Namespace, config: HarnessWebConfig) -> int:
+    import json
+
+    payload = {
+        "search_engine": config.search_engine,
+        "searxng_url": config.searxng_url,
+        "fetch_mode": config.fetch_mode,
+        "script": str(Path(__file__).resolve()),
+        "bootstrap": "ok",
+    }
+    print(json.dumps(payload, indent=2))
+    return 0
+
+
 def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
     sleep_sec = args.sleep if args.sleep is not None else config.rate_limit_ms / 1000.0
     if args.urls:
@@ -86,8 +100,8 @@ def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
         data = json.loads(Path(args.from_search).read_text(encoding="utf-8"))
         urls = [item["url"] for item in data.get("data", {}).get("web", []) if item.get("url")]
     else:
-        urls = search_ddg(args.query, limit=args.limit, config=config)
-        urls = [r["url"] for r in urls]
+        serp = search(args.query, limit=args.limit, config=config)
+        urls = [r["url"] for r in serp]
 
     if not urls:
         print("bulk-scrape: no URLs to fetch", file=sys.stderr)
@@ -111,11 +125,11 @@ def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
 def build_parser() -> argparse.ArgumentParser:
     p = argparse.ArgumentParser(
         prog="harness-web",
-        description="Harness web layer: search (DDG HTML) and scrape (Scrapling).",
+        description="Harness web layer: search (DDG HTML or SearXNG) and scrape (Scrapling).",
     )
     sub = p.add_subparsers(dest="command", required=True)
 
-    ps = sub.add_parser("search", help="Search via DuckDuckGo HTML SERP")
+    ps = sub.add_parser("search", help="Search via configured SERP (HARNESS_WEB_SEARCH_ENGINE)")
     ps.add_argument("query", help="Search query")
     ps.add_argument("-o", "--output", help="JSON output path (default: .web/search.json)")
     ps.add_argument("--limit", type=int, default=5)
@@ -160,6 +174,9 @@ def build_parser() -> argparse.ArgumentParser:
     pm.add_argument("--fast", action="store_true")
     pm.set_defaults(func=cmd_map)
 
+    pst = sub.add_parser("status", help="Print harness-web config as JSON (setup/diagnostics)")
+    pst.set_defaults(func=cmd_status)
+
     return p
 
 

package/.pi/scripts/harness_web/config.py

@@ -6,6 +6,8 @@ import os
 from dataclasses import dataclass
 from urllib.parse import urlparse
 
+SUPPORTED_SEARCH_ENGINES = frozenset({"ddg_html", "searxng"})
+
 
 def _int_env(name: str, default: int) -> int:
     raw = os.environ.get(name, "").strip()
@@ -24,6 +26,18 @@ def _fetch_mode() -> str:
     return "stealth"
 
 
+def _normalize_searxng_url(raw: str) -> str:
+    url = raw.strip().rstrip("/")
+    if not url:
+        return ""
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https") or not parsed.netloc:
+        raise SystemExit(
+            f"Invalid HARNESS_WEB_SEARXNG_URL={raw!r} — expected http(s)://host[:port]"
+        )
+    return url
+
+
 _STATIC_HOSTS = frozenset(
     {
         "example.com",
@@ -50,6 +64,7 @@ def host_is_static(url: str) -> bool:
 class HarnessWebConfig:
     fetch_mode: str
     search_engine: str
+    searxng_url: str | None
     proxy: str | None
     rate_limit_ms: int
     timeout_ms: int
@@ -68,13 +83,32 @@ class HarnessWebConfig:
         return False
 
 
+def validate_search_config(config: HarnessWebConfig) -> None:
+    engine = config.search_engine
+    if engine not in SUPPORTED_SEARCH_ENGINES:
+        supported = ", ".join(sorted(SUPPORTED_SEARCH_ENGINES))
+        raise SystemExit(
+            f"Unsupported HARNESS_WEB_SEARCH_ENGINE={engine!r} (supported: {supported})"
+        )
+    if engine == "searxng" and not config.searxng_url:
+        raise SystemExit(
+            "HARNESS_WEB_SEARCH_ENGINE=searxng requires HARNESS_WEB_SEARXNG_URL "
+            "(e.g. http://127.0.0.1:8080). Run /harness-setup and choose SearXNG, or set both in .env."
+        )
+
+
 def load_config() -> HarnessWebConfig:
     proxy = os.environ.get("HARNESS_WEB_PROXY", "").strip() or None
-    return HarnessWebConfig(
+    engine = os.environ.get("HARNESS_WEB_SEARCH_ENGINE", "ddg_html").strip() or "ddg_html"
+    searx_raw = os.environ.get("HARNESS_WEB_SEARXNG_URL", "").strip()
+    searxng_url = _normalize_searxng_url(searx_raw) if searx_raw else None
+    config = HarnessWebConfig(
         fetch_mode=_fetch_mode(),
-        search_engine=os.environ.get("HARNESS_WEB_SEARCH_ENGINE", "ddg_html").strip()
-        or "ddg_html",
+        search_engine=engine,
+        searxng_url=searxng_url,
         proxy=proxy,
         rate_limit_ms=_int_env("HARNESS_WEB_RATE_LIMIT_MS", 2000),
         timeout_ms=_int_env("HARNESS_WEB_TIMEOUT_MS", 30000),
     )
+    validate_search_config(config)
+    return config

package/.pi/scripts/harness_web/output.py

@@ -18,13 +18,19 @@ def write_json(path: Path, payload: Any) -> None:
     path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
 
 
-def write_search_results(path: Path, results: list[dict[str, str]], query: str) -> None:
+def write_search_results(
+    path: Path,
+    results: list[dict[str, str]],
+    query: str,
+    *,
+    engine: str,
+) -> None:
     """Firecrawl-compatible envelope: data.web[].url|title|description."""
     write_json(
         path,
         {
             "query": query,
-            "engine": "ddg_html",
+            "engine": engine,
             "data": {
                 "web": [
                     {

package/.pi/scripts/harness_web/search.py

@@ -0,0 +1,22 @@
+"""Route harness-web search to the configured SERP backend."""
+
+from __future__ import annotations
+
+from .config import HarnessWebConfig, validate_search_config
+from .search_ddg import search_ddg
+from .search_searxng import search_searxng
+
+
+def search(
+    query: str,
+    *,
+    limit: int,
+    config: HarnessWebConfig,
+) -> list[dict[str, str]]:
+    validate_search_config(config)
+    engine = config.search_engine
+    if engine == "searxng":
+        return search_searxng(query, limit=limit, config=config)
+    if engine == "ddg_html":
+        return search_ddg(query, limit=limit, config=config)
+    raise SystemExit(f"Unsupported HARNESS_WEB_SEARCH_ENGINE={engine!r}")

package/.pi/scripts/harness_web/search_ddg.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from typing import Any
 from urllib.parse import parse_qs, unquote, urlparse
 
 from scrapling.fetchers import Fetcher, StealthyFetcher
@@ -63,11 +64,6 @@ def search_ddg(
     config: HarnessWebConfig,
     impersonate: bool = True,
 ) -> list[dict[str, str]]:
-    if config.search_engine != "ddg_html":
-        raise SystemExit(
-            f"Unsupported HARNESS_WEB_SEARCH_ENGINE={config.search_engine!r} (only ddg_html)"
-        )
-
     kwargs: dict = {
         "params": {"q": query},
         "timeout": config.timeout_sec,