ultimate-pi 0.4.1 → 0.6.0

package/.pi/extensions/lib/harness-web/run-cli.ts

@@ -0,0 +1,92 @@
+import { spawnSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { resolveHarnessScript } from "../harness-paths.js";
+
+export interface RunHarnessWebResult {
+	ok: boolean;
+	exitCode: number;
+	stdout: string;
+	stderr: string;
+}
+
+export function runHarnessWeb(
+	moduleUrl: string,
+	args: string[],
+	cwd: string,
+): RunHarnessWebResult {
+	const script = resolveHarnessScript(moduleUrl, "harness-web.py");
+	const result = spawnSync("python3", [script, ...args], {
+		cwd,
+		env: process.env,
+		encoding: "utf-8",
+		maxBuffer: 16 * 1024 * 1024,
+	});
+	return {
+		ok: result.status === 0,
+		exitCode: result.status ?? 1,
+		stdout: (result.stdout ?? "").trim(),
+		stderr: (result.stderr ?? "").trim(),
+	};
+}
+
+export function readTextExcerpt(
+	filePath: string,
+	cwd: string,
+	maxChars = 2000,
+): string {
+	const full = resolve(cwd, filePath);
+	if (!existsSync(full)) return "";
+	const text = readFileSync(full, "utf-8");
+	if (text.length <= maxChars) return text;
+	return `${text.slice(0, maxChars)}\n… (truncated; use read tool for full file)`;
+}
+
+export interface SearchHit {
+	url: string;
+	title: string;
+	description: string;
+}
+
+export function summarizeSearchJson(filePath: string, cwd: string): string {
+	const full = resolve(cwd, filePath);
+	if (!existsSync(full)) return "";
+	try {
+		const data = JSON.parse(readFileSync(full, "utf-8")) as {
+			query?: string;
+			engine?: string;
+			data?: { web?: SearchHit[] };
+		};
+		const hits = data.data?.web ?? [];
+		const lines = [
+			`engine: ${data.engine ?? "unknown"}`,
+			`query: ${data.query ?? ""}`,
+			`results: ${hits.length}`,
+			"",
+		];
+		for (const [i, hit] of hits.entries()) {
+			lines.push(`${i + 1}. ${hit.title || "(no title)"}`);
+			lines.push(`   ${hit.url}`);
+			if (hit.description) {
+				const snip =
+					hit.description.length > 120
+						? `${hit.description.slice(0, 120)}…`
+						: hit.description;
+				lines.push(`   ${snip}`);
+			}
+		}
+		return lines.join("\n");
+	} catch {
+		return "";
+	}
+}
+
+export function harnessWebContextLine(): string {
+	const engine = process.env.HARNESS_WEB_SEARCH_ENGINE?.trim() || "ddg_html";
+	const searx = process.env.HARNESS_WEB_SEARXNG_URL?.trim();
+	const searxPart = searx ? ` searxng_url=${searx}` : "";
+	return (
+		`[HarnessWeb] search_engine=${engine}${searxPart} — use web_search / web_fetch tools; ` +
+		"never resolve UP_PKG, ls harness-web.py, or python3 -c import scrapling before searching."
+	);
+}

package/.pi/harness/env.harness.template

@@ -4,9 +4,11 @@
 # Telemetry (set false to disable harness PostHog events)
 HARNESS_TELEMETRY_ENABLED=true
 
-# harness-web (Scrapling) — local fetch/search; no API key
+# harness-web (Scrapling scrape + pluggable search)
 HARNESS_WEB_FETCH_MODE=stealth
 HARNESS_WEB_SEARCH_ENGINE=ddg_html
+# SearXNG (when HARNESS_WEB_SEARCH_ENGINE=searxng):
+# HARNESS_WEB_SEARXNG_URL=http://127.0.0.1:8080
 # HARNESS_WEB_PROXY=
 # HARNESS_WEB_RATE_LIMIT_MS=2000
 # HARNESS_WEB_TIMEOUT_MS=30000

package/.pi/prompts/harness-setup.md

@@ -134,10 +134,13 @@ export PATH="$HOME/.local/bin:$PATH"
 uv tool install "scrapling[fetchers]"
 scrapling install   # Chromium for default stealth scrape; may need sudo for OS libs on Linux
 mkdir -p .web
+python3 "$UP_PKG/.pi/scripts/harness-web.py" status   # JSON config (setup/diagnostics only)
 python3 "$UP_PKG/.pi/scripts/harness-web.py" search "ultimate-pi harness" -o .web/smoke-search.json --limit 3
 python3 "$UP_PKG/.pi/scripts/harness-web.py" scrape "https://example.com" -o .web/smoke-page.md --fast
 ```
 
+After pi loads extensions, agents should smoke **`web_search`** once (not `UP_PKG` / `import scrapling` preflight). Example intent: query `ultimate-pi harness`, `limit` 2.
+
 - **`--skip-tools`:** skip Step 2 (includes Scrapling verify).
 - On Linux/WSL, if stealth scrape fails, install browser libs from `harness-cli-verify.sh` output or use `--fast` for static targets.
 
@@ -303,7 +306,7 @@ if gh auth status &>/dev/null; then
 fi
 ```
 
-### 2.8 — sentrux (Architectural Quality Gate + MCP Sensor)
+### 2.8 — sentrux (Architectural Quality Gate)
 
 ```bash
 if ! command -v sentrux &>/dev/null || [ "$FORCE" = "true" ]; then
@@ -316,7 +319,7 @@ Install all 52 language plugins:
 sentrux plugin add-standard 2>/dev/null || echo "Plugins already installed or failed"
 ```
 
-Configure MCP server in `.pi/mcp.json` (see Step 4.2). **Rules.toml bootstrap runs in Step 4.3** (idempotent, merge-safe).
+Ensure the **sentrux** Pi skill is linked (see Step 4.2). **Rules.toml bootstrap runs in Step 4.3** (idempotent, merge-safe).
 
 ## Step 3 — Pi Extension Packages
 
@@ -421,6 +424,47 @@ If **no** `.env` at project root:
 - On **skip** or `--non-interactive`: warn in report (non-interactive skips creation)
 - If `ask_user` cancelled: stop with `needs_clarification`
 
+### 4.0b — harness-web search engine (non-destructive)
+
+Unless `--non-interactive`, **call `ask_user`** after Step 4.0 (harness-decisions skill):
+
+```json
+{
+  "question": "Which harness-web search backend should this project use?",
+  "context": "Scrapling still handles scrape/map/bulk. Search only: DuckDuckGo HTML needs no extra services. SearXNG must be self-hosted for agents — public instances often block JSON (403) and default to ~4 API requests/hour per IP.",
+  "options": [
+    {
+      "title": "DuckDuckGo HTML (default)",
+      "description": "HARNESS_WEB_SEARCH_ENGINE=ddg_html — no Docker"
+    },
+    {
+      "title": "Self-host SearXNG here (Docker)",
+      "description": "Bootstrap .searxng/ with official compose, enable JSON API, set harness env"
+    },
+    {
+      "title": "Use existing SearXNG instance",
+      "description": "You provide base URL; harness writes HARNESS_WEB_SEARXNG_URL"
+    }
+  ],
+  "allowFreeform": true
+}
+```
+
+| User choice | Actions |
+|-------------|---------|
+| **DDG** | Ensure `.env` has `HARNESS_WEB_SEARCH_ENGINE=ddg_html` via `harness-sync-env.mjs` (append only if missing; do not overwrite user values) |
+| **Self-host** | `node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs"` (requires Docker). Script sets `HARNESS_WEB_SEARCH_ENGINE=searxng` and `HARNESS_WEB_SEARXNG_URL` |
+| **Existing instance** | Parse base URL from freeform answer. Run `node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs" --set-url {url}` (health check + upsert `.env`) |
+| **Cancelled** | Stop with `needs_clarification` |
+| **`--non-interactive`** | Skip prompt; leave/default `ddg_html`; do not run Docker bootstrap |
+
+Post-choice smoke (report pass/fail):
+
+```bash
+mkdir -p .web
+python3 "$UP_PKG/.pi/scripts/harness-web.py" search "ultimate-pi harness" -o .web/setup-search.json --limit 2
+```
+
 Rules:
 
 - **Do not** `cp` over an existing `.env`.
@@ -436,6 +480,7 @@ Ensure `.gitignore` contains:
 ```
 .env
 .web/
+.searxng/
 .raw/
 .vault-meta/
 .pi/harness/critics/
@@ -451,28 +496,27 @@ Ensure `.gitignore` contains:
 !.sentrux/rules.toml
 ```
 
-### 4.2 — MCP Server Configuration
+### 4.2 — Sentrux Pi skill
 
-Add sentrux MCP server to `.pi/mcp.json`:
-```json
-{
-  "mcpServers": {
-    "context-mode": {
-      "command": "context-mode"
-    },
-    "sentrux": {
-      "command": "sentrux",
-      "args": ["--mcp"]
-    }
-  }
-}
+Pi does **not** load `.pi/mcp.json`. Agents use Sentrux via the **CLI** and the **`sentrux`** skill.
+
+From **project root**, ensure the skill is discoverable (idempotent):
+
+```bash
+UP_PKG="$(node -p "require('path').dirname(require.resolve('ultimate-pi/package.json'))")"
+SKILL_SRC="$UP_PKG/.agents/skills/sentrux"
+SKILL_DST=".pi/skills/sentrux"
+if [ -d "$SKILL_SRC" ] && [ ! -e "$SKILL_DST" ]; then
+  ln -s "../../.agents/skills/sentrux" "$SKILL_DST"
+  echo "✓ linked $SKILL_DST → sentrux skill"
+elif [ -e "$SKILL_DST" ]; then
+  echo "✓ sentrux skill already present at $SKILL_DST"
+else
+  echo "✗ missing $SKILL_SRC — reinstall ultimate-pi"
+fi
 ```
 
-This gives agents real-time access to structural health metrics:
-- `scan` — quality signal, file count, bottleneck detection
-- `session_start` / `session_end` — baseline comparison, degradation detection
-- `check_rules` — architectural constraint enforcement
-- `health`, `rescan`, `evolution`, `dsm`, `test_gaps`
+After `/reload`, agents can invoke **`/skill:sentrux`** for install paths, `sentrux check`, `sentrux gate --save` / `sentrux gate`, and harness integration. **context-mode** remains a separate `npm:context-mode` package in `.pi/settings.json` (its own MCP bridge inside that extension).
 
 ### 4.3 — Sentrux rules bootstrap (required)
 
@@ -646,6 +690,7 @@ Output summary table:
 | .gitignore | ✓/✗ | entries added (incl. `.env`) |
 | ./raw directory | ✓/✗ | Created for graphify source ingestion |
 | harness-web (Scrapling) | ✓/✗ | search + scrape smoke |
+| harness-web search engine | ddg / searxng / — | Step 4.0b choice; SearXNG URL if applicable |
 
 Next steps:
 1. If tools missing: re-run with `--force` or install individually

package/.pi/scripts/harness-cli-verify.sh

@@ -200,10 +200,19 @@ verify_scrapling() {
 		return
 	fi
 	mkdir -p .web
-	if python3 "$_hw" search "ultimate-pi harness" -o .web/verify-search.json --limit 2 2>/dev/null | grep -q wrote; then
-		pass "harness-web search smoke"
+	_search_engine="${HARNESS_WEB_SEARCH_ENGINE:-ddg_html}"
+	if [ "$_search_engine" = "searxng" ]; then
+		if [ -z "${HARNESS_WEB_SEARXNG_URL:-}" ]; then
+			fail "HARNESS_WEB_SEARCH_ENGINE=searxng but HARNESS_WEB_SEARXNG_URL is unset"
+		elif python3 "$_hw" search "ultimate-pi harness" -o .web/verify-search.json --limit 2 2>/dev/null | grep -q wrote; then
+			pass "harness-web search smoke (searxng)"
+		else
+			fail "harness-web search smoke failed (searxng at ${HARNESS_WEB_SEARXNG_URL})"
+		fi
+	elif python3 "$_hw" search "ultimate-pi harness" -o .web/verify-search.json --limit 2 2>/dev/null | grep -q wrote; then
+		pass "harness-web search smoke (ddg_html)"
 	else
-		fail "harness-web search smoke failed"
+		fail "harness-web search smoke failed (ddg_html)"
 	fi
 	if python3 "$_hw" scrape "https://example.com" -o .web/verify-page.md --fast 2>/dev/null | grep -q wrote; then
 		pass "harness-web scrape --fast smoke"

package/.pi/scripts/harness-searxng-bootstrap.mjs

@@ -0,0 +1,270 @@
+#!/usr/bin/env node
+/**
+ * Bootstrap a project-local SearXNG instance for harness-web (Docker Compose).
+ *
+ * - Creates .searxng/ with official upstream compose template
+ * - Writes core-config/settings.yml with json format + limiter off (local dev)
+ * - Starts containers and waits for JSON search health
+ * - Upserts HARNESS_WEB_SEARCH_ENGINE / HARNESS_WEB_SEARXNG_URL in project .env
+ *
+ * Usage:
+ *   node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs" [PROJECT_ROOT] [--url-only]
+ *   node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs" --set-url http://127.0.0.1:8080
+ *
+ * Requires: docker, docker compose, curl
+ */
+
+import {
+	access,
+	copyFile,
+	mkdir,
+	readFile,
+	writeFile,
+} from "node:fs/promises";
+import { constants } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn } from "node:child_process";
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const UP_PKG = join(SCRIPT_DIR, "..", "..");
+
+const SEARXNG_BASE =
+	"https://raw.githubusercontent.com/searxng/searxng/master/container";
+const DEFAULT_PORT = "8080";
+const HEALTH_PATH = "/search?q=harness&format=json";
+
+const MANAGED_START = "# --- harness:env:start ---";
+const MANAGED_END = "# --- harness:env:end ---";
+
+const args = process.argv.slice(2).filter((a) => !a.startsWith("-"));
+const flags = new Set(process.argv.slice(2).filter((a) => a.startsWith("-")));
+const urlOnly = flags.has("--url-only");
+const setUrlIdx = process.argv.indexOf("--set-url");
+const setUrl = setUrlIdx !== -1 ? process.argv[setUrlIdx + 1] : null;
+
+const PROJECT_ROOT = args[0] || process.cwd();
+const SEARXNG_DIR = join(PROJECT_ROOT, ".searxng");
+const CORE_CONFIG = join(SEARXNG_DIR, "core-config");
+const SETTINGS_PATH = join(CORE_CONFIG, "settings.yml");
+const COMPOSE_PATH = join(SEARXNG_DIR, "docker-compose.yml");
+const ENV_COMPOSE = join(SEARXNG_DIR, ".env");
+
+const HARNESS_SETTINGS = `use_default_settings: true
+
+search:
+  formats:
+    - html
+    - json
+
+server:
+  limiter: false
+  public_instance: false
+`;
+
+async function exists(path) {
+	try {
+		await access(path, constants.F_OK);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+function run(cmd, cmdArgs, opts = {}) {
+	return new Promise((resolve, reject) => {
+		const child = spawn(cmd, cmdArgs, {
+			stdio: opts.inherit ? "inherit" : "pipe",
+			cwd: opts.cwd,
+			env: { ...process.env, ...opts.env },
+		});
+		let stdout = "";
+		let stderr = "";
+		if (!opts.inherit) {
+			child.stdout?.on("data", (d) => {
+				stdout += d;
+			});
+			child.stderr?.on("data", (d) => {
+				stderr += d;
+			});
+		}
+		child.on("error", reject);
+		child.on("close", (code) => {
+			if (code === 0) resolve({ stdout, stderr });
+			else
+				reject(
+					new Error(
+						`${cmd} ${cmdArgs.join(" ")} exited ${code}\n${stderr || stdout}`,
+					),
+				);
+		});
+	});
+}
+
+async function requireDocker() {
+	for (const bin of ["docker"]) {
+		try {
+			await run(bin, ["--version"]);
+		} catch {
+			console.error(`✗ ${bin} not found`);
+			console.error(
+				"Install Docker: https://docs.searxng.org/admin/installation-docker.html",
+			);
+			process.exit(1);
+		}
+	}
+	try {
+		await run("docker", ["compose", "version"]);
+	} catch {
+		console.error("✗ docker compose not available");
+		console.error(
+			"Install Docker Compose v2: https://docs.docker.com/compose/install/",
+		);
+		process.exit(1);
+	}
+}
+
+async function curlToFile(url, dest) {
+	await run("curl", ["-fsSL", "-o", dest, url]);
+}
+
+async function readComposePort() {
+	if (!(await exists(ENV_COMPOSE))) return DEFAULT_PORT;
+	const text = await readFile(ENV_COMPOSE, "utf8");
+	for (const line of text.split("\n")) {
+		const m = line.match(/^SEARXNG_PORT=(.+)$/);
+		if (m) return m[1].trim().replace(/^["']|["']$/g, "") || DEFAULT_PORT;
+	}
+	return DEFAULT_PORT;
+}
+
+async function ensureSearxngLayout() {
+	await mkdir(CORE_CONFIG, { recursive: true });
+	if (!(await exists(COMPOSE_PATH))) {
+		console.log("Fetching SearXNG docker-compose.yml …");
+		await curlToFile(`${SEARXNG_BASE}/docker-compose.yml`, COMPOSE_PATH);
+	}
+	if (!(await exists(ENV_COMPOSE))) {
+		const example = join(SEARXNG_DIR, ".env.example");
+		if (!(await exists(example))) {
+			console.log("Fetching SearXNG .env.example …");
+			await curlToFile(`${SEARXNG_BASE}/.env.example`, example);
+		}
+		await copyFile(example, ENV_COMPOSE);
+	}
+	const needsSettings =
+		!(await exists(SETTINGS_PATH)) ||
+		!(await readFile(SETTINGS_PATH, "utf8")).includes("json");
+	if (needsSettings) {
+		await writeFile(SETTINGS_PATH, HARNESS_SETTINGS, "utf8");
+		console.log(`✓ Wrote ${SETTINGS_PATH} (json format, limiter off)`);
+	}
+}
+
+async function composeUp() {
+	console.log("Starting SearXNG (docker compose up -d) …");
+	await run("docker", ["compose", "up", "-d"], { cwd: SEARXNG_DIR, inherit: true });
+}
+
+async function waitForHealth(baseUrl) {
+	const url = `${baseUrl}${HEALTH_PATH}`;
+	const deadline = Date.now() + 90_000;
+	let lastErr = "";
+	while (Date.now() < deadline) {
+		try {
+			const res = await fetch(url, {
+				headers: { Accept: "application/json" },
+				signal: AbortSignal.timeout(10_000),
+			});
+			if (res.status === 403) {
+				throw new Error(
+					"SearXNG returned 403 for format=json — ensure search.formats includes json in .searxng/core-config/settings.yml",
+				);
+			}
+			if (res.ok) {
+				const data = await res.json();
+				if (data && typeof data === "object") {
+					console.log(`✓ SearXNG healthy at ${baseUrl}`);
+					return;
+				}
+			}
+			lastErr = `HTTP ${res.status}`;
+		} catch (err) {
+			lastErr = err instanceof Error ? err.message : String(err);
+		}
+		await new Promise((r) => setTimeout(r, 3000));
+	}
+	throw new Error(`SearXNG health check timed out (${url}): ${lastErr}`);
+}
+
+function upsertEnvKey(content, key, value) {
+	const line = `${key}=${value}`;
+	const re = new RegExp(`^${key}=.*$`, "m");
+	if (re.test(content)) {
+		return content.replace(re, line);
+	}
+	if (content.includes(MANAGED_START) && content.includes(MANAGED_END)) {
+		const end = content.indexOf(MANAGED_END);
+		return `${content.slice(0, end)}${line}\n${content.slice(end)}`;
+	}
+	const sep = content.endsWith("\n") || content.length === 0 ? "" : "\n";
+	return `${content}${sep}${MANAGED_START}\n# harness-web (SearXNG)\n${line}\n${MANAGED_END}\n`;
+}
+
+async function upsertHarnessEnv(baseUrl) {
+	const envPath = join(PROJECT_ROOT, ".env");
+	let content = "";
+	if (await exists(envPath)) {
+		content = await readFile(envPath, "utf8");
+	} else {
+		const template = join(UP_PKG, ".pi", "harness", "env.harness.template");
+		if (await exists(template)) {
+			content = await readFile(template, "utf8");
+		}
+	}
+	content = upsertEnvKey(content, "HARNESS_WEB_SEARCH_ENGINE", "searxng");
+	content = upsertEnvKey(content, "HARNESS_WEB_SEARXNG_URL", baseUrl);
+	await writeFile(envPath, content.endsWith("\n") ? content : `${content}\n`, "utf8");
+	console.log(`✓ Updated .env: HARNESS_WEB_SEARCH_ENGINE=searxng, HARNESS_WEB_SEARXNG_URL=${baseUrl}`);
+}
+
+function normalizeBaseUrl(raw) {
+	const url = raw.trim().replace(/\/+$/, "");
+	if (!/^https?:\/\//i.test(url)) {
+		throw new Error(`Invalid SearXNG URL: ${raw}`);
+	}
+	return url;
+}
+
+async function main() {
+	if (setUrl) {
+		const baseUrl = normalizeBaseUrl(setUrl);
+		await waitForHealth(baseUrl);
+		await upsertHarnessEnv(baseUrl);
+		process.exit(0);
+	}
+
+	if (urlOnly) {
+		const port = (await exists(ENV_COMPOSE)) ? await readComposePort() : DEFAULT_PORT;
+		console.log(`http://127.0.0.1:${port}`);
+		process.exit(0);
+	}
+
+	await requireDocker();
+	await ensureSearxngLayout();
+	const port = await readComposePort();
+	const baseUrl = `http://127.0.0.1:${port}`;
+	await composeUp();
+	await waitForHealth(baseUrl);
+	await upsertHarnessEnv(baseUrl);
+
+	console.log("");
+	console.log("SearXNG is ready for harness-web:");
+	console.log(`  HARNESS_WEB_SEARXNG_URL=${baseUrl}`);
+	console.log(`  Test: python3 "${join(UP_PKG, ".pi/scripts/harness-web.py")}" search "test" -o .web/search.json --limit 2`);
+}
+
+main().catch((err) => {
+	console.error(`✗ ${err.message || err}`);
+	process.exit(1);
+});

package/.pi/scripts/harness-web-search.md

@@ -1,12 +1,21 @@
 # harness-web search (internal)
 
-## Engine
+Routing: `harness_web/search.py` dispatches by `HARNESS_WEB_SEARCH_ENGINE`.
 
-Default: DuckDuckGo static HTML — `GET https://html.duckduckgo.com/html/?q=…`
+## Engines
 
-Implemented in `harness_web/search_ddg.py` via `Fetcher.get` (HTTP, not a browser per query).
+| Value | Module | Notes |
+|-------|--------|-------|
+| `ddg_html` (default) | `search_ddg.py` | DuckDuckGo HTML SERP via Scrapling HTTP (+ one stealth retry on challenge) |
+| `searxng` | `search_searxng.py` | Self-hosted JSON API — requires `HARNESS_WEB_SEARXNG_URL` |
 
-## Selectors
+Bootstrap local SearXNG: `node "$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs"`
+
+## DuckDuckGo HTML (`ddg_html`)
+
+`GET https://html.duckduckgo.com/html/?q=…`
+
+### Selectors
 
 | Field | CSS |
 |-------|-----|
@@ -16,10 +25,18 @@ Implemented in `harness_web/search_ddg.py` via `Fetcher.get` (HTTP, not a browse
 
 DDG redirect URLs (`//duckduckgo.com/l/?uddg=…`) are unwrapped to the target `uddg` parameter.
 
-## Challenge detection
+### Challenge detection
 
 If status 403 or HTML contains challenge markers (`anomaly-modal`, etc.), retry **once** with `StealthyFetcher`, then exit with a clear “search engine blocked” message.
 
+## SearXNG (`searxng`)
+
+`GET {HARNESS_WEB_SEARXNG_URL}/search?q=…&format=json&pageno=1`
+
+- No client API token (SearXNG has no standard search API key).
+- `search.formats` in instance `settings.yml` must include `json` or the API returns **403**.
+- Public instances are unsuitable (~4 JSON req/hr when limiter on; JSON often disabled). Use self-hosted bootstrap.
+
 ## Output
 
 `.web/search.json` — envelope compatible with legacy Firecrawl skills:
@@ -31,3 +48,5 @@ If status 403 or HTML contains challenge markers (`anomaly-modal`, etc.), retry
   "data": { "web": [{ "url", "title", "description" }] }
 }
 ```
+
+`engine` reflects the active backend (`ddg_html` or `searxng`).

package/.pi/scripts/harness-web.py

@@ -36,7 +36,7 @@ if str(SCRIPT_DIR) not in sys.path:
 from harness_web.config import HarnessWebConfig, load_config  # noqa: E402
 from harness_web.output import write_search_results  # noqa: E402
 from harness_web.scrape import bulk_scrape, map_url, scrape_url  # noqa: E402
-from harness_web.search_ddg import search_ddg  # noqa: E402
+from harness_web.search import search  # noqa: E402
 
 DEFAULT_WEB_DIR = ".web"
 
@@ -47,8 +47,8 @@ def _default_out(sub: str) -> Path:
 
 def cmd_search(args: argparse.Namespace, config: HarnessWebConfig) -> int:
     out = Path(args.output or _default_out("search.json"))
-    results = search_ddg(args.query, limit=args.limit, config=config)
-    write_search_results(out, results, args.query)
+    results = search(args.query, limit=args.limit, config=config)
+    write_search_results(out, results, args.query, engine=config.search_engine)
     print(f"wrote {out} ({len(results)} results)")
     return 0
 
@@ -76,6 +76,20 @@ def cmd_map(args: argparse.Namespace, config: HarnessWebConfig) -> int:
     return 0
 
 
+def cmd_status(_args: argparse.Namespace, config: HarnessWebConfig) -> int:
+    import json
+
+    payload = {
+        "search_engine": config.search_engine,
+        "searxng_url": config.searxng_url,
+        "fetch_mode": config.fetch_mode,
+        "script": str(Path(__file__).resolve()),
+        "bootstrap": "ok",
+    }
+    print(json.dumps(payload, indent=2))
+    return 0
+
+
 def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
     sleep_sec = args.sleep if args.sleep is not None else config.rate_limit_ms / 1000.0
     if args.urls:
@@ -86,8 +100,8 @@ def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
         data = json.loads(Path(args.from_search).read_text(encoding="utf-8"))
         urls = [item["url"] for item in data.get("data", {}).get("web", []) if item.get("url")]
     else:
-        urls = search_ddg(args.query, limit=args.limit, config=config)
-        urls = [r["url"] for r in urls]
+        serp = search(args.query, limit=args.limit, config=config)
+        urls = [r["url"] for r in serp]
 
     if not urls:
         print("bulk-scrape: no URLs to fetch", file=sys.stderr)
@@ -111,11 +125,11 @@ def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
 def build_parser() -> argparse.ArgumentParser:
     p = argparse.ArgumentParser(
         prog="harness-web",
-        description="Harness web layer: search (DDG HTML) and scrape (Scrapling).",
+        description="Harness web layer: search (DDG HTML or SearXNG) and scrape (Scrapling).",
     )
     sub = p.add_subparsers(dest="command", required=True)
 
-    ps = sub.add_parser("search", help="Search via DuckDuckGo HTML SERP")
+    ps = sub.add_parser("search", help="Search via configured SERP (HARNESS_WEB_SEARCH_ENGINE)")
     ps.add_argument("query", help="Search query")
     ps.add_argument("-o", "--output", help="JSON output path (default: .web/search.json)")
     ps.add_argument("--limit", type=int, default=5)
@@ -160,6 +174,9 @@ def build_parser() -> argparse.ArgumentParser:
     pm.add_argument("--fast", action="store_true")
     pm.set_defaults(func=cmd_map)
 
+    pst = sub.add_parser("status", help="Print harness-web config as JSON (setup/diagnostics)")
+    pst.set_defaults(func=cmd_status)
+
     return p