ultimate-pi 0.3.0 → 0.4.0

package/.pi/scripts/harness-sentrux-bootstrap.mjs

@@ -0,0 +1,146 @@
+#!/usr/bin/env node
+/**
+ * Idempotent Sentrux rules bootstrap for harness projects.
+ *
+ * 1. Seeds `.pi/harness/sentrux/architecture.manifest.json` from the package template when missing
+ * 2. Personalizes `project` on first seed from target package.json / directory name
+ * 3. Runs `sentrux-rules-sync.mjs` (merge-safe; preserves custom TOML outside managed markers)
+ *
+ * Usage:
+ *   node "$UP_PKG/.pi/scripts/harness-sentrux-bootstrap.mjs [PROJECT_ROOT] [--force] [--check]
+ *
+ * Bootstrap vs force:
+ *   - Default (harness-setup): no --force — skips write when manifest hash unchanged
+ *   - After editing architecture.manifest.json: `--force` or `/harness-sentrux-sync`
+ */
+
+import { readFile, writeFile, mkdir, access, copyFile } from "node:fs/promises";
+import { constants } from "node:fs";
+import { join, dirname, basename } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn } from "node:child_process";
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const UP_PKG = join(SCRIPT_DIR, "..", "..");
+
+const args = process.argv.slice(2).filter((a) => !a.startsWith("-"));
+const flags = process.argv.slice(2).filter((a) => a.startsWith("-"));
+const force = flags.includes("--force");
+const checkOnly = flags.includes("--check");
+
+const PROJECT_ROOT = args[0] || process.cwd();
+const MANIFEST = join(
+	PROJECT_ROOT,
+	".pi",
+	"harness",
+	"sentrux",
+	"architecture.manifest.json",
+);
+const MANIFEST_TEMPLATE = join(
+	UP_PKG,
+	".pi",
+	"harness",
+	"sentrux",
+	"architecture.manifest.json",
+);
+const SYNC_SCRIPT = join(SCRIPT_DIR, "sentrux-rules-sync.mjs");
+
+async function fileExists(path) {
+	try {
+		await access(path, constants.R_OK);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+async function resolveProjectName(root) {
+	const pkgPath = join(root, "package.json");
+	if (await fileExists(pkgPath)) {
+		try {
+			const pkg = JSON.parse(await readFile(pkgPath, "utf-8"));
+			if (typeof pkg.name === "string" && pkg.name.trim()) {
+				return pkg.name.trim();
+			}
+		} catch {
+			/* ignore */
+		}
+	}
+	return basename(root) || "project";
+}
+
+async function seedManifestIfMissing() {
+	if (await fileExists(MANIFEST)) {
+		return { seeded: false };
+	}
+	if (!(await fileExists(MANIFEST_TEMPLATE))) {
+		console.error(
+			"harness-sentrux-bootstrap: missing package template",
+			MANIFEST_TEMPLATE,
+		);
+		process.exit(1);
+	}
+	await mkdir(dirname(MANIFEST), { recursive: true });
+	await copyFile(MANIFEST_TEMPLATE, MANIFEST);
+	const projectName = await resolveProjectName(PROJECT_ROOT);
+	const manifest = JSON.parse(await readFile(MANIFEST, "utf-8"));
+	manifest.project = projectName;
+	await writeFile(MANIFEST, `${JSON.stringify(manifest, null, 2)}\n`, "utf-8");
+	console.log(
+		`harness-sentrux-bootstrap: seeded manifest -> ${MANIFEST} (project: ${projectName})`,
+	);
+	return { seeded: true, projectName };
+}
+
+function runSync(extraArgs) {
+	return new Promise((resolve) => {
+		const child = spawn(
+			process.execPath,
+			[SYNC_SCRIPT, ...extraArgs, PROJECT_ROOT],
+			{
+				cwd: PROJECT_ROOT,
+				stdio: ["ignore", "pipe", "pipe"],
+				env: process.env,
+			},
+		);
+		let out = "";
+		child.stdout?.on("data", (d) => {
+			out += d.toString();
+		});
+		child.stderr?.on("data", (d) => {
+			out += d.toString();
+		});
+		child.on("close", (code) => resolve({ code: code ?? 1, out: out.trim() }));
+		child.on("error", (err) =>
+			resolve({ code: 1, out: String(err.message) }),
+		);
+	});
+}
+
+async function main() {
+	const { seeded } = await seedManifestIfMissing();
+	if (!seeded) {
+		console.log(
+			"harness-sentrux-bootstrap: manifest present (edit layers/boundaries there, then re-run with --force)",
+		);
+	}
+
+	const syncArgs = [];
+	if (checkOnly) syncArgs.push("--check");
+	else if (force) syncArgs.push("--force");
+
+	const { code, out } = await runSync(syncArgs);
+	if (out) console.log(out);
+	if (code !== 0) process.exit(code);
+
+	if (!checkOnly && !force) {
+		console.log(
+			"harness-sentrux-bootstrap: done (idempotent). After manifest edits: node \"$UP_PKG/.pi/scripts/harness-sentrux-bootstrap.mjs\" --force",
+		);
+	}
+}
+
+main().catch((err) => {
+	console.error(err);
+	process.exit(1);
+});

package/.pi/scripts/harness-sync-env.mjs

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+/**
+ * Sync project-root `.env` with harness-required keys (non-destructive).
+ *
+ * - Never overwrites existing keys or values.
+ * - If `.env` is missing: exit 2 and print instructions (use --create-missing after user confirms).
+ * - If `.env` exists: append only missing keys inside a managed block at EOF.
+ *
+ * Usage:
+ *   node harness-sync-env.mjs [--create-missing] [--dry-run]
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+const UP_PKG = join(SCRIPT_DIR, "..", "..");
+const TEMPLATE_PATH = join(UP_PKG, ".pi", "harness", "env.harness.template");
+const ENV_PATH = join(process.cwd(), ".env");
+
+const MANAGED_START = "# --- harness:env:start ---";
+const MANAGED_END = "# --- harness:env:end ---";
+
+function parseEnvKeys(content) {
+	const keys = new Set();
+	for (const raw of content.split("\n")) {
+		const line = raw.trim();
+		if (!line || line.startsWith("#")) continue;
+		const eq = line.indexOf("=");
+		if (eq === -1) continue;
+		const key = line.slice(0, eq).trim();
+		if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) keys.add(key);
+	}
+	return keys;
+}
+
+/** @returns {{ key: string, line: string }[]} */
+function parseTemplateEntries(templateText) {
+	const entries = [];
+	for (const raw of templateText.split("\n")) {
+		const trimmed = raw.trim();
+		if (!trimmed || trimmed.startsWith("#")) {
+			if (trimmed) entries.push({ key: null, line: raw });
+			continue;
+		}
+		const eq = trimmed.indexOf("=");
+		if (eq === -1) continue;
+		const key = trimmed.slice(0, eq).trim();
+		if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) continue;
+		entries.push({ key, line: raw });
+	}
+	return entries;
+}
+
+function buildManagedBlock(missingEntries) {
+	const lines = [
+		"",
+		MANAGED_START,
+		"# Added by /harness-setup — edit values; existing keys elsewhere in .env are never changed.",
+	];
+	for (const entry of missingEntries) {
+		lines.push(entry.line);
+	}
+	lines.push(MANAGED_END);
+	lines.push("");
+	return lines.join("\n");
+}
+
+function insertIntoManagedBlock(content, missingEntries) {
+	const block = buildManagedBlock(missingEntries).trimEnd();
+	if (!content.includes(MANAGED_START)) {
+		const sep = content.endsWith("\n") || content.length === 0 ? "" : "\n";
+		return content + sep + block + "\n";
+	}
+	const start = content.indexOf(MANAGED_START);
+	const end = content.indexOf(MANAGED_END);
+	if (end === -1 || end < start) {
+		const sep = content.endsWith("\n") ? "" : "\n";
+		return content + sep + block + "\n";
+	}
+	const before = content.slice(0, end + MANAGED_END.length);
+	const after = content.slice(end + MANAGED_END.length);
+	const additions = missingEntries.map((e) => e.line).join("\n");
+	return `${before}\n${additions}${after}`;
+}
+
+function main() {
+	const createMissing = process.argv.includes("--create-missing");
+	const dryRun = process.argv.includes("--dry-run");
+
+	if (!existsSync(TEMPLATE_PATH)) {
+		console.error(`harness-sync-env: missing template ${TEMPLATE_PATH}`);
+		process.exit(1);
+	}
+
+	const templateText = readFileSync(TEMPLATE_PATH, "utf8");
+	const templateEntries = parseTemplateEntries(templateText);
+	const templateKeys = templateEntries.filter((e) => e.key);
+
+	if (!existsSync(ENV_PATH)) {
+		if (createMissing) {
+			const body = `${templateText.trimEnd()}\n`;
+			if (dryRun) {
+				console.log("[dry-run] would create .env from harness template");
+				process.stdout.write(body);
+				process.exit(0);
+			}
+			writeFileSync(ENV_PATH, body, "utf8");
+			console.log("✓ Created .env from harness template (edit secrets locally)");
+			process.exit(0);
+		}
+		console.log("✗ No .env at project root");
+		console.log("");
+		console.log("Create one, then re-run harness env sync:");
+		console.log(`  cp "${TEMPLATE_PATH}" .env`);
+		console.log("  # edit .env with your API keys");
+		console.log(`  node "${join(UP_PKG, ".pi", "scripts", "harness-sync-env.mjs")}"`);
+		console.log("");
+		console.log("Or, after user confirms:");
+		console.log(
+			`  node "${join(UP_PKG, ".pi", "scripts", "harness-sync-env.mjs")}" --create-missing`,
+		);
+		process.exit(2);
+	}
+
+	const existing = readFileSync(ENV_PATH, "utf8");
+	const existingKeys = parseEnvKeys(existing);
+	const missing = templateKeys.filter((e) => !existingKeys.has(e.key));
+
+	if (missing.length === 0) {
+		console.log("✓ .env contains all harness template keys — no changes");
+		process.exit(0);
+	}
+
+	const next = insertIntoManagedBlock(existing, missing);
+	if (dryRun) {
+		console.log(`[dry-run] would append ${missing.length} key(s): ${missing.map((m) => m.key).join(", ")}`);
+		process.exit(0);
+	}
+
+	writeFileSync(ENV_PATH, next, "utf8");
+	console.log(
+		`✓ Appended ${missing.length} harness env key(s) to .env (existing values preserved): ${missing.map((m) => m.key).join(", ")}`,
+	);
+}
+
+main();

package/.pi/scripts/harness-verify.mjs

@@ -40,8 +40,11 @@ const REQUIRED_EXTENSIONS = [
 	"observation-bus.ts",
 	"drift-monitor.ts",
 	"sentrux-rules-sync.ts",
+	"harness-subagents.ts",
 ];
 
+const AGENTS_MANIFEST = join(ROOT, ".pi", "harness", "agents.manifest.json");
+
 const SENTRUX_MANIFEST = join(
 	ROOT,
 	".pi",
@@ -203,6 +206,22 @@ async function main() {
 
 	await checkSentruxGate();
 
+	if (!(await fileExists(AGENTS_MANIFEST))) {
+		fail(
+			"missing .pi/harness/agents.manifest.json — run node \"$UP_PKG/.pi/scripts/harness-agents-manifest.mjs\" --write",
+		);
+	}
+	ok("agents.manifest.json present");
+
+	const { code: manifestCode, out: manifestOut } = await runNodeScript(
+		join(ROOT, ".pi", "scripts", "harness-agents-manifest.mjs"),
+		["--check"],
+	);
+	if (manifestCode !== 0) {
+		fail(manifestOut.trim() || "agents.manifest.json drift — regenerate with --write");
+	}
+	ok("agents.manifest.json in sync");
+
 	console.log("\nharness:verify PASS");
 }
 

package/.pi/scripts/harness-web-search.md

@@ -0,0 +1,33 @@
+# harness-web search (internal)
+
+## Engine
+
+Default: DuckDuckGo static HTML — `GET https://html.duckduckgo.com/html/?q=…`
+
+Implemented in `harness_web/search_ddg.py` via `Fetcher.get` (HTTP, not a browser per query).
+
+## Selectors
+
+| Field | CSS |
+|-------|-----|
+| Result block | `.result` |
+| Title + link | `.result__a` |
+| Snippet | `.result__snippet` |
+
+DDG redirect URLs (`//duckduckgo.com/l/?uddg=…`) are unwrapped to the target `uddg` parameter.
+
+## Challenge detection
+
+If status 403 or HTML contains challenge markers (`anomaly-modal`, etc.), retry **once** with `StealthyFetcher`, then exit with a clear “search engine blocked” message.
+
+## Output
+
+`.web/search.json` — envelope compatible with legacy Firecrawl skills:
+
+```json
+{
+  "query": "...",
+  "engine": "ddg_html",
+  "data": { "web": [{ "url", "title", "description" }] }
+}
+```

package/.pi/scripts/harness-web.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""harness-web — Scrapling-backed web search and scrape for ultimate-pi harness agents."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shutil
+import sys
+import time
+from pathlib import Path
+
+# Re-exec with scrapling's uv-tool Python when the library is not on default python3.
+def _bootstrap_scrapling() -> None:
+    try:
+        import scrapling  # noqa: F401
+    except ImportError:
+        scrapling_bin = shutil.which("scrapling")
+        if not scrapling_bin:
+            sys.exit(
+                'scrapling not installed. Run: uv tool install "scrapling[fetchers]" && scrapling install'
+            )
+        with open(scrapling_bin, encoding="utf-8") as fh:
+            shebang = fh.readline().strip()
+        if shebang.startswith("#!"):
+            os.execv(shebang[2:], [shebang[2:], *sys.argv])
+
+
+_bootstrap_scrapling()
+
+# Imports after bootstrap (scrapling must be available).
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from harness_web.config import HarnessWebConfig, load_config  # noqa: E402
+from harness_web.output import write_search_results  # noqa: E402
+from harness_web.scrape import bulk_scrape, map_url, scrape_url  # noqa: E402
+from harness_web.search_ddg import search_ddg  # noqa: E402
+
+DEFAULT_WEB_DIR = ".web"
+
+
+def _default_out(sub: str) -> Path:
+    return Path(DEFAULT_WEB_DIR) / sub
+
+
+def cmd_search(args: argparse.Namespace, config: HarnessWebConfig) -> int:
+    out = Path(args.output or _default_out("search.json"))
+    results = search_ddg(args.query, limit=args.limit, config=config)
+    write_search_results(out, results, args.query)
+    print(f"wrote {out} ({len(results)} results)")
+    return 0
+
+
+def cmd_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
+    out = Path(args.output or _default_out("page.md"))
+    fast = config.use_fast_for_url(args.url, args.fast)
+    scrape_url(
+        args.url,
+        str(out),
+        config=config,
+        fast=fast,
+        wait_ms=args.wait_for,
+    )
+    mode = "fast" if fast else "stealth"
+    print(f"wrote {out} ({mode})")
+    return 0
+
+
+def cmd_map(args: argparse.Namespace, config: HarnessWebConfig) -> int:
+    out = Path(args.output or _default_out("map.json"))
+    fast = config.use_fast_for_url(args.url, args.fast)
+    map_url(args.url, str(out), config=config, fast=fast, limit=args.limit)
+    print(f"wrote {out}")
+    return 0
+
+
+def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
+    sleep_sec = args.sleep if args.sleep is not None else config.rate_limit_ms / 1000.0
+    if args.urls:
+        urls = list(args.urls)
+    elif args.from_search:
+        import json
+
+        data = json.loads(Path(args.from_search).read_text(encoding="utf-8"))
+        urls = [item["url"] for item in data.get("data", {}).get("web", []) if item.get("url")]
+    else:
+        urls = search_ddg(args.query, limit=args.limit, config=config)
+        urls = [r["url"] for r in urls]
+
+    if not urls:
+        print("bulk-scrape: no URLs to fetch", file=sys.stderr)
+        return 1
+
+    out_dir = Path(args.output or _default_out("bulk"))
+    fast = args.fast or config.fetch_mode == "fast"
+    failures = bulk_scrape(
+        urls[: args.limit],
+        str(out_dir),
+        config=config,
+        fast=fast,
+        sleep_sec=sleep_sec,
+    )
+    print(f"wrote {len(urls[: args.limit]) - len(failures)} pages to {out_dir}")
+    for fail in failures:
+        print(f"  failed: {fail}", file=sys.stderr)
+    return 1 if failures and len(failures) == len(urls) else 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="harness-web",
+        description="Harness web layer: search (DDG HTML) and scrape (Scrapling).",
+    )
+    sub = p.add_subparsers(dest="command", required=True)
+
+    ps = sub.add_parser("search", help="Search via DuckDuckGo HTML SERP")
+    ps.add_argument("query", help="Search query")
+    ps.add_argument("-o", "--output", help="JSON output path (default: .web/search.json)")
+    ps.add_argument("--limit", type=int, default=5)
+    ps.set_defaults(func=cmd_search)
+
+    pc = sub.add_parser("scrape", help="Scrape a URL to markdown")
+    pc.add_argument("url")
+    pc.add_argument("-o", "--output", help="Markdown output (default: .web/page.md)")
+    pc.add_argument(
+        "--fast",
+        action="store_true",
+        help="HTTP get + ai-targeted extract (skip stealth browser)",
+    )
+    pc.add_argument(
+        "--wait-for",
+        type=int,
+        default=None,
+        metavar="MS",
+        help="Extra wait after load (stealth mode, milliseconds)",
+    )
+    pc.set_defaults(func=cmd_scrape)
+
+    pb = sub.add_parser("bulk-scrape", help="Search then scrape multiple URLs")
+    pb.add_argument("query", nargs="?", help="Search query when not using --from-search")
+    pb.add_argument("-o", "--output", help="Output directory (default: .web/bulk)")
+    pb.add_argument("--limit", type=int, default=3, help="Max URLs to scrape")
+    pb.add_argument("--from-search", metavar="JSON", help="Use URLs from search JSON file")
+    pb.add_argument("urls", nargs="*", help="Explicit URLs to scrape")
+    pb.add_argument("--fast", action="store_true")
+    pb.add_argument(
+        "--sleep",
+        type=float,
+        default=None,
+        help="Seconds between scrapes (default: HARNESS_WEB_RATE_LIMIT_MS)",
+    )
+    pb.set_defaults(func=cmd_bulk_scrape)
+
+    pm = sub.add_parser("map", help="List same-host links from a page")
+    pm.add_argument("url")
+    pm.add_argument("-o", "--output", help="JSON output (default: .web/map.json)")
+    pm.add_argument("--limit", type=int, default=100)
+    pm.add_argument("--fast", action="store_true")
+    pm.set_defaults(func=cmd_map)
+
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    config = load_config()
+    t0 = time.monotonic()
+    rc = args.func(args, config)
+    _ = time.monotonic() - t0
+    return rc
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/.pi/scripts/harness_web/__init__.py

@@ -0,0 +1 @@
+"""Harness web facade — Scrapling-backed search and scrape for ultimate-pi agents."""

package/.pi/scripts/harness_web/config.py

@@ -0,0 +1,80 @@
+"""Environment and defaults for harness-web."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from urllib.parse import urlparse
+
+
+def _int_env(name: str, default: int) -> int:
+    raw = os.environ.get(name, "").strip()
+    if not raw:
+        return default
+    try:
+        return int(raw)
+    except ValueError:
+        return default
+
+
+def _fetch_mode() -> str:
+    mode = os.environ.get("HARNESS_WEB_FETCH_MODE", "stealth").strip().lower()
+    if mode in ("stealth", "fast", "auto"):
+        return mode
+    return "stealth"
+
+
+_STATIC_HOSTS = frozenset(
+    {
+        "example.com",
+        "www.example.com",
+        "localhost",
+        "127.0.0.1",
+    }
+)
+
+
+def host_is_static(url: str) -> bool:
+    try:
+        host = (urlparse(url).hostname or "").lower()
+    except ValueError:
+        return False
+    if host in _STATIC_HOSTS:
+        return True
+    if host.endswith(".localhost"):
+        return True
+    return False
+
+
+@dataclass(frozen=True)
+class HarnessWebConfig:
+    fetch_mode: str
+    search_engine: str
+    proxy: str | None
+    rate_limit_ms: int
+    timeout_ms: int
+
+    @property
+    def timeout_sec(self) -> int:
+        return max(1, self.timeout_ms // 1000)
+
+    def use_fast_for_url(self, url: str, cli_fast: bool) -> bool:
+        if cli_fast:
+            return True
+        if self.fetch_mode == "fast":
+            return True
+        if self.fetch_mode == "auto" and host_is_static(url):
+            return True
+        return False
+
+
+def load_config() -> HarnessWebConfig:
+    proxy = os.environ.get("HARNESS_WEB_PROXY", "").strip() or None
+    return HarnessWebConfig(
+        fetch_mode=_fetch_mode(),
+        search_engine=os.environ.get("HARNESS_WEB_SEARCH_ENGINE", "ddg_html").strip()
+        or "ddg_html",
+        proxy=proxy,
+        rate_limit_ms=_int_env("HARNESS_WEB_RATE_LIMIT_MS", 2000),
+        timeout_ms=_int_env("HARNESS_WEB_TIMEOUT_MS", 30000),
+    )

package/.pi/scripts/harness_web/output.py

@@ -0,0 +1,55 @@
+"""Write harness-web artifacts (JSON + markdown)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from scrapling.core.shell import Convertor
+
+
+def ensure_parent(path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def write_json(path: Path, payload: Any) -> None:
+    ensure_parent(path)
+    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def write_search_results(path: Path, results: list[dict[str, str]], query: str) -> None:
+    """Firecrawl-compatible envelope: data.web[].url|title|description."""
+    write_json(
+        path,
+        {
+            "query": query,
+            "engine": "ddg_html",
+            "data": {
+                "web": [
+                    {
+                        "url": r["url"],
+                        "title": r.get("title", ""),
+                        "description": r.get("description", ""),
+                    }
+                    for r in results
+                ]
+            },
+        },
+    )
+
+
+def write_page_markdown(path: Path, page: Any, *, main_content_only: bool = True) -> None:
+    ensure_parent(path)
+    try:
+        Convertor.write_content_to_file(
+            page,
+            str(path.resolve()),
+            css_selector=None,
+            main_content_only=main_content_only,
+        )
+    except ModuleNotFoundError as err:
+        if "markdownify" not in str(err):
+            raise
+        text = page.get_all_text(strip=True) if hasattr(page, "get_all_text") else str(page)
+        path.write_text(text + "\n", encoding="utf-8")