ultimate-pi 0.19.0 → 0.20.0

package/.pi/lib/harness-web/cache.ts

@@ -0,0 +1,369 @@
+/**
+ * WRS local cache — pooled `.web/cache/` entries with freshness + search context.
+ * Workspace aliases (`.web/search-deep.json`, …) are copies/links for agent ergonomics.
+ */
+
+import {
+	copyFileSync,
+	existsSync,
+	mkdirSync,
+	readFileSync,
+	statSync,
+	writeFileSync,
+} from "node:fs";
+import { createHash } from "node:crypto";
+import { dirname, resolve } from "node:path";
+
+export const WEB_ROOT = ".web";
+export const WEB_CACHE_ROOT = `${WEB_ROOT}/cache`;
+
+export type WebCacheKind =
+	| "search"
+	| "search-deep"
+	| "fetch-page"
+	| "fetch-map"
+	| "fetch-highlights";
+
+export interface WebCacheMeta {
+	version: 1;
+	kind: WebCacheKind;
+	cacheKey: string;
+	createdAt: string;
+	expiresAt: string;
+	ttlSeconds: number;
+	context: Record<string, unknown>;
+	artifact: string;
+	hitCount: number;
+}
+
+export interface SearchCacheContext {
+	query: string;
+	tier: string;
+	engine: string;
+	limit: number;
+	category?: string;
+	expandHeuristic?: boolean;
+	anglesFingerprint?: string;
+}
+
+export interface FetchCacheContext {
+	url: string;
+	mode: "scrape" | "map";
+	fast: boolean;
+	highlightQuery?: string;
+	highlights: boolean;
+}
+
+export interface CacheLookupResult {
+	hit: boolean;
+	stale: boolean;
+	cacheKey: string;
+	entryDir: string;
+	artifactPath: string;
+	metaPath: string;
+	meta?: WebCacheMeta;
+	ageMs?: number;
+}
+
+function shaKey(payload: unknown): string {
+	return createHash("sha256")
+		.update(JSON.stringify(payload))
+		.digest("hex")
+		.slice(0, 20);
+}
+
+export function cacheEnabled(): boolean {
+	const raw = process.env.HARNESS_WEB_CACHE?.trim();
+	if (raw === "0" || raw?.toLowerCase() === "false") return false;
+	return true;
+}
+
+export function defaultCacheTtlSeconds(): number {
+	const raw = process.env.HARNESS_WEB_CACHE_TTL_SEC?.trim();
+	if (raw) {
+		const n = Number.parseInt(raw, 10);
+		if (Number.isFinite(n) && n > 0) return n;
+	}
+	return 86_400; // 24h
+}
+
+export function fingerprintFile(projectRoot: string, relPath: string): string | undefined {
+	const full = resolve(projectRoot, relPath);
+	if (!existsSync(full)) return undefined;
+	const text = readFileSync(full, "utf-8");
+	return createHash("sha256").update(text).digest("hex").slice(0, 16);
+}
+
+export function searchCacheKey(ctx: SearchCacheContext): string {
+	return shaKey({
+		v: 1,
+		kind: ctx.tier === "deep" || ctx.tier === "research" ? "search-deep" : "search",
+		query: ctx.query.trim().toLowerCase(),
+		tier: ctx.tier,
+		engine: ctx.engine,
+		limit: ctx.limit,
+		category: ctx.category?.trim().toLowerCase() || null,
+		expandHeuristic: Boolean(ctx.expandHeuristic),
+		angles: ctx.anglesFingerprint || null,
+	});
+}
+
+export function fetchCacheKey(ctx: FetchCacheContext): string {
+	return shaKey({
+		v: 1,
+		kind: ctx.highlights ? "fetch-highlights" : ctx.mode === "map" ? "fetch-map" : "fetch-page",
+		url: ctx.url.trim(),
+		mode: ctx.mode,
+		fast: ctx.fast,
+		highlightQuery: ctx.highlightQuery?.trim() || null,
+	});
+}
+
+function cacheKindFromTier(tier: string): WebCacheKind {
+	return tier === "deep" || tier === "research" ? "search-deep" : "search";
+}
+
+function entryDir(projectRoot: string, kind: WebCacheKind, cacheKey: string): string {
+	return `${WEB_CACHE_ROOT}/${kind}/${cacheKey}`;
+}
+
+function readMeta(metaPath: string): WebCacheMeta | undefined {
+	if (!existsSync(metaPath)) return undefined;
+	try {
+		const data = JSON.parse(readFileSync(metaPath, "utf-8")) as WebCacheMeta;
+		if (data?.version !== 1) return undefined;
+		return data;
+	} catch {
+		return undefined;
+	}
+}
+
+export function isCacheFresh(
+	meta: WebCacheMeta,
+	options?: { maxAgeSec?: number; nowMs?: number },
+): boolean {
+	const now = options?.nowMs ?? Date.now();
+	const expiresAt = Date.parse(meta.expiresAt);
+	if (Number.isFinite(expiresAt) && now > expiresAt) return false;
+	if (options?.maxAgeSec != null && options.maxAgeSec > 0) {
+		const createdAt = Date.parse(meta.createdAt);
+		if (Number.isFinite(createdAt) && now - createdAt > options.maxAgeSec * 1000) {
+			return false;
+		}
+	}
+	return true;
+}
+
+function bumpHitCount(metaPath: string, meta: WebCacheMeta): void {
+	const next: WebCacheMeta = { ...meta, hitCount: (meta.hitCount ?? 0) + 1 };
+	writeFileSync(metaPath, `${JSON.stringify(next, null, 2)}\n`, "utf-8");
+}
+
+export function lookupSearchCache(
+	projectRoot: string,
+	ctx: SearchCacheContext,
+	options?: { maxAgeSec?: number; ttlSeconds?: number },
+): CacheLookupResult {
+	const cacheKey = searchCacheKey(ctx);
+	const kind = cacheKindFromTier(ctx.tier);
+	const relDir = entryDir(projectRoot, kind, cacheKey);
+	const absDir = resolve(projectRoot, relDir);
+	const artifactName = kind === "search-deep" ? "search-deep.json" : "search.json";
+	const artifactPath = resolve(absDir, artifactName);
+	const metaPath = resolve(absDir, "meta.json");
+	const base: CacheLookupResult = {
+		hit: false,
+		stale: false,
+		cacheKey,
+		entryDir: relDir,
+		artifactPath: `${relDir}/${artifactName}`,
+		metaPath: `${relDir}/meta.json`,
+	};
+	if (!cacheEnabled() || !existsSync(artifactPath)) return base;
+	const meta = readMeta(metaPath);
+	if (!meta) return { ...base, hit: true, stale: true };
+	const fresh = isCacheFresh(meta, {
+		maxAgeSec: options?.maxAgeSec,
+	});
+	const createdAt = Date.parse(meta.createdAt);
+	const ageMs = Number.isFinite(createdAt) ? Date.now() - createdAt : undefined;
+	if (fresh) bumpHitCount(metaPath, meta);
+	return {
+		...base,
+		hit: true,
+		stale: !fresh,
+		meta,
+		ageMs,
+	};
+}
+
+export function lookupFetchCache(
+	projectRoot: string,
+	ctx: FetchCacheContext,
+	options?: { maxAgeSec?: number },
+): CacheLookupResult {
+	const cacheKey = fetchCacheKey(ctx);
+	const kind: WebCacheKind = ctx.highlights
+		? "fetch-highlights"
+		: ctx.mode === "map"
+			? "fetch-map"
+			: "fetch-page";
+	const relDir = entryDir(projectRoot, kind, cacheKey);
+	const absDir = resolve(projectRoot, relDir);
+	const artifactName = ctx.highlights
+		? "highlights.json"
+		: ctx.mode === "map"
+			? "map.json"
+			: "page.md";
+	const artifactPath = resolve(absDir, artifactName);
+	const metaPath = resolve(absDir, "meta.json");
+	const base: CacheLookupResult = {
+		hit: false,
+		stale: false,
+		cacheKey,
+		entryDir: relDir,
+		artifactPath: `${relDir}/${artifactName}`,
+		metaPath: `${relDir}/meta.json`,
+	};
+	if (!cacheEnabled() || !existsSync(artifactPath)) return base;
+	const meta = readMeta(metaPath);
+	if (!meta) return { ...base, hit: true, stale: true };
+	const fresh = isCacheFresh(meta, { maxAgeSec: options?.maxAgeSec });
+	const createdAt = Date.parse(meta.createdAt);
+	const ageMs = Number.isFinite(createdAt) ? Date.now() - createdAt : undefined;
+	if (fresh) bumpHitCount(metaPath, meta);
+	return {
+		...base,
+		hit: true,
+		stale: !fresh,
+		meta,
+		ageMs,
+	};
+}
+
+export function writeSearchCacheEntry(
+	projectRoot: string,
+	ctx: SearchCacheContext,
+	sourceArtifactPath: string,
+	options?: { anglesPath?: string; ttlSeconds?: number },
+): { cacheKey: string; entryDir: string; metaPath: string } {
+	const cacheKey = searchCacheKey(ctx);
+	const kind = cacheKindFromTier(ctx.tier);
+	const relDir = entryDir(projectRoot, kind, cacheKey);
+	const absDir = resolve(projectRoot, relDir);
+	mkdirSync(absDir, { recursive: true });
+	const artifactName = kind === "search-deep" ? "search-deep.json" : "search.json";
+	const destArtifact = resolve(absDir, artifactName);
+	copyFileSync(resolve(projectRoot, sourceArtifactPath), destArtifact);
+	if (options?.anglesPath && existsSync(resolve(projectRoot, options.anglesPath))) {
+		copyFileSync(resolve(projectRoot, options.anglesPath), resolve(absDir, "angles.yaml"));
+	}
+	const ttl = options?.ttlSeconds ?? defaultCacheTtlSeconds();
+	const now = new Date();
+	const expires = new Date(now.getTime() + ttl * 1000);
+	const meta: WebCacheMeta = {
+		version: 1,
+		kind,
+		cacheKey,
+		createdAt: now.toISOString(),
+		expiresAt: expires.toISOString(),
+		ttlSeconds: ttl,
+		context: {
+			query: ctx.query,
+			tier: ctx.tier,
+			engine: ctx.engine,
+			limit: ctx.limit,
+			category: ctx.category ?? null,
+			expandHeuristic: Boolean(ctx.expandHeuristic),
+			anglesFingerprint: ctx.anglesFingerprint ?? null,
+		},
+		artifact: artifactName,
+		hitCount: 0,
+	};
+	const metaPath = resolve(absDir, "meta.json");
+	writeFileSync(metaPath, `${JSON.stringify(meta, null, 2)}\n`, "utf-8");
+	return { cacheKey, entryDir: relDir, metaPath: `${relDir}/meta.json` };
+}
+
+export function writeFetchCacheEntry(
+	projectRoot: string,
+	ctx: FetchCacheContext,
+	sourceArtifactPath: string,
+	extra?: { highlightsPath?: string },
+): { cacheKey: string; entryDir: string } {
+	const cacheKey = fetchCacheKey(ctx);
+	const kind: WebCacheKind = ctx.highlights
+		? "fetch-highlights"
+		: ctx.mode === "map"
+			? "fetch-map"
+			: "fetch-page";
+	const relDir = entryDir(projectRoot, kind, cacheKey);
+	const absDir = resolve(projectRoot, relDir);
+	mkdirSync(absDir, { recursive: true });
+	const artifactName = ctx.highlights
+		? "highlights.json"
+		: ctx.mode === "map"
+			? "map.json"
+			: "page.md";
+	copyFileSync(resolve(projectRoot, sourceArtifactPath), resolve(absDir, artifactName));
+	if (extra?.highlightsPath && existsSync(resolve(projectRoot, extra.highlightsPath))) {
+		copyFileSync(
+			resolve(projectRoot, extra.highlightsPath),
+			resolve(absDir, "highlights.json"),
+		);
+	}
+	const ttl = defaultCacheTtlSeconds();
+	const now = new Date();
+	const expires = new Date(now.getTime() + ttl * 1000);
+	const meta: WebCacheMeta = {
+		version: 1,
+		kind,
+		cacheKey,
+		createdAt: now.toISOString(),
+		expiresAt: expires.toISOString(),
+		ttlSeconds: ttl,
+		context: {
+			url: ctx.url,
+			mode: ctx.mode,
+			fast: ctx.fast,
+			highlightQuery: ctx.highlightQuery ?? null,
+			highlights: ctx.highlights,
+		},
+		artifact: artifactName,
+		hitCount: 0,
+	};
+	writeFileSync(
+		resolve(absDir, "meta.json"),
+		`${JSON.stringify(meta, null, 2)}\n`,
+		"utf-8",
+	);
+	return { cacheKey, entryDir: relDir };
+}
+
+/** Copy cached artifact to a stable workspace path for agents (`.web/search-deep.json`, …). */
+export function publishWorkspaceAlias(
+	projectRoot: string,
+	cacheArtifactPath: string,
+	workspaceBasename: string,
+): string {
+	const workspacePath = `${WEB_ROOT}/${workspaceBasename}`;
+	const dest = resolve(projectRoot, workspacePath);
+	mkdirSync(dirname(dest), { recursive: true });
+	copyFileSync(resolve(projectRoot, cacheArtifactPath), dest);
+	return workspacePath;
+}
+
+export function formatCacheAge(ageMs: number | undefined): string {
+	if (ageMs == null || ageMs < 0) return "unknown";
+	if (ageMs < 60_000) return `${Math.round(ageMs / 1000)}s`;
+	if (ageMs < 3_600_000) return `${Math.round(ageMs / 60_000)}m`;
+	return `${(ageMs / 3_600_000).toFixed(1)}h`;
+}
+
+export function webCacheHint(): string {
+	return (
+		`[WRS cache] Pooled under ${WEB_CACHE_ROOT}/ with TTL (HARNESS_WEB_CACHE_TTL_SEC, default 24h). ` +
+		`Workspace aliases: ${WEB_ROOT}/search-deep.json, ${WEB_ROOT}/angles.yaml, ${WEB_ROOT}/page.md. ` +
+		`Use refreshCache:true to bypass. Same query+angles reuses SERP without network.`
+	);
+}

package/.pi/lib/harness-web/run-cli.ts

@@ -46,6 +46,8 @@ export interface SearchHit {
 	url: string;
 	title: string;
 	description: string;
+	score?: number;
+	angle_ids?: string[];
 }
 
 export function summarizeSearchJson(filePath: string, cwd: string): string {
@@ -55,11 +57,15 @@ export function summarizeSearchJson(filePath: string, cwd: string): string {
 		const data = JSON.parse(readFileSync(full, "utf-8")) as {
 			query?: string;
 			engine?: string;
+			tier?: string;
+			mode?: string;
 			data?: { web?: SearchHit[] };
 		};
 		const hits = data.data?.web ?? [];
+		const tier = data.tier ?? data.mode ?? "standard";
 		const lines = [
 			`engine: ${data.engine ?? "unknown"}`,
+			`tier: ${tier}`,
 			`query: ${data.query ?? ""}`,
 			`results: ${hits.length}`,
 			"",
@@ -67,6 +73,12 @@ export function summarizeSearchJson(filePath: string, cwd: string): string {
 		for (const [i, hit] of hits.entries()) {
 			lines.push(`${i + 1}. ${hit.title || "(no title)"}`);
 			lines.push(`   ${hit.url}`);
+			if (hit.score != null) {
+				lines.push(`   score: ${hit.score}`);
+			}
+			if (hit.angle_ids?.length) {
+				lines.push(`   angles: ${hit.angle_ids.join(", ")}`);
+			}
 			if (hit.description) {
 				const snip =
 					hit.description.length > 120
@@ -81,12 +93,40 @@ export function summarizeSearchJson(filePath: string, cwd: string): string {
 	}
 }
 
+export function summarizeDeepSearchJson(filePath: string, cwd: string): string {
+	const full = resolve(cwd, filePath);
+	if (!existsSync(full)) return "";
+	try {
+		const data = JSON.parse(readFileSync(full, "utf-8")) as {
+			query?: string;
+			angles?: Array<{ id: string; query: string }>;
+			data?: { web?: SearchHit[] };
+		};
+		const lines = [
+			summarizeSearchJson(filePath, cwd),
+			"",
+			`angles: ${data.angles?.length ?? 0}`,
+		];
+		for (const a of data.angles ?? []) {
+			lines.push(`  - ${a.id}: ${a.query}`);
+		}
+		lines.push("");
+		lines.push("Prefer URLs with multiple angle_ids. Use web_fetch highlights on top 3.");
+		return lines.join("\n");
+	} catch {
+		return summarizeSearchJson(filePath, cwd);
+	}
+}
+
 export function harnessWebContextLine(): string {
 	const engine = process.env.HARNESS_WEB_SEARCH_ENGINE?.trim() || "ddg_html";
 	const searx = process.env.HARNESS_WEB_SEARXNG_URL?.trim();
 	const searxPart = searx ? ` searxng_url=${searx}` : "";
 	return (
-		`[HarnessWeb] search_engine=${engine}${searxPart} — use web_search / web_fetch tools; ` +
-		"never resolve UP_PKG, ls harness-web.py, or python3 -c import scrapling before searching."
+		`[HarnessWeb] engine=${engine}${searxPart} | research: tier=deep + web-query-expander | ` +
+		"latency: tier=instant|standard or web-query-expander-fast | " +
+		"artifacts: .web/runs/<run_id>/ or .web/sessions/<session_id>/ (not flat .web/answer.md) | " +
+		"models: HARNESS_WEB_*_MODEL env (provider/model-id) | " +
+		"skill: web-retrieval"
 	);
 }

package/.pi/prompts/harness-plan.md

@@ -120,6 +120,7 @@ Gate: `harness_artifact_ready({ paths: ["artifacts/hypothesis.yaml"] })`.
 - Merge both into `research-brief.yaml` (`implementation:` + `stack:`) via parent `write_harness_yaml`.
 - **Partial failure:** if one lane fails, re-spawn that lane once; if still failing write `artifacts/plan-phase-status.yaml` with `plan_status: partial` and `ask_user`. Do not proceed to Phase 4b without both research artifacts unless `artifacts/plan-phase-waiver.yaml` records an explicit waiver.
 - **Web dedup:** implementation owns patterns/repos; stack owns libraries/versions — no overlapping queries.
+- **WRS bundle (parent pre-research):** When running web inline before spawn, use `web-retrieval` deep path: `.web/angles.yaml`, `.web/search-deep.json`, highlight fetches — attach paths in research task context so debate can cite fused SERP scores.
 
 Document `human_required` waiver in the run trace only when research is genuinely blocked.
 

package/.pi/prompts/harness-setup.md

@@ -140,6 +140,8 @@ python3 "$UP_PKG/.pi/scripts/harness-web.py" scrape "https://example.com" -o .we
 
 After pi loads extensions, agents should smoke **`web_search`** once (not `UP_PKG` / `import scrapling` preflight). Example intent: query `ultimate-pi harness`, `limit` 2.
 
+**WRS subagent models (optional):** set env vars to any Pi `provider/model-id` — `HARNESS_WEB_FAST_MODEL`, `HARNESS_WEB_EXPANDER_MODEL`, `HARNESS_WEB_QUALITY_MODEL` (see **web-retrieval** skill). Add to `.env` via `harness-sync-env.mjs` or export in shell.
+
 - **`--skip-tools`:** skip Step 2 (includes Scrapling verify).
 - On Linux/WSL, if stealth scrape fails, install browser libs from `harness-cli-verify.sh` output or use `--fast` for static targets.
 
@@ -188,7 +190,7 @@ python3 "$UP_PKG/.pi/scripts/harness-web.py" search "query" -o .web/search.json
 python3 "$UP_PKG/.pi/scripts/harness-web.py" scrape "https://example.com" -o .web/page.md --fast
 ```
 
-See `.agents/skills/scrapling-web/SKILL.md`.
+See `.agents/skills/web-retrieval/SKILL.md` (install + env).
 
 ### 2.2 — ctx7 (Context7 Library Docs + Skills Management)
 

package/.pi/prompts/harness-steer.md

@@ -19,7 +19,7 @@ Thin orchestrator for the **steer loop** (ADR 0044). Run only after `/harness-re
 2. Update `artifacts/steer-state.yaml` (`attempt`, `max_attempts`, `active: true`).
 3. Set policy phase to **execute** before spawning executor (required for mutating tools).
 4. One `ask_user` steer gate unless `run-context.steer_approved` is already true.
-5. Spawn **`harness/running/executor`** with `HarnessSpawnContext.mode: repair` and `repair_brief_path: artifacts/repair-brief.yaml`.
+5. Spawn **`harness/running/executor`** with `HarnessSpawnContext.mode: repair` and `repair_brief_path: artifacts/repair-brief.yaml`. Repair uses the same hash-anchored `read`/`edit`, batching, and pre-handoff verification rules as `/harness-run` (ADR 0051).
 6. Optional: `node "$UP_PKG/.pi/scripts/harness-sentrux-cli.mjs" gate --save` after repair to refresh baseline (ADR 0044).
 7. `next_command`: **`/harness-review`** (always re-verify; tiered adversary on attempts 2+ per practice-map).
 

package/.pi/scripts/gen-web-heuristic-angles-json.mjs

@@ -0,0 +1,24 @@
+#!/usr/bin/env node
+/** Regenerate .pi/harness/web-heuristic-angles.json from shipped Python defaults. */
+import { execFileSync } from "node:child_process";
+import { writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..", "harness");
+const out = join(root, "web-heuristic-angles.json");
+const py = join(dirname(fileURLToPath(import.meta.url)), "harness_web", "heuristic_angles_shipped.py");
+const json = execFileSync(
+	"python3",
+	[
+		"-c",
+		`import json, importlib.util
+spec = importlib.util.spec_from_file_location("shipped", ${JSON.stringify(py)})
+mod = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mod)
+print(json.dumps(mod.SHIPPED_HEURISTIC_ANGLES, indent=2))`,
+	],
+	{ encoding: "utf-8" },
+);
+writeFileSync(out, `${json}\n`, "utf-8");
+console.log(`wrote ${out}`);

package/.pi/scripts/harness-anchored-edit-smoke.mjs

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+/**
+ * Smoke test for native anchored edit apply (no Pi oldText shim).
+ */
+import { createRequire } from "node:module";
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const require = createRequire(join(root, "package.json"));
+
+const { applyAnchoredEditsToFile } = await import(
+	join(root, ".pi/lib/harness-anchored-edit/apply-anchored-edits.ts")
+);
+const { hashLinesStateful } = await import(
+	join(root, ".pi/lib/harness-anchored-edit/anchor-state.ts")
+);
+
+const dir = mkdtempSync(join(tmpdir(), "anchored-smoke-"));
+const file = join(dir, "t.ts");
+writeFileSync(file, "line one\nline two\n");
+const plain = readFileSync(file, "utf8");
+const hashed = hashLinesStateful(file, plain, "smoke");
+const line2 = hashed.split("\n").find((l) => l.includes("line two"));
+if (!line2) {
+	console.error("smoke FAIL: no anchor line");
+	process.exit(1);
+}
+const result = await applyAnchoredEditsToFile(
+	file,
+	[{ anchor: line2, text: "line TWO", edit_type: "replace" }],
+	"smoke",
+);
+if (!result.ok) {
+	console.error("smoke FAIL:", result.error);
+	process.exit(1);
+}
+const out = readFileSync(file, "utf8");
+if (!out.includes("line TWO")) {
+	console.error("smoke FAIL: file not updated:", out);
+	process.exit(1);
+}
+console.log("harness-anchored-edit-smoke OK");

package/.pi/scripts/harness-cli-verify.sh

@@ -214,6 +214,11 @@ verify_scrapling() {
 	else
 		fail "harness-web search smoke failed (ddg_html)"
 	fi
+	if python3 "$_hw" search-deep "ultimate-pi harness" --expand-heuristic -o .web/verify-search-deep.json --limit 3 2>/dev/null | grep -q wrote; then
+		pass "harness-web search-deep smoke (heuristic angles)"
+	else
+		fail "harness-web search-deep smoke failed"
+	fi
 	if python3 "$_hw" scrape "https://example.com" -o .web/verify-page.md --fast 2>/dev/null | grep -q wrote; then
 		pass "harness-web scrape --fast smoke"
 	else