ultimate-pi 0.19.0 → 0.19.1

package/.pi/harness/agents.manifest.json

@@ -1,9 +1,9 @@
 {
 	"schema_version": "1.0.0",
 	"package": "ultimate-pi",
-	"package_version": "0.18.1",
-	"generated_at": "2026-05-24T19:33:40.491Z",
-	"policy_sha256": "4c1cf8f0377e70bbcd34eecc26a2e02717fea950af3898ef03ebbab9260dfd17",
+	"package_version": "0.19.0",
+	"generated_at": "2026-05-26T06:39:19.250Z",
+	"policy_sha256": "f0eb41bc6a877a237087e87762ce11908e93755a42d58f19d6573665b198665f",
 	"agents": {
 		"pi-pi/agent-expert": {
 			"path": ".pi/agents/pi-pi/agent-expert.md",
@@ -61,6 +61,30 @@
 			"path": ".pi/agents/harness/trace-librarian.md",
 			"sha256": "083ce6a3508bab2f5bc57e8ab13256c55fff58e0eac95cb6542bdd47fd02de65"
 		},
+		"harness/web-retrieval/web-answerer": {
+			"path": ".pi/agents/harness/web-retrieval/web-answerer.md",
+			"sha256": "c9876fb34aa6704c1a8eb9f4db4ada71c5a762790db411a1b01104a81745215d"
+		},
+		"harness/web-retrieval/web-criteria-verifier": {
+			"path": ".pi/agents/harness/web-retrieval/web-criteria-verifier.md",
+			"sha256": "bb0a9b514964f6f64ef539c1d766ff28ee23c4d9199dc4cae0f41b0fe7fdc860"
+		},
+		"harness/web-retrieval/web-gap-analyzer": {
+			"path": ".pi/agents/harness/web-retrieval/web-gap-analyzer.md",
+			"sha256": "f4487993f561c7141b22c79f9afff52986466c5b3af4f3ffb0e75e30a6da025a"
+		},
+		"harness/web-retrieval/web-query-expander-fast": {
+			"path": ".pi/agents/harness/web-retrieval/web-query-expander-fast.md",
+			"sha256": "161c5ce82e013af7aaec219b4ccb6b8c557bf5768f8e0c85b1d8f243056b015c"
+		},
+		"harness/web-retrieval/web-query-expander": {
+			"path": ".pi/agents/harness/web-retrieval/web-query-expander.md",
+			"sha256": "b522532e15759c747ff92198aa6db2624accd0998a4e0af4cb086617734f5ce7"
+		},
+		"harness/web-retrieval/web-summarizer": {
+			"path": ".pi/agents/harness/web-retrieval/web-summarizer.md",
+			"sha256": "0c25f814ded534f6d190cc50ff141306d26dba98eff2ad6d51b56e42c143861d"
+		},
 		"harness/running/executor": {
 			"path": ".pi/agents/harness/running/executor.md",
 			"sha256": "219c9307567acc95a9c1b1340f899fac860406fb2c2e84f51b4a8c3ba3a0e2ec"
@@ -95,7 +119,7 @@
 		},
 		"harness/planning/implementation-researcher": {
 			"path": ".pi/agents/harness/planning/implementation-researcher.md",
-			"sha256": "8664ad35c63adcc3a6c52b84360d5e8217f9f581b0a9f23b565187a8f7c7bf22"
+			"sha256": "4559fc93ccf867ee199821a781cfd1b4a863e1f35ea8520d15ff389e61e17fd2"
 		},
 		"harness/planning/plan-adversary": {
 			"path": ".pi/agents/harness/planning/plan-adversary.md",
@@ -123,7 +147,7 @@
 		},
 		"harness/planning/stack-researcher": {
 			"path": ".pi/agents/harness/planning/stack-researcher.md",
-			"sha256": "641bcd714df327579ff28db04aeabf5d7ee5f36e333e5ae6971967c591863ca4"
+			"sha256": "0f19febc96bc65fd22b2d3f1c3bbed02e021f7c8129a7bb21423ea29d8429d67"
 		}
 	}
 }

package/.pi/harness/agents.policy.yaml

@@ -203,12 +203,44 @@ agents:
     max_turns: 14
     thinking: medium
     submit_tool: submit_hypothesis_brief
+  harness/web-retrieval/web-query-expander:
+    kind: other
+    extensions: false
+    max_turns: 8
+    thinking: low
+  harness/web-retrieval/web-query-expander-fast:
+    kind: other
+    extensions: false
+    max_turns: 5
+    thinking: off
+  harness/web-retrieval/web-gap-analyzer:
+    kind: other
+    extensions: false
+    max_turns: 8
+    thinking: low
+  harness/web-retrieval/web-answerer:
+    kind: other
+    extensions: false
+    max_turns: 12
+    thinking: medium
+  harness/web-retrieval/web-summarizer:
+    kind: other
+    extensions: false
+    max_turns: 6
+    thinking: low
+  harness/web-retrieval/web-criteria-verifier:
+    kind: other
+    extensions: false
+    max_turns: 14
+    thinking: medium
   harness/planning/implementation-researcher:
     kind: planner
     tools_add:
       - bash
       - web_search
       - web_fetch
+      - web_find_similar
+      - web_contents
       - submit_implementation_research
     extensions: false
     max_turns: 14
@@ -268,6 +300,8 @@ agents:
       - bash
       - web_search
       - web_fetch
+      - web_find_similar
+      - web_contents
       - submit_stack_brief
     extensions: false
     max_turns: 16

package/.pi/harness/docs/adrs/0050-agentic-web-retrieval-stack.md

@@ -0,0 +1,46 @@
+# ADR 0050: Agentic Web Retrieval Stack (WRS)
+
+- **Status:** Accepted
+- **Date:** 2026-05-26
+- **Deciders:** ultimate-pi harness team
+
+## Context
+
+Harness agents treated `web_search` as single-query SERP, yielding poor recall on ambiguous research questions. Exa-style outcomes (multi-angle discovery, fusion, evidence, synthesis) are needed without Exa API, MCP, or a neural index at Exa scale.
+
+## Decision
+
+Introduce **WRS** as the default non-API web layer:
+
+1. **Tiers** on `web_search`: `instant`, `standard`, **`deep`** (default for research), `research`.
+2. **Planning subagents** under `.pi/agents/harness/web-retrieval/` — e.g. `harness/web-retrieval/web-query-expander` produces `.web/angles.yaml`; parent runs `web_search(tier=deep, anglesFile=…)`.
+3. **Python fusion**: parallel metasearch per angle (DDG HTML or SearXNG) + RRF (`k=60`) + optional lexical rerank.
+4. **Extension tools**: `web_find_similar`, `web_contents`, `web_fetch` highlights.
+5. **Synthesis subagents** (same directory): `web-answerer`, `web-gap-analyzer`, `web-criteria-verifier`, `web-summarizer`, `web-query-expander-fast`.
+6. **web-retrieval** skill as canonical workflow; **SYSTEM.md** mandates deep default and anti-patterns.
+7. **context7** remains sole path for library API documentation.
+8. **User model routing:** env vars `HARNESS_WEB_FAST_MODEL`, `HARNESS_WEB_EXPANDER_MODEL`, `HARNESS_WEB_QUALITY_MODEL` (any Pi `provider/model-id`); else parent session or agent `model:` override.
+9. **Pooled local cache:** `.web/cache/` keyed by search/fetch context with TTL (`HARNESS_WEB_CACHE_TTL_SEC`); workspace aliases under `.web/`. Optional `HARNESS_WEB_ISOLATE=1` for per-run/session dirs.
+
+Subagents are **not** spawned inside tool `execute()`; parent orchestrates expander → deep → fetch.
+
+## Consequences
+
+### Positive
+
+- Higher recall on landscape / prior-art questions without paid search APIs.
+- Path-first `.web/` artifacts for harness-plan debate.
+- Contract checks in `harness-verify.mjs` keep guidance aligned with tools.
+
+### Negative / trade-offs
+
+- Deep search is slower (N parallel SERP calls).
+- Heuristic `--expand-heuristic` is weaker than expander subagent (templates from mergeable `.pi/harness/web-heuristic-angles.yaml`; projects extend via same path under their repo).
+- No embedding index; O3 precision is approximate vs Exa neural search.
+
+## References
+
+- `.pi/harness/docs/harness-web-search.md`
+- `.agents/skills/web-retrieval/SKILL.md`
+- `.pi/extensions/harness-web-tools.ts`
+- Plan: `.cursor/plans/exa-style_harness_web_fd231183.plan.md`

package/.pi/harness/docs/harness-web-search.md

@@ -0,0 +1,97 @@
+# Harness Web Retrieval Stack (WRS)
+
+Internal reference for multi-angle search, fusion, and agent workflows. User-facing procedures: **web-retrieval** skill (install + env).
+
+## Outcomes (Exa analog)
+
+| Outcome | Primitive |
+|---------|-----------|
+| Discovery / recall | `search-deep` + RRF |
+| Precision | Multi-angle + optional `HARNESS_WEB_RERANK=lexical` |
+| Evidence / highlights | `web_fetch(highlights)` |
+| Similar pages | `find-similar` CLI / `web_find_similar` |
+| Synthesis | `evidence-bundle` + `web-answerer` |
+
+## CLI
+
+```bash
+python3 .pi/scripts/harness-web.py search-deep "query" \
+  --angles-file .web/angles.yaml -o .web/search-deep.json
+python3 .pi/scripts/harness-web.py search-deep "query" --expand-heuristic -o .web/search-deep.json
+python3 .pi/scripts/harness-web.py find-similar "https://example.com" -o .web/search-deep.json
+python3 .pi/scripts/harness-web.py contents-batch --from-search .web/search-deep.json -o .web/contents/
+```
+
+## Python modules
+
+- `harness_web/query_angles.py` — parse expander YAML
+- `harness_web/multi_search.py` — parallel per-angle SERP
+- `harness_web/rank.py` — normalize URL, RRF, lexical rerank
+- `harness_web/deep_search.py` — orchestration
+- `harness_web/highlights.py` — excerpt scoring
+- `harness_web/evidence_bundle.py` — merge for answerer
+
+## Artifacts
+
+**Cache** (pooled): `.web/cache/<kind>/<cacheKey>/` with `meta.json` (search context, `createdAt`, `expiresAt`, `hitCount`).
+
+**Workspace** (default `.web/`): tool aliases agents read. `web_search` / `web_fetch` set `cacheHit`, `cacheKey`, `cachePath` in details.
+
+| Env / param | Effect |
+|-------------|--------|
+| `HARNESS_WEB_CACHE_TTL_SEC` | Default TTL (86400) |
+| `HARNESS_WEB_CACHE=0` | Disable cache |
+| `refreshCache: true` | Bypass cache |
+| `cacheMaxAge` | Max reuse age (seconds) |
+| `HARNESS_WEB_ISOLATE=1` | Per-run/session dirs (legacy) |
+
+| File | Content |
+|------|---------|
+| `angles.yaml` | Expander output |
+| `search-deep.json` | Fused SERP + scores + `angle_ids` |
+| `evidence-bundle.json` | URLs + snippets + highlights |
+| `answer.md` | Cited synthesis |
+
+## Subagents (`.pi/agents/harness/web-retrieval/`)
+
+| Spawn id | Role |
+|----------|------|
+| `harness/web-retrieval/web-query-expander` | Angles YAML (default research) |
+| `harness/web-retrieval/web-query-expander-fast` | 2–3 angles (latency) |
+| `harness/web-retrieval/web-gap-analyzer` | Follow-up angles |
+| `harness/web-retrieval/web-answerer` | Cited answer |
+| `harness/web-retrieval/web-summarizer` | Single-page digest |
+| `harness/web-retrieval/web-criteria-verifier` | Criteria scoring |
+
+## Heuristic angles config (user-extensible)
+
+Emergency templates for `expandHeuristic:true` / `--expand-heuristic` load from YAML:
+
+| File | Role |
+|------|------|
+| `<package>/.pi/harness/web-heuristic-angles.yaml` | Built-in defaults (code → github, stackoverflow, …) |
+| `<project>/.pi/harness/web-heuristic-angles.yaml` | **Your** extensions (merged on top) |
+
+Copy [examples/web-heuristic-angles.project.yaml](../examples/web-heuristic-angles.project.yaml) into an external project’s `.pi/harness/` to add sites per category or define new categories (use `category` on `web_search`).
+
+Query templates use `{query}` as the user search string. Same `id` in a category replaces the package angle.
+
+Optional: `HARNESS_WEB_HEURISTIC_ANGLES_FILE=/path/to/custom.yaml` (merged last).
+
+## Environment
+
+| Variable | Default |
+|----------|---------|
+| `HARNESS_WEB_SEARCH_ENGINE` | `ddg_html` |
+| `HARNESS_WEB_DEEP_CONCURRENCY` | `4` |
+| `HARNESS_WEB_RERANK` | `off` |
+| `HARNESS_WEB_FAST_MODEL` | expander-fast, summarizer, gap-analyzer |
+| `HARNESS_WEB_EXPANDER_MODEL` | full query expander |
+| `HARNESS_WEB_QUALITY_MODEL` | answerer, criteria-verifier |
+| `HARNESS_WEB_HEURISTIC_ANGLES_FILE` | Extra heuristic angles YAML (merged last) |
+| `HARNESS_PROJECT_ROOT` | Project root for `.pi/harness/web-heuristic-angles.yaml` |
+| `HARNESS_PKG_ROOT` | Package root for default heuristic YAML |
+
+Values use Pi `provider/model-id` format (any provider your install supports). Unset → subagent inherits parent session model. See **web-retrieval** skill.
+
+ADR: [0050-web-retrieval-retrieval-stack.md](adrs/0050-web-retrieval-retrieval-stack.md)

package/.pi/harness/env.harness.template

@@ -13,6 +13,15 @@ HARNESS_WEB_SEARCH_ENGINE=ddg_html
 # HARNESS_WEB_PROXY=
 # HARNESS_WEB_RATE_LIMIT_MS=2000
 # HARNESS_WEB_TIMEOUT_MS=30000
+# Heuristic angles (--expand-heuristic): edit <project>/.pi/harness/web-heuristic-angles.yaml
+# Optional override path (merged after package + project):
+# HARNESS_WEB_HEURISTIC_ANGLES_FILE=
+# Pooled WRS cache (Firecrawl-style freshness); workspace aliases under .web/
+# HARNESS_WEB_CACHE_TTL_SEC=86400
+# HARNESS_WEB_CACHE=0
+# HARNESS_WEB_ISOLATE=1
+# HARNESS_PROJECT_ROOT=
+# HARNESS_PKG_ROOT=
 
 # --- VCC compaction (env-only; no JSON config files) ---
 # Default: VCC handles /compact and auto-compaction. Set false for Pi LLM compaction:
@@ -21,7 +30,6 @@ HARNESS_WEB_SEARCH_ENGINE=ddg_html
 
 # --- PostHog (optional) ---
 # Project key — required for harness_* telemetry when HARNESS_TELEMETRY_ENABLED=true
-# WSL2: ultimate-pi loads 00-posthog-network-bootstrap.ts (IPv4 fetch for *.posthog.com).
 # If flush still fails, set POSTHOG_ENABLED=false or fix outbound HTTPS to PostHog.
 # POSTHOG_API_KEY=
 # POSTHOG_HOST=https://us.i.posthog.com

package/.pi/harness/examples/web-heuristic-angles.project.yaml

@@ -0,0 +1,22 @@
+# Example project extension — copy to:
+#   <your-project>/.pi/harness/web-heuristic-angles.yaml
+#
+# Merges on top of package .pi/harness/web-heuristic-angles.yaml.
+# Same angle id in a category replaces the package entry; new ids append.
+
+version: 1
+
+categories:
+  code:
+    # Add Rust-specific angles (keeps github + stackoverflow from package defaults)
+    - id: docs_rs
+      query: "{query} site:docs.rs"
+      rationale: Official Rust API docs
+    - id: crates_io
+      query: "{query} site:crates.io"
+      rationale: Crate ecosystem
+  # Package already ships category=security; extend or override ids here:
+  # security:
+  #   - id: cisa_kev
+  #     query: "{query} site:cisa.gov/known-exploited-vulnerabilities"
+  #     rationale: CISA KEV catalog

package/.pi/harness/web-heuristic-angles.json

@@ -0,0 +1,278 @@
+{
+  "version": 1,
+  "max_angles": 8,
+  "base": [
+    {
+      "id": "definitional",
+      "query": "{query}",
+      "rationale": "Core intent phrasing"
+    },
+    {
+      "id": "authoritative",
+      "query": "{query} official documentation OR specification OR RFC",
+      "rationale": "Primary specs and vendor docs"
+    }
+  ],
+  "categories": {
+    "code": [
+      {
+        "id": "github",
+        "query": "{query} site:github.com",
+        "rationale": "Source, issues, discussions"
+      },
+      {
+        "id": "stackoverflow",
+        "query": "{query} site:stackoverflow.com",
+        "rationale": "Debugging and API usage Q&A"
+      },
+      {
+        "id": "stackexchange",
+        "query": "{query} site:stackexchange.com",
+        "rationale": "Broader SE network (Super User, Server Fault, etc.)"
+      },
+      {
+        "id": "readthedocs",
+        "query": "{query} site:readthedocs.io",
+        "rationale": "OSS library documentation"
+      },
+      {
+        "id": "mdn",
+        "query": "{query} site:developer.mozilla.org",
+        "rationale": "Web platform and browser APIs"
+      },
+      {
+        "id": "package_registries",
+        "query": "{query} site:npmjs.com OR site:pypi.org OR site:pkg.go.dev OR site:crates.io",
+        "rationale": "Package metadata across major ecosystems"
+      },
+      {
+        "id": "microsoft_learn",
+        "query": "{query} site:learn.microsoft.com",
+        "rationale": ".NET, Azure, Windows, and enterprise stacks"
+      },
+      {
+        "id": "hacker_news",
+        "query": "{query} site:news.ycombinator.com",
+        "rationale": "High-signal practitioner discussion"
+      },
+      {
+        "id": "gitlab",
+        "query": "{query} site:gitlab.com",
+        "rationale": "Alternate host and CI-visible code"
+      },
+      {
+        "id": "devto",
+        "query": "{query} site:dev.to OR site:medium.com",
+        "rationale": "Tutorials and implementation writeups"
+      }
+    ],
+    "paper": [
+      {
+        "id": "arxiv",
+        "query": "{query} site:arxiv.org",
+        "rationale": "Preprints and latest ML/CS uploads"
+      },
+      {
+        "id": "semantic_scholar",
+        "query": "{query} site:semanticscholar.org",
+        "rationale": "Citations, influences, and PDF links"
+      },
+      {
+        "id": "google_scholar",
+        "query": "{query} site:scholar.google.com",
+        "rationale": "Broad academic discovery"
+      },
+      {
+        "id": "papers_with_code",
+        "query": "{query} site:paperswithcode.com",
+        "rationale": "Benchmarks tied to implementations"
+      },
+      {
+        "id": "openreview",
+        "query": "{query} site:openreview.net",
+        "rationale": "Peer reviews and ML conference submissions"
+      },
+      {
+        "id": "acl_anthology",
+        "query": "{query} site:aclanthology.org",
+        "rationale": "NLP and computational linguistics"
+      },
+      {
+        "id": "acm_dl",
+        "query": "{query} site:dl.acm.org",
+        "rationale": "ACM proceedings and journals"
+      },
+      {
+        "id": "pubmed",
+        "query": "{query} site:pubmed.ncbi.nlm.nih.gov",
+        "rationale": "Biomedical and life-sciences literature"
+      }
+    ],
+    "news": [
+      {
+        "id": "recent",
+        "query": "{query} news 2025 2026",
+        "rationale": "Recency-biased open web"
+      },
+      {
+        "id": "wire_reuters",
+        "query": "{query} site:reuters.com",
+        "rationale": "Wire-service reporting"
+      },
+      {
+        "id": "wire_ap",
+        "query": "{query} site:apnews.com",
+        "rationale": "Associated Press coverage"
+      },
+      {
+        "id": "tech_press",
+        "query": "{query} site:techcrunch.com OR site:theverge.com OR site:arstechnica.com",
+        "rationale": "Technology industry news"
+      },
+      {
+        "id": "business_press",
+        "query": "{query} site:bloomberg.com OR site:ft.com OR site:wsj.com",
+        "rationale": "Markets and business context"
+      },
+      {
+        "id": "analysis",
+        "query": "{query} in-depth analysis explainer",
+        "rationale": "Long-form journalism and explainers"
+      },
+      {
+        "id": "bbc",
+        "query": "{query} site:bbc.com/news",
+        "rationale": "International general news desk"
+      }
+    ],
+    "company": [
+      {
+        "id": "official_site",
+        "query": "{query} official website",
+        "rationale": "Company-controlled messaging"
+      },
+      {
+        "id": "crunchbase",
+        "query": "{query} site:crunchbase.com",
+        "rationale": "Funding, investors, and competitors"
+      },
+      {
+        "id": "linkedin_company",
+        "query": "{query} site:linkedin.com/company",
+        "rationale": "Headcount, hiring, and positioning"
+      },
+      {
+        "id": "sec_filings",
+        "query": "{query} site:sec.gov 10-K OR 10-Q OR S-1",
+        "rationale": "US public-company disclosures"
+      },
+      {
+        "id": "g2_reviews",
+        "query": "{query} site:g2.com OR site:capterra.com",
+        "rationale": "B2B software reviews and comparisons"
+      },
+      {
+        "id": "company_news",
+        "query": "{query} company announcement press release",
+        "rationale": "Launches, partnerships, and earnings"
+      },
+      {
+        "id": "glassdoor",
+        "query": "{query} site:glassdoor.com",
+        "rationale": "Employee sentiment and culture signals"
+      }
+    ],
+    "people": [
+      {
+        "id": "linkedin",
+        "query": "{query} site:linkedin.com/in",
+        "rationale": "Professional profiles"
+      },
+      {
+        "id": "github_person",
+        "query": "{query} site:github.com",
+        "rationale": "Open-source footprint for builders"
+      },
+      {
+        "id": "wikipedia",
+        "query": "{query} site:en.wikipedia.org",
+        "rationale": "Neutral biographical baseline"
+      },
+      {
+        "id": "scholar_person",
+        "query": "{query} site:scholar.google.com",
+        "rationale": "Publication record for researchers"
+      },
+      {
+        "id": "interviews",
+        "query": "{query} interview podcast keynote",
+        "rationale": "First-person statements and talks"
+      },
+      {
+        "id": "twitter_x",
+        "query": "{query} site:x.com OR site:twitter.com",
+        "rationale": "Public statements and discourse"
+      }
+    ],
+    "security": [
+      {
+        "id": "cve_nvd",
+        "query": "{query} CVE site:nvd.nist.gov",
+        "rationale": "National Vulnerability Database"
+      },
+      {
+        "id": "owasp",
+        "query": "{query} site:owasp.org",
+        "rationale": "AppSec standards and cheat sheets"
+      },
+      {
+        "id": "cwe",
+        "query": "{query} site:cwe.mitre.org",
+        "rationale": "Weakness taxonomy"
+      },
+      {
+        "id": "github_advisories",
+        "query": "{query} site:github.com/advisories OR dependabot",
+        "rationale": "Ecosystem security advisories"
+      },
+      {
+        "id": "snyk_blog",
+        "query": "{query} site:snyk.io/blog OR vulnerability",
+        "rationale": "Practitioner security writeups"
+      }
+    ],
+    "default": [
+      {
+        "id": "technical",
+        "query": "{query} how it works architecture internals",
+        "rationale": "Mechanism and design"
+      },
+      {
+        "id": "criticism",
+        "query": "{query} limitations criticism drawbacks",
+        "rationale": "Counterpoints and failure modes"
+      },
+      {
+        "id": "wikipedia",
+        "query": "{query} site:en.wikipedia.org",
+        "rationale": "Structured overview"
+      },
+      {
+        "id": "comparison",
+        "query": "{query} vs alternatives comparison benchmark",
+        "rationale": "Competitive landscape"
+      },
+      {
+        "id": "reddit",
+        "query": "{query} site:reddit.com",
+        "rationale": "Community experience reports"
+      },
+      {
+        "id": "hn_default",
+        "query": "{query} site:news.ycombinator.com",
+        "rationale": "Practitioner threads when category unknown"
+      }
+    ]
+  }
+}
+