ultimate-pi 0.18.1 → 0.19.1

package/.pi/scripts/harness_web/query_angles.py

@@ -0,0 +1,116 @@
+"""Parse and validate WRS search angles (YAML/JSON)."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+try:
+    import yaml  # type: ignore[import-untyped]
+except ImportError:
+    yaml = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True)
+class SearchAngle:
+    id: str
+    query: str
+    rationale: str = ""
+
+
+@dataclass(frozen=True)
+class AnglesPlan:
+    intent: str
+    angles: tuple[SearchAngle, ...]
+    category: str | None = None
+
+
+def _heuristic_angles(query: str, *, category: str | None = None) -> AnglesPlan:
+    """Emergency fallback when no expander output — templates from YAML config."""
+    from .heuristic_config import build_heuristic_angles, load_heuristic_angles_config
+
+    q = query.strip()
+    cfg = load_heuristic_angles_config()
+    built = build_heuristic_angles(q, category=category, config=cfg)
+    if len(built) < 2:
+        built = (
+            SearchAngle("definitional", q, "Core intent phrasing"),
+            SearchAngle("official", f"{q} official documentation", "Authoritative sources"),
+        )
+    return AnglesPlan(intent=q, angles=built, category=category or None)
+
+
+def _parse_angle_item(raw: Any, idx: int) -> SearchAngle:
+    if isinstance(raw, str):
+        s = raw.strip()
+        if not s:
+            raise ValueError(f"angles[{idx}]: empty query string")
+        return SearchAngle(id=f"angle_{idx + 1}", query=s)
+    if not isinstance(raw, dict):
+        raise ValueError(f"angles[{idx}]: expected object or string")
+    aid = str(raw.get("id") or raw.get("name") or f"angle_{idx + 1}").strip()
+    query = str(raw.get("query") or "").strip()
+    if not query:
+        raise ValueError(f"angles[{idx}]: missing query")
+    rationale = str(raw.get("rationale") or raw.get("reason") or "").strip()
+    return SearchAngle(id=aid or f"angle_{idx + 1}", query=query, rationale=rationale)
+
+
+def _load_structured(data: dict[str, Any]) -> AnglesPlan:
+    intent = str(data.get("intent") or data.get("query") or "").strip()
+    raw_angles = data.get("angles")
+    if not isinstance(raw_angles, list) or not raw_angles:
+        raise ValueError("angles: expected non-empty list")
+    angles = tuple(_parse_angle_item(item, i) for i, item in enumerate(raw_angles))
+    if len(angles) < 2:
+        raise ValueError("angles: need at least 2 entries for deep search")
+    if len(angles) > 8:
+        angles = angles[:8]
+    category = data.get("category")
+    cat_str = str(category).strip() if category else None
+    return AnglesPlan(intent=intent or angles[0].query, angles=angles, category=cat_str)
+
+
+def load_angles_file(path: Path) -> AnglesPlan:
+    text = path.read_text(encoding="utf-8")
+    # Strip markdown fences if present
+    fenced = re.search(r"```(?:ya?ml|json)?\s*\n([\s\S]*?)```", text)
+    if fenced:
+        text = fenced.group(1)
+    text = text.strip()
+    if not text:
+        raise ValueError(f"empty angles file: {path}")
+
+    data: Any
+    if text.startswith("{"):
+        data = json.loads(text)
+    elif yaml is not None:
+        data = yaml.safe_load(text)
+    else:
+        raise SystemExit(
+            "angles file is YAML but PyYAML is not installed. "
+            "Use JSON angles or: pip install pyyaml"
+        )
+    if not isinstance(data, dict):
+        raise ValueError("angles file root must be an object")
+    return _load_structured(data)
+
+
+def resolve_angles(
+    query: str,
+    *,
+    angles_file: Path | None = None,
+    expand_heuristic: bool = False,
+    category: str | None = None,
+) -> AnglesPlan:
+    if angles_file is not None:
+        return load_angles_file(angles_file)
+    if expand_heuristic:
+        return _heuristic_angles(query, category=category)
+    raise SystemExit(
+        "deep search requires --angles-file (.web/angles.yaml from web-query-expander) "
+        "or --expand-heuristic for emergency fallback"
+    )

package/.pi/scripts/harness_web/rank.py

@@ -0,0 +1,163 @@
+"""URL normalization and RRF fusion for multi-angle SERP results."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import parse_qs, urlparse, urlunparse
+
+RRF_K = 60
+
+_TRACKING_PARAMS = frozenset(
+    {
+        "utm_source",
+        "utm_medium",
+        "utm_campaign",
+        "utm_term",
+        "utm_content",
+        "fbclid",
+        "gclid",
+        "mc_cid",
+        "mc_eid",
+    }
+)
+
+
+@dataclass
+class RankedHit:
+    url: str
+    title: str
+    description: str
+    score: float
+    angle_ids: list[str] = field(default_factory=list)
+    ranks: dict[str, int] = field(default_factory=dict)
+
+    def to_web_dict(self) -> dict[str, Any]:
+        return {
+            "url": self.url,
+            "title": self.title,
+            "description": self.description,
+            "score": round(self.score, 6),
+            "angle_ids": list(self.angle_ids),
+            "ranks": dict(self.ranks),
+        }
+
+
+def normalize_url(url: str) -> str:
+    u = url.strip()
+    if not u:
+        return ""
+    parsed = urlparse(u)
+    scheme = (parsed.scheme or "https").lower()
+    host = (parsed.hostname or "").lower()
+    if not host:
+        return u
+    port = parsed.port
+    netloc = host
+    if port and not ((scheme == "http" and port == 80) or (scheme == "https" and port == 443)):
+        netloc = f"{host}:{port}"
+    path = parsed.path or "/"
+    if path != "/" and path.endswith("/"):
+        path = path.rstrip("/")
+    qs = parse_qs(parsed.query, keep_blank_values=False)
+    filtered = []
+    for key in sorted(qs.keys()):
+        if key.lower() in _TRACKING_PARAMS:
+            continue
+        for val in qs[key]:
+            filtered.append(f"{key}={val}")
+    query = "&".join(filtered)
+    return urlunparse((scheme, netloc, path, "", query, ""))
+
+
+def tokenize(text: str) -> set[str]:
+    return {t for t in re.findall(r"[a-z0-9]{3,}", text.lower()) if len(t) >= 3}
+
+
+def lexical_rerank(hits: list[RankedHit], intent: str) -> list[RankedHit]:
+    """Lightweight O3 boost when HARNESS_WEB_RERANK=lexical."""
+    intent_tokens = tokenize(intent)
+    if not intent_tokens:
+        return hits
+
+    def lex_score(h: RankedHit) -> float:
+        blob = f"{h.title} {h.description}".lower()
+        tokens = tokenize(blob)
+        if not tokens:
+            return 0.0
+        overlap = len(intent_tokens & tokens) / max(len(intent_tokens), 1)
+        return overlap
+
+    scored = [(h, h.score + 0.15 * lex_score(h)) for h in hits]
+    scored.sort(key=lambda x: x[1], reverse=True)
+    out: list[RankedHit] = []
+    for h, s in scored:
+        out.append(
+            RankedHit(
+                url=h.url,
+                title=h.title,
+                description=h.description,
+                score=s,
+                angle_ids=h.angle_ids,
+                ranks=h.ranks,
+            )
+        )
+    return out
+
+
+def fuse_angle_results(
+    per_angle: dict[str, list[dict[str, str]]],
+    *,
+    final_limit: int = 10,
+    intent: str = "",
+    rerank_mode: str = "off",
+) -> list[RankedHit]:
+    """Reciprocal Rank Fusion across angle result lists."""
+    accum: dict[str, dict[str, Any]] = {}
+
+    for angle_id, results in per_angle.items():
+        for rank_1based, item in enumerate(results, start=1):
+            raw_url = (item.get("url") or "").strip()
+            norm = normalize_url(raw_url)
+            if not norm or not norm.startswith("http"):
+                continue
+            entry = accum.setdefault(
+                norm,
+                {
+                    "url": raw_url,
+                    "title": "",
+                    "description": "",
+                    "score": 0.0,
+                    "angle_ids": [],
+                    "ranks": {},
+                },
+            )
+            entry["score"] += 1.0 / (RRF_K + rank_1based)
+            if angle_id not in entry["angle_ids"]:
+                entry["angle_ids"].append(angle_id)
+            entry["ranks"][angle_id] = rank_1based
+            title = (item.get("title") or "").strip()
+            desc = (item.get("description") or "").strip()
+            if title and not entry["title"]:
+                entry["title"] = title
+            if desc and (not entry["description"] or len(desc) > len(entry["description"])):
+                entry["description"] = desc
+
+    hits = [
+        RankedHit(
+            url=e["url"],
+            title=e["title"],
+            description=e["description"],
+            score=e["score"],
+            angle_ids=e["angle_ids"],
+            ranks=e["ranks"],
+        )
+        for e in accum.values()
+    ]
+    hits.sort(key=lambda h: (-h.score, -len(h.angle_ids), min(h.ranks.values()) if h.ranks else 999))
+
+    if rerank_mode == "lexical" and intent:
+        hits = lexical_rerank(hits, intent)
+
+    return hits[:final_limit]

package/.pi/scripts/harness_web/scrape.py

@@ -41,6 +41,36 @@ def scrape_url(
     write_page_markdown(Path(output), page, main_content_only=True)
 
 
+def scrape_url_with_highlights(
+    url: str,
+    markdown_output: str,
+    highlights_output: str | None,
+    *,
+    config: HarnessWebConfig,
+    fast: bool,
+    wait_ms: int | None,
+    highlight_query: str,
+) -> None:
+    import json
+    from pathlib import Path
+
+    from .highlights import extract_highlights
+
+    page = fetch_page(url, config=config, fast=fast, wait_ms=wait_ms)
+    md_path = Path(markdown_output)
+    write_page_markdown(md_path, page, main_content_only=True)
+    if highlights_output and highlight_query.strip():
+        text = md_path.read_text(encoding="utf-8")
+        spans = extract_highlights(text, highlight_query)
+        hp = Path(highlights_output)
+        hp.parent.mkdir(parents=True, exist_ok=True)
+        hp.write_text(
+            json.dumps({"url": url, "query": highlight_query, "highlights": spans}, indent=2)
+            + "\n",
+            encoding="utf-8",
+        )
+
+
 def map_url(
     url: str,
     output: str,

package/.pi/scripts/tests/test_harness_web_heuristic_config.py

@@ -0,0 +1,132 @@
+"""Unit tests for harness_web.heuristic_config."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from harness_web.heuristic_config import (
+    _embedded_builtin_dict,
+    _merge_config_dict,
+    build_heuristic_angles,
+    clear_heuristic_config_cache,
+    heuristic_config_from_merged,
+    load_heuristic_angles_config_cached,
+)
+
+
+class TestHeuristicConfig(unittest.TestCase):
+    def tearDown(self) -> None:
+        clear_heuristic_config_cache()
+
+    def test_builtin_code_includes_stackoverflow(self) -> None:
+        cfg = heuristic_config_from_merged(_embedded_builtin_dict())
+        angles = build_heuristic_angles("rust async", category="code", config=cfg)
+        ids = {a.id for a in angles}
+        self.assertIn("stackoverflow", ids)
+        self.assertIn("github", ids)
+        self.assertTrue(any("site:stackoverflow.com" in a.query for a in angles))
+
+    def test_shipped_code_includes_mdn_and_registries(self) -> None:
+        pkg = Path(__file__).resolve().parents[2] / "harness" / "web-heuristic-angles.yaml"
+        if not pkg.is_file():
+            self.skipTest("package yaml missing")
+        clear_heuristic_config_cache()
+        cfg = load_heuristic_angles_config_cached((str(pkg),))
+        angles = build_heuristic_angles("websocket api", category="code", config=cfg)
+        ids = {a.id for a in angles}
+        self.assertIn("mdn", ids)
+        self.assertIn("package_registries", ids)
+        self.assertLessEqual(len(angles), cfg.max_angles)
+
+    def test_shipped_security_category(self) -> None:
+        pkg = Path(__file__).resolve().parents[2] / "harness" / "web-heuristic-angles.yaml"
+        if not pkg.is_file():
+            self.skipTest("package yaml missing")
+        clear_heuristic_config_cache()
+        cfg = load_heuristic_angles_config_cached((str(pkg),))
+        angles = build_heuristic_angles("jwt validation", category="security", config=cfg)
+        ids = {a.id for a in angles}
+        self.assertIn("owasp", ids)
+        self.assertIn("cve_nvd", ids)
+
+    def test_merge_extends_code_category(self) -> None:
+        merged = _merge_config_dict(
+            _embedded_builtin_dict(),
+            {
+                "max_angles": 12,
+                "categories": {
+                    "code": [
+                        {
+                            "id": "docs_rs",
+                            "query": "{query} site:docs.rs",
+                            "rationale": "Rust docs",
+                        },
+                    ],
+                },
+            },
+        )
+        cfg = heuristic_config_from_merged(merged)
+        merged_ids = [a["id"] for a in merged["categories"]["code"]]
+        self.assertIn("docs_rs", merged_ids)
+        angles = build_heuristic_angles("tokio", category="code", config=cfg)
+        ids = {a.id for a in angles}
+        self.assertIn("stackoverflow", ids)
+        self.assertIn("github", ids)
+
+    def test_merge_adds_new_category(self) -> None:
+        merged = _merge_config_dict(
+            _embedded_builtin_dict(),
+            {
+                "categories": {
+                    "security": [
+                        {"id": "cve", "query": "{query} CVE", "rationale": "vulns"},
+                        {
+                            "id": "owasp",
+                            "query": "{query} site:owasp.org",
+                            "rationale": "guidance",
+                        },
+                    ],
+                },
+            },
+        )
+        cfg = heuristic_config_from_merged(merged)
+        angles = build_heuristic_angles("jwt auth", category="security", config=cfg)
+        ids = {a.id for a in angles}
+        self.assertIn("cve", ids)
+        self.assertIn("owasp", ids)
+
+    def test_json_project_file_merges(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            proj = Path(tmp)
+            harness_dir = proj / ".pi" / "harness"
+            harness_dir.mkdir(parents=True)
+            proj_file = harness_dir / "web-heuristic-angles.json"
+            proj_file.write_text(
+                json.dumps(
+                    {
+                        "categories": {
+                            "code": [
+                                {
+                                    "id": "crates_io",
+                                    "query": "{query} site:crates.io",
+                                    "rationale": "crates",
+                                },
+                            ],
+                        },
+                    }
+                ),
+                encoding="utf-8",
+            )
+            clear_heuristic_config_cache()
+            cfg = load_heuristic_angles_config_cached((str(proj_file),))
+            angles = build_heuristic_angles("serde", category="code", config=cfg)
+            ids = {a.id for a in angles}
+            self.assertIn("crates_io", ids)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/.pi/scripts/tests/test_harness_web_query_angles.py

@@ -0,0 +1,45 @@
+"""Unit tests for harness_web.query_angles (no network)."""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from harness_web.query_angles import AnglesPlan, load_angles_file, resolve_angles
+
+
+class TestResolveAngles(unittest.TestCase):
+    def test_heuristic_code_category(self) -> None:
+        plan = resolve_angles("rust async", expand_heuristic=True, category="code")
+        ids = {a.id for a in plan.angles}
+        self.assertIn("github", ids)
+        self.assertGreaterEqual(len(plan.angles), 2)
+        self.assertLessEqual(len(plan.angles), 5)
+
+
+class TestLoadFile(unittest.TestCase):
+    def test_load_json_file(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            p = Path(tmp) / "angles.json"
+            p.write_text(
+                json.dumps(
+                    {
+                        "intent": "load test",
+                        "angles": [
+                            {"id": "a", "query": "first angle query"},
+                            {"id": "b", "query": "second angle query"},
+                        ],
+                    }
+                ),
+                encoding="utf-8",
+            )
+            plan = load_angles_file(p)
+            self.assertIsInstance(plan, AnglesPlan)
+            self.assertEqual(plan.intent, "load test")
+            self.assertEqual(len(plan.angles), 2)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/.pi/scripts/tests/test_harness_web_rank.py

@@ -0,0 +1,56 @@
+"""Unit tests for harness_web.rank (no network)."""
+
+from __future__ import annotations
+
+import unittest
+
+from harness_web.rank import RankedHit, fuse_angle_results, lexical_rerank, normalize_url, tokenize
+
+
+class TestNormalizeUrl(unittest.TestCase):
+    def test_strips_tracking(self) -> None:
+        a = normalize_url("https://Example.com/path?utm_source=x&id=1")
+        b = normalize_url("https://example.com/path?id=1")
+        self.assertEqual(a, b)
+
+    def test_trailing_slash(self) -> None:
+        self.assertEqual(
+            normalize_url("https://example.com/foo/"),
+            normalize_url("https://example.com/foo"),
+        )
+
+
+class TestRrfFusion(unittest.TestCase):
+    def test_merges_duplicate_urls(self) -> None:
+        angle_results = {
+            "a": [
+                {"url": "https://x.com/1", "title": "T1", "description": "d1"},
+                {"url": "https://x.com/2", "title": "T2", "description": "d2"},
+            ],
+            "b": [
+                {"url": "https://x.com/1", "title": "T1b", "description": "d1b"},
+            ],
+        }
+        fused = fuse_angle_results(angle_results, final_limit=5)
+        self.assertEqual(len(fused), 2)
+        top = fused[0]
+        self.assertEqual(top.url, "https://x.com/1")
+        self.assertIn("a", top.angle_ids)
+        self.assertIn("b", top.angle_ids)
+        self.assertGreater(top.score, fused[1].score)
+
+
+class TestLexicalRerank(unittest.TestCase):
+    def test_boosts_intent_overlap(self) -> None:
+        hits = [
+            RankedHit("https://a", "unrelated", "noise", 0.52, ["a"]),
+            RankedHit("https://b", "kubernetes architecture", "how kubernetes works", 0.50, ["b"]),
+        ]
+        reranked = lexical_rerank(hits, "kubernetes architecture")
+        self.assertEqual(reranked[0].url, "https://b")
+        self.assertGreater(reranked[0].score, reranked[1].score)
+
+
+class TestTokenize(unittest.TestCase):
+    def test_min_length(self) -> None:
+        self.assertIn("hello", tokenize("hello hi"))