ultimate-pi 0.19.0 → 0.19.1

package/.pi/scripts/harness_web/heuristic_config.py

@@ -0,0 +1,251 @@
+"""Load and merge WRS heuristic angle templates from YAML."""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from .query_angles import SearchAngle
+
+try:
+    import yaml  # type: ignore[import-untyped]
+except ImportError:
+    yaml = None  # type: ignore[assignment]
+
+CONFIG_BASENAME = "web-heuristic-angles.yaml"
+CONFIG_JSON_BASENAME = "web-heuristic-angles.json"
+ENV_CONFIG_FILE = "HARNESS_WEB_HEURISTIC_ANGLES_FILE"
+
+
+@dataclass(frozen=True)
+class HeuristicAnglesConfig:
+    max_angles: int
+    base: tuple[SearchAngle, ...]
+    categories: dict[str, tuple[SearchAngle, ...]]
+
+
+def _project_root() -> Path:
+    env = os.environ.get("HARNESS_PROJECT_ROOT", "").strip()
+    if env:
+        return Path(env).resolve()
+    return Path.cwd().resolve()
+
+
+def _package_root() -> Path | None:
+    env = os.environ.get("HARNESS_PKG_ROOT", "").strip()
+    if env:
+        return Path(env).resolve()
+    here = Path(__file__).resolve()
+    for anc in here.parents:
+        if (anc / ".pi" / "harness" / CONFIG_BASENAME).is_file():
+            return anc
+    return None
+
+
+def discover_heuristic_config_paths() -> list[Path]:
+    """Package defaults first, then project override, then env file last (wins on id)."""
+    paths: list[Path] = []
+    pkg = _package_root()
+    if pkg is not None:
+        pkg_yaml = pkg / ".pi" / "harness" / CONFIG_BASENAME
+        pkg_json = pkg / ".pi" / "harness" / CONFIG_JSON_BASENAME
+        if pkg_yaml.is_file():
+            paths.append(pkg_yaml)
+        if pkg_json.is_file() and pkg_json not in paths:
+            paths.append(pkg_json)
+    proj_file = _project_root() / ".pi" / "harness" / CONFIG_BASENAME
+    if proj_file.is_file() and proj_file not in paths:
+        paths.append(proj_file)
+    env_path = os.environ.get(ENV_CONFIG_FILE, "").strip()
+    if env_path:
+        p = Path(env_path).expanduser().resolve()
+        if p.is_file() and p not in paths:
+            paths.append(p)
+    return paths
+
+
+def _format_query(template: str, query: str) -> str:
+    return template.replace("{query}", query.strip())
+
+
+def _parse_angle_list(raw: Any, *, source: str) -> list[SearchAngle]:
+    if raw is None:
+        return []
+    if not isinstance(raw, list):
+        raise ValueError(f"{source}: expected list of angle objects")
+    out: list[SearchAngle] = []
+    for i, item in enumerate(raw):
+        if not isinstance(item, dict):
+            raise ValueError(f"{source}[{i}]: expected object")
+        aid = str(item.get("id") or item.get("name") or f"angle_{i + 1}").strip()
+        qtpl = str(item.get("query") or "").strip()
+        if not aid or not qtpl:
+            raise ValueError(f"{source}[{i}]: id and query required")
+        rationale = str(item.get("rationale") or item.get("reason") or "").strip()
+        out.append(
+            SearchAngle(
+                id=aid,
+                query=qtpl,
+                rationale=rationale,
+            )
+        )
+    return out
+
+
+def _merge_config_dict(accum: dict[str, Any], layer: dict[str, Any]) -> dict[str, Any]:
+    out = dict(accum)
+    if "max_angles" in layer:
+        out["max_angles"] = layer["max_angles"]
+    if "version" in layer:
+        out["version"] = layer["version"]
+    base_acc = list(out.get("base") or [])
+    base_acc.extend(layer.get("base") or [])
+    out["base"] = base_acc
+    cats: dict[str, list[Any]] = dict(out.get("categories") or {})
+    layer_cats = layer.get("categories")
+    if isinstance(layer_cats, dict):
+        for key, angles in layer_cats.items():
+            cat = str(key).strip().lower()
+            if not cat:
+                continue
+            existing = list(cats.get(cat) or [])
+            if isinstance(angles, list):
+                existing.extend(angles)
+            cats[cat] = existing
+    out["categories"] = cats
+    return out
+
+
+def _load_config_file(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    if path.suffix.lower() == ".json":
+        data = json.loads(text)
+    elif yaml is not None:
+        data = yaml.safe_load(text)
+    else:
+        raise ValueError(f"PyYAML required to load {path} (or use .json)")
+    if not isinstance(data, dict):
+        raise ValueError(f"{path}: root must be a mapping")
+    return data
+
+
+def _embedded_builtin_dict() -> dict[str, Any]:
+    """Fallback when no config files load (stdlib shipped defaults)."""
+    pkg = _package_root()
+    if pkg is not None:
+        for name in (CONFIG_JSON_BASENAME, CONFIG_BASENAME):
+            pkg_file = pkg / ".pi" / "harness" / name
+            if not pkg_file.is_file():
+                continue
+            try:
+                return _load_config_file(pkg_file)
+            except (ValueError, json.JSONDecodeError, OSError):
+                continue
+    from .heuristic_angles_shipped import SHIPPED_HEURISTIC_ANGLES
+
+    return dict(SHIPPED_HEURISTIC_ANGLES)
+
+
+def heuristic_config_from_merged(merged: dict[str, Any]) -> HeuristicAnglesConfig:
+    max_angles = int(merged.get("max_angles") or 5)
+    max_angles = max(2, min(max_angles, 8))
+
+    base_templates = _parse_angle_list(merged.get("base"), source="base")
+    raw_cats = merged.get("categories")
+    categories: dict[str, tuple[SearchAngle, ...]] = {}
+    if isinstance(raw_cats, dict):
+        for key, raw_list in raw_cats.items():
+            cat = str(key).strip().lower()
+            if not cat:
+                continue
+            categories[cat] = tuple(
+                _parse_angle_list(raw_list, source=f"categories.{cat}")
+            )
+
+    if "default" not in categories:
+        categories["default"] = (
+            SearchAngle("technical", "{query} how it works architecture", "Technical"),
+            SearchAngle("criticism", "{query} limitations criticism", "Counterpoints"),
+        )
+
+    return HeuristicAnglesConfig(
+        max_angles=max_angles,
+        base=tuple(base_templates),
+        categories=categories,
+    )
+
+
+@lru_cache(maxsize=8)
+def load_heuristic_angles_config_cached(paths_key: tuple[str, ...]) -> HeuristicAnglesConfig:
+    paths = [Path(p) for p in paths_key] if paths_key else discover_heuristic_config_paths()
+    layers: list[dict[str, Any]] = []
+    for p in paths:
+        try:
+            layers.append(_load_config_file(p))
+        except (ValueError, json.JSONDecodeError, OSError):
+            continue
+    if layers:
+        merged: dict[str, Any] = {}
+        for layer in layers:
+            merged = _merge_config_dict(merged, layer)
+    else:
+        merged = _embedded_builtin_dict()
+    return heuristic_config_from_merged(merged)
+
+
+def load_heuristic_angles_config() -> HeuristicAnglesConfig:
+    paths = discover_heuristic_config_paths()
+    return load_heuristic_angles_config_cached(tuple(str(p) for p in paths))
+
+
+def clear_heuristic_config_cache() -> None:
+    load_heuristic_angles_config_cached.cache_clear()
+
+
+def build_heuristic_angles(
+    query: str,
+    *,
+    category: str | None = None,
+    config: HeuristicAnglesConfig | None = None,
+) -> tuple[SearchAngle, ...]:
+    cfg = config or load_heuristic_angles_config()
+    q = query.strip()
+    cat = (category or "").strip().lower()
+
+    angles: list[SearchAngle] = []
+    for tmpl in cfg.base:
+        angles.append(
+            SearchAngle(
+                id=tmpl.id,
+                query=_format_query(tmpl.query, q),
+                rationale=tmpl.rationale,
+            )
+        )
+
+    cat_angles = cfg.categories.get(cat) if cat else None
+    if not cat_angles:
+        cat_angles = cfg.categories.get("default", ())
+
+    for tmpl in cat_angles:
+        angles.append(
+            SearchAngle(
+                id=tmpl.id,
+                query=_format_query(tmpl.query, q),
+                rationale=tmpl.rationale,
+            )
+        )
+
+    # Stable dedupe by id (first wins — base before category)
+    seen: set[str] = set()
+    unique: list[SearchAngle] = []
+    for a in angles:
+        if a.id in seen:
+            continue
+        seen.add(a.id)
+        unique.append(a)
+
+    return tuple(unique[: cfg.max_angles])

package/.pi/scripts/harness_web/highlights.py

@@ -0,0 +1,47 @@
+"""Query-aligned excerpt extraction from page markdown."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def _tokenize(text: str) -> set[str]:
+    return {t for t in re.findall(r"[a-z0-9]{3,}", text.lower()) if len(t) >= 3}
+
+
+def extract_highlights(
+    markdown: str,
+    query: str,
+    *,
+    max_spans: int = 5,
+    max_chars_per_span: int = 400,
+) -> list[dict[str, Any]]:
+    q_tokens = _tokenize(query)
+    if not q_tokens:
+        return []
+
+    paragraphs = [p.strip() for p in re.split(r"\n\s*\n", markdown) if p.strip()]
+    if not paragraphs:
+        paragraphs = [line.strip() for line in markdown.splitlines() if line.strip()]
+
+    scored: list[tuple[float, int, str]] = []
+    for idx, para in enumerate(paragraphs):
+        if len(para) < 40:
+            continue
+        tokens = _tokenize(para)
+        if not tokens:
+            continue
+        overlap = len(q_tokens & tokens) / max(len(q_tokens), 1)
+        scored.append((overlap, idx, para))
+
+    scored.sort(key=lambda x: (-x[0], x[1]))
+    out: list[dict[str, Any]] = []
+    for score, idx, para in scored[:max_spans]:
+        if score <= 0:
+            continue
+        text = para[:max_chars_per_span]
+        if len(para) > max_chars_per_span:
+            text += "…"
+        out.append({"score": round(score, 4), "paragraph_index": idx, "text": text})
+    return out

package/.pi/scripts/harness_web/multi_search.py

@@ -0,0 +1,59 @@
+"""Parallel SERP queries per search angle."""
+
+from __future__ import annotations
+
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from .config import HarnessWebConfig
+from .query_angles import AnglesPlan, SearchAngle
+from .search import search
+
+
+def _concurrency() -> int:
+    raw = os.environ.get("HARNESS_WEB_DEEP_CONCURRENCY", "4").strip()
+    try:
+        return max(1, min(8, int(raw)))
+    except ValueError:
+        return 4
+
+
+def multi_search(
+    plan: AnglesPlan,
+    *,
+    per_angle_limit: int,
+    config: HarnessWebConfig,
+    rate_limit_ms: int | None = None,
+) -> dict[str, list[dict[str, str]]]:
+    """Run search() for each angle; return angle_id -> hits."""
+    sleep_sec = (rate_limit_ms if rate_limit_ms is not None else config.rate_limit_ms) / 1000.0
+    results: dict[str, list[dict[str, str]]] = {}
+    angles = list(plan.angles)
+
+    def run_one(angle: SearchAngle) -> tuple[str, list[dict[str, str]]]:
+        hits = search(angle.query, limit=per_angle_limit, config=config)
+        tagged = []
+        for i, h in enumerate(hits):
+            row = dict(h)
+            row["_angle_id"] = angle.id
+            row["_angle_rank"] = str(i + 1)
+            tagged.append(row)
+        return angle.id, tagged
+
+    if len(angles) == 1:
+        aid, hits = run_one(angles[0])
+        results[aid] = hits
+        return results
+
+    with ThreadPoolExecutor(max_workers=min(_concurrency(), len(angles))) as pool:
+        futures = {pool.submit(run_one, a): a for a in angles}
+        done = 0
+        for fut in as_completed(futures):
+            aid, hits = fut.result()
+            results[aid] = hits
+            done += 1
+            if done < len(angles) and sleep_sec > 0:
+                time.sleep(sleep_sec)
+
+    return results

package/.pi/scripts/harness_web/output.py

@@ -24,6 +24,7 @@ def write_search_results(
     query: str,
     *,
     engine: str,
+    tier: str = "standard",
 ) -> None:
     """Firecrawl-compatible envelope: data.web[].url|title|description."""
     write_json(
@@ -31,6 +32,7 @@ def write_search_results(
         {
             "query": query,
             "engine": engine,
+            "tier": tier,
             "data": {
                 "web": [
                     {
@@ -45,6 +47,28 @@ def write_search_results(
     )
 
 
+def write_deep_search_results(
+    path: Path,
+    *,
+    query: str,
+    engine: str,
+    tier: str,
+    plan_angles: list[dict],
+    ranked_web: list[dict],
+) -> None:
+    write_json(
+        path,
+        {
+            "query": query,
+            "engine": engine,
+            "mode": tier,
+            "tier": tier,
+            "angles": plan_angles,
+            "data": {"web": ranked_web},
+        },
+    )
+
+
 def write_page_markdown(path: Path, page: Any, *, main_content_only: bool = True) -> None:
     ensure_parent(path)
     try:

package/.pi/scripts/harness_web/query_angles.py

@@ -0,0 +1,116 @@
+"""Parse and validate WRS search angles (YAML/JSON)."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+try:
+    import yaml  # type: ignore[import-untyped]
+except ImportError:
+    yaml = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True)
+class SearchAngle:
+    id: str
+    query: str
+    rationale: str = ""
+
+
+@dataclass(frozen=True)
+class AnglesPlan:
+    intent: str
+    angles: tuple[SearchAngle, ...]
+    category: str | None = None
+
+
+def _heuristic_angles(query: str, *, category: str | None = None) -> AnglesPlan:
+    """Emergency fallback when no expander output — templates from YAML config."""
+    from .heuristic_config import build_heuristic_angles, load_heuristic_angles_config
+
+    q = query.strip()
+    cfg = load_heuristic_angles_config()
+    built = build_heuristic_angles(q, category=category, config=cfg)
+    if len(built) < 2:
+        built = (
+            SearchAngle("definitional", q, "Core intent phrasing"),
+            SearchAngle("official", f"{q} official documentation", "Authoritative sources"),
+        )
+    return AnglesPlan(intent=q, angles=built, category=category or None)
+
+
+def _parse_angle_item(raw: Any, idx: int) -> SearchAngle:
+    if isinstance(raw, str):
+        s = raw.strip()
+        if not s:
+            raise ValueError(f"angles[{idx}]: empty query string")
+        return SearchAngle(id=f"angle_{idx + 1}", query=s)
+    if not isinstance(raw, dict):
+        raise ValueError(f"angles[{idx}]: expected object or string")
+    aid = str(raw.get("id") or raw.get("name") or f"angle_{idx + 1}").strip()
+    query = str(raw.get("query") or "").strip()
+    if not query:
+        raise ValueError(f"angles[{idx}]: missing query")
+    rationale = str(raw.get("rationale") or raw.get("reason") or "").strip()
+    return SearchAngle(id=aid or f"angle_{idx + 1}", query=query, rationale=rationale)
+
+
+def _load_structured(data: dict[str, Any]) -> AnglesPlan:
+    intent = str(data.get("intent") or data.get("query") or "").strip()
+    raw_angles = data.get("angles")
+    if not isinstance(raw_angles, list) or not raw_angles:
+        raise ValueError("angles: expected non-empty list")
+    angles = tuple(_parse_angle_item(item, i) for i, item in enumerate(raw_angles))
+    if len(angles) < 2:
+        raise ValueError("angles: need at least 2 entries for deep search")
+    if len(angles) > 8:
+        angles = angles[:8]
+    category = data.get("category")
+    cat_str = str(category).strip() if category else None
+    return AnglesPlan(intent=intent or angles[0].query, angles=angles, category=cat_str)
+
+
+def load_angles_file(path: Path) -> AnglesPlan:
+    text = path.read_text(encoding="utf-8")
+    # Strip markdown fences if present
+    fenced = re.search(r"```(?:ya?ml|json)?\s*\n([\s\S]*?)```", text)
+    if fenced:
+        text = fenced.group(1)
+    text = text.strip()
+    if not text:
+        raise ValueError(f"empty angles file: {path}")
+
+    data: Any
+    if text.startswith("{"):
+        data = json.loads(text)
+    elif yaml is not None:
+        data = yaml.safe_load(text)
+    else:
+        raise SystemExit(
+            "angles file is YAML but PyYAML is not installed. "
+            "Use JSON angles or: pip install pyyaml"
+        )
+    if not isinstance(data, dict):
+        raise ValueError("angles file root must be an object")
+    return _load_structured(data)
+
+
+def resolve_angles(
+    query: str,
+    *,
+    angles_file: Path | None = None,
+    expand_heuristic: bool = False,
+    category: str | None = None,
+) -> AnglesPlan:
+    if angles_file is not None:
+        return load_angles_file(angles_file)
+    if expand_heuristic:
+        return _heuristic_angles(query, category=category)
+    raise SystemExit(
+        "deep search requires --angles-file (.web/angles.yaml from web-query-expander) "
+        "or --expand-heuristic for emergency fallback"
+    )

package/.pi/scripts/harness_web/rank.py

@@ -0,0 +1,163 @@
+"""URL normalization and RRF fusion for multi-angle SERP results."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import parse_qs, urlparse, urlunparse
+
+RRF_K = 60
+
+_TRACKING_PARAMS = frozenset(
+    {
+        "utm_source",
+        "utm_medium",
+        "utm_campaign",
+        "utm_term",
+        "utm_content",
+        "fbclid",
+        "gclid",
+        "mc_cid",
+        "mc_eid",
+    }
+)
+
+
+@dataclass
+class RankedHit:
+    url: str
+    title: str
+    description: str
+    score: float
+    angle_ids: list[str] = field(default_factory=list)
+    ranks: dict[str, int] = field(default_factory=dict)
+
+    def to_web_dict(self) -> dict[str, Any]:
+        return {
+            "url": self.url,
+            "title": self.title,
+            "description": self.description,
+            "score": round(self.score, 6),
+            "angle_ids": list(self.angle_ids),
+            "ranks": dict(self.ranks),
+        }
+
+
+def normalize_url(url: str) -> str:
+    u = url.strip()
+    if not u:
+        return ""
+    parsed = urlparse(u)
+    scheme = (parsed.scheme or "https").lower()
+    host = (parsed.hostname or "").lower()
+    if not host:
+        return u
+    port = parsed.port
+    netloc = host
+    if port and not ((scheme == "http" and port == 80) or (scheme == "https" and port == 443)):
+        netloc = f"{host}:{port}"
+    path = parsed.path or "/"
+    if path != "/" and path.endswith("/"):
+        path = path.rstrip("/")
+    qs = parse_qs(parsed.query, keep_blank_values=False)
+    filtered = []
+    for key in sorted(qs.keys()):
+        if key.lower() in _TRACKING_PARAMS:
+            continue
+        for val in qs[key]:
+            filtered.append(f"{key}={val}")
+    query = "&".join(filtered)
+    return urlunparse((scheme, netloc, path, "", query, ""))
+
+
+def tokenize(text: str) -> set[str]:
+    return {t for t in re.findall(r"[a-z0-9]{3,}", text.lower()) if len(t) >= 3}
+
+
+def lexical_rerank(hits: list[RankedHit], intent: str) -> list[RankedHit]:
+    """Lightweight O3 boost when HARNESS_WEB_RERANK=lexical."""
+    intent_tokens = tokenize(intent)
+    if not intent_tokens:
+        return hits
+
+    def lex_score(h: RankedHit) -> float:
+        blob = f"{h.title} {h.description}".lower()
+        tokens = tokenize(blob)
+        if not tokens:
+            return 0.0
+        overlap = len(intent_tokens & tokens) / max(len(intent_tokens), 1)
+        return overlap
+
+    scored = [(h, h.score + 0.15 * lex_score(h)) for h in hits]
+    scored.sort(key=lambda x: x[1], reverse=True)
+    out: list[RankedHit] = []
+    for h, s in scored:
+        out.append(
+            RankedHit(
+                url=h.url,
+                title=h.title,
+                description=h.description,
+                score=s,
+                angle_ids=h.angle_ids,
+                ranks=h.ranks,
+            )
+        )
+    return out
+
+
+def fuse_angle_results(
+    per_angle: dict[str, list[dict[str, str]]],
+    *,
+    final_limit: int = 10,
+    intent: str = "",
+    rerank_mode: str = "off",
+) -> list[RankedHit]:
+    """Reciprocal Rank Fusion across angle result lists."""
+    accum: dict[str, dict[str, Any]] = {}
+
+    for angle_id, results in per_angle.items():
+        for rank_1based, item in enumerate(results, start=1):
+            raw_url = (item.get("url") or "").strip()
+            norm = normalize_url(raw_url)
+            if not norm or not norm.startswith("http"):
+                continue
+            entry = accum.setdefault(
+                norm,
+                {
+                    "url": raw_url,
+                    "title": "",
+                    "description": "",
+                    "score": 0.0,
+                    "angle_ids": [],
+                    "ranks": {},
+                },
+            )
+            entry["score"] += 1.0 / (RRF_K + rank_1based)
+            if angle_id not in entry["angle_ids"]:
+                entry["angle_ids"].append(angle_id)
+            entry["ranks"][angle_id] = rank_1based
+            title = (item.get("title") or "").strip()
+            desc = (item.get("description") or "").strip()
+            if title and not entry["title"]:
+                entry["title"] = title
+            if desc and (not entry["description"] or len(desc) > len(entry["description"])):
+                entry["description"] = desc
+
+    hits = [
+        RankedHit(
+            url=e["url"],
+            title=e["title"],
+            description=e["description"],
+            score=e["score"],
+            angle_ids=e["angle_ids"],
+            ranks=e["ranks"],
+        )
+        for e in accum.values()
+    ]
+    hits.sort(key=lambda h: (-h.score, -len(h.angle_ids), min(h.ranks.values()) if h.ranks else 999))
+
+    if rerank_mode == "lexical" and intent:
+        hits = lexical_rerank(hits, intent)
+
+    return hits[:final_limit]