mcp-plesk-dev-docs 0.4.2__py3-none-any.whl

plesk_unified/benchmark_engines.py

@@ -0,0 +1,330 @@
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass
+from typing import Any
+
+_WORD_RE = re.compile(r"[A-Za-z0-9_./+-]+")
+_STOPWORDS = {
+    "a",
+    "an",
+    "and",
+    "are",
+    "for",
+    "from",
+    "how",
+    "in",
+    "is",
+    "of",
+    "on",
+    "or",
+    "the",
+    "to",
+    "via",
+    "with",
+    "what",
+    "which",
+    "when",
+    "where",
+    "who",
+    "why",
+}
+
+
+@dataclass(frozen=True)
+class SearchResult:
+    text: str
+    title: str
+    score: float
+    metadata: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class RoutingDecision:
+    engine: str
+    pilot_config: StructurePilotConfig | None
+    reason: str
+
+
+@dataclass(frozen=True)
+class StructurePilotConfig:
+    name: str
+    title_weight: float = 2.5
+    breadcrumb_weight: float = 1.8
+    filename_weight: float = 0.9
+    text_weight: float = 0.35
+    title_phrase_bonus: float = 1.5
+    breadcrumb_phrase_bonus: float = 1.0
+    rank_weight: float = 0.5
+
+
+DEFAULT_PILOT_CONFIG = StructurePilotConfig(name="base")
+
+PILOT_CONFIGS: list[StructurePilotConfig] = [
+    DEFAULT_PILOT_CONFIG,
+    StructurePilotConfig(
+        name="title-focused",
+        title_weight=3.0,
+        breadcrumb_weight=1.5,
+        filename_weight=0.8,
+        text_weight=0.25,
+        title_phrase_bonus=2.0,
+        breadcrumb_phrase_bonus=0.75,
+        rank_weight=0.35,
+    ),
+    StructurePilotConfig(
+        name="breadcrumb-focused",
+        title_weight=2.2,
+        breadcrumb_weight=2.4,
+        filename_weight=1.0,
+        text_weight=0.2,
+        title_phrase_bonus=1.25,
+        breadcrumb_phrase_bonus=1.5,
+        rank_weight=0.4,
+    ),
+    StructurePilotConfig(
+        name="metadata-heavy",
+        title_weight=2.8,
+        breadcrumb_weight=2.2,
+        filename_weight=1.2,
+        text_weight=0.15,
+        title_phrase_bonus=1.75,
+        breadcrumb_phrase_bonus=1.25,
+        rank_weight=0.3,
+    ),
+]
+
+
+def tokenize(text: str) -> list[str]:
+    tokens = [token.lower() for token in _WORD_RE.findall(text or "")]
+    return [token for token in tokens if token not in _STOPWORDS and len(token) > 1]
+
+
+def bucket_query(query: str) -> str:
+    """Classify a query as structural, lookup, or multi-hop for reporting/routing."""
+    normalized = (query or "").lower()
+
+    # N-gram markers for compound/multi-hop queries
+    multi_hop_markers = (
+        " and ",
+        " then ",
+        " also ",
+        "together",
+        "combined",
+        " followed by ",
+        " as well as ",
+    )
+
+    structural_markers = (
+        "how to",
+        "add ",
+        "create ",
+        "configure ",
+        "register ",
+        "package ",
+        "restart ",
+        "install ",
+        "set up",
+    )
+    lookup_markers = (
+        "list ",
+        "show ",
+        "find ",
+        "what is",
+        "where is",
+        "retrieve ",
+        "get ",
+        "authenticate ",
+    )
+
+    if any(marker in normalized for marker in multi_hop_markers):
+        return "multi-hop"
+    if any(marker in normalized for marker in structural_markers):
+        return "structural"
+    if any(marker in normalized for marker in lookup_markers):
+        return "lookup"
+    return "mixed"
+
+
+def _field_score(
+    query_terms: list[str], value: str | None, weight: float, phrase_bonus: float
+) -> float:
+    if not value:
+        return 0.0
+
+    value_lower = value.lower()
+    matches = sum(1 for term in query_terms if term in value_lower)
+    if matches == 0:
+        return 0.0
+
+    phrase_match = (
+        phrase_bonus
+        if " ".join(query_terms[:2]) in value_lower and len(query_terms) >= 2
+        else 0.0
+    )
+    return weight * (1.0 + math.log1p(matches) + phrase_match)
+
+
+def structure_pilot_score(
+    query: str,
+    result: dict[str, Any],
+    rank: int,
+    total: int,
+    config: StructurePilotConfig = DEFAULT_PILOT_CONFIG,
+) -> float:
+    """Compute a PageIndex-inspired score from title and breadcrumb structure."""
+    query_terms = tokenize(query)
+    if not query_terms:
+        return 0.0
+
+    title = result.get("title", "")
+    breadcrumb = result.get("breadcrumb", "")
+    filename = result.get("filename", "")
+    text = result.get("text", "")
+
+    base_rank_bonus = 0.0
+    if total > 1:
+        base_rank_bonus = (total - rank) / (total - 1)
+
+    score = 0.0
+    score += _field_score(
+        query_terms, title, config.title_weight, config.title_phrase_bonus
+    )
+    score += _field_score(
+        query_terms,
+        breadcrumb,
+        config.breadcrumb_weight,
+        config.breadcrumb_phrase_bonus,
+    )
+    score += _field_score(query_terms, filename, config.filename_weight, 0.0)
+    score += _field_score(query_terms, text, config.text_weight, 0.0)
+    score += base_rank_bonus * config.rank_weight
+
+    normalized_query = " ".join(query_terms)
+    if normalized_query and normalized_query in (title or "").lower():
+        score += config.title_phrase_bonus
+    if normalized_query and normalized_query in (breadcrumb or "").lower():
+        score += config.breadcrumb_phrase_bonus
+
+    return score
+
+
+def rerank_with_structure(
+    query: str,
+    results: list[dict[str, Any]],
+    config: StructurePilotConfig = DEFAULT_PILOT_CONFIG,
+) -> list[dict[str, Any]]:
+    """Return results sorted by a structure-aware pilot score."""
+    total = len(results)
+    scored = []
+    for rank, result in enumerate(results, start=1):
+        enriched = dict(result)
+        enriched["_pilot_bucket"] = bucket_query(query)
+        enriched["_pilot_config"] = config.name
+        enriched["_pilot_score"] = structure_pilot_score(
+            query, enriched, rank, total, config=config
+        )
+        scored.append(enriched)
+
+    scored.sort(
+        key=lambda item: (
+            item.get("_pilot_score", 0.0),
+            item.get("_score_tq", item.get("_distance", item.get("_score", 0.0))),
+        ),
+        reverse=True,
+    )
+    return scored
+
+
+def get_pilot_configs() -> list[StructurePilotConfig]:
+    return list(PILOT_CONFIGS)
+
+
+def get_pilot_config_by_name(name: str) -> StructurePilotConfig:
+    by_name = {cfg.name: cfg for cfg in PILOT_CONFIGS}
+    return by_name.get(name, DEFAULT_PILOT_CONFIG)
+
+
+def list_routing_policies() -> dict[str, str]:
+    return {
+        "baseline-only": "Always use baseline retrieval for every query.",
+        "adaptive": (
+            "Route multi-hop and targeted structural intents to pageindex-pilot; "
+            "keep lookup and generic intents on baseline."
+        ),
+        "aggressive": "Always use pageindex-pilot with breadcrumb-focused config.",
+    }
+
+
+def route_query(
+    query: str,
+    bucket: str,
+    routing_policy: str = "baseline-only",
+) -> RoutingDecision:
+    """Select retrieval engine per-query based on policy and intent markers."""
+    normalized_policy = (routing_policy or "baseline-only").strip().lower()
+    normalized_bucket = (bucket or bucket_query(query)).strip().lower()
+    query_lower = (query or "").lower()
+
+    if normalized_policy == "baseline-only":
+        return RoutingDecision(
+            engine="baseline",
+            pilot_config=None,
+            reason="policy-baseline-only",
+        )
+
+    if normalized_policy == "aggressive":
+        return RoutingDecision(
+            engine="pageindex-pilot",
+            pilot_config=get_pilot_config_by_name("breadcrumb-focused"),
+            reason="policy-aggressive",
+        )
+
+    if normalized_policy != "adaptive":
+        return RoutingDecision(
+            engine="baseline",
+            pilot_config=None,
+            reason="policy-unknown-fallback",
+        )
+
+    multi_hop_markers = (
+        " and ",
+        " then ",
+        " also ",
+        "together",
+        "combined",
+    )
+    structural_markers = (
+        "how to",
+        "add ",
+        "create ",
+        "where is",
+        "which section",
+        "which page",
+        "how do i",
+    )
+
+    if normalized_bucket == "multi-hop" or any(
+        m in query_lower for m in multi_hop_markers
+    ):
+        return RoutingDecision(
+            engine="pageindex-pilot",
+            pilot_config=get_pilot_config_by_name("breadcrumb-focused"),
+            reason="adaptive-multi-hop",
+        )
+
+    if normalized_bucket == "structural" and any(
+        m in query_lower for m in structural_markers
+    ):
+        return RoutingDecision(
+            engine="pageindex-pilot",
+            pilot_config=get_pilot_config_by_name("base"),
+            reason="adaptive-structural",
+        )
+
+    return RoutingDecision(
+        engine="baseline",
+        pilot_config=None,
+        reason="adaptive-baseline",
+    )

plesk_unified/benchmark_gates.py

@@ -0,0 +1,254 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from statistics import mean
+from typing import Any
+
+DEFAULT_GATE_CONFIG: dict[str, Any] = {
+    "regression": {
+        "hit_rate": {"max_drop": 0.01},
+        "mrr": {"max_drop": 0.02},
+        "avg_latency_s": {"max_increase_ratio": 0.20},
+    },
+    "absolute_minimums": {
+        "context_recall": 0.85,
+        "faithfulness": 0.90,
+    },
+    "required_metrics": [],
+}
+
+_NUMERIC_METRICS = (
+    "hit_rate",
+    "mrr",
+    "avg_latency_s",
+    "faithfulness",
+    "context_recall",
+    "context_precision",
+)
+
+
+def _identity(run: dict[str, Any]) -> tuple[str, str, str, str]:
+    return (
+        str(run.get("suite", "control")),
+        str(run.get("profile", "unknown")),
+        str(run.get("engine", "baseline")),
+        str(run.get("routing_policy", "baseline-only")),
+    )
+
+
+def _identity_string(run: dict[str, Any]) -> str:
+    suite, profile, engine, routing = _identity(run)
+    return (
+        f"suite={suite}, profile={profile}, engine={engine}, routing_policy={routing}"
+    )
+
+
+def aggregate_runs(runs: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Aggregate repeated runs by identity, averaging numeric quality metrics."""
+    grouped: dict[tuple[str, str, str, str], list[dict[str, Any]]] = {}
+    for run in runs:
+        grouped.setdefault(_identity(run), []).append(run)
+
+    aggregated: list[dict[str, Any]] = []
+    for key, items in grouped.items():
+        base = {
+            "suite": key[0],
+            "profile": key[1],
+            "engine": key[2],
+            "routing_policy": key[3],
+            "n_runs": len(items),
+        }
+        for metric in _NUMERIC_METRICS:
+            values = [
+                item[metric]
+                for item in items
+                if isinstance(item.get(metric), (int, float))
+            ]
+            if values:
+                base[metric] = float(mean(values))
+        aggregated.append(base)
+
+    aggregated.sort(key=_identity)
+    return aggregated
+
+
+def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    out = dict(base)
+    for key, value in override.items():
+        if isinstance(value, dict) and isinstance(out.get(key), dict):
+            out[key] = _deep_merge(out[key], value)
+        else:
+            out[key] = value
+    return out
+
+
+def load_gate_config(path: str | None) -> dict[str, Any]:
+    config = dict(DEFAULT_GATE_CONFIG)
+    if not path:
+        return config
+
+    override = json.loads(Path(path).read_text(encoding="utf-8"))
+    if not isinstance(override, dict):
+        raise ValueError("Gate config must be a JSON object.")
+    return _deep_merge(config, override)
+
+
+def build_baseline_payload(runs: list[dict[str, Any]]) -> dict[str, Any]:
+    return {
+        "version": 1,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "runs": aggregate_runs(runs),
+    }
+
+
+def write_baseline(path: str, runs: list[dict[str, Any]]) -> dict[str, Any]:
+    payload = build_baseline_payload(runs)
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    return payload
+
+
+def load_baseline(path: str) -> list[dict[str, Any]]:
+    payload = json.loads(Path(path).read_text(encoding="utf-8"))
+    if isinstance(payload, list):
+        return aggregate_runs(payload)
+    if isinstance(payload, dict) and isinstance(payload.get("runs"), list):
+        return aggregate_runs(payload["runs"])
+    raise ValueError("Baseline file must be a JSON list or object with 'runs'.")
+
+
+def _check_required_metrics(run, run_name, metrics, failures):
+    """Check that all required metrics are present in the run."""
+    for metric in metrics:
+        if metric not in run:
+            failures.append(
+                f"Missing required metric '{metric}' in current run: {run_name}"
+            )
+
+
+def _check_regression_gates(run, baseline, run_name, config, failures, warnings):
+    """Check metrics against baseline to detect regressions."""
+    for metric, cfg in config.items():
+        if metric not in run:
+            warnings.append(f"Metric '{metric}' missing in current run: {run_name}")
+            continue
+        if metric not in baseline:
+            warnings.append(f"Metric '{metric}' missing in baseline run: {run_name}")
+            continue
+
+        curr = float(run[metric])
+        base = float(baseline[metric])
+
+        if "max_drop" in cfg:
+            drop = base - curr
+            if drop > float(cfg["max_drop"]):
+                failures.append(
+                    f"Regression gate failed for {metric} ({run_name}): "
+                    f"drop={drop:.4f}, allowed={float(cfg['max_drop']):.4f}"
+                )
+        if "max_increase_ratio" in cfg and base > 0:
+            ratio = (curr - base) / base
+            if ratio > float(cfg["max_increase_ratio"]):
+                failures.append(
+                    f"Regression gate failed for {metric} ({run_name}): "
+                    f"increase_ratio={ratio:.4f}, "
+                    f"allowed={float(cfg['max_increase_ratio']):.4f}"
+                )
+
+
+def _check_absolute_minimums(run, run_name, config, failures, warnings):
+    """Check metrics against absolute minimum thresholds."""
+    for metric, threshold in config.items():
+        if metric not in run:
+            warnings.append(
+                f"Absolute gate metric '{metric}' missing in current run: {run_name}"
+            )
+            continue
+        if float(run[metric]) < float(threshold):
+            failures.append(
+                f"Absolute gate failed for {metric} ({run_name}): "
+                f"value={float(run[metric]):.4f}, threshold={float(threshold):.4f}"
+            )
+
+
+def _check_metrics(
+    run: dict[str, Any],
+    baseline: dict[str, Any],
+    run_name: str,
+    gate_config: dict[str, Any],
+    failures: list[str],
+    warnings: list[str],
+):
+    """Internal helper to check metrics for a single run against gate config."""
+    _check_required_metrics(
+        run, run_name, gate_config.get("required_metrics", []), failures
+    )
+    _check_regression_gates(
+        run,
+        baseline,
+        run_name,
+        gate_config.get("regression", {}),
+        failures,
+        warnings,
+    )
+    _check_absolute_minimums(
+        run, run_name, gate_config.get("absolute_minimums", {}), failures, warnings
+    )
+
+
+def evaluate_quality_gates(
+    current_runs: list[dict[str, Any]],
+    baseline_runs: list[dict[str, Any]],
+    gate_config: dict[str, Any],
+) -> dict[str, Any]:
+    current = aggregate_runs(current_runs)
+    baseline_map = {_identity(run): run for run in aggregate_runs(baseline_runs)}
+
+    failures: list[str] = []
+    warnings: list[str] = []
+
+    for run in current:
+        ident = _identity(run)
+        run_name = _identity_string(run)
+        baseline = baseline_map.get(ident)
+
+        if baseline is None:
+            warnings.append(f"No baseline run matched: {run_name}")
+            continue
+
+        _check_metrics(run, baseline, run_name, gate_config, failures, warnings)
+
+    return {
+        "passed": len(failures) == 0,
+        "failures": failures,
+        "warnings": warnings,
+        "current_count": len(current),
+        "baseline_count": len(baseline_map),
+    }
+
+
+def format_gate_report(report: dict[str, Any]) -> str:
+    lines = [
+        "\nQUALITY GATE REPORT",
+        "-" * 60,
+        f"Current runs : {report.get('current_count', 0)}",
+        f"Baseline runs: {report.get('baseline_count', 0)}",
+    ]
+
+    warnings = report.get("warnings", [])
+    failures = report.get("failures", [])
+
+    if warnings:
+        lines.append("Warnings:")
+        lines.extend(f"  - {w}" for w in warnings)
+
+    if failures:
+        lines.append("Failures:")
+        lines.extend(f"  - {f}" for f in failures)
+    else:
+        lines.append("All configured quality gates passed.")
+
+    return "\n".join(lines)

plesk_unified/benchmark_reporting.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+Terminal reporting helpers for benchmark results.
+
+Pure presentation logic — no internal package dependencies.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def print_result(result: dict[str, Any], final_k: int) -> None:
+    """Print a single profile/engine benchmark result."""
+    print(f"  Hit Rate (HR@{final_k}) : {result['hit_rate']:.1%}")
+    print(f"  MRR@{final_k}           : {result['mrr']:.3f}")
+    if "faithfulness" in result:
+        print(f"  Faithfulness       : {result['faithfulness']:.3f}")
+        print(f"  Context Recall     : {result['context_recall']:.3f}")
+        print(f"  Context Precision  : {result['context_precision']:.3f}")
+    print(f"  Avg latency      : {result['avg_latency_s']:.3f}s")
+    print(f"  Model RSS delta  : {result['model_rss_mb']:.0f} MB")
+    print(f"  Routing policy   : {result['routing_policy']}")
+    if result.get("bucket_metrics"):
+        for bucket_name, metrics in result["bucket_metrics"].items():
+            print(
+                f"  {bucket_name.title()} MRR  : {metrics.get('mrr', 0.0):.3f} "
+                f"(n={metrics.get('n', 0)})"
+            )
+
+    print("\n  Per-query results:")
+    for pq in result["per_query"]:
+        status = "HIT " if pq["hit"] else "MISS"
+        print(
+            f"    {status} [{pq['latency_s']:.2f}s] [{pq['bucket']}] "
+            f"[{pq['selected_engine']}] {pq['query'][:70]}"
+        )
+
+
+def print_summary_table(all_results: list[dict[str, Any]]) -> None:
+    """Print a side-by-side comparison table of all benchmark results."""
+    if len(all_results) <= 1:
+        return
+
+    print(f"\n{'=' * 60}")
+    print("SUMMARY")
+    print("=" * 60)
+    has_ragas = any("faithfulness" in r for r in all_results)
+    if has_ragas:
+        header = (
+            f"{'Profile':<10} {'Engine':<15} {'HR@5':>8} {'MRR@5':>8} "
+            f"{'Faith':>8} {'Recall':>8} {'Prec':>8} {'Latency':>10}"
+        )
+    else:
+        header = (
+            f"{'Profile':<10} {'Engine':<15} {'HR@5':>8} {'MRR@5':>8} "
+            f"{'Latency':>10} {'RSS MB':>10}"
+        )
+    print(header)
+    print("-" * len(header))
+    for result in all_results:
+        if has_ragas:
+            print(
+                f"{result['profile']:<10} {result['engine']:<15} "
+                f"{result['hit_rate']:>7.1%} "
+                f"{result['mrr']:>8.3f} "
+                f"{result.get('faithfulness', 0.0):>8.3f} "
+                f"{result.get('context_recall', 0.0):>8.3f} "
+                f"{result.get('context_precision', 0.0):>8.3f} "
+                f"{result['avg_latency_s']:>9.3f}s"
+            )
+        else:
+            print(
+                f"{result['profile']:<10} {result['engine']:<15} "
+                f"{result['hit_rate']:>7.1%} "
+                f"{result['mrr']:>8.3f} "
+                f"{result['avg_latency_s']:>9.3f}s "
+                f"{result['model_rss_mb']:>9.0f}"
+            )
+
+
+def print_autoresearch_summary(all_results: list[dict[str, Any]]) -> None:
+    """Print the autoresearch summary, highlighting the best structural config."""
+    if not all_results:
+        return
+
+    pageindex_runs = [r for r in all_results if r.get("engine") == "pageindex-pilot"]
+    if not pageindex_runs:
+        return
+
+    structural_best = max(
+        pageindex_runs,
+        key=lambda r: r.get("bucket_metrics", {}).get("structural", {}).get("mrr", 0.0),
+    )
+    structural_metrics = structural_best.get("bucket_metrics", {}).get("structural", {})
+    structural_mrr = structural_metrics.get("mrr", 0.0)
+    print("\nAUTORESEARCH SUMMARY")
+    print("-" * 60)
+    print(
+        f"Best structural config: {structural_best.get('pilot_config') or 'base'} "
+        f"(MRR={structural_mrr:.3f})"
+    )
+    print(
+        "Stop condition: if the structural MRR no longer improves across the "
+        "pilot configs, "
+        "the structure-aware ceiling has been reached."
+    )