solverpilot 0.1__py3-none-any.whl

solverpilot/benchmark/policy.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+from collections import defaultdict
+import json
+import math
+from pathlib import Path
+from statistics import median
+from typing import Mapping
+
+from solverpilot.evaluation import portfolio_metrics
+
+from .summary import (
+    _get_cost,
+    _paired_bootstrap_policy_ratio,
+    _require_comparable_environment,
+    _require_experiment_identity,
+    _require_instance_identity,
+    _require_protocol_identity,
+    _row_is_benchmark_success,
+    load_rows,
+)
+
+
+def _finite_nonnegative(value: object, *, label: str) -> float:
+    try:
+        out = float(value)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"{label} must be numeric") from exc
+    if not math.isfinite(out) or out < 0:
+        raise ValueError(f"{label} must be finite and non-negative")
+    return out
+
+
+def load_policy_map(path: str | Path) -> dict[str, dict]:
+    """Load a frozen external selector policy without inventing missing overhead."""
+    payload = json.loads(Path(path).read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise ValueError("policy map must be a JSON object keyed by instance")
+    out: dict[str, dict] = {}
+    for instance, raw in payload.items():
+        if isinstance(raw, str):
+            out[str(instance)] = {
+                "backend": raw,
+                "overhead_s": None,
+                "overhead_measured": False,
+                "overhead_components_s": {},
+            }
+            continue
+        if not isinstance(raw, dict) or not isinstance(raw.get("backend"), str):
+            raise ValueError(f"policy entry for {instance!r} must provide backend")
+        components_raw = raw.get("overhead_components_s", {})
+        if components_raw is None:
+            components_raw = {}
+        if not isinstance(components_raw, Mapping):
+            raise ValueError(f"policy overhead_components_s for {instance!r} must be an object")
+        components = {
+            str(name): _finite_nonnegative(value, label=f"policy overhead component {name!r} for {instance!r}")
+            for name, value in components_raw.items()
+        }
+        explicit_total = "overhead_s" in raw and raw.get("overhead_s") is not None
+        if explicit_total:
+            overhead = _finite_nonnegative(raw["overhead_s"], label=f"policy overhead for {instance!r}")
+            if components:
+                component_sum = sum(components.values())
+                if not math.isclose(overhead, component_sum, rel_tol=1e-9, abs_tol=1e-12):
+                    raise ValueError(f"policy overhead for {instance!r} disagrees with overhead_components_s: total={overhead}, components={component_sum}")
+            measured = True
+        elif components:
+            overhead = float(sum(components.values())); measured = True
+        else:
+            overhead = None; measured = False
+        out[str(instance)] = {
+            "backend": raw["backend"],
+            "overhead_s": overhead,
+            "overhead_measured": measured,
+            "overhead_components_s": components,
+        }
+    return out
+
+
+def evaluate_policy_map(
+    rows: list[dict],
+    policy: dict[str, dict],
+    *,
+    cutoff_s: float,
+    par_penalty: float = 10.0,
+    cost_field: str = "wall_s",
+    bootstrap_draws: int = 10000,
+    bootstrap_seed: int = 0,
+    allow_mixed_environments: bool = False,
+    allow_unmeasured_overhead: bool = False,
+) -> dict:
+    if cutoff_s <= 0 or par_penalty < 1:
+        raise ValueError("invalid cutoff/par_penalty")
+    environment_ids = _require_comparable_environment(rows, allow_mixed_environments=allow_mixed_environments)
+    experiment_ids = _require_experiment_identity(rows)
+    protocol_id = _require_protocol_identity(rows)
+    _require_instance_identity(rows)
+    direct_rows = [r for r in rows if r.get("backend") != "@auto"]
+    backends = sorted({str(r["backend"]) for r in direct_rows})
+    instances = sorted({str(r["instance"]) for r in direct_rows})
+    if set(policy) != set(instances):
+        missing = sorted(set(instances) - set(policy)); extra = sorted(set(policy) - set(instances))
+        raise ValueError(f"policy map must exactly cover benchmark instances; missing={missing[:10]}, extra={extra[:10]}")
+
+    unmeasured = sorted(instance for instance in instances if not bool(policy[instance].get("overhead_measured", policy[instance].get("overhead_s") is not None)))
+    if unmeasured and not allow_unmeasured_overhead:
+        raise ValueError(
+            "policy decision overhead is unmeasured for instances "
+            f"{unmeasured[:10]}; provide overhead_s/overhead_components_s or pass "
+            "allow_unmeasured_overhead=True explicitly for non-deployable analysis"
+        )
+
+    grouped: dict[tuple[str, str], list[dict]] = defaultdict(list)
+    for row in direct_rows:
+        grouped[(str(row["instance"]), str(row["backend"]))].append(row)
+    costs = {b: [] for b in backends}
+    for instance in instances:
+        for backend in backends:
+            good = []
+            for r in grouped.get((instance, backend), []):
+                if _row_is_benchmark_success(r):
+                    v = _get_cost(r, cost_field)
+                    if v is not None: good.append(min(v, cutoff_s))
+            costs[backend].append(float(median(good)) if good else cutoff_s * par_penalty)
+
+    choices = []; policy_costs = []; overhead_values: list[float] = []
+    for idx, instance in enumerate(instances):
+        entry = policy[instance]; backend = entry["backend"]
+        if backend not in costs:
+            raise ValueError(f"policy selected backend {backend!r} not present in benchmark")
+        choices.append(backend)
+        raw_overhead = entry.get("overhead_s")
+        overhead = 0.0 if raw_overhead is None else _finite_nonnegative(raw_overhead, label=f"policy overhead for {instance!r}")
+        overhead_values.append(overhead); policy_costs.append(float(costs[backend][idx] + overhead))
+
+    metrics = portfolio_metrics(costs, choices, metric=f"PAR{par_penalty:g}@{cutoff_s:g}s", policy_costs=policy_costs)
+    ratio = _paired_bootstrap_policy_ratio(policy_costs, costs[metrics.sbs_solver], draws=bootstrap_draws, seed=bootstrap_seed)
+    overhead_complete = not unmeasured
+    deployable_cost_scope = cost_field in {"wall_s", "worker_solve_wall_s"}
+    return {
+        "schema_version": "1.1",
+        "protocol_id": protocol_id,
+        "environment_ids": sorted(environment_ids),
+        "experiment_identities": sorted(experiment_ids),
+        "mixed_environments_allowed": bool(allow_mixed_environments),
+        "instances": len(instances),
+        "cost_field": cost_field,
+        "cutoff_s": cutoff_s,
+        "par_penalty": par_penalty,
+        "overhead_accounting": {
+            "complete": overhead_complete,
+            "unmeasured_instances": unmeasured,
+            "allow_unmeasured_overhead": bool(allow_unmeasured_overhead),
+            "assumed_zero_for_unmeasured": bool(unmeasured and allow_unmeasured_overhead),
+            "mean_measured_or_assumed_overhead_s": sum(overhead_values) / len(overhead_values),
+        },
+        "cost_accounting": {
+            "solver_cost_field": cost_field,
+            "external_policy_overhead_added": True,
+            "deployable_cost_scope": deployable_cost_scope,
+            "deployable_evidence": bool(overhead_complete and deployable_cost_scope and not allow_mixed_environments),
+        },
+        "vbs": {"cost": metrics.vbs_cost, "is_oracle": True, "deployable": False},
+        "policy_metrics": {
+            "metric": metrics.metric,
+            "solver_mean_costs": metrics.solver_mean_costs,
+            "sbs_solver": metrics.sbs_solver,
+            "sbs_cost": metrics.sbs_cost,
+            "vbs_cost": metrics.vbs_cost,
+            "policy_cost": metrics.policy_cost,
+            "gap_closure": metrics.gap_closure,
+            "policy_wins": metrics.policy_wins,
+            "policy_ties": metrics.policy_ties,
+            "policy_to_sbs_ratio_bootstrap_95pct": ratio,
+        },
+        "choices": {instance: policy[instance] for instance in instances},
+    }
+
+
+def evaluate_policy_jsonl(result_paths: list[str | Path], policy_path: str | Path, **kwargs) -> dict:
+    return evaluate_policy_map(load_rows(result_paths), load_policy_map(policy_path), **kwargs)

solverpilot/benchmark/registry.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from solverpilot.backends import BackendRegistry, BundledOSQPCAPIBackend, BundledHighsCAPIBackend
+from solverpilot.runtime.auto import builtin_backend_candidates
+
+
+def benchmark_registry(backends: tuple[str, ...]) -> BackendRegistry:
+    backends = tuple(b for b in backends if b != "@auto")
+    if not backends:
+        raise ValueError("benchmark registry needs at least one explicit backend; @auto is evaluated over that portfolio")
+    candidates = (*builtin_backend_candidates(), BundledOSQPCAPIBackend(), BundledHighsCAPIBackend())
+    by_name = {b.manifest.name: b for b in candidates}
+    unknown = sorted(set(backends) - set(by_name))
+    if unknown:
+        raise KeyError(f"unknown benchmark backends: {unknown}")
+    registry = BackendRegistry()
+    unavailable = []
+    unverified = []
+    for name in backends:
+        backend = by_name[name]
+        if not backend.is_available():
+            unavailable.append(name)
+            continue
+        verified_bridge_versions = backend.manifest.metadata.get("verified_bridge_package_versions")
+        bridge_version = backend.manifest.metadata.get("bridge_package_version")
+        if verified_bridge_versions is not None and bridge_version not in set(verified_bridge_versions):
+            unverified.append((name, f"bridge={bridge_version}", tuple(verified_bridge_versions)))
+            continue
+        verified_solver_versions = backend.manifest.metadata.get("verified_solver_versions")
+        if verified_solver_versions is not None and backend.manifest.version not in set(verified_solver_versions):
+            unverified.append((name, f"solver={backend.manifest.version}", tuple(verified_solver_versions)))
+            continue
+        registry.register(backend)
+    if unavailable:
+        raise RuntimeError(f"requested backends are unavailable: {unavailable}")
+    if unverified:
+        raise RuntimeError(f"requested verification bridges use unverified package versions: {unverified}")
+    return registry

solverpilot/benchmark/report.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+from typing import Any, Mapping
+
+from .campaign import CampaignSpec
+
+
+def scientific_report_payload(campaign: CampaignSpec, summary: Mapping[str,Any], *, campaign_run: Mapping[str,Any]|None=None, limitations: tuple[str,...]=()) -> dict[str,Any]:
+    vbs=summary.get("vbs") or {"cost":summary.get("vbs_cost"),"is_oracle":True,"deployable":False}
+    if vbs.get("deployable") is not False or vbs.get("is_oracle") is not True:
+        raise ValueError("scientific report requires VBS to be explicitly oracle/non-deployable")
+    provenance_ok=None if campaign_run is None else bool(campaign_run.get("scientific_provenance_verified"))
+    deployable=bool(provenance_ok and not summary.get("mixed_environments_allowed",False) and summary.get("instance_identity_verified") is True)
+    return {"schema_version":"solverpilot.scientific-report.v1","generated_at_utc":datetime.now(timezone.utc).isoformat(),"campaign":campaign.to_dict(),"campaign_sha256":campaign.sha256,"scientific_provenance_verified":provenance_ok,"deployable_scientific_evidence":deployable,"instances":summary.get("instances"),"backends":summary.get("backends"),"cost_field":summary.get("cost_field"),"sbs":{"backend":summary.get("sbs_solver"),"cost":summary.get("sbs_cost")},"vbs":vbs,"sbs_vbs_gap":summary.get("sbs_vbs_gap"),"relative_sbs_vbs_gap":summary.get("relative_sbs_vbs_gap"),"solver_stats":summary.get("solver_stats"),"auto_policy_metrics":summary.get("auto_policy_metrics"),"limitations":list(limitations),"claim_boundary":"performance claims may be made only when deployable_scientific_evidence=true; VBS is an oracle baseline and is never a deployable selector"}
+
+
+def render_scientific_report_markdown(payload: Mapping[str,Any])->str:
+    sbs=payload.get("sbs") or {}; vbs=payload.get("vbs") or {}
+    lines=["# SolverPilot Scientific Benchmark Report","",f"- Campaign: `{payload.get('campaign',{}).get('campaign_id')}`",f"- Campaign SHA-256: `{payload.get('campaign_sha256')}`",f"- Scientific provenance verified: `{payload.get('scientific_provenance_verified')}`",f"- Deployable scientific evidence: `{payload.get('deployable_scientific_evidence')}`",f"- Instances: `{payload.get('instances')}`",f"- Cost field: `{payload.get('cost_field')}`","","## Baselines","",f"- SBS: `{sbs.get('backend')}` — cost `{sbs.get('cost')}`",f"- VBS oracle cost: `{vbs.get('cost')}` (oracle=`{vbs.get('is_oracle')}`, deployable=`{vbs.get('deployable')}`)","","## Claim boundary","",str(payload.get("claim_boundary"))]
+    if payload.get("limitations"): lines += ["","## Limitations",""]+[f"- {x}" for x in payload["limitations"]]
+    return "\n".join(lines)+"\n"
+
+
+def write_scientific_report(payload:Mapping[str,Any], *, json_path:str|Path, markdown_path:str|Path)->None:
+    Path(json_path).write_text(json.dumps(payload,indent=2,sort_keys=True,default=str),encoding="utf-8")
+    Path(markdown_path).write_text(render_scientific_report_markdown(payload),encoding="utf-8")
+
+
+def compute_scientific_metrics(rows:list[dict[str,Any]], *, cutoff_s:float, cost_field:str="wall_s")->dict[str,Any]:
+    from .summary import summarize_rows
+    par2=summarize_rows(rows,cutoff_s=cutoff_s,par_penalty=2.0,cost_field=cost_field); par10=summarize_rows(rows,cutoff_s=cutoff_s,par_penalty=10.0,cost_field=cost_field)
+    direct=[r for r in rows if r.get("backend") != "@auto"]; backends=sorted({str(r.get("backend")) for r in direct})
+    timeout={b:sum(1 for r in direct if str(r.get("backend"))==b and (r.get("state")=="hard_timeout" or r.get("public_status")=="feasible_limit")) for b in backends}
+    gaps={b:[] for b in backends}
+    for row in direct:
+        raw=row.get("relative_gap",row.get("optimality_gap"))
+        if raw is None: continue
+        try: value=float(raw)
+        except (TypeError,ValueError): continue
+        if value>=0: gaps[str(row.get("backend"))].append(value)
+    return {"schema_version":"solverpilot.scientific-metrics.v1","cost_field":cost_field,"cutoff_s":float(cutoff_s),"par2":{"sbs_solver":par2["sbs_solver"],"sbs_cost":par2["sbs_cost"],"vbs_cost":par2["vbs_cost"],"solver_stats":par2["solver_stats"]},"par10":{"sbs_solver":par10["sbs_solver"],"sbs_cost":par10["sbs_cost"],"vbs_cost":par10["vbs_cost"],"solver_stats":par10["solver_stats"]},"timeout_count":timeout,"mean_reported_relative_gap":{b:(None if not v else sum(v)/len(v)) for b,v in gaps.items()},"optimality_gap_availability":"explicit_row_field_only; null means the backend/run did not expose a normalized gap","regret":"evaluate through solverpilot.evaluation oracle/regret APIs on a frozen held-out split","selection_overhead":"included only through deployable policy evaluation with measured overhead"}