solverpilot 0.1__py3-none-any.whl

solverpilot/evaluation/metrics.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from math import isfinite
+from statistics import mean
+from typing import Mapping, Sequence
+
+
+@dataclass(frozen=True, slots=True)
+class PortfolioMetrics:
+    metric: str
+    lower_is_better: bool
+    solver_mean_costs: dict[str, float]
+    sbs_solver: str
+    sbs_cost: float
+    vbs_cost: float
+    policy_cost: float
+    gap_closure: float | None
+    policy_wins: int
+    policy_ties: int
+    instances: int
+
+
+def _validate_cost_table(costs: Mapping[str, Sequence[float]]) -> int:
+    if not costs:
+        raise ValueError("costs must contain at least one solver")
+    lengths = {len(v) for v in costs.values()}
+    if len(lengths) != 1:
+        raise ValueError("all solver cost vectors must have the same length")
+    n = lengths.pop()
+    if n == 0:
+        raise ValueError("cost vectors must not be empty")
+    for solver, values in costs.items():
+        for value in values:
+            value = float(value)
+            if not isfinite(value) or value < 0:
+                raise ValueError(f"cost for {solver!r} must be finite and non-negative")
+    return n
+
+
+def portfolio_metrics(
+    costs: Mapping[str, Sequence[float]],
+    policy_choices: Sequence[str],
+    *,
+    metric: str = "wall_time_s",
+    tie_relative_tolerance: float = 0.01,
+    policy_costs: Sequence[float] | None = None,
+) -> PortfolioMetrics:
+    """Evaluate a lower-is-better algorithm-selection portfolio.
+
+    Costs must include every cost paid by a deployable policy (e.g. inspection/planning
+    overhead when evaluating an automatic policy). SBS is the best single solver by mean
+    instance cost. VBS is the per-instance oracle. Gap closure measures how much of the
+    SBS-to-VBS opportunity is captured by the policy.
+    """
+
+    n = _validate_cost_table(costs)
+    if len(policy_choices) != n:
+        raise ValueError("policy_choices length must equal the number of instances")
+    unknown = sorted(set(policy_choices) - set(costs))
+    if unknown:
+        raise ValueError(f"policy selected unknown solvers: {unknown}")
+    if tie_relative_tolerance < 0:
+        raise ValueError("tie_relative_tolerance must be non-negative")
+
+    means = {solver: float(mean(map(float, values))) for solver, values in costs.items()}
+    # Stable lexicographic tie-break prevents dict insertion order from becoming semantics.
+    sbs_solver = min(means, key=lambda s: (means[s], s))
+    sbs_cost = means[sbs_solver]
+
+    vbs_values = [min(float(costs[s][i]) for s in costs) for i in range(n)]
+    vbs_cost = float(mean(vbs_values))
+
+    chosen_solver_values = [float(costs[choice][i]) for i, choice in enumerate(policy_choices)]
+    if policy_costs is None:
+        policy_values = chosen_solver_values
+    else:
+        if len(policy_costs) != n:
+            raise ValueError("policy_costs length must equal the number of instances")
+        policy_values = [float(v) for v in policy_costs]
+        if any(not isfinite(v) or v < 0 for v in policy_values):
+            raise ValueError("policy_costs must be finite and non-negative")
+    policy_cost = float(mean(policy_values))
+
+    denominator = sbs_cost - vbs_cost
+    if denominator <= max(1e-15, 1e-12 * max(abs(sbs_cost), abs(vbs_cost), 1.0)):
+        gap_closure = None
+    else:
+        gap_closure = (sbs_cost - policy_cost) / denominator
+
+    wins = 0
+    ties = 0
+    for i, choice in enumerate(policy_choices):
+        chosen = chosen_solver_values[i]
+        best = vbs_values[i]
+        if best == 0.0:
+            is_tie = chosen == 0.0
+        else:
+            is_tie = (chosen - best) / best <= tie_relative_tolerance
+        if is_tie:
+            ties += 1
+        # Strict win means the chosen solver is uniquely faster beyond tolerance.
+        competitors = [float(costs[s][i]) for s in costs if s != choice]
+        if competitors and all(chosen < c * (1.0 - tie_relative_tolerance) for c in competitors):
+            wins += 1
+
+    return PortfolioMetrics(
+        metric=metric,
+        lower_is_better=True,
+        solver_mean_costs=means,
+        sbs_solver=sbs_solver,
+        sbs_cost=sbs_cost,
+        vbs_cost=vbs_cost,
+        policy_cost=policy_cost,
+        gap_closure=None if gap_closure is None else float(gap_closure),
+        policy_wins=wins,
+        policy_ties=ties,
+        instances=n,
+    )
+
+
+def performance_profile(
+    costs: Mapping[str, Sequence[float]],
+    taus: Sequence[float],
+) -> dict[str, list[float]]:
+    """Compute Dolan-Moré style performance profile values for finite costs.
+
+    For each tau >= 1, returns the fraction of instances where a solver's cost is at
+    most tau times the best cost on that instance.
+    """
+
+    n = _validate_cost_table(costs)
+    taus = [float(t) for t in taus]
+    if any(t < 1.0 or not isfinite(t) for t in taus):
+        raise ValueError("all taus must be finite and >= 1")
+
+    best = [min(float(costs[s][i]) for s in costs) for i in range(n)]
+    out: dict[str, list[float]] = {}
+    for solver, values in costs.items():
+        fractions: list[float] = []
+        for tau in taus:
+            count = 0
+            for value, b in zip(values, best):
+                value = float(value)
+                ratio = 1.0 if b == 0.0 and value == 0.0 else (float("inf") if b == 0.0 else value / b)
+                if ratio <= tau:
+                    count += 1
+            fractions.append(count / n)
+        out[solver] = fractions
+    return out

solverpilot/evaluation/oracle.py

@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from math import isclose, isfinite
+from typing import Iterable, Mapping, Any
+
+
+_REFERENCE_SUCCESS = {"objective_matches_optimum"}
+
+
+class ObjectiveSense(str, Enum):
+    MINIMIZE = "minimize"
+    MAXIMIZE = "maximize"
+
+    @classmethod
+    def parse(cls, value: str | "ObjectiveSense") -> "ObjectiveSense":
+        if isinstance(value, cls):
+            return value
+        normalized = str(value).strip().lower()
+        aliases = {"min": cls.MINIMIZE, "max": cls.MAXIMIZE}
+        if normalized in aliases:
+            return aliases[normalized]
+        try:
+            return cls(normalized)
+        except ValueError as exc:
+            raise ValueError("objective_sense must be 'minimize' or 'maximize'") from exc
+
+
+class OracleStatus(str, Enum):
+    AVAILABLE = "available"
+    NO_ELIGIBLE_CANDIDATE = "no_eligible_candidate"
+    PROTOCOL_MISMATCH = "protocol_mismatch"
+    ENVIRONMENT_MISMATCH = "environment_mismatch"
+    INSTANCE_IDENTITY_MISMATCH = "instance_identity_mismatch"
+
+
+class CandidateExclusionReason(str, Enum):
+    AUTO_BACKEND = "auto_backend"
+    NOT_SOLVED = "not_solved"
+    NOT_VALIDATED = "not_validated"
+    NOT_EXACT_SUCCESS = "not_exact_success"
+    MISSING_OBJECTIVE = "missing_objective"
+    NONFINITE_OBJECTIVE = "nonfinite_objective"
+    MISSING_RUNTIME = "missing_runtime"
+    NONFINITE_RUNTIME = "nonfinite_runtime"
+    NEGATIVE_RUNTIME = "negative_runtime"
+
+
+@dataclass(frozen=True, slots=True)
+class OraclePolicy:
+    objective_sense: ObjectiveSense | str = ObjectiveSense.MINIMIZE
+    objective_atol: float = 1e-7
+    objective_rtol: float = 1e-7
+    runtime_tie_breaker: bool = False
+    runtime_tolerance_s: float = 1e-9
+    require_same_protocol: bool = True
+    require_same_environment: bool = True
+    require_same_instance_sha256: bool = True
+    include_auto_backend: bool = False
+    runtime_field: str = "wall_s"
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "objective_sense", ObjectiveSense.parse(self.objective_sense))
+        for name in ("objective_atol", "objective_rtol", "runtime_tolerance_s"):
+            raw = getattr(self, name)
+            if isinstance(raw, bool) or type(raw) not in (int, float):
+                raise ValueError(f"{name} must be numeric")
+            value = float(raw)
+            if not isfinite(value) or value < 0:
+                raise ValueError(f"{name} must be finite and non-negative")
+            object.__setattr__(self, name, value)
+        for name in (
+            "runtime_tie_breaker",
+            "require_same_protocol",
+            "require_same_environment",
+            "require_same_instance_sha256",
+            "include_auto_backend",
+        ):
+            if type(getattr(self, name)) is not bool:
+                raise ValueError(f"{name} must be boolean")
+        if not isinstance(self.runtime_field, str) or not self.runtime_field.strip():
+            raise ValueError("runtime_field must be a non-empty dotted field name")
+        object.__setattr__(self, "runtime_field", self.runtime_field.strip())
+
+
+@dataclass(frozen=True, slots=True)
+class CandidateExclusion:
+    run_id: str
+    backend: str
+    reason: CandidateExclusionReason
+    detail: str
+
+
+@dataclass(frozen=True, slots=True)
+class OracleCandidate:
+    run_id: str
+    backend: str
+    objective: float
+    runtime_s: float
+
+
+@dataclass(frozen=True, slots=True)
+class OracleEntry:
+    instance: str
+    instance_sha256: str | None
+    protocol_id: str | None
+    environment_id: str | None
+    status: OracleStatus
+    objective_sense: ObjectiveSense
+    oracle_objective: float | None
+    oracle_run_ids: tuple[str, ...]
+    eligible_run_ids: tuple[str, ...]
+    fastest_oracle_runtime_s: float | None
+    candidates: tuple[OracleCandidate, ...]
+    exclusions: tuple[CandidateExclusion, ...]
+    reasons: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class OracleTable:
+    policy: OraclePolicy
+    entries: tuple[OracleEntry, ...]
+
+    def by_instance(self) -> dict[str, OracleEntry]:
+        return {entry.instance: entry for entry in self.entries}
+
+
+def _dotted_value(row: Mapping[str, Any], field: str) -> Any:
+    current: Any = row
+    for part in field.split("."):
+        if not isinstance(current, Mapping) or part not in current:
+            return None
+        current = current[part]
+    return current
+
+
+def _exact_success(row: Mapping[str, Any]) -> bool:
+    if row.get("state") != "solved" or row.get("validated") is not True:
+        return False
+    check = row.get("reference_check")
+    if check in _REFERENCE_SUCCESS:
+        return True
+    if check == "not_checkable":
+        evidence = row.get("optimality_evidence")
+        return (
+            row.get("public_status") == "valid_optimal"
+            and isinstance(evidence, Mapping)
+            and evidence.get("independently_verified_optimal") is True
+        )
+    return False
+
+
+def _candidate(row: Mapping[str, Any], policy: OraclePolicy) -> tuple[OracleCandidate | None, CandidateExclusion | None]:
+    run_id = str(row.get("run_id") or "")
+    backend = str(row.get("backend") or "")
+    if not run_id:
+        raise ValueError("every oracle row must provide a non-empty run_id")
+    if not backend:
+        raise ValueError(f"run {run_id!r} has no backend")
+    if backend == "@auto" and not policy.include_auto_backend:
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.AUTO_BACKEND, "automatic policy rows are excluded from the direct-run oracle")
+    if row.get("state") != "solved":
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.NOT_SOLVED, f"state={row.get('state')!r}")
+    if row.get("validated") is not True:
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.NOT_VALIDATED, "independent validation did not pass")
+    if not _exact_success(row):
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.NOT_EXACT_SUCCESS, f"reference_check={row.get('reference_check')!r}, public_status={row.get('public_status')!r}")
+    raw_objective = row.get("objective")
+    if raw_objective is None:
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.MISSING_OBJECTIVE, "eligible exact run has no objective")
+    try:
+        objective = float(raw_objective)
+    except (TypeError, ValueError):
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.NONFINITE_OBJECTIVE, "objective is not numeric")
+    if not isfinite(objective):
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.NONFINITE_OBJECTIVE, "objective is not finite")
+    raw_runtime = _dotted_value(row, policy.runtime_field)
+    if raw_runtime is None:
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.MISSING_RUNTIME, f"runtime field {policy.runtime_field!r} is missing")
+    try:
+        runtime = float(raw_runtime)
+    except (TypeError, ValueError):
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.NONFINITE_RUNTIME, "runtime is not numeric")
+    if not isfinite(runtime):
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.NONFINITE_RUNTIME, "runtime is not finite")
+    if runtime < 0:
+        return None, CandidateExclusion(run_id, backend, CandidateExclusionReason.NEGATIVE_RUNTIME, "runtime must be non-negative")
+    return OracleCandidate(run_id, backend, objective, runtime), None
+
+
+def _same_objective(a: float, b: float, policy: OraclePolicy) -> bool:
+    return isclose(a, b, abs_tol=policy.objective_atol, rel_tol=policy.objective_rtol)
+
+
+def _identity_value(row: Mapping[str, Any], field: str) -> str | None:
+    raw = row.get(field)
+    if raw is None:
+        return None
+    text = str(raw).strip()
+    return text or None
+
+
+def _valid_sha256(value: str | None) -> bool:
+    return value is not None and len(value) == 64 and all(ch in "0123456789abcdefABCDEF" for ch in value)
+
+
+def build_oracle_table(rows: Iterable[Mapping[str, Any]], policy: OraclePolicy | None = None) -> OracleTable:
+    policy = OraclePolicy() if policy is None else policy
+    data = tuple(rows)
+    if not data:
+        raise ValueError("oracle construction requires at least one benchmark row")
+    run_ids = [str(row.get("run_id") or "") for row in data]
+    if any(not run_id for run_id in run_ids):
+        raise ValueError("every oracle row must provide a non-empty run_id")
+    if len(set(run_ids)) != len(run_ids):
+        raise ValueError("run_id values must be unique")
+
+    grouped: dict[str, list[Mapping[str, Any]]] = {}
+    for row in data:
+        instance = str(row.get("instance") or "")
+        if not instance:
+            raise ValueError(f"run {row.get('run_id')!r} has no instance")
+        grouped.setdefault(instance, []).append(row)
+
+    entries: list[OracleEntry] = []
+    for instance in sorted(grouped):
+        group = grouped[instance]
+        protocols = {_identity_value(r, "protocol_id") for r in group}
+        environments = {_identity_value(r, "environment_id") for r in group}
+        instance_hashes = {_identity_value(r, "instance_sha256") for r in group}
+        status: OracleStatus | None = None
+        reasons: list[str] = []
+        if policy.require_same_protocol and (len(protocols) != 1 or None in protocols):
+            status = OracleStatus.PROTOCOL_MISMATCH
+            reasons.append("candidate rows must provide one identical non-empty protocol_id")
+        if status is None and policy.require_same_environment and (len(environments) != 1 or None in environments):
+            status = OracleStatus.ENVIRONMENT_MISMATCH
+            reasons.append("candidate rows must provide one identical non-empty environment_id")
+        if status is None and policy.require_same_instance_sha256:
+            if len(instance_hashes) != 1 or None in instance_hashes or not all(_valid_sha256(x) for x in instance_hashes):
+                status = OracleStatus.INSTANCE_IDENTITY_MISMATCH
+                reasons.append("candidate rows must provide one identical valid instance_sha256")
+
+        candidates: list[OracleCandidate] = []
+        exclusions: list[CandidateExclusion] = []
+        for row in sorted(group, key=lambda r: str(r.get("run_id"))):
+            candidate, exclusion = _candidate(row, policy)
+            if candidate is not None:
+                candidates.append(candidate)
+            if exclusion is not None:
+                exclusions.append(exclusion)
+
+        oracle_value: float | None = None
+        oracle_candidates: list[OracleCandidate] = []
+        fastest_runtime: float | None = None
+        if status is None:
+            if not candidates:
+                status = OracleStatus.NO_ELIGIBLE_CANDIDATE
+                reasons.append("no validated exact-success row with finite objective and runtime")
+            else:
+                values = [c.objective for c in candidates]
+                oracle_value = min(values) if policy.objective_sense is ObjectiveSense.MINIMIZE else max(values)
+                oracle_candidates = [c for c in candidates if _same_objective(c.objective, oracle_value, policy)]
+                fastest_runtime = min(c.runtime_s for c in oracle_candidates)
+                if policy.runtime_tie_breaker:
+                    oracle_candidates = [
+                        c for c in oracle_candidates
+                        if isclose(c.runtime_s, fastest_runtime, abs_tol=policy.runtime_tolerance_s, rel_tol=0.0)
+                    ]
+                status = OracleStatus.AVAILABLE
+
+        entries.append(OracleEntry(
+            instance=instance,
+            instance_sha256=next(iter(instance_hashes)) if len(instance_hashes) == 1 else None,
+            protocol_id=next(iter(protocols)) if len(protocols) == 1 else None,
+            environment_id=next(iter(environments)) if len(environments) == 1 else None,
+            status=status,
+            objective_sense=policy.objective_sense,
+            oracle_objective=oracle_value,
+            oracle_run_ids=tuple(sorted(c.run_id for c in oracle_candidates)),
+            eligible_run_ids=tuple(sorted(c.run_id for c in candidates)),
+            fastest_oracle_runtime_s=fastest_runtime,
+            candidates=tuple(candidates),
+            exclusions=tuple(exclusions),
+            reasons=tuple(reasons),
+        ))
+    return OracleTable(policy=policy, entries=tuple(entries))

solverpilot/evaluation/proof.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from enum import IntEnum
+from typing import Any, Mapping
+
+
+class ProofLevel(IntEnum):
+    NONE = 0
+    SOLVER_ASSERTED = 1
+    PRIMAL_VALIDATED = 2
+    BOUND_VERIFIED = 3
+    INDEPENDENT_CERTIFICATE = 4
+
+
+@dataclass(frozen=True, slots=True)
+class ProofAssessment:
+    level: ProofLevel
+    backend_reported_optimal: bool
+    primal_validated: bool
+    dual_verified: bool
+    gap_verified: bool
+    certificate_verified: bool
+    independently_verified_optimal: bool
+    reasons: tuple[str, ...] = ()
+
+    def to_dict(self) -> dict[str, Any]:
+        payload=asdict(self); payload["level"]=int(self.level); payload["level_name"]=self.level.name.lower(); return payload
+
+
+def _from_flags(*, backend_reported_optimal: bool, primal_validated: bool, dual_verified: bool, gap_verified: bool, certificate_verified: bool) -> ProofAssessment:
+    independently=bool(primal_validated and dual_verified and (gap_verified or certificate_verified)); reasons=[]
+    if certificate_verified and independently: level=ProofLevel.INDEPENDENT_CERTIFICATE
+    elif primal_validated and dual_verified and gap_verified: level=ProofLevel.BOUND_VERIFIED
+    elif primal_validated: level=ProofLevel.PRIMAL_VALIDATED
+    elif backend_reported_optimal: level=ProofLevel.SOLVER_ASSERTED
+    else: level=ProofLevel.NONE
+    if backend_reported_optimal and not primal_validated: reasons.append("backend optimality claim lacks an independently validated primal candidate")
+    if primal_validated and not independently: reasons.append("primal feasibility is validated, but complete optimality proof is not independently reconstructed")
+    return ProofAssessment(level,backend_reported_optimal,primal_validated,dual_verified,gap_verified,certificate_verified,independently,tuple(reasons))
+
+
+def assess_solve_result(result: object) -> ProofAssessment:
+    evidence=getattr(result,"optimality_evidence",None)
+    if evidence is None: return _from_flags(backend_reported_optimal=False,primal_validated=False,dual_verified=False,gap_verified=False,certificate_verified=False)
+    return _from_flags(backend_reported_optimal=bool(getattr(evidence,"backend_reported_optimal",False)),primal_validated=bool(getattr(evidence,"primal_validated",False)),dual_verified=bool(getattr(evidence,"dual_verified",False)),gap_verified=bool(getattr(evidence,"gap_verified",False)),certificate_verified=bool(getattr(evidence,"certificate_verified",False)))
+
+
+def assess_benchmark_row(row: Mapping[str, Any]) -> ProofAssessment:
+    evidence=row.get("optimality_evidence")
+    if isinstance(evidence,Mapping):
+        return _from_flags(backend_reported_optimal=bool(evidence.get("backend_reported_optimal",row.get("public_status")=="valid_optimal")),primal_validated=bool(evidence.get("primal_validated",row.get("validated") is True)),dual_verified=bool(evidence.get("dual_verified",False)),gap_verified=bool(evidence.get("gap_verified",False)),certificate_verified=bool(evidence.get("certificate_verified",False)))
+    return _from_flags(backend_reported_optimal=row.get("public_status")=="valid_optimal",primal_validated=row.get("validated") is True,dual_verified=False,gap_verified=False,certificate_verified=False)

solverpilot/evaluation/references.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from math import isfinite
+
+
+@dataclass(frozen=True, slots=True)
+class ObjectiveReference:
+    name: str
+    status: str
+    objective: float | None
+
+
+def parse_miplib_solu(text: str) -> dict[str, ObjectiveReference]:
+    """Parse a MIPLIB-style ``.solu`` objective/status file.
+
+    Supported records are ``=opt=``, ``=best=``, ``=inf=``, and ``=unkn=``.
+    The parser is intentionally strict about duplicate names and malformed objectives so
+    benchmark validation cannot silently use ambiguous reference data.
+    """
+
+    out: dict[str, ObjectiveReference] = {}
+    tag_map = {
+        "=opt=": "optimal",
+        "=best=": "best_known",
+        "=inf=": "infeasible",
+        "=unkn=": "unknown",
+    }
+    for line_no, raw in enumerate(text.splitlines(), start=1):
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        parts = line.split()
+        if len(parts) < 2 or parts[0] not in tag_map:
+            raise ValueError(f"line {line_no}: malformed .solu record")
+        tag, name = parts[0], parts[1]
+        if name in out:
+            raise ValueError(f"line {line_no}: duplicate .solu instance {name!r}")
+        status = tag_map[tag]
+        objective = None
+        if status in {"optimal", "best_known"}:
+            if len(parts) != 3:
+                raise ValueError(f"line {line_no}: {tag} record requires one objective")
+            try:
+                objective = float(parts[2])
+            except ValueError as exc:
+                raise ValueError(f"line {line_no}: invalid objective {parts[2]!r}") from exc
+            if not isfinite(objective):
+                raise ValueError(f"line {line_no}: objective must be finite")
+        elif len(parts) != 2:
+            raise ValueError(f"line {line_no}: {tag} record must not contain an objective")
+        out[name] = ObjectiveReference(name=name, status=status, objective=objective)
+    return out
+
+
+def parse_test_manifest(text: str) -> tuple[str, ...]:
+    """Parse a one-instance-per-line benchmark manifest such as MIPLIB ``.test``."""
+
+    names: list[str] = []
+    seen: set[str] = set()
+    for line_no, raw in enumerate(text.splitlines(), start=1):
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        if any(ch.isspace() for ch in line):
+            raise ValueError(f"line {line_no}: manifest entry must be a single filename")
+        if line in seen:
+            raise ValueError(f"line {line_no}: duplicate manifest entry {line!r}")
+        seen.add(line)
+        names.append(line)
+    if not names:
+        raise ValueError("manifest contains no instances")
+    return tuple(names)