solverpilot 0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- solverpilot/__init__.py +250 -0
- solverpilot/_immutability.py +27 -0
- solverpilot/_version.py +3 -0
- solverpilot/applications/__init__.py +3 -0
- solverpilot/applications/routing/__init__.py +19 -0
- solverpilot/applications/routing/compile_milp.py +321 -0
- solverpilot/applications/routing/diagnostics.py +78 -0
- solverpilot/applications/routing/errors.py +14 -0
- solverpilot/applications/routing/heuristics.py +167 -0
- solverpilot/applications/routing/model.py +259 -0
- solverpilot/applications/routing/reference.py +81 -0
- solverpilot/applications/routing/solution.py +89 -0
- solverpilot/applications/routing/timeline.py +79 -0
- solverpilot/applications/routing/validation.py +169 -0
- solverpilot/applications/tsp/__init__.py +41 -0
- solverpilot/applications/tsp/compile_cp.py +121 -0
- solverpilot/applications/tsp/compile_milp.py +139 -0
- solverpilot/applications/tsp/distance.py +60 -0
- solverpilot/applications/tsp/errors.py +17 -0
- solverpilot/applications/tsp/heuristics.py +112 -0
- solverpilot/applications/tsp/model.py +175 -0
- solverpilot/applications/tsp/reference.py +121 -0
- solverpilot/applications/tsp/solution.py +65 -0
- solverpilot/applications/tsp/validation.py +92 -0
- solverpilot/backends/__init__.py +51 -0
- solverpilot/backends/_casadi_highs_worker.py +92 -0
- solverpilot/backends/base.py +28 -0
- solverpilot/backends/bundled_capi.py +910 -0
- solverpilot/backends/casadi_conic.py +303 -0
- solverpilot/backends/health.py +240 -0
- solverpilot/backends/highspy_native.py +217 -0
- solverpilot/backends/nlopt_native.py +237 -0
- solverpilot/backends/osqp_native.py +297 -0
- solverpilot/backends/protocol_v2.py +119 -0
- solverpilot/backends/pyscipopt_native.py +200 -0
- solverpilot/backends/registry.py +38 -0
- solverpilot/backends/scipy_highs.py +138 -0
- solverpilot/backends/scipy_highs_lp.py +195 -0
- solverpilot/backends/scipy_slsqp_qp.py +133 -0
- solverpilot/backends/scipy_vendored_highs_dev.py +363 -0
- solverpilot/benchmark/__init__.py +23 -0
- solverpilot/benchmark/acquire.py +199 -0
- solverpilot/benchmark/baselines.py +70 -0
- solverpilot/benchmark/bundle.py +37 -0
- solverpilot/benchmark/campaign.py +150 -0
- solverpilot/benchmark/environment.py +115 -0
- solverpilot/benchmark/integrity.py +46 -0
- solverpilot/benchmark/model.py +146 -0
- solverpilot/benchmark/policy.py +182 -0
- solverpilot/benchmark/registry.py +38 -0
- solverpilot/benchmark/report.py +44 -0
- solverpilot/benchmark/runner.py +516 -0
- solverpilot/benchmark/slurm.py +55 -0
- solverpilot/benchmark/specs.py +75 -0
- solverpilot/benchmark/splits.py +181 -0
- solverpilot/benchmark/summary.py +384 -0
- solverpilot/benchmark/verify.py +142 -0
- solverpilot/benchmark/worker.py +71 -0
- solverpilot/bridges/__init__.py +19 -0
- solverpilot/bridges/engine.py +44 -0
- solverpilot/bridges/indicator.py +232 -0
- solverpilot/bridges/registry.py +78 -0
- solverpilot/bridges/transformation-tape.schema.json +53 -0
- solverpilot/bridges/types.py +149 -0
- solverpilot/bridges/validation.py +110 -0
- solverpilot/capabilities/__init__.py +32 -0
- solverpilot/capabilities/backend-capability-v2.schema.json +21 -0
- solverpilot/capabilities/conformance.py +151 -0
- solverpilot/capabilities/enums.py +27 -0
- solverpilot/capabilities/manifest.py +19 -0
- solverpilot/capabilities/ontology.py +73 -0
- solverpilot/capabilities/requirements.py +37 -0
- solverpilot/capabilities/resolve.py +30 -0
- solverpilot/capabilities/v2.py +711 -0
- solverpilot/cli/__init__.py +1 -0
- solverpilot/cli/backend_health.py +77 -0
- solverpilot/cli/benchmark.py +331 -0
- solverpilot/cli/benchmark_worker.py +33 -0
- solverpilot/cli/capabilities.py +65 -0
- solverpilot/conic/__init__.py +11 -0
- solverpilot/conic/backend.py +263 -0
- solverpilot/conic/compiler.py +310 -0
- solverpilot/conic/conformance.py +60 -0
- solverpilot/conic/conic-ir-v1.schema.json +24 -0
- solverpilot/conic/ir.py +299 -0
- solverpilot/conic/validation.py +149 -0
- solverpilot/cp/__init__.py +7 -0
- solverpilot/cp/conformance.py +32 -0
- solverpilot/cp/cp-ir-v1.schema.json +13 -0
- solverpilot/cp/ir.py +170 -0
- solverpilot/cp/model.py +143 -0
- solverpilot/cp/ortools_backend.py +423 -0
- solverpilot/cp/ortools_worker.py +75 -0
- solverpilot/cp/reference.py +51 -0
- solverpilot/cp/serialization.py +117 -0
- solverpilot/cp/validate.py +83 -0
- solverpilot/diagnose/__init__.py +33 -0
- solverpilot/diagnose/infeasibility.py +463 -0
- solverpilot/diagnose/model.py +87 -0
- solverpilot/evaluation/__init__.py +53 -0
- solverpilot/evaluation/metrics.py +150 -0
- solverpilot/evaluation/oracle.py +288 -0
- solverpilot/evaluation/proof.py +53 -0
- solverpilot/evaluation/references.py +73 -0
- solverpilot/evaluation/regret.py +281 -0
- solverpilot/exceptions.py +34 -0
- solverpilot/experimental/__init__.py +32 -0
- solverpilot/extensions/__init__.py +56 -0
- solverpilot/extensions/entrypoints.py +34 -0
- solverpilot/extensions/errors.py +35 -0
- solverpilot/extensions/manager.py +651 -0
- solverpilot/extensions/manifest.py +232 -0
- solverpilot/extensions/model.py +101 -0
- solverpilot/extensions/versioning.py +136 -0
- solverpilot/history/__init__.py +22 -0
- solverpilot/history/errors.py +15 -0
- solverpilot/history/model.py +129 -0
- solverpilot/history/sqlite.py +651 -0
- solverpilot/inspect/__init__.py +3 -0
- solverpilot/inspect/fingerprint.py +199 -0
- solverpilot/intelligence/__init__.py +36 -0
- solverpilot/intelligence/errors.py +16 -0
- solverpilot/intelligence/fingerprint.py +108 -0
- solverpilot/intelligence/leakage.py +69 -0
- solverpilot/intelligence/schema.py +314 -0
- solverpilot/intelligence/shift.py +208 -0
- solverpilot/io/__init__.py +41 -0
- solverpilot/io/binary64.py +49 -0
- solverpilot/io/csv.py +230 -0
- solverpilot/io/errors.py +25 -0
- solverpilot/io/hashing.py +111 -0
- solverpilot/io/json.py +90 -0
- solverpilot/io/mapping.py +195 -0
- solverpilot/io/model.py +407 -0
- solverpilot/io/numeric.py +108 -0
- solverpilot/io/provenance.py +114 -0
- solverpilot/io/result.py +45 -0
- solverpilot/io/source.py +104 -0
- solverpilot/io/strict_json.py +133 -0
- solverpilot/minlp/__init__.py +6 -0
- solverpilot/minlp/compiler.py +66 -0
- solverpilot/minlp/conformance.py +34 -0
- solverpilot/minlp/convexity.py +85 -0
- solverpilot/minlp/ir.py +35 -0
- solverpilot/minlp/minlp-ir-v1.schema.json +1 -0
- solverpilot/minlp/orchestrator.py +220 -0
- solverpilot/minlp/validation.py +28 -0
- solverpilot/model/__init__.py +27 -0
- solverpilot/model/compiler.py +1263 -0
- solverpilot/model/errors.py +22 -0
- solverpilot/model/expression.py +373 -0
- solverpilot/model/model.py +716 -0
- solverpilot/model/sets.py +60 -0
- solverpilot/model/types.py +36 -0
- solverpilot/nlp/__init__.py +7 -0
- solverpilot/nlp/ad.py +166 -0
- solverpilot/nlp/backend.py +113 -0
- solverpilot/nlp/compiler.py +116 -0
- solverpilot/nlp/conformance.py +58 -0
- solverpilot/nlp/ir.py +70 -0
- solverpilot/nlp/nlp-ir-v1.schema.json +1 -0
- solverpilot/nlp/validation.py +60 -0
- solverpilot/plan/__init__.py +33 -0
- solverpilot/plan/model.py +79 -0
- solverpilot/plan/planner.py +201 -0
- solverpilot/plan/production.py +627 -0
- solverpilot/plan/selective_lp.py +65 -0
- solverpilot/problem/__init__.py +16 -0
- solverpilot/problem/enums.py +20 -0
- solverpilot/problem/hashing.py +88 -0
- solverpilot/problem/linear.py +196 -0
- solverpilot/problem/mps.py +447 -0
- solverpilot/problem/quadratic.py +192 -0
- solverpilot/reporting/__init__.py +27 -0
- solverpilot/reporting/explain.py +327 -0
- solverpilot/reporting/model.py +264 -0
- solverpilot/reporting/render.py +55 -0
- solverpilot/runtime/__init__.py +19 -0
- solverpilot/runtime/auto.py +193 -0
- solverpilot/runtime/budgeting.py +43 -0
- solverpilot/runtime/executor.py +126 -0
- solverpilot/runtime/portfolio.py +197 -0
- solverpilot/runtime/result.py +72 -0
- solverpilot/session/__init__.py +29 -0
- solverpilot/session/conformance_v2.py +239 -0
- solverpilot/session/mutations.py +115 -0
- solverpilot/session/persistent-session-trace.schema.json +45 -0
- solverpilot/session/persistent.py +491 -0
- solverpilot/session/reuse.py +81 -0
- solverpilot/session/session.py +236 -0
- solverpilot/trace/__init__.py +3 -0
- solverpilot/trace/schema.py +47 -0
- solverpilot/validate/__init__.py +10 -0
- solverpilot/validate/core.py +179 -0
- solverpilot/validate/result.py +55 -0
- solverpilot-0.1.dist-info/METADATA +554 -0
- solverpilot-0.1.dist-info/RECORD +201 -0
- solverpilot-0.1.dist-info/WHEEL +5 -0
- solverpilot-0.1.dist-info/entry_points.txt +4 -0
- solverpilot-0.1.dist-info/licenses/LICENSE +202 -0
- solverpilot-0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import zipfile
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def sha256_bytes(data: bytes) -> str:
|
|
10
|
+
return hashlib.sha256(data).hexdigest()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def create_result_bundle(
|
|
14
|
+
output_zip: str | Path,
|
|
15
|
+
*,
|
|
16
|
+
files: list[str | Path],
|
|
17
|
+
metadata: dict | None = None,
|
|
18
|
+
) -> dict:
|
|
19
|
+
output = Path(output_zip)
|
|
20
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
21
|
+
entries = []
|
|
22
|
+
seen_names = set()
|
|
23
|
+
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
|
24
|
+
for raw in files:
|
|
25
|
+
p = Path(raw)
|
|
26
|
+
if not p.is_file():
|
|
27
|
+
raise FileNotFoundError(p)
|
|
28
|
+
arcname = p.name
|
|
29
|
+
if arcname in seen_names:
|
|
30
|
+
raise ValueError(f"duplicate bundle filename: {arcname}")
|
|
31
|
+
seen_names.add(arcname)
|
|
32
|
+
data = p.read_bytes()
|
|
33
|
+
zf.writestr(arcname, data)
|
|
34
|
+
entries.append({"name": arcname, "size_bytes": len(data), "sha256": sha256_bytes(data)})
|
|
35
|
+
manifest = {"schema_version": "1.0", "metadata": metadata or {}, "entries": entries}
|
|
36
|
+
zf.writestr("bundle-manifest.json", json.dumps(manifest, indent=2, sort_keys=True).encode())
|
|
37
|
+
return {"path": str(output), "sha256": hashlib.sha256(output.read_bytes()).hexdigest(), "entries": entries}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import asdict, dataclass, field
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from types import MappingProxyType
|
|
8
|
+
from typing import Any, Mapping
|
|
9
|
+
|
|
10
|
+
from solverpilot.evaluation import parse_test_manifest
|
|
11
|
+
from .acquire import dataset_spec_fingerprint, sha256_file
|
|
12
|
+
from .model import RunSpec
|
|
13
|
+
from .runner import run_benchmark
|
|
14
|
+
from .specs import MIPLIB2017_BENCHMARK_V2, QPLIB_ARCHIVE
|
|
15
|
+
from .splits import validate_split_file
|
|
16
|
+
from .verify import verify_acquired_dataset
|
|
17
|
+
|
|
18
|
+
CAMPAIGN_SCHEMA_VERSION = "solverpilot.benchmark-campaign.v1"
|
|
19
|
+
_DEFAULT_METRICS = ("par2","par10","solved_count","timeout_rate","optimality_gap","regret","selection_overhead")
|
|
20
|
+
_ALLOWED_COST_FIELDS = {"wall_s","worker_solve_wall_s","trace.solve_s","trace.total_s"}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _freeze_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
24
|
+
out={}
|
|
25
|
+
for key,item in value.items():
|
|
26
|
+
if isinstance(item,Mapping): out[str(key)]=_freeze_mapping(item)
|
|
27
|
+
elif isinstance(item,list): out[str(key)]=tuple(item)
|
|
28
|
+
else: out[str(key)]=item
|
|
29
|
+
return MappingProxyType(out)
|
|
30
|
+
|
|
31
|
+
def _canonical_sha256(payload: Mapping[str, Any]) -> str:
|
|
32
|
+
return hashlib.sha256(json.dumps(payload,sort_keys=True,separators=(",",":"),default=str).encode()).hexdigest()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True, slots=True)
|
|
36
|
+
class CampaignSpec:
|
|
37
|
+
campaign_id: str
|
|
38
|
+
dataset_name: str
|
|
39
|
+
dataset_spec_sha256: str
|
|
40
|
+
backends: tuple[str, ...]
|
|
41
|
+
repetitions: int = 1
|
|
42
|
+
cutoff_s: float = 600.0
|
|
43
|
+
hard_timeout_s: float | None = None
|
|
44
|
+
objective_atol: float = 1e-6
|
|
45
|
+
objective_rtol: float = 1e-7
|
|
46
|
+
seed: int = 0
|
|
47
|
+
thread_env_limit: int | None = 1
|
|
48
|
+
solver_threads: int | None = 1
|
|
49
|
+
worker_python_mode: str = "no_site"
|
|
50
|
+
cost_field: str = "wall_s"
|
|
51
|
+
metrics: tuple[str, ...] = _DEFAULT_METRICS
|
|
52
|
+
split_sha256: str | None = None
|
|
53
|
+
require_scientific_provenance: bool = True
|
|
54
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
55
|
+
schema_version: str = CAMPAIGN_SCHEMA_VERSION
|
|
56
|
+
|
|
57
|
+
def __post_init__(self):
|
|
58
|
+
if self.schema_version != CAMPAIGN_SCHEMA_VERSION: raise ValueError(f"unsupported campaign schema: {self.schema_version!r}")
|
|
59
|
+
if not self.campaign_id.strip() or not self.dataset_name.strip(): raise ValueError("campaign_id/dataset_name must be non-empty")
|
|
60
|
+
if len(self.dataset_spec_sha256)!=64 or any(c not in "0123456789abcdefABCDEF" for c in self.dataset_spec_sha256): raise ValueError("dataset_spec_sha256 must be a SHA-256 digest")
|
|
61
|
+
if self.split_sha256 is not None and (len(self.split_sha256)!=64 or any(c not in "0123456789abcdefABCDEF" for c in self.split_sha256)): raise ValueError("split_sha256 must be a SHA-256 digest")
|
|
62
|
+
if not self.backends or len(set(self.backends)) != len(self.backends): raise ValueError("backends must be non-empty and unique")
|
|
63
|
+
if self.repetitions < 1: raise ValueError("repetitions must be >= 1")
|
|
64
|
+
if self.cutoff_s <= 0: raise ValueError("cutoff_s must be positive")
|
|
65
|
+
if self.hard_timeout_s is not None and self.hard_timeout_s < self.cutoff_s: raise ValueError("hard_timeout_s must be >= cutoff_s when provided")
|
|
66
|
+
if self.thread_env_limit is not None and self.thread_env_limit < 1: raise ValueError("thread_env_limit must be >= 1")
|
|
67
|
+
if self.solver_threads is not None and self.solver_threads < 1: raise ValueError("solver_threads must be >= 1")
|
|
68
|
+
if self.worker_python_mode not in {"normal","no_site"}: raise ValueError("worker_python_mode must be 'normal' or 'no_site'")
|
|
69
|
+
if self.cost_field not in _ALLOWED_COST_FIELDS: raise ValueError(f"unsupported campaign cost_field: {self.cost_field!r}")
|
|
70
|
+
object.__setattr__(self,"backends",tuple(map(str,self.backends))); object.__setattr__(self,"metrics",tuple(map(str,self.metrics))); object.__setattr__(self,"metadata",_freeze_mapping(dict(self.metadata)))
|
|
71
|
+
|
|
72
|
+
def to_dict(self) -> dict[str,Any]:
|
|
73
|
+
def thaw(v):
|
|
74
|
+
if isinstance(v,Mapping): return {str(k):thaw(x) for k,x in v.items()}
|
|
75
|
+
if isinstance(v,tuple): return [thaw(x) for x in v]
|
|
76
|
+
return v
|
|
77
|
+
return {"campaign_id":self.campaign_id,"dataset_name":self.dataset_name,"dataset_spec_sha256":self.dataset_spec_sha256,"backends":list(self.backends),"repetitions":self.repetitions,"cutoff_s":self.cutoff_s,"hard_timeout_s":self.hard_timeout_s,"objective_atol":self.objective_atol,"objective_rtol":self.objective_rtol,"seed":self.seed,"thread_env_limit":self.thread_env_limit,"solver_threads":self.solver_threads,"worker_python_mode":self.worker_python_mode,"cost_field":self.cost_field,"metrics":list(self.metrics),"split_sha256":self.split_sha256,"require_scientific_provenance":self.require_scientific_provenance,"metadata":thaw(self.metadata),"schema_version":self.schema_version}
|
|
78
|
+
@property
|
|
79
|
+
def sha256(self): return _canonical_sha256(self.to_dict())
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass(frozen=True, slots=True)
|
|
83
|
+
class CampaignValidation:
|
|
84
|
+
ok: bool
|
|
85
|
+
campaign_sha256: str
|
|
86
|
+
scientific_provenance_verified: bool | None
|
|
87
|
+
split_verified: bool | None
|
|
88
|
+
errors: tuple[str,...]=()
|
|
89
|
+
warnings: tuple[str,...]=()
|
|
90
|
+
def to_dict(self): return asdict(self)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def save_campaign_spec(spec: CampaignSpec,path:str|Path)->None: Path(path).write_text(json.dumps(spec.to_dict(),indent=2,sort_keys=True),encoding="utf-8")
|
|
94
|
+
def load_campaign_spec(path:str|Path)->CampaignSpec:
|
|
95
|
+
p=json.loads(Path(path).read_text(encoding="utf-8")); p["backends"]=tuple(p.get("backends",())); p["metrics"]=tuple(p.get("metrics",_DEFAULT_METRICS)); return CampaignSpec(**p)
|
|
96
|
+
|
|
97
|
+
def miplib2017_campaign_template(backends: tuple[str,...], *, repetitions:int=1, cutoff_s:float=3600.0, hard_timeout_s:float|None=None, seed:int=0, solver_threads:int|None=1)->CampaignSpec:
|
|
98
|
+
return CampaignSpec(campaign_id="miplib2017-benchmark-v2",dataset_name=MIPLIB2017_BENCHMARK_V2.name,dataset_spec_sha256=dataset_spec_fingerprint(MIPLIB2017_BENCHMARK_V2),backends=backends,repetitions=repetitions,cutoff_s=cutoff_s,hard_timeout_s=hard_timeout_s,seed=seed,solver_threads=solver_threads,require_scientific_provenance=True,metadata={"benchmark_instances":240,"benchmark_version":2,"reference_solution_version":36,"reference_solution_release":"2026-01-26","official_source":"https://miplib.zib.de/download.html","status":"protocol-ready; execution requires fully pinned official inputs"})
|
|
99
|
+
|
|
100
|
+
def qplib_campaign_template(backends: tuple[str,...], *, cutoff_s:float=3600.0, seed:int=0)->CampaignSpec:
|
|
101
|
+
return CampaignSpec(campaign_id="qplib-supported-subset-v1",dataset_name=QPLIB_ARCHIVE.name,dataset_spec_sha256=dataset_spec_fingerprint(QPLIB_ARCHIVE),backends=backends,cutoff_s=cutoff_s,seed=seed,require_scientific_provenance=True,metadata={"official_continuous_instances":134,"official_discrete_instances":319,"supported_subset_count":None,"execution_supported":False,"blocked_on":"strict QPLIB parser/filter and pinned supported-subset manifest/reference","official_source":"https://qplib.zib.de/"})
|
|
102
|
+
|
|
103
|
+
def _lock_payload(target:Path):
|
|
104
|
+
lock=target/"acquisition-lock.json"
|
|
105
|
+
if not lock.exists(): raise FileNotFoundError(lock)
|
|
106
|
+
return json.loads(lock.read_text(encoding="utf-8"))
|
|
107
|
+
def _role_path(lock:Mapping[str,Any],role:str)->Path|None:
|
|
108
|
+
for item in lock.get("files",[]):
|
|
109
|
+
if isinstance(item,Mapping) and item.get("role")==role: return Path(str(item["path"]))
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
def validate_campaign(spec:CampaignSpec, *, dataset_target:str|Path|None=None, split_path:str|Path|None=None)->CampaignValidation:
|
|
113
|
+
errors=[]; warnings=[]; scientific=None; split_verified=None; expected_split_instances=None
|
|
114
|
+
if dataset_target is not None:
|
|
115
|
+
target=Path(dataset_target); report=verify_acquired_dataset(target); scientific=report.scientific_provenance_verified; lock=_lock_payload(target)
|
|
116
|
+
if str(lock.get("dataset"))!=spec.dataset_name: errors.append("campaign dataset_name does not match acquisition lock")
|
|
117
|
+
if str(lock.get("spec_fingerprint"))!=spec.dataset_spec_sha256: errors.append("campaign dataset_spec_sha256 does not match acquisition lock")
|
|
118
|
+
if not report.ok: errors.extend(report.errors)
|
|
119
|
+
if spec.require_scientific_provenance and not report.scientific_provenance_verified: errors.append("campaign requires scientifically verified provenance, but acquisition is not fully pinned")
|
|
120
|
+
if not report.scientific_provenance_verified: warnings.append("dataset integrity may be locally verified without authenticating the official snapshot")
|
|
121
|
+
manifest=_role_path(lock,"manifest")
|
|
122
|
+
if manifest is not None and manifest.exists():
|
|
123
|
+
names=parse_test_manifest(manifest.read_text(encoding="utf-8"))
|
|
124
|
+
def key(filename):
|
|
125
|
+
lower=filename.lower()
|
|
126
|
+
for suffix in (".mps.gz",".mps",".lp.gz",".lp"):
|
|
127
|
+
if lower.endswith(suffix): return filename[:-len(suffix)]
|
|
128
|
+
return Path(filename).stem
|
|
129
|
+
expected_split_instances=tuple(key(n) for n in names)
|
|
130
|
+
if split_path is None and spec.split_sha256 is not None: errors.append("campaign binds split_sha256 but no split_path was supplied")
|
|
131
|
+
if split_path is not None:
|
|
132
|
+
path=Path(split_path); digest=sha256_file(path)
|
|
133
|
+
if spec.split_sha256 is not None and digest.lower()!=spec.split_sha256.lower(): errors.append("split artifact SHA-256 does not match campaign")
|
|
134
|
+
sr=validate_split_file(path,expected_instances=expected_split_instances); split_verified=sr.ok
|
|
135
|
+
if not sr.ok: errors.extend(sr.errors)
|
|
136
|
+
return CampaignValidation(not errors,spec.sha256,scientific,split_verified,tuple(errors),tuple(warnings))
|
|
137
|
+
|
|
138
|
+
def run_campaign(spec:CampaignSpec, *, dataset_target:str|Path, output_jsonl:str|Path, split_path:str|Path|None=None, allow_unverified_dataset:bool=False, registry=None)->dict[str,Any]:
|
|
139
|
+
validation=validate_campaign(spec,dataset_target=dataset_target,split_path=split_path)
|
|
140
|
+
if not validation.ok:
|
|
141
|
+
if not allow_unverified_dataset: raise ValueError("campaign validation failed: "+"; ".join(validation.errors))
|
|
142
|
+
remaining=[e for e in validation.errors if "scientifically verified provenance" not in e]
|
|
143
|
+
if remaining: raise ValueError("campaign validation failed: "+"; ".join(remaining))
|
|
144
|
+
target=Path(dataset_target); lock=_lock_payload(target); manifest=_role_path(lock,"manifest"); reference=_role_path(lock,"reference")
|
|
145
|
+
if manifest is None: raise ValueError("campaign dataset acquisition has no manifest")
|
|
146
|
+
if spec.require_scientific_provenance and reference is None: raise ValueError("scientific campaign requires a reference file")
|
|
147
|
+
rs=RunSpec(dataset_dir=target/"instances",manifest=manifest,reference=reference,backends=spec.backends,output_jsonl=Path(output_jsonl),repetitions=spec.repetitions,time_limit_s=spec.cutoff_s,hard_timeout_s=spec.hard_timeout_s,objective_atol=spec.objective_atol,objective_rtol=spec.objective_rtol,seed=spec.seed,thread_env_limit=spec.thread_env_limit,solver_threads=spec.solver_threads,worker_python_mode=spec.worker_python_mode)
|
|
148
|
+
summary=run_benchmark(rs,registry=registry); result_path=Path(output_jsonl)
|
|
149
|
+
sidecar={"schema_version":"solverpilot.campaign-run.v1","campaign_id":spec.campaign_id,"campaign_sha256":spec.sha256,"scientific_provenance_verified":bool(validation.scientific_provenance_verified),"allow_unverified_dataset":bool(allow_unverified_dataset),"deployable_scientific_evidence":bool(validation.scientific_provenance_verified and not allow_unverified_dataset),"result_jsonl":str(result_path),"result_jsonl_sha256":sha256_file(result_path),"run_summary":summary}
|
|
150
|
+
Path(str(result_path)+".campaign.json").write_text(json.dumps(sidecar,indent=2,sort_keys=True,default=str),encoding="utf-8"); return sidecar
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import importlib.metadata
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import platform
|
|
9
|
+
import socket
|
|
10
|
+
import sys
|
|
11
|
+
from typing import Iterable
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
_THREAD_ENV_KEYS = (
|
|
15
|
+
"OMP_NUM_THREADS",
|
|
16
|
+
"OPENBLAS_NUM_THREADS",
|
|
17
|
+
"MKL_NUM_THREADS",
|
|
18
|
+
"NUMEXPR_NUM_THREADS",
|
|
19
|
+
"VECLIB_MAXIMUM_THREADS",
|
|
20
|
+
"BLIS_NUM_THREADS",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _cpu_model() -> str | None:
|
|
25
|
+
cpuinfo = Path("/proc/cpuinfo")
|
|
26
|
+
if cpuinfo.exists():
|
|
27
|
+
for line in cpuinfo.read_text(errors="ignore").splitlines():
|
|
28
|
+
if line.lower().startswith("model name") and ":" in line:
|
|
29
|
+
return line.split(":", 1)[1].strip()
|
|
30
|
+
return platform.processor() or None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _memory_bytes() -> int | None:
|
|
34
|
+
meminfo = Path("/proc/meminfo")
|
|
35
|
+
if meminfo.exists():
|
|
36
|
+
for line in meminfo.read_text(errors="ignore").splitlines():
|
|
37
|
+
if line.startswith("MemTotal:"):
|
|
38
|
+
parts = line.split()
|
|
39
|
+
if len(parts) >= 2:
|
|
40
|
+
return int(parts[1]) * 1024
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def thread_environment(limit: int | None) -> dict[str, str]:
|
|
45
|
+
"""Return environment variables used to cap common C/BLAS thread pools.
|
|
46
|
+
|
|
47
|
+
This does not prove that a solver's own internal thread pool is controlled.
|
|
48
|
+
Backend-level thread control is represented separately by ``solver_threads`` in
|
|
49
|
+
the benchmark protocol.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
if limit is None:
|
|
53
|
+
return {}
|
|
54
|
+
if limit < 1:
|
|
55
|
+
raise ValueError("thread environment limit must be >= 1")
|
|
56
|
+
value = str(int(limit))
|
|
57
|
+
return {key: value for key in _THREAD_ENV_KEYS}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def capture_environment(*, packages: Iterable[str] = ()) -> dict:
|
|
61
|
+
versions: dict[str, str | None] = {}
|
|
62
|
+
for name in packages:
|
|
63
|
+
try:
|
|
64
|
+
versions[name] = importlib.metadata.version(name)
|
|
65
|
+
except importlib.metadata.PackageNotFoundError:
|
|
66
|
+
versions[name] = None
|
|
67
|
+
return {
|
|
68
|
+
"python": sys.version,
|
|
69
|
+
"python_executable": sys.executable,
|
|
70
|
+
"platform": platform.platform(),
|
|
71
|
+
"machine": platform.machine(),
|
|
72
|
+
"hostname": socket.gethostname(),
|
|
73
|
+
"cpu_model": _cpu_model(),
|
|
74
|
+
"logical_cpus": os.cpu_count(),
|
|
75
|
+
"memory_bytes": _memory_bytes(),
|
|
76
|
+
"packages": versions,
|
|
77
|
+
"env_threads": {key: os.environ.get(key) for key in _THREAD_ENV_KEYS},
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def benchmark_environment_fingerprint(
|
|
82
|
+
environment: dict,
|
|
83
|
+
*,
|
|
84
|
+
thread_env_limit: int | None,
|
|
85
|
+
solver_threads: int | None,
|
|
86
|
+
worker_python_mode: str = "no_site",
|
|
87
|
+
) -> str:
|
|
88
|
+
"""Hash timing-relevant execution environment fields, excluding hostname.
|
|
89
|
+
|
|
90
|
+
Hostname is deliberately excluded so homogeneous cluster nodes can merge. CPU,
|
|
91
|
+
platform, Python/package versions, and the declared thread policy are retained.
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
payload = {
|
|
95
|
+
"schema_version": "1.0",
|
|
96
|
+
"python": environment.get("python"),
|
|
97
|
+
"platform": environment.get("platform"),
|
|
98
|
+
"machine": environment.get("machine"),
|
|
99
|
+
"cpu_model": environment.get("cpu_model"),
|
|
100
|
+
"logical_cpus": environment.get("logical_cpus"),
|
|
101
|
+
"memory_bytes": environment.get("memory_bytes"),
|
|
102
|
+
"packages": environment.get("packages", {}),
|
|
103
|
+
"thread_env_limit": thread_env_limit,
|
|
104
|
+
"solver_threads": solver_threads,
|
|
105
|
+
"worker_python_mode": worker_python_mode,
|
|
106
|
+
"effective_thread_env": thread_environment(thread_env_limit),
|
|
107
|
+
}
|
|
108
|
+
raw = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
|
|
109
|
+
return hashlib.sha256(raw).hexdigest()
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def write_environment(path: str | Path, *, packages: Iterable[str] = ()) -> dict:
|
|
113
|
+
payload = capture_environment(packages=packages)
|
|
114
|
+
Path(path).write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
|
|
115
|
+
return payload
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
from typing import Any, Mapping
|
|
6
|
+
|
|
7
|
+
CURRENT_BENCHMARK_ROW_SCHEMA = "1.5"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def canonical_sha256(value: Any) -> str:
|
|
11
|
+
raw = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str).encode("utf-8")
|
|
12
|
+
return hashlib.sha256(raw).hexdigest()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def experiment_identity(*, protocol_id: str, environment_id: str, backend_set_sha256: str, thread_policy: Mapping[str, Any]) -> str:
|
|
16
|
+
return canonical_sha256({
|
|
17
|
+
"protocol_id": protocol_id,
|
|
18
|
+
"environment_id": environment_id,
|
|
19
|
+
"backend_set_sha256": backend_set_sha256,
|
|
20
|
+
"thread_policy": dict(thread_policy),
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def row_sha256(row: Mapping[str, Any]) -> str:
|
|
25
|
+
return canonical_sha256({str(k): v for k, v in row.items() if k != "row_sha256"})
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def attach_row_integrity(row: Mapping[str, Any]) -> dict[str, Any]:
|
|
29
|
+
payload = dict(row)
|
|
30
|
+
payload.pop("row_sha256", None)
|
|
31
|
+
payload["row_sha256"] = row_sha256(payload)
|
|
32
|
+
return payload
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def verify_row_integrity(row: Mapping[str, Any], *, context: str = "benchmark row") -> None:
|
|
36
|
+
schema = str(row.get("schema_version") or "")
|
|
37
|
+
recorded = row.get("row_sha256")
|
|
38
|
+
if recorded is None:
|
|
39
|
+
if schema == CURRENT_BENCHMARK_ROW_SCHEMA:
|
|
40
|
+
raise ValueError(f"{context}: schema {schema} row is missing row_sha256")
|
|
41
|
+
return
|
|
42
|
+
text = str(recorded)
|
|
43
|
+
if len(text) != 64 or any(ch not in "0123456789abcdefABCDEF" for ch in text):
|
|
44
|
+
raise ValueError(f"{context}: invalid row_sha256")
|
|
45
|
+
if row_sha256(row).lower() != text.lower():
|
|
46
|
+
raise ValueError(f"{context}: row_sha256 mismatch")
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, Mapping
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
_PROVENANCE_LEVELS = {
|
|
9
|
+
"UNSPECIFIED",
|
|
10
|
+
"VERIFIED_FROZEN",
|
|
11
|
+
"PINNED_EXTERNAL",
|
|
12
|
+
"PINNED_ARCHIVE_ONLY",
|
|
13
|
+
"UNPINNED_EXTERNAL",
|
|
14
|
+
"LOCAL_ONLY",
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _FrozenDict(dict):
|
|
19
|
+
"""Immutable dict compatible with dataclasses.asdict/json workflows."""
|
|
20
|
+
|
|
21
|
+
def _blocked(self, *args, **kwargs):
|
|
22
|
+
raise TypeError("immutable mapping")
|
|
23
|
+
|
|
24
|
+
__setitem__ = __delitem__ = clear = pop = popitem = setdefault = update = _blocked
|
|
25
|
+
|
|
26
|
+
def __deepcopy__(self, memo):
|
|
27
|
+
return dict(self)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _freeze(value: Any) -> Any:
|
|
31
|
+
if isinstance(value, Mapping):
|
|
32
|
+
return _FrozenDict({k: _freeze(v) for k, v in value.items()})
|
|
33
|
+
if isinstance(value, list):
|
|
34
|
+
return tuple(_freeze(v) for v in value)
|
|
35
|
+
if isinstance(value, tuple):
|
|
36
|
+
return tuple(_freeze(v) for v in value)
|
|
37
|
+
if isinstance(value, set):
|
|
38
|
+
return frozenset(_freeze(v) for v in value)
|
|
39
|
+
return value
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True, slots=True)
|
|
43
|
+
class DatasetSpec:
|
|
44
|
+
name: str
|
|
45
|
+
kind: str
|
|
46
|
+
archive_url: str | None = None
|
|
47
|
+
manifest_url: str | None = None
|
|
48
|
+
reference_url: str | None = None
|
|
49
|
+
archive_sha256: str | None = None
|
|
50
|
+
manifest_sha256: str | None = None
|
|
51
|
+
reference_sha256: str | None = None
|
|
52
|
+
archive_filename: str | None = None
|
|
53
|
+
manifest_filename: str | None = None
|
|
54
|
+
reference_filename: str | None = None
|
|
55
|
+
instance_suffixes: tuple[str, ...] = (".mps.gz", ".mps")
|
|
56
|
+
provenance_level: str = "UNSPECIFIED"
|
|
57
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
58
|
+
|
|
59
|
+
def __post_init__(self) -> None:
|
|
60
|
+
level = str(self.provenance_level).strip().upper()
|
|
61
|
+
if level not in _PROVENANCE_LEVELS:
|
|
62
|
+
raise ValueError(f"unsupported provenance_level: {self.provenance_level!r}")
|
|
63
|
+
object.__setattr__(self, "provenance_level", level)
|
|
64
|
+
object.__setattr__(self, "metadata", _freeze(dict(self.metadata)))
|
|
65
|
+
object.__setattr__(self, "instance_suffixes", tuple(self.instance_suffixes))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True, slots=True)
|
|
69
|
+
class RunSpec:
|
|
70
|
+
dataset_dir: Path
|
|
71
|
+
manifest: Path
|
|
72
|
+
reference: Path | None
|
|
73
|
+
backends: tuple[str, ...]
|
|
74
|
+
output_jsonl: Path
|
|
75
|
+
repetitions: int = 1
|
|
76
|
+
time_limit_s: float | None = None
|
|
77
|
+
hard_timeout_s: float | None = None
|
|
78
|
+
objective_atol: float = 1e-6
|
|
79
|
+
objective_rtol: float = 1e-7
|
|
80
|
+
shard_index: int = 0
|
|
81
|
+
shard_count: int = 1
|
|
82
|
+
resume: bool = True
|
|
83
|
+
seed: int = 0
|
|
84
|
+
thread_env_limit: int | None = 1
|
|
85
|
+
solver_threads: int | None = None
|
|
86
|
+
worker_python_mode: str = "no_site"
|
|
87
|
+
|
|
88
|
+
def __post_init__(self) -> None:
|
|
89
|
+
if self.repetitions < 1:
|
|
90
|
+
raise ValueError("repetitions must be >= 1")
|
|
91
|
+
if self.time_limit_s is not None and self.time_limit_s <= 0:
|
|
92
|
+
raise ValueError("time_limit_s must be positive")
|
|
93
|
+
if self.hard_timeout_s is not None and self.hard_timeout_s <= 0:
|
|
94
|
+
raise ValueError("hard_timeout_s must be positive")
|
|
95
|
+
if self.thread_env_limit is not None and self.thread_env_limit < 1:
|
|
96
|
+
raise ValueError("thread_env_limit must be >= 1")
|
|
97
|
+
if self.solver_threads is not None and self.solver_threads < 1:
|
|
98
|
+
raise ValueError("solver_threads must be >= 1")
|
|
99
|
+
if self.worker_python_mode not in {"normal", "no_site"}:
|
|
100
|
+
raise ValueError("worker_python_mode must be 'normal' or 'no_site'")
|
|
101
|
+
if not self.backends:
|
|
102
|
+
raise ValueError("backends must not be empty")
|
|
103
|
+
if self.shard_count < 1:
|
|
104
|
+
raise ValueError("shard_count must be >= 1")
|
|
105
|
+
if not 0 <= self.shard_index < self.shard_count:
|
|
106
|
+
raise ValueError("shard_index must be in [0, shard_count)")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@dataclass(frozen=True, slots=True)
|
|
110
|
+
class AcquisitionFile:
|
|
111
|
+
role: str
|
|
112
|
+
url: str
|
|
113
|
+
path: str
|
|
114
|
+
size_bytes: int
|
|
115
|
+
sha256: str
|
|
116
|
+
expected_sha256: str | None
|
|
117
|
+
verified: bool
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass(frozen=True, slots=True)
|
|
121
|
+
class AcquisitionRecord:
|
|
122
|
+
dataset: str
|
|
123
|
+
spec_kind: str
|
|
124
|
+
target_dir: str
|
|
125
|
+
files: tuple[AcquisitionFile, ...]
|
|
126
|
+
extracted: bool
|
|
127
|
+
extracted_files: int
|
|
128
|
+
metadata: Mapping[str, Any]
|
|
129
|
+
schema_version: str
|
|
130
|
+
spec_fingerprint: str
|
|
131
|
+
provenance_level: str
|
|
132
|
+
all_remote_inputs_pinned: bool
|
|
133
|
+
unpinned_roles: tuple[str, ...]
|
|
134
|
+
|
|
135
|
+
def __post_init__(self) -> None:
|
|
136
|
+
if not self.schema_version:
|
|
137
|
+
raise ValueError("schema_version must not be empty")
|
|
138
|
+
if len(self.spec_fingerprint) != 64 or any(ch not in "0123456789abcdefABCDEF" for ch in self.spec_fingerprint):
|
|
139
|
+
raise ValueError("spec_fingerprint must be a SHA-256 hex digest")
|
|
140
|
+
level = str(self.provenance_level).strip().upper()
|
|
141
|
+
if level not in _PROVENANCE_LEVELS:
|
|
142
|
+
raise ValueError(f"unsupported provenance_level: {self.provenance_level!r}")
|
|
143
|
+
object.__setattr__(self, "provenance_level", level)
|
|
144
|
+
object.__setattr__(self, "metadata", _freeze(dict(self.metadata)))
|
|
145
|
+
object.__setattr__(self, "files", tuple(self.files))
|
|
146
|
+
object.__setattr__(self, "unpinned_roles", tuple(sorted(set(self.unpinned_roles))))
|