qtiming 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qtiming/__init__.py +0 -0
- qtiming/bench/__init__.py +327 -0
- qtiming/bench/diff.py +230 -0
- qtiming/bench/report.py +161 -0
- qtiming/bench/truth.py +88 -0
- qtiming/cli/__init__.py +0 -0
- qtiming/cli/cmd_evidence.py +35 -0
- qtiming/cli/main.py +398 -0
- qtiming/contract/__init__.py +0 -0
- qtiming/contract/compat.py +49 -0
- qtiming/contract/instances/README.md +30 -0
- qtiming/contract/instances/artiq.toml +45 -0
- qtiming/contract/instances/ibm_heron.toml +62 -0
- qtiming/contract/instances/ibm_heron_snapshot.json +22 -0
- qtiming/contract/instances/q1.toml +88 -0
- qtiming/contract/instances/qua.toml +65 -0
- qtiming/contract/loader.py +355 -0
- qtiming/contract/model.py +123 -0
- qtiming/contract/schema.py +106 -0
- qtiming/contract/target_import.py +46 -0
- qtiming/core/__init__.py +0 -0
- qtiming/core/ast.py +140 -0
- qtiming/core/errors.py +36 -0
- qtiming/core/ids.py +34 -0
- qtiming/core/labels.py +121 -0
- qtiming/core/provenance.py +77 -0
- qtiming/core/time.py +96 -0
- qtiming/dispatch/__init__.py +22 -0
- qtiming/dispatch/artiq.py +70 -0
- qtiming/dispatch/ibm.py +83 -0
- qtiming/dispatch/q1.py +102 -0
- qtiming/dispatch/qua.py +88 -0
- qtiming/elab/__init__.py +360 -0
- qtiming/evidence/__init__.py +15 -0
- qtiming/evidence/ingest.py +235 -0
- qtiming/exec/__init__.py +0 -0
- qtiming/exec/artiq.py +100 -0
- qtiming/exec/engine.py +235 -0
- qtiming/exec/explore.py +43 -0
- qtiming/exec/ibm.py +117 -0
- qtiming/exec/oracle.py +88 -0
- qtiming/exec/q1.py +95 -0
- qtiming/exec/qua.py +95 -0
- qtiming/frontend/__init__.py +40 -0
- qtiming/frontend/_shared.py +149 -0
- qtiming/frontend/annotations.py +81 -0
- qtiming/frontend/lint.py +116 -0
- qtiming/frontend/payload.py +113 -0
- qtiming/frontend/program.py +51 -0
- qtiming/frontend/qasm3_fe.py +341 -0
- qtiming/frontend/qiskit_expr.py +89 -0
- qtiming/frontend/qiskit_fe.py +220 -0
- qtiming/frontend/serialise.py +63 -0
- qtiming/frontend/stretch.py +71 -0
- qtiming/graph/__init__.py +0 -0
- qtiming/graph/builder.py +245 -0
- qtiming/graph/export.py +25 -0
- qtiming/graph/invariants.py +67 -0
- qtiming/graph/lteg.py +153 -0
- qtiming/graph/window.py +120 -0
- qtiming/oblig/__init__.py +0 -0
- qtiming/oblig/generate.py +145 -0
- qtiming/oblig/model.py +76 -0
- qtiming/py.typed +0 -0
- qtiming/qiskit_pass/__init__.py +7 -0
- qtiming/qiskit_pass/_common.py +21 -0
- qtiming/qiskit_pass/equalize.py +171 -0
- qtiming/qiskit_pass/plugin.py +23 -0
- qtiming/qiskit_pass/strip.py +42 -0
- qtiming/qiskit_pass/timing_check.py +40 -0
- qtiming/repair/__init__.py +0 -0
- qtiming/repair/suggest.py +146 -0
- qtiming/solve/__init__.py +0 -0
- qtiming/solve/concrete.py +148 -0
- qtiming/solve/dispatcher.py +55 -0
- qtiming/solve/interval.py +185 -0
- qtiming/solve/smt.py +167 -0
- qtiming/solve/smt_o5.py +1 -0
- qtiming/solve/synth/__init__.py +0 -0
- qtiming/solve/synth/bruteforce_dc.py +156 -0
- qtiming/solve/synth/model.py +59 -0
- qtiming/solve/synth/relax.py +93 -0
- qtiming/solve/synth/rigid.py +33 -0
- qtiming/solve/synth/sc.py +55 -0
- qtiming/solve/synth/stn.py +74 -0
- qtiming/solve/synth/stnu_dc.py +93 -0
- qtiming/verdict/__init__.py +0 -0
- qtiming/verdict/minimise.py +45 -0
- qtiming/verdict/model.py +107 -0
- qtiming/verdict/render.py +72 -0
- qtiming/verdict/render_html.py +80 -0
- qtiming/verdict/render_json.py +12 -0
- qtiming/verdict/render_sarif.py +104 -0
- qtiming/verdict/replay.py +77 -0
- qtiming/verdict/run.py +174 -0
- qtiming/verdict/rus.py +53 -0
- qtiming-0.1.0.dist-info/METADATA +262 -0
- qtiming-0.1.0.dist-info/RECORD +102 -0
- qtiming-0.1.0.dist-info/WHEEL +4 -0
- qtiming-0.1.0.dist-info/entry_points.txt +6 -0
- qtiming-0.1.0.dist-info/licenses/LICENSE +177 -0
- qtiming-0.1.0.dist-info/licenses/NOTICE +8 -0
qtiming/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
"""`qtiming-bench` (plan P13, 17.4): run a suite over contracts and policies, compare to golden
|
|
2
|
+
verdicts, exhaustive ground truth, the H1 differential and the report tables.
|
|
3
|
+
|
|
4
|
+
qtiming-bench run --suite benchmarks/families --contracts all --policies eq,sync,exp \
|
|
5
|
+
--repeat 5 --timeout 60 --out results/<run-id>/
|
|
6
|
+
qtiming-bench check results/<run-id>/ --golden benchmarks/expected.json [--update]
|
|
7
|
+
qtiming-bench truth --suite benchmarks/families --out benchmarks/families/
|
|
8
|
+
qtiming-bench diff --runs 10000 --seed 1 --out results/<run-id>/diff/
|
|
9
|
+
qtiming-bench report results/<run-id>/ --tables paper/tables/
|
|
10
|
+
|
|
11
|
+
A cell is one (program, contract, policy). Its status is the verdict, or `unsupported` when the
|
|
12
|
+
contract lacks data the program needs (E-CT-MISSING, E-GRAPH-UNSUPPORTED, ...), `timeout` when
|
|
13
|
+
it ran past `--timeout`, or `error` for anything else, so a gap is reported and never silently
|
|
14
|
+
skipped. `ms` is the median over `--repeat` runs. `--out` is a directory (results.jsonl,
|
|
15
|
+
manifest.json, and results.parquet when pyarrow is installed) or a `.jsonl` file path.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import hashlib
|
|
22
|
+
import json
|
|
23
|
+
import multiprocessing as mp
|
|
24
|
+
import platform
|
|
25
|
+
import queue
|
|
26
|
+
import statistics
|
|
27
|
+
import subprocess
|
|
28
|
+
import sys
|
|
29
|
+
import threading
|
|
30
|
+
import time
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
from qtiming.contract.loader import ContractLoadError, load_contract
|
|
35
|
+
from qtiming.core.errors import QtError
|
|
36
|
+
|
|
37
|
+
INSTANCES = Path(__file__).resolve().parents[1] / "contract" / "instances"
|
|
38
|
+
_UNSUPPORTED = ("E-CT-MISSING", "E-GRAPH-UNSUPPORTED", "E-ELAB-UNSUPPORTED", "E-GRAPH-CUT",
|
|
39
|
+
"E-ELAB-LOOP-LIMIT", "E-ELAB-Q1-NOT-EQUALISABLE", "E-ELAB-EQ-VARIABLE-ARM")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def programs(suite: Path) -> list[Path]:
|
|
43
|
+
return sorted(suite.rglob("*.qasm"))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def contracts(names: str) -> list[str]:
|
|
47
|
+
if names == "all":
|
|
48
|
+
return sorted(p.stem for p in INSTANCES.glob("*.toml"))
|
|
49
|
+
return [n for n in names.split(",") if n]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def load_cell(prog: Path, contract: str, assume: tuple[str, ...] = ()) -> Any:
|
|
53
|
+
"""(contract, program) for one cell; the program is parsed under that contract's dt."""
|
|
54
|
+
from qtiming.frontend import from_qasm3
|
|
55
|
+
|
|
56
|
+
c = load_contract(str(INSTANCES / f"{contract}.toml"), assume=assume)
|
|
57
|
+
return c, from_qasm3(prog.read_text(encoding="utf-8"), filename=str(prog), contract=c)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _tier_mix(v: Any) -> dict[str, int]:
|
|
61
|
+
mix: dict[str, int] = {}
|
|
62
|
+
for r in v.results:
|
|
63
|
+
k = "none" if r.res.tier is None else f"tier{r.res.tier}"
|
|
64
|
+
mix[k] = mix.get(k, 0) + 1
|
|
65
|
+
return mix
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _provenance_counts(v: Any, c: Any) -> tuple[int, int]:
|
|
69
|
+
"""(assumptions weaker than cycle-counted, of which rest on a placeholder number)."""
|
|
70
|
+
from qtiming.core.provenance import ProvClass
|
|
71
|
+
|
|
72
|
+
weak = [a for a in v.assumptions if a.prov.cls < ProvClass.CYC]
|
|
73
|
+
ph = 0
|
|
74
|
+
for a in weak:
|
|
75
|
+
try:
|
|
76
|
+
ph += bool(c.latency(a.key).placeholder)
|
|
77
|
+
except (KeyError, QtError):
|
|
78
|
+
pass
|
|
79
|
+
return len(weak), ph
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
UNROLL_LIMIT = 16 # below the 20-round QEC programs, so those are windowed (plan 17.1)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def run_cell(prog: Path, contract: str, policy: str, repeat: int,
|
|
86
|
+
assume: tuple[str, ...] = (), unroll_limit: int = UNROLL_LIMIT) -> dict[str, Any]:
|
|
87
|
+
from qtiming.verdict.run import check_program
|
|
88
|
+
|
|
89
|
+
rec: dict[str, Any] = {"program": prog.stem, "family": prog.parent.name,
|
|
90
|
+
"contract": contract, "policy": policy}
|
|
91
|
+
times: list[float] = []
|
|
92
|
+
try:
|
|
93
|
+
c, p = load_cell(prog, contract, assume)
|
|
94
|
+
for _ in range(max(1, repeat)):
|
|
95
|
+
t0 = time.perf_counter()
|
|
96
|
+
v = check_program(p.body, c, policy=policy, unroll_limit=unroll_limit)
|
|
97
|
+
times.append((time.perf_counter() - t0) * 1000)
|
|
98
|
+
weak, ph = _provenance_counts(v, c)
|
|
99
|
+
rec |= {"status": v.verdict, "obligations": len(v.results), "tiers": _tier_mix(v),
|
|
100
|
+
"weakest_class": v.weakest_class,
|
|
101
|
+
"assumptions": [a.key for a in v.assumptions], "uncertified": weak,
|
|
102
|
+
"placeholders": ph, "reasons": list(v.reasons)[:3]}
|
|
103
|
+
except ContractLoadError as e: # a QtError subclass: must come first
|
|
104
|
+
rec |= {"status": "error", "reasons": [str(x) for x in e.errors][:3]}
|
|
105
|
+
except QtError as e:
|
|
106
|
+
rec |= {"status": "unsupported" if e.code in _UNSUPPORTED else "error",
|
|
107
|
+
"reasons": [f"{e.code}: {e.message}"]}
|
|
108
|
+
except Exception as e: # a checker crash (an assert, ...) is an `error` cell, not a dead run
|
|
109
|
+
rec |= {"status": "error", "reasons": [f"internal {type(e).__name__}: {e}"]}
|
|
110
|
+
rec["ms"] = round(statistics.median(times), 3) if times else None
|
|
111
|
+
return rec
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def key(r: dict[str, Any]) -> str:
|
|
115
|
+
return f"{r['family']}/{r['program']}|{r['contract']}|{r['policy']}"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# ---------------------------------------------------------------- isolated workers (--timeout)
|
|
119
|
+
def _serve(conn: Any) -> None:
|
|
120
|
+
while True:
|
|
121
|
+
job = conn.recv()
|
|
122
|
+
if job is None:
|
|
123
|
+
return
|
|
124
|
+
try:
|
|
125
|
+
conn.send(run_cell(*job))
|
|
126
|
+
except Exception as e: # reported as an error cell, never a crash of the worker
|
|
127
|
+
conn.send({"program": job[0].stem, "family": job[0].parent.name, "contract": job[1],
|
|
128
|
+
"policy": job[2], "status": "error",
|
|
129
|
+
"reasons": [f"{type(e).__name__}: {e}"], "ms": None})
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class _Worker:
|
|
133
|
+
def __init__(self) -> None:
|
|
134
|
+
ctx = mp.get_context("spawn")
|
|
135
|
+
self.conn, child = ctx.Pipe()
|
|
136
|
+
self.proc = ctx.Process(target=_serve, args=(child,), daemon=True)
|
|
137
|
+
self.proc.start()
|
|
138
|
+
|
|
139
|
+
def run(self, job: tuple[Any, ...], timeout: float) -> dict[str, Any] | None:
|
|
140
|
+
self.conn.send(job)
|
|
141
|
+
return self.conn.recv() if self.conn.poll(timeout) else None # type: ignore[no-any-return]
|
|
142
|
+
|
|
143
|
+
def kill(self) -> None:
|
|
144
|
+
self.proc.kill()
|
|
145
|
+
self.proc.join()
|
|
146
|
+
|
|
147
|
+
def close(self) -> None:
|
|
148
|
+
try:
|
|
149
|
+
self.conn.send(None)
|
|
150
|
+
except OSError:
|
|
151
|
+
pass
|
|
152
|
+
self.proc.join(5)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def run_cells(jobs: list[tuple[Any, ...]], timeout: float, workers: int) -> list[dict[str, Any]]:
|
|
156
|
+
"""Cells in order. timeout <= 0: inline, no isolation. Else a worker process per thread,
|
|
157
|
+
killed and replaced when a cell passes `timeout` seconds (status `timeout`)."""
|
|
158
|
+
if timeout <= 0:
|
|
159
|
+
return [run_cell(*j) for j in jobs]
|
|
160
|
+
todo: queue.Queue[int] = queue.Queue()
|
|
161
|
+
for i in range(len(jobs)):
|
|
162
|
+
todo.put(i)
|
|
163
|
+
out: list[dict[str, Any] | None] = [None] * len(jobs)
|
|
164
|
+
|
|
165
|
+
def loop() -> None:
|
|
166
|
+
w = _Worker()
|
|
167
|
+
try:
|
|
168
|
+
while True:
|
|
169
|
+
try:
|
|
170
|
+
i = todo.get_nowait()
|
|
171
|
+
except queue.Empty:
|
|
172
|
+
return
|
|
173
|
+
prog, contract, policy = jobs[i][:3]
|
|
174
|
+
got = w.run(jobs[i], timeout)
|
|
175
|
+
if got is None:
|
|
176
|
+
w.kill()
|
|
177
|
+
w = _Worker()
|
|
178
|
+
got = {"program": prog.stem, "family": prog.parent.name, "contract": contract,
|
|
179
|
+
"policy": policy, "status": "timeout",
|
|
180
|
+
"reasons": [f"no verdict within {timeout:g} s"], "ms": None}
|
|
181
|
+
out[i] = got
|
|
182
|
+
finally:
|
|
183
|
+
w.close()
|
|
184
|
+
|
|
185
|
+
threads = [threading.Thread(target=loop) for _ in range(max(1, workers))]
|
|
186
|
+
for t in threads:
|
|
187
|
+
t.start()
|
|
188
|
+
for t in threads:
|
|
189
|
+
t.join()
|
|
190
|
+
return [r for r in out if r is not None]
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
# ---------------------------------------------------------------- manifest, outputs
|
|
194
|
+
def _sha(path: Path) -> str:
|
|
195
|
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _git(*args: str) -> str | None:
|
|
199
|
+
try:
|
|
200
|
+
r = subprocess.run(["git", *args], capture_output=True, text=True, timeout=20,
|
|
201
|
+
cwd=Path(__file__).parent, check=False)
|
|
202
|
+
return r.stdout.strip() if r.returncode == 0 else None
|
|
203
|
+
except (OSError, subprocess.SubprocessError):
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _version(pkg: str) -> str | None:
|
|
208
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
209
|
+
|
|
210
|
+
try:
|
|
211
|
+
return version(pkg)
|
|
212
|
+
except PackageNotFoundError:
|
|
213
|
+
return None
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def manifest(argv: list[str], names: list[str]) -> dict[str, Any]:
|
|
217
|
+
return {
|
|
218
|
+
"git_commit": _git("rev-parse", "HEAD"),
|
|
219
|
+
"git_dirty": bool(_git("status", "--porcelain")),
|
|
220
|
+
"python": sys.version.split()[0],
|
|
221
|
+
"platform": platform.platform(), "cpu": platform.processor(),
|
|
222
|
+
"versions": {p: _version(p) for p in ("qiskit", "z3-solver", "openqasm3", "pyarrow",
|
|
223
|
+
"qtiming")},
|
|
224
|
+
"contracts_sha256": {n: _sha(INSTANCES / f"{n}.toml") for n in names},
|
|
225
|
+
"fixtures_sha256": {p.name: _sha(p) for p in sorted(INSTANCES.glob("*.json"))},
|
|
226
|
+
"argv": argv,
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def out_paths(out: str) -> tuple[Path, Path]:
|
|
231
|
+
"""(results file, directory) for `--out`: a directory, or a .jsonl file path."""
|
|
232
|
+
p = Path(out)
|
|
233
|
+
if p.suffix == ".jsonl":
|
|
234
|
+
return p, p.parent
|
|
235
|
+
return p / "results.jsonl", p
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def write_rows(rows: list[dict[str, Any]], out: str) -> Path:
|
|
239
|
+
f, d = out_paths(out)
|
|
240
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
241
|
+
f.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8")
|
|
242
|
+
try: # Parquet only when pyarrow is installed (the `bench` extra); JSONL is always written
|
|
243
|
+
import pyarrow as pa # type: ignore[import-not-found,unused-ignore]
|
|
244
|
+
import pyarrow.parquet as pq # type: ignore[import-not-found,unused-ignore]
|
|
245
|
+
|
|
246
|
+
pq.write_table(pa.Table.from_pylist(rows), f.with_suffix(".parquet"))
|
|
247
|
+
except ImportError:
|
|
248
|
+
pass
|
|
249
|
+
return d
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def read_rows(path: str) -> list[dict[str, Any]]:
|
|
253
|
+
p = Path(path)
|
|
254
|
+
p = p / "results.jsonl" if p.is_dir() else p
|
|
255
|
+
return [json.loads(x) for x in p.read_text(encoding="utf-8").splitlines() if x.strip()]
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
# ---------------------------------------------------------------- commands
|
|
259
|
+
def _run(a: argparse.Namespace) -> int:
|
|
260
|
+
assume = tuple(a.assume or ())
|
|
261
|
+
names = contracts(a.contracts)
|
|
262
|
+
jobs = [(p, c, pol, a.repeat, assume, a.unroll_limit)
|
|
263
|
+
for p in programs(Path(a.suite)) for c in names
|
|
264
|
+
for pol in a.policies.split(",")]
|
|
265
|
+
cells = run_cells(jobs, a.timeout, a.jobs)
|
|
266
|
+
if a.out:
|
|
267
|
+
d = write_rows(cells, a.out)
|
|
268
|
+
m = manifest(sys.argv[1:] if a.argv is None else a.argv, names)
|
|
269
|
+
m["assume"] = list(assume)
|
|
270
|
+
m["unroll_limit"] = a.unroll_limit
|
|
271
|
+
(d / "manifest.json").write_text(json.dumps(m, indent=1) + "\n", encoding="utf-8")
|
|
272
|
+
counts: dict[str, int] = {}
|
|
273
|
+
for r in cells:
|
|
274
|
+
counts[r["status"]] = counts.get(r["status"], 0) + 1
|
|
275
|
+
print(f"{len(cells)} cells: " + ", ".join(f"{k}={v}" for k, v in sorted(counts.items())))
|
|
276
|
+
return 1 if counts.get("error") else 0
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _check(a: argparse.Namespace) -> int:
|
|
280
|
+
got = {key(r): r["status"] for r in read_rows(a.results)}
|
|
281
|
+
golden = Path(a.golden)
|
|
282
|
+
if a.update:
|
|
283
|
+
golden.write_text(json.dumps(dict(sorted(got.items())), indent=1) + "\n", encoding="utf-8")
|
|
284
|
+
print(f"wrote {len(got)} expectations to {golden}")
|
|
285
|
+
return 0
|
|
286
|
+
want: dict[str, str] = json.loads(golden.read_text(encoding="utf-8"))
|
|
287
|
+
bad = [(k, want.get(k), got.get(k)) for k in sorted(set(want) | set(got))
|
|
288
|
+
if want.get(k) != got.get(k)]
|
|
289
|
+
for k, w, g in bad:
|
|
290
|
+
print(f"MISMATCH {k}: expected {w}, got {g}")
|
|
291
|
+
print(f"{len(got) - len(bad)} of {len(got)} cells match the golden file")
|
|
292
|
+
return 1 if bad else 0
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def main(argv: list[str] | None = None) -> int:
|
|
296
|
+
from . import diff, report, truth
|
|
297
|
+
|
|
298
|
+
ap = argparse.ArgumentParser(prog="qtiming-bench")
|
|
299
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
300
|
+
r = sub.add_parser("run", help="verdict for every (program, contract, policy) cell")
|
|
301
|
+
r.add_argument("--suite", default="benchmarks/families")
|
|
302
|
+
r.add_argument("--contracts", default="all")
|
|
303
|
+
r.add_argument("--policies", default="eq,sync,exp")
|
|
304
|
+
r.add_argument("--repeat", type=int, default=1)
|
|
305
|
+
r.add_argument("--timeout", type=float, default=0,
|
|
306
|
+
help="seconds per cell, enforced in a worker process (0: inline, no limit)")
|
|
307
|
+
r.add_argument("--jobs", type=int, default=1, help="worker processes (with --timeout)")
|
|
308
|
+
r.add_argument("--assume", action="append", metavar="KEY=VALUE",
|
|
309
|
+
help="contract override, as `qtiming check --assume`; recorded in the manifest")
|
|
310
|
+
r.add_argument("--unroll-limit", type=int, default=UNROLL_LIMIT,
|
|
311
|
+
help="loops longer than this are windowed (IBM) or refused, not unrolled")
|
|
312
|
+
r.add_argument("--out", help="results directory (or a .jsonl path)")
|
|
313
|
+
r.set_defaults(fn=_run, argv=argv)
|
|
314
|
+
c = sub.add_parser("check", help="compare a run with the golden regression snapshot")
|
|
315
|
+
c.add_argument("results")
|
|
316
|
+
c.add_argument("--golden", default="benchmarks/expected.json")
|
|
317
|
+
c.add_argument("--update", action="store_true")
|
|
318
|
+
c.set_defaults(fn=_check)
|
|
319
|
+
truth.add_parser(sub)
|
|
320
|
+
diff.add_parser(sub)
|
|
321
|
+
report.add_parser(sub)
|
|
322
|
+
a = ap.parse_args(argv)
|
|
323
|
+
return int(a.fn(a))
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
if __name__ == "__main__":
|
|
327
|
+
sys.exit(main())
|
qtiming/bench/diff.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""`qtiming-bench diff`: the H1 differential (plan 17.2).
|
|
2
|
+
|
|
3
|
+
For every program the checker calls PROVED or PROVED-UNDER, run N random (omega, theta) executions
|
|
4
|
+
in the reference executor (which shares no code with the graph and solver side) and count the
|
|
5
|
+
runs that violate. The number must be ZERO. Two program sources, reported per personality:
|
|
6
|
+
|
|
7
|
+
* the benchmark suite, under the shipped contracts (`--suite`, `--contracts`);
|
|
8
|
+
* random programs (`--programs M`, from `--seed`), under small self-contained contracts of the same
|
|
9
|
+
four personalities (the dicts of tests/differential/test_graph_vs_executor.py, plus gate
|
|
10
|
+
durations), written as OpenQASM 3 so they go through the real front end.
|
|
11
|
+
|
|
12
|
+
Scope: the executor reports per-run violations only (RTIO underflow, Q1 stale read and RT
|
|
13
|
+
underflow, ARTIQ silent drops). O3, O4 and O7 compare times or two runs; they are covered by
|
|
14
|
+
tests/property/test_soundness.py, not here.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
import json
|
|
21
|
+
import random
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from qtiming.core.errors import QtError
|
|
26
|
+
|
|
27
|
+
STMT_BUDGET = 40_000 # runs x elaborated statements per proved cell
|
|
28
|
+
_GATES = {"x": 16, "h": 24, "cx": 40}
|
|
29
|
+
CONTRACTS: dict[str, dict[str, Any]] = {
|
|
30
|
+
"ibm": {"durations": {"measure": 90, "reset": 20},
|
|
31
|
+
"latencies": [{"key": "ff.default", "src": "measure", "dst": "branch", "lo": 5,
|
|
32
|
+
"hi": 60}]},
|
|
33
|
+
"qua": {"durations": {"measure": 90, "reset": 20}, "branch": {"allow_nested": True},
|
|
34
|
+
"params": {},
|
|
35
|
+
"latencies": [
|
|
36
|
+
{"key": "active_reset", "src": "measure", "dst": "branch", "lo": 0, "hi": 40},
|
|
37
|
+
{"key": "align.same_fem", "src": "align", "dst": "align", "tier": "same_fem",
|
|
38
|
+
"lo": 0, "hi": 12}]},
|
|
39
|
+
"q1": {"durations": {"measure": 30, "reset": 10}, "issue_cost": {"nop": 1},
|
|
40
|
+
"capacity": {"rt_queue_depth": 4}, "params": {"else_time_dt": 1},
|
|
41
|
+
"branch": {"allow_nested": True},
|
|
42
|
+
"latencies": [{"key": "trigger.network", "src": "measure", "dst": "branch", "lo": 3,
|
|
43
|
+
"hi": 20}]},
|
|
44
|
+
"artiq": {"durations": {"measure": 20, "reset": 5}, "issue_cost": {"cpu_issue": 3},
|
|
45
|
+
"capacity": {"lanes": 8, "coarse_mu": 8}, "commit_lead": 4,
|
|
46
|
+
"branch": {"class": "exposed", "default_policy": "exp", "allow_nested": True},
|
|
47
|
+
"latencies": [{"key": "in2cpu", "src": "input", "dst": "cpu", "lo": 0, "hi": 50}]},
|
|
48
|
+
}
|
|
49
|
+
_KLASS = {"ibm": "restricted", "q1": "predicated", "qua": "resync", "artiq": "exposed"}
|
|
50
|
+
_POLICY = {"ibm": "sync", "q1": "eq", "qua": "sync", "artiq": "exp"}
|
|
51
|
+
_LATE = {"ibm": "compile_error", "q1": "halt", "artiq": "mixed", "qua": "compile_error"}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def make_contract(pers: str) -> Any:
|
|
55
|
+
from qtiming.contract.loader import load_contract_dict
|
|
56
|
+
|
|
57
|
+
spec = CONTRACTS[pers]
|
|
58
|
+
raw: dict[str, Any] = {
|
|
59
|
+
"meta": {"name": f"bench_{pers}", "version": "0", "schema": "qtiming-contract/0.2",
|
|
60
|
+
"personality": pers},
|
|
61
|
+
"timebase": {"dt_ns": "1"},
|
|
62
|
+
"timeline": {"late_policy": _LATE[pers]},
|
|
63
|
+
"durations": {**spec["durations"], **_GATES},
|
|
64
|
+
"issue_cost": {k: {"ns": v, "prov": "cyc", "source": "bench"}
|
|
65
|
+
for k, v in spec.get("issue_cost", {}).items()},
|
|
66
|
+
"capacity": spec.get("capacity", {}),
|
|
67
|
+
"latency": [{"key": x["key"], "src": x["src"], "dst": x["dst"],
|
|
68
|
+
"tier": x.get("tier", "default"), "lo_ns": x["lo"], "hi_ns": x["hi"],
|
|
69
|
+
"prov": "vendor", "source": "bench"} for x in spec["latencies"]],
|
|
70
|
+
"branch": {"class": _KLASS[pers], "default_policy": _POLICY[pers], "allow_nested": False,
|
|
71
|
+
**spec.get("branch", {})},
|
|
72
|
+
"personality_params": {pers: spec.get("params", {})},
|
|
73
|
+
}
|
|
74
|
+
if spec.get("commit_lead") is not None:
|
|
75
|
+
raw["timeline"]["commit_lead"] = spec["commit_lead"]
|
|
76
|
+
return load_contract_dict(raw)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def random_qasm(pers: str, rng: random.Random) -> str:
|
|
80
|
+
"""A random valid program: gates, delays, measures, barriers and (nested-free) ifs."""
|
|
81
|
+
wide = pers not in ("q1", "artiq")
|
|
82
|
+
out: list[str] = []
|
|
83
|
+
nbits = 0
|
|
84
|
+
|
|
85
|
+
def simple() -> str:
|
|
86
|
+
if rng.random() < 0.3:
|
|
87
|
+
return f"delay[{rng.randint(1, 40)}ns] q[{rng.randrange(3)}];"
|
|
88
|
+
if wide and rng.random() < 0.3:
|
|
89
|
+
a, b = rng.sample(range(3), 2)
|
|
90
|
+
return f"cx q[{a}], q[{b}];"
|
|
91
|
+
return f"{rng.choice(['x', 'h'])} q[{rng.randrange(3)}];"
|
|
92
|
+
|
|
93
|
+
if pers == "artiq":
|
|
94
|
+
out.append(f"delay[{rng.randint(1, 80)}ns] q[0];")
|
|
95
|
+
for _ in range(rng.randint(3, 10)):
|
|
96
|
+
r = rng.random()
|
|
97
|
+
if r < 0.22 and nbits < 4:
|
|
98
|
+
out.append(f"c[{nbits}] = measure q[{rng.randrange(3)}];")
|
|
99
|
+
nbits += 1
|
|
100
|
+
elif r < 0.52 and nbits:
|
|
101
|
+
c = f"c[{rng.randrange(nbits)}]"
|
|
102
|
+
if nbits > 1 and rng.random() < 0.3:
|
|
103
|
+
c = f"{c} ^ c[{rng.randrange(nbits)}]"
|
|
104
|
+
if rng.random() < 0.3:
|
|
105
|
+
c = f"!({c})"
|
|
106
|
+
then = "\n".join(" " + simple() for _ in range(rng.randint(1, 3)))
|
|
107
|
+
els = ("" if rng.random() < 0.4 else " else {\n" + "\n".join(
|
|
108
|
+
" " + simple() for _ in range(rng.randint(1, 3))) + "\n}")
|
|
109
|
+
out.append(f"if ({c}) {{\n{then}\n}}{els}")
|
|
110
|
+
elif r < 0.62 and wide:
|
|
111
|
+
out.append("barrier q;")
|
|
112
|
+
else:
|
|
113
|
+
out.append(simple())
|
|
114
|
+
return "OPENQASM 3.0;\nqubit[3] q;\nbit[4] c;\n" + "\n".join(out) + "\n"
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _violations(body: Any, contract: Any, runs: int, seed: int) -> tuple[int, int, set[str]]:
|
|
118
|
+
"""(runs that violate, runs executed, kinds seen) of `runs` random executions."""
|
|
119
|
+
from qtiming.exec.engine import execute
|
|
120
|
+
from qtiming.exec.oracle import Random
|
|
121
|
+
|
|
122
|
+
bad, kinds = 0, set()
|
|
123
|
+
for k in range(runs):
|
|
124
|
+
res = execute(body, contract, Random(seed * 1_000_003 + k))
|
|
125
|
+
ks = {v.kind for v in res.violations} | {f"drop:{d.kind}" for d in res.drops}
|
|
126
|
+
bad += bool(ks)
|
|
127
|
+
kinds |= ks
|
|
128
|
+
return bad, runs, kinds
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def diff_program(prog: Any, contract: Any, policy: str, runs: int, seed: int,
|
|
132
|
+
) -> tuple[str, int, set[str], int] | None:
|
|
133
|
+
"""(verdict, violating runs, kinds, runs used) if PROVED or PROVED-UNDER, else None."""
|
|
134
|
+
from qtiming.core.ast import Policy
|
|
135
|
+
from qtiming.elab import elaborate
|
|
136
|
+
from qtiming.verdict.run import check_program, collect_locs, with_policy
|
|
137
|
+
|
|
138
|
+
from . import UNROLL_LIMIT
|
|
139
|
+
|
|
140
|
+
v = check_program(prog, contract, policy=policy, unroll_limit=UNROLL_LIMIT).verdict
|
|
141
|
+
if v not in ("PROVED", "PROVED-UNDER"):
|
|
142
|
+
return None
|
|
143
|
+
# the executor runs the program the verdict was about, fully unrolled (also a windowed loop)
|
|
144
|
+
el = elaborate(with_policy(prog, Policy(policy)), contract, unroll_limit=256) # type: ignore[arg-type]
|
|
145
|
+
# long unrolled programs get fewer runs (cost ~ runs x statements); the cap is reported
|
|
146
|
+
n = max(10, min(runs, STMT_BUDGET // max(1, len(collect_locs(el.body, {})))))
|
|
147
|
+
bad, _, kinds = _violations(el.body, contract, n, seed)
|
|
148
|
+
return v, bad, kinds, n
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _diff(a: argparse.Namespace) -> int:
|
|
152
|
+
from qtiming.contract.loader import load_contract
|
|
153
|
+
from qtiming.frontend import from_qasm3
|
|
154
|
+
|
|
155
|
+
from . import INSTANCES, contracts, key, programs
|
|
156
|
+
|
|
157
|
+
per: dict[str, dict[str, int]] = {}
|
|
158
|
+
failures: list[dict[str, Any]] = []
|
|
159
|
+
|
|
160
|
+
def skip(pers: str) -> None:
|
|
161
|
+
per.setdefault(pers, {"cells": 0, "proved_cells": 0, "runs": 0, "violating_runs": 0,
|
|
162
|
+
"skipped": 0})["skipped"] += 1
|
|
163
|
+
|
|
164
|
+
def tally(pers: str, name: str, res: tuple[str, int, set[str], int] | None) -> None:
|
|
165
|
+
s = per.setdefault(pers, {"cells": 0, "proved_cells": 0, "runs": 0, "violating_runs": 0,
|
|
166
|
+
"skipped": 0})
|
|
167
|
+
s["cells"] += 1
|
|
168
|
+
if res is None:
|
|
169
|
+
return
|
|
170
|
+
s["proved_cells"] += 1
|
|
171
|
+
s["runs"] += res[3]
|
|
172
|
+
s["violating_runs"] += res[1]
|
|
173
|
+
if res[1]:
|
|
174
|
+
failures.append({"cell": name, "verdict": res[0], "violating_runs": res[1],
|
|
175
|
+
"kinds": sorted(res[2])})
|
|
176
|
+
|
|
177
|
+
loaded = {cn: load_contract(str(INSTANCES / f"{cn}.toml")) for cn in contracts(a.contracts)}
|
|
178
|
+
for prog in programs(Path(a.suite)) if a.suite else []:
|
|
179
|
+
for cn in loaded:
|
|
180
|
+
for pol in a.policies.split(","):
|
|
181
|
+
name = key({"family": prog.parent.name, "program": prog.stem, "contract": cn,
|
|
182
|
+
"policy": pol})
|
|
183
|
+
c = loaded[cn]
|
|
184
|
+
try:
|
|
185
|
+
p = from_qasm3(prog.read_text(encoding="utf-8"), filename=str(prog),
|
|
186
|
+
contract=c)
|
|
187
|
+
tally(c.personality, name, diff_program(p.body, c, pol, a.runs, a.seed))
|
|
188
|
+
except QtError:
|
|
189
|
+
skip(c.personality)
|
|
190
|
+
for pers in CONTRACTS:
|
|
191
|
+
c = make_contract(pers)
|
|
192
|
+
for n in range(a.programs):
|
|
193
|
+
seed = a.seed * 100_003 + n
|
|
194
|
+
text = random_qasm(pers, random.Random(seed))
|
|
195
|
+
for pol in ("eq", "sync", "exp"):
|
|
196
|
+
try:
|
|
197
|
+
p = from_qasm3(text, filename=f"random/{pers}/{seed}", contract=c)
|
|
198
|
+
tally(pers, f"random/{pers}/{seed}|{pol}",
|
|
199
|
+
diff_program(p.body, c, pol, a.runs, seed))
|
|
200
|
+
except QtError:
|
|
201
|
+
skip(pers)
|
|
202
|
+
total = sum(s["violating_runs"] for s in per.values())
|
|
203
|
+
doc = {"runs_per_cell": a.runs, "seed": a.seed, "random_programs_per_personality": a.programs,
|
|
204
|
+
"per_personality": dict(sorted(per.items())), "violating_runs_total": total,
|
|
205
|
+
"failures": failures}
|
|
206
|
+
if a.out:
|
|
207
|
+
out = Path(a.out)
|
|
208
|
+
out = out if out.suffix == ".json" else out / "diff.json"
|
|
209
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
210
|
+
out.write_text(json.dumps(doc, indent=1) + "\n", encoding="utf-8")
|
|
211
|
+
for pers, s in sorted(per.items()):
|
|
212
|
+
print(f"{pers}: {s['proved_cells']} proved cells of {s['cells']}, {s['runs']} runs, "
|
|
213
|
+
f"{s['violating_runs']} violating, {s['skipped']} unsupported")
|
|
214
|
+
for f in failures[:20]:
|
|
215
|
+
print(f"VIOLATION {f}")
|
|
216
|
+
print(f"H1: {total} violating runs in total ({'PASS' if total == 0 else 'FAIL'})")
|
|
217
|
+
return 1 if total else 0
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def add_parser(sub: Any) -> None:
|
|
221
|
+
d = sub.add_parser("diff", help="H1 differential: executor violations on proved programs")
|
|
222
|
+
d.add_argument("--runs", type=int, default=1000, help="random executions per proved cell")
|
|
223
|
+
d.add_argument("--seed", type=int, default=1)
|
|
224
|
+
d.add_argument("--programs", type=int, default=0,
|
|
225
|
+
help="random programs per personality (0: benchmark suite only)")
|
|
226
|
+
d.add_argument("--suite", default="benchmarks/families", help="'' to skip the suite")
|
|
227
|
+
d.add_argument("--contracts", default="all")
|
|
228
|
+
d.add_argument("--policies", default="eq,sync,exp")
|
|
229
|
+
d.add_argument("--out")
|
|
230
|
+
d.set_defaults(fn=_diff)
|