offtarget 0.1.0__py3-none-any.whl

offload/engine/confirm/registry.py

@@ -0,0 +1,151 @@
+"""The rulings registry (catalog/rulings.yaml).
+
+Each entry turns a ruling of data/reports/rulings.md into something a rule
+can cite by id: its kind, the rulings.md text it comes from, its spec
+citations (version, section, printed page, quoted sentence) and what it
+applies to. ``check`` verifies an entry against rulings.md and the spec
+text; ``matching`` returns the entries that apply to a program and a
+failure reason."""
+from __future__ import annotations
+
+import bisect
+import re
+from dataclasses import dataclass, field
+from fnmatch import fnmatchcase
+from functools import lru_cache
+from pathlib import Path
+
+import yaml
+
+from offload.engine.root import ROOT  # noqa: E402
+REGISTRY = ROOT / "catalog" / "rulings.yaml"
+RULINGS_MD = ROOT / "data" / "reports" / "rulings.md"
+KINDS = ("legality", "spec_reading", "implementation_defined", "unspecified",
+         "no_fixed_value", "oracle", "mode", "environment", "process")
+# kinds under which the spec leaves the observed value open (an
+# implementation-defined *support* of a feature, e.g. a requirement, does
+# not: a supported feature must work)
+OPEN_KINDS = ("unspecified", "no_fixed_value")
+
+
+@dataclass(frozen=True)
+class Entry:
+    id: str
+    kind: str
+    task: str
+    rulings_md: str
+    citations: tuple
+    applies: dict = field(default_factory=dict, hash=False, compare=False)
+    test: str = ""
+    # A ruling that decides the support of the
+    # programs it applies to ({verdict, families, outcome}); the confirmer
+    # applies it before the family-witness rules (R11, R12b)
+    decides: dict = field(default_factory=dict, hash=False, compare=False)
+
+    def cite(self) -> str:
+        if not self.citations:
+            return f"{self.id} (rulings.md)"
+        c = self.citations[0]
+        return (f"{self.id} ({c['version']} §{c['section']} p.{c['page']}: "
+                f"\"{c['quote']}\")")
+
+
+def _norm(text: str) -> str:
+    return " ".join(text.replace("’", "'").split())
+
+
+def load(path: Path = REGISTRY) -> dict[str, Entry]:
+    data = yaml.safe_load(Path(path).read_text()) or {}
+    if data.get("version") != 1:
+        raise ValueError("rulings registry: unknown version")
+    out: dict[str, Entry] = {}
+    for d in data.get("entries") or []:
+        if d["id"] in out:
+            raise ValueError(f"duplicate registry id {d['id']}")
+        if d["kind"] not in KINDS:
+            raise ValueError(f"{d['id']}: unknown kind {d['kind']}")
+        out[d["id"]] = Entry(d["id"], d["kind"], str(d.get("task", "")),
+                             d["rulings_md"],
+                             tuple(d.get("citations") or ()),
+                             dict(d.get("applies") or {}), d.get("test", ""),
+                             dict(d.get("decides") or {}))
+    return out
+
+
+@lru_cache(maxsize=None)
+def entries() -> dict:
+    return load()
+
+
+def in_rulings_md(text: str, md: str | None = None) -> bool:
+    md = RULINGS_MD.read_text() if md is None else md
+    return _norm(text) in _norm(md.replace("\n  ", " "))
+
+
+@lru_cache(maxsize=None)
+def _spec(version: str):
+    from offload.engine.spec_text import _lines
+    text, starts, pages = "", [], []
+    for page, line in _lines(version):
+        starts.append(len(text))
+        pages.append(page)
+        text += line.replace("’", "'") + " "  # as _norm does the quote
+    return text, starts, pages
+
+
+def quote_pages(version: str, quote: str) -> list[tuple[int, int]]:
+    """(start page, end page) of every occurrence of `quote` in the spec
+    text (whitespace and hyphenation at line ends normalized)."""
+    text, starts, pages = _spec(version)
+    pat = re.compile(r"\s+".join(re.escape(w) for w in _norm(quote).split()))
+    out = []
+    for m in pat.finditer(text):
+        a = bisect.bisect_right(starts, m.start()) - 1
+        b = bisect.bisect_right(starts, m.end() - 1) - 1
+        out.append((pages[a], pages[b]))
+    return out
+
+
+def check(entry: Entry, md: str | None = None) -> list[str]:
+    """Problems of one entry: its rulings.md text missing, a quote not in
+    the spec, a cited page that does not hold it (the quote must start on
+    the cited page or the page after: an item is cited where it starts)."""
+    errs = []
+    if not in_rulings_md(entry.rulings_md, md):
+        errs.append(f"{entry.id}: rulings_md text not in rulings.md")
+    for c in entry.citations:
+        found = quote_pages(str(c["version"]), c["quote"])
+        if not found:
+            errs.append(f"{entry.id}: quote not in the {c['version']} text")
+        elif not any(a - 1 <= int(c["page"]) <= b for a, b in found):
+            errs.append(f"{entry.id}: quote is on p.{found} not "
+                        f"p.{c['page']}")
+    return errs
+
+
+def _requires(text: str) -> set[str]:
+    out = set()
+    for m in re.finditer(r"^\s*(?:!\$omp|#\s*pragma\s+omp)\s+requires\s+(.*)$",
+                         text, re.I | re.M):
+        out |= set(re.findall(r"[a-z_]+", m.group(1).lower()))
+    return out
+
+
+def matching(program_id: str, text: str = "", needs=(), reason: str = "",
+             kinds=None) -> list[Entry]:
+    """The entries whose `applies` matches: a program id pattern, a
+    program marker (needs), a requirement clause in the text, or a reason
+    pattern. An entry with an empty `applies` is cited by rules only."""
+    out = []
+    req = _requires(text) if text else set()
+    for e in entries().values():
+        if kinds and e.kind not in kinds:
+            continue
+        a = e.applies
+        hit = any(fnmatchcase(program_id, p) for p in a.get("programs", ())) \
+            or bool(set(a.get("needs", ())) & set(needs)) \
+            or bool(set(a.get("requires", ())) & req) \
+            or any(re.search(p, reason or "") for p in a.get("reasons", ()))
+        if hit:
+            out.append(e)
+    return out

offload/engine/confirm/regress.py

@@ -0,0 +1,278 @@
+"""Regression mode: ``confirm regress --toolchain NEW --like OLD``.
+
+Four steps, all writing under results/confirm/regress/NEW/:
+  plan     the jobs that run the catalog on NEW and the fixed fuzz sample
+           (tools/jobs/run_regress.sh), through the scheduler
+  fuzz     the sample (``FUZZ_SAMPLE``) on OLD and NEW in one job
+  diff     NEW's rows against OLD's stored rows per (program, mode): same,
+           regression (OLD passed, NEW fails), fix (OLD failed, NEW passes),
+           changed (both fail, differently)
+  confirm  each differing cell is rerun on OLD and NEW (k repeats, one node);
+           one that does not reproduce is ``not_reproduced``, otherwise the
+           rules decide the failing cell. Output: report.json
+``report`` writes data/reports/confirm-regress-NEW.md."""
+from __future__ import annotations
+
+import json
+import re
+import time
+from collections import Counter
+from pathlib import Path
+
+from offload.engine.confirm import ingest as I
+from offload.engine.confirm import machines as M
+from offload.engine.confirm.model import PASSING
+from offload.engine.toolchains import TOOLCHAINS
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+BASE = ROOT / "results" / "confirm" / "regress"
+# the fixed fuzz sample: the first 200 seeds of campaign s1-00's
+# configuration (generator fuzz-3, every base-language group)
+FUZZ_SAMPLE = {"gen_version": "fuzz-3",
+               "base": ("types", "derived", "alloc", "declare", "multitu"),
+               "seeds": (1, 200), "mode": "unified"}
+LANGS = ("f90", "c", "acc")
+
+
+def out_dir(new: str) -> Path:
+    return BASE / new
+
+
+def plan(new: str, like: str, submit: bool = False, scheduler=None
+         ) -> list[dict]:
+    """The run jobs of a regression (printed, or submitted through the
+    machine's scheduler)."""
+    from offload.engine.confirm import jobs as J
+    tc = TOOLCHAINS[new]
+    if TOOLCHAINS[like].machine != tc.machine:
+        raise ValueError("NEW and OLD must run on the same machine")
+    m = M.load()[tc.machine]
+    d = out_dir(new).relative_to(ROOT)
+    specs = []
+    for lang in LANGS:
+        for mode in m.modes:
+            x = m.row_key(mode)
+            specs.append(J.spec_for(m, "tools/jobs/run_regress.sh",
+                                    {"JOB": "matrix", "LANG_": lang,
+                                     "X": str(x), "NEW": new},
+                                    f"{d}/matrix.{lang}.x{x}.log"))
+    specs.append(J.spec_for(m, "tools/jobs/run_regress.sh",
+                            {"JOB": "fuzz", "NEW": new, "LIKE": like},
+                            f"{d}/fuzz.log"))
+    out = []
+    sched = scheduler or J.scheduler_for(m)
+    for s in specs:
+        rec = {"command": " ".join(sched.command(s)), "env": s.env}
+        if submit:
+            rec["job"] = sched.submit(s)
+        out.append(rec)
+    return out
+
+
+def run_fuzz(new: str, like: str, workers: int = 16, log=print) -> Path:
+    from offload.engine import fuzz_exec as fe
+    m = M.load()[TOOLCHAINS[new].machine]
+    fe.XNACK = m.row_key(FUZZ_SAMPLE["mode"])
+    d = out_dir(new) / "fuzz"
+    a, b = FUZZ_SAMPLE["seeds"]
+    meta = {"run_id": "fuzz", "gen_version": FUZZ_SAMPLE["gen_version"],
+            "base": list(FUZZ_SAMPLE["base"]), "seeds": [a, b],
+            "toolchains": [like, new], "xnack": fe.XNACK,
+            "started": time.strftime("%Y-%m-%dT%H:%M:%S")}
+    d.mkdir(parents=True, exist_ok=True)
+    (d / "meta.json").write_text(json.dumps({"sessions": [meta]}, indent=1))
+    fe.campaign(d, range(a, b + 1), workers, (like, new),
+                log=log, gen_version=FUZZ_SAMPLE["gen_version"],
+                base=FUZZ_SAMPLE["base"], xnack=fe.XNACK)
+    return d / "programs.jsonl"
+
+
+def new_rows(new: str) -> list[I.Row]:
+    out = []
+    d = out_dir(new)
+    for p in sorted(d.glob("matrix.*.jsonl")) + sorted(
+            d.glob("baseline.*.jsonl")):
+        if ".meta" in p.name:
+            continue
+        out += I.load_rows(p)
+    f = d / "fuzz" / "programs.jsonl"
+    if f.is_file():
+        out += I.load_rows(f)
+    return out
+
+
+def _reason_class(r: I.Row) -> str:
+    """The comparable part of a reason (no addresses, paths, numbers of
+    lines; a check value)."""
+    from offload.engine.confirm.findings import normalize
+    text = normalize(r.reason.split(";")[0])
+    text = re.sub(r"0x[0-9a-f]+|<tmpdir>\S*|:\d+:\d+|line \d+|"
+                  r"version \S+", "", text)
+    return " ".join(text.split())[:120]
+
+
+def classify(old: I.Row | None, new: I.Row) -> str:
+    if old is None:
+        return "new_cell"
+    po, pn = old.outcome in PASSING, new.outcome in PASSING
+    if po and pn:
+        return "same"
+    if po and not pn:
+        return "regression"
+    if pn and not po:
+        return "fix"
+    if old.outcome != new.outcome or old.value != new.value:
+        return "changed"
+    return "same"
+
+
+def diff(new: str, like: str, rows=None, index=None) -> list[dict]:
+    """Each cell of NEW with OLD's row for the same program and mode
+    (fuzz: OLD's row of the same side-by-side run)."""
+    rows = rows if rows is not None else new_rows(new)
+    idx = index or I.stored_index()
+    side = I.Index(rows)
+    out = []
+    for r in rows:
+        if r.toolchain != new:
+            continue
+        if r.kind == "fuzz":
+            old = side.get("fuzz", r.ident, like, r.mode)
+        else:
+            old = idx.get(r.kind, r.ident, like, r.mode)
+        cls = classify(old, r)
+        if cls == "same" and old is not None and old.outcome == r.outcome \
+                and r.outcome not in PASSING and \
+                _reason_class(old) != _reason_class(r):
+            cls = "changed"
+        out.append({"kind": r.kind, "ident": r.ident, "mode": r.mode,
+                    "class": cls, "legal": r.legal,
+                    "old": None if old is None else
+                    {"outcome": old.outcome, "value": old.value,
+                     "reason": old.reason[:200], "origin": old.origin},
+                    "new": {"outcome": r.outcome, "value": r.value,
+                            "reason": r.reason[:200], "origin": r.origin}})
+    return out
+
+
+def summary(cells: list[dict]) -> dict:
+    c = Counter((d["kind"], d["mode"], d["class"]) for d in cells)
+    return {f"{k}|{m}|{cl}": n for (k, m, cl), n in sorted(c.items())}
+
+
+def rerun_class(old: dict, new: dict) -> str:
+    """The class of a cell run again on OLD and NEW (each: outcomes and
+    values of k repeats). Both failing in the same outcome class with a
+    value that changes from run to run on both is "same" (one
+    nondeterministic failure, not a change between the versions)."""
+    po = all(o in PASSING for o in old["outcomes"])
+    pn = all(o in PASSING for o in new["outcomes"])
+    again = {(True, False): "regression", (False, True): "fix"}.get((po, pn))
+    if again is not None:
+        return again
+    if po:
+        return "same"
+    if set(old["outcomes"]) != set(new["outcomes"]):
+        return "changed"
+    if set(old["values"]) == set(new["values"]):
+        return "same"
+    if len(set(old["values"])) > 1 and len(set(new["values"])) > 1:
+        return "same"
+    return "changed"
+
+
+def confirm_differences(new: str, like: str, cells: list[dict], runner,
+                        repeats: int = 3, workers: int = 8, store=None,
+                        log=print) -> list[dict]:
+    """Run each differing cell again on OLD and NEW (this node), then the
+    rules on the cell that still differs."""
+    from concurrent.futures import ThreadPoolExecutor
+
+    from offload.engine.confirm import collectors as C
+    from offload.engine.confirm import engine as EN
+    from offload.engine.confirm.store import Store
+    todo = [d for d in cells if d["class"] in ("regression", "fix",
+                                               "changed") and d["legal"]]
+    rows = new_rows(new)
+    idx = I.Index(list(I.stored_index().by_cell.values()) + rows)
+    ctx = C.Context(idx, store or Store(), runner=runner, allow_run=True,
+                    repeats=repeats)
+    nidx = I.Index(rows)
+
+    def one(d):
+        row = nidx.get(d["kind"], d["ident"], new, d["mode"])
+        case = I.case_of_row(row)
+        ref = case.program
+        res = {}
+        for t in (like, new):
+            runs, prov = runner.run(ref, t, d["mode"], "-O2", "mandatory",
+                                    repeats)
+            res[t] = {"outcomes": [r.outcome for r in runs],
+                      "values": [r.value for r in runs],
+                      "reason": runs[0].reason[:200], "host": prov.host}
+        again = rerun_class(res[like], res[new])
+        out = dict(d, rerun=res, rerun_class=again)
+        if again == "same":
+            out["status"] = "not_reproduced"
+            return out
+        target = new if again in ("regression", "changed") else like
+        trow = idx.get(d["kind"], d["ident"], target, d["mode"])
+        tcase = I.case_of_row(trow, ref)
+        verdict = EN.confirm(tcase, ctx)
+        out["status"] = "confirmed"
+        out["confirm"] = {k: verdict[k] for k in (
+            "verdict", "rule", "reason", "citations", "findings",
+            "witnesses", "withdrawn")}
+        out["confirm"]["toolchain"] = target
+        return out
+
+    results = []
+    with ThreadPoolExecutor(max(1, workers)) as pool:
+        for k, r in enumerate(pool.map(one, todo), 1):
+            results.append(r)
+            log(f"{k}/{len(todo)} {r['kind']} {r['ident']} {r['mode']}: "
+                f"{r['class']} -> {r.get('rerun_class')} "
+                f"{r.get('confirm', {}).get('verdict', '')}")
+    return results
+
+
+def render(new: str, like: str, cells: list[dict], confirmed: list[dict],
+           meta: dict) -> str:
+    from offload.engine.toolchains import compiler_version
+    s = Counter((d["kind"], d["class"]) for d in cells)
+    kinds = sorted({k for k, _ in s})
+    classes = ["same", "regression", "fix", "changed", "new_cell"]
+    lines = [f"# Regression: {new} against {like}", "",
+             f"NEW `{new}`: {meta.get('new_compiler', '')}; OLD `{like}`: "
+             f"{meta.get('old_compiler', '')}. Rows of NEW: "
+             "results/confirm/regress/" + new + "/ (git-ignored; not in the "
+             "headline tables). OLD: the stored full result files; the fuzz "
+             "sample ran both side by side.", "",
+             "| set | " + " | ".join(classes) + " |",
+             "|---|" + "---|" * len(classes)]
+    for k in kinds:
+        lines.append(f"| {k} | " + " | ".join(str(s[(k, c)]) for c in
+                                               classes) + " |")
+    st = Counter(d.get("status", "") for d in confirmed)
+    diffs = sum(s[(k, c)] for k in kinds for c in ("regression", "fix",
+                                                    "changed"))
+    lines += ["", f"Result: {diffs} differing cells; run again on one node "
+              f"(3 repeats of each version): {st['confirmed']} reproduced, "
+              f"{st['not_reproduced']} not reproduced (the versions behave "
+              "the same; a stored difference was run-to-run variation)."]
+    lines += ["", "## Differences, run again on one node and confirmed", "",
+              "| set | program | mode | stored | rerun | verdict (rule) on "
+              "| finding |", "|---|---|---|---|---|---|---|"]
+    for d in sorted(confirmed, key=lambda d: (d["kind"], d["ident"],
+                                               d["mode"])):
+        c = d.get("confirm") or {}
+        f = (c.get("findings") or [{}])[0].get("file", "") if c else ""
+        lines.append(
+            f"| {d['kind']} | {d['ident']} | {d['mode']} | {d['class']}: "
+            f"{(d['old'] or {}).get('outcome')} -> {d['new']['outcome']} | "
+            f"{d.get('rerun_class')} | "
+            + (f"{c['verdict']} ({c['rule']}) on {c['toolchain']}"
+               if c else d.get("status", "") + (f": {d['note']}" if
+                                                d.get("note") else ""))
+            + f" | {f} |")
+    return "\n".join(lines) + "\n"