offtarget 0.1.0__py3-none-any.whl

offload/engine/confirm/machines.py

@@ -0,0 +1,254 @@
+"""Machines and memory modes (catalog/machines.yaml).
+
+A memory mode is ``unified``, ``copy`` or ``managed``; a machine
+declares the modes it has, how each is selected (``env``: a run-time
+environment, the AMD HSA_XNACK; ``flags``: a compile flag, NVHPC's
+``-gpu=mem:...``), the value result rows store for it (``row_key``, the
+rows' ``xnack`` field), its headline mode, its scheduler and how it is
+reached (``access``: ``local`` or ``{ssh, root}``). ``not_applicable``
+records a mode the machine cannot run and why (machine data, not a
+failure). Rules use modes only; a mode a machine lacks raises
+``NotApplicable``, which collectors turn into ``not_applicable``
+evidence."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from functools import lru_cache
+from pathlib import Path
+
+import yaml
+
+from offload.engine import localcfg
+from offload.engine.root import ROOT  # noqa: E402
+MACHINES_FILE = ROOT / "catalog" / "machines.yaml"
+# ``managed`` (NVHPC -gpu=mem:managed: dynamically allocated data
+# in CUDA managed memory, everything else copied) is neither unified nor
+# copy; no rule names it, so the mode rules never fire on it
+MODES = ("unified", "copy", "managed")
+
+
+class NotApplicable(Exception):
+    """The machine has no such mode (or arch): not a failure, no evidence
+    is missing."""
+
+
+@dataclass(frozen=True)
+class Machine:
+    name: str
+    arch: str
+    device: str
+    access: object
+    scheduler: dict
+    headline_mode: str
+    mode_table: dict = field(default_factory=dict)
+    # the toolchains the data lists (test data); the committed machines
+    # take theirs from catalog/toolchains.yaml (``toolchains``)
+    listed_toolchains: tuple | None = None
+    # Modes the machine cannot run, with the reason (data)
+    not_applicable: dict = field(default_factory=dict)
+    # the gate's environment (c) is another GPU of this one shared host
+    one_host: bool = False
+    # whether the site plans its finding-matrix runs here (a toolchain's
+    # ``site_plan_runs`` overrides)
+    site_plan_runs: bool = True
+
+    @property
+    def toolchains(self) -> tuple:
+        """The measured toolchains of the machine (a regression-only
+        toolchain is not one): catalog/toolchains.yaml, unless the data
+        lists them."""
+        if self.listed_toolchains is not None:
+            return self.listed_toolchains
+        from offload.engine.toolchains import registry
+        return registry().of_machine(self.name)
+
+    @property
+    def modes(self) -> tuple[str, ...]:
+        return tuple(m for m in MODES if m in self.mode_table)
+
+    def has(self, mode: str) -> bool:
+        return mode in self.mode_table
+
+    def env(self, mode: str) -> dict[str, str]:
+        if not self.has(mode):
+            raise NotApplicable(f"{self.name} has no {mode} mode")
+        return {k: str(v) for k, v in
+                (self.mode_table[mode].get("env") or {}).items()}
+
+    def flags(self, mode: str, vendor: str = "") -> str:
+        """The compile flag that selects `mode` ("" when the mode is a
+        run-time setting, ``env``). A compiler vendor (Toolchain.kind) with
+        its own spelling has it in the mode's ``vendor_flags`` (GCC's
+        separate memory is its default: no flag)."""
+        if not self.has(mode):
+            raise NotApplicable(f"{self.name} has no {mode} mode")
+        spec = self.mode_table[mode]
+        own = spec.get("vendor_flags") or {}
+        if vendor and vendor in own:
+            return str(own[vendor])
+        return str(spec.get("flags") or "")
+
+    @property
+    def compile_time_modes(self) -> bool:
+        """Whether the modes are compile flags (NVHPC), so each
+        mode needs its own build."""
+        return any(spec.get("flags") for spec in self.mode_table.values())
+
+    def row_key(self, mode: str) -> int:
+        if not self.has(mode):
+            raise NotApplicable(f"{self.name} has no {mode} mode")
+        return int(self.mode_table[mode]["row_key"])
+
+    def mode_of_row_key(self, key) -> str:
+        for m in self.modes:
+            if int(self.mode_table[m]["row_key"]) == int(key):
+                return m
+        raise NotApplicable(f"{self.name} has no mode stored as {key}")
+
+    def other_modes(self, mode: str) -> tuple[str, ...]:
+        return tuple(m for m in self.modes if m != mode)
+
+
+def _check(name: str, d: dict) -> Machine:
+    for key in ("arch", "access", "scheduler", "headline_mode", "modes"):
+        if key not in d:
+            raise ValueError(f"machine {name}: no {key}")
+    modes = d["modes"] or {}
+    if not modes:
+        raise ValueError(f"machine {name}: no memory mode")
+    for m, spec in modes.items():
+        if m not in MODES:
+            raise ValueError(f"machine {name}: unknown mode {m} "
+                             f"(known: {', '.join(MODES)})")
+        if "row_key" not in spec:
+            raise ValueError(f"machine {name}: mode {m} has no row_key")
+        if spec.get("env") and spec.get("flags"):
+            raise ValueError(f"machine {name}: mode {m} has both env and "
+                             "flags")
+        vf = spec.get("vendor_flags")
+        if vf is not None and (not isinstance(vf, dict) or not spec.get(
+                "flags") or not all(isinstance(v, str)
+                                    for v in vf.values())):
+            raise ValueError(f"machine {name}: mode {m}: vendor_flags is "
+                             "a map vendor -> flag of a mode with flags")
+    na = d.get("not_applicable") or {}
+    for m, why in na.items():
+        if m not in MODES or m in modes or not why:
+            raise ValueError(f"machine {name}: bad not_applicable mode {m}")
+    if d["headline_mode"] not in modes:
+        raise ValueError(f"machine {name}: headline mode "
+                         f"{d['headline_mode']} is not one of its modes")
+    keys = [spec["row_key"] for spec in modes.values()]
+    if len(keys) != len(set(keys)):
+        raise ValueError(f"machine {name}: two modes share a row_key")
+    if (d["scheduler"] or {}).get("kind") not in ("flux", "slurm", "local",
+                                                  "direct"):
+        raise ValueError(f"machine {name}: unknown scheduler kind")
+    listed = tuple(d["toolchains"] or ()) if "toolchains" in d else None
+    return Machine(name, d["arch"], d.get("device", ""), d["access"],
+                   dict(d["scheduler"]), d["headline_mode"], dict(modes),
+                   listed, dict(na), bool(d.get("one_host")),
+                   bool(d.get("site_plan_runs", True)))
+
+
+def load(path: Path = MACHINES_FILE, data: dict | None = None
+         ) -> dict[str, Machine]:
+    """The machines of `path` (or of `data`), checked."""
+    if data is None:
+        return dict(_load_file(str(path)))
+    if data.get("version") != 1:
+        raise ValueError("machines: unknown version")
+    _under_test_only(data)
+    return {n: _check(n, d) for n, d in (data.get("machines") or {}).items()}
+
+
+def _read(path: str) -> dict:
+    """The file's data; for the catalog itself, with the user's local settings laid over."""
+    data = yaml.safe_load(Path(path).read_text()) or {}
+    return localcfg.overlay("machines.yaml", data) if Path(path) == MACHINES_FILE else data
+
+
+@lru_cache(maxsize=None)
+def _data(path: str = str(MACHINES_FILE)) -> dict:
+    return _read(path)
+
+
+def family_of(toolchain: str) -> str:
+    """The compiler family of a toolchain (catalog/toolchains.yaml
+    ``family``; test data may list ``families`` in the machines data); a
+    toolchain in no family is its own."""
+    for fam, members in (_data().get("families") or {}).items():
+        if toolchain in members:
+            return fam
+    from offload.engine.toolchains import registry
+    fam = registry().family_of(toolchain)
+    return fam.id if fam is not None else toolchain
+
+
+def _under_test_only(data: dict) -> frozenset:
+    """The checked ``under_test_only`` families of `data` (test data: each
+    a known family of its ``families`` with its reason); the committed
+    machines have none: the families are catalog/toolchains.yaml's."""
+    fams = set(data.get("families") or {})
+    out = set()
+    for fam, why in (data.get("under_test_only") or {}).items():
+        if fam not in fams or not why:
+            raise ValueError(f"machines: bad under_test_only entry {fam} "
+                             f"(families: {sorted(fams)}; a reason)")
+        out.add(fam)
+    return frozenset(out)
+
+
+def under_test_only() -> frozenset:
+    """Compiler families that are only ever compilers under test
+    (catalog/toolchains.yaml ``families.<f>.under_test_only``: GCC, the
+    the standing rule): their cells are never collected as evidence for
+    another family's case and they are never a legality witness."""
+    from offload.engine.toolchains import registry
+    return _under_test_only(_data()) | frozenset(registry().under_test_only())
+
+
+def compare_with(machine: str) -> tuple[str, ...]:
+    """The machines whose toolchains a machine's cases are
+    compared with (``compare_with``). The AMD machines are
+    compared with wingtip (an NVHPC cell is a family witness for an
+    AMD-side defect, in the modes both machines have)."""
+    d = ((_data().get("machines") or {}).get(machine) or {})
+    return tuple(d.get("compare_with") or ())
+
+
+def no_present_check() -> frozenset:
+    """(family, mode) pairs whose runs do not enforce the present check
+    (catalog/machines.yaml ``no_present_check``): no witness for a
+    presence or overlap abort."""
+    out = set()
+    for d in _data().get("no_present_check") or ():
+        if d.get("mode") not in MODES or not d.get("family"):
+            raise ValueError(f"machines: bad no_present_check entry {d}")
+        out.add((d["family"], d["mode"]))
+    return frozenset(out)
+
+
+def frontend_checkers() -> dict:
+    """name -> {family, lang} of the legality checkers that are
+    compilers."""
+    return dict(_data().get("frontend_checkers") or {})
+
+
+@lru_cache(maxsize=None)
+def _load_file(path: str) -> tuple:
+    data = _read(path)
+    return tuple(load(data=data).items())
+
+
+def machine_of(toolchain: str) -> Machine:
+    """The machine a toolchain runs on (offload.engine.toolchains)."""
+    from offload.engine.toolchains import TOOLCHAINS
+    return load()[TOOLCHAINS[toolchain].machine]
+
+
+def machine_of_arch(arch: str) -> Machine | None:
+    for m in load().values():
+        if m.arch == arch:
+            return m
+    return None

offload/engine/confirm/minimize.py

@@ -0,0 +1,268 @@
+"""The general minimizer: any case, Fortran or C, one file or a
+set. A candidate is kept only when the reduced program (a copy; corpus/
+is never touched) passes the legality gate and gets the SAME verdict
+under the same rules with the same failure group on the same toolchain
+and mode (``Reconfirm``: the candidate's own evidence is collected anew).
+
+Candidates: a fuzz-format Fortran program (``! fuzz: kernels`` marks)
+uses offload.engine.fuzz_min's transformations (blocks, nest levels,
+statements, clauses, right-hand sides, files of a set, cleanup); any other
+program uses the generic ones here: delete a directive with its end
+directive (Fortran) or a pragma line (C), delete a clause or one item of a
+clause's list, delete a statement line, delete a file of a set. Greedy:
+the first candidate in order that keeps the verdict wins, repeated until
+none does."""
+from __future__ import annotations
+
+import hashlib
+import re
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+
+from offload.engine.confirm import programs as P
+from offload.engine.confirm.model import Case, ProgramRef
+
+DIRECTIVE = re.compile(r"^\s*(?:!\$(?:omp|acc)|#\s*pragma\s+(?:omp|acc))\s+",
+                       re.I)
+END_DIRECTIVE = re.compile(r"^\s*!\$(?:omp|acc)\s+end\s+", re.I)
+KEEP = re.compile(r"^\s*(program|end\s+program|module|end\s+module|"
+                  r"contains|implicit|use\s|subroutine|end\s+subroutine|"
+                  r"function|end\s+function|integer function|"
+                  r"#include|int\s+main|return\s+0|}\s*$|print|printf|"
+                  r"write\s*\()", re.I)
+
+
+@dataclass
+class MinResult:
+    source: str
+    lines_before: int
+    lines_after: int
+    steps: list = field(default_factory=list)
+    calls: int = 0
+    verdict: str = ""
+
+    def summary(self) -> str:
+        return (f"minimized: {self.lines_before} -> {self.lines_after} lines "
+                f"in {len(self.steps)} steps ({self.calls} candidates "
+                f"judged), verdict {self.verdict} kept")
+
+
+def _clause_spans(text: str) -> list[tuple[int, int]]:
+    """(start, end) of each top-level clause of a directive line's tail
+    (after the directive name words)."""
+    spans, depth, start = [], 0, None
+    i = 0
+    while i < len(text):
+        ch = text[i]
+        if ch == "(":
+            depth += 1
+        elif ch == ")":
+            depth -= 1
+            if depth == 0 and start is not None:
+                spans.append((start, i + 1))
+                start = None
+        elif depth == 0 and re.match(r"[a-z_]", ch, re.I) and start is None \
+                and (i == 0 or text[i - 1] in " ,"):
+            m = re.match(r"[a-z_]+", text[i:], re.I)
+            j = i + m.end()
+            if j < len(text) and text[j] == "(":
+                start = i
+                i = j
+                continue
+            spans.append((i, j))
+            i = j
+            continue
+        i += 1
+    return spans
+
+
+def _end_of(lines: list[str], k: int) -> int | None:
+    """The matching end directive of a Fortran block directive at k."""
+    m = DIRECTIVE.match(lines[k])
+    if not m or lines[k].lstrip().startswith("#"):
+        return None
+    words = lines[k][m.end():].split("(")[0].split()
+    if not words:
+        return None
+    first = words[0].lower()
+    depth = 0
+    for j in range(k + 1, len(lines)):
+        mm = DIRECTIVE.match(lines[j])
+        if not mm:
+            continue
+        w = lines[j][mm.end():].split()
+        if not w:
+            continue
+        if w[0].lower() == first:
+            depth += 1
+        if w[0].lower() == "end" and len(w) > 1 and w[1].lower() == first:
+            if depth == 0:
+                return j
+            depth -= 1
+    return None
+
+
+def generic_candidates(text: str) -> list[tuple[str, str]]:
+    files = P.split_files(text)
+    if files and len(files) > 1:
+        out = [(f"delete file {n}", P.join_files([f for f in files
+                                                   if f[0] != n]))
+               for n, _ in files[1:]]
+        return out + [(f"in {n}: {w}", P.join_files(
+            [(m, t2 if m == n else t) for m, t in files]))
+            for n, t in files for w, t2 in generic_candidates(t)]
+    lines = text.splitlines()
+    out = []
+
+    def without(*ks):
+        return "\n".join(ln for i, ln in enumerate(lines)
+                         if i not in ks) + "\n"
+    for k, ln in enumerate(lines):
+        if DIRECTIVE.match(ln) and not END_DIRECTIVE.match(ln):
+            end = _end_of(lines, k)
+            if end is not None:
+                out.append((f"unwrap directive at line {k + 1}",
+                            without(k, end)))
+            elif ln.lstrip().startswith("#"):
+                out.append((f"delete pragma at line {k + 1}", without(k)))
+    for k, ln in enumerate(lines):
+        m = DIRECTIVE.match(ln)
+        if not m or END_DIRECTIVE.match(ln):
+            continue
+        tail = ln[m.end():]
+        for a, b in _clause_spans(tail)[1:]:
+            new = (ln[:m.end()] + tail[:a] + tail[b:]).rstrip()
+            new = re.sub(r"\s+,|,\s*$", "", new)
+            out.append((f"delete clause {tail[a:b][:30]} at line {k + 1}",
+                        "\n".join(lines[:k] + [new] + lines[k + 1:]) + "\n"))
+    for k, ln in enumerate(lines):
+        s = ln.strip()
+        if not s or s.startswith(("!", "//")) or DIRECTIVE.match(ln) or \
+                KEEP.match(ln) or s.endswith("{") or s.startswith("}") or \
+                re.match(r"^(do|end\s*do|if|end\s*if|else|for)\b", s, re.I) \
+                or "::" in s:
+            continue
+        out.append((f"delete statement at line {k + 1}", without(k)))
+    return out
+
+
+class Reconfirm:
+    """Does a candidate keep the verdict? (cached by text hash)"""
+
+    def __init__(self, case: Case, ctx, verdict: str, confirm=None,
+                 gate=None, rule: str | None = None):
+        self.case, self.ctx, self.verdict = case, ctx, verdict
+        self.rule = rule
+        self.cache: dict[str, tuple[bool, str]] = {}
+        self.calls = 0
+        from offload.engine.confirm import engine as EN
+        from offload.engine.confirm import legality as L
+        self.confirm = confirm or EN.confirm
+        if gate is None:
+            from offload.engine.confirm.machines import family_of
+            fam = (family_of(case.toolchain),)
+            gate = (lambda ref: L.gate(ref, under_test=fam))
+        self.gate = gate
+
+    def ref(self, text: str) -> ProgramRef:
+        p = self.case.program
+        return ProgramRef("file", f"{p.ident}~min", p.lang, p.version,
+                          p.model, True, p.needs, p.evidence, text,
+                          p.expected)
+
+    def __call__(self, text: str) -> tuple[bool, str]:
+        key = hashlib.sha256(text.encode()).hexdigest()
+        if key not in self.cache:
+            self.cache[key] = self._judge(text)
+        return self.cache[key]
+
+    def _judge(self, text: str) -> tuple[bool, str]:
+        from offload.engine.confirm import collectors as C
+        from offload.engine.confirm import findings as FI
+        self.calls += 1
+        ref = self.ref(text)
+        lg = self.gate(ref)
+        if lg.status != "legal":
+            return False, f"legality {lg.status}: {lg.basis[:120]}"
+        ev = C.run_cell(self.ctx, "minimize", Case(
+            ref, self.case.toolchain, self.case.machine, self.case.arch,
+            self.case.mode, "unknown"), self.case.toolchain, self.case.mode)
+        if ev is None or not ev.runs:
+            return False, "no run"
+        r = ev.runs[0]
+        if FI.group(r.outcome, r.reason) != FI.group(self.case.outcome,
+                                                     self.case.reason):
+            return False, f"outcome {r.outcome}"
+        # Spot check: the same failure, not only the same group
+        # (an ICE at another site, another diagnostic or another signal
+        # is another defect)
+        if signature_of(r.outcome, r.reason) != signature_of(
+                self.case.outcome, self.case.reason):
+            return False, (f"signature {signature_of(r.outcome, r.reason)}"
+                           " differs")
+        cand = Case(ref, self.case.toolchain, self.case.machine,
+                    self.case.arch, self.case.mode, r.outcome, r.reason,
+                    r.value, r.ran_on_host, ev.detail.startswith("values"),
+                    len(ev.runs), self.case.opt)
+        res = self.confirm(cand, self.ctx, legality=lg)
+        same = res["verdict"] == self.verdict and (
+            self.rule is None or res["rule"] == self.rule)
+        return same, f"{res['verdict']} {res['rule']}"
+
+
+def signature_of(outcome: str, reason: str) -> str:
+    """What must stay the same while minimizing: an ICE's compiler site
+    (or its message without positions), a rejection's diagnostic key, a
+    crash's signal; a wrong answer is kept by the verdict alone."""
+    from offload.engine.confirm import findings as FI
+    from offload.engine.confirm.facts import diagnostic_key
+    g = FI.group(outcome, reason)
+    if g == "ice":
+        return FI.site_of(reason) or FI.strip_site(FI.normalize(reason))
+    if g == "rejected":
+        return diagnostic_key(reason)
+    if g == "abort":
+        return FI.signal_of(reason) or g
+    return g
+
+
+def minimize_case(case: Case, ctx, verdict: str, workers: int = 4,
+                  max_steps: int = 200, judge=None,
+                  rule: str | None = None) -> MinResult:
+    judge = judge or Reconfirm(case, ctx, verdict, rule=rule)
+    text = case.program.text
+    before = len(text.splitlines())
+    steps = []
+    if "! fuzz: kernels" in text and case.program.lang == "f90":
+        from offload.engine import fuzz_min as FM
+        try:
+            res = FM.minimize(text, judge, workers=workers)
+        except ValueError as e:      # the failure does not reproduce
+            return MinResult(text, before, before, [str(e)[:300]],
+                             judge.calls, verdict)
+        return MinResult(res.source, before, res.lines_after, res.steps,
+                         judge.calls, verdict)
+    ok, why = judge(text)
+    if not ok:
+        return MinResult(text, before, before, ["the case does not "
+                                                "reproduce: " + why],
+                         judge.calls, verdict)
+    with ThreadPoolExecutor(max(1, workers)) as pool:
+        for _ in range(max_steps):
+            cands = generic_candidates(text)
+            hit = None
+            for b in range(0, len(cands), workers):
+                batch = cands[b:b + workers]
+                res = list(pool.map(lambda c: judge(c[1]), batch))
+                for c, (good, _) in zip(batch, res):
+                    if good:
+                        hit = c
+                        break
+                if hit:
+                    break
+            if not hit:
+                break
+            steps.append(hit[0])
+            text = hit[1]
+    return MinResult(text, before, len(text.splitlines()), steps,
+                     judge.calls, verdict)