offtarget 0.1.0__py3-none-any.whl

offload/engine/campaigns/__init__.py

@@ -0,0 +1,2 @@
+"""The campaigns behind the interface of offload.engine.campaign: one module
+each, registered in ``offload.engine.campaign.MODULES``."""

offload/engine/campaigns/hecbench.py

@@ -0,0 +1,153 @@
+"""The ``hecbench`` campaign: ORNL HeCBench OpenMP-offload benchmarks (``src/<name>-omp``)
+built and run by every toolchain and memory mode, to find failures in real code
+(data/reports/hecbench.md).
+
+Programs are recipes (offload.engine.hecbench) derived from each benchmark's Makefile.
+A hash of the directory name splits them into a dev half (data/bench/hecbench/recipes.yaml)
+and a holdout half (data/bench/hecbench-holdout/recipes.yaml); the run named ``holdout``
+is never reported, clustered or confirmed here. A benchmark excluded for a
+non-compiler reason is refused at the pre-run with the reason. Sources stay in
+the git-ignored clone corpus/apps/HeCBench-ornl; the run directory's
+programs.jsonl carries the recipes. A failure counts as evidence only when the
+reproduction gate confirms it and a compiler of another family runs the same
+program correctly (offload.engine.hecbench_report)."""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+
+from offload.engine import campaign as C
+from offload.engine import hecbench as H
+
+HOLDOUT_RUN = "holdout"
+
+
+@dataclass
+class Program:
+    id: str
+    dir: str
+    half: str
+    recipe: dict
+    lang: str = "c"                 # the interface's language slot (the sources are C or C++)
+    model: str = "omp"
+    version: int = 52
+    legality: str = "pending"
+    why: str = "real code: legality is not judged; evidence needs the gate and a witness family"
+    expected: dict = field(default_factory=dict)
+    cites: tuple = ()
+    modes: tuple = ("copy", "unified")
+    ref: dict | None = field(default=None, compare=False)
+
+    @property
+    def source(self) -> str:
+        return H.recipe_json(self.recipe)
+
+    @property
+    def text(self) -> str:
+        return self.source
+
+    def harness(self):
+        return self
+
+    def reference_output(self) -> str | None:
+        r = self.ref
+        return r["out"] if r and r.get("ok") else None
+
+
+def _full_recipes(half: str) -> dict:
+    """dir -> the whole recipe of the half: the tracked file's, and for a gpl-style
+    benchmark (tracked as an id only) the private copy under results/."""
+    path = H.RECIPES if half == "dev" else H.HOLDOUT_RECIPES
+    tracked = H.load_recipes(path)
+    priv = H.RESULTS / f"recipes_{half}_full.json"
+    full = {}
+    if priv.is_file():
+        full = {r["dir"]: r for r in json.loads(priv.read_text())}
+    return {d: (full.get(d) if r["licence"] in H.WITHHELD and d in full else r)
+            for d, r in tracked.items()}
+
+
+class Hecbench(C.Campaign):
+    name = "hecbench"
+    labels = ("pending",)
+    caps = C.Capabilities(langs=("c",), models=("omp",),
+                          families={"tuolumne": ("afar", "cce"), "tioga": None, "wingtip": None},
+                          regression=True, draw_options=("half",))
+    printed_values = False
+    compile_limit_s = H.COMPILE_TIMEOUT_S
+    timeout_s = H.RUN_TIMEOUT_S
+    doc = "data/reports/hecbench.md"
+    stored_programs = True
+    sealed_runs = (HOLDOUT_RUN,)
+
+    def programs(self, seed: int = 0, n: int = 0, half: str = "dev", **opts) -> list:
+        if half not in ("dev", "holdout"):
+            raise ValueError(f"half: dev or holdout, not {half!r}")
+        recipes = _full_recipes(half)
+        names = sorted(recipes)
+        if n:
+            names = names[:n]
+        return [Program(recipes[d]["id"], d, half, recipes[d]) for d in names]
+
+    def programs_from_records(self, rows: list) -> list:
+        return [Program(r["id"], r["recipe"]["dir"], r["recipe"]["half"], r["recipe"])
+                for r in sorted(rows, key=lambda r: r["id"])]
+
+    def is_sealed(self, programs: list) -> bool:
+        return any(p.half == "holdout" for p in programs)
+
+    def bind_run(self, run_dir, programs: list) -> None:
+        """Give each program its serial host reference (the pre-run rows)."""
+        refs = {r["id"]: r.get("ref") for r in C.read_jsonl(run_dir / "prerun.jsonl")}
+        for p in programs:
+            p.ref = refs.get(p.id)
+
+    def record(self, p) -> dict:
+        d = super().record(p)
+        d["recipe"] = p.recipe
+        return d
+
+    def src_name(self, p) -> str:
+        return f"{p.id}.recipe.json"
+
+    def ext(self, p) -> str:
+        return "c"
+
+    def gated(self, p) -> bool:
+        return p.recipe["status"] == "ok"
+
+    def make_runner(self):
+        from offload.engine.hecbench_run import HecbenchRunner
+        return HecbenchRunner(self)
+
+    def gate_filter(self, items: list, rows: list) -> list:
+        """A failure counts as evidence only with a compiler of another family running the
+        program correctly in the same memory mode (offload.engine.hecbench_report.witnessed), so a
+        cell without such a witness is not worth the gate's four reruns: it stays unconfirmed."""
+        from offload.engine import hecbench_report as RP
+        by = {}
+        for r in rows:
+            by.setdefault(r["program_id"], []).append(r)
+        return [it for it in items
+                if RP.witnessed({"toolchain": it["toolchain"], "xnack": it["xnack"]},
+                                by.get(it["ident"], []), C.mode_name)]
+
+    def row_tags(self, p) -> dict:
+        return {"lang": p.recipe.get("lang") or "cpp", "model": "omp", "legality": "pending",
+                "half": p.half, "licence": p.recipe["licence"]}
+
+    def mark(self, p) -> str:
+        return ""
+
+    def facets(self, p) -> set:
+        r = p.recipe
+        out = {f"lang={r.get('lang')}", f"std={r.get('std') or 'default'}",
+               f"files={len(r['compiles'])}"}
+        out |= {f"lib={x}" for x in r.get("libs", []) if x != "m"}
+        return out
+
+    def parse_observed(self, stdout: str) -> dict:
+        return {}
+
+
+CAMPAIGN = C.register(Hecbench())

offload/engine/campaigns/nested.py

@@ -0,0 +1,76 @@
+"""nested-data behind the campaign interface: pointer-linked data structures
+mapped to the device, in C and Fortran, OpenMP 5.2 and OpenACC 3.4
+(offload/engine/nested_gen.py; docs/tooling/nested-data.md). The draw
+is ``nested_gen.pilot``; its options (``min_cover``, ``legacy``,
+``max_tries``) are the draw's and are recorded in meta.json, so the second
+draw (seed 11, n 1000, min_cover 8, legacy False, max_tries 600: the 1055
+programs of results/nested/big) regenerates unchanged.
+
+Measured on every family of Tuolumne and wingtip (regression releases
+included) and on GCC on Tioga; the gate kind is ``nested``; a program whose
+legality is ``unspecified``, ``illegal`` or ``pending`` runs, but its
+failures are observations, never findings."""
+from __future__ import annotations
+
+import re
+
+from offload.engine import campaign as C
+
+# what NVHPC says about the map modifiers, declare mapper and the
+# allocatable runtime it lacks, on top of the shared list
+NOT_A_VERDICT = re.compile(
+    C.NOT_A_VERDICT.pattern +
+    r"|invalid text in pragma|syntax error at or near|openmp gpu - 'target'"
+    r" is used with map-type|call to nvhpc runtime function|openacc data "
+    r"clause expected|no acc routine information", re.I)
+
+
+class Nested(C.Campaign):
+    name = "nested"
+    caps = C.Capabilities(
+        langs=("c", "f90"), models=("omp", "acc"),
+        families={"tuolumne": ("afar", "cce"), "tioga": ("gcc",), "wingtip": None},
+        regression=True, draw_options=("min_cover", "legacy", "max_tries"))
+    not_a_verdict = NOT_A_VERDICT
+    fe_timeout_s = 90           # one front-end check; a hang is no verdict
+    doc = "docs/tooling/nested-data.md"
+
+    def programs(self, seed: int = 1, n: int = 400, **opts) -> list:
+        from offload.engine import nested_gen as G
+        return G.pilot(n, seed, min_cover=opts.get("min_cover", G.MIN_COVER),
+                       legacy=opts.get("legacy", True),
+                       max_tries=opts.get("max_tries", 80))
+
+    def record(self, p) -> dict:
+        return {"id": p.id, "family": p.family, "lang": p.lang,
+                "model": p.model, "version": p.version,
+                "legality": p.legality, "why": p.why,
+                "expected": p.expected, "rules": list(p.rules),
+                "cites": [dict(r) for r in p.cites], "axes": p.axes,
+                "title": p.title}
+
+    def parse_observed(self, stdout: str) -> dict:
+        from offload.engine import nested_gen as G
+        return G.parse_observed(stdout)
+
+    def row_tags(self, p) -> dict:
+        return {"family": p.family, "lang": p.lang, "model": p.model,
+                "legality": p.legality}
+
+    def gate_tags(self, r: dict) -> dict:
+        return {"family": r["family"]}
+
+    def short_id(self, p) -> str:
+        return p.id[3:]
+
+    def facets(self, p) -> set:
+        out = set()
+        for k, v in p.axes.items():
+            if k in ("constructs", "objects"):
+                continue
+            for x in (v if isinstance(v, list) else [v]):
+                out.add(f"{k}={x}")
+        return out
+
+
+CAMPAIGN = C.register(Nested())

offload/engine/campaigns/oop.py

@@ -0,0 +1,61 @@
+"""fortran-oop behind the campaign interface: the derived-type and
+object-oriented offload families (offload/engine/oop_gen.py). A hand-listed campaign: ``programs``
+is ``oop_gen.programs()`` whatever the draw arguments (its run directory
+has no meta.json, and none is needed).
+
+Fortran only, OpenMP and OpenACC; measured on every family of Tuolumne and
+wingtip (regression releases included) and on GCC on Tioga; the gate kind is
+``oop``. Every program is gated (a failure of an ``unspecified`` program is
+confirmed against the stored evidence too, with legality ``unknown``: the
+confirmer's legality keys on the label, never on ``gated``), as tools/campaigns/oop.py
+always did."""
+from __future__ import annotations
+
+from offload.engine import campaign as C
+
+
+class Oop(C.Campaign):
+    name = "oop"
+    labels = ("defined", "unspecified")
+    hand_listed = True
+    caps = C.Capabilities(
+        langs=("f90",), models=("omp", "acc"),
+        families={"tuolumne": ("afar", "cce"), "tioga": ("gcc",), "wingtip": None},
+        regression=True)
+
+    def programs(self, seed: int = 0, n: int = 0, **opts) -> list:
+        from offload.engine import oop_gen as G
+        return G.programs()
+
+    def record(self, p) -> dict:
+        return {"id": p.id, "family": p.family, "variant": p.variant,
+                "model": p.model, "version": p.version,
+                "legality": p.legality, "why": p.why,
+                "expected": p.expected, "modes": list(p.modes),
+                "cites": list(p.cites), "title": p.title}
+
+    def parse_observed(self, stdout: str) -> dict:
+        from offload.engine import oop_gen as G
+        return G.parse_observed(stdout)
+
+    def gated(self, p) -> bool:
+        return True
+
+    def unknown_why(self, p) -> str:
+        return f"unspecified behaviour: {p.why}"[:300]
+
+    def row_tags(self, p) -> dict:
+        return {"family": p.family, "variant": p.variant, "model": p.model,
+                "legality": p.legality}
+
+    def gate_tags(self, r: dict) -> dict:
+        return {"family": r["family"]}
+
+    def short_id(self, p) -> str:
+        return p.id[4:]
+
+    def mark(self, p) -> str:
+        return "U" if p.legality != "defined" else ""
+
+
+CAMPAIGN = C.register(Oop())

offload/engine/canary.py

@@ -0,0 +1,275 @@
+"""Canaries. Every campaign runs five programs per toolchain with a known
+outcome (catalog/canaries.yaml). Its report is not written if one changed.
+* ``pass``: runs correctly on the device on every toolchain;
+* ``ice``: a known internal compiler error or rejection;
+* ``wrong``: a known wrong answer;
+* ``racy``: a racy program the pre-run check (offload.engine.prerun) must refuse
+  (outcome ``refused``, never compiled);
+* ``hang``: never ends; the run timeout must end it (outcome
+  ``wrong_answer``, reason ``timeout``).
+
+``check`` names the changed canaries and ``require_unchanged`` stops the
+report. Canary rows are harness rows (offload.engine.harness.Result as a dict)
+plus ``canary`` (the kind) and ``canary_program``. A campaign writes them to
+``<campaign output>.canaries.jsonl`` (matrix file) or
+``<run dir>/canaries.jsonl`` (fuzz, stress); ``run_for`` makes them."""
+from __future__ import annotations
+
+import dataclasses
+import json
+from functools import lru_cache
+from pathlib import Path
+
+import yaml
+
+from offload.engine import localcfg
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+CANARIES = ROOT / "catalog" / "canaries.yaml"
+KINDS = ("pass", "ice", "wrong", "racy", "hang")
+
+
+class CanaryChanged(RuntimeError):
+    """A canary's outcome differs from catalog/canaries.yaml."""
+
+
+@lru_cache(maxsize=None)
+def _load(path: str) -> dict:
+    d = yaml.safe_load(Path(path).read_text()) or {}
+    if Path(path) == CANARIES:
+        d = localcfg.overlay("canaries.yaml", d)       # a user's own toolchains
+    if d.get("version") != 1:
+        raise ValueError("canaries: unknown version")
+    for tc, kinds in (d.get("toolchains") or {}).items():
+        if set(kinds) != set(KINDS):
+            raise ValueError(f"canaries: {tc} must name {KINDS}")
+        for k, v in kinds.items():
+            if v["program"] not in d["programs"]:
+                raise ValueError(f"canaries: {tc} {k}: unknown program")
+            if "outcome" not in v:
+                raise ValueError(f"canaries: {tc} {k}: no outcome")
+    return d
+
+
+def load(path: Path | None = None) -> dict:
+    return _load(str(path or CANARIES))
+
+
+def program(name: str, spec: dict | None = None):
+    """The Program of a canary program entry (a catalog id or a file)."""
+    spec = spec or load()
+    p = spec["programs"][name]
+    if "catalog" in p:
+        from offload.engine.confirm.programs import catalog_program
+        prog = catalog_program(p["catalog"])
+        if prog is None:
+            raise KeyError(f"canary program {p['catalog']} is no longer "
+                           "generated")
+        return prog
+    from offload.engine.harness import program_from_file
+    return program_from_file(ROOT / p["file"], id=f"canary-{name}")
+
+
+def _toolchain(prog, name: str):
+    from offload.engine.confirm.execute import toolchain_for
+    from offload.engine.confirm.programs import catalog_ref
+    from offload.engine.toolchains import TOOLCHAINS
+    if prog.id.startswith(("omp", "acc")) and not prog.id.startswith(
+            "canary-"):
+        return toolchain_for(catalog_ref(prog.id, prog), name)
+    return TOOLCHAINS[name]
+
+
+def run_for(toolchain: str, xnack: int, timeout_s: int = 120,
+            spec: dict | None = None, tc=None, runner=None,
+            prerun=None) -> list[dict]:
+    """The canary rows of one toolchain and memory mode: the racy canary
+    through the pre-run check only (never compiled), the others through
+    the harness with the campaign's run timeout. `tc`, `runner` and
+    `prerun` replace the toolchain, harness.run_program and
+    offload.engine.prerun.check (tests)."""
+    from offload.engine import harness
+    spec = spec or load()
+    kinds = spec["toolchains"][toolchain]
+    run = runner or harness.run_program
+    out = []
+    for kind in KINDS:
+        name = kinds[kind]["program"]
+        prog = program(name, spec)
+        if kind == "racy":
+            from offload.engine import prerun as PR
+            ok, why = (prerun or PR.check)(prog)
+            row = {"program_id": prog.id, "toolchain": toolchain,
+                   "xnack": xnack, "outcome": "accepted" if ok else
+                   "refused", "reason": "; ".join(why)[:600]}
+        else:
+            t = tc or _toolchain(prog, toolchain)
+            res = run(prog, t, xnack, "-O2", timeout_s=timeout_s, repeats=1)
+            row = dataclasses.asdict(res)
+            row["outcome"] = str(row["outcome"])
+            row["toolchain"] = toolchain
+        row["canary"] = kind
+        row["canary_program"] = name
+        out.append(row)
+    return out
+
+
+def _matches(row: dict, want: dict) -> bool:
+    if str(row.get("outcome")) != want["outcome"]:
+        return False
+    reason = row.get("reason") or ""
+    pre = want.get("reason")
+    return pre is None or reason.startswith(pre) or pre in reason
+
+
+def check(campaign_rows, spec: dict | None = None) -> list[str]:
+    """The canaries whose outcome differs from catalog/canaries.yaml, one
+    message each (rows without a ``canary`` field are the campaign's own
+    and are skipped; a canary of a toolchain the file does not list is
+    reported)."""
+    spec = spec or load()
+    out = []
+    for r in campaign_rows:
+        kind = r.get("canary")
+        if not kind:
+            continue
+        tc = r.get("toolchain")
+        want = (spec.get("toolchains") or {}).get(tc, {}).get(kind)
+        if want is None:
+            out.append(f"canary {kind} on {tc}: no expected outcome in "
+                       "catalog/canaries.yaml")
+            continue
+        if not _matches(r, want):
+            out.append(
+                f"canary {kind} ({want['program']}) on {tc} at mode key "
+                f"{r.get('xnack')}: expected {want['outcome']}"
+                + (f" ({want['reason']})" if want.get("reason") else "")
+                + f", got {r.get('outcome')}: {(r.get('reason') or '')[:160]}")
+    return out
+
+
+def rows_of(paths) -> list[dict]:
+    out = []
+    for p in paths:
+        p = Path(p)
+        if not p.is_file():
+            continue
+        for ln in p.read_text().splitlines():
+            if ln.strip():
+                out.append(json.loads(ln))
+    return out
+
+
+def files_for(results: Path) -> list[Path]:
+    """The canary files of a campaign output: ``<file>.canaries.jsonl``
+    next to a matrix file (and those of the per-mode runs merged into it:
+    ``<stem>.*.canaries.jsonl``), ``canaries.jsonl`` in a run directory."""
+    results = Path(results)
+    if results.is_dir():
+        return sorted(results.glob("canaries.jsonl"))
+    stem = results.name[:-len(".jsonl")] if results.name.endswith(
+        ".jsonl") else results.name
+    return sorted(set(results.parent.glob(f"{stem}.canaries.jsonl")) |
+                  set(results.parent.glob(f"{stem}.*.canaries.jsonl")))
+
+
+def _post_task45(campaign_rows) -> bool:
+    """A campaign recorded with provenance: its rows carry git_rev
+    or gate."""
+    return any(r.get("git_rev") or r.get("gate") for r in campaign_rows)
+
+
+def missing(campaign_rows, canary_rows, spec: dict | None = None
+            ) -> list[str]:
+    """The (toolchain, mode, kind) canaries
+    a campaign recorded with provenance lacks: the toolchains and modes
+    whose rows carry provenance (a file merged from older runs and newer
+    ones needs them for the newer only); a toolchain absent from
+    catalog/canaries.yaml is one message. An explicit exemption row
+    (``canary_exempt``: the reason) exempts the campaign."""
+    spec = spec or load()
+    if not _post_task45(campaign_rows):
+        return []
+    if any(r.get("canary_exempt") for r in canary_rows):
+        return []
+    have = {(r.get("toolchain"), int(r.get("xnack", -1)), r.get("canary"))
+            for r in canary_rows if r.get("canary")}
+    cells = {(r.get("toolchain"), int(r.get("xnack", r.get("mode", -1))))
+             for r in campaign_rows if r.get("toolchain") and _post_task45([r])}
+    out = []
+    for tc, x in sorted(cells, key=str):
+        if tc not in (spec.get("toolchains") or {}):
+            out.append(f"toolchain {tc} has no canaries in "
+                       "catalog/canaries.yaml")
+            continue
+        for k in KINDS:
+            if (tc, x, k) not in have:
+                out.append(f"canary {k} missing on {tc} at mode key {x}")
+    return out
+
+
+def require_unchanged(rows_or_paths, what: str = "the campaign",
+                      campaign_rows=None) -> int:
+    """Stop the report (CanaryChanged) when a canary changed, or (with
+    `campaign_rows`) when a campaign recorded with provenance lacks a
+    canary of one of its toolchains and modes and is not marked exempt
+    (CanaryMissing); the number of canary rows checked (0: a campaign
+    recorded without provenance, which has none)."""
+    rows = list(rows_or_paths)
+    if rows and not isinstance(rows[0], dict):
+        rows = rows_of(rows)
+    bad = check(rows)
+    if bad:
+        raise CanaryChanged(f"{what}: {len(bad)} canar"
+                            f"{'y' if len(bad) == 1 else 'ies'} changed; the "
+                            "report is not written until they are explained "
+                            "(catalog/canaries.yaml):\n  " + "\n  ".join(bad))
+    if campaign_rows is not None:
+        gone = missing(list(campaign_rows), rows)
+        if gone:
+            raise CanaryMissing(
+                f"{what}: {len(gone)} canaries missing; the report is not "
+                "written until the campaign runs them or is marked exempt "
+                "with a reason (--canaries-exempt):\n  "
+                + "\n  ".join(gone[:20]))
+    return sum(1 for r in rows if r.get("canary"))
+
+
+class CanaryMissing(CanaryChanged):
+    """A campaign recorded with provenance has no canary of a toolchain,
+    mode and kind it ran, and no exemption."""
+
+
+def exempt(path: Path, reason: str) -> None:
+    """Mark a campaign exempt from canaries, with the reason (one row in
+    its canaries file)."""
+    if not reason.strip():
+        raise ValueError("an exemption needs a reason")
+    write([{"canary_exempt": reason.strip()}], Path(path))
+
+
+def write(rows: list[dict], path: Path) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a") as fh:
+        for r in rows:
+            fh.write(json.dumps(r) + "\n")
+
+
+def add_to_campaign(path: Path, toolchains, modes, timeout_s: int = 120,
+                    log=print) -> list[dict]:
+    """Run the canaries of every toolchain and mode of a campaign and
+    append them to `path` (a campaign runner's last step)."""
+    spec = load()
+    rows = []
+    for t in toolchains:
+        if t not in (spec.get("toolchains") or {}):
+            raise KeyError(f"canaries: toolchain {t} has none in "
+                           "catalog/canaries.yaml (an error)")
+        for x in modes:
+            rows += run_for(t, x, timeout_s, spec)
+    write(rows, path)
+    bad = check(rows, spec)
+    log(f"canaries: {len(rows)} rows -> {path}"
+        + (f"; CHANGED: {'; '.join(bad)}" if bad else "; unchanged"))
+    return rows