offtarget 0.1.0__py3-none-any.whl

offload/engine/harvest2/hand.py

@@ -0,0 +1,150 @@
+"""Programs from training material, example collections and applications,
+made self-checking by recorded edits (catalog/harvest2_hand.yaml;
+``tools/campaigns/harvest2.py hand``).
+
+An entry names a source file at a pinned commit and lists its edits. Each
+edit is an exact text replacement (``old`` must occur exactly once) with a
+reason (``why``). The edits add a self-check (a host recomputation or the
+source's documented values, compared with a stated tolerance; a wrong result
+exits nonzero) and only what a standalone legal program needs besides (a
+missing #include, a kernel driver, a smaller problem size).
+offload.engine.harvest.convert then renames main and appends the self-check
+(OpenACC uses the host probe of offload.engine.harvest2.gomp).
+
+The oracle is ``exit_status``, or ``filecheck`` with CHECK lines
+(``checks``). An entry with no check and no ``self_checking: true`` is refused."""
+from __future__ import annotations
+
+import hashlib
+import re
+
+from offload.engine.harvest import checks as K
+from offload.engine.harvest import convert as HC
+from offload.engine.harvest2 import gomp as G
+
+
+class EditError(ValueError):
+    """An edit that does not apply, or an entry without a self-check."""
+
+
+def apply_edits(text: str, edits: list[dict]) -> tuple[str, list[str]]:
+    notes = []
+    for e in edits or []:
+        n = text.count(e["old"])
+        want = e.get("count", 1)
+        if n != want:
+            raise EditError(f"edit {e['why']!r}: the old text occurs {n} "
+                            f"times, not {want}")
+        text = text.replace(e["old"], e["new"])
+        notes.append(e["why"])
+    return text, notes
+
+
+def lang_of(path: str) -> str:
+    return "c" if path.endswith(".c") else "f90"
+
+
+def convert(e: dict, text: str, appended: dict | None = None,
+            root=None) -> dict:
+    """``appended``: path -> text of the entry's ``append`` and ``prepend``
+    files (the other translation units of a multi-file example, appended
+    in order, or put first: a Fortran module before its use)."""
+    lang = lang_of(e["path"])
+    inlined: list = []
+    if e.get("inline"):
+        from pathlib import Path
+        skip = tuple(e.get("inline_skip") or ())
+        text, inlined = G.inline_includes(text, Path(root) / e["path"],
+                                          skip=skip)
+        appended = dict(appended or {})
+        for q in (e.get("append") or []) + (e.get("prepend") or []):
+            appended[q], more = G.inline_includes(appended[q],
+                                                  Path(root) / q, skip=skip)
+            inlined += more
+        if None in inlined:
+            raise EditError(f"{e['id']}: a local include is missing")
+    for path in e.get("append") or []:
+        mark = (f"/* Appended {path} */" if lang == "c" else
+                f"! Appended {path}")
+        text = text.rstrip("\n") + f"\n{mark}\n" + (appended or {})[path]
+    for path in reversed(e.get("prepend") or []):
+        mark = (f"/* Prepended {path} */" if lang == "c" else
+                f"! Prepended {path}")
+        text = f"{mark}\n" + (appended or {})[path].rstrip("\n") + \
+            f"\n{mark.replace('prepended', 'end of')}\n" + text
+    crlf = "\r\n" in text
+    if crlf:
+        text = text.replace("\r\n", "\n")
+    if e.get("detab"):
+        text = text.expandtabs(8)
+    body, notes = apply_edits(text, e.get("edits") or [])
+    if crlf:
+        notes.insert(0, "CRLF line ends as LF")
+    if inlined:
+        notes.insert(0, "local includes inlined: " +
+                     ", ".join(sorted(set(i for i in inlined if i))))
+    if e.get("detab"):
+        notes.insert(0, "tabs replaced by spaces (a tab is not in the Fortran "
+                     "character set)")
+    if e.get("append"):
+        notes.insert(0, "the example's other files appended: " +
+                     ", ".join(e["append"]))
+    if e.get("prepend"):
+        notes.insert(0, "the example's other files put first (modules "
+                     "before their use): " + ", ".join(e["prepend"]))
+    checks = e.get("checks") or []
+    if not notes and not checks and not e.get("self_checking"):
+        raise EditError(f"{e['id']}: no self-check (edits, checks or "
+                        "self_checking)")
+    model = e.get("model") or G.model_of(body)
+    if model not in ("omp", "acc"):
+        raise EditError(f"{e['id']}: model {model}")
+    cs = K.parse_checks("\n".join("// " + c for c in checks), ("CHECK",)) \
+        if checks else []
+    defines = e.get("defines") or []
+    v = {"flags": defines, "prefixes": ["CHECK"] if cs else []}
+    if lang == "c":
+        src, cedits = HC.convert_c(body, cs, v)
+    else:
+        src, cedits = HC.convert_f(body, cs, v)
+    if model == "acc":
+        src = G.acc_runtime(src, lang)
+        cedits.append("the self-check's host probe is OpenACC")
+    version, gerr = HC.spec_version(body, lang, [], model)
+    oracle = "filecheck" if cs else "exit_status"
+    modes = e.get("modes") or ["copy", "unified"]
+    orig = hashlib.sha256(text.encode()).hexdigest()
+    ver = ("OpenACC 3.4" if model == "acc" else
+           f"OpenMP {version // 10}.{version % 10}")
+    head = [
+        "Harvested program (tools/campaigns/harvest2.py hand; do not edit)",
+        f"source: {e['url']}",
+        f"license: {e['license']}",
+        f"original sha256: {orig}",
+        f"modes: {', '.join(modes)}"
+        + (f"; defines: {' '.join(defines)}" if defines else ""),
+        f"spec version: {ver} (the grammar reads every directive at it)",
+        f"expected: exit status 0; {e['why']}" + (
+            f"; the output matches {len(cs)} CHECK lines" if cs else ""),
+        "self-check: RESULT ran_on_host=<0|1> check=<pass|fail>",
+        "edits: " + "; ".join(notes + cedits + ["self-check appended"]),
+    ]
+    if lang == "c":
+        head = ["/*"] + [" * " + ln.replace("*/", "* /") for ln in head] + \
+            [" */"]
+        pre = HC.CR.PRELUDE
+        src = pre + "\n".join(head) + "\n" + src[len(pre):]
+    else:
+        src = "\n".join("! " + ln for ln in head) + "\n" + src
+    return {"id": e["id"], "record": f"{e['repo']}/{e['path']}", "lang": lang,
+            "model": model, "version": version, "source": src,
+            "oracle": oracle, "checks": len(cs),
+            "prefixes": ["CHECK"] if cs else [], "modes": modes,
+            "edits": notes + cedits, "grammar_errors": gerr, "url": e["url"],
+            "license": e["license"], "project": e["repo"],
+            "suite": e.get("suite", e["repo"]), "commit": e["commit"],
+            "gpl": bool(re.match(r"(?:L|A)?GPL", e["license"])),
+            "copyleft": bool(re.match(r"(?:(?:L|A)?GPL|CC-BY-SA)",
+                                      e["license"])),
+            "original_sha256": orig,
+            "source_kind": e.get("kind", "training")}

offload/engine/harvest2/split.py

@@ -0,0 +1,78 @@
+"""The dev/holdout split of the harvest2 programs.
+
+The holdout is the lowest 30% of the sha256 space of a program's code
+(offload.engine.harvest.convert.holdout_by_code: comments, the header among
+them, and blank runs ignored), so programs that differ
+only in comments fall on one side:
+data/bench/harvest2-holdout, opaque case files harvest2-holdout-NNN.<c|f90>
+numbered in program-id order, cases.yaml (provenance) and SEAL (the digest
+of tools/campaigns/fresh_bench.py over the case files, and the sha256 of cases.yaml;
+outcomes are added by phase 2's runs). Everything else is the dev set,
+data/bench/harvest2, one file per program named by its id.
+
+A copyleft program (GPL: the GCC testsuite; CC-BY-SA) goes to a
+``copyleft/`` subdirectory of either set, beside its license texts; the
+maintainers decide on publishing it."""
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+from offload.engine.harvest import convert as HC
+
+HOLDOUT_NAME = "harvest2-holdout"
+
+
+def _ext(lang: str) -> str:
+    return ".c" if lang == "c" else ".f90"
+
+
+def plan(progs: list[dict]) -> list[dict]:
+    """Each program with ``set`` (dev or holdout), ``path`` (relative to
+    the set's directory) and, for the holdout, ``case``."""
+    out = []
+    k = 0
+    for p in sorted(progs, key=lambda q: q["id"]):
+        sub = "copyleft/" if p.get("copyleft") else ""
+        if HC.holdout_by_code(p["source"], p["lang"]):
+            k += 1
+            case = f"{HOLDOUT_NAME}-{k:03d}"
+            out.append({**p, "set": "holdout", "case": case,
+                        "path": sub + case + _ext(p["lang"])})
+        else:
+            out.append({**p, "set": "dev",
+                        "path": sub + p["id"] + _ext(p["lang"])})
+    return out
+
+
+def dev_similar(plan_rows: list[dict]) -> dict:
+    """Each dev program's `similar` list without holdout programs (no
+    holdout id in the dev set's cases)."""
+    hold = {p["id"] for p in plan_rows if p["set"] == "holdout"}
+    return {p["id"]: [x for x in p.get("similar") or []
+                      if not (x.get("set") == "harvest2" and x["id"] in hold)]
+            for p in plan_rows if p["set"] == "dev"}
+
+
+def digest(paths) -> str:
+    """tools/campaigns/fresh_bench.py's digest: sha256 over the case files sorted by
+    name, each as name, NUL, bytes, NUL."""
+    h = hashlib.sha256()
+    for p in sorted(paths, key=lambda q: q.name):
+        h.update(p.name.encode() + b"\0" + p.read_bytes() + b"\0")
+    return h.hexdigest()
+
+
+def seal_text(commit: str, paths, meta: Path) -> str:
+    paths = list(paths)
+    return (
+        f"# Seal of bench/{HOLDOUT_NAME} (the harvest2 holdout: "
+        "30% of the programs by sha256)\n"
+        f"commit: {commit}\n"
+        f"cases: {len(paths)}\n"
+        f"method: sha256 over the case files ({HOLDOUT_NAME}-*.f90, "
+        # the sealed text names the tool by its original path (tools/): the committed SEAL files say so
+        f"{HOLDOUT_NAME}-*.c, copyleft/ included) sorted by name, each as "
+        "name, NUL, bytes, NUL (tools/fresh_bench.py digest)\n"
+        f"sha256: {digest(paths)}\n"
+        f"cases_yaml_sha256: {hashlib.sha256(meta.read_bytes()).hexdigest()}\n")

offload/engine/harvest_known/__init__.py

@@ -0,0 +1,2 @@
+"""Extraction side of the known-issues harvest (centre pages,
+NVIDIA forum and release notes, application workaround history)."""

offload/engine/harvest_known/apps.py

@@ -0,0 +1,65 @@
+"""The workaround history of open offload applications.
+
+Commits, pull requests and issues that add or remove a workaround naming a
+compiler (CCE, Cray, nvfortran, NVHPC, amdflang, AOMP, XNACK, "compiler
+bug"). The callers read ``git log`` of a clone in the untrusted download
+area and the GitHub API's issue/PR JSON (read-only); these functions only
+parse text."""
+from __future__ import annotations
+
+import re
+
+from offload.engine.harvest_known import textscan
+
+_HIT = re.compile(
+    r"\bCCE\b|\bcray\b|\bcrayftn\b|\bftn-\d+|\bnvfortran\b|\bnvc\b|\bnvhpc\b|"
+    r"\bNV\s?HPC\b|\bpgi\b|\bpgfortran\b|\bamdflang\b|\bamd[- ]flang\b|"
+    r"\bLLVMFlang\b|\bflang-new\b|\bAFAR\b|\bAOMP\b|XNACK|"
+    r"compiler\s+(bug|issue|error|crash|problem)|internal compiler error|"
+    r"\bICE\b|miscompil|\bfort2\b|work[- ]?around", re.I)
+_PR = re.compile(r"(?:\(|\bpull request\s+|\bsee\s+|\band\s+|^|\s)#(\d+)\b",
+                 re.I)
+_FENCE = re.compile(r"```[^\n]*\n(.*?)```", re.S)
+
+
+def workaround_hit(text: str) -> bool:
+    return bool(_HIT.search(text))
+
+
+def pr_numbers(text: str) -> list[int]:
+    return [int(x) for x in _PR.findall(text)]
+
+
+def parse_log(log: str) -> list[dict]:
+    """``git log --format=%x1e%H %ad%n%B --date=short`` into records (the
+    ASCII record separator starts each commit; bodies may hold anything)."""
+    out: list[dict] = []
+    for chunk in log.split("\x1e")[1:]:
+        head, _, rest = chunk.partition("\n")
+        commit, date = head.split()[:2]
+        lines = rest.strip("\n").split("\n")
+        subject = lines[0] if lines else ""
+        body = "\n".join(lines[1:]).strip()
+        out.append({"commit": commit, "date": date, "subject": subject,
+                    "body": body})
+    return out
+
+
+def issue_record(issue: dict, repo: str) -> dict:
+    body = issue.get("body") or ""
+    text = (issue.get("title") or "") + "\n" + body
+    codes = _FENCE.findall(body)
+    mods: set[str] = set()
+    for c in codes:
+        mods |= textscan.models(c)
+    mods |= textscan.models(body)
+    if "omp-target" in mods:
+        mods.discard("omp-host")
+    return {"repo": repo, "number": issue["number"],
+            "kind": "pr" if issue.get("pull_request") else "issue",
+            "url": issue.get("html_url"), "title": issue.get("title"),
+            "created": (issue.get("created_at") or "")[:10],
+            "state": issue.get("state"), "n_code": len(codes),
+            "models": sorted(mods),
+            "compilers": sorted(textscan.compilers(text)),
+            "hit": workaround_hit(text)}

offload/engine/harvest_known/cases.py

@@ -0,0 +1,128 @@
+"""The harvested cases (data/bench/harvest-known/hk-NNN.<f90|c>, the
+data/bench/fresh case format: a header with ``fresh case:``, ``source:``,
+``edits:``, ``expected: RESULT: pass``).
+
+legality_rows runs the pre-run legality check (offload.engine.prerun.check:
+the rules witness and the front-end witness, never a compiler under test
+judging itself, plus the race rule) on each case. Refused cases are
+recorded with the reasons and never run."""
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def lang_of(path: Path) -> str:
+    return "c" if path.suffix == ".c" else "f90"
+
+
+def legality_rows(paths, frontend: bool = True,
+                  timeout_s: int = 600) -> list[dict]:
+    from offload.engine import prerun
+    rows = []
+    for p in paths:
+        text = Path(p).read_text()
+        ok, why = prerun.check((text, lang_of(Path(p))), frontend=frontend,
+                               timeout_s=timeout_s)
+        rows.append({"id": Path(p).stem, "ok": bool(ok), "reasons": why,
+                     "kind": None if ok else prerun.refusal_kind(why)})
+    return rows
+
+
+import re as _re
+
+_NV = _re.compile(r"forums\.developer\.nvidia\.com/t/(?:[^/]+/)?(\d+)")
+_GH = _re.compile(r"github\.com/([^/]+)/([^/]+)/(?:issues|pull)/(\d+)")
+_COMMIT = _re.compile(r"/commit/([0-9a-f]{7,40})")
+
+
+def url_key(url: str) -> str:
+    """A source URL in a form two sets can compare: the forum topic id, the
+    GitHub issue or pull-request number, the first 7 hex digits of a
+    commit; any other URL as written (trailing slash dropped)."""
+    u = str(url).strip()
+    m = _NV.search(u)
+    if m:
+        return f"nvforum:{m.group(1)}"
+    m = _GH.search(u)
+    if m:
+        return f"gh:{m.group(1).lower()}/{m.group(2).lower()}#{m.group(3)}"
+    m = _COMMIT.search(u)
+    if m:
+        return f"commit:{m.group(1)[:7]}"
+    return u.rstrip("/")
+
+
+def known_keys(dirs) -> dict[str, str]:
+    """url_key -> the case id (or 'excluded') of every source URL another
+    set holds (cases.yaml: each case's url and also; excluded sources).
+    For a sealed set the caller only tests membership; nothing here is
+    printed."""
+    import yaml
+    out: dict[str, str] = {}
+    for d in dirs:
+        meta = Path(d) / "cases.yaml"
+        if not meta.is_file():
+            continue
+        y = yaml.safe_load(meta.read_text()) or {}
+        for c in y.get("cases") or []:
+            for u in [c.get("url"), *(c.get("also") or [])]:
+                if u:
+                    out.setdefault(url_key(u), c["id"])
+        for e in y.get("excluded") or []:
+            if e.get("source"):
+                out.setdefault(url_key(e["source"]), "excluded")
+    return out
+
+
+# --- the 30% holdout and the export to the runner --------------------------------
+
+import hashlib as _hashlib
+
+HOLDOUT_FRACTION = 0.30
+# toolchains (offload.engine.toolchains names) by what they compile: the AMD
+# gfx942 toolchains, their gfx90a twins and the other toolchains of the
+# merged result tables (NVHPC; GCC, under test only, is no target); OpenACC is CCE Fortran and NVHPC only (AFAR
+# has none; CCE has no OpenACC C): catalog/toolchains.yaml families
+from offload.engine.toolchains import GFX90A_TOOLCHAINS as _G90  # noqa: E402
+from offload.engine.toolchains import GFX942_TOOLCHAINS as _G942  # noqa: E402
+from offload.engine.toolchains import TOOLCHAINS as _TC, registry as _registry  # noqa: E402
+ALL_TC = list(_G942 + _G90 + tuple(t for t in _registry().merged() if t not in _G942
+                                    and _TC[t].family_id not in _registry().under_test_only()))
+ACC_F_TC = [t for t in ALL_TC if _TC[t].family.why_unsupported("acc", "f90") is None]
+ACC_C_TC = [t for t in ALL_TC if _TC[t].family.why_unsupported("acc", "c") is None]
+
+
+def holdout(text: str) -> bool:
+    """The rule (offload.engine.harvest.convert.holdout): the program
+    whose sha256 lies in the lowest 30% of the hash space."""
+    h = int(_hashlib.sha256(text.encode()).hexdigest(), 16)
+    return h < int(HOLDOUT_FRACTION * (1 << 256))
+
+
+def toolchains_for(lang: str, model: str) -> list[str]:
+    if model == "acc":
+        return list(ACC_C_TC if lang == "c" else ACC_F_TC)
+    return list(ALL_TC)
+
+
+def export_rows(bench) -> list[dict]:
+    """The cases of ``bench`` as rows of the runner's programs.jsonl (id,
+    record, variant, version, oracle, file, lang, model, modes, holdout),
+    plus the toolchains that compile the model and the text's sha256."""
+    import yaml
+    bench = Path(bench)
+    meta = yaml.safe_load((bench / "cases.yaml").read_text())
+    out = []
+    for c in meta["cases"]:
+        text = (bench / c["file"]).read_text()
+        model = c["model"]
+        version = 34 if model == "acc" else int(c.get("omp_version", 52))
+        out.append({"id": c["id"], "record": c["id"], "variant": 0,
+                    "version": version, "oracle": "selfcheck",
+                    "file": c["file"], "lang": c["lang"], "model": model,
+                    "modes": list(c.get("modes", ["copy", "unified"])),
+                    "holdout": holdout(text),
+                    "sha256": _hashlib.sha256(text.encode()).hexdigest(),
+                    "toolchains": toolchains_for(c["lang"], model),
+                    "source": c.get("source_kind", "")})
+    return out

offload/engine/harvest_known/centres.py

@@ -0,0 +1,184 @@
+"""Supercomputing centres' known-issues pages as a bug source.
+
+Pages come as HTML (the live page and its web.archive.org snapshots, e.g.
+OLCF, whose known issues are injected from JIRA at build time) or as the
+Markdown/RST sources of a docs repository whose history is the archive
+(NERSC, ALCF, LUMI, CSC). Each parser splits a page into sections
+(heading, text, code blocks); relevant() keeps the sections about an
+offload compiler defect; entry_key() names an entry across snapshots
+(the ticket id when it has one)."""
+from __future__ import annotations
+
+import html as _html
+import re
+
+from offload.engine.harvest_known import textscan
+
+_H = re.compile(r"<h([1-6])[^>]*>(.*?)</h\1>", re.S | re.I)
+_PRE = re.compile(r"<pre[^>]*>(.*?)</pre>", re.S | re.I)
+_TAG = re.compile(r"<[^>]+>")
+
+
+def _clean(s: str) -> str:
+    return _html.unescape(_TAG.sub("", s))
+
+
+def _ws(s: str) -> str:
+    return " ".join(s.split())
+
+
+def html_sections(page: str) -> list[dict]:
+    out = []
+    hs = list(_H.finditer(page))
+    for k, m in enumerate(hs):
+        end = hs[k + 1].start() if k + 1 < len(hs) else len(page)
+        body = page[m.end():end]
+        codes = [_clean(c).strip("\n") for c in _PRE.findall(body)]
+        text = _ws(_clean(_PRE.sub(" ", body)))
+        out.append({"heading": _ws(_clean(m.group(2))).rstrip("¶ ").strip(),
+                    "level": int(m.group(1)), "text": text, "codes": codes})
+    return out
+
+
+_MD_H = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
+_FENCE = re.compile(r"^\s*(```|~~~)")
+
+
+def md_sections(md: str) -> list[dict]:
+    out: list[dict] = []
+    cur = None
+    code: list[str] | None = None
+    for line in md.splitlines():
+        if code is not None:
+            if _FENCE.match(line):
+                if cur is not None:
+                    cur["codes"].append("\n".join(code))
+                code = None
+            else:
+                code.append(line)
+            continue
+        if _FENCE.match(line):
+            code = []
+            continue
+        m = _MD_H.match(line)
+        if m:
+            cur = {"heading": m.group(2).strip(), "level": len(m.group(1)),
+                   "text": "", "codes": []}
+            out.append(cur)
+        elif cur is not None:
+            cur["text"] += " " + line
+    for s in out:
+        s["text"] = _ws(s["text"])
+    return out
+
+
+_RST_UL = re.compile(r"^([=\-~^\"'`#*+])\1{2,}\s*$")
+
+
+def _dedent_block(lines: list[str]) -> str:
+    ind = min((len(x) - len(x.lstrip()) for x in lines if x.strip()),
+              default=0)
+    while lines and not lines[-1].strip():
+        lines.pop()
+    return "\n".join(x[ind:] for x in lines)
+
+
+def rst_sections(rst: str) -> list[dict]:
+    lines = rst.splitlines()
+    out: list[dict] = []
+    cur = None
+    levels: list[str] = []
+    i = 0
+    while i < len(lines):
+        ln = lines[i]
+        nxt = lines[i + 1] if i + 1 < len(lines) else ""
+        if ln.strip() and _RST_UL.match(nxt) and len(nxt.strip()) >= \
+                len(ln.strip()) and not _RST_UL.match(ln):
+            ch = nxt.strip()[0]
+            if ch not in levels:
+                levels.append(ch)
+            cur = {"heading": ln.strip(), "level": levels.index(ch) + 1,
+                   "text": "", "codes": []}
+            out.append(cur)
+            i += 2
+            continue
+        lit = ln.rstrip().endswith("::") or \
+            re.match(r"^\s*\.\.\s+(code-block|code|sourcecode)::", ln)
+        if lit and cur is not None:
+            if not ln.strip().startswith(".."):
+                cur["text"] += " " + ln.rstrip()[:-1]
+            base = len(ln) - len(ln.lstrip())
+            j = i + 1
+            while j < len(lines) and (not lines[j].strip() or
+                                      lines[j].strip().startswith(":")):
+                j += 1
+            blk: list[str] = []
+            while j < len(lines) and (not lines[j].strip() or
+                                      len(lines[j]) - len(lines[j].lstrip())
+                                      > base):
+                blk.append(lines[j])
+                j += 1
+            if any(x.strip() for x in blk):
+                cur["codes"].append(_dedent_block(blk))
+            i = j
+            continue
+        if cur is not None:
+            cur["text"] += " " + ln
+        i += 1
+    for s in out:
+        s["text"] = _ws(s["text"])
+    return out
+
+
+_DEVICE_WORDS = re.compile(
+    r"\bOpenMP\b.{0,80}\b(offload|target|GPU|device)\b|\b(offload|target)"
+    r"\b.{0,80}\bOpenMP\b|\bOpenACC\b|!\$acc|!\$omp\s+target|#pragma\s+"
+    r"(acc|omp\s+target)|\bomp\s+target\b|\bdeclare\s+target\b|"
+    r"-fopenmp-targets|-mp=gpu|-acc\b|-hacc\b|-fopenmp\b.{0,40}--offload|"
+    r"--offload-arch|\bomp_target_\w+|\bacc_\w+\(|\bdo\s+concurrent\b.{0,80}"
+    r"\b(GPU|offload|stdpar)\b|-stdpar", re.I | re.S)
+
+
+def relevant(sec: dict) -> bool:
+    """A section about a defect of an offload compiler: it carries offload
+    code (on a centre's page that code is for the centre's compilers), or
+    its text names both a compiler and a device model."""
+    blob = " ".join([sec["heading"], sec["text"]])
+    code = "\n".join(sec["codes"])
+    if {"acc", "omp-target"} & textscan.models(code):
+        return True
+    return bool(_DEVICE_WORDS.search(blob)) and \
+        bool(textscan.compilers(blob + "\n" + code))
+
+
+_TICKET = re.compile(r"\b([A-Z][A-Z0-9]{2,}[-_]\d{1,7})\b")
+
+
+def entry_key(heading: str) -> str:
+    m = _TICKET.search(heading)
+    if m:
+        return m.group(1).replace("_", "-")
+    return _ws(heading).lower()
+
+
+def union(rows: list[dict]) -> list[dict]:
+    """One entry per (source, key) over every snapshot that shows it: the
+    first and last snapshot dates, and the richest text and code seen."""
+    by: dict[tuple, dict] = {}
+    for r in sorted(rows, key=lambda r: r["snapshot"]):
+        k = (r["source"], entry_key(r["heading"]))
+        e = by.get(k)
+        if e is None:
+            e = by[k] = {**r, "key": k[1], "first_seen": r["snapshot"],
+                         "last_seen": r["snapshot"], "snapshots": 0,
+                         "pages": []}
+            e.pop("snapshot", None)
+        e["snapshots"] += 1
+        e["last_seen"] = r["snapshot"]
+        if r.get("page") not in e["pages"]:
+            e["pages"].append(r.get("page"))
+        if len(r["text"]) > len(e["text"]):
+            e["text"] = r["text"]
+        if sum(map(len, r["codes"])) > sum(map(len, e["codes"])):
+            e["codes"] = r["codes"]
+    return sorted(by.values(), key=lambda e: (e["source"], e["key"]))