offtarget 0.1.0__py3-none-any.whl

offload/engine/confirm/decisions.py

@@ -0,0 +1,202 @@
+"""Agent decisions and spot checks over a batch of confirmer results.
+
+The confirmer decides each failing cell by its rules. An agent decides only
+the cells left ``needs_human`` and re-reads a seeded random 10% of each
+verdict class. Both are data in catalog/triage_28.yaml:
+
+* ``decisions``: match needs_human results by rule id, toolchain, outcome and
+  regexes over reason and signature, and give a verdict with its basis. A
+  decision never changes a verdict the rules gave.
+* ``spot_checks``: one entry per signature group of the sampled cases
+  (``spot_sample``, ``spot_groups``): agree true/false and a note.
+* ``case_spot_checks``: one entry per sampled case (``case_sample``, at most
+  CAP per class, agent-decided cells as class "agent-decided");
+  ``case_agreement`` counts them."""
+from __future__ import annotations
+
+import random
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+DATA = ROOT / "catalog" / "triage_28.yaml"
+
+
+def _msg_key(text: str) -> str:
+    t = re.sub(r"File = [^|]*\|", "", text or "")
+    t = re.sub(r"ftn: ERROR \S+", "ftn: ERROR", t)
+    t = re.sub(r'loc\("[^"]*":\d+:\d+\)', "loc", t)
+    t = re.sub(r"<tmpdir>\S*|/\S+", "<path>", t)
+    t = re.sub(r'"[A-Za-z0-9_$%]+"', '"X"', t)
+    t = re.sub(r"checksum=-?\d+|expected=-?\d+", "", t)
+    t = re.sub(r"0x[0-9a-fA-F]+|\d+", "N", t)
+    return " ".join(t.split("; repeats")[0].split())[:120]
+
+
+def signature(r: dict) -> str:
+    """toolchain | outcome | rule | cause: the later form R11c names, the
+    finding a bug is an instance of, else the message key."""
+    c = r["case"]
+    m = re.search(r"later form (\S+) \(", r.get("reason") or "")
+    if m:
+        cause = f"form {m.group(1)}"
+    elif r.get("findings"):
+        cause = "finding " + Path(r["findings"][0]["file"]).name
+    else:
+        cause = _msg_key(c.get("reason", ""))
+    return f"{c['toolchain']}|{c['outcome']}|{r['rule']}|{cause}"
+
+
+@dataclass(frozen=True)
+class Decision:
+    id: str
+    verdict: str
+    basis: str
+    rule: str = "*"
+    toolchains: tuple = ()
+    outcomes: tuple = ()
+    reason: str = ""
+    sig: str = ""
+    cases: tuple = ()
+    own_sample_class: bool = False   # sampled as a class of its own
+
+    def matches(self, r: dict) -> bool:
+        from fnmatch import fnmatchcase
+        c = r["case"]
+        if r["verdict"] != "needs_human":
+            return False
+        if self.cases and c["case_id"] not in self.cases:
+            return False
+        if not fnmatchcase(r["rule"], self.rule):
+            return False
+        if self.toolchains and c["toolchain"] not in self.toolchains:
+            return False
+        if self.outcomes and c["outcome"] not in self.outcomes:
+            return False
+        if self.reason and not re.search(self.reason, c.get("reason", "")):
+            return False
+        if self.sig and not re.search(self.sig, signature(r)):
+            return False
+        return True
+
+
+def load(path: Path = DATA) -> dict:
+    if not Path(path).is_file():
+        return {"decisions": [], "spot_checks": [], "case_spot_checks": []}
+    d = yaml.safe_load(Path(path).read_text()) or {}
+    if d.get("version") != 1:
+        raise ValueError("triage_28: unknown version")
+    decs = []
+    for x in d.get("decisions") or ():
+        m = x.get("match") or {}
+        decs.append(Decision(x["id"], x["verdict"], x["basis"],
+                             m.get("rule", "*"),
+                             tuple(m.get("toolchains") or ()),
+                             tuple(m.get("outcomes") or ()),
+                             m.get("reason", ""), m.get("signature", ""),
+                             tuple(m.get("cases") or ()),
+                             bool(x.get("own_sample_class", False))))
+    return {"decisions": decs, "spot_checks": list(d.get("spot_checks")
+                                                    or ()),
+            "case_spot_checks": list(d.get("case_spot_checks") or ())}
+
+
+def apply(results: list[dict], decisions) -> list[dict]:
+    """Each result with `final` (verdict) and `decided_by` ("rules" or
+    the decision id): the first matching decision decides a needs_human
+    result."""
+    out = []
+    for r in results:
+        r = dict(r)
+        r["final"], r["decided_by"] = r["verdict"], "rules"
+        if r["verdict"] == "needs_human":
+            for d in decisions:
+                if d.matches(r):
+                    r["final"], r["decided_by"] = d.verdict, d.id
+                    break
+        out.append(r)
+    return out
+
+
+def spot_sample(results: list[dict], frac: float = 0.1, seed: int = 28
+                ) -> dict[str, list[str]]:
+    """A seeded random `frac` (at least one) of each rule-decided verdict
+    class, by case id."""
+    by = defaultdict(set)
+    for r in results:
+        if r["verdict"] != "needs_human":
+            by[r["verdict"]].add(r["case"]["case_id"])
+    rnd = random.Random(seed)
+    out = {}
+    for v in sorted(by):
+        ids = sorted(by[v])
+        out[v] = sorted(rnd.sample(ids, max(1, round(len(ids) * frac))))
+    return out
+
+
+def spot_groups(results: list[dict], sample: dict) -> dict[str, list[str]]:
+    """The sampled case ids by signature (verdict | signature)."""
+    want = {c for v in sample.values() for c in v}
+    out = defaultdict(list)
+    for r in results:
+        if r["case"]["case_id"] in want:
+            out[f"{r['verdict']}|{signature(r)}"].append(
+                r["case"]["case_id"])
+    return {k: sorted(v) for k, v in sorted(out.items())}
+
+
+CAP = 60
+CASE_SEED = 2801          # A fresh draw
+
+
+def own_classes(decisions) -> tuple:
+    """The ids of the decisions sampled as a class of their own
+    (``own_sample_class: true`` in the decisions file: a decision that
+    would otherwise fill the agent-decided sample, e.g. one that came
+    later with thousands of cells)."""
+    return tuple(d.id for d in decisions if d.own_sample_class)
+
+
+def case_sample(final: list[dict], frac: float = 0.1, cap: int = CAP,
+                seed: int = CASE_SEED, own_class=()
+                ) -> dict[str, list[str]]:
+    """The case-level spot sample of results with `final`/`decided_by`
+    (apply): a seeded `frac` of each final verdict class decided by the
+    rules (needs_human included), at least one and at most `cap` cases,
+    and the same of the agent-decided cells as class "agent-decided" (a
+    decision in `own_class` as "agent-decided <id>")."""
+    by = defaultdict(set)
+    for r in final:
+        d = r.get("decided_by", "rules")
+        k = r["final"] if d == "rules" else \
+            f"agent-decided {d}" if d in own_class else "agent-decided"
+        by[k].add(r["case"]["case_id"])
+    rnd = random.Random(seed)
+    out = {}
+    for v in sorted(by):
+        ids = sorted(by[v])
+        n = min(cap, max(1, round(len(ids) * frac)))
+        out[v] = sorted(rnd.sample(ids, n))
+    return out
+
+
+def sample_of(checks: list[dict]) -> dict[str, list[str]]:
+    """The recorded case-level sample (each check names the class its
+    case had when the sample was drawn)."""
+    out = defaultdict(list)
+    for c in checks:
+        out[c["class"]].append(c["case"])
+    return {k: sorted(v) for k, v in sorted(out.items())}
+
+
+def case_agreement(sample: dict, checks: list[dict]) -> dict:
+    """Per sampled class: cases sampled, read (a check entry) and agreed."""
+    by = {c["case"]: c for c in checks}
+    return {v: {"sampled": len(ids),
+                "read": sum(1 for c in ids if c in by),
+                "agree": sum(1 for c in ids if c in by and by[c]["agree"])}
+            for v, ids in sample.items()}

offload/engine/confirm/draft.py

@@ -0,0 +1,195 @@
+"""Finding drafts and their promotion.
+
+A draft is a directory results/confirm/drafts/<case id>/ with
+``finding.md`` (header: compilers and versions, arch, memory modes,
+expected vs actual, the evidence table, the rule id, the ruling citations
+or "citation needed"), ``program.<f90|c>`` (the minimized program when
+there is one, else the case's program) and ``result.json``. Drafts never
+go to findings by themselves: ``promote`` copies the program there
+with the header as comments and appends an entry to
+data/findings/index.yaml (an explicit step)."""
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+DRAFTS = ROOT / "results" / "confirm" / "drafts"
+FINDINGS = ROOT / "data" / "findings"
+
+
+def _slug(text: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("_")[:120]
+
+
+def _compiler(ev: list[dict], tc: str) -> str:
+    from offload.engine.toolchains import TOOLCHAINS, compiler_version
+    try:
+        return compiler_version(TOOLCHAINS[tc]) or tc
+    except Exception:
+        return tc
+
+
+# gfortran and gcc are never a legality witness (standing rule). The gate's base-language syntax check (gfortran -std=f2008
+# or gcc -std=c11, without OpenMP) stays in result.json's checks, but a
+# draft's legality line never names it as a checker.
+_BASE_CHECKER = re.compile(r"\b(gfortran|gcc)\b")
+
+
+def _checker_list(basis: str) -> tuple[list[str], str] | None:
+    """The checkers of a "legal by the checkers (...)" basis and the text
+    after its closing parenthesis, the list split at top-level commas (a
+    checker name may contain parentheses); None for another basis."""
+    head = "legal by the checkers ("
+    if not basis.startswith(head):
+        return None
+    depth, cur, items = 0, "", []
+    for i, ch in enumerate(basis[len(head):]):
+        if ch == "(":
+            depth += 1
+        elif ch == ")":
+            if depth == 0:
+                items.append(cur)
+                return ([x.strip() for x in items if x.strip()],
+                        basis[len(head) + i + 1:])
+            depth -= 1
+        elif ch == "," and depth == 0:
+            items.append(cur)
+            cur = ""
+            continue
+        cur += ch
+    return None
+
+
+def legality_text(leg: dict) -> str:
+    """The draft's legality line: the gate's status and basis without the
+    base-language checkers. A "legal by the checkers (...)" basis keeps
+    its other checkers; a basis that rests on gfortran or gcc is replaced
+    by a pointer to the spec reading."""
+    status = leg.get("status", "unknown")
+    basis = leg.get("basis", "")
+    parsed = _checker_list(basis)
+    if parsed is not None:
+        kept = [x for x in parsed[0] if not _BASE_CHECKER.search(x)]
+        tail = parsed[1]
+        if _BASE_CHECKER.search(tail):
+            tail = ""
+        basis = ("checked by " + ", ".join(kept) + tail
+                 + "; the legality rests on the spec reading in the "
+                 "finding")
+    elif _BASE_CHECKER.search(basis):
+        basis = ("the base-language syntax check rejects it (see "
+                 "result.json); the legality needs a spec reading")
+    return f"{status} ({basis})"
+
+
+def render(result: dict, compiler_text: str | None = None) -> str:
+    c = result["case"]
+    p = c["program"]
+    exp = result.get("expected") or {}
+    rows = []
+    for e in result.get("evidence", []):
+        if not e.get("runs") and e.get("status") != "value":
+            continue
+        outs = ", ".join(sorted({r["outcome"] for r in e.get("runs", [])}))
+        rows.append(f"| {e['collector']} | {e.get('toolchain', '')} | "
+                    f"{e.get('arch', '')} | {e.get('mode', '')} | "
+                    f"{e.get('variant', '')} | {outs} | "
+                    f"{(e.get('value') or '')[:60]} | "
+                    f"{e.get('provenance', {}).get('origin', '')} |")
+    cites = result.get("citations") or ["citation needed"]
+    lines = [
+        f"# Draft finding: {p['ident']} on {c['toolchain']}",
+        "",
+        f"- compiler: {compiler_text or c['toolchain']}",
+        f"- toolchain: {c['toolchain']}; arch: {c['arch']}; machine: "
+        f"{c['machine']}; memory mode: {c['mode']}; opt: {c['opt']}",
+        f"- program: {p['kind']} {p['ident']} ({p['lang']}, "
+        f"version {p['version']}, sha256 {p['sha256'][:16]})",
+        f"- expected: {exp.get('value')} ({exp.get('status')}; "
+        f"{' + '.join(exp.get('sources', []))})",
+        f"- actual: {c['outcome']} {c.get('value') or ''} "
+        f"({c.get('reason', '')[:200]})",
+        f"- verdict: {result['verdict']} by rule {result['rule']}: "
+        f"{result['rationale']}",
+        f"- reason: {result['reason']}",
+        "- citations: " + "; ".join(cites),
+        f"- legality: {legality_text(result['legality'])}",
+        f"- witnesses: {', '.join(result.get('witnesses') or []) or 'none'}",
+        "- withdrawn: " + (", ".join(f"{w['toolchain']} ({w['finding']})"
+                                     for w in result.get("withdrawn", []))
+                           or "none"),
+        "",
+        "## Evidence",
+        "",
+        "| collector | toolchain | arch | mode | variant | outcomes | value |"
+        " origin |",
+        "|---|---|---|---|---|---|---|---|",
+        *rows,
+        "",
+        "## Actions",
+        "",
+        *[f"- {a}" for a in result.get("actions", [])],
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def write_draft(result: dict, base: Path | None = None) -> Path:
+    c = result["case"]
+    d = Path(base or DRAFTS) / _slug(c["case_id"])
+    d.mkdir(parents=True, exist_ok=True)
+    (d / "finding.md").write_text(render(result))
+    ext = "c" if c["program"]["lang"] == "c" else "f90"
+    src = result.get("minimized") or result.get("source") or ""
+    if not src:
+        from offload.engine.confirm.ingest import Row  # noqa: F401
+        src = result.get("program_text", "")
+    (d / f"program.{ext}").write_text(src)
+    (d / "result.json").write_text(json.dumps(result, indent=1,
+                                              default=str))
+    return d
+
+
+def promote(draft: Path, name: str, findings: Path = FINDINGS,
+            index: Path | None = None) -> Path:
+    """Copy a draft's program to data/findings/<name> with the draft's
+    header as comments and add an index entry."""
+    draft = Path(draft)
+    result = json.loads((draft / "result.json").read_text())
+    c = result["case"]
+    lang = c["program"]["lang"]
+    ext = "c" if lang == "c" else "f90"
+    if not name.endswith(f".{ext}"):
+        name = f"{name}.{ext}"
+    target = Path(findings) / name
+    if target.exists():
+        raise FileExistsError(target)
+    lead = "//" if lang == "c" else "!"
+    head = []
+    for ln in (draft / "finding.md").read_text().split("## Evidence")[0] \
+            .splitlines():
+        ln = f"{lead} {ln}" if ln else lead
+        if lang == "c":
+            head.append(ln)
+        else:
+            from offload.engine.hdrwrap import wrap_line
+            head.extend(wrap_line(ln))
+    body = (draft / f"program.{ext}").read_text()
+    if lang != "c":
+        from offload.engine.hdrwrap import wrap_comments
+        body = wrap_comments(body)
+    target.write_text("\n".join(head) + "\n" + body)
+    index = Path(index or Path(findings) / "index.yaml")
+    exp = (result.get("expected") or {}).get("value") or ""
+    entry = (f"  - file: {name}\n    toolchains: [{c['toolchain']}]\n"
+             f"    lang: {lang}\n    outcome: {c['outcome']}\n"
+             f"    modes: [{c['mode']}]\n    arch: [{c['arch']}]\n")
+    if c["outcome"] in ("ice", "rejected", "crash"):
+        msg = c.get("reason", "").split(":", 1)[-1].strip()[:80]
+        entry += f"    message: {json.dumps(re.escape(msg))}\n"
+    if exp:
+        entry += f"    expected: {json.dumps(exp)}\n"
+    with index.open("a") as f:
+        f.write(entry)
+    return target

offload/engine/confirm/engine.py

@@ -0,0 +1,157 @@
+"""``confirm(case)``: the legality gate, the rules, then the actions
+(dedupe, minimize, draft) for a failure the rules call a bug.
+
+The JSON result per case: the case, its legality, the verdict with its
+rule id, rationale, reason and citations (registry entries as
+"<id> (<version> §<section> p.<page>: "<quote>")"), every withdrawn
+witness with its finding, the findings whose signature the failure has,
+the evidence (with provenance), the actions taken and the drafts
+written."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from offload.engine.confirm import collectors as C
+from offload.engine.confirm import findings as FI
+from offload.engine.confirm import legality as L
+from offload.engine.confirm import registry as REG
+from offload.engine.confirm import rules as RU
+from offload.engine.confirm.facts import Facts
+from offload.engine.confirm.model import Case
+
+BUG_VERDICTS = ("compiler_bug", "copy_mode_failure", "unified_memory_failure",
+                "arch_specific")
+# An instance of a known finding is still minimized and
+# drafted when no finding it matches has a reproducer of its shape
+
+
+def _fuzz_check(case: Case, ctx) -> str | None:
+    if case.program.kind != "fuzz":
+        return None
+    row = ctx.index.get("fuzz", case.program.ident, case.toolchain,
+                        case.mode)
+    return row.extra.get("check") if row is not None else None
+
+
+def facts_of(case: Case, ctx, legality=None) -> Facts:
+    from offload.engine.confirm.machines import family_of
+    lg = legality or L.gate(case.program, _fuzz_check(case, ctx),
+                            under_test=(family_of(case.toolchain),))
+    reg = REG.matching(case.program.ident, case.program.text,
+                       case.program.needs, case.reason)
+    row = ctx.index.get(case.program.kind, case.program.ident,
+                        case.toolchain, case.mode)
+    extra = dict(row.extra) if row is not None else {}
+    # the host builds of the same program (host check)
+    extra["host_values"] = {
+        r.toolchain: r.extra.get("host_checksum")
+        for r in ctx.index.rows_of(case.program.kind, case.program.ident)
+        if r.mode == case.mode and "host_checksum" in r.extra}
+    # the same atom's 5.2 program on this toolchain and mode (is a 6.0
+    # rejection one of the 6.0 form only?)
+    import re as _re
+    m = _re.match(r"omp60-(.*)$", case.program.ident)
+    if case.program.kind == "catalog" and m:
+        r52 = ctx.index.get("catalog", f"omp52-{m.group(1)}",
+                            case.toolchain, case.mode)
+        extra["v52_outcome"] = r52.outcome if r52 is not None else None
+        extra["v52_reason"] = r52.reason if r52 is not None else ""
+    return Facts(case, legality=lg, registry=reg, extra=extra)
+
+
+def other_cells(f: Facts) -> list[dict]:
+    """Every device cell of another toolchain, mode or arch, passing or
+    failing, with the finding a failure has the signature of (no
+    disagreeing evidence is dropped silently)."""
+    out = []
+    for e in f.cells():
+        if e.toolchain == f.case.toolchain and e.mode == f.case.mode and \
+                e.variant == "original":
+            continue
+        failed = not all(o in ("correct_on_device", "unobservable")
+                         for o in e.outcomes)
+        hits = FI.dedupe(e.toolchain, f.case.program.lang, e.outcomes[0],
+                         e.runs[0].reason, f.case.program.text,
+                         f.case.program.ident, e.mode,
+                         arch=e.arch, opt=f.case.opt) if failed else []
+        out.append({"collector": e.collector, "toolchain": e.toolchain,
+                    "arch": e.arch, "mode": e.mode, "variant": e.variant,
+                    "outcome": e.outcomes[0], "value": e.value,
+                    "finding": hits[0]["file"] if hits else None})
+    return out
+
+
+def confirm(case: Case, ctx, legality=None, minimize: bool = False,
+            draft: bool = False, draft_dir: Path | None = None) -> dict:
+    f = facts_of(case, ctx, legality)
+    d = RU.decide(f, ctx)
+    reg = REG.entries()
+    out = {"case": case.to_dict(), "legality": f.legality.to_dict(),
+           "verdict": d.verdict, "rule": d.rule, "rationale": d.rationale,
+           "reason": d.reason,
+           "citations": [reg[c].cite() for c in d.cites if c in reg] or
+           (["citation needed"] if d.verdict in BUG_VERDICTS else []),
+           "withdrawn": d.withdrawn, "expected": d.expected,
+           "witnesses": d.witnesses, "fail_rate": d.fail_rate,
+           "collectors_run": d.collectors_run,
+           "registry": [e.id for e in f.registry],
+           "evidence": [e.to_dict() for es in f.evidence.values()
+                        for e in es],
+           "actions": [], "drafts": [], "findings": []}
+    if case.failing:
+        # The reproduction gate's flag on the case's
+        # cell. "unchecked": not rerun, the verdict above is the confirmer's
+        # on stored data only; such a row is never evidence for a new
+        # finding
+        from offload.engine import gate as G
+        gv = G.verdict_of_case(case)
+        if gv is not None:
+            out["gate"] = gv.get("verdict")
+            if gv.get("decision"):
+                out["gate_decision"] = gv["decision"]
+        out["other_cells"] = other_cells(f)
+        out["missing"] = f.missing_evidence()
+        out["families_without_cell"] = f.missing_families() \
+            if case.failing else []
+    if d.verdict in BUG_VERDICTS:
+        output = f.extra.get("stderr_tail", "")
+        hits = FI.dedupe(case.toolchain, case.program.lang, case.outcome,
+                         case.reason, case.program.text, case.program.ident,
+                         case.mode, arch=case.arch, output=output,
+                         related=True, opt=case.opt)
+        dups = [h for h in hits if h["kind"] == "duplicate"]
+        out["findings"] = dups
+        out["related"] = [h for h in hits if h["kind"] == "related"]
+        if dups:
+            out["actions"].append(f"dedupe: instance of {dups[0]['file']} "
+                                  f"({dups[0]['why']})")
+        else:
+            out["actions"].append("dedupe: new failure signature"
+                                  + (f" (related: {out['related'][0]['file']})"
+                                     if out["related"] else ""))
+        # a new signature, or an instance whose findings have no
+        # reproducer of its shape, is minimized and drafted; any case when
+        # asked explicitly (minimize="always")
+        shaped = any(h["same_shape"] for h in dups)
+        if not shaped or minimize == "always":
+            if dups:
+                out["actions"].append("no finding has a reproducer of this "
+                                      "shape" if not shaped else
+                                      "minimized on request")
+            if minimize:
+                from offload.engine.confirm import minimize as MZ
+                res = MZ.minimize_case(case, ctx, d.verdict, workers=8,
+                                       rule=d.rule)
+                out["actions"].append(res.summary())
+                out["minimized"] = res.source
+            if draft:
+                from offload.engine.confirm import draft as DR
+                out["source"] = case.program.text
+                path = DR.write_draft(out, draft_dir)
+                out["drafts"].append(str(path))
+    return out
+
+
+def dumps(result: dict) -> str:
+    return json.dumps(result, indent=1, default=str)