osaudit 0.1.0__py3-none-any.whl

osaudit/__init__.py

@@ -0,0 +1,3 @@
+"""osaudit: trace every claim of a scientific paper to the evidence that supports it."""
+
+__version__ = "0.1.0"

osaudit/__main__.py

@@ -0,0 +1,7 @@
+"""Run `python -m osaudit`."""
+
+from __future__ import annotations
+
+from ._cli import main
+
+raise SystemExit(main())

osaudit/_cli.py

@@ -0,0 +1,114 @@
+"""The `osaudit` command line: silent when the paper is clean, one line per finding otherwise.
+
+Exit status: 0 no finding, 1 findings, 2 the audit could not run (configuration, input).
+`osaudit run` writes a results file with its provenance; it prints nothing on success.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Callable, Sequence
+from pathlib import Path
+
+from . import __version__, _numbers, _provenance, _quotes, _results_check
+from ._config import Config, ConfigError, instruments, load
+from ._findings import Finding
+from ._git import GitError
+from ._pdf import PdfError
+
+
+def _results(cfg: Config, args: argparse.Namespace) -> list[Finding]:
+    return _results_check.check(cfg, inputs=getattr(args, "inputs", False),
+                                rerun=getattr(args, "rerun", False),
+                                timeout=getattr(args, "timeout", None))
+
+
+CHECKS: dict[str, tuple[Callable[[Config, argparse.Namespace], list[Finding]], str]] = {
+    "numbers": (lambda cfg, _: _numbers.check(cfg),
+                "every number of the paper traced to results, derivation or allow"),
+    "quotes": (lambda cfg, _: _quotes.check(cfg),
+               "every cited verbatim quote in a verified [[quotes]] entry, with its page"),
+    "results": (_results,"every results file has a provenance that still holds "
+                          "(static; --inputs, --rerun to go further)"),
+}
+
+
+def parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="osaudit",
+        description="Audit a paper repository: report every claim not traced to its evidence. "
+                    "Silent when clean; `file:line: rule: message` per finding. "
+                    "Conventions: CONVENTIONS.md. Without a check name, runs every check "
+                    "(static parts only).",
+        epilog="exit status: 0 clean, 1 findings, 2 the audit could not run",
+    )
+    p.add_argument("--version", action="version", version=f"osaudit {__version__}")
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--root", type=Path, default=Path("."),
+                        help="paper repository (default: current directory)")
+    common.add_argument("--config", type=Path, help="audit.toml (default: <root>/audit.toml)")
+    common.add_argument("--json", action="store_true",
+                        help="print findings as a JSON array ([] when clean)")
+    sub = p.add_subparsers(dest="check", metavar="CHECK")
+    for name, (_, help_) in CHECKS.items():
+        sp = sub.add_parser(name, parents=[common], help=help_, description=help_)
+        if name == "results":
+            sp.add_argument("--inputs", action="store_true",
+                            help="rehash every recorded input (needs the inputs)")
+            sp.add_argument("--rerun", action="store_true",
+                            help="rerun every recorded command at its time anchor and compare")
+            sp.add_argument("--timeout", type=float, help="seconds per rerun")
+    sub.add_parser("all", parents=[common], help="run every check (the default)")
+    run = sub.add_parser(
+        "run", help="run an instrument, write its JSON results with their provenance",
+        description="Run CMD (its stdout must be a JSON object) with the time anchor in "
+                    f"${_provenance.ENV_NOW}, and write the object to OUTPUT with a provenance "
+                    "block: command, inputs and their sha256, git commit, time anchor. "
+                    "Refuses to run on a tree with uncommitted changes.",
+        epilog="example: osaudit run --input data/answers.csv -o results/survey.json -- "
+               "python study/survey.py data/answers.csv")
+    run.add_argument("--root", type=Path, default=Path("."), help="repository of the instruments")
+    run.add_argument("--input", type=Path, action="append", default=[], dest="input_paths",
+                     help="an input file or directory (repeat for each)")
+    run.add_argument("-o", "--output", type=Path, required=True, help="results file to write")
+    run.add_argument("cmd", nargs=argparse.REMAINDER, help="-- CMD [ARG ...]")
+    return p
+
+
+def _run(args: argparse.Namespace) -> int:
+    cmd = args.cmd[1:] if args.cmd[:1] == ["--"] else args.cmd
+    if not cmd:
+        print("osaudit run: no command (put it after --)", file=sys.stderr)
+        return 2
+    root = args.root.resolve()
+    try:
+        output = args.output.resolve()
+        out_rel = str(output.relative_to(root)) if output.is_relative_to(root) else str(output)
+        _provenance.run(root, cmd, args.input_paths, args.output, instruments(root, out_rel))
+    except (_provenance.RunError, ConfigError, GitError, OSError) as e:
+        print(f"osaudit run: {e}", file=sys.stderr)
+        return 2
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if not (argv and argv[0] in (*CHECKS, "all", "run", "-h", "--help", "--version")):
+        argv = ["all", *argv]
+    args = parser().parse_args(argv)
+    if args.check == "run":
+        return _run(args)
+    names = list(CHECKS) if args.check == "all" else [args.check]
+    try:
+        cfg = load(args.root.resolve(), args.config.resolve() if args.config else None)
+        findings = [f for n in names for f in CHECKS[n][0](cfg, args)]
+    except (ConfigError, PdfError, GitError, OSError, ValueError) as e:
+        print(f"osaudit: {e}", file=sys.stderr)
+        return 2
+    if args.json:
+        print(json.dumps([f.as_json() for f in findings], indent=1))
+    else:
+        sys.stdout.write("".join(f"{f}\n" for f in findings))
+    return 1 if findings else 0

osaudit/_config.py

@@ -0,0 +1,283 @@
+"""`audit.toml`: where the paper, the results and the instruments are, and how they are audited.
+
+    [paper]
+    tex = ["paper/sections/*.tex"]     # LaTeX sources, audited when present
+    pdf = "paper/main.pdf"             # audited when no tex is declared
+
+    [results]
+    files = ["study/results/*.json"]
+    instruments = ["study/*.py"]       # the code that produces the results (staleness, clock)
+
+    [[results.depends]]                # optional: the instruments one results file depends on
+    file = "study/results/survey.json" # (default: every instrument)
+    instruments = ["study/survey.py", "study/_common.py"]
+
+    [[results.manual]]                 # a results file made by a person, not an instrument
+    file = "study/results/labels_check.json"
+    by = "A. Author"
+    date = "2026-10-17"
+    reason = "the author checked 20 labels of the classifier"
+
+    [[numbers.derive]]
+    value = "1,204"
+    expr = "survey.responses - survey.partial"
+    file = "paper/results.tex"         # optional: only there
+
+    [[numbers.allow]]
+    value = "2,000,000"
+    reason = "population size published by the cited census"
+    cite = "census2020"                # optional
+    file = "paper/method.tex"          # optional
+
+    [[quotes]]                         # a quote as read in its source
+    key = "doe2001theory"              # BibTeX key
+    text = "The full sentence, verbatim, of which the paper may quote a part."
+    version = "publisher"              # publisher|author-pdf|preprint|reprint|abstract|secondary
+    url = "https://doi.org/10.1000/xyz"
+    page = "107"                       # omitted for an abstract
+    date = "2026-10-17"                # read on
+    status = "verified"                # verified|unverified
+    claim = "what the citation supports"   # optional
+
+`[results]` is optional: a paper without a study audits its quotes and references only.
+"""
+
+from __future__ import annotations
+
+import re
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from ._number import Num, parse
+
+
+class ConfigError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class Entry:
+    """A `[[numbers.derive]]` or `[[numbers.allow]]` entry; `line` is its header in audit.toml."""
+
+    kind: str
+    num: Num
+    line: int
+    file: str | None
+    expr: str | None = None
+    reason: str | None = None
+    cite: str | None = None
+
+    def covers(self, n: Num) -> bool:
+        return n.key == self.num.key and (self.file is None or self.file == n.file)
+
+
+@dataclass(frozen=True)
+class Manual:
+    """A `[[results.manual]]` entry: a results file made by a person."""
+
+    file: str
+    by: str
+    date: str
+    reason: str
+    line: int
+
+
+@dataclass(frozen=True)
+class Depends:
+    """A `[[results.depends]]` entry: the instruments one results file depends on."""
+
+    file: str
+    instruments: tuple[Path, ...]
+    line: int
+
+
+VERSIONS = ("publisher", "author-pdf", "preprint", "reprint", "abstract", "secondary")
+STATUSES = ("verified", "unverified")
+
+
+@dataclass(frozen=True)
+class Quote:
+    """A `[[quotes]]` entry: a quote as read in a source."""
+
+    key: str
+    text: str
+    version: str
+    url: str
+    page: str | None
+    date: str
+    status: str
+    claim: str | None
+    line: int
+
+
+@dataclass(frozen=True)
+class Config:
+    root: Path
+    path: Path
+    tex: tuple[Path, ...]
+    pdf: Path | None
+    results: tuple[Path, ...]
+    entries: tuple[Entry, ...]
+    instruments: tuple[Path, ...] = ()
+    manual: tuple[Manual, ...] = ()
+    depends: tuple[Depends, ...] = ()
+    quotes: tuple[Quote, ...] = ()
+
+    def rel(self, p: Path) -> str:
+        try:
+            return str(p.relative_to(self.root))
+        except ValueError:
+            return str(p)
+
+    def instruments_of(self, results_file: str) -> tuple[Path, ...]:
+        """The instruments `results_file` depends on: its `[[results.depends]]`, or all."""
+        for d in self.depends:
+            if d.file == results_file:
+                return d.instruments
+        return self.instruments
+
+
+def _glob(root: Path, patterns: Any, what: str) -> tuple[Path, ...]:
+    if isinstance(patterns, str):
+        patterns = [patterns]
+    if not isinstance(patterns, list) or not all(isinstance(p, str) for p in patterns):
+        raise ConfigError(f"{what} must be a string or a list of strings")
+    out: list[Path] = []
+    for p in patterns:
+        hits = sorted(root.glob(p))
+        if not hits:
+            raise ConfigError(f"{what}: {p!r} matches no file")
+        out += hits
+    return tuple(out)
+
+
+def _header_lines(text: str, table: str) -> list[int]:
+    pat = re.compile(rf"^[ \t]*\[\[[ \t]*{re.escape(table)}[ \t]*\]\]", re.MULTILINE)
+    return [text.count("\n", 0, m.start()) + 1 for m in pat.finditer(text)]
+
+
+def _entries(text: str, numbers: dict[str, Any], kind: str) -> list[Entry]:
+    raw = numbers.get(kind, [])
+    if not isinstance(raw, list):
+        raise ConfigError(f"numbers.{kind} must be an array of tables ([[numbers.{kind}]])")
+    lines = _header_lines(text, f"numbers.{kind}")
+    out = []
+    for i, e in enumerate(raw):
+        line = lines[i] if i < len(lines) else 0
+        where = f"audit.toml:{line}: numbers.{kind}"
+        value = e.get("value")
+        num = parse(value) if isinstance(value, str) else None
+        if num is None:
+            raise ConfigError(f"{where}: value must be a string, the number as printed")
+        if kind == "derive" and not isinstance(e.get("expr"), str):
+            raise ConfigError(f"{where}: expr is mandatory")
+        if kind == "allow" and not str(e.get("reason", "")).strip():
+            raise ConfigError(f"{where}: reason is mandatory")
+        out.append(Entry(kind, num, line, e.get("file"), e.get("expr"), e.get("reason"),
+                         e.get("cite")))
+    return out
+
+
+def load(root: Path, path: Path | None = None) -> Config:
+    path = path or root / "audit.toml"
+    if not path.is_file():
+        raise ConfigError(f"{path}: no audit.toml (see CONVENTIONS.md)")
+    text = path.read_text()
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError as e:
+        raise ConfigError(f"{path}: {e}") from e
+    paper = data.get("paper", {})
+    tex = _glob(root, paper["tex"], "paper.tex") if "tex" in paper else ()
+    pdf = None
+    if not tex:
+        if "pdf" not in paper:
+            raise ConfigError(f"{path}: [paper] declares neither tex nor pdf")
+        pdf = root / paper["pdf"]
+        if not pdf.is_file():
+            raise ConfigError(f"paper.pdf: {paper['pdf']!r} is not a file")
+    res = data.get("results", {})
+    results = _glob(root, res["files"], "results.files") if "files" in res else ()
+    numbers = data.get("numbers", {})
+    entries = _entries(text, numbers, "derive") + _entries(text, numbers, "allow")
+    instruments = res.get("instruments", [])
+    return Config(root, path, tex, pdf, results, tuple(entries),
+                  _glob(root, instruments, "results.instruments") if instruments else (),
+                  tuple(_manual(text, res.get("manual", []))),
+                  tuple(_depends(root, text, res.get("depends", []))),
+                  tuple(_quotes(text, data.get("quotes", []))))
+
+
+def _quotes(text: str, raw: Any) -> list[Quote]:
+    if not isinstance(raw, list):
+        raise ConfigError("quotes must be an array of tables ([[quotes]])")
+    lines = _header_lines(text, "quotes")
+    out = []
+    for i, e in enumerate(raw):
+        line = lines[i] if i < len(lines) else 0
+        where = f"audit.toml:{line}: quotes"
+        missing = [k for k in ("key", "text", "version", "url", "date", "status")
+                   if not str(e.get(k, "")).strip()]
+        if missing:
+            raise ConfigError(f"{where}: {', '.join(missing)} mandatory")
+        if e["version"] not in VERSIONS:
+            raise ConfigError(f"{where}: version must be one of {', '.join(VERSIONS)}")
+        if e["status"] not in STATUSES:
+            raise ConfigError(f"{where}: status must be one of {', '.join(STATUSES)}")
+        page = e.get("page")
+        out.append(Quote(e["key"], e["text"], e["version"], e["url"],
+                         None if page is None else str(page), str(e["date"]), e["status"],
+                         e.get("claim"), line))
+    return out
+
+
+def instruments(root: Path, output: str) -> list[str] | None:
+    """The instruments `output` depends on, per `<root>/audit.toml`, relative to root; None if
+    undeclared.
+
+    For `osaudit run`, which may create the first results file, so `load` would refuse.
+    """
+    path = root / "audit.toml"
+    if not path.is_file():
+        return None
+    text = path.read_text()
+    try:
+        results = tomllib.loads(text).get("results", {})
+    except tomllib.TOMLDecodeError as e:
+        raise ConfigError(f"{path}: {e}") from e
+    for d in _depends(root, text, results.get("depends", [])):
+        if d.file == output:
+            return [str(p.relative_to(root)) for p in d.instruments]
+    if not results.get("instruments"):
+        return None
+    return [str(p.relative_to(root))
+            for p in _glob(root, results["instruments"], "results.instruments")]
+
+
+def _depends(root: Path, text: str, raw: Any) -> list[Depends]:
+    if not isinstance(raw, list):
+        raise ConfigError("results.depends must be an array of tables ([[results.depends]])")
+    lines = _header_lines(text, "results.depends")
+    out = []
+    for i, e in enumerate(raw):
+        line = lines[i] if i < len(lines) else 0
+        if not isinstance(e.get("file"), str) or "instruments" not in e:
+            raise ConfigError(f"audit.toml:{line}: results.depends: file, instruments mandatory")
+        out.append(Depends(e["file"], _glob(root, e["instruments"], "results.depends"), line))
+    return out
+
+
+def _manual(text: str, raw: Any) -> list[Manual]:
+    if not isinstance(raw, list):
+        raise ConfigError("results.manual must be an array of tables ([[results.manual]])")
+    lines = _header_lines(text, "results.manual")
+    out = []
+    for i, e in enumerate(raw):
+        line = lines[i] if i < len(lines) else 0
+        missing = [k for k in ("file", "by", "date", "reason") if not str(e.get(k, "")).strip()]
+        if missing:
+            raise ConfigError(f"audit.toml:{line}: results.manual: {', '.join(missing)} mandatory")
+        out.append(Manual(e["file"], e["by"], str(e["date"]), e["reason"], line))
+    return out

osaudit/_findings.py

@@ -0,0 +1,23 @@
+"""A finding: one line `file:line: rule: message`, or one JSON object."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Finding:
+    file: str
+    line: int
+    rule: str
+    message: str
+    candidates: tuple[str, ...] = field(default=())
+
+    def __str__(self) -> str:
+        return f"{self.file}:{self.line}: {self.rule}: {self.message}"
+
+    def as_json(self) -> dict[str, Any]:
+        d = asdict(self)
+        d["candidates"] = list(self.candidates)
+        return d

osaudit/_git.py

@@ -0,0 +1,45 @@
+"""The few git queries the audit needs."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+class GitError(Exception):
+    pass
+
+
+def _git(root: Path, *args: str) -> str:
+    try:
+        r = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True)
+    except OSError as e:
+        raise GitError(f"git: {e}") from e
+    if r.returncode != 0:
+        raise GitError(f"git {' '.join(args)}: {r.stderr.strip()}")
+    return r.stdout
+
+
+def head(root: Path) -> str:
+    return _git(root, "rev-parse", "HEAD").strip()
+
+
+def dirty(root: Path) -> list[str]:
+    """Tracked files modified, staged or deleted, relative to `root` (untracked do not count)."""
+    return _git(root, "diff", "--name-only", "--relative", "HEAD").splitlines()
+
+
+def has_commit(root: Path, commit: str) -> bool:
+    try:
+        _git(root, "cat-file", "-e", f"{commit}^{{commit}}")
+    except GitError:
+        return False
+    return True
+
+
+def changed_since(root: Path, commit: str, paths: list[str]) -> list[str]:
+    """Of `paths`, those that differ between `commit` and the working tree."""
+    if not paths:
+        return []
+    out = _git(root, "diff", "--name-only", "--relative", commit, "--", *paths)
+    return out.splitlines()

osaudit/_number.py

@@ -0,0 +1,63 @@
+"""A number as printed in a paper, and when it matches a value."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from decimal import Decimal
+
+# A number not glued to a word, a path or a command: `123d0d04`, `3.0cm`, `RQ1` and `\\2` are not
+# numbers. The lookahead refuses a trailing `.x`/`,x`, so backtracking cannot cut `3.0cm` to `3`.
+NUMBER = r"(?<![\w.\\/])(\d{1,3}(?:,\d{3})+|\d+)(\.\d+)?(?![.,]?\w)"
+# A number that names a part of a document, not a quantity: "Section 4.3", "Table~1", "p. 107".
+PART = re.compile(
+    r"\b(?:sections?|sec\.|tables?|tab\.|figures?|fig\.|definitions?|equations?|eq\.|appendix"
+    r"|chapters?|ch\.|pages?|pp?\.)(?:\s|~)*\d+(?:\.\d+)*(?:\s*[-–]\s*\d+)?",
+    re.IGNORECASE)
+PRINTED = re.compile(r"\s*(\d{1,3}(?:,\d{3})+|\d+)(\.\d+)?\s*(\\?%)?\s*")
+
+
+@dataclass(frozen=True)
+class Num:
+    """A printed number: `text` as in the paper, `value` without separators, d `decimals`."""
+
+    file: str
+    line: int
+    text: str
+    value: Decimal
+    decimals: int
+    percent: bool
+
+    @property
+    def key(self) -> tuple[Decimal, int, bool]:
+        """What makes two printed numbers the same number (`1,204` and `1204`)."""
+        return (self.value, self.decimals, self.percent)
+
+
+def make(file: str, line: int, text: str, integer: str, frac: str | None, percent: bool) -> Num:
+    digits = integer.replace(",", "") + (frac or "")
+    return Num(file, line, text, Decimal(digits), len(frac) - 1 if frac else 0, percent)
+
+
+def parse(text: str) -> Num | None:
+    """Parse a value written in `audit.toml` (`"1,204"`, `"80.3%"`, `"80.3\\%"`)."""
+    m = PRINTED.fullmatch(text)
+    if not m:
+        return None
+    return make("", 0, text.strip(), m.group(1), m.group(2), m.group(3) is not None)
+
+
+def close(num: Num, v: Decimal) -> bool:
+    """`num` is `v` printed with its decimals: equal within half a unit of its last digit.
+
+    A plain integer matches only an integral value: `7` is not a rounding of 7.2. A percent
+    matches v as a share (100·v), or as a value already in percent when v is not a count (a
+    JSON integer): `57.0%` is not the count 57.
+    """
+    half = Decimal(5) * Decimal(10) ** -(num.decimals + 1)
+    if num.percent:
+        in_percent = v.as_tuple().exponent != 0 and abs(num.value - v) <= half
+        return abs(num.value - 100 * v) <= half or in_percent
+    if num.decimals == 0:
+        return v == v.to_integral_value() and v == num.value
+    return abs(num.value - v) <= half

osaudit/_numbers.py

@@ -0,0 +1,122 @@
+"""`osaudit numbers`: every number of the paper is a results value, a derivation, or allowed.
+
+Rules:
+  numbers/untraced         a printed number matches no results value, derivation or allow entry
+  numbers/derive-mismatch  a derivation evaluates to another value than the one it declares
+  numbers/derive-unknown   a derivation names a results value that does not exist
+  numbers/unused           a derivation or allow entry matches no number of the paper
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Iterable, Iterator
+from decimal import Decimal
+
+from . import _pdf, _tex
+from ._config import Config
+from ._findings import Finding
+from ._number import Num, close
+from ._results import Results
+
+MAX_CANDIDATES = 3
+
+
+def paper_numbers(cfg: Config) -> Iterator[Num]:
+    for f in cfg.tex:
+        yield from _tex.numbers(cfg.rel(f), f.read_text())
+    if cfg.pdf is not None:
+        yield from _pdf.numbers(cfg.rel(cfg.pdf), _pdf.text_of(cfg.pdf))
+
+
+def _printed(v: Decimal) -> str:
+    return format(v.normalize(), "f") if v == v.to_integral_value() else format(v, ".6g")
+
+
+def _shown(n: Num, v: Decimal) -> str:
+    """`v` rounded as `n` is printed, for messages."""
+    x = 100 * v if n.percent else v
+    q = x.quantize(Decimal(10) ** -n.decimals)
+    return f"{q:,}" + ("%" if n.percent else "")
+
+
+def candidates(n: Num, values: dict[str, Decimal]) -> list[str]:
+    """Up to MAX_CANDIDATES single-operation expressions over results values that print as `n`.
+
+    Operands are counts (JSON integers): a sum or difference of counts, or a share of two counts.
+    Expressions within one results file come first, then those whose operands are closest in
+    the results tree (`survey.by_age.young.complete / survey.total`), then the shortest.
+    """
+    named = sorted((k, v) for k, v in values.items()
+                   if not k.endswith(" (key)") and v.as_tuple().exponent == 0)
+    by_value: dict[Decimal, list[str]] = {}
+    for k, v in named:
+        by_value.setdefault(v, []).append(k)
+    out: list[str] = []
+    if not n.percent and n.decimals == 0:
+        for a, va in named:
+            for b in by_value.get(va - n.value, []):
+                out.append(f"{a} - {b}")
+            for b in by_value.get(n.value - va, []):
+                if a < b:
+                    out.append(f"{a} + {b}")
+    else:
+        for a, va in named:
+            for b, vb in named:
+                if a != b and vb != 0 and close(n, va / vb):
+                    out.append(f"{a} / {b}")
+    def rank(e: str) -> tuple[int, int, int, str]:
+        a, b = (re.split(r"[.\[]", x) for x in e.split(" ", 2)[::2])
+        shared = len(os.path.commonprefix([a, b]))
+        return (a[0] != b[0], len(a) + len(b) - 2 * shared, len(e), e)
+
+    return sorted(out, key=rank)[:MAX_CANDIDATES]
+
+
+def check(cfg: Config) -> list[Finding]:
+    results = Results.load(list(cfg.results))
+    values = list(results.values.values())
+    toml = cfg.rel(cfg.path)
+    findings: list[Finding] = []
+
+    broken: set[int] = set()
+    for i, e in enumerate(cfg.entries):
+        if e.kind != "derive":
+            continue
+        assert e.expr is not None
+        try:
+            v = results.evaluate(e.expr)
+        except ValueError as err:
+            findings.append(Finding(toml, e.line, "numbers/derive-unknown", str(err)))
+            broken.add(i)
+            continue
+        if not close(e.num, v):
+            findings.append(Finding(
+                toml, e.line, "numbers/derive-mismatch",
+                f"{e.num.text} declared, but {e.expr} = {_printed(v)}, printed {_shown(e.num, v)}"))
+            broken.add(i)
+
+    # A broken entry still covers its number: the defect is reported once, on the entry.
+    used: set[int] = set()
+    for n in paper_numbers(cfg):
+        hits = [i for i, e in enumerate(cfg.entries) if e.covers(n)]
+        used.update(hits)
+        if hits or any(close(n, v) for v in values):
+            continue
+        cands = candidates(n, results.values)
+        msg = f"{n.text} matches no results value, derivation or allow entry"
+        if cands:
+            msg += "; candidate expr: " + " | ".join(cands)
+        findings.append(Finding(n.file, n.line, "numbers/untraced", msg, tuple(cands)))
+
+    for i, e in enumerate(cfg.entries):
+        if i not in used and i not in broken:
+            where = f" in {e.file}" if e.file else ""
+            findings.append(Finding(toml, e.line, "numbers/unused",
+                                    f"{e.kind} {e.num.text} matches no number of the paper{where}"))
+    return sorted(findings, key=lambda f: (f.file, f.line, f.rule, f.message))
+
+
+def render(findings: Iterable[Finding]) -> str:
+    return "".join(f"{f}\n" for f in findings)