veracite 0.1.1__py3-none-any.whl

veracite/__init__.py

@@ -0,0 +1,17 @@
+"""VeraCite -- a bibliography health checker for LaTeX projects.
+
+Public API: parse a .bib, run the checks, render a report. See cli.main for the
+command-line entry point.
+"""
+
+from .config import VERSION, load_settings
+from .parser import parse_bib
+from .report import Finding, Report, Severity
+from .rules import run_static, syntax_pass
+from .webcheck import check_bib_text
+
+__version__ = VERSION
+__all__ = [
+    "parse_bib", "run_static", "syntax_pass", "check_bib_text",
+    "Report", "Finding", "Severity", "load_settings", "__version__",
+]

veracite/__main__.py

@@ -0,0 +1,8 @@
+"""Enable `python -m veracite`."""
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

veracite/catalog.py

@@ -0,0 +1,170 @@
+"""The rule catalog: a publisher's audit sheet, derived from the source of truth.
+
+`veracite --list-rules` prints, for every finding category VeraCite can emit, its
+default severity, its group (syntax/semantic/context), what (if anything)
+supersedes it, and a one-line description. A publisher reads this table to decide
+where their house standard disagrees, then encodes the disagreements in a
+settings file's `severity` block -- no code change needed.
+
+The catalog is *introspected*, never hand-maintained: the set of categories is
+scanned from the `category="..."` literals in the package source, and the four
+columns are joined from the existing tables (DEFAULT_SETTINGS['severity'],
+report.CATEGORY_GROUP, report.SUPERSEDES, report.CATEGORY_DOC). So it cannot drift
+from what the code actually emits -- and `tests/test_catalog.py` asserts exactly
+that, which is what stops the table from ever going stale.
+"""
+
+import json
+import os
+import re
+
+from .config import DEFAULT_SETTINGS
+from .report import (CATEGORY_DOC, CATEGORY_GROUP, finding_group,
+                     resolve_severity, SEVERITY_NAMES, SUPERSEDES)
+
+_PKG_DIR = os.path.dirname(os.path.abspath(__file__))
+_CATEGORY_RE = re.compile(r'category="([a-z_]+)"')
+_DEF_RE = re.compile(r'^(\s*)(?:async\s+)?def (\w+)\s*\(')
+
+# Categories deliberately NOT pinned in DEFAULT_SETTINGS['severity']: their checks
+# emit MORE THAN ONE severity (author_format: a note for ALL-CAPS surnames, a
+# warning for an 'and' glued to a name), and pinning a category flattens all its
+# findings to one level. Listed here -- the single source of truth, referenced by
+# config.py's comment and asserted by tests/test_catalog.py -- so a new
+# mixed-severity category is a conscious choice, not an accidental gap.
+INTENTIONALLY_UNPINNED = frozenset({"author_format"})
+
+
+def category_sources():
+    """Map each emittable category to the source locations that emit it.
+
+    A single static scan of the package: it walks every `category="..."` literal
+    and attributes it to the innermost enclosing `def`, recording (file, function,
+    line). The catalog's index is built from this, and `emitted_categories()` is
+    just its key set -- so there is ONE scanner, and the index can never disagree
+    with the category list it indexes.
+
+    The relationship is genuinely many-to-many (one rule function can emit several
+    categories; one category, e.g. 'style', is emitted by many functions), so each
+    category maps to a *list* of sources, sorted for stable output."""
+    sources = {}
+    for name in sorted(os.listdir(_PKG_DIR)):
+        if not name.endswith(".py") or name == "catalog.py":
+            continue  # catalog.py's own _CATEGORY_RE literal is not an emit site
+        path = os.path.join(_PKG_DIR, name)
+        with open(path, encoding="utf-8") as fh:
+            # Track the function enclosing each line by indentation: a `def` owns
+            # every following line more-indented than it, until a sibling/dedent.
+            stack = []  # (indent, function name)
+            for lineno, line in enumerate(fh, 1):
+                if not line.strip() or line.lstrip().startswith("#"):
+                    continue
+                indent = len(line) - len(line.lstrip())
+                m = _DEF_RE.match(line)
+                if m:
+                    while stack and stack[-1][0] >= indent:
+                        stack.pop()
+                    stack.append((indent, m.group(2)))
+                else:
+                    while stack and stack[-1][0] >= indent:
+                        stack.pop()
+                for cat in _CATEGORY_RE.findall(line):
+                    fn = stack[-1][1] if stack else "<module>"
+                    sources.setdefault(cat, set()).add((name, fn, lineno))
+    return {c: sorted(locs) for c, locs in sources.items()}
+
+
+def emitted_categories():
+    """Every category any rule or layer can emit, scanned from the package source.
+
+    This is the authoritative set: a static scan of the `category="..."` literals
+    catches categories from the online layers and multi-category rules that
+    introspecting the ENTRY_RULES/FILE_RULES registries alone would miss."""
+    return set(category_sources())
+
+
+def default_severity_label(category):
+    """The default severity a category resolves to with no user override:
+    'error'/'warning'/'note' for a pinned category, or 'mixed' for one that is
+    deliberately unpinned (its checks emit several severities; see
+    INTENTIONALLY_UNPINNED). Reads DEFAULT_SETTINGS['severity'] -- the same table
+    resolve_severity() consults -- so it matches a real run's behaviour."""
+    configured = DEFAULT_SETTINGS.get("severity", {}).get(category)
+    if configured and str(configured).lower() in SEVERITY_NAMES:
+        return str(configured).lower()
+    if category in INTENTIONALLY_UNPINNED:
+        return "mixed"
+    return None  # an unexpected gap: caller/test surfaces it
+
+
+def catalog():
+    """The full catalog as a sorted list of dicts, one per emittable category.
+
+    Each row also carries `sources`: the function(s) and file:line(s) that emit the
+    category. This makes the catalog a faithful *index* into the detection logic --
+    it cannot reproduce a check (the algorithm lives in the function body), but it
+    points at exactly the code to read to see what a check does."""
+    srcmap = category_sources()
+    rows = []
+    for cat in sorted(emitted_categories()):
+        sup = SUPERSEDES.get(cat)
+        rows.append({
+            "category": cat,
+            "default_severity": default_severity_label(cat),
+            "group": finding_group(cat),
+            "superseded_by": sup[1] if sup else None,
+            "description": CATEGORY_DOC.get(cat, ""),
+            "sources": [{"function": fn, "file": f, "line": ln}
+                        for (f, fn, ln) in srcmap.get(cat, [])],
+        })
+    return rows
+
+
+def _source_functions(row, limit=None):
+    """The distinct function names that emit a category. Many-to-many is normal
+    (e.g. 'style' is emitted by ten functions). For the table a `limit` keeps rows
+    scannable, eliding the tail as '+N more'; the JSON `sources` lists them all,
+    with file:line."""
+    seen = []
+    for s in row["sources"]:
+        if s["function"] not in seen:
+            seen.append(s["function"])
+    if limit and len(seen) > limit:
+        return ", ".join(seen[:limit]) + f" +{len(seen) - limit} more"
+    return ", ".join(seen)
+
+
+def _fmt_table(rows):
+    cols = [("category", "category"), ("default_severity", "severity"),
+            ("group", "group"), ("superseded_by", "superseded by"),
+            ("rules", "rules (in source)"), ("description", "description")]
+    def cell(r, key):
+        if key == "rules":
+            return _source_functions(r, limit=3) or "-"
+        v = r.get(key)
+        return "-" if v in (None, "") else str(v)
+    widths = {k: max(len(hdr), *(len(cell(r, k)) for r in rows)) for k, hdr in cols}
+    line = lambda vals: "  ".join(v.ljust(widths[k]) for (k, _), v in zip(cols, vals))
+    out = [line([hdr for _, hdr in cols]),
+           line(["-" * widths[k] for k, _ in cols])]
+    out += [line([cell(r, k) for k, _ in cols]) for r in rows]
+    return "\n".join(out)
+
+
+def print_catalog(as_json=False, stream=None):
+    """Print the catalog to `stream` (default stdout). The audit sheet a publisher
+    reviews against their house standard; `as_json` for machine consumption."""
+    import sys
+    stream = stream or sys.stdout
+    rows = catalog()
+    if as_json:
+        json.dump({"rules": rows}, stream, indent=2)
+        stream.write("\n")
+        return
+    stream.write(_fmt_table(rows) + "\n")
+    stream.write(
+        f"\n{len(rows)} finding categories. The 'rules' column names the function(s) "
+        "that emit each\ncategory -- read those in veracite/ for the exact logic "
+        "('--list-rules json' gives\nfull file:line). Re-rank any category in a "
+        "settings file's \"severity\" block, e.g.\n"
+        '  {"severity": {"style": "warning", "title_case": "note"}}\n')

veracite/checkpoint.py

@@ -0,0 +1,321 @@
+"""Checkpointing: persist a (possibly partial) run to the --json report as an
+append-only NDJSON log, and rebuild it on a later run -- so a large or interrupted
+job is resumable and can be completed in phases (offline -> online -> llm).
+
+The on-disk format is **NDJSON**: one self-contained JSON record per line. Most
+lines are one bibliography ENTRY, keyed by its citation key and carrying everything
+about it -- which `phases` it has, its verification `status`/`confidence`,
+`identifiers`, the matched `canonical_record`, the `sources`, and its `issues`
+(findings):
+
+    {"key": "k0", "phases": {...}, "status": "VERIFIED", "issues": [...], ...}
+    {"key": "k1", ...}
+    {"key": "<file>", "issues": [...]}      # file-level findings (duplicates, ...)
+    {"key": "<summary>", "summary": {...}}  # the integrity roll-up / offline stub
+
+Why NDJSON: a new entry is a single O(1) APPEND (no rewrite of the whole growing
+report), so checkpointing after every entry stays cheap even at 10k references, and
+a crash mid-run leaves every prior line intact (a torn final line is just skipped on
+load). Re-running an entry simply APPENDS a fresh record for that key; on load, the
+LAST record per key wins, so an updated entry supersedes its earlier state with no
+in-place edit. At the end of a clean run the file is COMPACTED -- rewritten once,
+atomically, with exactly one line per key in bibliography order.
+
+This module owns all of that format knowledge (the per-key record shape, append,
+load with last-wins, compaction) so the CLI driver and report.py stay agnostic.
+"""
+
+import json
+import os
+
+from .models import Record
+from .record import Resolution
+from .report import Finding, Severity
+
+# The phase order, weakest first. A run "requests" a set of phases; an entry is
+# (re)processed for a requested phase it does not already have.
+PHASES = ("offline", "online", "llm")
+
+# Reserved keys for the non-entry records.
+FILE_KEY = "<file>"
+SUMMARY_KEY = "<summary>"
+
+# Which phase produces a finding, by its category. Anything not listed is treated
+# as 'offline' (the static/syntax rules), the conservative default -- those
+# findings are cheap to recompute and never depend on the network. Only the
+# online/llm categories must be named, since those are the ones we must NOT throw
+# away when resuming a job that already paid for them.
+_ONLINE_CATEGORIES = {
+    "metadata_mismatch", "source_conflict", "record_unresolved", "dead_doi",
+    "retraction", "related_work", "preprint_superseded",
+    "id_resolves_wrong_record", "doi_available", "pid_missing", "pid_optional",
+    "container_granularity",
+}
+_LLM_CATEGORIES = {"llm_relevance", "wrong_paper", "llm_config"}
+
+
+def finding_phase(f):
+    """The phase that produced finding `f`. LLM and online categories are named
+    explicitly; everything else is an offline static/syntax finding."""
+    from .report import Report   # lazy: avoid an import cycle at module load
+    if f.category in _LLM_CATEGORIES or f.layer == "llm":
+        return "llm"
+    if f.category in _ONLINE_CATEGORIES or f.layer in Report._ONLINE_LAYERS:
+        return "online"
+    return "offline"
+
+
+def requested_phases(online, llm):
+    """The phases this invocation is asking to (re)compute. Offline always runs;
+    online unless --offline; llm only with --llm (which implies online)."""
+    req = {"offline"}
+    if online:
+        req.add("online")
+    if llm:
+        req.add("llm")
+    return req
+
+
+# --- per-entry record (the NDJSON line) ------------------------------------
+
+def entry_record(key, res, status, conf, phases, issues, verify=None):
+    """Build the persisted record for one bib entry: a self-contained dict with its
+    phases, verification status, identifiers, canonical record, sources and issues.
+    `res` is a Resolution (or None for an offline-only entry); `issues` is a list of
+    finding dicts (Report._finding_dict shape). This is the inverse of the loader's
+    `_resolution_from_record`, so a round trip reproduces the same report."""
+    rec = (res.record if res else None) or {}
+    return {
+        "key": key,
+        "phases": {p: (p in phases) for p in PHASES},
+        "status": status,
+        "confidence": conf,
+        "verify": verify,
+        "identifiers": {"doi": (res.doi if res else "") or None,
+                        "arxiv": (res.arxiv_id if res else "") or None,
+                        "isbn": (res.isbn if res else "") or None},
+        "sources": sorted(res.sources) if res else [],
+        "canonical_record": {k: rec.get(k) for k in
+                             ("title", "year", "journal", "volume",
+                              "number", "pages")} if rec else None,
+        "issues": issues,
+    }
+
+
+def file_record(issues):
+    """The reserved file-level record: findings not tied to one entry (duplicates,
+    brace balance, a cited key with no entry)."""
+    return {"key": FILE_KEY, "issues": issues}
+
+
+def summary_record(summary):
+    """The reserved summary record: the integrity roll-up (or offline stub), stamped
+    with the VeraCite version that produced the report so a saved/shared report is
+    traceable to the exact tool revision (checks and scoring can change between
+    versions)."""
+    from .config import VERSION
+    return {"key": SUMMARY_KEY, "veracite_version": VERSION, "summary": summary}
+
+
+def append_record(path, record):
+    """Append one NDJSON record (a single line) to the checkpoint. O(1): no rewrite
+    of the existing file. Each line is a complete JSON value terminated by '\\n', so
+    a crash after a full line leaves a loadable file and a crash mid-line is skipped
+    on load. Returns True on success; an OSError is reported, never raised."""
+    try:
+        with open(path, "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(record, ensure_ascii=False) + "\n")
+        return True
+    except OSError as ex:
+        import sys
+        print(f"\nwarning: could not write checkpoint to {path}: {ex}", file=sys.stderr)
+        return False
+
+
+def compact(path, ordered_keys):
+    """Rewrite the checkpoint with exactly one line per key, in `ordered_keys` order
+    (entry keys in bib order), followed by the <file> and <summary> records. Done
+    once at the end of a clean run so a finished report has no superseded duplicate
+    lines. Atomic (temp + os.replace), so an interruption during compaction cannot
+    corrupt the still-valid append log it replaces. Returns True on success."""
+    records, _ = _read_records(path)        # last-wins map: key -> record
+    if records is None:
+        return False
+    order = list(ordered_keys) + [FILE_KEY, SUMMARY_KEY]
+    seen = set()
+    lines = []
+    for k in order:
+        if k in records and k not in seen:
+            seen.add(k)
+            lines.append(json.dumps(records[k], ensure_ascii=False))
+    # Any record whose key was not in `ordered_keys` (shouldn't happen, but be safe)
+    # is appended at the end so nothing is silently dropped.
+    for k, rec in records.items():
+        if k not in seen:
+            lines.append(json.dumps(rec, ensure_ascii=False))
+    tmp = f"{path}.tmp.{os.getpid()}"
+    try:
+        with open(tmp, "w", encoding="utf-8") as fh:
+            fh.write("\n".join(lines) + ("\n" if lines else ""))
+        os.replace(tmp, path)
+        return True
+    except OSError as ex:
+        import sys
+        print(f"\nwarning: could not compact checkpoint {path}: {ex}", file=sys.stderr)
+        try:
+            os.remove(tmp)
+        except OSError:
+            pass
+        return False
+
+
+def _read_records(path):
+    """Read the NDJSON checkpoint into an ORDERED last-wins map {key: record}. A
+    later line for a key supersedes an earlier one (an updated entry), keeping the
+    key's first-seen position so bib order is stable. A blank or unparseable line
+    (e.g. a torn final line from a crash) is skipped. Returns (records, n_lines) or
+    (None, 0) if the file is absent/unreadable."""
+    if not path or not os.path.isfile(path):
+        return None, 0
+    records = {}
+    n = 0
+    try:
+        with open(path, encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    rec = json.loads(line)
+                except ValueError:
+                    continue            # torn / partial line -- skip it
+                if not isinstance(rec, dict) or "key" not in rec:
+                    continue
+                n += 1
+                records[rec["key"]] = rec   # last wins; insertion order preserved
+    except OSError:
+        return None, 0
+    return records, n
+
+
+class Checkpoint:
+    """A loaded prior run rebuilt from an NDJSON checkpoint. `phases_by_key[key]` is
+    the set of phases already computed for an entry; `needs(key, requested)` says
+    which requested phases are still missing. The replayed findings, resolutions,
+    statuses and verify links are exposed so the driver can reproduce prior work
+    without recomputing it."""
+
+    def __init__(self, path):
+        self.path = path
+        self.findings = []                  # replayed Finding objects (all keys)
+        self.results = {}                   # key -> Resolution
+        self.statuses = {}                  # key -> (status, confidence)
+        self.links = {}                     # key -> verify url
+        self.phases_by_key = {}             # key -> set(phases done)
+        self.summary = None                 # the saved summary record, if any
+        self._findings_by_key = {}
+        self.loaded = False
+
+    @classmethod
+    def load(cls, path):
+        """Load an NDJSON checkpoint if it exists and parses. Returns a Checkpoint
+        (with .loaded True) or None if the file is absent or holds no usable record
+        (so a stray/empty/foreign path just starts fresh)."""
+        records, _ = _read_records(path)
+        if not records:
+            return None
+        cp = cls(path)
+        cp._replay(records)
+        cp.loaded = True
+        return cp
+
+    def _replay(self, records):
+        for key, rec in records.items():
+            if key == SUMMARY_KEY:
+                self.summary = rec.get("summary")
+                continue
+            for fd in rec.get("issues", []):
+                f = _finding_from_dict(fd, key)
+                if f is not None:
+                    self.findings.append(f)
+                    self._findings_by_key.setdefault(key, []).append(f)
+            if key == FILE_KEY:
+                continue
+            self.phases_by_key[key] = {p for p, on in (rec.get("phases") or {}).items() if on}
+            self.results[key] = _resolution_from_record(rec)
+            self.statuses[key] = (rec.get("status"),
+                                  float(rec.get("confidence") or 0.0))
+            if rec.get("verify"):
+                self.links[key] = rec["verify"]
+
+    # -- phase coverage -----------------------------------------------------
+
+    def has(self, key, phase):
+        return phase in self.phases_by_key.get(key, set())
+
+    def needs(self, key, requested):
+        """The requested phases this key has NOT already computed -- the work left
+        to do for it. Empty means the saved entry already satisfies the request.
+
+        The LLM rating needs the work's abstract, which only the online layer
+        fetches and which is NOT persisted (it is an LLM input, not a result). So
+        when the llm phase must run, the online phase is run with it -- otherwise a
+        resumed --llm pass would have no abstract to rate. This is the one phase
+        coupling; offline is always independent."""
+        todo = {p for p in requested if not self.has(key, phase=p)}
+        if "llm" in todo:
+            todo.add("online")
+        return todo
+
+    def seed_findings_for(self, key, keep_phases):
+        """The saved findings for `key` that belong to a phase in `keep_phases` --
+        i.e. the prior findings to replay because this run is NOT recomputing their
+        phase. Offline findings are always recomputed live, so they are never in
+        `keep_phases` and never replayed (avoids duplicating a static finding)."""
+        return [f for f in self._findings_by_key.get(key, [])
+                if finding_phase(f) in keep_phases]
+
+    def file_findings(self):
+        """The replayed file-level (<file>) findings, so a resumed run that does not
+        recompute them (it always does, but defensively) does not lose them."""
+        return list(self._findings_by_key.get(FILE_KEY, []))
+
+
+def _finding_from_dict(fd, key):
+    """Rebuild a Finding from a persisted issue dict. The key comes from the record
+    (issues no longer carry their own key). Tolerates a missing/odd severity by
+    defaulting to a note rather than crashing on a hand-edited report."""
+    try:
+        sev = Severity[fd.get("severity", "INFO")]
+    except KeyError:
+        sev = Severity.INFO
+    return Finding(severity=sev, key=key,
+                   line=int(fd.get("line") or 0), message=fd.get("message", ""),
+                   layer=fd.get("layer", "static"), category=fd.get("category", ""),
+                   suggested=fd.get("suggested"))
+
+
+def _resolution_from_record(rec):
+    """Rebuild a Resolution from a saved entry record -- enough for the integrity
+    score (ids, sources) and the report to regenerate identically. Abstracts are not
+    persisted (they are an LLM input, not a result), so a resumed LLM phase re-fetches
+    the abstract via the online layer."""
+    ids = rec.get("identifiers") or {}
+    res = Resolution()
+    res.doi = ids.get("doi") or ""
+    res.arxiv_id = ids.get("arxiv") or ""
+    res.isbn = ids.get("isbn") or ""
+    crec = rec.get("canonical_record")
+    if crec:
+        res.record = Record(title=crec.get("title") or "", year=crec.get("year"),
+                            journal=crec.get("journal") or "",
+                            volume=crec.get("volume") or "",
+                            number=crec.get("number") or "",
+                            pages=crec.get("pages") or "")
+        res.source = (rec.get("sources") or [""])[0]
+    for s in (rec.get("sources") or []):
+        # Per-source records are not persisted individually; a placeholder keeps
+        # len(sources) correct for the confidence/cross-source logic and the
+        # `sources` list. The primary record carries the comparable fields.
+        res.sources[s] = res.record if s == res.source else {}
+    return res