reference-audit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reference_audit/__init__.py +8 -0
- reference_audit/alignmentcheck.py +136 -0
- reference_audit/bookcheck.py +98 -0
- reference_audit/cache/__init__.py +1 -0
- reference_audit/cache/db.py +108 -0
- reference_audit/cache/store.py +128 -0
- reference_audit/cli.py +190 -0
- reference_audit/config.py +179 -0
- reference_audit/fieldcheck.py +599 -0
- reference_audit/inputs.py +96 -0
- reference_audit/llm/__init__.py +6 -0
- reference_audit/llm/client.py +85 -0
- reference_audit/llm/prompts.py +206 -0
- reference_audit/llm/schemas.py +38 -0
- reference_audit/matching/__init__.py +6 -0
- reference_audit/matching/adjudicate.py +63 -0
- reference_audit/matching/features.py +185 -0
- reference_audit/matching/names.py +238 -0
- reference_audit/matching/pool.py +344 -0
- reference_audit/matching/sameobject.py +150 -0
- reference_audit/matching/scoring.py +75 -0
- reference_audit/matching/verdict.py +56 -0
- reference_audit/matching/webcheck.py +189 -0
- reference_audit/models.py +412 -0
- reference_audit/parsing/__init__.py +1 -0
- reference_audit/parsing/bib.py +392 -0
- reference_audit/parsing/context.py +55 -0
- reference_audit/parsing/entities.py +22 -0
- reference_audit/parsing/identifiers.py +165 -0
- reference_audit/parsing/latex.py +83 -0
- reference_audit/parsing/tex.py +159 -0
- reference_audit/pdf/__init__.py +16 -0
- reference_audit/pdf/embedded.py +117 -0
- reference_audit/pdf/grobid.py +234 -0
- reference_audit/pdf/tei.py +470 -0
- reference_audit/pipeline.py +1692 -0
- reference_audit/report.py +330 -0
- reference_audit/sources/__init__.py +6 -0
- reference_audit/sources/arxiv.py +81 -0
- reference_audit/sources/base.py +54 -0
- reference_audit/sources/clickhouse.py +575 -0
- reference_audit/sources/crossref.py +90 -0
- reference_audit/sources/dblp.py +140 -0
- reference_audit/sources/google_books.py +101 -0
- reference_audit/sources/http.py +260 -0
- reference_audit/sources/normalize.py +464 -0
- reference_audit/sources/openalex.py +93 -0
- reference_audit/sources/openlibrary.py +158 -0
- reference_audit/sources/publisher.py +186 -0
- reference_audit/sources/registry.py +117 -0
- reference_audit/sources/render.py +115 -0
- reference_audit/sources/semantic_scholar.py +68 -0
- reference_audit/sources/titlewords.py +31 -0
- reference_audit/sources/web.py +256 -0
- reference_audit/versioning.py +125 -0
- reference_audit-0.1.0.dist-info/METADATA +715 -0
- reference_audit-0.1.0.dist-info/RECORD +60 -0
- reference_audit-0.1.0.dist-info/WHEEL +4 -0
- reference_audit-0.1.0.dist-info/entry_points.txt +2 -0
- reference_audit-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""reference_audit — audit .bib/.tex references for correctness and hallucination.
|
|
2
|
+
|
|
3
|
+
The package is a plain async Python application (project governance uses the
|
|
4
|
+
`constructor-studio` CLI, installed separately; the package does not depend on it). See `architecture/` for the
|
|
5
|
+
governed design artifacts and `README.md` for the audit specification.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Citation alignment: is a reference cited for a claim its source actually makes?
|
|
2
|
+
|
|
3
|
+
Once step 1 has matched an entry to exactly one real work, the work's *identity* is settled — but it
|
|
4
|
+
may still be cited for something the work does not claim. This module pairs each citing context (the
|
|
5
|
+
sentence(s) that invoke the reference, from `parsing/tex.py`) with the cited work's ABSTRACT and
|
|
6
|
+
classifies the usage:
|
|
7
|
+
|
|
8
|
+
- ``supported`` the abstract corroborates the citing claim.
|
|
9
|
+
- ``contradicted`` the abstract asserts the opposite — the one "loud" finding.
|
|
10
|
+
- ``not_in_abstract`` the abstract is silent on the claim. An abstract is only a summary, so its
|
|
11
|
+
silence is NOT misuse — the full text may well support the claim.
|
|
12
|
+
- ``unverifiable`` no abstract was retrievable, the LLM is unavailable, or the check failed.
|
|
13
|
+
|
|
14
|
+
Per the reliability contract, a silent/absent abstract or ANY failure yields
|
|
15
|
+
``not_in_abstract``/``unverifiable`` — NEVER ``contradicted``. Each citation is judged in isolation
|
|
16
|
+
(one failure never affects the others), and the check is advisory: it never changes the step-1
|
|
17
|
+
verdict. LLM decisions are cached by ``(prompt, kind, model)``.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import asyncio
|
|
23
|
+
|
|
24
|
+
from reference_audit.cache.store import AuditCache, prompt_hash
|
|
25
|
+
from reference_audit.config import AuditConfig
|
|
26
|
+
from reference_audit.llm.client import LLMClient, LLMError
|
|
27
|
+
from reference_audit.llm.prompts import CITATION_ALIGNMENT_SYSTEM, citation_alignment_user
|
|
28
|
+
from reference_audit.models import (
|
|
29
|
+
AlignmentFinding,
|
|
30
|
+
BibEntry,
|
|
31
|
+
CitationAlignmentResult,
|
|
32
|
+
CitationContext,
|
|
33
|
+
MatchedArtifact,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
_KIND = "citation_alignment"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def artifact_abstract(artifact: MatchedArtifact) -> str:
|
|
40
|
+
"""The fullest abstract available anywhere on the matched artifact.
|
|
41
|
+
|
|
42
|
+
Scans every record/version (and the best record) rather than trusting one representative, so an
|
|
43
|
+
abstract that rode in on any merged source is found. Empty string when none carries one — which
|
|
44
|
+
the caller reports as ``unverifiable``, never a guess.
|
|
45
|
+
"""
|
|
46
|
+
records = [*artifact.records, *artifact.versions]
|
|
47
|
+
if artifact.best_record is not None:
|
|
48
|
+
records.append(artifact.best_record)
|
|
49
|
+
return max(((r.abstract or "").strip() for r in records), key=len, default="")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _unverifiable(ctx: CitationContext, detail: str) -> AlignmentFinding:
|
|
53
|
+
return AlignmentFinding(
|
|
54
|
+
key=ctx.key, context_text=ctx.text, ordinal=ctx.ordinal, status="unverifiable", detail=detail
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _apply(ctx: CitationContext, result: CitationAlignmentResult) -> AlignmentFinding:
|
|
59
|
+
return AlignmentFinding(
|
|
60
|
+
key=ctx.key,
|
|
61
|
+
context_text=ctx.text,
|
|
62
|
+
ordinal=ctx.ordinal,
|
|
63
|
+
status=result.classification,
|
|
64
|
+
evidence_quote=result.evidence_quote,
|
|
65
|
+
detail=result.reason,
|
|
66
|
+
confidence=result.confidence,
|
|
67
|
+
via_llm=True,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
async def _classify_one(
|
|
72
|
+
entry: BibEntry,
|
|
73
|
+
ctx: CitationContext,
|
|
74
|
+
abstract: str,
|
|
75
|
+
llm: LLMClient | None,
|
|
76
|
+
cache: AuditCache | None,
|
|
77
|
+
) -> AlignmentFinding:
|
|
78
|
+
"""Classify one citing context. Never raises — any failure becomes ``unverifiable`` so a single
|
|
79
|
+
citation's problem cannot abort the entry or the other citations (per-citation isolation)."""
|
|
80
|
+
if not abstract:
|
|
81
|
+
return _unverifiable(ctx, "no abstract available for the cited work; alignment not checked")
|
|
82
|
+
if llm is None:
|
|
83
|
+
return _unverifiable(ctx, "LLM unavailable; citation alignment not checked")
|
|
84
|
+
user = citation_alignment_user(entry, ctx.text, abstract)
|
|
85
|
+
p_hash = prompt_hash(CITATION_ALIGNMENT_SYSTEM + "\n" + user)
|
|
86
|
+
if cache is not None:
|
|
87
|
+
cached = cache.get_llm_decision(p_hash, _KIND)
|
|
88
|
+
if cached is not None:
|
|
89
|
+
return _apply(ctx, CitationAlignmentResult.model_validate_json(cached))
|
|
90
|
+
try:
|
|
91
|
+
result = await llm.structured(
|
|
92
|
+
CITATION_ALIGNMENT_SYSTEM, user, CitationAlignmentResult, _KIND
|
|
93
|
+
)
|
|
94
|
+
except asyncio.CancelledError:
|
|
95
|
+
raise
|
|
96
|
+
except LLMError as exc:
|
|
97
|
+
return _unverifiable(ctx, f"citation alignment check failed ({exc})")
|
|
98
|
+
except Exception as exc: # noqa: BLE001 — advisory; isolate this citation, never abort the entry
|
|
99
|
+
return _unverifiable(ctx, f"citation alignment check errored ({exc})")
|
|
100
|
+
if cache is not None:
|
|
101
|
+
cache.put_llm_decision(p_hash, _KIND, result.model_dump_json())
|
|
102
|
+
return _apply(ctx, result)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
async def resolve_alignment_findings(
|
|
106
|
+
entry: BibEntry,
|
|
107
|
+
artifact: MatchedArtifact,
|
|
108
|
+
contexts: list[CitationContext],
|
|
109
|
+
llm: LLMClient | None,
|
|
110
|
+
config: AuditConfig, # noqa: ARG001 — symmetry with resolve_field_findings; reserved
|
|
111
|
+
cache: AuditCache | None,
|
|
112
|
+
) -> list[AlignmentFinding]:
|
|
113
|
+
"""Alignment findings for every citing context of one matched entry (concurrent, isolated)."""
|
|
114
|
+
if not contexts:
|
|
115
|
+
return []
|
|
116
|
+
abstract = artifact_abstract(artifact)
|
|
117
|
+
return list(
|
|
118
|
+
await asyncio.gather(*(_classify_one(entry, c, abstract, llm, cache) for c in contexts))
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def alignment_note(f: AlignmentFinding) -> str:
|
|
123
|
+
"""One-line human summary for the text report (only actionable findings are surfaced)."""
|
|
124
|
+
where = f" [cite #{f.ordinal + 1}]" if f.ordinal else ""
|
|
125
|
+
if f.status == "contradicted":
|
|
126
|
+
quote = f" — abstract says: “{f.evidence_quote}”" if f.evidence_quote else ""
|
|
127
|
+
return (
|
|
128
|
+
f"citation{where} may misrepresent the source: the cited abstract appears to contradict "
|
|
129
|
+
f"“{f.context_text}” — {f.detail}{quote}"
|
|
130
|
+
)
|
|
131
|
+
if f.status == "not_in_abstract":
|
|
132
|
+
return (
|
|
133
|
+
f"citation{where} not confirmed by the abstract (the abstract is silent on "
|
|
134
|
+
f"“{f.context_text}”; full text may support it) — {f.detail}"
|
|
135
|
+
)
|
|
136
|
+
return f"citation{where} alignment could not be verified — {f.detail}"
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Edition-aware book verification, grounded in Open Library editions.
|
|
2
|
+
|
|
3
|
+
A .bib book entry usually cites a *specific edition* — a (year, publisher) pair, sometimes an ISBN.
|
|
4
|
+
The generic step-1 matcher pools whatever records the sources return, and for books that often
|
|
5
|
+
includes a *newer* edition pulled from Crossref (e.g. a 2018 Routledge reprint of a 1976 book). The
|
|
6
|
+
generic field check then compares the cited original edition's year/publisher against that newer
|
|
7
|
+
record and wrongly flags them as mistakes.
|
|
8
|
+
|
|
9
|
+
This module fixes that by working off the actual Open Library editions of the book:
|
|
10
|
+
|
|
11
|
+
1. `match_cited_edition` finds the edition the .bib is citing (by ISBN, else year+publisher, else
|
|
12
|
+
year). When found, the cited edition "checks out" and its own year/publisher are the canonical
|
|
13
|
+
values to verify against — so the original edition is no longer flagged against a reprint. When
|
|
14
|
+
*not* found (Open Library doesn't have the book, or not that edition), the caller reports the
|
|
15
|
+
gap to the user rather than silently passing — see the reliability contract.
|
|
16
|
+
2. `latest_edition` / `better_edition_note` then point at the most recent edition as a possible
|
|
17
|
+
better version to cite (step 2).
|
|
18
|
+
|
|
19
|
+
Pure and network-free: the editions are fetched by the Open Library adapter and passed in.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import re
|
|
25
|
+
|
|
26
|
+
from reference_audit.models import BibEntry, SourceRecord
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _fold(s: str | None) -> str:
|
|
30
|
+
return re.sub(r"[^a-z0-9]+", " ", (s or "").lower()).strip()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _publisher_compatible(bib_publisher: str, edition_publisher: str) -> bool:
|
|
34
|
+
"""The cited publisher matches the edition's when one folded name contains the other
|
|
35
|
+
('W. A. Benjamin' ⊆ 'W. A. Benjamin, Advanced Book Program')."""
|
|
36
|
+
a, b = _fold(bib_publisher), _fold(edition_publisher)
|
|
37
|
+
return bool(a and b and (a == b or a in b or b in a))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def match_cited_edition(
|
|
41
|
+
entry: BibEntry, editions: list[SourceRecord]
|
|
42
|
+
) -> SourceRecord | None:
|
|
43
|
+
"""The Open Library edition the .bib cites, or None if no edition corresponds.
|
|
44
|
+
|
|
45
|
+
Strongest signal first: the cited ISBN, then year+publisher agreement, then year alone.
|
|
46
|
+
"""
|
|
47
|
+
if not editions:
|
|
48
|
+
return None
|
|
49
|
+
cited_isbns = entry.ids.all_isbn13()
|
|
50
|
+
if cited_isbns:
|
|
51
|
+
for e in editions:
|
|
52
|
+
if e.ids.all_isbn13() & cited_isbns:
|
|
53
|
+
return e
|
|
54
|
+
if entry.year is not None:
|
|
55
|
+
if entry.publisher:
|
|
56
|
+
for e in editions:
|
|
57
|
+
if (
|
|
58
|
+
e.year == entry.year
|
|
59
|
+
and e.publisher
|
|
60
|
+
and _publisher_compatible(entry.publisher, e.publisher)
|
|
61
|
+
):
|
|
62
|
+
return e
|
|
63
|
+
for e in editions:
|
|
64
|
+
if e.year == entry.year:
|
|
65
|
+
return e
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def latest_edition(editions: list[SourceRecord]) -> SourceRecord | None:
|
|
70
|
+
"""The most recently published edition among those carrying a year."""
|
|
71
|
+
dated = [e for e in editions if e.year is not None]
|
|
72
|
+
return max(dated, key=lambda e: e.year) if dated else None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def better_edition_note(
|
|
76
|
+
matched: SourceRecord | None, latest: SourceRecord | None
|
|
77
|
+
) -> str | None:
|
|
78
|
+
"""A step-2 upgrade notice when a later edition than the cited one exists; else None."""
|
|
79
|
+
if matched is None or latest is None or matched.year is None or latest.year is None:
|
|
80
|
+
return None
|
|
81
|
+
if latest.year <= matched.year:
|
|
82
|
+
return None
|
|
83
|
+
where = ", ".join(p for p in (str(latest.year), latest.publisher) if p)
|
|
84
|
+
isbn = f" (isbn {latest.ids.isbn13})" if latest.ids.isbn13 else ""
|
|
85
|
+
return (
|
|
86
|
+
f"citing the {matched.year} edition; a newer edition is available: {where}{isbn}"
|
|
87
|
+
" — verify whether you should cite the latest edition"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def describe_cited_edition(entry: BibEntry) -> str:
|
|
92
|
+
"""Human description of the edition the .bib cites, for the could-not-verify report."""
|
|
93
|
+
bits = [str(entry.year)] if entry.year else []
|
|
94
|
+
if entry.publisher:
|
|
95
|
+
bits.append(entry.publisher)
|
|
96
|
+
if entry.ids.isbn13:
|
|
97
|
+
bits.append(f"isbn {entry.ids.isbn13}")
|
|
98
|
+
return ", ".join(bits) or "this edition"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""SQLite memoization so repeat runs on the same .bib trigger zero DB/LLM calls."""
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""SQLite connection + schema (adapted from sciwrite-lint workspace_db `_core`).
|
|
2
|
+
|
|
3
|
+
Three cache layers (README: avoid re-running DB/LLM calls):
|
|
4
|
+
- source_query_cache : raw adapter responses, keyed by (entry_hash, source, query_kind).
|
|
5
|
+
Only successful (ok=1) results are stored; errors are never cached (so they retry), preserving
|
|
6
|
+
the error≠not-found invariant.
|
|
7
|
+
- llm_decision_cache : LLM verdicts keyed by (prompt_hash, kind, model) — model in key ⇒ a model
|
|
8
|
+
switch re-runs (added in M4).
|
|
9
|
+
- entry_verdict_cache: whole-entry fast path, keyed by (entry_hash, backend) and gated by
|
|
10
|
+
(pipeline_version, model). `backend` is the source backend that produced the verdict (`api` or
|
|
11
|
+
`clickhouse`): the two read different snapshots of the same databases, so neither may serve the
|
|
12
|
+
other's verdict.
|
|
13
|
+
- doi_resolution_cache: doi.org's verdict on a DOI (does the handle resolve?) — a world-fact, so
|
|
14
|
+
keyed by the bare DOI and independent of pipeline_version/model. Only definitive True/False are
|
|
15
|
+
stored; an outage is never cached (mirrors the never-cache-errors invariant).
|
|
16
|
+
- db_quirks : mirror of docs/db_quirks.md (README principle 2).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import sqlite3
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
SCHEMA = """
|
|
25
|
+
CREATE TABLE IF NOT EXISTS source_query_cache (
|
|
26
|
+
entry_hash TEXT NOT NULL,
|
|
27
|
+
source TEXT NOT NULL,
|
|
28
|
+
query_kind TEXT NOT NULL,
|
|
29
|
+
result_json TEXT NOT NULL,
|
|
30
|
+
fetched_at TEXT NOT NULL,
|
|
31
|
+
ok INTEGER NOT NULL DEFAULT 1,
|
|
32
|
+
PRIMARY KEY (entry_hash, source, query_kind)
|
|
33
|
+
);
|
|
34
|
+
CREATE TABLE IF NOT EXISTS llm_decision_cache (
|
|
35
|
+
prompt_hash TEXT NOT NULL,
|
|
36
|
+
kind TEXT NOT NULL,
|
|
37
|
+
model TEXT NOT NULL,
|
|
38
|
+
result_json TEXT NOT NULL,
|
|
39
|
+
created_at TEXT NOT NULL,
|
|
40
|
+
PRIMARY KEY (prompt_hash, kind, model)
|
|
41
|
+
);
|
|
42
|
+
CREATE TABLE IF NOT EXISTS entry_verdict_cache (
|
|
43
|
+
entry_hash TEXT NOT NULL,
|
|
44
|
+
backend TEXT NOT NULL DEFAULT 'api',
|
|
45
|
+
verdict_json TEXT NOT NULL,
|
|
46
|
+
pipeline_version TEXT NOT NULL,
|
|
47
|
+
model TEXT NOT NULL,
|
|
48
|
+
created_at TEXT NOT NULL,
|
|
49
|
+
PRIMARY KEY (entry_hash, backend)
|
|
50
|
+
);
|
|
51
|
+
CREATE TABLE IF NOT EXISTS doi_resolution_cache (
|
|
52
|
+
doi TEXT PRIMARY KEY,
|
|
53
|
+
resolves INTEGER NOT NULL,
|
|
54
|
+
checked_at TEXT NOT NULL
|
|
55
|
+
);
|
|
56
|
+
CREATE TABLE IF NOT EXISTS db_quirks (
|
|
57
|
+
source TEXT NOT NULL,
|
|
58
|
+
quirk TEXT NOT NULL,
|
|
59
|
+
example_entry TEXT,
|
|
60
|
+
noted_at TEXT NOT NULL
|
|
61
|
+
);
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _migrate(conn: sqlite3.Connection) -> None:
|
|
66
|
+
"""Bring a cache written by an older version up to the current schema, in place.
|
|
67
|
+
|
|
68
|
+
`entry_verdict_cache` gained a `backend` column (part of the key). Every verdict an older cache
|
|
69
|
+
holds was produced through the public APIs, so it is kept and labelled `api`.
|
|
70
|
+
"""
|
|
71
|
+
cols = {row["name"] for row in conn.execute("PRAGMA table_info(entry_verdict_cache)")}
|
|
72
|
+
if cols and "backend" not in cols:
|
|
73
|
+
conn.executescript(
|
|
74
|
+
"""
|
|
75
|
+
BEGIN;
|
|
76
|
+
ALTER TABLE entry_verdict_cache RENAME TO entry_verdict_cache_old;
|
|
77
|
+
CREATE TABLE entry_verdict_cache (
|
|
78
|
+
entry_hash TEXT NOT NULL,
|
|
79
|
+
backend TEXT NOT NULL DEFAULT 'api',
|
|
80
|
+
verdict_json TEXT NOT NULL,
|
|
81
|
+
pipeline_version TEXT NOT NULL,
|
|
82
|
+
model TEXT NOT NULL,
|
|
83
|
+
created_at TEXT NOT NULL,
|
|
84
|
+
PRIMARY KEY (entry_hash, backend)
|
|
85
|
+
);
|
|
86
|
+
INSERT INTO entry_verdict_cache
|
|
87
|
+
(entry_hash, backend, verdict_json, pipeline_version, model, created_at)
|
|
88
|
+
SELECT entry_hash, 'api', verdict_json, pipeline_version, model, created_at
|
|
89
|
+
FROM entry_verdict_cache_old;
|
|
90
|
+
DROP TABLE entry_verdict_cache_old;
|
|
91
|
+
COMMIT;
|
|
92
|
+
"""
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def connect(path: str | Path) -> sqlite3.Connection:
|
|
97
|
+
"""Open (creating dirs + schema) a WAL-mode connection."""
|
|
98
|
+
p = Path(path)
|
|
99
|
+
if p.parent and str(p.parent) not in ("", "."):
|
|
100
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
101
|
+
conn = sqlite3.connect(str(p))
|
|
102
|
+
conn.row_factory = sqlite3.Row
|
|
103
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
104
|
+
conn.execute("PRAGMA busy_timeout=5000")
|
|
105
|
+
_migrate(conn)
|
|
106
|
+
conn.executescript(SCHEMA)
|
|
107
|
+
conn.commit()
|
|
108
|
+
return conn
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Typed CRUD over the cache DB. The pipeline reads here before any network/LLM call."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime
|
|
6
|
+
import hashlib
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from reference_audit.cache import db as _db
|
|
10
|
+
from reference_audit.models import SourceQueryResult, Verdict
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _now() -> str:
|
|
14
|
+
# Fixed width (microseconds always present), so stamps order correctly as text.
|
|
15
|
+
return datetime.datetime.now(datetime.UTC).isoformat(timespec="microseconds")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def prompt_hash(text: str) -> str:
|
|
19
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AuditCache:
|
|
23
|
+
"""Backed by SQLite. `model`/`pipeline_version` gate the whole-entry verdict fast path, and
|
|
24
|
+
`backend` (the source backend, `api` or `clickhouse`) keys it.
|
|
25
|
+
|
|
26
|
+
`fresh=True` (`--fresh`) ignores every row stored before this cache was opened: reads only see
|
|
27
|
+
what was written since, and writes overwrite as usual. Nothing is deleted, so other
|
|
28
|
+
bibliographies sharing the DB keep their cached results."""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
path: str | Path,
|
|
33
|
+
*,
|
|
34
|
+
pipeline_version: str = "0.1",
|
|
35
|
+
model: str = "",
|
|
36
|
+
backend: str = "api",
|
|
37
|
+
fresh: bool = False,
|
|
38
|
+
):
|
|
39
|
+
self.conn = _db.connect(path)
|
|
40
|
+
self.pipeline_version = pipeline_version
|
|
41
|
+
self.model = model
|
|
42
|
+
self.backend = backend
|
|
43
|
+
# Rows stamped before this are invisible to reads (ISO-8601 UTC stamps compare as text).
|
|
44
|
+
self.since = _now() if fresh else ""
|
|
45
|
+
|
|
46
|
+
# --- source query cache (only successful results are stored) ---
|
|
47
|
+
def get_source_query(
|
|
48
|
+
self, entry_hash: str, source: str, query_kind: str
|
|
49
|
+
) -> SourceQueryResult | None:
|
|
50
|
+
row = self.conn.execute(
|
|
51
|
+
"SELECT result_json FROM source_query_cache "
|
|
52
|
+
"WHERE entry_hash=? AND source=? AND query_kind=? AND ok=1 AND fetched_at>=?",
|
|
53
|
+
(entry_hash, source, query_kind, self.since),
|
|
54
|
+
).fetchone()
|
|
55
|
+
return SourceQueryResult.model_validate_json(row["result_json"]) if row else None
|
|
56
|
+
|
|
57
|
+
def put_source_query(
|
|
58
|
+
self, entry_hash: str, result: SourceQueryResult, *, source: str | None = None
|
|
59
|
+
) -> None:
|
|
60
|
+
"""Store a successful result under `source` (default `result.source`). An adapter passes its
|
|
61
|
+
`cache_source`, so a local-mirror response never answers for the public API or vice versa."""
|
|
62
|
+
if result.error is not None:
|
|
63
|
+
return # never cache errors — they must retry (error ≠ not-found)
|
|
64
|
+
self.conn.execute(
|
|
65
|
+
"INSERT OR REPLACE INTO source_query_cache "
|
|
66
|
+
"(entry_hash, source, query_kind, result_json, fetched_at, ok) VALUES (?,?,?,?,?,1)",
|
|
67
|
+
(entry_hash, source or result.source, result.query_kind, result.model_dump_json(), _now()),
|
|
68
|
+
)
|
|
69
|
+
self.conn.commit()
|
|
70
|
+
|
|
71
|
+
# --- whole-entry verdict fast path ---
|
|
72
|
+
def get_entry_verdict(self, entry_hash: str) -> Verdict | None:
|
|
73
|
+
row = self.conn.execute(
|
|
74
|
+
"SELECT verdict_json FROM entry_verdict_cache "
|
|
75
|
+
"WHERE entry_hash=? AND backend=? AND pipeline_version=? AND model=? AND created_at>=?",
|
|
76
|
+
(entry_hash, self.backend, self.pipeline_version, self.model, self.since),
|
|
77
|
+
).fetchone()
|
|
78
|
+
return Verdict.model_validate_json(row["verdict_json"]) if row else None
|
|
79
|
+
|
|
80
|
+
# @cpt-dod:cpt-referenceaudit-dod-identification-caching:p1
|
|
81
|
+
def put_entry_verdict(self, entry_hash: str, verdict: Verdict) -> None:
|
|
82
|
+
self.conn.execute(
|
|
83
|
+
"INSERT OR REPLACE INTO entry_verdict_cache "
|
|
84
|
+
"(entry_hash, backend, verdict_json, pipeline_version, model, created_at) "
|
|
85
|
+
"VALUES (?,?,?,?,?,?)",
|
|
86
|
+
(
|
|
87
|
+
entry_hash, self.backend, verdict.model_dump_json(), self.pipeline_version,
|
|
88
|
+
self.model, _now(),
|
|
89
|
+
),
|
|
90
|
+
)
|
|
91
|
+
self.conn.commit()
|
|
92
|
+
|
|
93
|
+
# --- DOI resolution cache (does doi.org's handle system know this DOI?) ---
|
|
94
|
+
def get_doi_resolution(self, doi: str) -> bool | None:
|
|
95
|
+
"""Cached doi.org verdict for `doi`: True/False if known, None if never checked."""
|
|
96
|
+
row = self.conn.execute(
|
|
97
|
+
"SELECT resolves FROM doi_resolution_cache WHERE doi=? AND checked_at>=?",
|
|
98
|
+
(doi, self.since),
|
|
99
|
+
).fetchone()
|
|
100
|
+
return bool(row["resolves"]) if row else None
|
|
101
|
+
|
|
102
|
+
def put_doi_resolution(self, doi: str, resolves: bool) -> None:
|
|
103
|
+
self.conn.execute(
|
|
104
|
+
"INSERT OR REPLACE INTO doi_resolution_cache (doi, resolves, checked_at) "
|
|
105
|
+
"VALUES (?,?,?)",
|
|
106
|
+
(doi, 1 if resolves else 0, _now()),
|
|
107
|
+
)
|
|
108
|
+
self.conn.commit()
|
|
109
|
+
|
|
110
|
+
# --- LLM decision cache (used from M4) ---
|
|
111
|
+
def get_llm_decision(self, p_hash: str, kind: str) -> str | None:
|
|
112
|
+
row = self.conn.execute(
|
|
113
|
+
"SELECT result_json FROM llm_decision_cache "
|
|
114
|
+
"WHERE prompt_hash=? AND kind=? AND model=? AND created_at>=?",
|
|
115
|
+
(p_hash, kind, self.model, self.since),
|
|
116
|
+
).fetchone()
|
|
117
|
+
return row["result_json"] if row else None
|
|
118
|
+
|
|
119
|
+
def put_llm_decision(self, p_hash: str, kind: str, result_json: str) -> None:
|
|
120
|
+
self.conn.execute(
|
|
121
|
+
"INSERT OR REPLACE INTO llm_decision_cache "
|
|
122
|
+
"(prompt_hash, kind, model, result_json, created_at) VALUES (?,?,?,?,?)",
|
|
123
|
+
(p_hash, kind, self.model, result_json, _now()),
|
|
124
|
+
)
|
|
125
|
+
self.conn.commit()
|
|
126
|
+
|
|
127
|
+
def close(self) -> None:
|
|
128
|
+
self.conn.close()
|
reference_audit/cli.py
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""`reference-audit` command-line interface (Typer).
|
|
2
|
+
|
|
3
|
+
`audit` runs the async pipeline (parse + identify, cached) over either a `.tex` + `.bib` pair or a
|
|
4
|
+
single PDF. `--no-network` gives the parse-only report (for a PDF only when it embeds the BibTeX records
|
|
5
|
+
of its references — otherwise its reference list comes from an HTTP call to GROBID, so there is no
|
|
6
|
+
offline parse for it).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
|
|
16
|
+
from reference_audit.config import AuditConfig
|
|
17
|
+
from reference_audit.inputs import InvalidInputError, PdfInput, default_cache_path, resolve_input
|
|
18
|
+
from reference_audit.pdf.embedded import read_embedded_references
|
|
19
|
+
from reference_audit.pdf.grobid import GrobidError
|
|
20
|
+
from reference_audit.pipeline import (
|
|
21
|
+
EmptyBibliographyError,
|
|
22
|
+
build_embedded_parse_report,
|
|
23
|
+
build_parse_report,
|
|
24
|
+
run_audit,
|
|
25
|
+
run_pdf_audit,
|
|
26
|
+
)
|
|
27
|
+
from reference_audit.report import render_json, render_text
|
|
28
|
+
from reference_audit.sources.clickhouse import ClickHouseUnavailableError
|
|
29
|
+
|
|
30
|
+
app = typer.Typer(
|
|
31
|
+
add_completion=False,
|
|
32
|
+
help="Audit .bib/.tex or PDF references: identify artifacts, screen for hallucinations.",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@app.callback()
|
|
37
|
+
def main() -> None:
|
|
38
|
+
"""Reference auditor — identify exact artifacts and screen for hallucinated references."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@app.command()
|
|
42
|
+
def audit(
|
|
43
|
+
document: Path = typer.Argument(
|
|
44
|
+
...,
|
|
45
|
+
exists=True,
|
|
46
|
+
dir_okay=False,
|
|
47
|
+
help="Manuscript .tex (paired with a .bib), or a .pdf whose references GROBID extracts.",
|
|
48
|
+
),
|
|
49
|
+
bib: Path | None = typer.Argument(
|
|
50
|
+
None,
|
|
51
|
+
exists=True,
|
|
52
|
+
dir_okay=False,
|
|
53
|
+
help="Bibliography .bib to audit. Required with a .tex; must be omitted with a .pdf.",
|
|
54
|
+
),
|
|
55
|
+
fmt: str = typer.Option("text", "--format", "-f", help="Output format: text | json | both."),
|
|
56
|
+
no_network: bool = typer.Option(
|
|
57
|
+
False, "--no-network", help="Parse-only: identifiers + cited/uncited, no DB calls."
|
|
58
|
+
),
|
|
59
|
+
no_llm: bool = typer.Option(
|
|
60
|
+
False, "--no-llm", help="Formal-only: skip LLM adjudication (deterministic)."
|
|
61
|
+
),
|
|
62
|
+
check_citations: bool = typer.Option(
|
|
63
|
+
False, "--check-citations",
|
|
64
|
+
help="Advisory: check each citing context against the cited work's abstract (needs the LLM).",
|
|
65
|
+
),
|
|
66
|
+
fresh: bool = typer.Option(False, "--fresh", help="Ignore cached results and re-query."),
|
|
67
|
+
cache: Path | None = typer.Option(
|
|
68
|
+
None, "--cache", help="Cache DB path (default: <bib_dir>/.reference_audit/cache.db)."
|
|
69
|
+
),
|
|
70
|
+
model: str | None = typer.Option(None, "--model", help="LLM model override."),
|
|
71
|
+
backend: str | None = typer.Option(
|
|
72
|
+
None, "--backend",
|
|
73
|
+
help="Where Semantic Scholar, OpenAlex and DBLP are read from: api | clickhouse "
|
|
74
|
+
"(default: SOURCE_BACKEND, else api).",
|
|
75
|
+
),
|
|
76
|
+
grobid: str | None = typer.Option(
|
|
77
|
+
None, "--grobid", help="GROBID base URL for PDF input (default http://localhost:8070)."
|
|
78
|
+
),
|
|
79
|
+
ignore_embedded: bool = typer.Option(
|
|
80
|
+
False, "--ignore-embedded",
|
|
81
|
+
help="For a PDF that embeds the BibTeX records of its cited works (AI4X template), extract "
|
|
82
|
+
"the references with GROBID anyway.",
|
|
83
|
+
),
|
|
84
|
+
fail_on: str | None = typer.Option(
|
|
85
|
+
None, "--fail-on", help="Exit non-zero if any verdict matches: hallucinated | multiple."
|
|
86
|
+
),
|
|
87
|
+
partial_authors: str | None = typer.Option(
|
|
88
|
+
None, "--partial-authors",
|
|
89
|
+
help="A cited author list that omits some of the work's authors without 'and others': "
|
|
90
|
+
"ignore | warn | error (default: PARTIAL_AUTHORS, else warn).",
|
|
91
|
+
),
|
|
92
|
+
) -> None:
|
|
93
|
+
"""Audit a .bib with its .tex, or a PDF on its own.
|
|
94
|
+
|
|
95
|
+
With a PDF, GROBID supplies both the reference list and the in-text citations, so no .bib is
|
|
96
|
+
needed — unless the PDF embeds the BibTeX records of its cited works (the AI4X template does),
|
|
97
|
+
which are then audited instead. Either way each reference is identified and screened for
|
|
98
|
+
hallucinations.
|
|
99
|
+
"""
|
|
100
|
+
if fmt not in ("text", "json", "both"):
|
|
101
|
+
raise typer.BadParameter("format must be one of: text, json, both")
|
|
102
|
+
if backend not in (None, "api", "clickhouse"):
|
|
103
|
+
raise typer.BadParameter("--backend must be 'api' or 'clickhouse'")
|
|
104
|
+
if partial_authors not in (None, "ignore", "warn", "error"):
|
|
105
|
+
raise typer.BadParameter("--partial-authors must be 'ignore', 'warn' or 'error'")
|
|
106
|
+
if fail_on not in (None, "hallucinated", "multiple"):
|
|
107
|
+
raise typer.BadParameter("--fail-on must be 'hallucinated' or 'multiple'")
|
|
108
|
+
if fail_on and no_network:
|
|
109
|
+
# The parse-only report has no verdicts, so the gate could never trip: a CI job relying on
|
|
110
|
+
# it would always pass. Reject the combination rather than pass silently.
|
|
111
|
+
raise typer.BadParameter(
|
|
112
|
+
"--fail-on cannot be combined with --no-network: the parse-only report has no verdicts "
|
|
113
|
+
"to check"
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
source = resolve_input(document, bib)
|
|
118
|
+
except InvalidInputError as exc:
|
|
119
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
120
|
+
|
|
121
|
+
embedded_bib: str | None = None
|
|
122
|
+
if no_network and isinstance(source, PdfInput):
|
|
123
|
+
# A PDF that embeds its references' BibTeX records parses offline. Any other PDF needs
|
|
124
|
+
# GROBID, and --no-network is a contract ("this run contacts nothing") — --grobid can point
|
|
125
|
+
# at any host, so we cannot honestly weaken it to "no *remote* network". Reject it rather
|
|
126
|
+
# than quietly redefining what the flag promises.
|
|
127
|
+
embedded = None if ignore_embedded else read_embedded_references(source.pdf_path)
|
|
128
|
+
if embedded is None or embedded.bib_text is None:
|
|
129
|
+
why = (
|
|
130
|
+
"--ignore-embedded was given" if embedded is None
|
|
131
|
+
else embedded.problem or "it embeds no BibTeX records of its references"
|
|
132
|
+
)
|
|
133
|
+
raise typer.BadParameter(
|
|
134
|
+
f"--no-network cannot be combined with this PDF ({why}): its reference list and "
|
|
135
|
+
"in-text citations would be extracted by the GROBID service over HTTP, so it has no "
|
|
136
|
+
"offline parse path. Pass a .tex + .bib pair for the offline parse-only report."
|
|
137
|
+
)
|
|
138
|
+
embedded_bib = embedded.bib_text
|
|
139
|
+
|
|
140
|
+
try:
|
|
141
|
+
if no_network and embedded_bib is not None:
|
|
142
|
+
report = build_embedded_parse_report(source.pdf_path, embedded_bib)
|
|
143
|
+
elif no_network:
|
|
144
|
+
report = build_parse_report(source.tex_path, source.bib_path)
|
|
145
|
+
else:
|
|
146
|
+
updates: dict = {}
|
|
147
|
+
if model:
|
|
148
|
+
updates["model"] = model
|
|
149
|
+
if no_llm:
|
|
150
|
+
updates["use_llm"] = False
|
|
151
|
+
if check_citations:
|
|
152
|
+
updates["check_alignment"] = True
|
|
153
|
+
if grobid:
|
|
154
|
+
updates["grobid_url"] = grobid
|
|
155
|
+
if ignore_embedded:
|
|
156
|
+
updates["use_embedded_references"] = False
|
|
157
|
+
if backend:
|
|
158
|
+
updates["source_backend"] = backend
|
|
159
|
+
if partial_authors:
|
|
160
|
+
updates["partial_authors"] = partial_authors
|
|
161
|
+
config = AuditConfig().model_copy(update=updates)
|
|
162
|
+
cache_path = cache or default_cache_path(source)
|
|
163
|
+
common = {
|
|
164
|
+
"config": config,
|
|
165
|
+
"cache_path": cache_path,
|
|
166
|
+
"fresh": fresh,
|
|
167
|
+
"progress": sys.stderr.isatty(),
|
|
168
|
+
}
|
|
169
|
+
if isinstance(source, PdfInput):
|
|
170
|
+
report = run_pdf_audit(source.pdf_path, **common)
|
|
171
|
+
else:
|
|
172
|
+
report = run_audit(source.tex_path, source.bib_path, **common)
|
|
173
|
+
except (EmptyBibliographyError, GrobidError, ClickHouseUnavailableError) as exc:
|
|
174
|
+
typer.echo(f"error: {exc}", err=True)
|
|
175
|
+
raise typer.Exit(code=2) from exc
|
|
176
|
+
|
|
177
|
+
if fmt in ("json", "both"):
|
|
178
|
+
typer.echo(render_json(report))
|
|
179
|
+
if fmt in ("text", "both"):
|
|
180
|
+
typer.echo(render_text(report))
|
|
181
|
+
|
|
182
|
+
if fail_on:
|
|
183
|
+
verdicts = report.summary.get("verdicts", {})
|
|
184
|
+
trigger = {"hallucinated": "none", "multiple": "multiple"}[fail_on]
|
|
185
|
+
if verdicts.get(trigger, 0) > 0:
|
|
186
|
+
raise typer.Exit(code=1)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
if __name__ == "__main__":
|
|
190
|
+
app()
|