reference-audit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. reference_audit/__init__.py +8 -0
  2. reference_audit/alignmentcheck.py +136 -0
  3. reference_audit/bookcheck.py +98 -0
  4. reference_audit/cache/__init__.py +1 -0
  5. reference_audit/cache/db.py +108 -0
  6. reference_audit/cache/store.py +128 -0
  7. reference_audit/cli.py +190 -0
  8. reference_audit/config.py +179 -0
  9. reference_audit/fieldcheck.py +599 -0
  10. reference_audit/inputs.py +96 -0
  11. reference_audit/llm/__init__.py +6 -0
  12. reference_audit/llm/client.py +85 -0
  13. reference_audit/llm/prompts.py +206 -0
  14. reference_audit/llm/schemas.py +38 -0
  15. reference_audit/matching/__init__.py +6 -0
  16. reference_audit/matching/adjudicate.py +63 -0
  17. reference_audit/matching/features.py +185 -0
  18. reference_audit/matching/names.py +238 -0
  19. reference_audit/matching/pool.py +344 -0
  20. reference_audit/matching/sameobject.py +150 -0
  21. reference_audit/matching/scoring.py +75 -0
  22. reference_audit/matching/verdict.py +56 -0
  23. reference_audit/matching/webcheck.py +189 -0
  24. reference_audit/models.py +412 -0
  25. reference_audit/parsing/__init__.py +1 -0
  26. reference_audit/parsing/bib.py +392 -0
  27. reference_audit/parsing/context.py +55 -0
  28. reference_audit/parsing/entities.py +22 -0
  29. reference_audit/parsing/identifiers.py +165 -0
  30. reference_audit/parsing/latex.py +83 -0
  31. reference_audit/parsing/tex.py +159 -0
  32. reference_audit/pdf/__init__.py +16 -0
  33. reference_audit/pdf/embedded.py +117 -0
  34. reference_audit/pdf/grobid.py +234 -0
  35. reference_audit/pdf/tei.py +470 -0
  36. reference_audit/pipeline.py +1692 -0
  37. reference_audit/report.py +330 -0
  38. reference_audit/sources/__init__.py +6 -0
  39. reference_audit/sources/arxiv.py +81 -0
  40. reference_audit/sources/base.py +54 -0
  41. reference_audit/sources/clickhouse.py +575 -0
  42. reference_audit/sources/crossref.py +90 -0
  43. reference_audit/sources/dblp.py +140 -0
  44. reference_audit/sources/google_books.py +101 -0
  45. reference_audit/sources/http.py +260 -0
  46. reference_audit/sources/normalize.py +464 -0
  47. reference_audit/sources/openalex.py +93 -0
  48. reference_audit/sources/openlibrary.py +158 -0
  49. reference_audit/sources/publisher.py +186 -0
  50. reference_audit/sources/registry.py +117 -0
  51. reference_audit/sources/render.py +115 -0
  52. reference_audit/sources/semantic_scholar.py +68 -0
  53. reference_audit/sources/titlewords.py +31 -0
  54. reference_audit/sources/web.py +256 -0
  55. reference_audit/versioning.py +125 -0
  56. reference_audit-0.1.0.dist-info/METADATA +715 -0
  57. reference_audit-0.1.0.dist-info/RECORD +60 -0
  58. reference_audit-0.1.0.dist-info/WHEEL +4 -0
  59. reference_audit-0.1.0.dist-info/entry_points.txt +2 -0
  60. reference_audit-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,1692 @@
1
+ """Audit pipeline.
2
+
3
+ M1 ships the parse-only path (`build_parse_report`, no network). Later milestones add the
4
+ async `AuditPipeline.run`: parse → generate candidates → score → adjudicate → equivalence →
5
+ verdict → report.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+ from collections.abc import Awaitable, Callable
12
+ from dataclasses import dataclass, field
13
+ from pathlib import Path
14
+
15
+ from tqdm import tqdm
16
+
17
+ from reference_audit.cache.store import AuditCache
18
+ from reference_audit.config import AuditConfig
19
+ from reference_audit.llm.client import LLMClient
20
+ from reference_audit.matching.adjudicate import adjudicate_entry
21
+ from reference_audit.matching.features import _published_doi, compute_features, title_ratio
22
+ from reference_audit.matching.names import author_overlap
23
+ from reference_audit.matching.pool import pool_candidates
24
+ from reference_audit.matching.sameobject import cluster_accepted
25
+ from reference_audit.matching.scoring import bucket
26
+ from reference_audit.matching.verdict import build_verdict
27
+ from reference_audit.matching.webcheck import check_web_reference
28
+ from reference_audit.pdf.embedded import (
29
+ ATTACHMENT_NAME,
30
+ read_embedded_references,
31
+ reference_list_text,
32
+ title_printed,
33
+ )
34
+ from reference_audit.pdf.grobid import GrobidClient, GrobidRequestError
35
+ from reference_audit.pdf.tei import TeiParseError, parse_tei
36
+ from reference_audit.models import (
37
+ AuditReport,
38
+ BibEntry,
39
+ CandidateAssessment,
40
+ EntryAudit,
41
+ EntryType,
42
+ FieldFinding,
43
+ Identifiers,
44
+ MatchedArtifact,
45
+ SourceQueryResult,
46
+ SourceRecord,
47
+ Verdict,
48
+ )
49
+ from reference_audit.bookcheck import (
50
+ better_edition_note,
51
+ describe_cited_edition,
52
+ latest_edition,
53
+ match_cited_edition,
54
+ )
55
+ from reference_audit.fieldcheck import (
56
+ check_book_edition_fields,
57
+ consulted_sources,
58
+ finding_note,
59
+ resolve_field_findings,
60
+ )
61
+ from reference_audit.alignmentcheck import alignment_note, resolve_alignment_findings
62
+ from reference_audit.parsing.bib import parse_bib, parse_bib_text
63
+ from reference_audit.parsing.tex import parse_cited_keys, parse_citation_contexts
64
+ from reference_audit.sources.base import SourceAdapter
65
+ from reference_audit.versioning import better_version_notes
66
+ from reference_audit.sources.registry import (
67
+ build_default_adapters,
68
+ route_entry,
69
+ venue_allows_no_doi,
70
+ )
71
+
72
+ _NEEDS_ISBN = {EntryType.BOOK, EntryType.INCOLLECTION}
73
+ _NEEDS_DOI = {EntryType.ARTICLE, EntryType.INPROCEEDINGS}
74
+
75
+
76
+ @dataclass
77
+ class _BookResolution:
78
+ """Outcome of consulting Open Library (the authority of record for books) for one entry.
79
+
80
+ `error` set ⇒ Open Library could not be reached (reported, never read as 'no editions').
81
+ `matched` is the cited edition (confirms identity + sources year/publisher); `latest` is the most
82
+ recent edition (the better-version target).
83
+ """
84
+
85
+ error: str | None = None
86
+ editions: list[SourceRecord] = field(default_factory=list)
87
+ matched: SourceRecord | None = None
88
+ latest: SourceRecord | None = None
89
+ # Set when the entry carried no ISBN of its own and we located the book via an ISBN backfilled
90
+ # from the cited DOI's record (a chapter-level DOI of the book). Drives the explanatory note.
91
+ backfilled_isbn: str | None = None
92
+
93
+
94
+ class EmptyBibliographyError(ValueError):
95
+ """Raised when the .bib parses to zero auditable entries — nothing to audit."""
96
+
97
+
98
+ def _parse_issues(entry: BibEntry) -> list[str]:
99
+ """Deterministic, no-network issues visible from the .bib alone."""
100
+ issues: list[str] = []
101
+ raw_doi = (entry.raw_fields.get("doi") or "").strip()
102
+ if entry.ids.doi and ("doi.org" in raw_doi.lower() or raw_doi.lower().startswith("http")):
103
+ issues.append(f"DOI normalized from URL form ('{raw_doi}' → '{entry.ids.doi}')")
104
+ if entry.entry_type in _NEEDS_ISBN and not entry.ids.isbn13:
105
+ issues.append("book has no ISBN (will attempt ISBN backfill)")
106
+ if entry.entry_type in _NEEDS_DOI and not entry.ids.doi and not entry.ids.arxiv_id:
107
+ issues.append("no DOI/arXiv id (will attempt DOI backfill)")
108
+ if entry.entry_type == EntryType.MISC and not entry.ids.any_present():
109
+ issues.append("no identifier or URL")
110
+ if not entry.title:
111
+ issues.append("missing title")
112
+ if not entry.authors:
113
+ issues.append("missing author list")
114
+ return issues
115
+
116
+
117
+ # @cpt-flow:cpt-referenceaudit-flow-parsing-build-report:p1
118
+ # @cpt-dod:cpt-referenceaudit-dod-parsing-bookkeeping:p1
119
+ def build_parse_report(tex_path: str | Path | None, bib_path: str | Path) -> AuditReport:
120
+ """Parse-only report (M1): identifiers + cited/uncited bookkeeping + deterministic issues.
121
+
122
+ `tex_path` may be None (audit a .bib without a manuscript); then nothing is 'uncited'.
123
+ """
124
+ # @cpt-begin:cpt-referenceaudit-flow-parsing-build-report:p1:inst-parse-bib
125
+ entries, twins, unparsed = parse_bib(bib_path)
126
+ if not entries:
127
+ notes = []
128
+ if twins:
129
+ notes.append(f"{len(twins)} commented-out twin(s) ignored")
130
+ if unparsed:
131
+ notes.append(
132
+ f"{len(unparsed)} unparseable entr{'y' if len(unparsed) == 1 else 'ies'}: "
133
+ + "; ".join(f"{u.key} (line {u.line}): {u.reason}" for u in unparsed[:5])
134
+ )
135
+ detail = f" ({'; '.join(notes)})" if notes else ""
136
+ raise EmptyBibliographyError(
137
+ f"No auditable bibliography entries found in {bib_path}{detail}. "
138
+ "Is it a valid .bib file, and are the .bib/.tex arguments in the right order?"
139
+ )
140
+ bib_keys = {e.key for e in entries}
141
+ unparsed_keys = {u.key for u in unparsed}
142
+ # @cpt-end:cpt-referenceaudit-flow-parsing-build-report:p1:inst-parse-bib
143
+
144
+ # @cpt-begin:cpt-referenceaudit-flow-parsing-build-report:p1:inst-parse-tex
145
+ cited_keys: set[str] = set()
146
+ nocite_star = False
147
+ missing_includes: list[str] = []
148
+ if tex_path is not None:
149
+ cited_keys, nocite_star, missing_includes = parse_cited_keys(tex_path)
150
+ # @cpt-end:cpt-referenceaudit-flow-parsing-build-report:p1:inst-parse-tex
151
+
152
+ audits: list[EntryAudit] = []
153
+ for e in entries:
154
+ # @cpt-begin:cpt-referenceaudit-flow-parsing-build-report:p1:inst-mark-cited
155
+ e.cited = nocite_star or (e.key in cited_keys)
156
+ # @cpt-end:cpt-referenceaudit-flow-parsing-build-report:p1:inst-mark-cited
157
+ # @cpt-begin:cpt-referenceaudit-flow-parsing-build-report:p1:inst-collect-issues
158
+ audits.append(EntryAudit(entry=e, verdict=None, issues=_parse_issues(e)))
159
+ # @cpt-end:cpt-referenceaudit-flow-parsing-build-report:p1:inst-collect-issues
160
+
161
+ # A cited key whose entry exists but could not be parsed is reported under `unparsed`, not as a
162
+ # dangling citation.
163
+ cited_but_missing = sorted(
164
+ k for k in cited_keys if k not in bib_keys and k not in unparsed_keys
165
+ )
166
+ uncited = [] if (tex_path is None or nocite_star) else sorted(
167
+ k for k in bib_keys if k not in cited_keys
168
+ )
169
+
170
+ by_type: dict[str, int] = {}
171
+ for e in entries:
172
+ by_type[e.entry_type.value] = by_type.get(e.entry_type.value, 0) + 1
173
+
174
+ # @cpt-begin:cpt-referenceaudit-flow-parsing-build-report:p1:inst-build-report
175
+ summary = {
176
+ "total_entries": len(entries),
177
+ "cited": sum(1 for e in entries if e.cited),
178
+ "uncited": len(uncited),
179
+ "cited_but_missing": len(cited_but_missing),
180
+ "commented_twins": len(twins),
181
+ "missing_includes": len(missing_includes),
182
+ "unparsed": len(unparsed),
183
+ "entries_with_issues": sum(1 for a in audits if a.issues),
184
+ "by_type": by_type,
185
+ }
186
+
187
+ return AuditReport(
188
+ entries=audits,
189
+ cited_but_missing=cited_but_missing,
190
+ uncited=uncited,
191
+ commented_twins=[t.key for t in twins],
192
+ missing_includes=missing_includes,
193
+ unparsed=unparsed,
194
+ summary=summary,
195
+ )
196
+ # @cpt-end:cpt-referenceaudit-flow-parsing-build-report:p1:inst-build-report
197
+
198
+
199
+ @dataclass
200
+ class PdfParse:
201
+ """One PDF's parse output: the report plus the citing contexts recovered from the same TEI.
202
+
203
+ The contexts ride along rather than being re-derived because getting them again would mean
204
+ re-uploading the PDF — the TEI that carries the reference list carries the citations too.
205
+ """
206
+
207
+ report: AuditReport
208
+ contexts: dict[str, list[CitationContext]]
209
+ contexts_available: bool
210
+
211
+
212
+ # @cpt-flow:cpt-referenceaudit-flow-pdf-input-audit-pdf:p1
213
+ async def build_pdf_parse_report(pdf_path: str | Path, *, client: GrobidClient) -> PdfParse:
214
+ """Parse a PDF into an `AuditReport` plus citing contexts, via one GROBID request.
215
+
216
+ GROBID supplies BOTH the reference list (TEI `<listBibl>/<biblStruct>` → `BibEntry`) and the
217
+ in-text citations (`<ref type="bibr">` → `CitationContext`), so neither a `.bib` nor a `.tex` is
218
+ involved. The `.bib`-specific bookkeeping is reported honestly rather than fabricated: see the
219
+ `AuditReport` field comments for what each one means for a PDF.
220
+
221
+ Raises `GrobidError` (service unreachable / failing / this PDF unparseable) or
222
+ `EmptyBibliographyError` (TEI parsed, but it contains no reference list).
223
+ """
224
+ path = Path(pdf_path)
225
+ tei_xml = await client.fulltext_tei(path)
226
+ try:
227
+ parsed = parse_tei(tei_xml)
228
+ except TeiParseError as exc:
229
+ # Surfaced as a request error: from the user's side the service returned something unusable.
230
+ raise GrobidRequestError(str(exc)) from exc
231
+
232
+ if not parsed.entries:
233
+ raise EmptyBibliographyError(
234
+ f"GROBID found no reference list in {path.name} (0 bibliography entries in the TEI). "
235
+ "Is this a full paper with a bibliography, rather than a slide deck, poster, or an "
236
+ "excerpt with its references removed?"
237
+ )
238
+
239
+ # Citedness depends on GROBID having linked in-text markers to bibliography items. When it linked
240
+ # none, we do not know which references are cited — reporting them all as 'uncited' would be a
241
+ # fabrication, so citedness is declared unavailable and the uncited list is suppressed.
242
+ linking_available = parsed.markers_seen > 0
243
+ linked = parsed.linked_keys()
244
+
245
+ audits: list[EntryAudit] = []
246
+ for entry in parsed.entries:
247
+ entry.cited = entry.key in linked
248
+ issues = _parse_issues(entry) + parsed.extraction_failures.get(entry.key, [])
249
+ audits.append(EntryAudit(entry=entry, verdict=None, issues=issues))
250
+
251
+ uncited = (
252
+ sorted(e.key for e in parsed.entries if e.key not in linked) if linking_available else []
253
+ )
254
+ by_type: dict[str, int] = {}
255
+ for e in parsed.entries:
256
+ by_type[e.entry_type.value] = by_type.get(e.entry_type.value, 0) + 1
257
+
258
+ no_anchor = sum(1 for e in parsed.entries if not e.title and not e.ids.any_present())
259
+ notes = [
260
+ f"references and in-text citations were extracted from {path.name} by GROBID; both are "
261
+ "extraction results, not authored source files",
262
+ ]
263
+ if parsed.unresolved_markers:
264
+ notes.append(
265
+ f"{parsed.unresolved_markers} in-text citation marker(s) could not be linked to a "
266
+ "reference by GROBID; they are not counted as citations"
267
+ )
268
+ if not linking_available:
269
+ notes.append(
270
+ "GROBID linked no in-text citation markers in this document, so cited/uncited "
271
+ "bookkeeping is unavailable — no reference is reported as uncited"
272
+ )
273
+ if no_anchor:
274
+ notes.append(
275
+ f"{no_anchor} reference(s) yielded neither a title nor an identifier and could not be "
276
+ "checked at all; they are listed with the raw text GROBID read"
277
+ )
278
+
279
+ summary = {
280
+ "total_entries": len(parsed.entries),
281
+ "cited": sum(1 for e in parsed.entries if e.cited),
282
+ "uncited": len(uncited),
283
+ "cited_but_missing": len(parsed.dangling_targets),
284
+ "commented_twins": 0, # inapplicable: a PDF has no commented-out entries
285
+ "missing_includes": 0, # inapplicable: a PDF has no \input targets
286
+ "entries_with_issues": sum(1 for a in audits if a.issues),
287
+ "by_type": by_type,
288
+ "citation_markers": parsed.markers_seen,
289
+ "unresolved_citation_markers": parsed.unresolved_markers,
290
+ "references_without_title_or_id": no_anchor,
291
+ }
292
+
293
+ report = AuditReport(
294
+ entries=audits,
295
+ # A TEI target points into the same document's own bibliography, so a dangling one means
296
+ # malformed TEI, not a missing reference. Normally empty; reported, never swallowed.
297
+ cited_but_missing=list(parsed.dangling_targets),
298
+ uncited=uncited,
299
+ commented_twins=[],
300
+ missing_includes=[],
301
+ summary=summary,
302
+ input_kind="pdf",
303
+ citation_linking="available" if linking_available else "unavailable",
304
+ notes=notes,
305
+ )
306
+ return PdfParse(report=report, contexts=parsed.contexts, contexts_available=linking_available)
307
+
308
+
309
+ def build_embedded_parse_report(pdf_path: str | Path, bib_text: str) -> AuditReport:
310
+ """Parse the BibTeX records a PDF embeds for its cited works (`pdf.embedded`).
311
+
312
+ The records are authored source, so the entries are built exactly as from a `.bib` — and share
313
+ its verdict cache. Every record is cited: the bibliography style exports only the entries BibTeX
314
+ printed. A record is what the author compiled, which a hand-edited `.bbl` can make differ from
315
+ what is printed, so each title is looked up in the PDF's reference list.
316
+
317
+ Raises `EmptyBibliographyError` when the records hold no readable entry.
318
+ """
319
+ path = Path(pdf_path)
320
+ entries, _twins, unparsed = parse_bib_text(bib_text)
321
+ if not entries:
322
+ detail = f" ({len(unparsed)} unparseable)" if unparsed else ""
323
+ raise EmptyBibliographyError(
324
+ f"the {ATTACHMENT_NAME} attachment of {path.name} holds no readable BibTeX entry{detail}"
325
+ )
326
+
327
+ notes = [
328
+ f"references were read from the BibTeX records of the cited works embedded in {path.name} "
329
+ f"(attachment {ATTACHMENT_NAME}), not extracted from the typeset text",
330
+ ]
331
+ printed: str | None
332
+ try:
333
+ printed = reference_list_text(path)
334
+ except Exception as exc: # noqa: BLE001 — pypdf raises many types; reported, never swallowed
335
+ printed = None
336
+ notes.append(
337
+ f"the text of {path.name} could not be extracted ({type(exc).__name__}: {exc}), so the "
338
+ "embedded records were not compared with the printed reference list"
339
+ )
340
+
341
+ audits: list[EntryAudit] = []
342
+ unprinted = 0
343
+ for e in entries:
344
+ e.cited = True
345
+ issues = _parse_issues(e)
346
+ if printed is not None and title_printed(e.title, printed) is False:
347
+ unprinted += 1
348
+ issues.append(
349
+ "the title of this embedded record was not found in the PDF's printed reference "
350
+ "list — the record checked may differ from the printed reference"
351
+ )
352
+ audits.append(EntryAudit(entry=e, verdict=None, issues=issues))
353
+ if unprinted:
354
+ notes.append(
355
+ f"{unprinted} embedded record(s) have a title that was not found in the printed "
356
+ "reference list; compare them with the PDF, or re-run with --ignore-embedded to audit "
357
+ "the references extracted from the typeset text"
358
+ )
359
+
360
+ by_type: dict[str, int] = {}
361
+ for e in entries:
362
+ by_type[e.entry_type.value] = by_type.get(e.entry_type.value, 0) + 1
363
+ summary = {
364
+ "total_entries": len(entries),
365
+ "cited": len(entries),
366
+ "uncited": 0,
367
+ "cited_but_missing": 0,
368
+ "commented_twins": 0, # inapplicable: the style exports no commented-out entries
369
+ "missing_includes": 0, # inapplicable: a PDF has no \input targets
370
+ "unparsed": len(unparsed),
371
+ "entries_with_issues": sum(1 for a in audits if a.issues),
372
+ "by_type": by_type,
373
+ "titles_not_in_printed_references": unprinted,
374
+ }
375
+ return AuditReport(
376
+ entries=audits,
377
+ unparsed=unparsed,
378
+ summary=summary,
379
+ input_kind="pdf_embedded",
380
+ notes=notes,
381
+ )
382
+
383
+
384
+ def _verdict_summary(report: AuditReport) -> dict[str, int]:
385
+ counts = {"none": 0, "exactly_one": 0, "multiple": 0, "unresolved": 0}
386
+ for a in report.entries:
387
+ counts["unresolved" if a.verdict is None else a.verdict.kind] += 1
388
+ return counts
389
+
390
+
391
+ def _shares_strong_id(a: Identifiers, b: Identifiers) -> bool:
392
+ """True if two identifier sets agree on any strong identifier (DOI / ISBN13 / arXiv)."""
393
+ return bool(
394
+ (a.doi and a.doi == b.doi)
395
+ or bool(a.all_isbn13() & b.all_isbn13())
396
+ or (a.arxiv_id and a.arxiv_id == b.arxiv_id)
397
+ )
398
+
399
+
400
+ def _lookup_ids(ids: Identifiers) -> tuple:
401
+ """The identifiers a by-id lookup is routed and queried by."""
402
+ return (ids.doi, ids.arxiv_id, ids.all_isbn13(), ids.pmid, ids.openalex, ids.google_books)
403
+
404
+
405
+ def _verdict_records(verdict: Verdict | None) -> list[SourceRecord]:
406
+ """Flatten the records behind a verdict's artifacts (used to re-derive book findings from a
407
+ cached verdict — the records carry the ISBN that originally located the book)."""
408
+ if verdict is None:
409
+ return []
410
+ return [r for a in verdict.artifacts for r in a.records]
411
+
412
+
413
+ def _is_url_only_web(entry: BibEntry) -> bool:
414
+ """A web @misc identified ONLY by a URL (no DOI/arXiv/ISBN) — the `web.py` check's domain."""
415
+ return bool(
416
+ entry.entry_type == EntryType.MISC
417
+ and entry.ids.url
418
+ and not (entry.ids.doi or entry.ids.arxiv_id or entry.ids.isbn13)
419
+ )
420
+
421
+
422
+ def _web_fetch_predates_render(cached: SourceQueryResult) -> bool:
423
+ """True for a web fetch cached *before* SPA rendering existed. The source-query cache is not
424
+ versioned by pipeline_version, so an old cached shell would otherwise be served and skip the new
425
+ render path. Every live (non-dead) web record now carries ``raw['render']``; its absence marks a
426
+ pre-feature fetch that must be re-fetched once so an SPA shell actually gets rendered."""
427
+ for rec in cached.records:
428
+ raw = rec.raw or {}
429
+ if not raw.get("dead") and "render" not in raw:
430
+ return True
431
+ return False
432
+
433
+
434
+ class AuditPipeline:
435
+ """Async audit pipeline.
436
+
437
+ M3: parse → route → (id-lookup + metadata-search, concurrent & cached) → pool candidates by
438
+ identifier/version-edge → score (full feature vector) → 3-way verdict; backfills missing
439
+ DOIs/ISBNs. The LLM funnel over the *adjudicate* bucket (M4) and the full SAME-OBJECT rule +
440
+ LLM tie-break (M5) extend `_audit_entry` / `build_verdict` later.
441
+ """
442
+
443
+ def __init__(
444
+ self,
445
+ config: AuditConfig,
446
+ *,
447
+ cache: AuditCache | None = None,
448
+ adapters: list[SourceAdapter] | None = None,
449
+ llm: LLMClient | None = None,
450
+ grobid: GrobidClient | None = None,
451
+ ):
452
+ self.config = config
453
+ self.cache = cache
454
+ self.adapters: list[SourceAdapter] = (
455
+ adapters if adapters is not None else build_default_adapters(config)
456
+ )
457
+ if llm is not None:
458
+ self.llm: LLMClient | None = llm
459
+ elif config.llm_enabled():
460
+ self.llm = LLMClient(
461
+ model=config.model,
462
+ api_key=config.openai_api_key,
463
+ base_url=config.openai_base_url,
464
+ concurrency=config.llm_concurrency,
465
+ temperature=config.llm_temperature,
466
+ )
467
+ else:
468
+ self.llm = None
469
+ # Per-key citing contexts for the alignment check, populated by `run`/`run_pdf`.
470
+ self._citation_contexts: dict[str, list[CitationContext]] = {}
471
+ # False only when a PDF's in-text citation markers could not be linked at all — then a
472
+ # missing context is a reportable extraction failure, not an expected absence.
473
+ self._contexts_available = True
474
+ # Built lazily: a .bib/.tex run must never construct a GROBID client, let alone contact it.
475
+ self._grobid_client = grobid
476
+ self._owns_grobid = grobid is None
477
+
478
+ def _grobid(self) -> GrobidClient:
479
+ if self._grobid_client is None:
480
+ self._grobid_client = GrobidClient(self.config.grobid_url)
481
+ return self._grobid_client
482
+
483
+ async def aclose(self) -> None:
484
+ for adapter in self.adapters:
485
+ await adapter.aclose()
486
+ if self.llm is not None:
487
+ await self.llm.aclose()
488
+ if self._owns_grobid and self._grobid_client is not None:
489
+ await self._grobid_client.aclose()
490
+
491
+ async def preflight(self) -> None:
492
+ """Fail the whole run up front when a configured source cannot be queried at all.
493
+
494
+ A local ClickHouse mirror that is down, or that lacks the title index the search relies on,
495
+ would otherwise turn every entry into a per-entry error (or a minutes-long scan); one named
496
+ error with the fix is the useful report. The public-API adapters have nothing to check here —
497
+ their outages are transient and per query, and are reported per entry.
498
+ """
499
+ for adapter in self.adapters:
500
+ await adapter.preflight()
501
+
502
+ async def run(
503
+ self, tex_path: str | Path | None, bib_path: str | Path, *, progress: bool = False
504
+ ) -> AuditReport:
505
+ report = build_parse_report(tex_path, bib_path)
506
+ await self.preflight()
507
+ # Citation-alignment (opt-in) needs the citing context from the .tex; extract it once, offline,
508
+ # up front. Without a manuscript there is no context, so the check simply produces nothing.
509
+ if self.config.check_alignment and tex_path is not None:
510
+ self._citation_contexts = parse_citation_contexts(tex_path)
511
+ return await self._audit_all(report, progress=progress)
512
+
513
+ async def run_pdf(self, pdf_path: str | Path, *, progress: bool = False) -> AuditReport:
514
+ """Audit a PDF: from the BibTeX records it embeds, if any, otherwise from GROBID's
515
+ extraction of its reference list and in-text citations.
516
+
517
+ A sibling of `run` rather than an extra optional parameter on it — the two inputs need
518
+ different arguments, and an `isinstance`/`None`-sniffing signature would make every existing
519
+ call site ambiguous for no gain.
520
+ """
521
+ report: AuditReport | None = None
522
+ fallback_notes: list[str] = []
523
+ if self.config.use_embedded_references:
524
+ embedded = read_embedded_references(pdf_path)
525
+ if embedded.bib_text is not None:
526
+ try:
527
+ report = build_embedded_parse_report(pdf_path, embedded.bib_text)
528
+ except EmptyBibliographyError as exc:
529
+ fallback_notes.append(
530
+ f"{exc}; the references were extracted from the typeset text by GROBID instead"
531
+ )
532
+ elif embedded.problem is not None:
533
+ fallback_notes.append(
534
+ f"{embedded.problem}; the references were extracted from the typeset text by "
535
+ "GROBID instead"
536
+ )
537
+
538
+ if report is not None:
539
+ if self.config.check_alignment:
540
+ # The records carry no citing sentences. Said once, for the document, rather than
541
+ # as an issue on every entry.
542
+ report.notes.append(
543
+ "citation alignment was not checked: the embedded records carry no citing "
544
+ "sentences; re-run with --ignore-embedded to take them from GROBID"
545
+ )
546
+ else:
547
+ parsed = await build_pdf_parse_report(pdf_path, client=self._grobid())
548
+ report = parsed.report
549
+ report.notes.extend(fallback_notes)
550
+ # Contexts come free from the same TEI we already had to fetch, so they are always kept;
551
+ # the alignment check itself still gates on `config.check_alignment`.
552
+ self._citation_contexts = parsed.contexts
553
+ self._contexts_available = parsed.contexts_available
554
+ await self.preflight()
555
+ return await self._audit_all(report, progress=progress)
556
+
557
+ async def _audit_all(self, report: AuditReport, *, progress: bool) -> AuditReport:
558
+ tasks = [asyncio.ensure_future(self._audit_entry(a)) for a in report.entries]
559
+ # Advance a bar as each entry resolves; verdicts land in-place on the audit objects, so
560
+ # completion order is irrelevant. `disable` keeps tests and library callers silent.
561
+ for task in tqdm(
562
+ asyncio.as_completed(tasks),
563
+ total=len(tasks),
564
+ desc="Auditing references",
565
+ unit="ref",
566
+ disable=not progress,
567
+ ):
568
+ await task
569
+ report.summary["verdicts"] = _verdict_summary(report)
570
+ report.summary["source_backend"] = self.config.source_backend
571
+ coverage = {
572
+ a.name: end.isoformat()
573
+ for a in self.adapters
574
+ if (end := getattr(a, "coverage_end", None)) is not None
575
+ }
576
+ if coverage:
577
+ report.summary["source_coverage"] = coverage
578
+ return report
579
+
580
+ async def _audit_entry(self, audit: EntryAudit) -> None:
581
+ # Isolation: a failure auditing one reference must never abort the run (or cancel the
582
+ # other in-flight entries). Anything unexpected leaves this entry unresolved — an error is
583
+ # not a 'not found', so it is retried on the next run rather than reported as a miss.
584
+ try:
585
+ await self._audit_entry_inner(audit)
586
+ except asyncio.CancelledError:
587
+ raise
588
+ except Exception as exc:
589
+ audit.verdict = None
590
+ audit.unresolved_reasons = [f"audit failed: {type(exc).__name__}: {exc}"]
591
+ audit.issues.append(f"audit failed (left unresolved, will retry next run): {exc}")
592
+
593
+ # @cpt-flow:cpt-referenceaudit-flow-identification-audit-entry:p1
594
+ # @cpt-dod:cpt-referenceaudit-dod-identification-identify-artifact:p1
595
+ async def _audit_entry_inner(self, audit: EntryAudit) -> None:
596
+ entry = audit.entry
597
+ # A reference with neither a title nor any identifier gives the matcher nothing to search on.
598
+ # A title-less metadata query would return an arbitrary paper, which the scorer could then
599
+ # accept as a match — inventing a verdict out of nothing. Report it as uncheckable instead.
600
+ # Near-impossible from a .bib; routine when GROBID cannot structure a printed reference.
601
+ if not entry.title and not entry.ids.any_present():
602
+ audit.verdict = None
603
+ audit.unresolved_reasons = ["no title and no identifier to search on"]
604
+ audit.issues.append(
605
+ "no title and no identifier — nothing to search on; this reference could not be "
606
+ "checked"
607
+ )
608
+ return
609
+ if self.cache is not None:
610
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-cache-lookup
611
+ cached = self.cache.get_entry_verdict(entry.content_hash)
612
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-cache-lookup
613
+ if cached is not None:
614
+ audit.verdict = cached
615
+ audit.from_cache = True
616
+ self._note_snapshot_coverage(audit, cached)
617
+ # The cited Work id's lookup is in the source-query cache, so this re-reads it.
618
+ self._check_cited_work(audit, cached, await self._cited_work_query(entry))
619
+ # Issues/field findings aren't part of the cached verdict; recompute them from the
620
+ # cached artifact records (all deterministic, and per-field LLM decisions are
621
+ # themselves cached) so a cached run reports identically to a --fresh one.
622
+ await self._enrich_artifact(audit, cached)
623
+ await self._note_backfill(audit, cached)
624
+ self._note_better_version(audit, cached)
625
+ await self._check_fields(audit, cached)
626
+ # Web @misc: re-derive the page-check issues (the fetch + any LLM decision are cached,
627
+ # so this re-runs against the caches and reproduces the same verdict it discards).
628
+ await self._resolve_web(audit, cached)
629
+ # Books: re-derive the Open Library edition findings (editions are cached, so this is
630
+ # the same single fetch a --fresh run made) and report them identically. The cached
631
+ # artifact's records carry the ISBN (incl. one backfilled from a chapter DOI), so the
632
+ # book is re-located the same way and the report can't contradict the cached verdict.
633
+ await self._report_book(
634
+ audit, await self._resolve_book(audit, _verdict_records(cached))
635
+ )
636
+ # Citation alignment: re-derive from the cached artifact's abstract + cached per-
637
+ # context LLM decisions, so a cached run reports identically to a --fresh one.
638
+ await self._check_alignment(audit, cached)
639
+ return
640
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-route
641
+ route = route_entry(entry, self.adapters)
642
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-route
643
+ if not route.id_adapters and not route.metadata_adapters:
644
+ # No scholarly source routed (the default set always routes @misc to the aggregators, so
645
+ # this is a reduced-adapter configuration). A URL-only web @misc is still verifiable
646
+ # against its own page; anything else is simply left unresolved.
647
+ verdict = await self._resolve_web(audit, None)
648
+ audit.verdict = verdict
649
+ if verdict is None:
650
+ self._record_unresolved(audit, [], [], None)
651
+ self._check_cited_work(audit, verdict, await self._cited_work_query(entry))
652
+ if self.cache is not None and verdict is not None:
653
+ self.cache.put_entry_verdict(entry.content_hash, verdict)
654
+ return
655
+
656
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-gather
657
+ records, source_errors, cited_work = await self._gather_candidates(entry, route)
658
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-gather
659
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-assess
660
+ pooled = pool_candidates(records)
661
+ # A bare URL is not a matching anchor (no feature compares it), so a conference paper cited
662
+ # only by its proceedings URL is, for scoring, as anchorless as one with no id — eligible for
663
+ # the strict title+author backfill path. This is what lets a DBLP-confirmed NeurIPS/ICLR/ICML
664
+ # paper (no DOI) reach a deterministic verdict instead of depending on the LLM.
665
+ entry_has_id = entry.ids.has_strong_id()
666
+ audit.candidates = [self._assess(entry, r, entry_has_id=entry_has_id) for r in pooled]
667
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-assess
668
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-verdict
669
+ verdict = await self._verdict(audit, errored=bool(source_errors))
670
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-verdict
671
+
672
+ # README: "unless a returned record is a 100% match, use an LLM to filter results one by
673
+ # one." A formal exactly_one IS the 100%-match short-circuit, so only invoke the per-
674
+ # candidate LLM filter when the formal rules left the entry unresolved.
675
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-llm-adjudicate
676
+ llm_errors: list[str] = []
677
+ if verdict is None and self.llm is not None:
678
+ llm_errors = await adjudicate_entry(audit, self.llm, self.config, self.cache)
679
+ verdict = await self._verdict(audit, errored=bool(source_errors or llm_errors))
680
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-llm-adjudicate
681
+
682
+ # URL-only web @misc (a blog/software page no scholarly DB indexes): fetch the cited page and
683
+ # verify it via its HTML metadata, then an LLM fallback. Runs only when the DB path found
684
+ # nothing real, so it never overrides a genuine scholarly match.
685
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-web
686
+ verdict = await self._resolve_web(audit, verdict)
687
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-web
688
+
689
+ # OpenAlex Work id: a cited openalex.org Work URL is authoritative identity (the author
690
+ # pointed us at the exact Work). When the by-id lookup returns that Work and it matches the
691
+ # entry's title+author, pin it — never let the article-centric pooler dissolve it into a
692
+ # similar-titled foreign-DOI record and backfill the wrong identifiers. A cited DOI that is
693
+ # another Work is then reported by the cited-DOI field check.
694
+ verdict = self._apply_openalex_identity(verdict, entry, cited_work)
695
+
696
+ # Google Books volume id: a cited books.google.…/books?id=… is likewise author-supplied,
697
+ # authoritative identity for a trade/book title. Pin it when the by-id lookup confirms it, so
698
+ # a same-titled journal-article (e.g. a book *review* reusing the book's title+authors, which
699
+ # carries a DOI the book lacks) can't be matched and backfill its wrong DOI onto the book.
700
+ verdict = self._apply_google_books_identity(verdict, entry, records)
701
+
702
+ # Books: Open Library is the authority of record for identity. It is edition-aware, so it can
703
+ # confirm a real book the article-centric matcher rejected (a 1976 original whose only
704
+ # DOI-bearing candidate is a 2018 reprint — a 42-year/ISBN gap that score-rejects). The
705
+ # confirmed cited edition becomes the matched artifact, settled BEFORE caching.
706
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-book
707
+ book = await self._resolve_book(audit, records)
708
+ verdict = self._apply_book_identity(verdict, book)
709
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-book
710
+
711
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-best-output
712
+ await self._note_backfill(audit, verdict)
713
+ self._note_better_version(audit, verdict)
714
+ # Enrich BEFORE caching the verdict, so the cached artifact already carries the authoritative
715
+ # by-id / publisher-of-record records (identity is settled; this only improves field sourcing).
716
+ await self._enrich_artifact(audit, verdict)
717
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-best-output
718
+ # @cpt-begin:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-cache-store
719
+ audit.verdict = verdict
720
+ if verdict is None:
721
+ self._record_unresolved(audit, source_errors, llm_errors, book)
722
+ self._note_snapshot_coverage(audit, verdict)
723
+ if self.cache is not None and verdict is not None:
724
+ self.cache.put_entry_verdict(entry.content_hash, verdict)
725
+ # @cpt-end:cpt-referenceaudit-flow-identification-audit-entry:p1:inst-cache-store
726
+ self._check_cited_work(audit, verdict, cited_work)
727
+ await self._check_fields(audit, verdict)
728
+ await self._report_book(audit, book)
729
+ await self._check_alignment(audit, verdict)
730
+
731
+ # @cpt-algo:cpt-referenceaudit-algo-identification-verdict:p1
732
+ async def _verdict(self, audit: EntryAudit, *, errored: bool):
733
+ """Cluster the accepted candidates into artifacts (SAME-OBJECT), then count them."""
734
+ # @cpt-begin:cpt-referenceaudit-algo-identification-verdict:p1:inst-cluster
735
+ accepted = [c for c in audit.candidates if c.bucket == "auto_accept"]
736
+ artifacts, cluster_errored = await cluster_accepted(
737
+ accepted, self.config, self.llm, self.cache
738
+ )
739
+ # @cpt-end:cpt-referenceaudit-algo-identification-verdict:p1:inst-cluster
740
+ # @cpt-begin:cpt-referenceaudit-algo-identification-verdict:p1:inst-build
741
+ return build_verdict(audit.candidates, artifacts, errored=errored or cluster_errored)
742
+ # @cpt-end:cpt-referenceaudit-algo-identification-verdict:p1:inst-build
743
+
744
+ async def _query(self, entry: BibEntry, adapter: SourceAdapter, kind: str) -> SourceQueryResult:
745
+ """One adapter query for `entry` (by id or by metadata), cached under the entry's hash."""
746
+ cached = (
747
+ self.cache.get_source_query(entry.content_hash, adapter.cache_source, kind)
748
+ if self.cache is not None
749
+ else None
750
+ )
751
+ if cached is not None:
752
+ return cached
753
+ result = (
754
+ await adapter.lookup_by_id(entry.ids)
755
+ if kind == "id"
756
+ else await adapter.search_by_metadata(entry)
757
+ )
758
+ if self.cache is not None:
759
+ self.cache.put_source_query(entry.content_hash, result, source=adapter.cache_source)
760
+ return result
761
+
762
+ async def _gather_candidates(
763
+ self, entry, route
764
+ ) -> tuple[list[SourceRecord], list[str], SourceQueryResult | None]:
765
+ """Query the routed adapters (cached). Returns the records, one line per failed query (naming
766
+ the source, so an unresolved entry can say which source it could not reach), and the cited
767
+ OpenAlex Work id's own lookup (`_cited_work_query`; None when the entry cites no Work id)."""
768
+ queries = [(a, "id") for a in route.id_adapters]
769
+ queries += [(a, "metadata") for a in route.metadata_adapters]
770
+ tasks = [self._query(entry, a, kind) for a, kind in queries]
771
+ # The OpenAlex by-id lookup takes the DOI, else the arXiv id, else the Work id — so an entry
772
+ # citing a Work id next to a DOI/arXiv id needs the Work looked up on its own as well.
773
+ probe_work = bool(entry.ids.openalex and (entry.ids.doi or entry.ids.arxiv_id))
774
+ if probe_work:
775
+ tasks.append(self._cited_work_query(entry))
776
+ results = await asyncio.gather(*tasks)
777
+ records: list[SourceRecord] = []
778
+ errors: list[str] = []
779
+ for res in results:
780
+ if res is None:
781
+ continue
782
+ if res.error:
783
+ errors.append(f"{res.source} {res.query_kind} query failed: {res.error}")
784
+ records.extend(res.records)
785
+ if probe_work:
786
+ cited_work = results[-1]
787
+ else:
788
+ cited_work = next(
789
+ (
790
+ res for (a, kind), res in zip(queries, results)
791
+ if entry.ids.openalex and a.name == "openalex" and kind == "id"
792
+ ),
793
+ None,
794
+ )
795
+ return records, errors, cited_work
796
+
797
+ async def _cited_work_query(self, entry: BibEntry) -> SourceQueryResult | None:
798
+ """The OpenAlex lookup of the entry's cited Work id on its own (cached), or None when the entry
799
+ cites no Work id or OpenAlex is not configured.
800
+
801
+ An entry citing only a Work id is looked up by it directly (the routed by-id query). One that
802
+ also carries a DOI/arXiv id is probed with the Work id alone, cached under that probe's own
803
+ hash, so the DOI-keyed lookup — and responses cached before this probe existed — never stand
804
+ in for the Work.
805
+ """
806
+ adapter = next((a for a in self.adapters if a.name == "openalex"), None)
807
+ if not entry.ids.openalex or adapter is None:
808
+ return None
809
+ if entry.ids.doi or entry.ids.arxiv_id:
810
+ entry = BibEntry(
811
+ key=entry.key, entry_type=entry.entry_type,
812
+ ids=Identifiers(openalex=entry.ids.openalex),
813
+ )
814
+ return await self._query(entry, adapter, "id")
815
+
816
+ def _note_snapshot_coverage(self, audit: EntryAudit, verdict: Verdict | None) -> None:
817
+ """Caveat a `none` verdict reached on local snapshots that may predate the cited work.
818
+
819
+ A snapshot holds nothing published after its date, so for an entry from that year or later a
820
+ 'no match' may be a coverage gap rather than a hallucination. The verdict stands (the other
821
+ sources were live); the report names the snapshots. Year-level, since entries carry no date.
822
+ """
823
+ if verdict is None or verdict.kind != "none" or audit.entry.year is None:
824
+ return
825
+ stale = [
826
+ (a.name, end) for a in self.adapters
827
+ if (end := getattr(a, "coverage_end", None)) is not None
828
+ and audit.entry.entry_type in a.handles
829
+ and audit.entry.year >= end.year
830
+ ]
831
+ if stale:
832
+ ends = ", ".join(f"{name} up to {end.isoformat()}" for name, end in stale)
833
+ audit.issues.append(
834
+ f"cited year {audit.entry.year} is not before the end of the local snapshot(s) "
835
+ f"({ends}); a work published later is missing from them, so this no-match may be "
836
+ "a coverage gap"
837
+ )
838
+
839
+ def _record_unresolved(
840
+ self,
841
+ audit: EntryAudit,
842
+ source_errors: list[str],
843
+ llm_errors: list[str],
844
+ book: _BookResolution | None,
845
+ ) -> None:
846
+ """Say why the entry was left without a verdict, one line per cause.
847
+
848
+ Each cause is stored on `audit.unresolved_reasons` (for machine consumers) and as an issue.
849
+ Causes the web and book checks already reported as issues are referred to, not repeated.
850
+ """
851
+ reasons = [*source_errors, *(f"LLM adjudication failed for {e}" for e in llm_errors)]
852
+ if book is not None and book.error:
853
+ reasons.append(
854
+ f"Open Library (the authority for books) could not be queried: {book.error}"
855
+ )
856
+ pending = [c for c in audit.candidates if c.bucket == "adjudicate"]
857
+ if pending and self.llm is None:
858
+ why = (
859
+ "disabled (--no-llm)" if not self.config.use_llm
860
+ else "not configured (no OPENAI_API_KEY)"
861
+ )
862
+ reasons.append(
863
+ f"{len(pending)} candidate(s) need LLM adjudication, but the LLM is {why}"
864
+ )
865
+ elif pending:
866
+ low = [c for c in pending if c.llm is not None]
867
+ if low:
868
+ best = max(low, key=lambda c: c.features.composite).record
869
+ reasons.append(
870
+ f"LLM adjudication was inconclusive (low confidence) for {len(low)} "
871
+ f"candidate(s); closest: {best.source} '{best.title}'"
872
+ )
873
+ unasked = len(pending) - len(low) - len(llm_errors)
874
+ if unasked > 0:
875
+ reasons.append(
876
+ f"{unasked} candidate(s) beyond the per-entry LLM cap "
877
+ f"(llm_max_candidates={self.config.llm_max_candidates}) were not adjudicated"
878
+ )
879
+ if not reasons and _is_url_only_web(audit.entry):
880
+ reasons.append("the cited web page could not be confirmed (see the web-check issue)")
881
+ if not reasons:
882
+ reasons.append("no identification rule reached a verdict (see the issues reported)")
883
+ audit.unresolved_reasons = reasons
884
+ audit.issues.extend(f"unresolved: {r}" for r in reasons)
885
+
886
+ async def _gather_by_id(
887
+ self, entry: BibEntry, ids: Identifiers, adapters: list[SourceAdapter]
888
+ ) -> tuple[list[SourceRecord], list[str]]:
889
+ """Query the given adapters by `ids` (cached per entry). Returns (records, error notes)."""
890
+
891
+ async def one(adapter: SourceAdapter) -> SourceQueryResult:
892
+ cached = (
893
+ self.cache.get_source_query(entry.content_hash, adapter.cache_source, "id")
894
+ if self.cache is not None
895
+ else None
896
+ )
897
+ if cached is not None:
898
+ return cached
899
+ result = await adapter.lookup_by_id(ids)
900
+ if self.cache is not None:
901
+ self.cache.put_source_query(entry.content_hash, result, source=adapter.cache_source)
902
+ return result
903
+
904
+ results = await asyncio.gather(*(one(a) for a in adapters))
905
+ records: list[SourceRecord] = []
906
+ errors: list[str] = []
907
+ for res in results:
908
+ records.extend(res.records)
909
+ if res.error:
910
+ errors.append(res.error)
911
+ return records, errors
912
+
913
+ async def _enrich_artifact(self, audit: EntryAudit, verdict) -> None:
914
+ """Step 2.5 (advisory): enrich an exactly-one artifact with authoritative by-id records — the
915
+ publisher-of-record citation export and any DOI-keyed registration records the initial
916
+ metadata search missed — before field checking.
917
+
918
+ Identity is already settled (exactly_one), so enrichment only improves *field canonicalization*
919
+ and never changes the verdict. A discovered identifier (T3 backfill) is the most authoritative
920
+ key we will ever have for the entry; here it is finally used to query, not merely reported. A
921
+ blocked authority of record is surfaced (reliability: report the gap, never guess).
922
+ """
923
+ if not self.config.check_fields:
924
+ return
925
+ if verdict is None or verdict.kind != "exactly_one" or not verdict.artifacts:
926
+ return
927
+ if audit.entry.entry_type in _NEEDS_ISBN:
928
+ # Books are identity-resolved by Open Library to a SPECIFIC edition; re-pooling by the
929
+ # work's ISBN would dissolve that edition back into the work and lose its year/publisher.
930
+ return
931
+ artifact = verdict.artifacts[0]
932
+ ids = artifact.merged_ids
933
+ if not (ids.doi or ids.isbn13):
934
+ return # no strong identifier to enrich by
935
+ probe = BibEntry(
936
+ key=audit.entry.key, entry_type=audit.entry.entry_type, title=audit.entry.title, ids=ids
937
+ )
938
+ # Authoritative by-id sources for the (now-known) identifier, plus the publisher of record —
939
+ # which lives only here, never in identity routing (a blocked publisher must not affect the
940
+ # verdict).
941
+ id_adapters = list(route_entry(probe, self.adapters).id_adapters)
942
+ publisher = next((a for a in self.adapters if a.name == "publisher"), None)
943
+ if publisher is not None and ids.doi and publisher not in id_adapters:
944
+ id_adapters.append(publisher)
945
+ if not id_adapters:
946
+ return
947
+ # Cache under the entry only when the lookup is the one identification made (same ids). With
948
+ # other ids (a backfilled or matched DOI), the entry's "id" slot would be overwritten by the
949
+ # matched work's records, and the next run's identification would read them as the entry's.
950
+ cache_key = audit.entry if _lookup_ids(ids) == _lookup_ids(audit.entry.ids) else BibEntry(
951
+ key=audit.entry.key, entry_type=audit.entry.entry_type, ids=ids
952
+ )
953
+ try:
954
+ new_records, errors = await self._gather_by_id(cache_key, ids, id_adapters)
955
+ except asyncio.CancelledError:
956
+ raise
957
+ except Exception as exc: # noqa: BLE001 — advisory; report, never fail the entry
958
+ audit.issues.append(f"metadata enrichment skipped (verdict unaffected): {exc}")
959
+ return
960
+ for err in errors:
961
+ audit.issues.append(f"authority of record not auto-retrievable (field check may be incomplete) — {err}")
962
+ if not new_records:
963
+ return
964
+ pooled = pool_candidates(list(artifact.records) + new_records)
965
+ # Identity is fixed; keep the representative(s) that still carry the matched identifier.
966
+ matched = [r for r in pooled if _shares_strong_id(r.ids, ids)] or pooled
967
+ artifact.records = matched
968
+ artifact.versions = matched
969
+ artifact.best_record = max(
970
+ matched,
971
+ key=lambda r: (0 if r.is_preprint else 1, 1 if r.ids.doi else 0, r.citation_count),
972
+ )
973
+
974
+ # @cpt-algo:cpt-referenceaudit-algo-identification-score:p1
975
+ def _assess(
976
+ self, entry: BibEntry, record: SourceRecord, *, entry_has_id: bool
977
+ ) -> CandidateAssessment:
978
+ # @cpt-begin:cpt-referenceaudit-algo-identification-score:p1:inst-features
979
+ features = compute_features(
980
+ entry, record, tail_threshold=self.config.prefix_trap_tail_jaccard
981
+ )
982
+ # @cpt-end:cpt-referenceaudit-algo-identification-score:p1:inst-features
983
+ # @cpt-begin:cpt-referenceaudit-algo-identification-score:p1:inst-bucket
984
+ return CandidateAssessment(
985
+ record=record,
986
+ features=features,
987
+ bucket=bucket(features, self.config, entry_has_id=entry_has_id),
988
+ )
989
+ # @cpt-end:cpt-referenceaudit-algo-identification-score:p1:inst-bucket
990
+
991
+ async def _resolve_web(self, audit: EntryAudit, verdict):
992
+ """Verify a URL-only @misc by fetching its cited page (HTML metadata, then an LLM fallback).
993
+
994
+ A URL-only web artifact (a blog/software page) is not a hallucination just because no
995
+ scholarly DB indexes it, so this steps in precisely when the DB path found nothing real
996
+ (verdict None or `none`). A genuine scholarly match is left untouched. Idempotent over the
997
+ caches (fetch + any LLM decision are cached), so the cached fast path re-runs it to re-derive
998
+ the same issues; see `check_web_reference` for the funnel and the verdict semantics.
999
+ """
1000
+ entry = audit.entry
1001
+ if not _is_url_only_web(entry):
1002
+ return verdict
1003
+ # A real scholarly match (or ambiguity) stands; only own the None/`none` outcome — except a
1004
+ # prior web match, which we re-derive (the cached fast path passes the cached web verdict in).
1005
+ if verdict is not None and verdict.kind in ("exactly_one", "multiple"):
1006
+ best = verdict.artifacts[0].best_record if verdict.artifacts else None
1007
+ if not (best and best.source == "web"):
1008
+ return verdict
1009
+ web = next((a for a in self.adapters if a.name == "web"), None)
1010
+ if web is None:
1011
+ audit.issues.append("URL-only web reference not checked (web adapter not configured)")
1012
+ return None
1013
+ fetched = await self._fetch_web(entry, web)
1014
+ web_verdict, issues = await check_web_reference(
1015
+ entry, fetched, self.llm, self.config, self.cache
1016
+ )
1017
+ audit.issues.extend(issues)
1018
+ return web_verdict
1019
+
1020
+ async def _fetch_web(self, entry: BibEntry, web: SourceAdapter) -> SourceQueryResult:
1021
+ """Fetch the entry's cited URL (cached per entry, like any source query)."""
1022
+ if self.cache is not None:
1023
+ cached = self.cache.get_source_query(entry.content_hash, web.name, "web")
1024
+ if cached is not None and not _web_fetch_predates_render(cached):
1025
+ return cached
1026
+ result = await web.fetch_page(entry.ids.url)
1027
+ if self.cache is not None:
1028
+ self.cache.put_source_query(entry.content_hash, result)
1029
+ return result
1030
+
1031
+ async def _note_backfill(self, audit: EntryAudit, verdict) -> None:
1032
+ """Record identifiers discovered for an entry that lacked them (T3 backfill)."""
1033
+ if verdict is None or verdict.kind != "exactly_one" or not verdict.artifacts:
1034
+ return
1035
+ best = verdict.artifacts[0].best_record
1036
+ entry = audit.entry
1037
+ if best is None:
1038
+ return
1039
+ if not entry.ids.doi and best.ids.doi:
1040
+ audit.issues.append(await self._backfilled_doi_note(best.ids.doi, best.source))
1041
+ if not entry.ids.isbn13 and best.ids.isbn13:
1042
+ audit.issues.append(f"ISBN found via {best.source}: {best.ids.isbn13}")
1043
+ if (
1044
+ not entry.ids.doi
1045
+ and not best.ids.doi
1046
+ and venue_allows_no_doi(entry.venue)
1047
+ ):
1048
+ audit.issues.append("no DOI expected for this venue (allowlisted)")
1049
+
1050
+ async def _backfilled_doi_note(self, doi: str, source: str) -> str:
1051
+ """Backfill message for a *discovered* DOI, gated on whether it actually resolves.
1052
+
1053
+ A discovered DOI is the most authoritative key we report for the entry, so it must be real.
1054
+ An unregistered one (doi.org 404 — commonly an author-supplied placeholder echoed from
1055
+ preprint metadata, the origin of this entry's `10.5555/...`) is reported as a defect, never
1056
+ presented as a clean find. With no resolver available we fall back to trusting the source
1057
+ (prior behaviour); a doi.org outage leaves the find 'unconfirmed' — reported, never asserted
1058
+ valid or invalid.
1059
+ """
1060
+ publisher = next((a for a in self.adapters if a.name == "publisher"), None)
1061
+ if publisher is None:
1062
+ return f"DOI found via {source}: {doi}"
1063
+ resolves = await self._doi_resolves(doi, publisher)
1064
+ if resolves is False:
1065
+ return (
1066
+ f"DOI found via {source} ({doi}) does NOT resolve at doi.org "
1067
+ "(404 — DOI Not Found): likely an invalid identifier (e.g. author-supplied in "
1068
+ "preprint/arXiv metadata). The work was matched, but this DOI is unusable."
1069
+ )
1070
+ if resolves is None:
1071
+ return (
1072
+ f"DOI found via {source}: {doi} "
1073
+ "(could not confirm it resolves at doi.org — verify before use)"
1074
+ )
1075
+ return f"DOI found via {source}: {doi}"
1076
+
1077
+ async def _doi_resolves(self, doi: str, publisher: SourceAdapter) -> bool | None:
1078
+ """doi.org's verdict on a DOI (cached): True=registered, False=404, None=undetermined.
1079
+
1080
+ Only definitive True/False are cached; an undetermined outcome (outage) must retry, mirroring
1081
+ the never-cache-errors invariant.
1082
+ """
1083
+ if self.cache is not None:
1084
+ cached = self.cache.get_doi_resolution(doi)
1085
+ if cached is not None:
1086
+ return cached
1087
+ try:
1088
+ resolves = await publisher.doi_registered(doi)
1089
+ except asyncio.CancelledError:
1090
+ raise
1091
+ except Exception: # noqa: BLE001 — advisory; a failed check is undetermined, never invalid
1092
+ return None
1093
+ if resolves is not None and self.cache is not None:
1094
+ self.cache.put_doi_resolution(doi, resolves)
1095
+ return resolves
1096
+
1097
+ # @cpt-dod:cpt-referenceaudit-dod-identification-best-version:p1
1098
+ def _note_better_version(self, audit: EntryAudit, verdict) -> None:
1099
+ """Step 2: report if a better version of the matched artifact is available.
1100
+
1101
+ Books are excluded here — their better-edition advice is edition-grounded in `_check_book`.
1102
+ """
1103
+ if audit.entry.entry_type in _NEEDS_ISBN:
1104
+ return
1105
+ if verdict is None or verdict.kind != "exactly_one" or not verdict.artifacts:
1106
+ return
1107
+ for note in better_version_notes(audit.entry, verdict.artifacts[0]):
1108
+ audit.issues.append(note)
1109
+
1110
+ async def _check_fields(self, audit: EntryAudit, verdict) -> None:
1111
+ """Step 3: verify each field of an exactly-one match against the canonical record.
1112
+
1113
+ Advisory only — never alters the verdict. Errors / could-not-verify findings are surfaced
1114
+ as issues; the full per-field result (including benign formatting variants) stays on
1115
+ `audit.field_findings` for machine consumers.
1116
+ """
1117
+ if not self.config.check_fields:
1118
+ return
1119
+ if verdict is None or verdict.kind != "exactly_one" or not verdict.artifacts:
1120
+ return
1121
+ # A URL-only web @misc is confirmed against the page itself (`_resolve_web`); the page's
1122
+ # self-declared og/citation metadata is not an authoritative bibliographic record, so running
1123
+ # the article field check against it only yields spurious "could not verify" noise. Skip it.
1124
+ if _is_url_only_web(audit.entry):
1125
+ return
1126
+ # Advisory step: a field-check failure must never clear the (already-decided) verdict or
1127
+ # abort the entry, so isolate it here rather than letting it reach `_audit_entry`.
1128
+ # Books delegate year/publisher to the edition-grounded `_check_book`, so skip them here
1129
+ # (comparing an original edition against a pooled reprint produced false positives).
1130
+ skip_fields = (
1131
+ frozenset({"year", "publisher"})
1132
+ if audit.entry.entry_type in _NEEDS_ISBN
1133
+ else frozenset()
1134
+ )
1135
+ try:
1136
+ findings = await resolve_field_findings(
1137
+ audit.entry, verdict.artifacts[0], self.llm, self.config, self.cache,
1138
+ skip_fields=skip_fields,
1139
+ )
1140
+ doi_finding = await self._check_cited_doi(audit.entry, verdict.artifacts[0])
1141
+ if doi_finding is not None:
1142
+ findings.append(doi_finding)
1143
+ except asyncio.CancelledError:
1144
+ raise
1145
+ except Exception as exc: # noqa: BLE001 — advisory; report, don't fail the entry
1146
+ audit.issues.append(f"field correctness check failed (verdict unaffected): {exc}")
1147
+ return
1148
+ audit.field_findings = findings
1149
+ for f in findings:
1150
+ if f.status in ("error", "uncertain"):
1151
+ audit.issues.append(finding_note(f))
1152
+ # Collapse the (often several) could-not-verify fields into a single line to keep the report
1153
+ # readable while still reporting the gap (reliability: never silently pass an unchecked field).
1154
+ # An unverifiable DOI has its own cause (no record, doi.org unreachable), not a missing field.
1155
+ for f in findings:
1156
+ if f.field == "doi" and f.status == "unverifiable":
1157
+ audit.issues.append(finding_note(f))
1158
+ unverifiable = [
1159
+ f.field for f in findings if f.status == "unverifiable" and f.field != "doi"
1160
+ ]
1161
+ if unverifiable:
1162
+ # Name the sources we actually consulted — never claim universal absence (a null field on
1163
+ # the sources we reached is not proof the datum does not exist; cf. the publisher export).
1164
+ where = ", ".join(consulted_sources(verdict.artifacts[0])) or "any source we could reach"
1165
+ audit.issues.append(
1166
+ f"could not verify field(s) {', '.join(unverifiable)} — "
1167
+ f"not present in the metadata from {where}"
1168
+ )
1169
+
1170
+ async def _check_cited_doi(
1171
+ self, entry: BibEntry, artifact: MatchedArtifact
1172
+ ) -> FieldFinding | None:
1173
+ """Is the cited DOI the matched work's? A `doi` field finding, or None if there is no DOI.
1174
+
1175
+ Identification can match a real work on title and authors while the cited DOI points
1176
+ elsewhere: to another paper, or to nothing at all. The DOI's own by-id records settle it:
1177
+ - the matched work carries the DOI → `ok`;
1178
+ - a source has a record for it that is another work → `error`, naming that work;
1179
+ - a record with the same title and authors that was not merged → `uncertain`;
1180
+ - no record, and doi.org answers 404 → `error`;
1181
+ - no record, but doi.org knows it → `uncertain` (it cannot be tied to the matched work);
1182
+ - no record and no answer from doi.org → `unverifiable`.
1183
+
1184
+ Books are skipped: they are identified by Open Library editions, which carry no DOI, and a
1185
+ book is legitimately cited by a chapter-level DOI that names the chapter, not the book.
1186
+ An arXiv DataCite DOI is a preprint id, not a competing published DOI, and is skipped too.
1187
+ """
1188
+ cited = _published_doi((entry.ids.doi or "").lower())
1189
+ if not cited or entry.entry_type in _NEEDS_ISBN:
1190
+ return None
1191
+ records = [*artifact.records, *artifact.versions]
1192
+ if artifact.best_record is not None:
1193
+ records.append(artifact.best_record)
1194
+ carriers = [
1195
+ r for r in records
1196
+ if cited in {
1197
+ d.lower() for d in [r.ids.doi or "", *((r.raw or {}).get("merged_dois") or [])]
1198
+ }
1199
+ ]
1200
+ matched_doi = artifact.merged_ids.doi or next((r.ids.doi for r in records if r.ids.doi), "")
1201
+
1202
+ def finding(status: str, detail: str, sources: list[str]) -> FieldFinding:
1203
+ return FieldFinding(
1204
+ field="doi", bib_value=cited, canonical_value=matched_doi or "",
1205
+ sources=sorted(set(sources)), status=status, detail=detail,
1206
+ )
1207
+
1208
+ if carriers or (artifact.merged_ids.doi or "").lower() == cited:
1209
+ return finding(
1210
+ "ok", "the matched work carries the cited DOI", [r.source for r in carriers]
1211
+ )
1212
+
1213
+ # The cited DOI on its own, cached under a probe entry keyed by that DOI alone. Only reached
1214
+ # when the match does not carry the DOI, so this extra lookup is rare.
1215
+ probe = BibEntry(key=entry.key, entry_type=entry.entry_type, ids=Identifiers(doi=cited))
1216
+ adapters = list(route_entry(probe, self.adapters).id_adapters)
1217
+ holders: list[SourceRecord] = []
1218
+ errors: list[str] = []
1219
+ if adapters:
1220
+ found, errors = await self._gather_by_id(probe, probe.ids, adapters)
1221
+ holders = [r for r in found if (r.ids.doi or "").lower() == cited]
1222
+ matched = f" (the matched work's DOI is {matched_doi})" if matched_doi else ""
1223
+ if holders:
1224
+ h = holders[0]
1225
+ best = artifact.best_record
1226
+ same_work = best is not None and (
1227
+ title_ratio(h.title, best.title) >= self.config.title_accept
1228
+ and author_overlap(best.authors, h.authors) >= self.config.author_accept
1229
+ )
1230
+ year = f", {h.year}" if h.year else ""
1231
+ if same_work:
1232
+ return finding(
1233
+ "uncertain",
1234
+ f"the cited DOI belongs to a record with the same title and authors "
1235
+ f"('{h.title}'{year}) that was not merged with the matched work; check that "
1236
+ f"it is a version of it{matched}",
1237
+ [r.source for r in holders],
1238
+ )
1239
+ return finding(
1240
+ "error",
1241
+ f"the cited DOI belongs to a different work: '{h.title}'{year}{matched}",
1242
+ [r.source for r in holders],
1243
+ )
1244
+
1245
+ publisher = next((a for a in self.adapters if a.name == "publisher"), None)
1246
+ resolves = await self._doi_resolves(cited, publisher) if publisher is not None else None
1247
+ if resolves is False:
1248
+ return finding(
1249
+ "error",
1250
+ f"the cited DOI does not resolve at doi.org: the DOI system has no record of it"
1251
+ f"{matched}",
1252
+ ["doi.org"],
1253
+ )
1254
+ if resolves is True:
1255
+ return finding(
1256
+ "uncertain",
1257
+ "the cited DOI is registered at doi.org, but no source we queried has a record "
1258
+ f"for it, so it could not be tied to the matched work{matched}",
1259
+ ["doi.org"],
1260
+ )
1261
+ why = "; ".join(errors) if errors else "no source has a record for it"
1262
+ reach = (
1263
+ "doi.org could not be reached" if publisher is not None
1264
+ else "no doi.org check is configured"
1265
+ )
1266
+ return finding(
1267
+ "unverifiable", f"could not check the cited DOI: {why}, and {reach}{matched}", [],
1268
+ )
1269
+
1270
+ async def _check_alignment(self, audit: EntryAudit, verdict) -> None:
1271
+ """Citation alignment: check each citing context against the cited work's abstract.
1272
+
1273
+ Advisory only — never alters the verdict. Runs on an `exactly_one` match when enabled and the
1274
+ entry has citing contexts. A `contradicted` finding is surfaced loudly per citation; the
1275
+ (often several) `unverifiable` findings collapse to one line (reliability: report the gap,
1276
+ never silently pass); benign `not_in_abstract` / `supported` stay on `audit.alignment_findings`
1277
+ for machine consumers and the dedicated report section. Isolated so a failure cannot abort the
1278
+ entry.
1279
+ """
1280
+ if not self.config.check_alignment:
1281
+ return
1282
+ if verdict is None or verdict.kind != "exactly_one" or not verdict.artifacts:
1283
+ return
1284
+ contexts = self._citation_contexts.get(audit.entry.key, [])
1285
+ if not contexts:
1286
+ # For a .bib/.tex run, no context is an expected absence (uncited entry, or no manuscript
1287
+ # given) and stays silent. For a PDF whose markers GROBID could not link at all, it is an
1288
+ # extraction failure the user asked for a check on — say so rather than passing quietly.
1289
+ if not self._contexts_available:
1290
+ audit.issues.append(
1291
+ "no in-text citation markers could be linked in this document, so citation "
1292
+ "alignment was not checked for any reference"
1293
+ )
1294
+ return
1295
+ try:
1296
+ findings = await resolve_alignment_findings(
1297
+ audit.entry, verdict.artifacts[0], contexts, self.llm, self.config, self.cache
1298
+ )
1299
+ except asyncio.CancelledError:
1300
+ raise
1301
+ except Exception as exc: # noqa: BLE001 — advisory; report, don't fail the entry
1302
+ audit.issues.append(f"citation alignment check failed (verdict unaffected): {exc}")
1303
+ return
1304
+ audit.alignment_findings = findings
1305
+ for f in findings:
1306
+ if f.status == "contradicted":
1307
+ audit.issues.append(alignment_note(f))
1308
+ unverifiable = sum(1 for f in findings if f.status == "unverifiable")
1309
+ if unverifiable:
1310
+ first = next(f for f in findings if f.status == "unverifiable")
1311
+ audit.issues.append(
1312
+ f"citation alignment could not be checked for {unverifiable} citation(s) — {first.detail}"
1313
+ )
1314
+
1315
+ def _book_backfill_isbns(self, entry: BibEntry, records: list[SourceRecord]) -> tuple[str, ...]:
1316
+ """ISBNs to locate the book in Open Library when the entry carries none of its own.
1317
+
1318
+ A book is often cited by a *chapter-level* DOI (Oxford Scholarship Online mints one DOI per
1319
+ chapter: `10.1093/{isbn10}.003.0002`). That DOI's own record carries the containing book's
1320
+ ISBNs, and Open Library — which whiffs on subtitle-bearing title searches — resolves the book
1321
+ cleanly by ISBN. Return the *whole* ISBN set (OL indexes only some of a book's ISBNs, so we
1322
+ try them all). Gated on the entry actually carrying a DOI (this is specifically the chapter-DOI
1323
+ case); we trust an ISBN only from a record that provably belongs to this book: the author-cited
1324
+ DOI's own record (same published DOI), or an Open Library edition (the authority itself, which
1325
+ is what the cached artifact holds when re-deriving from a cached verdict). We deliberately do
1326
+ NOT trust a mere same-author record, which could be a different book by the same author."""
1327
+ cited_doi = _published_doi(entry.ids.doi)
1328
+ if entry.ids.isbn13 or not cited_doi:
1329
+ return ()
1330
+ for r in records:
1331
+ isbns = r.ids.all_isbn13()
1332
+ if not isbns:
1333
+ continue
1334
+ if _published_doi(r.ids.doi) == cited_doi or r.source == "openlibrary":
1335
+ return tuple(sorted(isbns))
1336
+ return ()
1337
+
1338
+ async def _resolve_book(
1339
+ self, audit: EntryAudit, records: list[SourceRecord] | None = None
1340
+ ) -> _BookResolution | None:
1341
+ """Consult Open Library — the authority of record for books — for the entry's editions.
1342
+
1343
+ Returns None for non-books. Otherwise fetches the editions and selects the cited one
1344
+ (`matched`) and the most recent one (`latest`); a transport/HTTP failure is carried as
1345
+ `error` (reliability: an outage is reported, never read as 'no editions'). The result drives
1346
+ both the identity override (`_apply_book_identity`) and the report (`_report_book`).
1347
+
1348
+ When the entry has no ISBN of its own, an ISBN backfilled from the cited DOI's record (see
1349
+ `_book_backfill_isbn`) is used to *fetch* the editions — but the cited edition is still matched
1350
+ against the original entry's year/publisher, so a book cited by a chapter DOI grounds on the
1351
+ edition the author actually cited (e.g. the 1989 original, not a 1992 reprint the DOI rides on).
1352
+ """
1353
+ if audit.entry.entry_type not in _NEEDS_ISBN:
1354
+ return None
1355
+ ol = next((a for a in self.adapters if a.name == "openlibrary"), None)
1356
+ if ol is None:
1357
+ return _BookResolution(error="Open Library adapter not configured")
1358
+ backfilled = self._book_backfill_isbns(audit.entry, records or [])
1359
+ query_entry = audit.entry
1360
+ if backfilled:
1361
+ query_entry = audit.entry.model_copy(
1362
+ update={
1363
+ "ids": audit.entry.ids.model_copy(
1364
+ update={"isbn13": backfilled[0], "isbn13s": backfilled}
1365
+ )
1366
+ }
1367
+ )
1368
+ try:
1369
+ result = await self._fetch_editions(audit.entry, ol, query_entry=query_entry)
1370
+ except asyncio.CancelledError:
1371
+ raise
1372
+ except Exception as exc: # noqa: BLE001 — report, don't fail the entry
1373
+ return _BookResolution(error=f"edition lookup failed: {exc}")
1374
+ if result.error:
1375
+ return _BookResolution(error=result.error)
1376
+ editions = result.records
1377
+ # Cited edition by the original entry's year/publisher; fall back to the backfilled-ISBN
1378
+ # edition so a book whose cited year matches no edition is still confirmed (not a miss).
1379
+ matched = match_cited_edition(audit.entry, editions)
1380
+ if matched is None and backfilled:
1381
+ matched = match_cited_edition(query_entry, editions)
1382
+ return _BookResolution(
1383
+ editions=editions,
1384
+ matched=matched,
1385
+ latest=latest_edition(editions),
1386
+ backfilled_isbn=(matched.ids.isbn13 if (backfilled and matched) else None),
1387
+ )
1388
+
1389
+ def _cited_work(self, entry: BibEntry, cited_work: SourceQueryResult | None):
1390
+ """The record the cited Work id resolves to (OpenAlex answers a merged id with the Work it was
1391
+ merged into), and whether it passes the title+author gate. (None, False) when there is none."""
1392
+ if not entry.ids.openalex or cited_work is None or not cited_work.records:
1393
+ return None, False
1394
+ rec = next(
1395
+ (r for r in cited_work.records if r.ids.openalex == entry.ids.openalex),
1396
+ cited_work.records[0],
1397
+ )
1398
+ features = compute_features(
1399
+ entry, rec, tail_threshold=self.config.prefix_trap_tail_jaccard
1400
+ )
1401
+ return rec, bucket(features, self.config, entry_has_id=True) == "auto_accept"
1402
+
1403
+ def _apply_openalex_identity(self, verdict, entry: BibEntry, cited_work):
1404
+ """Pin the matched artifact to the cited OpenAlex Work when its by-id lookup confirms it.
1405
+
1406
+ A cited openalex.org Work id is an author-supplied identifier, as authoritative as a DOI/ISBN:
1407
+ when the lookup returns that exact Work and it passes the title+author gate, that record IS
1408
+ the identity. We override here so the explicitly-cited Work — not a similar-titled foreign-DOI
1409
+ record the title pooler may have merged it into — represents the entry (which keeps a wrong
1410
+ DOI/ISBN from being backfilled). A cited id whose Work has a mismatched title/author does NOT
1411
+ confirm (it scores below auto_accept) and is left to the generic verdict;
1412
+ `_check_cited_work` reports it.
1413
+ """
1414
+ rec, confirmed = self._cited_work(entry, cited_work)
1415
+ if rec is None or not confirmed:
1416
+ # The lookup errored, found nothing, or found another work — never a confirmation.
1417
+ return verdict
1418
+ artifact = MatchedArtifact(
1419
+ records=[rec], versions=[rec], best_record=rec, merged_ids=rec.ids
1420
+ )
1421
+ return Verdict(
1422
+ kind="exactly_one",
1423
+ artifacts=[artifact],
1424
+ confidence="high",
1425
+ rationale="confirmed via OpenAlex (cited Work id resolved)",
1426
+ )
1427
+
1428
+ def _check_cited_work(
1429
+ self, audit: EntryAudit, verdict, cited_work: SourceQueryResult | None
1430
+ ) -> None:
1431
+ """Report a cited OpenAlex Work id that is not the entry's work, or could not be checked.
1432
+
1433
+ Runs for every verdict: the Work id is the author's own identifier, so whether the entry came
1434
+ out matched to another record, ambiguous, unmatched or unresolved, a Work id that resolves to
1435
+ a different work (or to nothing) is reported, naming the Work. The verdict is not changed.
1436
+ """
1437
+ work_id = audit.entry.ids.openalex
1438
+ if not work_id:
1439
+ return
1440
+ if cited_work is None:
1441
+ audit.issues.append(
1442
+ f"could not check the cited OpenAlex Work id {work_id}: OpenAlex is not among the "
1443
+ "configured sources"
1444
+ )
1445
+ return
1446
+ if cited_work.error:
1447
+ audit.issues.append(
1448
+ f"could not check the cited OpenAlex Work id {work_id}: the OpenAlex lookup failed "
1449
+ f"({cited_work.error})"
1450
+ )
1451
+ return
1452
+ rec, confirmed = self._cited_work(audit.entry, cited_work)
1453
+ if rec is None:
1454
+ audit.issues.append(
1455
+ f"the cited OpenAlex Work id {work_id} does not resolve: OpenAlex has no such Work"
1456
+ )
1457
+ return
1458
+ matched_ids = {
1459
+ r.ids.openalex
1460
+ for a in (verdict.artifacts if verdict is not None else [])
1461
+ for r in [*a.records, *a.versions, *([a.best_record] if a.best_record else [])]
1462
+ if r.ids.openalex
1463
+ }
1464
+ if confirmed or rec.ids.openalex in matched_ids:
1465
+ return
1466
+ year = f", {rec.year}" if rec.year else ""
1467
+ authors = ""
1468
+ if rec.authors:
1469
+ more = " et al." if len(rec.authors) > 3 else ""
1470
+ authors = f" by {', '.join(rec.authors[:3])}{more}"
1471
+ merged = (
1472
+ f" (as {rec.ids.openalex})" if rec.ids.openalex and rec.ids.openalex != work_id else ""
1473
+ )
1474
+ audit.issues.append(
1475
+ f"the cited OpenAlex Work id {work_id} belongs to a different work{merged}: "
1476
+ f"'{rec.title}'{authors}{year}"
1477
+ )
1478
+
1479
+ def _apply_google_books_identity(self, verdict, entry: BibEntry, records: list[SourceRecord]):
1480
+ """Pin the matched artifact to the cited Google Books volume when its by-id lookup confirms it.
1481
+
1482
+ Mirrors `_apply_openalex_identity`: a cited volume id is an author-supplied key, so when the
1483
+ lookup returns that exact volume and it passes the title+author gate, that record IS the
1484
+ identity — overriding a same-titled journal-article a title pooler may otherwise match (a book
1485
+ review carries the book's title+authors AND a DOI, which would be wrongly backfilled onto a
1486
+ @book). A volume whose title/author mismatches scores below auto_accept and is left alone.
1487
+ """
1488
+ if not entry.ids.google_books:
1489
+ return verdict
1490
+ rec = next(
1491
+ (
1492
+ r
1493
+ for r in records
1494
+ if r.ids.google_books and r.ids.google_books == entry.ids.google_books
1495
+ ),
1496
+ None,
1497
+ )
1498
+ if rec is None:
1499
+ return verdict # the lookup errored or returned nothing — never read as a confirmation
1500
+ features = compute_features(
1501
+ entry, rec, tail_threshold=self.config.prefix_trap_tail_jaccard
1502
+ )
1503
+ if bucket(features, self.config, entry_has_id=True) != "auto_accept":
1504
+ return verdict
1505
+ artifact = MatchedArtifact(
1506
+ records=[rec], versions=[rec], best_record=rec, merged_ids=rec.ids
1507
+ )
1508
+ return Verdict(
1509
+ kind="exactly_one",
1510
+ artifacts=[artifact],
1511
+ confidence="high",
1512
+ rationale="confirmed via Google Books (cited volume id resolved)",
1513
+ )
1514
+
1515
+ def _apply_book_identity(self, verdict, book: _BookResolution | None):
1516
+ """For a book, an Open Library edition match is authoritative identity: it confirms the cited
1517
+ edition as the matched artifact, overriding the article-centric matcher (which can't bridge a
1518
+ 1976 original to its only DOI-bearing record, a 2018 reprint). Non-books and unconfirmed books
1519
+ keep whatever the generic matcher decided.
1520
+
1521
+ One exception, for reliability: when Open Library — the book authority of record — could not be
1522
+ reached (`book.error`), a `none` ('possible hallucination') verdict from the article-centric
1523
+ matcher is NOT trustworthy. That matcher routinely fails to confirm a real book (it sees only a
1524
+ reprint, or nothing), so without the authority we have not actually checked the book. Leave it
1525
+ unresolved (reported, retried next run) rather than assert a hallucination we did not establish.
1526
+ """
1527
+ if book is None:
1528
+ return verdict
1529
+ if book.matched is None:
1530
+ if book.error and verdict is not None and verdict.kind == "none":
1531
+ return None
1532
+ return verdict
1533
+ edition = book.matched
1534
+ artifact = MatchedArtifact(
1535
+ records=[edition], versions=[edition], best_record=edition, merged_ids=edition.ids
1536
+ )
1537
+ return Verdict(
1538
+ kind="exactly_one",
1539
+ artifacts=[artifact],
1540
+ confidence="high",
1541
+ rationale="confirmed via Open Library (cited edition matched)",
1542
+ )
1543
+
1544
+ async def _report_book(self, audit: EntryAudit, book: _BookResolution | None) -> None:
1545
+ """Report the Open Library book check: confirm the cited edition (or raise the gap to the
1546
+ user), check its year/publisher, and point at the latest edition as a better version.
1547
+
1548
+ Runs after `_check_fields` so it appends to — never clobbers — the field findings.
1549
+ """
1550
+ if book is None:
1551
+ return
1552
+ cited = describe_cited_edition(audit.entry)
1553
+ if book.error:
1554
+ audit.issues.append(
1555
+ f"could not verify the cited edition ({cited}) against Open Library "
1556
+ f"(reported, will retry next run): {book.error}"
1557
+ )
1558
+ return
1559
+ if not book.editions:
1560
+ audit.issues.append(
1561
+ f"could not verify the cited edition ({cited}) against Open Library "
1562
+ "— the book was not found there"
1563
+ )
1564
+ return
1565
+ if book.matched is None:
1566
+ audit.issues.append(
1567
+ f"the cited edition ({cited}) was not found among Open Library's "
1568
+ f"{len(book.editions)} known edition(s) — verify the year/publisher"
1569
+ )
1570
+ return
1571
+
1572
+ # Step 1: year/publisher correctness, grounded in the confirmed cited edition.
1573
+ if self.config.check_fields:
1574
+ try:
1575
+ findings = await check_book_edition_fields(
1576
+ audit.entry, book.matched, self.llm, self.config, self.cache
1577
+ )
1578
+ except asyncio.CancelledError:
1579
+ raise
1580
+ except Exception as exc: # noqa: BLE001 — advisory; report, don't fail the entry
1581
+ audit.issues.append(f"book edition field check failed (verdict unaffected): {exc}")
1582
+ findings = []
1583
+ audit.field_findings = list(audit.field_findings) + findings
1584
+ for f in findings:
1585
+ if f.status in ("error", "uncertain"):
1586
+ audit.issues.append(finding_note(f))
1587
+
1588
+ if book.backfilled_isbn is not None:
1589
+ audit.issues.append(
1590
+ f"the cited DOI ({audit.entry.ids.doi}) is a chapter/component DOI; the book itself "
1591
+ f"was confirmed via Open Library by its ISBN ({book.backfilled_isbn}) — consider "
1592
+ "citing the book's ISBN or book-level DOI"
1593
+ )
1594
+
1595
+ # Step 2: a later edition than the one cited is a better version to consider.
1596
+ note = better_edition_note(book.matched, book.latest)
1597
+ if note:
1598
+ audit.issues.append(note)
1599
+
1600
+ async def _fetch_editions(
1601
+ self, entry: BibEntry, ol: SourceAdapter, *, query_entry: BibEntry | None = None
1602
+ ):
1603
+ """Open Library editions for `entry` (cached per entry, like any source query).
1604
+
1605
+ `query_entry` (default `entry`) is what's actually sent to Open Library — it may carry an ISBN
1606
+ backfilled from the cited DOI — while the cache key stays the *original* entry's hash, so the
1607
+ cached-verdict re-derivation reuses the same editions regardless of how the ISBN was found."""
1608
+ if self.cache is not None:
1609
+ cached = self.cache.get_source_query(entry.content_hash, ol.name, "editions")
1610
+ if cached is not None:
1611
+ return cached
1612
+ result = await ol.fetch_editions(query_entry or entry)
1613
+ if self.cache is not None:
1614
+ self.cache.put_source_query(entry.content_hash, result)
1615
+ return result
1616
+
1617
+
1618
+ async def _run_async(
1619
+ tex_path: str | Path | None,
1620
+ bib_path: str | Path,
1621
+ config: AuditConfig,
1622
+ cache: AuditCache | None,
1623
+ progress: bool,
1624
+ ) -> AuditReport:
1625
+ pipeline = AuditPipeline(config, cache=cache)
1626
+ try:
1627
+ return await pipeline.run(tex_path, bib_path, progress=progress)
1628
+ finally:
1629
+ await pipeline.aclose()
1630
+
1631
+
1632
+ async def _run_pipeline(
1633
+ run: Callable[[AuditPipeline], Awaitable[AuditReport]],
1634
+ config: AuditConfig,
1635
+ cache: AuditCache | None,
1636
+ ) -> AuditReport:
1637
+ pipeline = AuditPipeline(config, cache=cache)
1638
+ try:
1639
+ return await run(pipeline)
1640
+ finally:
1641
+ await pipeline.aclose()
1642
+
1643
+
1644
+ def _run_sync(
1645
+ run: Callable[[AuditPipeline], Awaitable[AuditReport]],
1646
+ config: AuditConfig | None,
1647
+ cache_path: str | Path | None,
1648
+ fresh: bool,
1649
+ ) -> AuditReport:
1650
+ """Shared body of the synchronous entry points: build config + cache, run, tear down."""
1651
+ config = config or AuditConfig()
1652
+ cache: AuditCache | None = None
1653
+ if cache_path is not None:
1654
+ cache = AuditCache(
1655
+ cache_path,
1656
+ pipeline_version=config.pipeline_version,
1657
+ model=config.model,
1658
+ backend=config.source_backend,
1659
+ fresh=fresh,
1660
+ )
1661
+ try:
1662
+ return asyncio.run(_run_pipeline(run, config, cache))
1663
+ finally:
1664
+ if cache is not None:
1665
+ cache.close()
1666
+
1667
+
1668
+ def run_audit(
1669
+ tex_path: str | Path | None,
1670
+ bib_path: str | Path,
1671
+ *,
1672
+ config: AuditConfig | None = None,
1673
+ cache_path: str | Path | None = None,
1674
+ fresh: bool = False,
1675
+ progress: bool = False,
1676
+ ) -> AuditReport:
1677
+ """Synchronous entry point used by the CLI: build config + cache, run, tear down."""
1678
+ return _run_sync(
1679
+ lambda p: p.run(tex_path, bib_path, progress=progress), config, cache_path, fresh
1680
+ )
1681
+
1682
+
1683
+ def run_pdf_audit(
1684
+ pdf_path: str | Path,
1685
+ *,
1686
+ config: AuditConfig | None = None,
1687
+ cache_path: str | Path | None = None,
1688
+ fresh: bool = False,
1689
+ progress: bool = False,
1690
+ ) -> AuditReport:
1691
+ """Synchronous entry point for a PDF input — the `run_audit` counterpart used by the CLI."""
1692
+ return _run_sync(lambda p: p.run_pdf(pdf_path, progress=progress), config, cache_path, fresh)