reference-audit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. reference_audit/__init__.py +8 -0
  2. reference_audit/alignmentcheck.py +136 -0
  3. reference_audit/bookcheck.py +98 -0
  4. reference_audit/cache/__init__.py +1 -0
  5. reference_audit/cache/db.py +108 -0
  6. reference_audit/cache/store.py +128 -0
  7. reference_audit/cli.py +190 -0
  8. reference_audit/config.py +179 -0
  9. reference_audit/fieldcheck.py +599 -0
  10. reference_audit/inputs.py +96 -0
  11. reference_audit/llm/__init__.py +6 -0
  12. reference_audit/llm/client.py +85 -0
  13. reference_audit/llm/prompts.py +206 -0
  14. reference_audit/llm/schemas.py +38 -0
  15. reference_audit/matching/__init__.py +6 -0
  16. reference_audit/matching/adjudicate.py +63 -0
  17. reference_audit/matching/features.py +185 -0
  18. reference_audit/matching/names.py +238 -0
  19. reference_audit/matching/pool.py +344 -0
  20. reference_audit/matching/sameobject.py +150 -0
  21. reference_audit/matching/scoring.py +75 -0
  22. reference_audit/matching/verdict.py +56 -0
  23. reference_audit/matching/webcheck.py +189 -0
  24. reference_audit/models.py +412 -0
  25. reference_audit/parsing/__init__.py +1 -0
  26. reference_audit/parsing/bib.py +392 -0
  27. reference_audit/parsing/context.py +55 -0
  28. reference_audit/parsing/entities.py +22 -0
  29. reference_audit/parsing/identifiers.py +165 -0
  30. reference_audit/parsing/latex.py +83 -0
  31. reference_audit/parsing/tex.py +159 -0
  32. reference_audit/pdf/__init__.py +16 -0
  33. reference_audit/pdf/embedded.py +117 -0
  34. reference_audit/pdf/grobid.py +234 -0
  35. reference_audit/pdf/tei.py +470 -0
  36. reference_audit/pipeline.py +1692 -0
  37. reference_audit/report.py +330 -0
  38. reference_audit/sources/__init__.py +6 -0
  39. reference_audit/sources/arxiv.py +81 -0
  40. reference_audit/sources/base.py +54 -0
  41. reference_audit/sources/clickhouse.py +575 -0
  42. reference_audit/sources/crossref.py +90 -0
  43. reference_audit/sources/dblp.py +140 -0
  44. reference_audit/sources/google_books.py +101 -0
  45. reference_audit/sources/http.py +260 -0
  46. reference_audit/sources/normalize.py +464 -0
  47. reference_audit/sources/openalex.py +93 -0
  48. reference_audit/sources/openlibrary.py +158 -0
  49. reference_audit/sources/publisher.py +186 -0
  50. reference_audit/sources/registry.py +117 -0
  51. reference_audit/sources/render.py +115 -0
  52. reference_audit/sources/semantic_scholar.py +68 -0
  53. reference_audit/sources/titlewords.py +31 -0
  54. reference_audit/sources/web.py +256 -0
  55. reference_audit/versioning.py +125 -0
  56. reference_audit-0.1.0.dist-info/METADATA +715 -0
  57. reference_audit-0.1.0.dist-info/RECORD +60 -0
  58. reference_audit-0.1.0.dist-info/WHEEL +4 -0
  59. reference_audit-0.1.0.dist-info/entry_points.txt +2 -0
  60. reference_audit-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,344 @@
1
+ """Candidate pooling: merge source records that are the same work into one candidate.
2
+
3
+ Merging is IDENTIFIER-GROUNDED only — never title fuzz — so distinct works (distinct DOIs, T2) stay
4
+ separate, while:
5
+ * records sharing an exact id (DOI / arXiv / ISBN13 / OpenAlex Work id) collapse, and
6
+ * a preprint and its published version merge when one record's id appears in the other's
7
+ `version_links` (the arXiv↔published edge — T1).
8
+ The richer record (DOI-bearing, then higher citation count) represents the group; ids and version
9
+ links are unioned, and both source records are retained on the merged candidate for the report.
10
+ The ambiguous, identifier-disjoint zone (similar titles, no link) is left to M5's LLM tie-break.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from reference_audit.matching.features import is_repository_venue, title_prefix_trap, title_ratio
16
+ from reference_audit.matching.names import authors_compatible
17
+ from reference_audit.models import Identifiers, SourceRecord
18
+ from reference_audit.parsing.identifiers import extract_arxiv_id, normalize_doi
19
+
20
+ # Title threshold for the preprint↔published version merge (tight: titles are "usually correct").
21
+ _VERSION_MERGE_TITLE = 0.93
22
+
23
+
24
+ def _own_keys(rec: SourceRecord) -> set[str]:
25
+ keys: set[str] = set()
26
+ if rec.ids.doi:
27
+ keys.add(f"doi:{rec.ids.doi}")
28
+ if rec.ids.arxiv_id:
29
+ keys.add(f"arxiv:{rec.ids.arxiv_id.lower()}")
30
+ # Pool on the canonical ISBN only — deliberately NOT the whole `all_isbn13()` set. A book
31
+ # *chapter*'s scholarly record inherits its containing volume's ISBNs, so pooling on any shared
32
+ # ISBN would fuse sibling chapters (and the chapter with its parent book) into one record whose
33
+ # representative is picked by citation-richness, not title — which could bury the title-matching
34
+ # chapter. Same-work editions (print vs electronic) are still unified downstream by the
35
+ # SAME-OBJECT clustering at verdict time, which compares title+author, not ISBN membership alone.
36
+ if rec.ids.isbn13:
37
+ keys.add(f"isbn:{rec.ids.isbn13}")
38
+ if rec.openalex_work_id:
39
+ keys.add(f"oa:{rec.openalex_work_id}")
40
+ return keys
41
+
42
+
43
+ def _link_to_keys(link: str) -> set[str]:
44
+ """Identifier keys referenced by a version-link string (URL or bare id)."""
45
+ keys: set[str] = set()
46
+ arxiv = extract_arxiv_id(None, None, fallback_text=link)
47
+ if arxiv:
48
+ keys.add(f"arxiv:{arxiv.lower()}")
49
+ doi = normalize_doi(link)
50
+ if doi and not doi.startswith("10.48550/arxiv"): # the arXiv DOI is already captured as arxiv:
51
+ keys.add(f"doi:{doi}")
52
+ return keys
53
+
54
+
55
+ def _link_keys(rec: SourceRecord) -> set[str]:
56
+ keys: set[str] = set()
57
+ for link in rec.version_links:
58
+ keys |= _link_to_keys(link)
59
+ return keys
60
+
61
+
62
+ def _merge_ids(a: Identifiers, b: Identifiers) -> Identifiers:
63
+ return Identifiers(
64
+ doi=a.doi or b.doi,
65
+ arxiv_id=a.arxiv_id or b.arxiv_id,
66
+ isbn13=a.isbn13 or b.isbn13,
67
+ isbn13s=tuple(dict.fromkeys((*a.isbn13s, *b.isbn13s, *a.all_isbn13(), *b.all_isbn13()))),
68
+ pmid=a.pmid or b.pmid,
69
+ bibcode=a.bibcode or b.bibcode,
70
+ openalex=a.openalex or b.openalex,
71
+ url=a.url or b.url,
72
+ )
73
+
74
+
75
+ def _richness(rec: SourceRecord) -> tuple[int, int, int]:
76
+ return (1 if rec.ids.doi else 0, rec.citation_count, len(rec.authors))
77
+
78
+
79
+ def _underlying_sources(rec: SourceRecord) -> list[str]:
80
+ """Sources actually behind a record: the merge set if it is a pooled representative, else its own
81
+ source. Pooling must be idempotent — re-pooling an already-merged record (the enrichment pass)
82
+ must not lose provenance. (The prior two-phase pooling rebuilt representatives from earlier
83
+ representatives and ranked them by a single lossy `.source` label, so a preprint copy could
84
+ outrank a richer published record hidden inside another representative — the Flow-Lenia bug.)"""
85
+ merged_from = rec.raw.get("merged_from") if isinstance(rec.raw, dict) else None
86
+ return list(merged_from) if merged_from else [rec.source]
87
+
88
+
89
+ # Bibliographic fields are filled on the merged representative from the most authoritative source
90
+ # that carries them, independent of which record is the citation-richest. The publisher of record
91
+ # (the DOI landing-page citation export) outranks the aggregators; Crossref/OpenAlex (the
92
+ # registration-grade sources) outrank Semantic Scholar, whose venue strings are often truncated
93
+ # ('Complexity' → 'Complex') and which omits volume/issue/pages. Records are ranked by their *best
94
+ # underlying* source, so a pooled representative is ranked by the richest source inside it — not by
95
+ # its lossy `.source` label. This realizes the SPEC's "compile all available information". DBLP
96
+ # carries the conference of a CS paper that the aggregators often index only as its arXiv copy.
97
+ _FIELD_SOURCE_PRIORITY: dict[str, tuple[str, ...]] = {
98
+ "venue": ("publisher", "crossref", "openalex", "dblp", "openlibrary"),
99
+ "volume": ("publisher", "crossref", "openalex"),
100
+ "issue": ("publisher", "crossref", "openalex"),
101
+ "pages": ("publisher", "crossref", "openalex"),
102
+ "publisher": ("publisher", "crossref", "openlibrary", "openalex"),
103
+ }
104
+
105
+
106
+ _YEAR_SOURCE_PRIORITY = ("publisher", "crossref", "openalex")
107
+
108
+ # The author list scoring and adjudication see. The citation-richest record is often Semantic
109
+ # Scholar's, whose lists can be defective (a name twice, one missing: Neural Collapse's 'Hung Tran'
110
+ # for Stanley Osher), and the LLM rejected the real paper over it. DBLP and the registration-grade
111
+ # sources keep complete, clean lists; arXiv's is author-supplied.
112
+ _AUTHOR_SOURCE_PRIORITY = ("publisher", "dblp", "crossref", "arxiv", "openalex", "semantic_scholar")
113
+
114
+
115
+ def _rank(rec: SourceRecord, priority: tuple[str, ...]) -> int:
116
+ """Priority rank of a record by its best (lowest-rank) underlying source."""
117
+ return min(
118
+ (priority.index(s) if s in priority else len(priority))
119
+ for s in _underlying_sources(rec)
120
+ )
121
+
122
+
123
+ def _best_field(recs: list[SourceRecord], attr: str, priority: tuple[str, ...]) -> str:
124
+ """Most authoritative non-empty value for `attr` across a same-work group ('' if none has it).
125
+
126
+ For the venue, a preprint server or repository ('arXiv (Cornell University)', DBLP's 'CoRR') is
127
+ used only when no member names a journal or conference: OpenAlex's arXiv copy of an ICLR paper
128
+ must not outrank DBLP's 'ICLR'.
129
+ """
130
+ ranked = sorted(recs, key=lambda r: _rank(r, priority))
131
+ if attr == "venue":
132
+ ranked.sort(key=lambda r: is_repository_venue(r.venue)) # stable: rank kept within each
133
+ for r in ranked:
134
+ value = (getattr(r, attr) or "").strip()
135
+ if value:
136
+ return value
137
+ return ""
138
+
139
+
140
+ def _best_authors(recs: list[SourceRecord]) -> list[str]:
141
+ """The author list of the most reliable source in a same-work group ([] if none has one)."""
142
+ for r in sorted(recs, key=lambda r: _rank(r, _AUTHOR_SOURCE_PRIORITY)):
143
+ if r.authors:
144
+ return list(r.authors)
145
+ return []
146
+
147
+
148
+ def _best_abstract(recs: list[SourceRecord]) -> str:
149
+ """The fullest abstract across a same-work group (longest non-empty).
150
+
151
+ Abstracts come from OpenAlex / Semantic Scholar; the citation-richest representative may be a
152
+ source (e.g. Crossref) that carries none, so — like the other bibliographic fields — the abstract
153
+ is compiled across the group rather than taken only from the representative, so it survives onto
154
+ the matched artifact (and the cached verdict) for the citation-alignment check.
155
+ """
156
+ return max(((r.abstract or "").strip() for r in recs), key=len, default="")
157
+
158
+
159
+ def _best_year(recs: list[SourceRecord]) -> int | None:
160
+ """Publication year from the registration-grade source (publisher/Crossref/OpenAlex) when
161
+ available.
162
+
163
+ Semantic Scholar often reports the online/preprint year (it gave 2018 for the 2019 Complex
164
+ Systems 'Lenia' DOI whose Crossref record is 2019), so its richer-citation record must not
165
+ override the DOI registrant's year.
166
+ """
167
+ for r in sorted(recs, key=lambda r: _rank(r, _YEAR_SOURCE_PRIORITY)):
168
+ if r.year:
169
+ return r.year
170
+ return None
171
+
172
+
173
+ def pool_candidates(records: list[SourceRecord]) -> list[SourceRecord]:
174
+ """Merge records that describe the same work into one representative candidate each.
175
+
176
+ A single union-find over all records with two kinds of (transitively closed) edge:
177
+ * identifier edge — both records carry ids and they overlap, or one side's version link
178
+ bridges the other's id (shared DOI/arXiv/ISBN13/OpenAlex id; the arXiv↔published edge).
179
+ Grounded in identifiers, never title fuzz.
180
+ * version edge — `_same_work_version`: a preprint and its published version, or two editions of
181
+ one book, agreeing on title+authors. This is the *only* place title similarity is allowed,
182
+ and only to UNITE versions of one work — two distinct published DOIs are still held apart by
183
+ the V1 veto inside `_same_work_version`.
184
+
185
+ Transitive closure (order-independent union-find) is essential and is why this replaced the
186
+ earlier greedy first-match grouping: a Semantic Scholar record that carries BOTH the arXiv id and
187
+ the published DOI must fuse the preprint group and the published group into one candidate. When
188
+ they stayed split, canonical fields were sourced from the preprint copy (the Flow-Lenia bug).
189
+ """
190
+ n = len(records)
191
+ if n <= 1:
192
+ return list(records)
193
+
194
+ own = [_own_keys(r) for r in records]
195
+ links = [_link_keys(r) for r in records]
196
+ parent = list(range(n))
197
+ # The records in each cluster, keyed by its root.
198
+ cluster = {i: [r] for i, r in enumerate(records)}
199
+
200
+ def find(i: int) -> int:
201
+ while parent[i] != i:
202
+ parent[i] = parent[parent[i]]
203
+ i = parent[i]
204
+ return i
205
+
206
+ def union(i: int, j: int) -> None:
207
+ ri, rj = find(i), find(j)
208
+ parent[rj] = ri
209
+ cluster[ri].extend(cluster.pop(rj))
210
+
211
+ def authors_agree_across(i: int, j: int) -> bool:
212
+ """A version relation joins two clusters only if every record of one has authors compatible
213
+ with every record of the other. A conference paper and its journal extension (CrossFormer at
214
+ ICLR, CrossFormer++ in TPAMI) are two works; fusing them hides the record that matches the
215
+ entry behind the other one's metadata. Checking each pair, not each edge, keeps a record that
216
+ fits both (a one-author supplementary-material DOI) from bridging them. Version *links* are
217
+ held to this too: they come from the aggregators' own work-merging, which can be wrong."""
218
+ return all(
219
+ authors_compatible(a.authors, b.authors)
220
+ for a in cluster[find(i)]
221
+ for b in cluster[find(j)]
222
+ )
223
+
224
+ for i in range(n):
225
+ for j in range(i + 1, n):
226
+ if find(i) == find(j):
227
+ continue
228
+ shared_id = bool(own[i] & own[j]) # the same DOI/arXiv/ISBN/OpenAlex id: one record
229
+ linked = bool(own[i] and own[j] and ((own[i] & links[j]) or (links[i] & own[j])))
230
+ if shared_id or (
231
+ (linked or _same_work_version(records[i], records[j]))
232
+ and authors_agree_across(i, j)
233
+ ):
234
+ union(i, j)
235
+
236
+ clusters: dict[int, list[SourceRecord]] = {}
237
+ for idx in range(n):
238
+ clusters.setdefault(find(idx), []).append(records[idx])
239
+ return [_representative(g) if len(g) > 1 else g[0] for g in clusters.values()]
240
+
241
+
242
+ def _as_member(rec: SourceRecord) -> SourceRecord:
243
+ return rec.model_copy(update={"raw": {}, "abstract": "", "members": []}, deep=True)
244
+
245
+
246
+ def _members(rec: SourceRecord) -> list[SourceRecord]:
247
+ """The source records behind `rec`: its members if it is pooled, else itself."""
248
+ return list(rec.members) if rec.members else [_as_member(rec)]
249
+
250
+
251
+ def _representative(recs: list[SourceRecord]) -> SourceRecord:
252
+ best = max(recs, key=_richness)
253
+ merged = best.model_copy(deep=True)
254
+ for r in recs:
255
+ merged.ids = _merge_ids(merged.ids, r.ids)
256
+ for link in r.version_links:
257
+ if link not in merged.version_links:
258
+ merged.version_links.append(link)
259
+ if r.openalex_work_id and not merged.openalex_work_id:
260
+ merged.openalex_work_id = r.openalex_work_id
261
+ merged.is_preprint = merged.is_preprint and r.is_preprint
262
+ # Compile the best bibliographic metadata across the group (not just the richest record's).
263
+ for attr, priority in _FIELD_SOURCE_PRIORITY.items():
264
+ value = _best_field(recs, attr, priority)
265
+ if value:
266
+ setattr(merged, attr, value)
267
+ best_year = _best_year(recs)
268
+ if best_year is not None:
269
+ merged.year = best_year
270
+ best_authors = _best_authors(recs)
271
+ if best_authors:
272
+ merged.authors = best_authors
273
+ best_abstract = _best_abstract(recs)
274
+ if best_abstract:
275
+ merged.abstract = best_abstract
276
+ merged.raw = {
277
+ "merged_from": sorted({s for r in recs for s in _underlying_sources(r)}),
278
+ # `ids.doi` keeps one DOI (the richest record's, often an arXiv DataCite DOI); every member's
279
+ # DOI is kept here so a cited published DOI can still be recognised as this work's.
280
+ "merged_dois": sorted({d for r in recs for d in _member_dois(r)}),
281
+ }
282
+ merged.members = _distinct([m for r in recs for m in _members(r)])
283
+ return merged
284
+
285
+
286
+ def _distinct(members: list[SourceRecord]) -> list[SourceRecord]:
287
+ """Members without repeats: enrichment re-pools a work with the same sources' by-id records."""
288
+ seen: set[tuple] = set()
289
+ out = []
290
+ for m in members:
291
+ key = (m.source, m.source_native_id) if m.source_native_id else (
292
+ m.source, m.ids.doi, m.ids.arxiv_id, m.title.casefold(), m.year
293
+ )
294
+ if key not in seen:
295
+ seen.add(key)
296
+ out.append(m)
297
+ return out
298
+
299
+
300
+ def _member_dois(rec: SourceRecord) -> list[str]:
301
+ """The record's own DOI plus those of the records it already merged (pooling is re-run)."""
302
+ merged = rec.raw.get("merged_dois") if isinstance(rec.raw, dict) else None
303
+ return [*(merged or []), *([rec.ids.doi] if rec.ids.doi else [])]
304
+
305
+
306
+ def _is_preprintish(rec: SourceRecord) -> bool:
307
+ return rec.is_preprint or bool(rec.ids.doi and rec.ids.doi.startswith("10.48550/arxiv"))
308
+
309
+
310
+ def _published_doi(rec: SourceRecord) -> str | None:
311
+ doi = rec.ids.doi
312
+ return doi if doi and not doi.startswith("10.48550/arxiv") else None
313
+
314
+
315
+ def _title_authors_agree(a: SourceRecord, b: SourceRecord) -> bool:
316
+ if title_ratio(a.title, b.title) < _VERSION_MERGE_TITLE:
317
+ return False
318
+ if title_prefix_trap(a.title, b.title, tail_threshold=0.34):
319
+ return False # V3: shared prefix, divergent tail (bagrov vs kravchenko)
320
+ # Person-level, not a fuzzy surname average: short surnames (Wang, Chen, Lin, He, Liu) let
321
+ # CrossFormer's authors "overlap" CrossFormer++'s at 0.85 with two of seven people different.
322
+ # A title alone never makes a version: both records must name their authors.
323
+ return bool(a.authors and b.authors) and authors_compatible(a.authors, b.authors)
324
+
325
+
326
+ def _same_work_version(a: SourceRecord, b: SourceRecord) -> bool:
327
+ """Two records describe one work when titles+authors agree AND either:
328
+
329
+ R1 (preprint↔published): exactly one side is a preprint — a version relationship that overrides
330
+ the two-distinct-DOIs signal (the published DOI and the arXiv preprint DOI legitimately
331
+ differ); or
332
+ R2 (same edition/printing): the two do NOT both carry distinct *published* DOIs (one or both
333
+ lack one) — different editions/reprints of one book, or a metadata-only duplicate.
334
+
335
+ Two distinct *published* DOIs with neither side a preprint is left UNMERGED (the design's V1
336
+ veto): it is a genuine "same surface" case for M5's LLM tie-break (e.g. duplicate DOIs for one
337
+ paper), not a formal merge.
338
+ """
339
+ if not _title_authors_agree(a, b):
340
+ return False
341
+ if _is_preprintish(a) != _is_preprintish(b): # R1
342
+ return True
343
+ pa, pb = _published_doi(a), _published_doi(b)
344
+ return not (pa and pb and pa != pb) # R2: merge unless two distinct published DOIs
@@ -0,0 +1,150 @@
1
+ """SAME-OBJECT clustering over an entry's ACCEPTED candidates → matched artifacts.
2
+
3
+ `pool.py` already merged the identifier-grounded same-work cases (shared id, version edge, preprint
4
+ pair, same edition). What remains for an entry with ≥2 accepted candidates is the genuinely
5
+ ambiguous zone: two records with near-identical title+authors but two DISTINCT published DOIs (the
6
+ design's V1 veto holds them apart formally — e.g. one paper registered under two DOIs). Those, and
7
+ only those, escalate to the `SAME_WORK` LLM tie-break. The distinct-work vetoes V2 (disjoint pages),
8
+ V3 (title-prefix trap) and V4 (author-set distinct) are decisive and never reach the LLM.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from typing import Literal
14
+
15
+ from reference_audit.cache.store import AuditCache, prompt_hash
16
+ from reference_audit.config import AuditConfig
17
+ from reference_audit.llm.client import LLMClient, LLMError
18
+ from reference_audit.llm.prompts import SAME_WORK_SYSTEM, same_work_user
19
+ from reference_audit.matching.features import (
20
+ author_set_jaccard,
21
+ id_agreement,
22
+ pages_conflict_records,
23
+ title_prefix_trap,
24
+ title_ratio,
25
+ )
26
+ from reference_audit.matching.names import author_overlap, author_subset
27
+ from reference_audit.models import (
28
+ CandidateAssessment,
29
+ Identifiers,
30
+ MatchedArtifact,
31
+ SameWorkResult,
32
+ SourceRecord,
33
+ )
34
+
35
+ Relation = Literal["same", "distinct", "ambiguous"]
36
+
37
+ _SAME_TITLE = 0.90
38
+ _SAME_AUTHOR = 0.7
39
+
40
+
41
+ def formal_relation(a: SourceRecord, b: SourceRecord, config: AuditConfig) -> Relation:
42
+ """Formal same-object relation between two accepted records (no LLM)."""
43
+ if id_agreement(a.ids, b.ids) == "match":
44
+ return "same" # shared exact identifier (M1/M3)
45
+ # Distinct-work vetoes (decisive).
46
+ if pages_conflict_records(a, b):
47
+ return "distinct" # V2
48
+ if title_prefix_trap(a.title, b.title, tail_threshold=config.prefix_trap_tail_jaccard):
49
+ return "distinct" # V3
50
+ if (
51
+ author_set_jaccard(a.authors, b.authors) < config.author_set_distinct_jaccard
52
+ and not author_subset(a.authors, b.authors)
53
+ ):
54
+ return "distinct" # V4
55
+ # Near-identical title + authors but no shared id ⇒ ambiguous (→ SAME_WORK LLM).
56
+ titles_agree = title_ratio(a.title, b.title) >= _SAME_TITLE
57
+ authors_agree = author_overlap(a.authors, b.authors) >= _SAME_AUTHOR or author_subset(
58
+ a.authors, b.authors
59
+ )
60
+ if titles_agree and authors_agree:
61
+ return "ambiguous"
62
+ return "distinct"
63
+
64
+
65
+ def _merge_ids(a: Identifiers, b: Identifiers) -> Identifiers:
66
+ return Identifiers(
67
+ doi=a.doi or b.doi,
68
+ arxiv_id=a.arxiv_id or b.arxiv_id,
69
+ isbn13=a.isbn13 or b.isbn13,
70
+ isbn13s=tuple(dict.fromkeys((*a.isbn13s, *b.isbn13s, *a.all_isbn13(), *b.all_isbn13()))),
71
+ pmid=a.pmid or b.pmid,
72
+ bibcode=a.bibcode or b.bibcode,
73
+ openalex=a.openalex or b.openalex,
74
+ url=a.url or b.url,
75
+ )
76
+
77
+
78
+ def _artifact_from_records(records: list[SourceRecord]) -> MatchedArtifact:
79
+ # Prefer a published (non-preprint), DOI-bearing, highly-cited record as the canonical best.
80
+ best = max(
81
+ records,
82
+ key=lambda r: (0 if r.is_preprint else 1, 1 if r.ids.doi else 0, r.citation_count),
83
+ )
84
+ merged = Identifiers()
85
+ for r in records:
86
+ merged = _merge_ids(merged, r.ids)
87
+ return MatchedArtifact(
88
+ records=list(records), merged_ids=merged, versions=list(records), best_record=best
89
+ )
90
+
91
+
92
+ async def _same_work_llm(
93
+ a: SourceRecord, b: SourceRecord, llm: LLMClient, cache: AuditCache | None
94
+ ) -> tuple[SameWorkResult | None, bool]:
95
+ user = same_work_user(a, b)
96
+ p_hash = prompt_hash(SAME_WORK_SYSTEM + "\n" + user)
97
+ if cache is not None:
98
+ cached = cache.get_llm_decision(p_hash, "same_work")
99
+ if cached is not None:
100
+ return SameWorkResult.model_validate_json(cached), False
101
+ try:
102
+ result = await llm.structured(SAME_WORK_SYSTEM, user, SameWorkResult, "same_work")
103
+ except LLMError:
104
+ return None, True
105
+ if cache is not None:
106
+ cache.put_llm_decision(p_hash, "same_work", result.model_dump_json())
107
+ return result, False
108
+
109
+
110
+ async def cluster_accepted(
111
+ accepted: list[CandidateAssessment],
112
+ config: AuditConfig,
113
+ llm: LLMClient | None,
114
+ cache: AuditCache | None,
115
+ ) -> tuple[list[MatchedArtifact], bool]:
116
+ """Cluster accepted candidates into artifacts (union-find). Returns (artifacts, errored)."""
117
+ records = [c.record for c in accepted]
118
+ n = len(records)
119
+ if n <= 1:
120
+ return ([_artifact_from_records(records)] if records else []), False
121
+
122
+ parent = list(range(n))
123
+
124
+ def find(i: int) -> int:
125
+ while parent[i] != i:
126
+ parent[i] = parent[parent[i]]
127
+ i = parent[i]
128
+ return i
129
+
130
+ def union(i: int, j: int) -> None:
131
+ parent[find(j)] = find(i)
132
+
133
+ errored = False
134
+ for i in range(n):
135
+ for j in range(i + 1, n):
136
+ if find(i) == find(j):
137
+ continue
138
+ rel = formal_relation(records[i], records[j], config)
139
+ if rel == "same":
140
+ union(i, j)
141
+ elif rel == "ambiguous" and llm is not None:
142
+ result, err = await _same_work_llm(records[i], records[j], llm, cache)
143
+ errored = errored or err
144
+ if result and result.relation in ("same_artifact", "versions_of_same_work"):
145
+ union(i, j)
146
+
147
+ clusters: dict[int, list[SourceRecord]] = {}
148
+ for idx, rec in enumerate(records):
149
+ clusters.setdefault(find(idx), []).append(rec)
150
+ return [_artifact_from_records(group) for group in clusters.values()], errored
@@ -0,0 +1,75 @@
1
+ """Bucketing: auto_accept / auto_reject / adjudicate — the funnel before any LLM call.
2
+
3
+ The 100%-match short-circuit (README "unless a returned record is a 100% match, use an LLM")
4
+ reads the feature VECTOR, not the scalar: an id match alone is not enough (a hallucinated DOI that
5
+ resolves must still pass title+author), and a prefix-trap/pages-conflict forces adjudication even
6
+ at high composite (stops T2 from auto-accepting).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import Literal
12
+
13
+ from reference_audit.config import AuditConfig
14
+ from reference_audit.models import FeatureVector
15
+
16
+ Bucket = Literal["auto_accept", "auto_reject", "adjudicate"]
17
+
18
+
19
+ def bucket(features: FeatureVector, config: AuditConfig, *, entry_has_id: bool = True) -> Bucket:
20
+ """Bucket one (entry, candidate) by its feature vector.
21
+
22
+ `entry_has_id` enables the strict backfill path: when the bib entry carries no *strong* anchor
23
+ (DOI / ISBN / arXiv / OpenAlex / Google Books — a bare URL does not count, see
24
+ `Identifiers.has_strong_id`), a near-exact title + strong authors (and no distinct-work veto) is
25
+ accepted so the matched DOI can be backfilled. The threshold is stricter than the id-based path
26
+ because there is no identifier to anchor the match.
27
+ """
28
+ # A low author-set overlap forces adjudication ONLY when neither set is a subset of the other:
29
+ # a subset is consistent with an abbreviated "et al." list (same work), not a distinct work.
30
+ author_set_distinct = (
31
+ features.author_set_jaccard < config.author_set_distinct_jaccard
32
+ and not features.author_subset
33
+ )
34
+ authors_ok = features.author_overlap >= config.author_accept or features.author_subset
35
+
36
+ # A disjoint page range in the same venue normally signals a DISTINCT work (laughlin T2c: two
37
+ # different-titled papers on consecutive pages). But when the title is a near-exact match AND the
38
+ # authors agree, two different works is not a credible explanation — the likely one is a wrong
39
+ # `pages` field in the citation (soros2014: cited 306--313 vs the identical paper's canonical
40
+ # 793--800). Treat that as a field error to be REPORTED by the field check downstream, not as
41
+ # grounds to call a real, title-and-author-identical paper a possible hallucination. (prefix_trap
42
+ # stays load-bearing: it fires on divergent title TAILS, i.e. when the titles are NOT near-exact.)
43
+ title_authors_lock = features.title_ratio >= config.title_backfill and authors_ok
44
+ pages_conflict_distinct = features.pages_conflict and not title_authors_lock
45
+ forced_adjudicate = (
46
+ features.title_prefix_trap or pages_conflict_distinct or author_set_distinct
47
+ )
48
+
49
+ # Path A — entry has an identifier that the candidate matches (authoritative; README: IDs
50
+ # uniquely identify). Accept even if the DB holds a subset of authors (wilson1974/Kogut); the
51
+ # title gate still rejects a hallucinated DOI resolving to a different paper (T4-by-bad-id).
52
+ if (
53
+ features.id_agreement == "match"
54
+ and features.title_ratio >= config.title_accept
55
+ and authors_ok
56
+ and not forced_adjudicate
57
+ ):
58
+ return "auto_accept"
59
+
60
+ # Path B — entry has NO identifier; accept a near-exact title + strong authors to backfill the
61
+ # candidate's DOI/ISBN. Stricter title floor; all distinct-work vetoes still apply (keeps T2a's
62
+ # bagrov/kravchenko prefix-trap from auto-accepting).
63
+ if (
64
+ not entry_has_id
65
+ and features.id_agreement != "conflict"
66
+ and features.title_ratio >= config.title_backfill
67
+ and features.author_overlap >= config.author_accept
68
+ and not forced_adjudicate
69
+ ):
70
+ return "auto_accept"
71
+
72
+ if features.id_agreement != "match" and features.composite < config.composite_reject:
73
+ return "auto_reject"
74
+
75
+ return "adjudicate"
@@ -0,0 +1,56 @@
1
+ """Turn scored + clustered candidates into a 3-way Verdict (README 5.1/5.2/5.3).
2
+
3
+ `artifacts` is the SAME-OBJECT clustering of the *accepted* candidates (see `sameobject.py`): each
4
+ element is one distinct real-world work. The verdict counts them.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from reference_audit.models import CandidateAssessment, MatchedArtifact, Verdict
10
+
11
+
12
+ # @cpt-dod:cpt-referenceaudit-dod-identification-three-way-verdict:p1
13
+ # @cpt-dod:cpt-referenceaudit-dod-identification-hallucination-screen:p1
14
+ def build_verdict(
15
+ assessments: list[CandidateAssessment],
16
+ artifacts: list[MatchedArtifact],
17
+ *,
18
+ errored: bool,
19
+ ) -> Verdict | None:
20
+ if artifacts:
21
+ if len(artifacts) == 1:
22
+ best = artifacts[0].best_record
23
+ src = best.source if best else ""
24
+ n_versions = len(artifacts[0].versions)
25
+ extra = f" ({n_versions} versions)" if n_versions > 1 else ""
26
+ return Verdict(
27
+ kind="exactly_one",
28
+ artifacts=artifacts,
29
+ confidence="high",
30
+ rationale=f"Matched a single work via {src}{extra}.",
31
+ )
32
+ return Verdict(
33
+ kind="multiple",
34
+ artifacts=artifacts,
35
+ confidence="medium",
36
+ rationale=f"Matched {len(artifacts)} distinct works.",
37
+ )
38
+
39
+ if errored:
40
+ return None # transient failure — leave unresolved, retry next run (error ≠ not-found)
41
+
42
+ if not assessments:
43
+ return Verdict(
44
+ kind="none",
45
+ confidence="high",
46
+ rationale="No database record found for this reference (possible hallucination).",
47
+ )
48
+
49
+ if all(a.bucket == "auto_reject" for a in assessments):
50
+ return Verdict(
51
+ kind="none",
52
+ confidence="medium",
53
+ rationale="Database records were found but none matched the entry's title/authors.",
54
+ )
55
+
56
+ return None # adjudicate bucket remaining — deferred to the LLM funnel