reference-audit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reference_audit/__init__.py +8 -0
- reference_audit/alignmentcheck.py +136 -0
- reference_audit/bookcheck.py +98 -0
- reference_audit/cache/__init__.py +1 -0
- reference_audit/cache/db.py +108 -0
- reference_audit/cache/store.py +128 -0
- reference_audit/cli.py +190 -0
- reference_audit/config.py +179 -0
- reference_audit/fieldcheck.py +599 -0
- reference_audit/inputs.py +96 -0
- reference_audit/llm/__init__.py +6 -0
- reference_audit/llm/client.py +85 -0
- reference_audit/llm/prompts.py +206 -0
- reference_audit/llm/schemas.py +38 -0
- reference_audit/matching/__init__.py +6 -0
- reference_audit/matching/adjudicate.py +63 -0
- reference_audit/matching/features.py +185 -0
- reference_audit/matching/names.py +238 -0
- reference_audit/matching/pool.py +344 -0
- reference_audit/matching/sameobject.py +150 -0
- reference_audit/matching/scoring.py +75 -0
- reference_audit/matching/verdict.py +56 -0
- reference_audit/matching/webcheck.py +189 -0
- reference_audit/models.py +412 -0
- reference_audit/parsing/__init__.py +1 -0
- reference_audit/parsing/bib.py +392 -0
- reference_audit/parsing/context.py +55 -0
- reference_audit/parsing/entities.py +22 -0
- reference_audit/parsing/identifiers.py +165 -0
- reference_audit/parsing/latex.py +83 -0
- reference_audit/parsing/tex.py +159 -0
- reference_audit/pdf/__init__.py +16 -0
- reference_audit/pdf/embedded.py +117 -0
- reference_audit/pdf/grobid.py +234 -0
- reference_audit/pdf/tei.py +470 -0
- reference_audit/pipeline.py +1692 -0
- reference_audit/report.py +330 -0
- reference_audit/sources/__init__.py +6 -0
- reference_audit/sources/arxiv.py +81 -0
- reference_audit/sources/base.py +54 -0
- reference_audit/sources/clickhouse.py +575 -0
- reference_audit/sources/crossref.py +90 -0
- reference_audit/sources/dblp.py +140 -0
- reference_audit/sources/google_books.py +101 -0
- reference_audit/sources/http.py +260 -0
- reference_audit/sources/normalize.py +464 -0
- reference_audit/sources/openalex.py +93 -0
- reference_audit/sources/openlibrary.py +158 -0
- reference_audit/sources/publisher.py +186 -0
- reference_audit/sources/registry.py +117 -0
- reference_audit/sources/render.py +115 -0
- reference_audit/sources/semantic_scholar.py +68 -0
- reference_audit/sources/titlewords.py +31 -0
- reference_audit/sources/web.py +256 -0
- reference_audit/versioning.py +125 -0
- reference_audit-0.1.0.dist-info/METADATA +715 -0
- reference_audit-0.1.0.dist-info/RECORD +60 -0
- reference_audit-0.1.0.dist-info/WHEEL +4 -0
- reference_audit-0.1.0.dist-info/entry_points.txt +2 -0
- reference_audit-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
"""Candidate pooling: merge source records that are the same work into one candidate.
|
|
2
|
+
|
|
3
|
+
Merging is IDENTIFIER-GROUNDED only — never title fuzz — so distinct works (distinct DOIs, T2) stay
|
|
4
|
+
separate, while:
|
|
5
|
+
* records sharing an exact id (DOI / arXiv / ISBN13 / OpenAlex Work id) collapse, and
|
|
6
|
+
* a preprint and its published version merge when one record's id appears in the other's
|
|
7
|
+
`version_links` (the arXiv↔published edge — T1).
|
|
8
|
+
The richer record (DOI-bearing, then higher citation count) represents the group; ids and version
|
|
9
|
+
links are unioned, and both source records are retained on the merged candidate for the report.
|
|
10
|
+
The ambiguous, identifier-disjoint zone (similar titles, no link) is left to M5's LLM tie-break.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from reference_audit.matching.features import is_repository_venue, title_prefix_trap, title_ratio
|
|
16
|
+
from reference_audit.matching.names import authors_compatible
|
|
17
|
+
from reference_audit.models import Identifiers, SourceRecord
|
|
18
|
+
from reference_audit.parsing.identifiers import extract_arxiv_id, normalize_doi
|
|
19
|
+
|
|
20
|
+
# Title threshold for the preprint↔published version merge (tight: titles are "usually correct").
|
|
21
|
+
_VERSION_MERGE_TITLE = 0.93
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _own_keys(rec: SourceRecord) -> set[str]:
|
|
25
|
+
keys: set[str] = set()
|
|
26
|
+
if rec.ids.doi:
|
|
27
|
+
keys.add(f"doi:{rec.ids.doi}")
|
|
28
|
+
if rec.ids.arxiv_id:
|
|
29
|
+
keys.add(f"arxiv:{rec.ids.arxiv_id.lower()}")
|
|
30
|
+
# Pool on the canonical ISBN only — deliberately NOT the whole `all_isbn13()` set. A book
|
|
31
|
+
# *chapter*'s scholarly record inherits its containing volume's ISBNs, so pooling on any shared
|
|
32
|
+
# ISBN would fuse sibling chapters (and the chapter with its parent book) into one record whose
|
|
33
|
+
# representative is picked by citation-richness, not title — which could bury the title-matching
|
|
34
|
+
# chapter. Same-work editions (print vs electronic) are still unified downstream by the
|
|
35
|
+
# SAME-OBJECT clustering at verdict time, which compares title+author, not ISBN membership alone.
|
|
36
|
+
if rec.ids.isbn13:
|
|
37
|
+
keys.add(f"isbn:{rec.ids.isbn13}")
|
|
38
|
+
if rec.openalex_work_id:
|
|
39
|
+
keys.add(f"oa:{rec.openalex_work_id}")
|
|
40
|
+
return keys
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _link_to_keys(link: str) -> set[str]:
|
|
44
|
+
"""Identifier keys referenced by a version-link string (URL or bare id)."""
|
|
45
|
+
keys: set[str] = set()
|
|
46
|
+
arxiv = extract_arxiv_id(None, None, fallback_text=link)
|
|
47
|
+
if arxiv:
|
|
48
|
+
keys.add(f"arxiv:{arxiv.lower()}")
|
|
49
|
+
doi = normalize_doi(link)
|
|
50
|
+
if doi and not doi.startswith("10.48550/arxiv"): # the arXiv DOI is already captured as arxiv:
|
|
51
|
+
keys.add(f"doi:{doi}")
|
|
52
|
+
return keys
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _link_keys(rec: SourceRecord) -> set[str]:
|
|
56
|
+
keys: set[str] = set()
|
|
57
|
+
for link in rec.version_links:
|
|
58
|
+
keys |= _link_to_keys(link)
|
|
59
|
+
return keys
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _merge_ids(a: Identifiers, b: Identifiers) -> Identifiers:
|
|
63
|
+
return Identifiers(
|
|
64
|
+
doi=a.doi or b.doi,
|
|
65
|
+
arxiv_id=a.arxiv_id or b.arxiv_id,
|
|
66
|
+
isbn13=a.isbn13 or b.isbn13,
|
|
67
|
+
isbn13s=tuple(dict.fromkeys((*a.isbn13s, *b.isbn13s, *a.all_isbn13(), *b.all_isbn13()))),
|
|
68
|
+
pmid=a.pmid or b.pmid,
|
|
69
|
+
bibcode=a.bibcode or b.bibcode,
|
|
70
|
+
openalex=a.openalex or b.openalex,
|
|
71
|
+
url=a.url or b.url,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _richness(rec: SourceRecord) -> tuple[int, int, int]:
|
|
76
|
+
return (1 if rec.ids.doi else 0, rec.citation_count, len(rec.authors))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _underlying_sources(rec: SourceRecord) -> list[str]:
|
|
80
|
+
"""Sources actually behind a record: the merge set if it is a pooled representative, else its own
|
|
81
|
+
source. Pooling must be idempotent — re-pooling an already-merged record (the enrichment pass)
|
|
82
|
+
must not lose provenance. (The prior two-phase pooling rebuilt representatives from earlier
|
|
83
|
+
representatives and ranked them by a single lossy `.source` label, so a preprint copy could
|
|
84
|
+
outrank a richer published record hidden inside another representative — the Flow-Lenia bug.)"""
|
|
85
|
+
merged_from = rec.raw.get("merged_from") if isinstance(rec.raw, dict) else None
|
|
86
|
+
return list(merged_from) if merged_from else [rec.source]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# Bibliographic fields are filled on the merged representative from the most authoritative source
|
|
90
|
+
# that carries them, independent of which record is the citation-richest. The publisher of record
|
|
91
|
+
# (the DOI landing-page citation export) outranks the aggregators; Crossref/OpenAlex (the
|
|
92
|
+
# registration-grade sources) outrank Semantic Scholar, whose venue strings are often truncated
|
|
93
|
+
# ('Complexity' → 'Complex') and which omits volume/issue/pages. Records are ranked by their *best
|
|
94
|
+
# underlying* source, so a pooled representative is ranked by the richest source inside it — not by
|
|
95
|
+
# its lossy `.source` label. This realizes the SPEC's "compile all available information". DBLP
|
|
96
|
+
# carries the conference of a CS paper that the aggregators often index only as its arXiv copy.
|
|
97
|
+
_FIELD_SOURCE_PRIORITY: dict[str, tuple[str, ...]] = {
|
|
98
|
+
"venue": ("publisher", "crossref", "openalex", "dblp", "openlibrary"),
|
|
99
|
+
"volume": ("publisher", "crossref", "openalex"),
|
|
100
|
+
"issue": ("publisher", "crossref", "openalex"),
|
|
101
|
+
"pages": ("publisher", "crossref", "openalex"),
|
|
102
|
+
"publisher": ("publisher", "crossref", "openlibrary", "openalex"),
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
_YEAR_SOURCE_PRIORITY = ("publisher", "crossref", "openalex")
|
|
107
|
+
|
|
108
|
+
# The author list scoring and adjudication see. The citation-richest record is often Semantic
|
|
109
|
+
# Scholar's, whose lists can be defective (a name twice, one missing: Neural Collapse's 'Hung Tran'
|
|
110
|
+
# for Stanley Osher), and the LLM rejected the real paper over it. DBLP and the registration-grade
|
|
111
|
+
# sources keep complete, clean lists; arXiv's is author-supplied.
|
|
112
|
+
_AUTHOR_SOURCE_PRIORITY = ("publisher", "dblp", "crossref", "arxiv", "openalex", "semantic_scholar")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _rank(rec: SourceRecord, priority: tuple[str, ...]) -> int:
|
|
116
|
+
"""Priority rank of a record by its best (lowest-rank) underlying source."""
|
|
117
|
+
return min(
|
|
118
|
+
(priority.index(s) if s in priority else len(priority))
|
|
119
|
+
for s in _underlying_sources(rec)
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _best_field(recs: list[SourceRecord], attr: str, priority: tuple[str, ...]) -> str:
|
|
124
|
+
"""Most authoritative non-empty value for `attr` across a same-work group ('' if none has it).
|
|
125
|
+
|
|
126
|
+
For the venue, a preprint server or repository ('arXiv (Cornell University)', DBLP's 'CoRR') is
|
|
127
|
+
used only when no member names a journal or conference: OpenAlex's arXiv copy of an ICLR paper
|
|
128
|
+
must not outrank DBLP's 'ICLR'.
|
|
129
|
+
"""
|
|
130
|
+
ranked = sorted(recs, key=lambda r: _rank(r, priority))
|
|
131
|
+
if attr == "venue":
|
|
132
|
+
ranked.sort(key=lambda r: is_repository_venue(r.venue)) # stable: rank kept within each
|
|
133
|
+
for r in ranked:
|
|
134
|
+
value = (getattr(r, attr) or "").strip()
|
|
135
|
+
if value:
|
|
136
|
+
return value
|
|
137
|
+
return ""
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _best_authors(recs: list[SourceRecord]) -> list[str]:
|
|
141
|
+
"""The author list of the most reliable source in a same-work group ([] if none has one)."""
|
|
142
|
+
for r in sorted(recs, key=lambda r: _rank(r, _AUTHOR_SOURCE_PRIORITY)):
|
|
143
|
+
if r.authors:
|
|
144
|
+
return list(r.authors)
|
|
145
|
+
return []
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _best_abstract(recs: list[SourceRecord]) -> str:
|
|
149
|
+
"""The fullest abstract across a same-work group (longest non-empty).
|
|
150
|
+
|
|
151
|
+
Abstracts come from OpenAlex / Semantic Scholar; the citation-richest representative may be a
|
|
152
|
+
source (e.g. Crossref) that carries none, so — like the other bibliographic fields — the abstract
|
|
153
|
+
is compiled across the group rather than taken only from the representative, so it survives onto
|
|
154
|
+
the matched artifact (and the cached verdict) for the citation-alignment check.
|
|
155
|
+
"""
|
|
156
|
+
return max(((r.abstract or "").strip() for r in recs), key=len, default="")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _best_year(recs: list[SourceRecord]) -> int | None:
|
|
160
|
+
"""Publication year from the registration-grade source (publisher/Crossref/OpenAlex) when
|
|
161
|
+
available.
|
|
162
|
+
|
|
163
|
+
Semantic Scholar often reports the online/preprint year (it gave 2018 for the 2019 Complex
|
|
164
|
+
Systems 'Lenia' DOI whose Crossref record is 2019), so its richer-citation record must not
|
|
165
|
+
override the DOI registrant's year.
|
|
166
|
+
"""
|
|
167
|
+
for r in sorted(recs, key=lambda r: _rank(r, _YEAR_SOURCE_PRIORITY)):
|
|
168
|
+
if r.year:
|
|
169
|
+
return r.year
|
|
170
|
+
return None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def pool_candidates(records: list[SourceRecord]) -> list[SourceRecord]:
|
|
174
|
+
"""Merge records that describe the same work into one representative candidate each.
|
|
175
|
+
|
|
176
|
+
A single union-find over all records with two kinds of (transitively closed) edge:
|
|
177
|
+
* identifier edge — both records carry ids and they overlap, or one side's version link
|
|
178
|
+
bridges the other's id (shared DOI/arXiv/ISBN13/OpenAlex id; the arXiv↔published edge).
|
|
179
|
+
Grounded in identifiers, never title fuzz.
|
|
180
|
+
* version edge — `_same_work_version`: a preprint and its published version, or two editions of
|
|
181
|
+
one book, agreeing on title+authors. This is the *only* place title similarity is allowed,
|
|
182
|
+
and only to UNITE versions of one work — two distinct published DOIs are still held apart by
|
|
183
|
+
the V1 veto inside `_same_work_version`.
|
|
184
|
+
|
|
185
|
+
Transitive closure (order-independent union-find) is essential and is why this replaced the
|
|
186
|
+
earlier greedy first-match grouping: a Semantic Scholar record that carries BOTH the arXiv id and
|
|
187
|
+
the published DOI must fuse the preprint group and the published group into one candidate. When
|
|
188
|
+
they stayed split, canonical fields were sourced from the preprint copy (the Flow-Lenia bug).
|
|
189
|
+
"""
|
|
190
|
+
n = len(records)
|
|
191
|
+
if n <= 1:
|
|
192
|
+
return list(records)
|
|
193
|
+
|
|
194
|
+
own = [_own_keys(r) for r in records]
|
|
195
|
+
links = [_link_keys(r) for r in records]
|
|
196
|
+
parent = list(range(n))
|
|
197
|
+
# The records in each cluster, keyed by its root.
|
|
198
|
+
cluster = {i: [r] for i, r in enumerate(records)}
|
|
199
|
+
|
|
200
|
+
def find(i: int) -> int:
|
|
201
|
+
while parent[i] != i:
|
|
202
|
+
parent[i] = parent[parent[i]]
|
|
203
|
+
i = parent[i]
|
|
204
|
+
return i
|
|
205
|
+
|
|
206
|
+
def union(i: int, j: int) -> None:
|
|
207
|
+
ri, rj = find(i), find(j)
|
|
208
|
+
parent[rj] = ri
|
|
209
|
+
cluster[ri].extend(cluster.pop(rj))
|
|
210
|
+
|
|
211
|
+
def authors_agree_across(i: int, j: int) -> bool:
|
|
212
|
+
"""A version relation joins two clusters only if every record of one has authors compatible
|
|
213
|
+
with every record of the other. A conference paper and its journal extension (CrossFormer at
|
|
214
|
+
ICLR, CrossFormer++ in TPAMI) are two works; fusing them hides the record that matches the
|
|
215
|
+
entry behind the other one's metadata. Checking each pair, not each edge, keeps a record that
|
|
216
|
+
fits both (a one-author supplementary-material DOI) from bridging them. Version *links* are
|
|
217
|
+
held to this too: they come from the aggregators' own work-merging, which can be wrong."""
|
|
218
|
+
return all(
|
|
219
|
+
authors_compatible(a.authors, b.authors)
|
|
220
|
+
for a in cluster[find(i)]
|
|
221
|
+
for b in cluster[find(j)]
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
for i in range(n):
|
|
225
|
+
for j in range(i + 1, n):
|
|
226
|
+
if find(i) == find(j):
|
|
227
|
+
continue
|
|
228
|
+
shared_id = bool(own[i] & own[j]) # the same DOI/arXiv/ISBN/OpenAlex id: one record
|
|
229
|
+
linked = bool(own[i] and own[j] and ((own[i] & links[j]) or (links[i] & own[j])))
|
|
230
|
+
if shared_id or (
|
|
231
|
+
(linked or _same_work_version(records[i], records[j]))
|
|
232
|
+
and authors_agree_across(i, j)
|
|
233
|
+
):
|
|
234
|
+
union(i, j)
|
|
235
|
+
|
|
236
|
+
clusters: dict[int, list[SourceRecord]] = {}
|
|
237
|
+
for idx in range(n):
|
|
238
|
+
clusters.setdefault(find(idx), []).append(records[idx])
|
|
239
|
+
return [_representative(g) if len(g) > 1 else g[0] for g in clusters.values()]
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _as_member(rec: SourceRecord) -> SourceRecord:
|
|
243
|
+
return rec.model_copy(update={"raw": {}, "abstract": "", "members": []}, deep=True)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _members(rec: SourceRecord) -> list[SourceRecord]:
|
|
247
|
+
"""The source records behind `rec`: its members if it is pooled, else itself."""
|
|
248
|
+
return list(rec.members) if rec.members else [_as_member(rec)]
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _representative(recs: list[SourceRecord]) -> SourceRecord:
|
|
252
|
+
best = max(recs, key=_richness)
|
|
253
|
+
merged = best.model_copy(deep=True)
|
|
254
|
+
for r in recs:
|
|
255
|
+
merged.ids = _merge_ids(merged.ids, r.ids)
|
|
256
|
+
for link in r.version_links:
|
|
257
|
+
if link not in merged.version_links:
|
|
258
|
+
merged.version_links.append(link)
|
|
259
|
+
if r.openalex_work_id and not merged.openalex_work_id:
|
|
260
|
+
merged.openalex_work_id = r.openalex_work_id
|
|
261
|
+
merged.is_preprint = merged.is_preprint and r.is_preprint
|
|
262
|
+
# Compile the best bibliographic metadata across the group (not just the richest record's).
|
|
263
|
+
for attr, priority in _FIELD_SOURCE_PRIORITY.items():
|
|
264
|
+
value = _best_field(recs, attr, priority)
|
|
265
|
+
if value:
|
|
266
|
+
setattr(merged, attr, value)
|
|
267
|
+
best_year = _best_year(recs)
|
|
268
|
+
if best_year is not None:
|
|
269
|
+
merged.year = best_year
|
|
270
|
+
best_authors = _best_authors(recs)
|
|
271
|
+
if best_authors:
|
|
272
|
+
merged.authors = best_authors
|
|
273
|
+
best_abstract = _best_abstract(recs)
|
|
274
|
+
if best_abstract:
|
|
275
|
+
merged.abstract = best_abstract
|
|
276
|
+
merged.raw = {
|
|
277
|
+
"merged_from": sorted({s for r in recs for s in _underlying_sources(r)}),
|
|
278
|
+
# `ids.doi` keeps one DOI (the richest record's, often an arXiv DataCite DOI); every member's
|
|
279
|
+
# DOI is kept here so a cited published DOI can still be recognised as this work's.
|
|
280
|
+
"merged_dois": sorted({d for r in recs for d in _member_dois(r)}),
|
|
281
|
+
}
|
|
282
|
+
merged.members = _distinct([m for r in recs for m in _members(r)])
|
|
283
|
+
return merged
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _distinct(members: list[SourceRecord]) -> list[SourceRecord]:
|
|
287
|
+
"""Members without repeats: enrichment re-pools a work with the same sources' by-id records."""
|
|
288
|
+
seen: set[tuple] = set()
|
|
289
|
+
out = []
|
|
290
|
+
for m in members:
|
|
291
|
+
key = (m.source, m.source_native_id) if m.source_native_id else (
|
|
292
|
+
m.source, m.ids.doi, m.ids.arxiv_id, m.title.casefold(), m.year
|
|
293
|
+
)
|
|
294
|
+
if key not in seen:
|
|
295
|
+
seen.add(key)
|
|
296
|
+
out.append(m)
|
|
297
|
+
return out
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _member_dois(rec: SourceRecord) -> list[str]:
|
|
301
|
+
"""The record's own DOI plus those of the records it already merged (pooling is re-run)."""
|
|
302
|
+
merged = rec.raw.get("merged_dois") if isinstance(rec.raw, dict) else None
|
|
303
|
+
return [*(merged or []), *([rec.ids.doi] if rec.ids.doi else [])]
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _is_preprintish(rec: SourceRecord) -> bool:
|
|
307
|
+
return rec.is_preprint or bool(rec.ids.doi and rec.ids.doi.startswith("10.48550/arxiv"))
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _published_doi(rec: SourceRecord) -> str | None:
|
|
311
|
+
doi = rec.ids.doi
|
|
312
|
+
return doi if doi and not doi.startswith("10.48550/arxiv") else None
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _title_authors_agree(a: SourceRecord, b: SourceRecord) -> bool:
|
|
316
|
+
if title_ratio(a.title, b.title) < _VERSION_MERGE_TITLE:
|
|
317
|
+
return False
|
|
318
|
+
if title_prefix_trap(a.title, b.title, tail_threshold=0.34):
|
|
319
|
+
return False # V3: shared prefix, divergent tail (bagrov vs kravchenko)
|
|
320
|
+
# Person-level, not a fuzzy surname average: short surnames (Wang, Chen, Lin, He, Liu) let
|
|
321
|
+
# CrossFormer's authors "overlap" CrossFormer++'s at 0.85 with two of seven people different.
|
|
322
|
+
# A title alone never makes a version: both records must name their authors.
|
|
323
|
+
return bool(a.authors and b.authors) and authors_compatible(a.authors, b.authors)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _same_work_version(a: SourceRecord, b: SourceRecord) -> bool:
|
|
327
|
+
"""Two records describe one work when titles+authors agree AND either:
|
|
328
|
+
|
|
329
|
+
R1 (preprint↔published): exactly one side is a preprint — a version relationship that overrides
|
|
330
|
+
the two-distinct-DOIs signal (the published DOI and the arXiv preprint DOI legitimately
|
|
331
|
+
differ); or
|
|
332
|
+
R2 (same edition/printing): the two do NOT both carry distinct *published* DOIs (one or both
|
|
333
|
+
lack one) — different editions/reprints of one book, or a metadata-only duplicate.
|
|
334
|
+
|
|
335
|
+
Two distinct *published* DOIs with neither side a preprint is left UNMERGED (the design's V1
|
|
336
|
+
veto): it is a genuine "same surface" case for M5's LLM tie-break (e.g. duplicate DOIs for one
|
|
337
|
+
paper), not a formal merge.
|
|
338
|
+
"""
|
|
339
|
+
if not _title_authors_agree(a, b):
|
|
340
|
+
return False
|
|
341
|
+
if _is_preprintish(a) != _is_preprintish(b): # R1
|
|
342
|
+
return True
|
|
343
|
+
pa, pb = _published_doi(a), _published_doi(b)
|
|
344
|
+
return not (pa and pb and pa != pb) # R2: merge unless two distinct published DOIs
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""SAME-OBJECT clustering over an entry's ACCEPTED candidates → matched artifacts.
|
|
2
|
+
|
|
3
|
+
`pool.py` already merged the identifier-grounded same-work cases (shared id, version edge, preprint
|
|
4
|
+
pair, same edition). What remains for an entry with ≥2 accepted candidates is the genuinely
|
|
5
|
+
ambiguous zone: two records with near-identical title+authors but two DISTINCT published DOIs (the
|
|
6
|
+
design's V1 veto holds them apart formally — e.g. one paper registered under two DOIs). Those, and
|
|
7
|
+
only those, escalate to the `SAME_WORK` LLM tie-break. The distinct-work vetoes V2 (disjoint pages),
|
|
8
|
+
V3 (title-prefix trap) and V4 (author-set distinct) are decisive and never reach the LLM.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import Literal
|
|
14
|
+
|
|
15
|
+
from reference_audit.cache.store import AuditCache, prompt_hash
|
|
16
|
+
from reference_audit.config import AuditConfig
|
|
17
|
+
from reference_audit.llm.client import LLMClient, LLMError
|
|
18
|
+
from reference_audit.llm.prompts import SAME_WORK_SYSTEM, same_work_user
|
|
19
|
+
from reference_audit.matching.features import (
|
|
20
|
+
author_set_jaccard,
|
|
21
|
+
id_agreement,
|
|
22
|
+
pages_conflict_records,
|
|
23
|
+
title_prefix_trap,
|
|
24
|
+
title_ratio,
|
|
25
|
+
)
|
|
26
|
+
from reference_audit.matching.names import author_overlap, author_subset
|
|
27
|
+
from reference_audit.models import (
|
|
28
|
+
CandidateAssessment,
|
|
29
|
+
Identifiers,
|
|
30
|
+
MatchedArtifact,
|
|
31
|
+
SameWorkResult,
|
|
32
|
+
SourceRecord,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
Relation = Literal["same", "distinct", "ambiguous"]
|
|
36
|
+
|
|
37
|
+
_SAME_TITLE = 0.90
|
|
38
|
+
_SAME_AUTHOR = 0.7
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def formal_relation(a: SourceRecord, b: SourceRecord, config: AuditConfig) -> Relation:
|
|
42
|
+
"""Formal same-object relation between two accepted records (no LLM)."""
|
|
43
|
+
if id_agreement(a.ids, b.ids) == "match":
|
|
44
|
+
return "same" # shared exact identifier (M1/M3)
|
|
45
|
+
# Distinct-work vetoes (decisive).
|
|
46
|
+
if pages_conflict_records(a, b):
|
|
47
|
+
return "distinct" # V2
|
|
48
|
+
if title_prefix_trap(a.title, b.title, tail_threshold=config.prefix_trap_tail_jaccard):
|
|
49
|
+
return "distinct" # V3
|
|
50
|
+
if (
|
|
51
|
+
author_set_jaccard(a.authors, b.authors) < config.author_set_distinct_jaccard
|
|
52
|
+
and not author_subset(a.authors, b.authors)
|
|
53
|
+
):
|
|
54
|
+
return "distinct" # V4
|
|
55
|
+
# Near-identical title + authors but no shared id ⇒ ambiguous (→ SAME_WORK LLM).
|
|
56
|
+
titles_agree = title_ratio(a.title, b.title) >= _SAME_TITLE
|
|
57
|
+
authors_agree = author_overlap(a.authors, b.authors) >= _SAME_AUTHOR or author_subset(
|
|
58
|
+
a.authors, b.authors
|
|
59
|
+
)
|
|
60
|
+
if titles_agree and authors_agree:
|
|
61
|
+
return "ambiguous"
|
|
62
|
+
return "distinct"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _merge_ids(a: Identifiers, b: Identifiers) -> Identifiers:
|
|
66
|
+
return Identifiers(
|
|
67
|
+
doi=a.doi or b.doi,
|
|
68
|
+
arxiv_id=a.arxiv_id or b.arxiv_id,
|
|
69
|
+
isbn13=a.isbn13 or b.isbn13,
|
|
70
|
+
isbn13s=tuple(dict.fromkeys((*a.isbn13s, *b.isbn13s, *a.all_isbn13(), *b.all_isbn13()))),
|
|
71
|
+
pmid=a.pmid or b.pmid,
|
|
72
|
+
bibcode=a.bibcode or b.bibcode,
|
|
73
|
+
openalex=a.openalex or b.openalex,
|
|
74
|
+
url=a.url or b.url,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _artifact_from_records(records: list[SourceRecord]) -> MatchedArtifact:
|
|
79
|
+
# Prefer a published (non-preprint), DOI-bearing, highly-cited record as the canonical best.
|
|
80
|
+
best = max(
|
|
81
|
+
records,
|
|
82
|
+
key=lambda r: (0 if r.is_preprint else 1, 1 if r.ids.doi else 0, r.citation_count),
|
|
83
|
+
)
|
|
84
|
+
merged = Identifiers()
|
|
85
|
+
for r in records:
|
|
86
|
+
merged = _merge_ids(merged, r.ids)
|
|
87
|
+
return MatchedArtifact(
|
|
88
|
+
records=list(records), merged_ids=merged, versions=list(records), best_record=best
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
async def _same_work_llm(
|
|
93
|
+
a: SourceRecord, b: SourceRecord, llm: LLMClient, cache: AuditCache | None
|
|
94
|
+
) -> tuple[SameWorkResult | None, bool]:
|
|
95
|
+
user = same_work_user(a, b)
|
|
96
|
+
p_hash = prompt_hash(SAME_WORK_SYSTEM + "\n" + user)
|
|
97
|
+
if cache is not None:
|
|
98
|
+
cached = cache.get_llm_decision(p_hash, "same_work")
|
|
99
|
+
if cached is not None:
|
|
100
|
+
return SameWorkResult.model_validate_json(cached), False
|
|
101
|
+
try:
|
|
102
|
+
result = await llm.structured(SAME_WORK_SYSTEM, user, SameWorkResult, "same_work")
|
|
103
|
+
except LLMError:
|
|
104
|
+
return None, True
|
|
105
|
+
if cache is not None:
|
|
106
|
+
cache.put_llm_decision(p_hash, "same_work", result.model_dump_json())
|
|
107
|
+
return result, False
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
async def cluster_accepted(
|
|
111
|
+
accepted: list[CandidateAssessment],
|
|
112
|
+
config: AuditConfig,
|
|
113
|
+
llm: LLMClient | None,
|
|
114
|
+
cache: AuditCache | None,
|
|
115
|
+
) -> tuple[list[MatchedArtifact], bool]:
|
|
116
|
+
"""Cluster accepted candidates into artifacts (union-find). Returns (artifacts, errored)."""
|
|
117
|
+
records = [c.record for c in accepted]
|
|
118
|
+
n = len(records)
|
|
119
|
+
if n <= 1:
|
|
120
|
+
return ([_artifact_from_records(records)] if records else []), False
|
|
121
|
+
|
|
122
|
+
parent = list(range(n))
|
|
123
|
+
|
|
124
|
+
def find(i: int) -> int:
|
|
125
|
+
while parent[i] != i:
|
|
126
|
+
parent[i] = parent[parent[i]]
|
|
127
|
+
i = parent[i]
|
|
128
|
+
return i
|
|
129
|
+
|
|
130
|
+
def union(i: int, j: int) -> None:
|
|
131
|
+
parent[find(j)] = find(i)
|
|
132
|
+
|
|
133
|
+
errored = False
|
|
134
|
+
for i in range(n):
|
|
135
|
+
for j in range(i + 1, n):
|
|
136
|
+
if find(i) == find(j):
|
|
137
|
+
continue
|
|
138
|
+
rel = formal_relation(records[i], records[j], config)
|
|
139
|
+
if rel == "same":
|
|
140
|
+
union(i, j)
|
|
141
|
+
elif rel == "ambiguous" and llm is not None:
|
|
142
|
+
result, err = await _same_work_llm(records[i], records[j], llm, cache)
|
|
143
|
+
errored = errored or err
|
|
144
|
+
if result and result.relation in ("same_artifact", "versions_of_same_work"):
|
|
145
|
+
union(i, j)
|
|
146
|
+
|
|
147
|
+
clusters: dict[int, list[SourceRecord]] = {}
|
|
148
|
+
for idx, rec in enumerate(records):
|
|
149
|
+
clusters.setdefault(find(idx), []).append(rec)
|
|
150
|
+
return [_artifact_from_records(group) for group in clusters.values()], errored
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Bucketing: auto_accept / auto_reject / adjudicate — the funnel before any LLM call.
|
|
2
|
+
|
|
3
|
+
The 100%-match short-circuit (README "unless a returned record is a 100% match, use an LLM")
|
|
4
|
+
reads the feature VECTOR, not the scalar: an id match alone is not enough (a hallucinated DOI that
|
|
5
|
+
resolves must still pass title+author), and a prefix-trap/pages-conflict forces adjudication even
|
|
6
|
+
at high composite (stops T2 from auto-accepting).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Literal
|
|
12
|
+
|
|
13
|
+
from reference_audit.config import AuditConfig
|
|
14
|
+
from reference_audit.models import FeatureVector
|
|
15
|
+
|
|
16
|
+
Bucket = Literal["auto_accept", "auto_reject", "adjudicate"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def bucket(features: FeatureVector, config: AuditConfig, *, entry_has_id: bool = True) -> Bucket:
|
|
20
|
+
"""Bucket one (entry, candidate) by its feature vector.
|
|
21
|
+
|
|
22
|
+
`entry_has_id` enables the strict backfill path: when the bib entry carries no *strong* anchor
|
|
23
|
+
(DOI / ISBN / arXiv / OpenAlex / Google Books — a bare URL does not count, see
|
|
24
|
+
`Identifiers.has_strong_id`), a near-exact title + strong authors (and no distinct-work veto) is
|
|
25
|
+
accepted so the matched DOI can be backfilled. The threshold is stricter than the id-based path
|
|
26
|
+
because there is no identifier to anchor the match.
|
|
27
|
+
"""
|
|
28
|
+
# A low author-set overlap forces adjudication ONLY when neither set is a subset of the other:
|
|
29
|
+
# a subset is consistent with an abbreviated "et al." list (same work), not a distinct work.
|
|
30
|
+
author_set_distinct = (
|
|
31
|
+
features.author_set_jaccard < config.author_set_distinct_jaccard
|
|
32
|
+
and not features.author_subset
|
|
33
|
+
)
|
|
34
|
+
authors_ok = features.author_overlap >= config.author_accept or features.author_subset
|
|
35
|
+
|
|
36
|
+
# A disjoint page range in the same venue normally signals a DISTINCT work (laughlin T2c: two
|
|
37
|
+
# different-titled papers on consecutive pages). But when the title is a near-exact match AND the
|
|
38
|
+
# authors agree, two different works is not a credible explanation — the likely one is a wrong
|
|
39
|
+
# `pages` field in the citation (soros2014: cited 306--313 vs the identical paper's canonical
|
|
40
|
+
# 793--800). Treat that as a field error to be REPORTED by the field check downstream, not as
|
|
41
|
+
# grounds to call a real, title-and-author-identical paper a possible hallucination. (prefix_trap
|
|
42
|
+
# stays load-bearing: it fires on divergent title TAILS, i.e. when the titles are NOT near-exact.)
|
|
43
|
+
title_authors_lock = features.title_ratio >= config.title_backfill and authors_ok
|
|
44
|
+
pages_conflict_distinct = features.pages_conflict and not title_authors_lock
|
|
45
|
+
forced_adjudicate = (
|
|
46
|
+
features.title_prefix_trap or pages_conflict_distinct or author_set_distinct
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# Path A — entry has an identifier that the candidate matches (authoritative; README: IDs
|
|
50
|
+
# uniquely identify). Accept even if the DB holds a subset of authors (wilson1974/Kogut); the
|
|
51
|
+
# title gate still rejects a hallucinated DOI resolving to a different paper (T4-by-bad-id).
|
|
52
|
+
if (
|
|
53
|
+
features.id_agreement == "match"
|
|
54
|
+
and features.title_ratio >= config.title_accept
|
|
55
|
+
and authors_ok
|
|
56
|
+
and not forced_adjudicate
|
|
57
|
+
):
|
|
58
|
+
return "auto_accept"
|
|
59
|
+
|
|
60
|
+
# Path B — entry has NO identifier; accept a near-exact title + strong authors to backfill the
|
|
61
|
+
# candidate's DOI/ISBN. Stricter title floor; all distinct-work vetoes still apply (keeps T2a's
|
|
62
|
+
# bagrov/kravchenko prefix-trap from auto-accepting).
|
|
63
|
+
if (
|
|
64
|
+
not entry_has_id
|
|
65
|
+
and features.id_agreement != "conflict"
|
|
66
|
+
and features.title_ratio >= config.title_backfill
|
|
67
|
+
and features.author_overlap >= config.author_accept
|
|
68
|
+
and not forced_adjudicate
|
|
69
|
+
):
|
|
70
|
+
return "auto_accept"
|
|
71
|
+
|
|
72
|
+
if features.id_agreement != "match" and features.composite < config.composite_reject:
|
|
73
|
+
return "auto_reject"
|
|
74
|
+
|
|
75
|
+
return "adjudicate"
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Turn scored + clustered candidates into a 3-way Verdict (README 5.1/5.2/5.3).
|
|
2
|
+
|
|
3
|
+
`artifacts` is the SAME-OBJECT clustering of the *accepted* candidates (see `sameobject.py`): each
|
|
4
|
+
element is one distinct real-world work. The verdict counts them.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from reference_audit.models import CandidateAssessment, MatchedArtifact, Verdict
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# @cpt-dod:cpt-referenceaudit-dod-identification-three-way-verdict:p1
|
|
13
|
+
# @cpt-dod:cpt-referenceaudit-dod-identification-hallucination-screen:p1
|
|
14
|
+
def build_verdict(
|
|
15
|
+
assessments: list[CandidateAssessment],
|
|
16
|
+
artifacts: list[MatchedArtifact],
|
|
17
|
+
*,
|
|
18
|
+
errored: bool,
|
|
19
|
+
) -> Verdict | None:
|
|
20
|
+
if artifacts:
|
|
21
|
+
if len(artifacts) == 1:
|
|
22
|
+
best = artifacts[0].best_record
|
|
23
|
+
src = best.source if best else ""
|
|
24
|
+
n_versions = len(artifacts[0].versions)
|
|
25
|
+
extra = f" ({n_versions} versions)" if n_versions > 1 else ""
|
|
26
|
+
return Verdict(
|
|
27
|
+
kind="exactly_one",
|
|
28
|
+
artifacts=artifacts,
|
|
29
|
+
confidence="high",
|
|
30
|
+
rationale=f"Matched a single work via {src}{extra}.",
|
|
31
|
+
)
|
|
32
|
+
return Verdict(
|
|
33
|
+
kind="multiple",
|
|
34
|
+
artifacts=artifacts,
|
|
35
|
+
confidence="medium",
|
|
36
|
+
rationale=f"Matched {len(artifacts)} distinct works.",
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
if errored:
|
|
40
|
+
return None # transient failure — leave unresolved, retry next run (error ≠ not-found)
|
|
41
|
+
|
|
42
|
+
if not assessments:
|
|
43
|
+
return Verdict(
|
|
44
|
+
kind="none",
|
|
45
|
+
confidence="high",
|
|
46
|
+
rationale="No database record found for this reference (possible hallucination).",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
if all(a.bucket == "auto_reject" for a in assessments):
|
|
50
|
+
return Verdict(
|
|
51
|
+
kind="none",
|
|
52
|
+
confidence="medium",
|
|
53
|
+
rationale="Database records were found but none matched the entry's title/authors.",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
return None # adjudicate bucket remaining — deferred to the LLM funnel
|