sciwrite-lint 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sciwrite_lint/__init__.py +3 -0
- sciwrite_lint/__main__.py +527 -0
- sciwrite_lint/_network.py +195 -0
- sciwrite_lint/api.py +1484 -0
- sciwrite_lint/checks/__init__.py +1 -0
- sciwrite_lint/checks/_section_utils.py +111 -0
- sciwrite_lint/checks/cite_purpose.py +122 -0
- sciwrite_lint/checks/claim_support.py +96 -0
- sciwrite_lint/checks/cross_section_consistency.py +185 -0
- sciwrite_lint/checks/dangling_cite.py +93 -0
- sciwrite_lint/checks/dangling_ref.py +116 -0
- sciwrite_lint/checks/full_paper_consistency.py +604 -0
- sciwrite_lint/checks/ref_internal_checks.py +919 -0
- sciwrite_lint/checks/reference_accuracy.py +277 -0
- sciwrite_lint/checks/reference_exists.py +119 -0
- sciwrite_lint/checks/reference_unreliable.py +244 -0
- sciwrite_lint/checks/registry.py +136 -0
- sciwrite_lint/checks/retracted_cite.py +96 -0
- sciwrite_lint/checks/structure_promises.py +115 -0
- sciwrite_lint/checks/unreferenced_figure.py +70 -0
- sciwrite_lint/claims.py +330 -0
- sciwrite_lint/claude_backend.py +94 -0
- sciwrite_lint/claude_cli.py +405 -0
- sciwrite_lint/cli/__init__.py +1 -0
- sciwrite_lint/cli/check.py +480 -0
- sciwrite_lint/cli/config.py +229 -0
- sciwrite_lint/cli/fetch.py +250 -0
- sciwrite_lint/cli/misc.py +1202 -0
- sciwrite_lint/cli/rank.py +470 -0
- sciwrite_lint/cli/verify.py +437 -0
- sciwrite_lint/config.py +646 -0
- sciwrite_lint/cross_paper.py +174 -0
- sciwrite_lint/eval_claims.py +1196 -0
- sciwrite_lint/fulltext.py +851 -0
- sciwrite_lint/llm_utils.py +386 -0
- sciwrite_lint/local_pdfs.py +122 -0
- sciwrite_lint/manuscript_store.py +674 -0
- sciwrite_lint/models.py +139 -0
- sciwrite_lint/pdf/__init__.py +1 -0
- sciwrite_lint/pdf/grobid.py +785 -0
- sciwrite_lint/pdf/pdf_download.py +258 -0
- sciwrite_lint/pipeline.py +2694 -0
- sciwrite_lint/prompt_safety.py +30 -0
- sciwrite_lint/rate_limiter.py +227 -0
- sciwrite_lint/references/__init__.py +1 -0
- sciwrite_lint/references/citations.py +715 -0
- sciwrite_lint/references/crossref.py +282 -0
- sciwrite_lint/references/embedding_store.py +380 -0
- sciwrite_lint/references/matching.py +273 -0
- sciwrite_lint/references/metadata.py +273 -0
- sciwrite_lint/references/reference_store.py +823 -0
- sciwrite_lint/references/retraction_watch.py +178 -0
- sciwrite_lint/references/workspace_db.py +1390 -0
- sciwrite_lint/report.py +163 -0
- sciwrite_lint/schemas.py +260 -0
- sciwrite_lint/scoring/__init__.py +1 -0
- sciwrite_lint/scoring/chain.py +716 -0
- sciwrite_lint/scoring/contribution.py +322 -0
- sciwrite_lint/scoring/scilint_score.py +611 -0
- sciwrite_lint/tex_parser.py +248 -0
- sciwrite_lint/usage.py +594 -0
- sciwrite_lint/vision/__init__.py +1 -0
- sciwrite_lint/vision/cache.py +140 -0
- sciwrite_lint/vision/describe.py +311 -0
- sciwrite_lint/vision/image_extraction.py +491 -0
- sciwrite_lint/vision/pipeline.py +207 -0
- sciwrite_lint/vllm/__init__.py +1 -0
- sciwrite_lint/vllm/metrics.py +157 -0
- sciwrite_lint/vllm/vllm_server.py +445 -0
- sciwrite_lint/web.py +369 -0
- sciwrite_lint-0.2.0.dist-info/METADATA +268 -0
- sciwrite_lint-0.2.0.dist-info/RECORD +76 -0
- sciwrite_lint-0.2.0.dist-info/WHEEL +5 -0
- sciwrite_lint-0.2.0.dist-info/entry_points.txt +2 -0
- sciwrite_lint-0.2.0.dist-info/licenses/LICENSE +21 -0
- sciwrite_lint-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,716 @@
|
|
|
1
|
+
"""Bibliography existence verification for cited references.
|
|
2
|
+
|
|
3
|
+
After GROBID parses each reference PDF and stores its bibliography entries
|
|
4
|
+
in workspace.db, this module batch-verifies those entries against
|
|
5
|
+
OpenAlex and Semantic Scholar to detect hallucinated references,
|
|
6
|
+
retracted papers, and metadata mismatches.
|
|
7
|
+
|
|
8
|
+
This runs as part of the default pipeline (no special flags needed).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
import sqlite3
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from pydantic import BaseModel, Field
|
|
19
|
+
|
|
20
|
+
from loguru import logger
|
|
21
|
+
|
|
22
|
+
from sciwrite_lint.config import LintConfig
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# ---------------------------------------------------------------------------
|
|
26
|
+
# Data types
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class RefBibCheck(BaseModel):
|
|
31
|
+
"""Result of checking a reference's own bibliography for existence and metadata."""
|
|
32
|
+
|
|
33
|
+
key: str
|
|
34
|
+
total_entries: int
|
|
35
|
+
found: int
|
|
36
|
+
not_found: int
|
|
37
|
+
retracted: int = 0
|
|
38
|
+
metadata_mismatches: int = 0
|
|
39
|
+
mismatch_details: list[str] = Field(default_factory=list)
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def hallucination_rate(self) -> float:
|
|
43
|
+
return self.not_found / self.total_entries if self.total_entries else 0.0
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def mismatch_rate(self) -> float:
|
|
47
|
+
return self.metadata_mismatches / self.found if self.found else 0.0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# ---------------------------------------------------------------------------
|
|
51
|
+
# Bibliography parsing helpers
|
|
52
|
+
# ---------------------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
# Pattern for references section entries like "1. Author (2020). Title..."
|
|
55
|
+
_REF_ENTRY_RE = re.compile(
|
|
56
|
+
r"^(\d+)\.\s+(.+?)(?:\.\s+|\n)",
|
|
57
|
+
re.MULTILINE,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
_TITLE_RE = re.compile(
|
|
61
|
+
r"^\d+\.\s+" # "1. "
|
|
62
|
+
r"(?:[A-Z][^.]*?\.\s+)*" # author block: "Author, A. et al. "
|
|
63
|
+
r"(?:\(\d{4}\)\.\s*)?" # optional year: "(2020). "
|
|
64
|
+
r"(.+?)(?:\.|$)", # title: everything up to next period
|
|
65
|
+
re.MULTILINE,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
_DOI_RE = re.compile(r"(?:doi:\s*|doi\.org/)(10\.\d{4,}/[^\s,;]+)", re.IGNORECASE)
|
|
69
|
+
_ARXIV_RE = re.compile(
|
|
70
|
+
r"(?:arXiv:\s*|arxiv\.org/abs/)(\d{4}\.\d{4,5}(?:v\d+)?)", re.IGNORECASE
|
|
71
|
+
)
|
|
72
|
+
_PMID_RE = re.compile(r"(?:PMID:\s*|pubmed/)(\d{6,9})", re.IGNORECASE)
|
|
73
|
+
_BRACKET_CITE_RE = re.compile(r"\[(\d+(?:[,;\s]+\d+)*)\]")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class ExtractedCitation(BaseModel):
|
|
77
|
+
"""A citation extracted from GROBID-parsed markdown."""
|
|
78
|
+
|
|
79
|
+
index: int
|
|
80
|
+
context: str
|
|
81
|
+
ref_entry: str = ""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def extract_citations_from_markdown(text: str) -> list[ExtractedCitation]:
|
|
85
|
+
"""Extract inline bracket citations from GROBID-parsed markdown.
|
|
86
|
+
|
|
87
|
+
Returns citations with their surrounding paragraph context.
|
|
88
|
+
Only extracts from body text (stops at References/Bibliography heading).
|
|
89
|
+
"""
|
|
90
|
+
bib_start = _find_bibliography_start(text)
|
|
91
|
+
body = text[:bib_start] if bib_start else text
|
|
92
|
+
bib_text = text[bib_start:] if bib_start else ""
|
|
93
|
+
bib_entries = _parse_bib_entries(bib_text)
|
|
94
|
+
|
|
95
|
+
results: list[ExtractedCitation] = []
|
|
96
|
+
seen: set[tuple[int, str]] = set()
|
|
97
|
+
|
|
98
|
+
for match in _BRACKET_CITE_RE.finditer(body):
|
|
99
|
+
indices_str = match.group(1)
|
|
100
|
+
start = max(0, body.rfind("\n\n", 0, match.start()))
|
|
101
|
+
end = body.find("\n\n", match.end())
|
|
102
|
+
if end == -1:
|
|
103
|
+
end = len(body)
|
|
104
|
+
context = body[start:end].strip()
|
|
105
|
+
|
|
106
|
+
for idx_str in re.split(r"[,;]\s*", indices_str):
|
|
107
|
+
idx_str = idx_str.strip()
|
|
108
|
+
if not idx_str.isdigit():
|
|
109
|
+
continue
|
|
110
|
+
idx = int(idx_str)
|
|
111
|
+
dedup_key = (idx, context[:100])
|
|
112
|
+
if dedup_key in seen:
|
|
113
|
+
continue
|
|
114
|
+
seen.add(dedup_key)
|
|
115
|
+
results.append(
|
|
116
|
+
ExtractedCitation(
|
|
117
|
+
index=idx,
|
|
118
|
+
context=context,
|
|
119
|
+
ref_entry=bib_entries.get(idx, ""),
|
|
120
|
+
)
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
return results
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class _BibEntry(BaseModel):
|
|
127
|
+
"""A parsed bibliography entry with extractable IDs and metadata."""
|
|
128
|
+
|
|
129
|
+
doi: str = ""
|
|
130
|
+
arxiv_id: str = ""
|
|
131
|
+
pmid: str = ""
|
|
132
|
+
title: str = ""
|
|
133
|
+
authors: list[str] = Field(default_factory=list)
|
|
134
|
+
year: str = ""
|
|
135
|
+
venue: str = ""
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def has_id(self) -> bool:
|
|
139
|
+
return bool(self.doi or self.arxiv_id or self.pmid)
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def has_metadata(self) -> bool:
|
|
143
|
+
"""Whether this entry has structured metadata for comparison."""
|
|
144
|
+
return bool(self.title and (self.authors or self.year))
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _prioritize_entries(entries: list[_BibEntry], limit: int) -> list[_BibEntry]:
|
|
148
|
+
"""Sort entries so formal-looking ones come first, then cap at *limit*.
|
|
149
|
+
|
|
150
|
+
Entries with structured IDs (DOI/arXiv/PMID) are most likely to be formal
|
|
151
|
+
academic papers and most useful for API verification. Entries with a title
|
|
152
|
+
but no ID can still be title-searched. Entries with neither are least
|
|
153
|
+
useful. Within each tier the original order is preserved (stable sort).
|
|
154
|
+
"""
|
|
155
|
+
entries.sort(
|
|
156
|
+
key=lambda e: (e.has_id, bool(e.title)),
|
|
157
|
+
reverse=True,
|
|
158
|
+
)
|
|
159
|
+
return entries[:limit]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _find_bibliography_start(text: str) -> int | None:
|
|
163
|
+
"""Find where the references/bibliography section starts."""
|
|
164
|
+
pattern = re.compile(
|
|
165
|
+
r"(?m)^#{1,3}\s+(?:References|Bibliography|Works Cited)\s*$",
|
|
166
|
+
re.IGNORECASE,
|
|
167
|
+
)
|
|
168
|
+
m = pattern.search(text)
|
|
169
|
+
return m.start() if m else None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _parse_bib_entries(bib_text: str) -> dict[int, str]:
|
|
173
|
+
"""Parse numbered bibliography entries."""
|
|
174
|
+
entries: dict[int, str] = {}
|
|
175
|
+
for match in _REF_ENTRY_RE.finditer(bib_text):
|
|
176
|
+
idx = int(match.group(1))
|
|
177
|
+
# Grab the full entry — up to next numbered entry or end
|
|
178
|
+
start = match.start()
|
|
179
|
+
next_match = _REF_ENTRY_RE.search(bib_text, match.end())
|
|
180
|
+
end = next_match.start() if next_match else len(bib_text)
|
|
181
|
+
entries[idx] = bib_text[start:end].strip()
|
|
182
|
+
return entries
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _extract_bib_entries(bib_entries: dict[int, str]) -> list[_BibEntry]:
|
|
186
|
+
"""Extract IDs and titles from GROBID bibliography entries.
|
|
187
|
+
|
|
188
|
+
Extracts DOIs, arXiv IDs, and PMIDs for batch verification.
|
|
189
|
+
Titles are used for entries without any structured ID.
|
|
190
|
+
"""
|
|
191
|
+
results: list[_BibEntry] = []
|
|
192
|
+
for _idx, entry in sorted(bib_entries.items()):
|
|
193
|
+
doi = ""
|
|
194
|
+
arxiv_id = ""
|
|
195
|
+
pmid = ""
|
|
196
|
+
title = ""
|
|
197
|
+
|
|
198
|
+
doi_match = _DOI_RE.search(entry)
|
|
199
|
+
if doi_match:
|
|
200
|
+
doi = doi_match.group(1).rstrip(".")
|
|
201
|
+
|
|
202
|
+
arxiv_match = _ARXIV_RE.search(entry)
|
|
203
|
+
if arxiv_match:
|
|
204
|
+
arxiv_id = arxiv_match.group(1)
|
|
205
|
+
|
|
206
|
+
pmid_match = _PMID_RE.search(entry)
|
|
207
|
+
if pmid_match:
|
|
208
|
+
pmid = pmid_match.group(1)
|
|
209
|
+
|
|
210
|
+
title_match = _TITLE_RE.match(entry)
|
|
211
|
+
if title_match:
|
|
212
|
+
t = title_match.group(1).strip()
|
|
213
|
+
if 10 < len(t) < 300:
|
|
214
|
+
title = t
|
|
215
|
+
|
|
216
|
+
if doi or arxiv_id or pmid or title:
|
|
217
|
+
results.append(
|
|
218
|
+
_BibEntry(doi=doi, arxiv_id=arxiv_id, pmid=pmid, title=title)
|
|
219
|
+
)
|
|
220
|
+
return results
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
# ---------------------------------------------------------------------------
|
|
224
|
+
# Metadata comparison helpers
|
|
225
|
+
# ---------------------------------------------------------------------------
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _oa_work_to_api_data(work: dict[str, Any]) -> dict[str, Any]:
|
|
229
|
+
"""Normalize an OpenAlex work record to a common metadata dict."""
|
|
230
|
+
authors: list[str] = []
|
|
231
|
+
for authorship in work.get("authorships") or []:
|
|
232
|
+
name = (authorship.get("author") or {}).get("display_name", "")
|
|
233
|
+
if name:
|
|
234
|
+
authors.append(name)
|
|
235
|
+
venue = ""
|
|
236
|
+
loc = work.get("primary_location") or {}
|
|
237
|
+
source = loc.get("source") or {}
|
|
238
|
+
venue = source.get("display_name") or ""
|
|
239
|
+
return {
|
|
240
|
+
"title": work.get("title") or "",
|
|
241
|
+
"authors": authors,
|
|
242
|
+
"year": str(work.get("publication_year") or ""),
|
|
243
|
+
"venue": venue,
|
|
244
|
+
"retracted": bool(work.get("is_retracted", False)),
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _s2_paper_to_api_data(paper: dict[str, Any]) -> dict[str, Any]:
|
|
249
|
+
"""Normalize a Semantic Scholar paper record to a common metadata dict."""
|
|
250
|
+
authors = [a.get("name", "") for a in (paper.get("authors") or []) if a.get("name")]
|
|
251
|
+
return {
|
|
252
|
+
"title": paper.get("title") or "",
|
|
253
|
+
"authors": authors,
|
|
254
|
+
"year": str(paper.get("year") or ""),
|
|
255
|
+
"venue": paper.get("venue") or "",
|
|
256
|
+
"retracted": bool(
|
|
257
|
+
paper.get("isRetracted", False)
|
|
258
|
+
or paper.get("publicationTypes", None) == ["Retracted"]
|
|
259
|
+
),
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _compare_bib_entry_metadata(
|
|
264
|
+
entry: _BibEntry, api_data: dict[str, Any]
|
|
265
|
+
) -> list[str]:
|
|
266
|
+
"""Compare a bibliography entry's metadata against API data.
|
|
267
|
+
|
|
268
|
+
Returns list of mismatch description strings (empty if all match).
|
|
269
|
+
"""
|
|
270
|
+
from sciwrite_lint.references.matching import (
|
|
271
|
+
AUTHOR_THRESHOLD,
|
|
272
|
+
TITLE_THRESHOLD,
|
|
273
|
+
VENUE_THRESHOLD,
|
|
274
|
+
author_similarity,
|
|
275
|
+
title_similarity,
|
|
276
|
+
venue_similarity,
|
|
277
|
+
year_match,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
issues: list[str] = []
|
|
281
|
+
|
|
282
|
+
api_title = api_data.get("title", "")
|
|
283
|
+
if entry.title and api_title:
|
|
284
|
+
sim = title_similarity(entry.title, api_title)
|
|
285
|
+
if sim < TITLE_THRESHOLD:
|
|
286
|
+
issues.append(
|
|
287
|
+
f"Title mismatch (sim={sim:.2f}): "
|
|
288
|
+
f"bib='{entry.title[:60]}', API='{api_title[:60]}'"
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
api_authors: list[str] = api_data.get("authors", [])
|
|
292
|
+
if entry.authors and api_authors:
|
|
293
|
+
sim = author_similarity(entry.authors, api_authors)
|
|
294
|
+
if sim < AUTHOR_THRESHOLD:
|
|
295
|
+
bib_str = ", ".join(entry.authors[:3])
|
|
296
|
+
api_str = ", ".join(api_authors[:3])
|
|
297
|
+
issues.append(
|
|
298
|
+
f"Author mismatch (sim={sim:.2f}): bib='{bib_str}', API='{api_str}'"
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
api_year = api_data.get("year", "")
|
|
302
|
+
if entry.year and api_year and not year_match(entry.year, api_year):
|
|
303
|
+
issues.append(f"Year mismatch: bib={entry.year}, API={api_year}")
|
|
304
|
+
|
|
305
|
+
api_venue = api_data.get("venue", "")
|
|
306
|
+
if entry.venue and api_venue:
|
|
307
|
+
sim = venue_similarity(entry.venue, api_venue)
|
|
308
|
+
if sim < VENUE_THRESHOLD:
|
|
309
|
+
issues.append(
|
|
310
|
+
f"Venue mismatch (sim={sim:.2f}): "
|
|
311
|
+
f"bib='{entry.venue[:60]}', API='{api_venue[:60]}'"
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
return issues
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
# ---------------------------------------------------------------------------
|
|
318
|
+
# BibVerifier — batch bibliography verification
|
|
319
|
+
# ---------------------------------------------------------------------------
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
class BibVerifier:
|
|
323
|
+
"""Collects bibliography entries from multiple references, then
|
|
324
|
+
batch-verifies all of them in minimal API calls.
|
|
325
|
+
|
|
326
|
+
Usage::
|
|
327
|
+
|
|
328
|
+
verifier = BibVerifier(config)
|
|
329
|
+
verifier.add_reference("smith2020", ref_text_smith)
|
|
330
|
+
verifier.add_reference("jones2021", ref_text_jones)
|
|
331
|
+
await verifier.verify_all()
|
|
332
|
+
result = verifier.get_result("smith2020") # RefBibCheck
|
|
333
|
+
|
|
334
|
+
Two-pass strategy across ALL collected entries:
|
|
335
|
+
1. Batch DOI lookup — all DOIs from all references in one OpenAlex
|
|
336
|
+
request (up to 200 per request, paginated if needed).
|
|
337
|
+
2. Parallel title search — remaining entries without DOIs.
|
|
338
|
+
|
|
339
|
+
10 refs × 30 entries = 300 total. If 60% have DOIs: 2 batch requests
|
|
340
|
+
+ ~120 title searches = ~122 requests instead of 300.
|
|
341
|
+
"""
|
|
342
|
+
|
|
343
|
+
def __init__(self, config: LintConfig | None = None) -> None:
|
|
344
|
+
self._config = config or LintConfig()
|
|
345
|
+
# {ref_key: list of entries}
|
|
346
|
+
self._entries: dict[str, list[_BibEntry]] = {}
|
|
347
|
+
# {ref_key: RefBibCheck} — populated after verify_all()
|
|
348
|
+
self._results: dict[str, RefBibCheck] = {}
|
|
349
|
+
|
|
350
|
+
# Maximum bibliography entries to verify per cited paper.
|
|
351
|
+
# Books and monographs can have hundreds of references;
|
|
352
|
+
# verifying all would be slow and provide diminishing signal.
|
|
353
|
+
MAX_BIB_ENTRIES = 100
|
|
354
|
+
|
|
355
|
+
def add_reference(self, ref_key: str, ref_text: str) -> None:
|
|
356
|
+
"""Extract and store bibliography entries from a parsed reference.
|
|
357
|
+
|
|
358
|
+
Regex-based extraction from markdown text. Prefer
|
|
359
|
+
``add_reference_from_registry`` when structured GROBID data is
|
|
360
|
+
available in workspace.db.
|
|
361
|
+
"""
|
|
362
|
+
bib_start = _find_bibliography_start(ref_text)
|
|
363
|
+
bib_text = ref_text[bib_start:] if bib_start else ""
|
|
364
|
+
raw_entries = _parse_bib_entries(bib_text)
|
|
365
|
+
entries = _extract_bib_entries(raw_entries)
|
|
366
|
+
if len(entries) > self.MAX_BIB_ENTRIES:
|
|
367
|
+
logger.info(
|
|
368
|
+
"Capping bibliography verification for {} ({} entries > {} limit)",
|
|
369
|
+
ref_key,
|
|
370
|
+
len(entries),
|
|
371
|
+
self.MAX_BIB_ENTRIES,
|
|
372
|
+
)
|
|
373
|
+
entries = _prioritize_entries(entries, self.MAX_BIB_ENTRIES)
|
|
374
|
+
self._entries[ref_key] = entries
|
|
375
|
+
|
|
376
|
+
def add_reference_from_registry(
|
|
377
|
+
self,
|
|
378
|
+
ref_key: str,
|
|
379
|
+
conn: "sqlite3.Connection",
|
|
380
|
+
) -> bool:
|
|
381
|
+
"""Load bibliography entries from workspace.db instead of parsing text.
|
|
382
|
+
|
|
383
|
+
Returns True if entries were found, False if registry has no data
|
|
384
|
+
for this parent_key (caller should use ``add_reference`` text path).
|
|
385
|
+
"""
|
|
386
|
+
from sciwrite_lint.references.workspace_db import load_bibliography_entries
|
|
387
|
+
|
|
388
|
+
rows = load_bibliography_entries(conn, parent_key=ref_key, depth=1)
|
|
389
|
+
if not rows:
|
|
390
|
+
return False
|
|
391
|
+
|
|
392
|
+
entries = [
|
|
393
|
+
_BibEntry(
|
|
394
|
+
doi=r.get("doi") or "",
|
|
395
|
+
arxiv_id=r.get("arxiv_id") or "",
|
|
396
|
+
pmid=r.get("pmid") or "",
|
|
397
|
+
title=r.get("title") or "",
|
|
398
|
+
authors=r.get("authors") or [],
|
|
399
|
+
year=r.get("year") or "",
|
|
400
|
+
venue=r.get("venue") or "",
|
|
401
|
+
)
|
|
402
|
+
for r in rows
|
|
403
|
+
]
|
|
404
|
+
if len(entries) > self.MAX_BIB_ENTRIES:
|
|
405
|
+
logger.info(
|
|
406
|
+
"Capping bibliography verification for {} ({} entries > {} limit)",
|
|
407
|
+
ref_key,
|
|
408
|
+
len(entries),
|
|
409
|
+
self.MAX_BIB_ENTRIES,
|
|
410
|
+
)
|
|
411
|
+
entries = _prioritize_entries(entries, self.MAX_BIB_ENTRIES)
|
|
412
|
+
self._entries[ref_key] = entries
|
|
413
|
+
logger.debug(
|
|
414
|
+
"{}: loaded {} bibliography entries from registry",
|
|
415
|
+
ref_key,
|
|
416
|
+
len(entries),
|
|
417
|
+
)
|
|
418
|
+
return True
|
|
419
|
+
|
|
420
|
+
def get_result(self, ref_key: str) -> RefBibCheck:
|
|
421
|
+
"""Get verification result for a reference. Call after verify_all()."""
|
|
422
|
+
return self._results.get(
|
|
423
|
+
ref_key, RefBibCheck(key=ref_key, total_entries=0, found=0, not_found=0)
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
async def verify_all(self) -> None:
|
|
427
|
+
"""Batch-verify all collected bibliography entries.
|
|
428
|
+
|
|
429
|
+
Three-pass strategy across ALL collected entries:
|
|
430
|
+
1. OpenAlex batch DOI (one request per 200 DOIs)
|
|
431
|
+
2. S2 batch for arXiv + PMID IDs (one request per 500 IDs)
|
|
432
|
+
3. Parallel title search for entries with no ID
|
|
433
|
+
|
|
434
|
+
When entries have structured metadata (from GROBID via registry),
|
|
435
|
+
also compares title/authors/year/venue against API responses.
|
|
436
|
+
"""
|
|
437
|
+
import asyncio
|
|
438
|
+
|
|
439
|
+
import httpx
|
|
440
|
+
|
|
441
|
+
from sciwrite_lint.rate_limiter import MonotonicRateLimiter
|
|
442
|
+
|
|
443
|
+
# Partition entries by best available ID
|
|
444
|
+
all_doi: list[tuple[str, _BibEntry]] = []
|
|
445
|
+
all_s2_id: list[tuple[str, _BibEntry]] = [] # arXiv + PMID
|
|
446
|
+
all_title_only: list[tuple[str, _BibEntry]] = []
|
|
447
|
+
|
|
448
|
+
for ref_key, entries in self._entries.items():
|
|
449
|
+
for e in entries:
|
|
450
|
+
if e.doi:
|
|
451
|
+
all_doi.append((ref_key, e))
|
|
452
|
+
elif e.arxiv_id or e.pmid:
|
|
453
|
+
all_s2_id.append((ref_key, e))
|
|
454
|
+
elif e.title:
|
|
455
|
+
all_title_only.append((ref_key, e))
|
|
456
|
+
|
|
457
|
+
# Track found entries per ref
|
|
458
|
+
ref_found: dict[str, int] = {}
|
|
459
|
+
# Track metadata mismatches: {ref_key: list of detail strings}
|
|
460
|
+
ref_mismatches: dict[str, list[str]] = {}
|
|
461
|
+
# Track retracted entries per ref
|
|
462
|
+
ref_retracted: dict[str, int] = {}
|
|
463
|
+
|
|
464
|
+
def _mark_found(rk: str) -> None:
|
|
465
|
+
ref_found[rk] = ref_found.get(rk, 0) + 1
|
|
466
|
+
|
|
467
|
+
def _check_metadata(
|
|
468
|
+
rk: str, entry: _BibEntry, api_data: dict[str, Any]
|
|
469
|
+
) -> None:
|
|
470
|
+
"""Compare entry metadata against API response, record mismatches."""
|
|
471
|
+
if api_data.get("retracted"):
|
|
472
|
+
ref_retracted[rk] = ref_retracted.get(rk, 0) + 1
|
|
473
|
+
ref_mismatches.setdefault(rk, []).append(
|
|
474
|
+
f"RETRACTED: '{entry.title[:60]}'" if entry.title else "RETRACTED"
|
|
475
|
+
)
|
|
476
|
+
if not entry.has_metadata:
|
|
477
|
+
return
|
|
478
|
+
issues = _compare_bib_entry_metadata(entry, api_data)
|
|
479
|
+
if issues:
|
|
480
|
+
ref_mismatches.setdefault(rk, []).extend(issues)
|
|
481
|
+
|
|
482
|
+
_OA_SELECT = (
|
|
483
|
+
"id,doi,title,authorships,publication_year,primary_location,is_retracted"
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
|
|
487
|
+
# Pass 1: OpenAlex batch DOI (one request per 200)
|
|
488
|
+
# Index API results by DOI for metadata comparison
|
|
489
|
+
doi_api_data: dict[str, dict[str, Any]] = {}
|
|
490
|
+
oa_params: dict[str, str | int] = {"select": _OA_SELECT}
|
|
491
|
+
if self._config.polite_email:
|
|
492
|
+
oa_params["mailto"] = self._config.polite_email
|
|
493
|
+
|
|
494
|
+
for i in range(0, len(all_doi), 200):
|
|
495
|
+
batch = all_doi[i : i + 200]
|
|
496
|
+
doi_filter = "|".join(f"https://doi.org/{e.doi}" for _, e in batch)
|
|
497
|
+
try:
|
|
498
|
+
resp = await client.get(
|
|
499
|
+
"https://api.openalex.org/works",
|
|
500
|
+
params={
|
|
501
|
+
**oa_params,
|
|
502
|
+
"filter": f"doi:{doi_filter}",
|
|
503
|
+
"per_page": 200,
|
|
504
|
+
},
|
|
505
|
+
)
|
|
506
|
+
if resp.status_code == 200:
|
|
507
|
+
for work in resp.json().get("results", []):
|
|
508
|
+
doi = (
|
|
509
|
+
(work.get("doi") or "")
|
|
510
|
+
.replace("https://doi.org/", "")
|
|
511
|
+
.lower()
|
|
512
|
+
)
|
|
513
|
+
if doi:
|
|
514
|
+
doi_api_data[doi] = _oa_work_to_api_data(work)
|
|
515
|
+
except httpx.HTTPError as e:
|
|
516
|
+
logger.debug("OpenAlex DOI batch lookup failed: {}", e)
|
|
517
|
+
|
|
518
|
+
for ref_key, entry in all_doi:
|
|
519
|
+
api_data = doi_api_data.get(entry.doi.lower())
|
|
520
|
+
if api_data:
|
|
521
|
+
_mark_found(ref_key)
|
|
522
|
+
_check_metadata(ref_key, entry, api_data)
|
|
523
|
+
elif entry.arxiv_id or entry.pmid:
|
|
524
|
+
all_s2_id.append((ref_key, entry))
|
|
525
|
+
elif entry.title:
|
|
526
|
+
all_title_only.append((ref_key, entry))
|
|
527
|
+
|
|
528
|
+
# Pass 2: S2 batch for arXiv + PMID (one request per 500)
|
|
529
|
+
if all_s2_id:
|
|
530
|
+
s2_ids: list[tuple[str, str]] = []
|
|
531
|
+
s2_entries: list[tuple[str, _BibEntry]] = []
|
|
532
|
+
for ref_key, entry in all_s2_id:
|
|
533
|
+
if entry.arxiv_id:
|
|
534
|
+
s2_ids.append((ref_key, f"ARXIV:{entry.arxiv_id}"))
|
|
535
|
+
s2_entries.append((ref_key, entry))
|
|
536
|
+
elif entry.pmid:
|
|
537
|
+
s2_ids.append((ref_key, f"PMID:{entry.pmid}"))
|
|
538
|
+
s2_entries.append((ref_key, entry))
|
|
539
|
+
|
|
540
|
+
for i in range(0, len(s2_ids), 500):
|
|
541
|
+
s2_batch = s2_ids[i : i + 500]
|
|
542
|
+
s2_entry_batch = s2_entries[i : i + 500]
|
|
543
|
+
ids = [sid for _, sid in s2_batch]
|
|
544
|
+
try:
|
|
545
|
+
resp = await client.post(
|
|
546
|
+
"https://api.semanticscholar.org/graph/v1/paper/batch",
|
|
547
|
+
json={"ids": ids},
|
|
548
|
+
params={
|
|
549
|
+
"fields": "externalIds,title,authors,year,venue,isRetracted",
|
|
550
|
+
},
|
|
551
|
+
)
|
|
552
|
+
if resp.status_code == 200:
|
|
553
|
+
papers = resp.json()
|
|
554
|
+
for paper, (rk, _), (_, entry) in zip(
|
|
555
|
+
papers, s2_batch, s2_entry_batch
|
|
556
|
+
):
|
|
557
|
+
if paper: # S2 returns null for not-found
|
|
558
|
+
_mark_found(rk)
|
|
559
|
+
api_data = _s2_paper_to_api_data(paper)
|
|
560
|
+
_check_metadata(rk, entry, api_data)
|
|
561
|
+
elif rk not in [r for r, _ in all_title_only]:
|
|
562
|
+
orig = next(
|
|
563
|
+
(
|
|
564
|
+
e
|
|
565
|
+
for r, e in all_s2_id
|
|
566
|
+
if r == rk and e.title
|
|
567
|
+
),
|
|
568
|
+
None,
|
|
569
|
+
)
|
|
570
|
+
if orig:
|
|
571
|
+
all_title_only.append((rk, orig))
|
|
572
|
+
except httpx.HTTPError as e:
|
|
573
|
+
logger.debug("S2 batch lookup failed: {}", e)
|
|
574
|
+
|
|
575
|
+
# Pass 3: parallel title search for entries with no ID
|
|
576
|
+
title_results: list[tuple[str, bool, _BibEntry]] = []
|
|
577
|
+
if all_title_only:
|
|
578
|
+
limiter = MonotonicRateLimiter(10, 0.15)
|
|
579
|
+
|
|
580
|
+
async def _check_title(
|
|
581
|
+
rk: str,
|
|
582
|
+
entry: _BibEntry,
|
|
583
|
+
) -> tuple[str, bool, _BibEntry]:
|
|
584
|
+
async with limiter:
|
|
585
|
+
pass
|
|
586
|
+
query = re.sub(r"[^\w\s-]", "", entry.title).strip()
|
|
587
|
+
if not query:
|
|
588
|
+
return rk, False, entry
|
|
589
|
+
try:
|
|
590
|
+
resp = await client.get(
|
|
591
|
+
"https://api.openalex.org/works",
|
|
592
|
+
params={
|
|
593
|
+
**oa_params,
|
|
594
|
+
"filter": f"title.search:{query}",
|
|
595
|
+
"per_page": 1,
|
|
596
|
+
},
|
|
597
|
+
)
|
|
598
|
+
if resp.status_code == 200:
|
|
599
|
+
results = resp.json().get("results", [])
|
|
600
|
+
if results:
|
|
601
|
+
api_data = _oa_work_to_api_data(results[0])
|
|
602
|
+
_check_metadata(rk, entry, api_data)
|
|
603
|
+
return rk, True, entry
|
|
604
|
+
except httpx.HTTPError as e:
|
|
605
|
+
logger.debug("OpenAlex title search failed for {}: {}", rk, e)
|
|
606
|
+
return rk, False, entry
|
|
607
|
+
|
|
608
|
+
title_results = await asyncio.gather(
|
|
609
|
+
*[_check_title(rk, e) for rk, e in all_title_only]
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
for rk, found, _entry in title_results:
|
|
613
|
+
if found:
|
|
614
|
+
_mark_found(rk)
|
|
615
|
+
|
|
616
|
+
# Build RefBibCheck per reference
|
|
617
|
+
for ref_key, entries in self._entries.items():
|
|
618
|
+
total = len(entries)
|
|
619
|
+
n_found = ref_found.get(ref_key, 0)
|
|
620
|
+
details = ref_mismatches.get(ref_key, [])
|
|
621
|
+
n_retracted = ref_retracted.get(ref_key, 0)
|
|
622
|
+
# Don't count retraction strings as metadata mismatches
|
|
623
|
+
n_mismatches = len(details) - n_retracted
|
|
624
|
+
self._results[ref_key] = RefBibCheck(
|
|
625
|
+
key=ref_key,
|
|
626
|
+
total_entries=total,
|
|
627
|
+
found=n_found,
|
|
628
|
+
not_found=total - n_found,
|
|
629
|
+
retracted=n_retracted,
|
|
630
|
+
metadata_mismatches=n_mismatches,
|
|
631
|
+
mismatch_details=details,
|
|
632
|
+
)
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
# ---------------------------------------------------------------------------
|
|
636
|
+
# Standalone bibliography verification (runs in default pipeline)
|
|
637
|
+
# ---------------------------------------------------------------------------
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
async def run_bib_verification(
|
|
641
|
+
references_dir: Path,
|
|
642
|
+
config: LintConfig | None = None,
|
|
643
|
+
) -> list[RefBibCheck]:
|
|
644
|
+
"""Verify bibliography entries for all parsed formal references.
|
|
645
|
+
|
|
646
|
+
Loads structured bibliography data from workspace.db (stored at parse
|
|
647
|
+
time), verifies existence + metadata + retraction via APIs, and returns
|
|
648
|
+
per-reference results.
|
|
649
|
+
|
|
650
|
+
This runs as part of the default pipeline (no ``--chain`` needed).
|
|
651
|
+
"""
|
|
652
|
+
config = config or LintConfig()
|
|
653
|
+
|
|
654
|
+
from sciwrite_lint.references.metadata import load_all_metadata
|
|
655
|
+
from sciwrite_lint.references.workspace_db import get_db, is_formal_cached_db
|
|
656
|
+
|
|
657
|
+
all_meta = load_all_metadata(references_dir)
|
|
658
|
+
|
|
659
|
+
verifier = BibVerifier(config)
|
|
660
|
+
refs_loaded = 0
|
|
661
|
+
|
|
662
|
+
with get_db(references_dir) as conn:
|
|
663
|
+
for key, meta in all_meta.items():
|
|
664
|
+
# Only formal PDFs have reliable bibliography entries
|
|
665
|
+
local_file = meta.access.get("local_file") or ""
|
|
666
|
+
if local_file.endswith(".pdf") and not is_formal_cached_db(conn, key):
|
|
667
|
+
continue
|
|
668
|
+
# Skip entries without local files (T3 references)
|
|
669
|
+
if not local_file:
|
|
670
|
+
continue
|
|
671
|
+
# Try structured registry data
|
|
672
|
+
if verifier.add_reference_from_registry(key, conn):
|
|
673
|
+
refs_loaded += 1
|
|
674
|
+
|
|
675
|
+
if not refs_loaded:
|
|
676
|
+
return []
|
|
677
|
+
|
|
678
|
+
logger.info(
|
|
679
|
+
"Bibliography verification: {} references with stored entries",
|
|
680
|
+
refs_loaded,
|
|
681
|
+
)
|
|
682
|
+
await verifier.verify_all()
|
|
683
|
+
|
|
684
|
+
results: list[RefBibCheck] = []
|
|
685
|
+
for key in verifier._entries:
|
|
686
|
+
bib_check = verifier.get_result(key)
|
|
687
|
+
if bib_check.total_entries > 0:
|
|
688
|
+
results.append(bib_check)
|
|
689
|
+
parts = [
|
|
690
|
+
f"{bib_check.found}/{bib_check.total_entries} found"
|
|
691
|
+
f" ({bib_check.hallucination_rate:.0%} hallucination)",
|
|
692
|
+
]
|
|
693
|
+
if bib_check.retracted:
|
|
694
|
+
parts.append(f"{bib_check.retracted} retracted")
|
|
695
|
+
if bib_check.metadata_mismatches:
|
|
696
|
+
parts.append(f"{bib_check.metadata_mismatches} metadata mismatches")
|
|
697
|
+
logger.info(" {}: bibliography {}", key, ", ".join(parts))
|
|
698
|
+
|
|
699
|
+
return results
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
# Keep single-reference convenience function for backward compatibility
|
|
703
|
+
async def verify_ref_bibliography(
|
|
704
|
+
ref_key: str,
|
|
705
|
+
ref_text: str,
|
|
706
|
+
config: LintConfig | None = None,
|
|
707
|
+
) -> RefBibCheck:
|
|
708
|
+
"""Check how many of a reference's own citations exist in OpenAlex.
|
|
709
|
+
|
|
710
|
+
Convenience wrapper around BibVerifier for single-reference use.
|
|
711
|
+
For multiple references, use BibVerifier directly to batch across refs.
|
|
712
|
+
"""
|
|
713
|
+
verifier = BibVerifier(config)
|
|
714
|
+
verifier.add_reference(ref_key, ref_text)
|
|
715
|
+
await verifier.verify_all()
|
|
716
|
+
return verifier.get_result(ref_key)
|