sciwrite-lint 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. sciwrite_lint/__init__.py +3 -0
  2. sciwrite_lint/__main__.py +527 -0
  3. sciwrite_lint/_network.py +195 -0
  4. sciwrite_lint/api.py +1484 -0
  5. sciwrite_lint/checks/__init__.py +1 -0
  6. sciwrite_lint/checks/_section_utils.py +111 -0
  7. sciwrite_lint/checks/cite_purpose.py +122 -0
  8. sciwrite_lint/checks/claim_support.py +96 -0
  9. sciwrite_lint/checks/cross_section_consistency.py +185 -0
  10. sciwrite_lint/checks/dangling_cite.py +93 -0
  11. sciwrite_lint/checks/dangling_ref.py +116 -0
  12. sciwrite_lint/checks/full_paper_consistency.py +604 -0
  13. sciwrite_lint/checks/ref_internal_checks.py +919 -0
  14. sciwrite_lint/checks/reference_accuracy.py +277 -0
  15. sciwrite_lint/checks/reference_exists.py +119 -0
  16. sciwrite_lint/checks/reference_unreliable.py +244 -0
  17. sciwrite_lint/checks/registry.py +136 -0
  18. sciwrite_lint/checks/retracted_cite.py +96 -0
  19. sciwrite_lint/checks/structure_promises.py +115 -0
  20. sciwrite_lint/checks/unreferenced_figure.py +70 -0
  21. sciwrite_lint/claims.py +330 -0
  22. sciwrite_lint/claude_backend.py +94 -0
  23. sciwrite_lint/claude_cli.py +405 -0
  24. sciwrite_lint/cli/__init__.py +1 -0
  25. sciwrite_lint/cli/check.py +480 -0
  26. sciwrite_lint/cli/config.py +229 -0
  27. sciwrite_lint/cli/fetch.py +250 -0
  28. sciwrite_lint/cli/misc.py +1202 -0
  29. sciwrite_lint/cli/rank.py +470 -0
  30. sciwrite_lint/cli/verify.py +437 -0
  31. sciwrite_lint/config.py +646 -0
  32. sciwrite_lint/cross_paper.py +174 -0
  33. sciwrite_lint/eval_claims.py +1196 -0
  34. sciwrite_lint/fulltext.py +851 -0
  35. sciwrite_lint/llm_utils.py +386 -0
  36. sciwrite_lint/local_pdfs.py +122 -0
  37. sciwrite_lint/manuscript_store.py +674 -0
  38. sciwrite_lint/models.py +139 -0
  39. sciwrite_lint/pdf/__init__.py +1 -0
  40. sciwrite_lint/pdf/grobid.py +785 -0
  41. sciwrite_lint/pdf/pdf_download.py +258 -0
  42. sciwrite_lint/pipeline.py +2694 -0
  43. sciwrite_lint/prompt_safety.py +30 -0
  44. sciwrite_lint/rate_limiter.py +227 -0
  45. sciwrite_lint/references/__init__.py +1 -0
  46. sciwrite_lint/references/citations.py +715 -0
  47. sciwrite_lint/references/crossref.py +282 -0
  48. sciwrite_lint/references/embedding_store.py +380 -0
  49. sciwrite_lint/references/matching.py +273 -0
  50. sciwrite_lint/references/metadata.py +273 -0
  51. sciwrite_lint/references/reference_store.py +823 -0
  52. sciwrite_lint/references/retraction_watch.py +178 -0
  53. sciwrite_lint/references/workspace_db.py +1390 -0
  54. sciwrite_lint/report.py +163 -0
  55. sciwrite_lint/schemas.py +260 -0
  56. sciwrite_lint/scoring/__init__.py +1 -0
  57. sciwrite_lint/scoring/chain.py +716 -0
  58. sciwrite_lint/scoring/contribution.py +322 -0
  59. sciwrite_lint/scoring/scilint_score.py +611 -0
  60. sciwrite_lint/tex_parser.py +248 -0
  61. sciwrite_lint/usage.py +594 -0
  62. sciwrite_lint/vision/__init__.py +1 -0
  63. sciwrite_lint/vision/cache.py +140 -0
  64. sciwrite_lint/vision/describe.py +311 -0
  65. sciwrite_lint/vision/image_extraction.py +491 -0
  66. sciwrite_lint/vision/pipeline.py +207 -0
  67. sciwrite_lint/vllm/__init__.py +1 -0
  68. sciwrite_lint/vllm/metrics.py +157 -0
  69. sciwrite_lint/vllm/vllm_server.py +445 -0
  70. sciwrite_lint/web.py +369 -0
  71. sciwrite_lint-0.2.0.dist-info/METADATA +268 -0
  72. sciwrite_lint-0.2.0.dist-info/RECORD +76 -0
  73. sciwrite_lint-0.2.0.dist-info/WHEEL +5 -0
  74. sciwrite_lint-0.2.0.dist-info/entry_points.txt +2 -0
  75. sciwrite_lint-0.2.0.dist-info/licenses/LICENSE +21 -0
  76. sciwrite_lint-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,716 @@
1
+ """Bibliography existence verification for cited references.
2
+
3
+ After GROBID parses each reference PDF and stores its bibliography entries
4
+ in workspace.db, this module batch-verifies those entries against
5
+ OpenAlex and Semantic Scholar to detect hallucinated references,
6
+ retracted papers, and metadata mismatches.
7
+
8
+ This runs as part of the default pipeline (no special flags needed).
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import re
14
+ import sqlite3
15
+ from pathlib import Path
16
+ from typing import Any
17
+
18
+ from pydantic import BaseModel, Field
19
+
20
+ from loguru import logger
21
+
22
+ from sciwrite_lint.config import LintConfig
23
+
24
+
25
+ # ---------------------------------------------------------------------------
26
+ # Data types
27
+ # ---------------------------------------------------------------------------
28
+
29
+
30
+ class RefBibCheck(BaseModel):
31
+ """Result of checking a reference's own bibliography for existence and metadata."""
32
+
33
+ key: str
34
+ total_entries: int
35
+ found: int
36
+ not_found: int
37
+ retracted: int = 0
38
+ metadata_mismatches: int = 0
39
+ mismatch_details: list[str] = Field(default_factory=list)
40
+
41
+ @property
42
+ def hallucination_rate(self) -> float:
43
+ return self.not_found / self.total_entries if self.total_entries else 0.0
44
+
45
+ @property
46
+ def mismatch_rate(self) -> float:
47
+ return self.metadata_mismatches / self.found if self.found else 0.0
48
+
49
+
50
+ # ---------------------------------------------------------------------------
51
+ # Bibliography parsing helpers
52
+ # ---------------------------------------------------------------------------
53
+
54
+ # Pattern for references section entries like "1. Author (2020). Title..."
55
+ _REF_ENTRY_RE = re.compile(
56
+ r"^(\d+)\.\s+(.+?)(?:\.\s+|\n)",
57
+ re.MULTILINE,
58
+ )
59
+
60
+ _TITLE_RE = re.compile(
61
+ r"^\d+\.\s+" # "1. "
62
+ r"(?:[A-Z][^.]*?\.\s+)*" # author block: "Author, A. et al. "
63
+ r"(?:\(\d{4}\)\.\s*)?" # optional year: "(2020). "
64
+ r"(.+?)(?:\.|$)", # title: everything up to next period
65
+ re.MULTILINE,
66
+ )
67
+
68
+ _DOI_RE = re.compile(r"(?:doi:\s*|doi\.org/)(10\.\d{4,}/[^\s,;]+)", re.IGNORECASE)
69
+ _ARXIV_RE = re.compile(
70
+ r"(?:arXiv:\s*|arxiv\.org/abs/)(\d{4}\.\d{4,5}(?:v\d+)?)", re.IGNORECASE
71
+ )
72
+ _PMID_RE = re.compile(r"(?:PMID:\s*|pubmed/)(\d{6,9})", re.IGNORECASE)
73
+ _BRACKET_CITE_RE = re.compile(r"\[(\d+(?:[,;\s]+\d+)*)\]")
74
+
75
+
76
+ class ExtractedCitation(BaseModel):
77
+ """A citation extracted from GROBID-parsed markdown."""
78
+
79
+ index: int
80
+ context: str
81
+ ref_entry: str = ""
82
+
83
+
84
+ def extract_citations_from_markdown(text: str) -> list[ExtractedCitation]:
85
+ """Extract inline bracket citations from GROBID-parsed markdown.
86
+
87
+ Returns citations with their surrounding paragraph context.
88
+ Only extracts from body text (stops at References/Bibliography heading).
89
+ """
90
+ bib_start = _find_bibliography_start(text)
91
+ body = text[:bib_start] if bib_start else text
92
+ bib_text = text[bib_start:] if bib_start else ""
93
+ bib_entries = _parse_bib_entries(bib_text)
94
+
95
+ results: list[ExtractedCitation] = []
96
+ seen: set[tuple[int, str]] = set()
97
+
98
+ for match in _BRACKET_CITE_RE.finditer(body):
99
+ indices_str = match.group(1)
100
+ start = max(0, body.rfind("\n\n", 0, match.start()))
101
+ end = body.find("\n\n", match.end())
102
+ if end == -1:
103
+ end = len(body)
104
+ context = body[start:end].strip()
105
+
106
+ for idx_str in re.split(r"[,;]\s*", indices_str):
107
+ idx_str = idx_str.strip()
108
+ if not idx_str.isdigit():
109
+ continue
110
+ idx = int(idx_str)
111
+ dedup_key = (idx, context[:100])
112
+ if dedup_key in seen:
113
+ continue
114
+ seen.add(dedup_key)
115
+ results.append(
116
+ ExtractedCitation(
117
+ index=idx,
118
+ context=context,
119
+ ref_entry=bib_entries.get(idx, ""),
120
+ )
121
+ )
122
+
123
+ return results
124
+
125
+
126
+ class _BibEntry(BaseModel):
127
+ """A parsed bibliography entry with extractable IDs and metadata."""
128
+
129
+ doi: str = ""
130
+ arxiv_id: str = ""
131
+ pmid: str = ""
132
+ title: str = ""
133
+ authors: list[str] = Field(default_factory=list)
134
+ year: str = ""
135
+ venue: str = ""
136
+
137
+ @property
138
+ def has_id(self) -> bool:
139
+ return bool(self.doi or self.arxiv_id or self.pmid)
140
+
141
+ @property
142
+ def has_metadata(self) -> bool:
143
+ """Whether this entry has structured metadata for comparison."""
144
+ return bool(self.title and (self.authors or self.year))
145
+
146
+
147
+ def _prioritize_entries(entries: list[_BibEntry], limit: int) -> list[_BibEntry]:
148
+ """Sort entries so formal-looking ones come first, then cap at *limit*.
149
+
150
+ Entries with structured IDs (DOI/arXiv/PMID) are most likely to be formal
151
+ academic papers and most useful for API verification. Entries with a title
152
+ but no ID can still be title-searched. Entries with neither are least
153
+ useful. Within each tier the original order is preserved (stable sort).
154
+ """
155
+ entries.sort(
156
+ key=lambda e: (e.has_id, bool(e.title)),
157
+ reverse=True,
158
+ )
159
+ return entries[:limit]
160
+
161
+
162
+ def _find_bibliography_start(text: str) -> int | None:
163
+ """Find where the references/bibliography section starts."""
164
+ pattern = re.compile(
165
+ r"(?m)^#{1,3}\s+(?:References|Bibliography|Works Cited)\s*$",
166
+ re.IGNORECASE,
167
+ )
168
+ m = pattern.search(text)
169
+ return m.start() if m else None
170
+
171
+
172
+ def _parse_bib_entries(bib_text: str) -> dict[int, str]:
173
+ """Parse numbered bibliography entries."""
174
+ entries: dict[int, str] = {}
175
+ for match in _REF_ENTRY_RE.finditer(bib_text):
176
+ idx = int(match.group(1))
177
+ # Grab the full entry — up to next numbered entry or end
178
+ start = match.start()
179
+ next_match = _REF_ENTRY_RE.search(bib_text, match.end())
180
+ end = next_match.start() if next_match else len(bib_text)
181
+ entries[idx] = bib_text[start:end].strip()
182
+ return entries
183
+
184
+
185
+ def _extract_bib_entries(bib_entries: dict[int, str]) -> list[_BibEntry]:
186
+ """Extract IDs and titles from GROBID bibliography entries.
187
+
188
+ Extracts DOIs, arXiv IDs, and PMIDs for batch verification.
189
+ Titles are used for entries without any structured ID.
190
+ """
191
+ results: list[_BibEntry] = []
192
+ for _idx, entry in sorted(bib_entries.items()):
193
+ doi = ""
194
+ arxiv_id = ""
195
+ pmid = ""
196
+ title = ""
197
+
198
+ doi_match = _DOI_RE.search(entry)
199
+ if doi_match:
200
+ doi = doi_match.group(1).rstrip(".")
201
+
202
+ arxiv_match = _ARXIV_RE.search(entry)
203
+ if arxiv_match:
204
+ arxiv_id = arxiv_match.group(1)
205
+
206
+ pmid_match = _PMID_RE.search(entry)
207
+ if pmid_match:
208
+ pmid = pmid_match.group(1)
209
+
210
+ title_match = _TITLE_RE.match(entry)
211
+ if title_match:
212
+ t = title_match.group(1).strip()
213
+ if 10 < len(t) < 300:
214
+ title = t
215
+
216
+ if doi or arxiv_id or pmid or title:
217
+ results.append(
218
+ _BibEntry(doi=doi, arxiv_id=arxiv_id, pmid=pmid, title=title)
219
+ )
220
+ return results
221
+
222
+
223
+ # ---------------------------------------------------------------------------
224
+ # Metadata comparison helpers
225
+ # ---------------------------------------------------------------------------
226
+
227
+
228
+ def _oa_work_to_api_data(work: dict[str, Any]) -> dict[str, Any]:
229
+ """Normalize an OpenAlex work record to a common metadata dict."""
230
+ authors: list[str] = []
231
+ for authorship in work.get("authorships") or []:
232
+ name = (authorship.get("author") or {}).get("display_name", "")
233
+ if name:
234
+ authors.append(name)
235
+ venue = ""
236
+ loc = work.get("primary_location") or {}
237
+ source = loc.get("source") or {}
238
+ venue = source.get("display_name") or ""
239
+ return {
240
+ "title": work.get("title") or "",
241
+ "authors": authors,
242
+ "year": str(work.get("publication_year") or ""),
243
+ "venue": venue,
244
+ "retracted": bool(work.get("is_retracted", False)),
245
+ }
246
+
247
+
248
+ def _s2_paper_to_api_data(paper: dict[str, Any]) -> dict[str, Any]:
249
+ """Normalize a Semantic Scholar paper record to a common metadata dict."""
250
+ authors = [a.get("name", "") for a in (paper.get("authors") or []) if a.get("name")]
251
+ return {
252
+ "title": paper.get("title") or "",
253
+ "authors": authors,
254
+ "year": str(paper.get("year") or ""),
255
+ "venue": paper.get("venue") or "",
256
+ "retracted": bool(
257
+ paper.get("isRetracted", False)
258
+ or paper.get("publicationTypes", None) == ["Retracted"]
259
+ ),
260
+ }
261
+
262
+
263
+ def _compare_bib_entry_metadata(
264
+ entry: _BibEntry, api_data: dict[str, Any]
265
+ ) -> list[str]:
266
+ """Compare a bibliography entry's metadata against API data.
267
+
268
+ Returns list of mismatch description strings (empty if all match).
269
+ """
270
+ from sciwrite_lint.references.matching import (
271
+ AUTHOR_THRESHOLD,
272
+ TITLE_THRESHOLD,
273
+ VENUE_THRESHOLD,
274
+ author_similarity,
275
+ title_similarity,
276
+ venue_similarity,
277
+ year_match,
278
+ )
279
+
280
+ issues: list[str] = []
281
+
282
+ api_title = api_data.get("title", "")
283
+ if entry.title and api_title:
284
+ sim = title_similarity(entry.title, api_title)
285
+ if sim < TITLE_THRESHOLD:
286
+ issues.append(
287
+ f"Title mismatch (sim={sim:.2f}): "
288
+ f"bib='{entry.title[:60]}', API='{api_title[:60]}'"
289
+ )
290
+
291
+ api_authors: list[str] = api_data.get("authors", [])
292
+ if entry.authors and api_authors:
293
+ sim = author_similarity(entry.authors, api_authors)
294
+ if sim < AUTHOR_THRESHOLD:
295
+ bib_str = ", ".join(entry.authors[:3])
296
+ api_str = ", ".join(api_authors[:3])
297
+ issues.append(
298
+ f"Author mismatch (sim={sim:.2f}): bib='{bib_str}', API='{api_str}'"
299
+ )
300
+
301
+ api_year = api_data.get("year", "")
302
+ if entry.year and api_year and not year_match(entry.year, api_year):
303
+ issues.append(f"Year mismatch: bib={entry.year}, API={api_year}")
304
+
305
+ api_venue = api_data.get("venue", "")
306
+ if entry.venue and api_venue:
307
+ sim = venue_similarity(entry.venue, api_venue)
308
+ if sim < VENUE_THRESHOLD:
309
+ issues.append(
310
+ f"Venue mismatch (sim={sim:.2f}): "
311
+ f"bib='{entry.venue[:60]}', API='{api_venue[:60]}'"
312
+ )
313
+
314
+ return issues
315
+
316
+
317
+ # ---------------------------------------------------------------------------
318
+ # BibVerifier — batch bibliography verification
319
+ # ---------------------------------------------------------------------------
320
+
321
+
322
+ class BibVerifier:
323
+ """Collects bibliography entries from multiple references, then
324
+ batch-verifies all of them in minimal API calls.
325
+
326
+ Usage::
327
+
328
+ verifier = BibVerifier(config)
329
+ verifier.add_reference("smith2020", ref_text_smith)
330
+ verifier.add_reference("jones2021", ref_text_jones)
331
+ await verifier.verify_all()
332
+ result = verifier.get_result("smith2020") # RefBibCheck
333
+
334
+ Two-pass strategy across ALL collected entries:
335
+ 1. Batch DOI lookup — all DOIs from all references in one OpenAlex
336
+ request (up to 200 per request, paginated if needed).
337
+ 2. Parallel title search — remaining entries without DOIs.
338
+
339
+ 10 refs × 30 entries = 300 total. If 60% have DOIs: 2 batch requests
340
+ + ~120 title searches = ~122 requests instead of 300.
341
+ """
342
+
343
+ def __init__(self, config: LintConfig | None = None) -> None:
344
+ self._config = config or LintConfig()
345
+ # {ref_key: list of entries}
346
+ self._entries: dict[str, list[_BibEntry]] = {}
347
+ # {ref_key: RefBibCheck} — populated after verify_all()
348
+ self._results: dict[str, RefBibCheck] = {}
349
+
350
+ # Maximum bibliography entries to verify per cited paper.
351
+ # Books and monographs can have hundreds of references;
352
+ # verifying all would be slow and provide diminishing signal.
353
+ MAX_BIB_ENTRIES = 100
354
+
355
+ def add_reference(self, ref_key: str, ref_text: str) -> None:
356
+ """Extract and store bibliography entries from a parsed reference.
357
+
358
+ Regex-based extraction from markdown text. Prefer
359
+ ``add_reference_from_registry`` when structured GROBID data is
360
+ available in workspace.db.
361
+ """
362
+ bib_start = _find_bibliography_start(ref_text)
363
+ bib_text = ref_text[bib_start:] if bib_start else ""
364
+ raw_entries = _parse_bib_entries(bib_text)
365
+ entries = _extract_bib_entries(raw_entries)
366
+ if len(entries) > self.MAX_BIB_ENTRIES:
367
+ logger.info(
368
+ "Capping bibliography verification for {} ({} entries > {} limit)",
369
+ ref_key,
370
+ len(entries),
371
+ self.MAX_BIB_ENTRIES,
372
+ )
373
+ entries = _prioritize_entries(entries, self.MAX_BIB_ENTRIES)
374
+ self._entries[ref_key] = entries
375
+
376
+ def add_reference_from_registry(
377
+ self,
378
+ ref_key: str,
379
+ conn: "sqlite3.Connection",
380
+ ) -> bool:
381
+ """Load bibliography entries from workspace.db instead of parsing text.
382
+
383
+ Returns True if entries were found, False if registry has no data
384
+ for this parent_key (caller should use ``add_reference`` text path).
385
+ """
386
+ from sciwrite_lint.references.workspace_db import load_bibliography_entries
387
+
388
+ rows = load_bibliography_entries(conn, parent_key=ref_key, depth=1)
389
+ if not rows:
390
+ return False
391
+
392
+ entries = [
393
+ _BibEntry(
394
+ doi=r.get("doi") or "",
395
+ arxiv_id=r.get("arxiv_id") or "",
396
+ pmid=r.get("pmid") or "",
397
+ title=r.get("title") or "",
398
+ authors=r.get("authors") or [],
399
+ year=r.get("year") or "",
400
+ venue=r.get("venue") or "",
401
+ )
402
+ for r in rows
403
+ ]
404
+ if len(entries) > self.MAX_BIB_ENTRIES:
405
+ logger.info(
406
+ "Capping bibliography verification for {} ({} entries > {} limit)",
407
+ ref_key,
408
+ len(entries),
409
+ self.MAX_BIB_ENTRIES,
410
+ )
411
+ entries = _prioritize_entries(entries, self.MAX_BIB_ENTRIES)
412
+ self._entries[ref_key] = entries
413
+ logger.debug(
414
+ "{}: loaded {} bibliography entries from registry",
415
+ ref_key,
416
+ len(entries),
417
+ )
418
+ return True
419
+
420
+ def get_result(self, ref_key: str) -> RefBibCheck:
421
+ """Get verification result for a reference. Call after verify_all()."""
422
+ return self._results.get(
423
+ ref_key, RefBibCheck(key=ref_key, total_entries=0, found=0, not_found=0)
424
+ )
425
+
426
+ async def verify_all(self) -> None:
427
+ """Batch-verify all collected bibliography entries.
428
+
429
+ Three-pass strategy across ALL collected entries:
430
+ 1. OpenAlex batch DOI (one request per 200 DOIs)
431
+ 2. S2 batch for arXiv + PMID IDs (one request per 500 IDs)
432
+ 3. Parallel title search for entries with no ID
433
+
434
+ When entries have structured metadata (from GROBID via registry),
435
+ also compares title/authors/year/venue against API responses.
436
+ """
437
+ import asyncio
438
+
439
+ import httpx
440
+
441
+ from sciwrite_lint.rate_limiter import MonotonicRateLimiter
442
+
443
+ # Partition entries by best available ID
444
+ all_doi: list[tuple[str, _BibEntry]] = []
445
+ all_s2_id: list[tuple[str, _BibEntry]] = [] # arXiv + PMID
446
+ all_title_only: list[tuple[str, _BibEntry]] = []
447
+
448
+ for ref_key, entries in self._entries.items():
449
+ for e in entries:
450
+ if e.doi:
451
+ all_doi.append((ref_key, e))
452
+ elif e.arxiv_id or e.pmid:
453
+ all_s2_id.append((ref_key, e))
454
+ elif e.title:
455
+ all_title_only.append((ref_key, e))
456
+
457
+ # Track found entries per ref
458
+ ref_found: dict[str, int] = {}
459
+ # Track metadata mismatches: {ref_key: list of detail strings}
460
+ ref_mismatches: dict[str, list[str]] = {}
461
+ # Track retracted entries per ref
462
+ ref_retracted: dict[str, int] = {}
463
+
464
+ def _mark_found(rk: str) -> None:
465
+ ref_found[rk] = ref_found.get(rk, 0) + 1
466
+
467
+ def _check_metadata(
468
+ rk: str, entry: _BibEntry, api_data: dict[str, Any]
469
+ ) -> None:
470
+ """Compare entry metadata against API response, record mismatches."""
471
+ if api_data.get("retracted"):
472
+ ref_retracted[rk] = ref_retracted.get(rk, 0) + 1
473
+ ref_mismatches.setdefault(rk, []).append(
474
+ f"RETRACTED: '{entry.title[:60]}'" if entry.title else "RETRACTED"
475
+ )
476
+ if not entry.has_metadata:
477
+ return
478
+ issues = _compare_bib_entry_metadata(entry, api_data)
479
+ if issues:
480
+ ref_mismatches.setdefault(rk, []).extend(issues)
481
+
482
+ _OA_SELECT = (
483
+ "id,doi,title,authorships,publication_year,primary_location,is_retracted"
484
+ )
485
+
486
+ async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
487
+ # Pass 1: OpenAlex batch DOI (one request per 200)
488
+ # Index API results by DOI for metadata comparison
489
+ doi_api_data: dict[str, dict[str, Any]] = {}
490
+ oa_params: dict[str, str | int] = {"select": _OA_SELECT}
491
+ if self._config.polite_email:
492
+ oa_params["mailto"] = self._config.polite_email
493
+
494
+ for i in range(0, len(all_doi), 200):
495
+ batch = all_doi[i : i + 200]
496
+ doi_filter = "|".join(f"https://doi.org/{e.doi}" for _, e in batch)
497
+ try:
498
+ resp = await client.get(
499
+ "https://api.openalex.org/works",
500
+ params={
501
+ **oa_params,
502
+ "filter": f"doi:{doi_filter}",
503
+ "per_page": 200,
504
+ },
505
+ )
506
+ if resp.status_code == 200:
507
+ for work in resp.json().get("results", []):
508
+ doi = (
509
+ (work.get("doi") or "")
510
+ .replace("https://doi.org/", "")
511
+ .lower()
512
+ )
513
+ if doi:
514
+ doi_api_data[doi] = _oa_work_to_api_data(work)
515
+ except httpx.HTTPError as e:
516
+ logger.debug("OpenAlex DOI batch lookup failed: {}", e)
517
+
518
+ for ref_key, entry in all_doi:
519
+ api_data = doi_api_data.get(entry.doi.lower())
520
+ if api_data:
521
+ _mark_found(ref_key)
522
+ _check_metadata(ref_key, entry, api_data)
523
+ elif entry.arxiv_id or entry.pmid:
524
+ all_s2_id.append((ref_key, entry))
525
+ elif entry.title:
526
+ all_title_only.append((ref_key, entry))
527
+
528
+ # Pass 2: S2 batch for arXiv + PMID (one request per 500)
529
+ if all_s2_id:
530
+ s2_ids: list[tuple[str, str]] = []
531
+ s2_entries: list[tuple[str, _BibEntry]] = []
532
+ for ref_key, entry in all_s2_id:
533
+ if entry.arxiv_id:
534
+ s2_ids.append((ref_key, f"ARXIV:{entry.arxiv_id}"))
535
+ s2_entries.append((ref_key, entry))
536
+ elif entry.pmid:
537
+ s2_ids.append((ref_key, f"PMID:{entry.pmid}"))
538
+ s2_entries.append((ref_key, entry))
539
+
540
+ for i in range(0, len(s2_ids), 500):
541
+ s2_batch = s2_ids[i : i + 500]
542
+ s2_entry_batch = s2_entries[i : i + 500]
543
+ ids = [sid for _, sid in s2_batch]
544
+ try:
545
+ resp = await client.post(
546
+ "https://api.semanticscholar.org/graph/v1/paper/batch",
547
+ json={"ids": ids},
548
+ params={
549
+ "fields": "externalIds,title,authors,year,venue,isRetracted",
550
+ },
551
+ )
552
+ if resp.status_code == 200:
553
+ papers = resp.json()
554
+ for paper, (rk, _), (_, entry) in zip(
555
+ papers, s2_batch, s2_entry_batch
556
+ ):
557
+ if paper: # S2 returns null for not-found
558
+ _mark_found(rk)
559
+ api_data = _s2_paper_to_api_data(paper)
560
+ _check_metadata(rk, entry, api_data)
561
+ elif rk not in [r for r, _ in all_title_only]:
562
+ orig = next(
563
+ (
564
+ e
565
+ for r, e in all_s2_id
566
+ if r == rk and e.title
567
+ ),
568
+ None,
569
+ )
570
+ if orig:
571
+ all_title_only.append((rk, orig))
572
+ except httpx.HTTPError as e:
573
+ logger.debug("S2 batch lookup failed: {}", e)
574
+
575
+ # Pass 3: parallel title search for entries with no ID
576
+ title_results: list[tuple[str, bool, _BibEntry]] = []
577
+ if all_title_only:
578
+ limiter = MonotonicRateLimiter(10, 0.15)
579
+
580
+ async def _check_title(
581
+ rk: str,
582
+ entry: _BibEntry,
583
+ ) -> tuple[str, bool, _BibEntry]:
584
+ async with limiter:
585
+ pass
586
+ query = re.sub(r"[^\w\s-]", "", entry.title).strip()
587
+ if not query:
588
+ return rk, False, entry
589
+ try:
590
+ resp = await client.get(
591
+ "https://api.openalex.org/works",
592
+ params={
593
+ **oa_params,
594
+ "filter": f"title.search:{query}",
595
+ "per_page": 1,
596
+ },
597
+ )
598
+ if resp.status_code == 200:
599
+ results = resp.json().get("results", [])
600
+ if results:
601
+ api_data = _oa_work_to_api_data(results[0])
602
+ _check_metadata(rk, entry, api_data)
603
+ return rk, True, entry
604
+ except httpx.HTTPError as e:
605
+ logger.debug("OpenAlex title search failed for {}: {}", rk, e)
606
+ return rk, False, entry
607
+
608
+ title_results = await asyncio.gather(
609
+ *[_check_title(rk, e) for rk, e in all_title_only]
610
+ )
611
+
612
+ for rk, found, _entry in title_results:
613
+ if found:
614
+ _mark_found(rk)
615
+
616
+ # Build RefBibCheck per reference
617
+ for ref_key, entries in self._entries.items():
618
+ total = len(entries)
619
+ n_found = ref_found.get(ref_key, 0)
620
+ details = ref_mismatches.get(ref_key, [])
621
+ n_retracted = ref_retracted.get(ref_key, 0)
622
+ # Don't count retraction strings as metadata mismatches
623
+ n_mismatches = len(details) - n_retracted
624
+ self._results[ref_key] = RefBibCheck(
625
+ key=ref_key,
626
+ total_entries=total,
627
+ found=n_found,
628
+ not_found=total - n_found,
629
+ retracted=n_retracted,
630
+ metadata_mismatches=n_mismatches,
631
+ mismatch_details=details,
632
+ )
633
+
634
+
635
+ # ---------------------------------------------------------------------------
636
+ # Standalone bibliography verification (runs in default pipeline)
637
+ # ---------------------------------------------------------------------------
638
+
639
+
640
+ async def run_bib_verification(
641
+ references_dir: Path,
642
+ config: LintConfig | None = None,
643
+ ) -> list[RefBibCheck]:
644
+ """Verify bibliography entries for all parsed formal references.
645
+
646
+ Loads structured bibliography data from workspace.db (stored at parse
647
+ time), verifies existence + metadata + retraction via APIs, and returns
648
+ per-reference results.
649
+
650
+ This runs as part of the default pipeline (no ``--chain`` needed).
651
+ """
652
+ config = config or LintConfig()
653
+
654
+ from sciwrite_lint.references.metadata import load_all_metadata
655
+ from sciwrite_lint.references.workspace_db import get_db, is_formal_cached_db
656
+
657
+ all_meta = load_all_metadata(references_dir)
658
+
659
+ verifier = BibVerifier(config)
660
+ refs_loaded = 0
661
+
662
+ with get_db(references_dir) as conn:
663
+ for key, meta in all_meta.items():
664
+ # Only formal PDFs have reliable bibliography entries
665
+ local_file = meta.access.get("local_file") or ""
666
+ if local_file.endswith(".pdf") and not is_formal_cached_db(conn, key):
667
+ continue
668
+ # Skip entries without local files (T3 references)
669
+ if not local_file:
670
+ continue
671
+ # Try structured registry data
672
+ if verifier.add_reference_from_registry(key, conn):
673
+ refs_loaded += 1
674
+
675
+ if not refs_loaded:
676
+ return []
677
+
678
+ logger.info(
679
+ "Bibliography verification: {} references with stored entries",
680
+ refs_loaded,
681
+ )
682
+ await verifier.verify_all()
683
+
684
+ results: list[RefBibCheck] = []
685
+ for key in verifier._entries:
686
+ bib_check = verifier.get_result(key)
687
+ if bib_check.total_entries > 0:
688
+ results.append(bib_check)
689
+ parts = [
690
+ f"{bib_check.found}/{bib_check.total_entries} found"
691
+ f" ({bib_check.hallucination_rate:.0%} hallucination)",
692
+ ]
693
+ if bib_check.retracted:
694
+ parts.append(f"{bib_check.retracted} retracted")
695
+ if bib_check.metadata_mismatches:
696
+ parts.append(f"{bib_check.metadata_mismatches} metadata mismatches")
697
+ logger.info(" {}: bibliography {}", key, ", ".join(parts))
698
+
699
+ return results
700
+
701
+
702
+ # Keep single-reference convenience function for backward compatibility
703
+ async def verify_ref_bibliography(
704
+ ref_key: str,
705
+ ref_text: str,
706
+ config: LintConfig | None = None,
707
+ ) -> RefBibCheck:
708
+ """Check how many of a reference's own citations exist in OpenAlex.
709
+
710
+ Convenience wrapper around BibVerifier for single-reference use.
711
+ For multiple references, use BibVerifier directly to batch across refs.
712
+ """
713
+ verifier = BibVerifier(config)
714
+ verifier.add_reference(ref_key, ref_text)
715
+ await verifier.verify_all()
716
+ return verifier.get_result(ref_key)