bibcite-cli 0.6.4__tar.gz → 0.6.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/PKG-INFO +9 -4
  2. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/Readme.md +8 -3
  3. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/pyproject.toml +1 -1
  4. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/src/bibcite/sources.py +134 -124
  5. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/src/bibcite/venues.py +1 -1
  6. bibcite_cli-0.6.5/tests/test_dblp.py +152 -0
  7. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_no_google_scholar.py +5 -9
  8. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_source_retries.py +77 -13
  9. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_status_semantics.py +1 -12
  10. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/uv.lock +1 -1
  11. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/.github/workflows/ci.yml +0 -0
  12. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/.github/workflows/publish.yml +0 -0
  13. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/.gitignore +0 -0
  14. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/LICENSE +0 -0
  15. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/assets/bibcite.svg +0 -0
  16. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/skills/bibcite/SKILL.md +0 -0
  17. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/src/bibcite/__init__.py +0 -0
  18. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/src/bibcite/bibfile.py +0 -0
  19. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/src/bibcite/cache.py +0 -0
  20. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/src/bibcite/cli.py +0 -0
  21. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/src/bibcite/data/strings.bib +0 -0
  22. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/src/bibcite/normalize.py +0 -0
  23. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/src/bibcite/resolve.py +0 -0
  24. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_bibfile.py +0 -0
  25. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_bugfixes.py +0 -0
  26. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_cli_status.py +0 -0
  27. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_entry_types.py +0 -0
  28. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_normalize.py +0 -0
  29. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_public_service.py +0 -0
  30. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_round2.py +0 -0
  31. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_round3.py +0 -0
  32. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_strings_override.py +0 -0
  33. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_venues.py +0 -0
  34. {bibcite_cli-0.6.4 → bibcite_cli-0.6.5}/tests/test_webpages.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: bibcite-cli
3
- Version: 0.6.4
3
+ Version: 0.6.5
4
4
  Summary: Resolve papers (arXiv id / DOI / title) to canonical, normalized BibTeX for agents and humans
5
5
  Project-URL: Repository, https://github.com/leo1oel/bibcite
6
6
  License-Expression: MIT
@@ -158,10 +158,15 @@ Mark a confirmed preprint-only entry with `pubstate = {preprint}` if you want `c
158
158
 
159
159
  ## How resolution works
160
160
 
161
- For arXiv IDs and titles, `bibcite` collects paper metadata and checks publication sources in a cascade derived from [PaperMemory](https://github.com/vict0rsch/PaperMemory): DBLP, Semantic Scholar, Crossref, Unpaywall, and OpenAlex.
161
+ For arXiv IDs and titles, `bibcite` collects paper metadata and checks publication sources in a cascade derived from [PaperMemory](https://github.com/vict0rsch/PaperMemory): DBLP, Semantic Scholar, Crossref, and OpenAlex.
162
162
  Google Scholar scraping is excluded so CAPTCHA challenges cannot delay publication lookup.
163
163
  A published match must have the same normalized title or pass a guarded title-drift check, have a plausible publication year, and name a non-preprint venue.
164
164
 
165
+ DBLP requests are single-attempt, with short connection/read timeouts and a shared four-second scheduling budget for exact and title-drift lookup.
166
+ Socket timeouts apply per network phase, so this budget is not a hard end-to-end deadline.
167
+ Published DBLP records are constructed from the search response, without another request for a BibTeX export.
168
+ A failed or unfinished DBLP check stays incomplete rather than being treated as evidence that a paper is unpublished.
169
+
165
170
  Successful published matches are cached at `~/.cache/bibcite/published.json`.
166
171
  Preprint-only results are never cached because a paper may be published later.
167
172
  Use `--no-cache` or set `BIBCITE_NO_CACHE=1` to bypass the cache.
@@ -176,7 +181,7 @@ These optional environment variables improve source reliability:
176
181
  | --- | --- |
177
182
  | `OPENALEX_API_KEY` | Uses your OpenAlex quota instead of the anonymous shared pool. |
178
183
  | `S2_API_KEY` | Uses a private Semantic Scholar quota. |
179
- | `BIBCITE_MAILTO` | Sends your contact email to the Crossref, OpenAlex, and Unpaywall polite pools. |
184
+ | `BIBCITE_MAILTO` | Sends your contact email to the Crossref and OpenAlex polite pools. |
180
185
  | `BIBCITE_PUBLIC_SERVICE_URL` | Routes keyless OpenAlex, Semantic Scholar, and Crossref requests through an HTTPS `/v1/query` service supplied by the embedding application. |
181
186
  | `BIBCITE_CORE_SOURCES` | Overrides the sources required for a trustworthy publication check. |
182
187
  | `BIBCITE_NO_CACHE=1` | Disables the local publication cache. |
@@ -213,7 +218,7 @@ uv tool install --editable .
213
218
 
214
219
  ## Acknowledgements
215
220
 
216
- Inspired by [PaperMemory](https://github.com/vict0rsch/PaperMemory), with formatting by [bibtex-tidy](https://github.com/FlamingTempura/bibtex-tidy) and metadata from arXiv, DBLP, Semantic Scholar, Crossref, Unpaywall, and OpenAlex.
221
+ Inspired by [PaperMemory](https://github.com/vict0rsch/PaperMemory), with formatting by [bibtex-tidy](https://github.com/FlamingTempura/bibtex-tidy) and metadata from arXiv, DBLP, Semantic Scholar, Crossref, and OpenAlex.
217
222
 
218
223
  ## License
219
224
 
@@ -145,10 +145,15 @@ Mark a confirmed preprint-only entry with `pubstate = {preprint}` if you want `c
145
145
 
146
146
  ## How resolution works
147
147
 
148
- For arXiv IDs and titles, `bibcite` collects paper metadata and checks publication sources in a cascade derived from [PaperMemory](https://github.com/vict0rsch/PaperMemory): DBLP, Semantic Scholar, Crossref, Unpaywall, and OpenAlex.
148
+ For arXiv IDs and titles, `bibcite` collects paper metadata and checks publication sources in a cascade derived from [PaperMemory](https://github.com/vict0rsch/PaperMemory): DBLP, Semantic Scholar, Crossref, and OpenAlex.
149
149
  Google Scholar scraping is excluded so CAPTCHA challenges cannot delay publication lookup.
150
150
  A published match must have the same normalized title or pass a guarded title-drift check, have a plausible publication year, and name a non-preprint venue.
151
151
 
152
+ DBLP requests are single-attempt, with short connection/read timeouts and a shared four-second scheduling budget for exact and title-drift lookup.
153
+ Socket timeouts apply per network phase, so this budget is not a hard end-to-end deadline.
154
+ Published DBLP records are constructed from the search response, without another request for a BibTeX export.
155
+ A failed or unfinished DBLP check stays incomplete rather than being treated as evidence that a paper is unpublished.
156
+
152
157
  Successful published matches are cached at `~/.cache/bibcite/published.json`.
153
158
  Preprint-only results are never cached because a paper may be published later.
154
159
  Use `--no-cache` or set `BIBCITE_NO_CACHE=1` to bypass the cache.
@@ -163,7 +168,7 @@ These optional environment variables improve source reliability:
163
168
  | --- | --- |
164
169
  | `OPENALEX_API_KEY` | Uses your OpenAlex quota instead of the anonymous shared pool. |
165
170
  | `S2_API_KEY` | Uses a private Semantic Scholar quota. |
166
- | `BIBCITE_MAILTO` | Sends your contact email to the Crossref, OpenAlex, and Unpaywall polite pools. |
171
+ | `BIBCITE_MAILTO` | Sends your contact email to the Crossref and OpenAlex polite pools. |
167
172
  | `BIBCITE_PUBLIC_SERVICE_URL` | Routes keyless OpenAlex, Semantic Scholar, and Crossref requests through an HTTPS `/v1/query` service supplied by the embedding application. |
168
173
  | `BIBCITE_CORE_SOURCES` | Overrides the sources required for a trustworthy publication check. |
169
174
  | `BIBCITE_NO_CACHE=1` | Disables the local publication cache. |
@@ -200,7 +205,7 @@ uv tool install --editable .
200
205
 
201
206
  ## Acknowledgements
202
207
 
203
- Inspired by [PaperMemory](https://github.com/vict0rsch/PaperMemory), with formatting by [bibtex-tidy](https://github.com/FlamingTempura/bibtex-tidy) and metadata from arXiv, DBLP, Semantic Scholar, Crossref, Unpaywall, and OpenAlex.
208
+ Inspired by [PaperMemory](https://github.com/vict0rsch/PaperMemory), with formatting by [bibtex-tidy](https://github.com/FlamingTempura/bibtex-tidy) and metadata from arXiv, DBLP, Semantic Scholar, Crossref, and OpenAlex.
204
209
 
205
210
  ## License
206
211
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "bibcite-cli"
3
- version = "0.6.4"
3
+ version = "0.6.5"
4
4
  description = "Resolve papers (arXiv id / DOI / title) to canonical, normalized BibTeX for agents and humans"
5
5
  readme = "Readme.md"
6
6
  license = "MIT"
@@ -1,7 +1,7 @@
1
1
  """API clients for the publication-matching cascade.
2
2
 
3
3
  Order and matching rules ported from PaperMemory's bibMatcher:
4
- DBLP -> Semantic Scholar -> CrossRef -> Unpaywall -> OpenAlex.
4
+ DBLP -> Semantic Scholar -> CrossRef -> OpenAlex.
5
5
  All matchers verify identity via normalized-title equality and reject
6
6
  preprint venues (arXiv / CoRR / bioRxiv / ...).
7
7
  """
@@ -78,7 +78,11 @@ def _request_timeout(cap: float = TIMEOUT) -> float:
78
78
 
79
79
 
80
80
  def _get(
81
- c: httpx.Client, url: str, *, timeout: float = TIMEOUT, **kwargs
81
+ c: httpx.Client,
82
+ url: str,
83
+ *,
84
+ timeout: float | httpx.Timeout = TIMEOUT,
85
+ **kwargs,
82
86
  ) -> httpx.Response:
83
87
  """GET directly or route keyless supported APIs through the public service."""
84
88
  public_url = os.environ.get("BIBCITE_PUBLIC_SERVICE_URL")
@@ -98,7 +102,10 @@ def _get(
98
102
  "crossref": bool(os.environ.get("BIBCITE_MAILTO")),
99
103
  }
100
104
  if not public_url or not provider or personal_access[provider]:
101
- return c.get(url, timeout=_request_timeout(timeout), **kwargs)
105
+ effective_timeout = (
106
+ timeout if isinstance(timeout, httpx.Timeout) else _request_timeout(timeout)
107
+ )
108
+ return c.get(url, timeout=effective_timeout, **kwargs)
102
109
 
103
110
  service = urlsplit(public_url)
104
111
  loopback = service.hostname in {"localhost", "127.0.0.1", "::1"}
@@ -183,7 +190,7 @@ def _s2_headers() -> dict:
183
190
 
184
191
 
185
192
  def _mailto() -> str:
186
- """Contact email for the polite pools (CrossRef/OpenAlex/Unpaywall).
193
+ """Contact email for the polite pools (CrossRef/OpenAlex).
187
194
  Set BIBCITE_MAILTO to use your own."""
188
195
  return os.environ.get("BIBCITE_MAILTO") or "bibcite@gmail.com"
189
196
 
@@ -340,7 +347,26 @@ def _paced_get(
340
347
 
341
348
  # DBLP throttles at roughly 1-2 req/s and escalates to temporary IP bans.
342
349
  def _dblp_get(c: httpx.Client, url: str, params: dict | None = None) -> httpx.Response:
343
- return _paced_get(c, url, "dblp", 0.8, params=params)
350
+ wait = 0.8 - (time.monotonic() - _LAST_REQUEST.get("dblp", 0.0))
351
+ if wait > 0:
352
+ _sleep(wait)
353
+ _LAST_REQUEST["dblp"] = time.monotonic()
354
+ remaining = _request_timeout(2.5)
355
+ timeout = httpx.Timeout(
356
+ connect=min(1.5, remaining),
357
+ read=min(2.5, remaining),
358
+ write=min(1.0, remaining),
359
+ pool=min(1.0, remaining),
360
+ )
361
+ try:
362
+ response = _get(c, url, params=params, timeout=timeout)
363
+ except httpx.HTTPError as e:
364
+ raise TransientSourceError(f"dblp unreachable ({type(e).__name__})") from e
365
+ if response.status_code == 429:
366
+ raise SourceUnavailable("dblp rate-limited (429)")
367
+ if response.status_code >= 500:
368
+ raise TransientSourceError(f"dblp server error ({response.status_code})")
369
+ return response
344
370
 
345
371
 
346
372
  def _dblp_sanitize(q: str) -> str:
@@ -352,19 +378,76 @@ def _dblp_sanitize(q: str) -> str:
352
378
 
353
379
 
354
380
  def _dblp_search(c: httpx.Client, q: str, h: int = 100) -> list:
355
- """One search request; a 500 (broad all-common-words query × large h
356
- times out their backend) is retried once with a small result window
357
- before giving up on this query variant."""
381
+ """Run one DBLP search while retaining the full result window."""
358
382
  url = "https://dblp.org/search/publ/api"
359
383
  q = _dblp_sanitize(q)
360
384
  r = _dblp_get(c, url, params={"q": q, "format": "json", "h": h})
361
- if r.status_code == 500 and h > 10:
362
- _log("[dblp] 500 on broad query — retrying with h=10")
363
- r = _dblp_get(c, url, params={"q": q, "format": "json", "h": 10})
364
385
  r.raise_for_status()
365
386
  return r.json().get("result", {}).get("hits", {}).get("hit", []) or []
366
387
 
367
388
 
389
+ def _dblp_bibtex(info: dict, venue: str) -> str:
390
+ """Build the DBLP record from search JSON, avoiding a second .bib request."""
391
+ from .bibfile import entry_to_bibtex
392
+
393
+ record_type = str(info.get("type", "")).lower()
394
+ key = str(info.get("key") or "dblp")
395
+ if record_type == "editorship":
396
+ entry_type = "proceedings" if key.startswith("conf/") else "book"
397
+ elif key.startswith("phd/"):
398
+ entry_type = "phdthesis"
399
+ elif record_type == "parts in books or collections":
400
+ entry_type = "incollection"
401
+ elif record_type == "books and theses":
402
+ entry_type = "book"
403
+ else:
404
+ entry_type = (
405
+ "inproceedings"
406
+ if "conference" in record_type or key.startswith("conf/")
407
+ else "article"
408
+ )
409
+ fields = {
410
+ "ID": key,
411
+ "ENTRYTYPE": entry_type,
412
+ "author": " and ".join(_dblp_hit_authors(info)),
413
+ "title": clean_title(html.unescape(str(info.get("title", "")))),
414
+ "year": str(info.get("year", "")),
415
+ "pages": str(info.get("pages", "")),
416
+ "volume": str(info.get("volume", "")),
417
+ "number": str(info.get("number", "")),
418
+ "doi": str(info.get("doi", "")),
419
+ "url": str(info.get("ee") or info.get("url") or ""),
420
+ "publisher": str(info.get("publisher", "")),
421
+ "editor": str(info.get("editor", "")),
422
+ "isbn": str(info.get("isbn", "")),
423
+ }
424
+ if entry_type in {"inproceedings", "incollection"}:
425
+ fields["booktitle"] = venue
426
+ elif entry_type == "article":
427
+ fields["journal"] = venue
428
+ if record_type == "editorship":
429
+ fields["editor"] = fields["editor"] or fields["author"]
430
+ fields.pop("author")
431
+ return entry_to_bibtex(fields)
432
+
433
+
434
+ def _dblp_match(info: dict, source: str) -> Match:
435
+ venue = info["venue"]
436
+ if isinstance(venue, list):
437
+ venue = venue[0]
438
+ title = clean_title(html.unescape(info.get("title", "")))
439
+ return Match(
440
+ source=source,
441
+ venue=str(venue),
442
+ title=title,
443
+ year=str(info.get("year", "")),
444
+ authors=_dblp_hit_authors(info),
445
+ doi=info.get("doi", ""),
446
+ bibtex=_dblp_bibtex(info, str(venue)),
447
+ url=info.get("ee", "") or info.get("url", ""),
448
+ )
449
+
450
+
368
451
  def try_dblp(title: str, author_hint: str = "") -> Match | None:
369
452
  """DBLP search. Generic titles ("X is all you need") drown in DBLP's
370
453
  ranking, so when we know the first author we query with their last name
@@ -386,27 +469,10 @@ def try_dblp(title: str, author_hint: str = "") -> Match | None:
386
469
  continue
387
470
  if info.get("venue") == "CoRR" or not info.get("venue"):
388
471
  continue
389
- venue = info["venue"]
390
- if isinstance(venue, list):
391
- venue = venue[0]
392
- bibtex = ""
393
- if info.get("url"):
394
- try:
395
- br = _dblp_get(c, info["url"] + ".bib")
396
- if br.status_code == 200:
397
- bibtex = br.text
398
- except SourceUnavailable:
399
- pass # keep the match; construct from fields
472
+ match = _dblp_match(info, "dblp")
473
+ venue = match.venue
400
474
  _log(f"[dblp] match: {venue} {info.get('year', '')}")
401
- return Match(
402
- source="dblp",
403
- venue=str(venue),
404
- title=clean_title(html.unescape(info.get("title", ""))),
405
- year=str(info.get("year", "")),
406
- doi=info.get("doi", ""),
407
- bibtex=bibtex,
408
- url=info.get("ee", "") or info.get("url", ""),
409
- )
475
+ return match
410
476
  return None
411
477
 
412
478
 
@@ -414,7 +480,11 @@ def _dblp_hit_authors(info: dict) -> list[str]:
414
480
  authors = (info.get("authors") or {}).get("author") or []
415
481
  if isinstance(authors, dict):
416
482
  authors = [authors]
417
- return [a.get("text", "") for a in authors if isinstance(a, dict)]
483
+ return [
484
+ re.sub(r"\s+\d{4}$", "", a.get("text", ""))
485
+ for a in authors
486
+ if isinstance(a, dict) and a.get("text")
487
+ ]
418
488
 
419
489
 
420
490
  def try_dblp_fuzzy(title: str, author_hint: str, year: str = "") -> Match | None:
@@ -446,30 +516,13 @@ def try_dblp_fuzzy(title: str, author_hint: str, year: str = "") -> Match | None
446
516
  hit_authors = mini_hash(" ".join(_dblp_hit_authors(info)))
447
517
  if author_hint not in hit_authors:
448
518
  continue
449
- venue = info["venue"]
450
- if isinstance(venue, list):
451
- venue = venue[0]
452
- bibtex = ""
453
- if info.get("url"):
454
- try:
455
- br = _dblp_get(c, info["url"] + ".bib")
456
- if br.status_code == 200:
457
- bibtex = br.text
458
- except SourceUnavailable:
459
- pass # keep the match; construct from fields
519
+ match = _dblp_match(info, "dblp-fuzzy")
520
+ venue = match.venue
460
521
  _log(
461
522
  f"[dblp-fuzzy] match with title drift: '{hit_title}' "
462
523
  f"@ {venue} {info.get('year', '')}"
463
524
  )
464
- return Match(
465
- source="dblp-fuzzy",
466
- venue=str(venue),
467
- title=hit_title,
468
- year=str(info.get("year", "")),
469
- doi=info.get("doi", ""),
470
- bibtex=bibtex,
471
- url=info.get("ee", "") or info.get("url", ""),
472
- )
525
+ return match
473
526
  return None
474
527
 
475
528
 
@@ -615,9 +668,8 @@ def try_crossref(title: str) -> Match | None:
615
668
  if r.status_code == 429:
616
669
  raise SourceUnavailable("CrossRef rate-limited (429)")
617
670
  if r.status_code >= 500:
618
- # A dead endpoint (Unpaywall search 500s for days at a time)
619
- # gets benched for the run instead of adding latency + noise
620
- # to every remaining query.
671
+ # A dead endpoint gets benched for the run instead of adding
672
+ # latency and noise to every remaining query.
621
673
  raise SourceUnavailable(f"CrossRef server error ({r.status_code})")
622
674
  r.raise_for_status()
623
675
  payload = r.json()
@@ -660,52 +712,6 @@ def try_crossref(title: str) -> Match | None:
660
712
  return None
661
713
 
662
714
 
663
- # ---------------------------------------------------------------------------
664
- # Unpaywall
665
- # ---------------------------------------------------------------------------
666
-
667
- def try_unpaywall(title: str) -> Match | None:
668
- with _client() as c:
669
- r = _get(
670
- c,
671
- "https://api.unpaywall.org/v2/search",
672
- params={"query": title, "is_oa": "true", "email": _mailto()},
673
- )
674
- if r.status_code == 429:
675
- raise SourceUnavailable("Unpaywall rate-limited (429)")
676
- if r.status_code >= 500:
677
- # A dead endpoint (Unpaywall search 500s for days at a time)
678
- # gets benched for the run instead of adding latency + noise
679
- # to every remaining query.
680
- raise SourceUnavailable(f"Unpaywall server error ({r.status_code})")
681
- r.raise_for_status()
682
- ref = norm_title(title)
683
- for res in r.json().get("results") or []:
684
- resp = res.get("response", {})
685
- if norm_title(resp.get("title", "")) != ref:
686
- continue
687
- venue = (resp.get("journal_name") or "").strip()
688
- if not _is_published_venue(venue):
689
- continue
690
- doi = resp.get("doi", "")
691
- if ARXIV_DOI.match(doi):
692
- continue
693
- authors = [
694
- " ".join(filter(None, [a.get("given"), a.get("family")]))
695
- for a in resp.get("z_authors") or []
696
- ]
697
- _log(f"[unpaywall] match: {venue} {resp.get('year', '')}")
698
- return Match(
699
- source="unpaywall",
700
- venue=venue,
701
- title=clean_title(resp.get("title", "")),
702
- year=str(resp.get("year") or ""),
703
- authors=[a for a in authors if a],
704
- doi=doi,
705
- )
706
- return None
707
-
708
-
709
715
  # ---------------------------------------------------------------------------
710
716
  # OpenAlex (not in PaperMemory; unauthenticated with generous rate limits, so
711
717
  # it doubles as the metadata fallback when the arXiv API / S2 are throttled)
@@ -722,9 +728,8 @@ def openalex_search(title: str) -> dict | None:
722
728
  if r.status_code == 429:
723
729
  raise SourceUnavailable("OpenAlex rate-limited (429)")
724
730
  if r.status_code >= 500:
725
- # A dead endpoint (Unpaywall search 500s for days at a time)
726
- # gets benched for the run instead of adding latency + noise
727
- # to every remaining query.
731
+ # A dead endpoint gets benched for the run instead of adding
732
+ # latency and noise to every remaining query.
728
733
  raise SourceUnavailable(f"OpenAlex server error ({r.status_code})")
729
734
  r.raise_for_status()
730
735
  ref = norm_title(title)
@@ -829,7 +834,6 @@ CASCADE = (
829
834
  ("dblp", lambda t, y, a, au: try_dblp(t, au)),
830
835
  ("semanticscholar", lambda t, y, a, au: try_semantic_scholar(t, y, a)),
831
836
  ("crossref", lambda t, y, a, au: try_crossref(t)),
832
- ("unpaywall", lambda t, y, a, au: try_unpaywall(t)),
833
837
  ("openalex", lambda t, y, a, au: try_openalex(t)),
834
838
  )
835
839
 
@@ -840,8 +844,7 @@ CASCADE = (
840
844
  _DISABLED: dict[str, str] = {}
841
845
 
842
846
  # Only these sources are authoritative enough that losing one taints a miss
843
- # into "incomplete". Unpaywall flakiness is routine and must not stop
844
- # "not_found" from ever being trustworthy.
847
+ # into "incomplete".
845
848
  # Override with BIBCITE_CORE_SOURCES="dblp,semanticscholar" if one of these
846
849
  # is down for days and keeps every verdict incomplete.
847
850
  CORE_SOURCES = frozenset(
@@ -880,7 +883,9 @@ def find_published(
880
883
  # version (the common case) misses everywhere, and used to pay the *sum* of
881
884
  # each source's latency; now the wall-clock is the slowest single source.
882
885
  # The first verified hit by CASCADE priority still wins.
883
- deadline = time.monotonic() + PUBLICATION_TIMEOUT
886
+ started = time.monotonic()
887
+ deadline = started + PUBLICATION_TIMEOUT
888
+ dblp_deadline = min(deadline, started + 4.0)
884
889
  active = [(name, fn) for name, fn in CASCADE if name not in _DISABLED]
885
890
  outcomes: dict[str, tuple] = {}
886
891
  if active:
@@ -888,7 +893,7 @@ def find_published(
888
893
  futures = {
889
894
  pool.submit(
890
895
  _with_deadline,
891
- deadline,
896
+ dblp_deadline if name == "dblp" else deadline,
892
897
  fn,
893
898
  title,
894
899
  year,
@@ -940,23 +945,28 @@ def find_published(
940
945
  author_hint
941
946
  and dblp_outcome is not None
942
947
  and dblp_outcome[0] == "miss"
943
- and time.monotonic() < deadline
944
948
  ):
945
- try:
946
- m = _with_deadline(deadline, try_dblp_fuzzy, title, author_hint, year)
947
- if m:
948
- cache.put(cache_key, m.__dict__)
949
- return m, "found"
950
- clean_misses += 1
951
- except TransientSourceError as e:
952
- incomplete = True
953
- _log(f"[dblp-fuzzy] transient failure for this entry: {e}")
954
- except SourceUnavailable as e:
955
- _DISABLED["dblp"] = str(e)
956
- incomplete = True
957
- except Exception as e:
949
+ if time.monotonic() >= dblp_deadline:
958
950
  incomplete = True
959
- _log(f"[dblp-fuzzy] error: {type(e).__name__}: {e}")
951
+ _log("[dblp-fuzzy] skipped: DBLP lookup budget exhausted")
952
+ else:
953
+ try:
954
+ m = _with_deadline(
955
+ dblp_deadline, try_dblp_fuzzy, title, author_hint, year
956
+ )
957
+ if m:
958
+ cache.put(cache_key, m.__dict__)
959
+ return m, "found"
960
+ clean_misses += 1
961
+ except TransientSourceError as e:
962
+ incomplete = True
963
+ _log(f"[dblp-fuzzy] transient failure for this entry: {e}")
964
+ except SourceUnavailable as e:
965
+ _DISABLED["dblp"] = str(e)
966
+ incomplete = True
967
+ except Exception as e:
968
+ incomplete = True
969
+ _log(f"[dblp-fuzzy] error: {type(e).__name__}: {e}")
960
970
  if not clean_misses:
961
971
  return None, "unavailable"
962
972
  return None, ("incomplete" if incomplete else "not_found")
@@ -2,7 +2,7 @@
2
2
 
3
3
  Parses the vendored ``data/strings.bib`` @string table (journals /
4
4
  conferences / workshops) and maps venue strings returned by DBLP, Semantic
5
- Scholar, CrossRef, Unpaywall, etc. onto the canonical names.
5
+ Scholar, CrossRef, OpenAlex, etc. onto the canonical names.
6
6
  """
7
7
 
8
8
  import re
@@ -0,0 +1,152 @@
1
+ import httpx
2
+ import pytest
3
+
4
+ import bibcite.sources as sources
5
+ from bibcite.bibfile import parse_bibtex_entry
6
+
7
+
8
+ @pytest.fixture(autouse=True)
9
+ def isolated(monkeypatch):
10
+ monkeypatch.setattr(sources, "_LAST_REQUEST", {})
11
+ monkeypatch.setattr(sources.time, "sleep", lambda _: None)
12
+
13
+
14
+ def _hit(title="A Published Paper"):
15
+ return {
16
+ "info": {
17
+ "authors": {
18
+ "author": [
19
+ {"text": "Ada Lovelace 0001"},
20
+ {"text": "Grace Hopper"},
21
+ ]
22
+ },
23
+ "title": title,
24
+ "venue": "Test Conference",
25
+ "year": "2025",
26
+ "pages": "12-24",
27
+ "volume": "42",
28
+ "number": "3",
29
+ "doi": "10.1000/test",
30
+ "ee": "https://doi.org/10.1000/test",
31
+ "url": "https://dblp.org/rec/conf/test/Lovelace25",
32
+ "key": "conf/test/Lovelace25",
33
+ "type": "Conference and Workshop Papers",
34
+ "publisher": "Example Press",
35
+ "editor": "Test Editor",
36
+ "isbn": "978-0-00-000000-0",
37
+ }
38
+ }
39
+
40
+
41
+ @pytest.mark.parametrize("fuzzy", [False, True])
42
+ def test_dblp_match_uses_search_json_without_bib_download(monkeypatch, fuzzy):
43
+ calls = []
44
+ query_title = (
45
+ "An Information-Theoretic Perspective on Variance-Invariance-Covariance Regularization"
46
+ if fuzzy
47
+ else "A Published Paper"
48
+ )
49
+ hit = _hit(
50
+ "An Information Theory Perspective on Variance-Invariance-Covariance Regularization"
51
+ if fuzzy
52
+ else query_title
53
+ )
54
+
55
+ def get(client, url, params=None):
56
+ calls.append((url, params))
57
+ return httpx.Response(
58
+ 200,
59
+ request=httpx.Request("GET", url),
60
+ json={"result": {"hits": {"hit": [hit]}}},
61
+ )
62
+
63
+ monkeypatch.setattr(sources, "_dblp_get", get)
64
+ match = (
65
+ sources.try_dblp_fuzzy(query_title, "lovelace", "2024")
66
+ if fuzzy
67
+ else sources.try_dblp(query_title)
68
+ )
69
+
70
+ assert match is not None
71
+ assert len(calls) == 1
72
+ assert calls[0][0].endswith("/search/publ/api")
73
+ entry = parse_bibtex_entry(match.bibtex)
74
+ assert entry["ENTRYTYPE"] == "inproceedings"
75
+ assert entry["author"] == "Ada Lovelace and Grace Hopper"
76
+ assert entry["title"] == hit["info"]["title"]
77
+ assert entry["booktitle"] == "Test Conference"
78
+ for field in (
79
+ "year",
80
+ "pages",
81
+ "volume",
82
+ "number",
83
+ "doi",
84
+ "url",
85
+ "publisher",
86
+ "editor",
87
+ "isbn",
88
+ ):
89
+ expected = hit["info"]["ee"] if field == "url" else hit["info"][field]
90
+ assert entry[field] == expected
91
+
92
+
93
+ def test_dblp_json_type_distinguishes_journal_articles():
94
+ info = _hit()["info"]
95
+ info.update(
96
+ {
97
+ "type": "Journal Articles",
98
+ "key": "journals/test/Lovelace25",
99
+ "venue": "Test Journal",
100
+ }
101
+ )
102
+
103
+ entry = parse_bibtex_entry(sources._dblp_match(info, "dblp").bibtex)
104
+
105
+ assert entry["ENTRYTYPE"] == "article"
106
+ assert entry["journal"] == "Test Journal"
107
+ assert "booktitle" not in entry
108
+
109
+
110
+ @pytest.mark.parametrize(
111
+ ("record_type", "key", "expected"),
112
+ [
113
+ ("Books and Theses", "books/example/Test", "book"),
114
+ ("Books and Theses", "phd/example/Test", "phdthesis"),
115
+ ("Parts in Books or Collections", "books/example/TestChapter", "incollection"),
116
+ ("Editorship", "conf/example/2025", "proceedings"),
117
+ ],
118
+ )
119
+ def test_search_json_preserves_other_record_types(record_type, key, expected):
120
+ info = _hit()["info"]
121
+ info.update(type=record_type, key=key)
122
+ entry = parse_bibtex_entry(sources._dblp_match(info, "dblp").bibtex)
123
+ assert entry["ENTRYTYPE"] == expected
124
+ assert "journal" not in entry
125
+ if expected == "proceedings":
126
+ assert entry["editor"] == "Test Editor"
127
+ assert "author" not in entry
128
+
129
+
130
+ def test_dblp_exact_still_rejects_a_different_title(monkeypatch):
131
+ monkeypatch.setattr(sources, "_dblp_search", lambda *args: [_hit("Other Paper")])
132
+
133
+ assert sources.try_dblp("A Published Paper") is None
134
+
135
+
136
+ @pytest.mark.parametrize(
137
+ ("author", "year"),
138
+ [("different", "2024"), ("lovelace", "2010")],
139
+ )
140
+ def test_dblp_fuzzy_still_requires_author_and_plausible_year(monkeypatch, author, year):
141
+ drifted = _hit(
142
+ "An Information Theory Perspective on Variance-Invariance-Covariance Regularization"
143
+ )
144
+ monkeypatch.setattr(sources, "_dblp_search", lambda *args: [drifted])
145
+
146
+ match = sources.try_dblp_fuzzy(
147
+ "An Information-Theoretic Perspective on Variance-Invariance-Covariance Regularization",
148
+ author,
149
+ year,
150
+ )
151
+
152
+ assert match is None
@@ -12,7 +12,7 @@ resolver = importlib.import_module("bibcite.resolve")
12
12
 
13
13
 
14
14
  @pytest.mark.parametrize("operation", ["get", "add", "upgrade"])
15
- def test_cli_never_queries_google_scholar(operation, monkeypatch, tmp_path, capsys):
15
+ def test_cli_never_queries_removed_sources(operation, monkeypatch, tmp_path, capsys):
16
16
  monkeypatch.setattr(cache, "DISABLED", True)
17
17
  monkeypatch.setattr(sources, "_DISABLED", {})
18
18
  monkeypatch.setattr(
@@ -27,7 +27,7 @@ def test_cli_never_queries_google_scholar(operation, monkeypatch, tmp_path, caps
27
27
  ),
28
28
  )
29
29
  visited = []
30
- for name in ("dblp", "semantic_scholar", "crossref", "unpaywall", "openalex"):
30
+ for name in ("dblp", "semantic_scholar", "crossref", "openalex"):
31
31
 
32
32
  def miss(*args, source=name):
33
33
  visited.append(source)
@@ -59,11 +59,7 @@ def test_cli_never_queries_google_scholar(operation, monkeypatch, tmp_path, caps
59
59
  args = ["upgrade", str(path), "--dry-run"]
60
60
  cli.main(args)
61
61
  json.loads(capsys.readouterr().out)
62
- assert set(visited) == {
63
- "dblp",
64
- "semantic_scholar",
65
- "crossref",
66
- "unpaywall",
67
- "openalex",
68
- }
62
+ assert set(visited) == {"dblp", "semantic_scholar", "crossref", "openalex"}
69
63
  assert requests == []
64
+ assert not hasattr(sources, "try_unpaywall")
65
+ assert all("scholar.google" not in url and "unpaywall" not in url for url in requests)
@@ -35,7 +35,7 @@ class _ReadErrorClient:
35
35
  return httpx.Response(200, request=request, json={})
36
36
 
37
37
 
38
- def test_paced_get_retries_a_single_read_error():
38
+ def test_paced_get_retries_a_single_read_error_for_other_sources():
39
39
  client = _ReadErrorClient(failures=1)
40
40
 
41
41
  response = sources._paced_get(client, "https://dblp.org/test", "dblp", 0)
@@ -44,15 +44,46 @@ def test_paced_get_retries_a_single_read_error():
44
44
  assert client.calls == 2
45
45
 
46
46
 
47
+ @pytest.mark.parametrize("failure", ["timeout", 500, 429])
48
+ def test_dblp_get_never_retries(failure):
49
+ class Client:
50
+ calls = 0
51
+
52
+ def get(self, url, params=None, timeout=None):
53
+ self.calls += 1
54
+ request = httpx.Request("GET", url)
55
+ if failure == "timeout":
56
+ raise httpx.ReadTimeout("slow", request=request)
57
+ return httpx.Response(failure, request=request)
58
+
59
+ client = Client()
60
+ expected = SourceUnavailable if failure == 429 else TransientSourceError
61
+ with pytest.raises(expected):
62
+ sources._dblp_get(client, "https://dblp.org/test")
63
+ assert client.calls == 1
64
+
65
+
66
+ def test_dblp_timeout_has_short_phase_caps(monkeypatch):
67
+ observed = None
68
+
69
+ class Client:
70
+ def get(self, url, params=None, timeout=None):
71
+ nonlocal observed
72
+ observed = timeout
73
+ return httpx.Response(200, request=httpx.Request("GET", url))
74
+
75
+ sources._dblp_get(Client(), "https://dblp.org/test")
76
+ assert isinstance(observed, httpx.Timeout)
77
+ assert observed.connect == 1.5
78
+ assert observed.read == 2.5
79
+ assert observed.write == 1.0
80
+ assert observed.pool == 1.0
81
+
82
+
47
83
  def test_dblp_read_failures_do_not_disable_later_batch_entries(monkeypatch):
48
84
  def dblp(title, year, arxiv_id, author_hint):
49
85
  if title == "First paper":
50
- return sources._paced_get(
51
- _ReadErrorClient(failures=3),
52
- "https://dblp.org/test",
53
- "dblp",
54
- 0,
55
- )
86
+ return sources._dblp_get(_ReadErrorClient(failures=3), "https://dblp.org/test")
56
87
  return Match(source="dblp", venue="TMLR", title=title, year="2025")
57
88
 
58
89
  monkeypatch.setattr(
@@ -110,6 +141,44 @@ def test_requests_use_only_the_remaining_publication_budget(monkeypatch):
110
141
  del sources._REQUEST_DEADLINE.value
111
142
 
112
143
 
144
+ def test_exact_and_fuzzy_share_one_four_second_dblp_budget(monkeypatch):
145
+ deadlines = []
146
+ real_with_deadline = sources._with_deadline
147
+
148
+ def capture(deadline, fn, *args):
149
+ if fn in {exact, fuzzy}:
150
+ deadlines.append(deadline)
151
+ return real_with_deadline(deadline, fn, *args)
152
+
153
+ def exact(*args):
154
+ return None
155
+
156
+ def fuzzy(*args):
157
+ return None
158
+
159
+ started = sources.time.monotonic()
160
+ monkeypatch.setattr(sources, "CASCADE", (("dblp", exact),))
161
+ monkeypatch.setattr(sources, "try_dblp_fuzzy", fuzzy)
162
+ monkeypatch.setattr(sources, "_with_deadline", capture)
163
+
164
+ match, status = find_published("A paper", author_hint="author")
165
+
166
+ assert (match, status) == (None, "not_found")
167
+ assert len(deadlines) == 2
168
+ assert deadlines[0] == deadlines[1]
169
+ assert deadlines[0] <= started + 4.01
170
+
171
+
172
+ def test_expired_budget_prevents_author_fuzzy_from_claiming_not_found(monkeypatch):
173
+ clock = iter((0.0, 4.0))
174
+ monkeypatch.setattr(sources.time, "monotonic", lambda: next(clock, 4.0))
175
+ monkeypatch.setattr(sources, "CASCADE", (("dblp", lambda *args: None),))
176
+
177
+ match, status = find_published("A paper", author_hint="author")
178
+
179
+ assert (match, status) == (None, "incomplete")
180
+
181
+
113
182
  def test_upgrade_retries_dblp_after_previous_entry_read_failures(
114
183
  tmp_path, monkeypatch
115
184
  ):
@@ -133,12 +202,7 @@ def test_upgrade_retries_dblp_after_previous_entry_read_failures(
133
202
 
134
203
  def dblp(title, year, arxiv_id, author_hint):
135
204
  if title == "First paper":
136
- return sources._paced_get(
137
- _ReadErrorClient(failures=3),
138
- "https://dblp.org/test",
139
- "dblp",
140
- 0,
141
- )
205
+ return sources._dblp_get(_ReadErrorClient(failures=3), "https://dblp.org/test")
142
206
  return Match(source="dblp", venue="TMLR", title=title, year="2025")
143
207
 
144
208
  monkeypatch.setattr(
@@ -43,7 +43,7 @@ def test_core_source_429_taints_verdict(monkeypatch):
43
43
  monkeypatch.setattr(
44
44
  sources,
45
45
  "CASCADE",
46
- _cascade(dblp="raise", unpaywall=None, crossref=None),
46
+ _cascade(dblp="raise", openalex=None, crossref=None),
47
47
  )
48
48
  match, status = find_published("Some Title", author_hint="smith")
49
49
  assert (match, status) == (None, "incomplete")
@@ -59,17 +59,6 @@ def test_previously_disabled_core_source_taints_next_queries(monkeypatch):
59
59
  assert (match, status) == (None, "incomplete")
60
60
 
61
61
 
62
- def test_noncore_outage_does_not_taint(monkeypatch):
63
- # Unpaywall outages are routine; a miss stays trustworthy.
64
- monkeypatch.setattr(
65
- sources,
66
- "CASCADE",
67
- _cascade(dblp=None, unpaywall="raise", crossref=None),
68
- )
69
- match, status = find_published("Some Title", author_hint="smith")
70
- assert (match, status) == (None, "not_found")
71
-
72
-
73
62
  def test_everything_down_is_unavailable(monkeypatch):
74
63
  monkeypatch.setattr(
75
64
  sources, "CASCADE", _cascade(dblp="raise", semanticscholar="raise")
@@ -18,7 +18,7 @@ wheels = [
18
18
 
19
19
  [[package]]
20
20
  name = "bibcite-cli"
21
- version = "0.6.4"
21
+ version = "0.6.5"
22
22
  source = { editable = "." }
23
23
  dependencies = [
24
24
  { name = "bibtexparser" },
File without changes
File without changes