bibcite-cli 0.6.4__tar.gz → 0.6.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/PKG-INFO +9 -4
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/Readme.md +8 -3
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/pyproject.toml +1 -1
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/src/bibcite/sources.py +277 -166
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/src/bibcite/venues.py +1 -1
- bibcite_cli-0.6.6/tests/test_dblp.py +266 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_no_google_scholar.py +5 -9
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_round3.py +19 -6
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_source_retries.py +77 -13
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_status_semantics.py +1 -12
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/uv.lock +1 -1
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/.github/workflows/ci.yml +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/.github/workflows/publish.yml +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/.gitignore +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/LICENSE +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/assets/bibcite.svg +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/skills/bibcite/SKILL.md +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/src/bibcite/__init__.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/src/bibcite/bibfile.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/src/bibcite/cache.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/src/bibcite/cli.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/src/bibcite/data/strings.bib +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/src/bibcite/normalize.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/src/bibcite/resolve.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_bibfile.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_bugfixes.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_cli_status.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_entry_types.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_normalize.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_public_service.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_round2.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_strings_override.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_venues.py +0 -0
- {bibcite_cli-0.6.4 → bibcite_cli-0.6.6}/tests/test_webpages.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bibcite-cli
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.6
|
|
4
4
|
Summary: Resolve papers (arXiv id / DOI / title) to canonical, normalized BibTeX for agents and humans
|
|
5
5
|
Project-URL: Repository, https://github.com/leo1oel/bibcite
|
|
6
6
|
License-Expression: MIT
|
|
@@ -158,10 +158,15 @@ Mark a confirmed preprint-only entry with `pubstate = {preprint}` if you want `c
|
|
|
158
158
|
|
|
159
159
|
## How resolution works
|
|
160
160
|
|
|
161
|
-
For arXiv IDs and titles, `bibcite` collects paper metadata and checks publication sources in a cascade derived from [PaperMemory](https://github.com/vict0rsch/PaperMemory): DBLP, Semantic Scholar, Crossref,
|
|
161
|
+
For arXiv IDs and titles, `bibcite` collects paper metadata and checks publication sources in a cascade derived from [PaperMemory](https://github.com/vict0rsch/PaperMemory): DBLP, Semantic Scholar, Crossref, and OpenAlex.
|
|
162
162
|
Google Scholar scraping is excluded so CAPTCHA challenges cannot delay publication lookup.
|
|
163
163
|
A published match must have the same normalized title or pass a guarded title-drift check, have a plausible publication year, and name a non-preprint venue.
|
|
164
164
|
|
|
165
|
+
DBLP requests are single-attempt, with short connection/read timeouts and a shared four-second scheduling budget for exact and title-drift lookup.
|
|
166
|
+
Socket timeouts apply per network phase, so this budget is not a hard end-to-end deadline.
|
|
167
|
+
Published DBLP records are constructed from the search response, without another request for a BibTeX export.
|
|
168
|
+
A failed or unfinished DBLP check stays incomplete rather than being treated as evidence that a paper is unpublished.
|
|
169
|
+
|
|
165
170
|
Successful published matches are cached at `~/.cache/bibcite/published.json`.
|
|
166
171
|
Preprint-only results are never cached because a paper may be published later.
|
|
167
172
|
Use `--no-cache` or set `BIBCITE_NO_CACHE=1` to bypass the cache.
|
|
@@ -176,7 +181,7 @@ These optional environment variables improve source reliability:
|
|
|
176
181
|
| --- | --- |
|
|
177
182
|
| `OPENALEX_API_KEY` | Uses your OpenAlex quota instead of the anonymous shared pool. |
|
|
178
183
|
| `S2_API_KEY` | Uses a private Semantic Scholar quota. |
|
|
179
|
-
| `BIBCITE_MAILTO` | Sends your contact email to the Crossref
|
|
184
|
+
| `BIBCITE_MAILTO` | Sends your contact email to the Crossref and OpenAlex polite pools. |
|
|
180
185
|
| `BIBCITE_PUBLIC_SERVICE_URL` | Routes keyless OpenAlex, Semantic Scholar, and Crossref requests through an HTTPS `/v1/query` service supplied by the embedding application. |
|
|
181
186
|
| `BIBCITE_CORE_SOURCES` | Overrides the sources required for a trustworthy publication check. |
|
|
182
187
|
| `BIBCITE_NO_CACHE=1` | Disables the local publication cache. |
|
|
@@ -213,7 +218,7 @@ uv tool install --editable .
|
|
|
213
218
|
|
|
214
219
|
## Acknowledgements
|
|
215
220
|
|
|
216
|
-
Inspired by [PaperMemory](https://github.com/vict0rsch/PaperMemory), with formatting by [bibtex-tidy](https://github.com/FlamingTempura/bibtex-tidy) and metadata from arXiv, DBLP, Semantic Scholar, Crossref,
|
|
221
|
+
Inspired by [PaperMemory](https://github.com/vict0rsch/PaperMemory), with formatting by [bibtex-tidy](https://github.com/FlamingTempura/bibtex-tidy) and metadata from arXiv, DBLP, Semantic Scholar, Crossref, and OpenAlex.
|
|
217
222
|
|
|
218
223
|
## License
|
|
219
224
|
|
|
@@ -145,10 +145,15 @@ Mark a confirmed preprint-only entry with `pubstate = {preprint}` if you want `c
|
|
|
145
145
|
|
|
146
146
|
## How resolution works
|
|
147
147
|
|
|
148
|
-
For arXiv IDs and titles, `bibcite` collects paper metadata and checks publication sources in a cascade derived from [PaperMemory](https://github.com/vict0rsch/PaperMemory): DBLP, Semantic Scholar, Crossref,
|
|
148
|
+
For arXiv IDs and titles, `bibcite` collects paper metadata and checks publication sources in a cascade derived from [PaperMemory](https://github.com/vict0rsch/PaperMemory): DBLP, Semantic Scholar, Crossref, and OpenAlex.
|
|
149
149
|
Google Scholar scraping is excluded so CAPTCHA challenges cannot delay publication lookup.
|
|
150
150
|
A published match must have the same normalized title or pass a guarded title-drift check, have a plausible publication year, and name a non-preprint venue.
|
|
151
151
|
|
|
152
|
+
DBLP requests are single-attempt, with short connection/read timeouts and a shared four-second scheduling budget for exact and title-drift lookup.
|
|
153
|
+
Socket timeouts apply per network phase, so this budget is not a hard end-to-end deadline.
|
|
154
|
+
Published DBLP records are constructed from the search response, without another request for a BibTeX export.
|
|
155
|
+
A failed or unfinished DBLP check stays incomplete rather than being treated as evidence that a paper is unpublished.
|
|
156
|
+
|
|
152
157
|
Successful published matches are cached at `~/.cache/bibcite/published.json`.
|
|
153
158
|
Preprint-only results are never cached because a paper may be published later.
|
|
154
159
|
Use `--no-cache` or set `BIBCITE_NO_CACHE=1` to bypass the cache.
|
|
@@ -163,7 +168,7 @@ These optional environment variables improve source reliability:
|
|
|
163
168
|
| --- | --- |
|
|
164
169
|
| `OPENALEX_API_KEY` | Uses your OpenAlex quota instead of the anonymous shared pool. |
|
|
165
170
|
| `S2_API_KEY` | Uses a private Semantic Scholar quota. |
|
|
166
|
-
| `BIBCITE_MAILTO` | Sends your contact email to the Crossref
|
|
171
|
+
| `BIBCITE_MAILTO` | Sends your contact email to the Crossref and OpenAlex polite pools. |
|
|
167
172
|
| `BIBCITE_PUBLIC_SERVICE_URL` | Routes keyless OpenAlex, Semantic Scholar, and Crossref requests through an HTTPS `/v1/query` service supplied by the embedding application. |
|
|
168
173
|
| `BIBCITE_CORE_SOURCES` | Overrides the sources required for a trustworthy publication check. |
|
|
169
174
|
| `BIBCITE_NO_CACHE=1` | Disables the local publication cache. |
|
|
@@ -200,7 +205,7 @@ uv tool install --editable .
|
|
|
200
205
|
|
|
201
206
|
## Acknowledgements
|
|
202
207
|
|
|
203
|
-
Inspired by [PaperMemory](https://github.com/vict0rsch/PaperMemory), with formatting by [bibtex-tidy](https://github.com/FlamingTempura/bibtex-tidy) and metadata from arXiv, DBLP, Semantic Scholar, Crossref,
|
|
208
|
+
Inspired by [PaperMemory](https://github.com/vict0rsch/PaperMemory), with formatting by [bibtex-tidy](https://github.com/FlamingTempura/bibtex-tidy) and metadata from arXiv, DBLP, Semantic Scholar, Crossref, and OpenAlex.
|
|
204
209
|
|
|
205
210
|
## License
|
|
206
211
|
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
"""API clients for the publication-matching cascade.
|
|
2
2
|
|
|
3
3
|
Order and matching rules ported from PaperMemory's bibMatcher:
|
|
4
|
-
DBLP -> Semantic Scholar -> CrossRef ->
|
|
4
|
+
DBLP -> Semantic Scholar -> CrossRef -> OpenAlex.
|
|
5
5
|
All matchers verify identity via normalized-title equality and reject
|
|
6
6
|
preprint venues (arXiv / CoRR / bioRxiv / ...).
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
import html
|
|
10
|
+
import json
|
|
10
11
|
import os
|
|
11
12
|
import re
|
|
12
13
|
import sys
|
|
@@ -19,7 +20,14 @@ from urllib.parse import urlsplit
|
|
|
19
20
|
|
|
20
21
|
import httpx
|
|
21
22
|
|
|
22
|
-
from .normalize import
|
|
23
|
+
from .normalize import (
|
|
24
|
+
clean_title,
|
|
25
|
+
first_author_last_name,
|
|
26
|
+
mini_hash,
|
|
27
|
+
norm_title,
|
|
28
|
+
sig_tokens,
|
|
29
|
+
titles_similar,
|
|
30
|
+
)
|
|
23
31
|
|
|
24
32
|
UA = "bibcite/0.6 (https://github.com/leo1oel/bibcite)"
|
|
25
33
|
BROWSER_UA = (
|
|
@@ -78,7 +86,11 @@ def _request_timeout(cap: float = TIMEOUT) -> float:
|
|
|
78
86
|
|
|
79
87
|
|
|
80
88
|
def _get(
|
|
81
|
-
c: httpx.Client,
|
|
89
|
+
c: httpx.Client,
|
|
90
|
+
url: str,
|
|
91
|
+
*,
|
|
92
|
+
timeout: float | httpx.Timeout = TIMEOUT,
|
|
93
|
+
**kwargs,
|
|
82
94
|
) -> httpx.Response:
|
|
83
95
|
"""GET directly or route keyless supported APIs through the public service."""
|
|
84
96
|
public_url = os.environ.get("BIBCITE_PUBLIC_SERVICE_URL")
|
|
@@ -98,7 +110,10 @@ def _get(
|
|
|
98
110
|
"crossref": bool(os.environ.get("BIBCITE_MAILTO")),
|
|
99
111
|
}
|
|
100
112
|
if not public_url or not provider or personal_access[provider]:
|
|
101
|
-
|
|
113
|
+
effective_timeout = (
|
|
114
|
+
timeout if isinstance(timeout, httpx.Timeout) else _request_timeout(timeout)
|
|
115
|
+
)
|
|
116
|
+
return c.get(url, timeout=effective_timeout, **kwargs)
|
|
102
117
|
|
|
103
118
|
service = urlsplit(public_url)
|
|
104
119
|
loopback = service.hostname in {"localhost", "127.0.0.1", "::1"}
|
|
@@ -183,7 +198,7 @@ def _s2_headers() -> dict:
|
|
|
183
198
|
|
|
184
199
|
|
|
185
200
|
def _mailto() -> str:
|
|
186
|
-
"""Contact email for the polite pools (CrossRef/OpenAlex
|
|
201
|
+
"""Contact email for the polite pools (CrossRef/OpenAlex).
|
|
187
202
|
Set BIBCITE_MAILTO to use your own."""
|
|
188
203
|
return os.environ.get("BIBCITE_MAILTO") or "bibcite@gmail.com"
|
|
189
204
|
|
|
@@ -340,73 +355,216 @@ def _paced_get(
|
|
|
340
355
|
|
|
341
356
|
# DBLP throttles at roughly 1-2 req/s and escalates to temporary IP bans.
|
|
342
357
|
def _dblp_get(c: httpx.Client, url: str, params: dict | None = None) -> httpx.Response:
|
|
343
|
-
|
|
344
|
-
|
|
358
|
+
wait = 0.8 - (time.monotonic() - _LAST_REQUEST.get("dblp", 0.0))
|
|
359
|
+
if wait > 0:
|
|
360
|
+
_sleep(wait)
|
|
361
|
+
_LAST_REQUEST["dblp"] = time.monotonic()
|
|
362
|
+
remaining = _request_timeout(2.5)
|
|
363
|
+
timeout = httpx.Timeout(
|
|
364
|
+
connect=min(1.5, remaining),
|
|
365
|
+
read=min(2.5, remaining),
|
|
366
|
+
write=min(1.0, remaining),
|
|
367
|
+
pool=min(1.0, remaining),
|
|
368
|
+
)
|
|
369
|
+
try:
|
|
370
|
+
response = _get(c, url, params=params, timeout=timeout)
|
|
371
|
+
except httpx.HTTPError as e:
|
|
372
|
+
raise TransientSourceError(f"dblp unreachable ({type(e).__name__})") from e
|
|
373
|
+
if response.status_code == 429:
|
|
374
|
+
raise SourceUnavailable("dblp rate-limited (429)")
|
|
375
|
+
if response.status_code >= 500:
|
|
376
|
+
raise TransientSourceError(f"dblp server error ({response.status_code})")
|
|
377
|
+
return response
|
|
345
378
|
|
|
346
|
-
def _dblp_sanitize(q: str) -> str:
|
|
347
|
-
"""DBLP's search parser 500s deterministically on queries containing its
|
|
348
|
-
syntax characters (':' in subtitled papers, '?' in question titles).
|
|
349
|
-
They tokenize on punctuation anyway, so replacing with spaces loses
|
|
350
|
-
nothing."""
|
|
351
|
-
return re.sub(r"[^\w\s.-]", " ", q)
|
|
352
379
|
|
|
380
|
+
def _dblp_title_search(c: httpx.Client, title: str) -> list:
|
|
381
|
+
"""Indexed title candidates, not a full-dataset regex or a retry cascade.
|
|
353
382
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
383
|
+
QLever's literal text index avoids the slow CompleteSearch endpoint.
|
|
384
|
+
The caller still verifies the entire normalized title and author identity.
|
|
385
|
+
Limit publications BEFORE expanding author signatures, otherwise a paper
|
|
386
|
+
with many authors can silently lose authors or crowd out the actual match.
|
|
387
|
+
"""
|
|
388
|
+
tokens = sorted(
|
|
389
|
+
set(re.findall(r"[^\W_]+", title.lower())), key=lambda t: (-len(t), t)
|
|
390
|
+
)[:5]
|
|
391
|
+
if not tokens:
|
|
392
|
+
raise TransientSourceError("dblp title has no searchable words")
|
|
393
|
+
words = json.dumps(" ".join(tokens), ensure_ascii=False)
|
|
394
|
+
query = f"""PREFIX dblp: <https://dblp.org/rdf/schema#>
|
|
395
|
+
PREFIX ql: <http://qlever.cs.uni-freiburg.de/builtin-functions/>
|
|
396
|
+
SELECT ?publ ?subject ?predicate ?value WHERE {{
|
|
397
|
+
{{ SELECT DISTINCT ?publ ?title WHERE {{
|
|
398
|
+
?text ql:contains-word {words} ; ql:contains-entity ?title .
|
|
399
|
+
?publ dblp:title ?title .
|
|
400
|
+
}} ORDER BY STRLEN(STR(?title)) ?publ LIMIT 101 }}
|
|
401
|
+
?publ dblp:hasSignature? ?subject .
|
|
402
|
+
?subject ?predicate ?value .
|
|
403
|
+
}}"""
|
|
404
|
+
response = _dblp_get(
|
|
405
|
+
c,
|
|
406
|
+
"https://sparql.dblp.org/sparql",
|
|
407
|
+
params={"query": query, "action": "sparql_json_export"},
|
|
408
|
+
)
|
|
409
|
+
response.raise_for_status()
|
|
410
|
+
# Do not turn an HTML challenge or malformed/partial response into a clean miss.
|
|
411
|
+
rows = response.json()["results"]["bindings"]
|
|
412
|
+
# Read the bounded publication/signature triples instead of joining every
|
|
413
|
+
# optional field with every author. Those joins made live queries ~5s;
|
|
414
|
+
# this indexed property path returns the same fields without a cross product.
|
|
415
|
+
records: dict[str, dict[str, dict[str, list[str]]]] = {}
|
|
416
|
+
for row in rows:
|
|
417
|
+
values = {key: value["value"] for key, value in row.items()}
|
|
418
|
+
url = values["publ"]
|
|
419
|
+
if not url.startswith("https://dblp.org/rec/"):
|
|
420
|
+
raise TransientSourceError(
|
|
421
|
+
"dblp returned an invalid publication identifier"
|
|
422
|
+
)
|
|
423
|
+
subject = records.setdefault(url, {}).setdefault(values["subject"], {})
|
|
424
|
+
subject.setdefault(values["predicate"].rsplit("#", 1)[-1], []).append(
|
|
425
|
+
values["value"]
|
|
426
|
+
)
|
|
427
|
+
kinds = {
|
|
428
|
+
"Article": "Journal Articles",
|
|
429
|
+
"Inproceedings": "Conference and Workshop Papers",
|
|
430
|
+
"Incollection": "Parts in Books or Collections",
|
|
431
|
+
"Book": "Books and Theses",
|
|
432
|
+
"Editorship": "Editorship",
|
|
433
|
+
"Informal": "Informal and Other Publications",
|
|
434
|
+
"Reference": "Parts in Books or Collections",
|
|
435
|
+
}
|
|
436
|
+
fields = {
|
|
437
|
+
"title": "title",
|
|
438
|
+
"yearOfPublication": "year",
|
|
439
|
+
"publishedIn": "venue",
|
|
440
|
+
"pagination": "pages",
|
|
441
|
+
"publishedInJournalVolume": "volume",
|
|
442
|
+
"publishedInJournalVolumeIssue": "number",
|
|
443
|
+
"publishedBy": "publisher",
|
|
444
|
+
"primaryDocumentPage": "ee",
|
|
445
|
+
}
|
|
446
|
+
hits = []
|
|
447
|
+
for url, subjects in records.items():
|
|
448
|
+
record = subjects[url]
|
|
449
|
+
info = {
|
|
450
|
+
target: record[prop][0]
|
|
451
|
+
for prop, target in fields.items()
|
|
452
|
+
if record.get(prop)
|
|
453
|
+
}
|
|
454
|
+
info.update(url=url, key=url.removeprefix("https://dblp.org/rec/"))
|
|
455
|
+
for kind in record.get("type", []):
|
|
456
|
+
if kind.rsplit("#", 1)[-1] in kinds:
|
|
457
|
+
info["type"] = kinds[kind.rsplit("#", 1)[-1]]
|
|
458
|
+
if record.get("doi"):
|
|
459
|
+
info["doi"] = record["doi"][0].removeprefix("https://doi.org/")
|
|
460
|
+
if record.get("isbn"):
|
|
461
|
+
info["isbn"] = record["isbn"][0].removeprefix("urn:isbn:")
|
|
462
|
+
if not info.get("ee") and record.get("documentPage"):
|
|
463
|
+
info["ee"] = record["documentPage"][0]
|
|
464
|
+
authors, editors = [], []
|
|
465
|
+
for signature in record.get("hasSignature", []):
|
|
466
|
+
data = subjects[signature]
|
|
467
|
+
name = data["signatureDblpName"][0]
|
|
468
|
+
ordinal = int(data["signatureOrdinal"][0])
|
|
469
|
+
target = (
|
|
470
|
+
editors
|
|
471
|
+
if any(t.endswith("#EditorSignature") for t in data.get("type", []))
|
|
472
|
+
else authors
|
|
473
|
+
)
|
|
474
|
+
target.append((ordinal, name))
|
|
475
|
+
info["authors"] = {"author": [{"text": name} for _, name in sorted(authors)]}
|
|
476
|
+
info["editor"] = " and ".join(name for _, name in sorted(editors))
|
|
477
|
+
hits.append({"info": info})
|
|
478
|
+
return hits
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _dblp_bibtex(info: dict, venue: str) -> str:
|
|
482
|
+
"""Build the DBLP record from search JSON, avoiding a second .bib request."""
|
|
483
|
+
from .bibfile import entry_to_bibtex
|
|
484
|
+
|
|
485
|
+
record_type = str(info.get("type", "")).lower()
|
|
486
|
+
key = str(info.get("key") or "dblp")
|
|
487
|
+
if record_type == "editorship":
|
|
488
|
+
entry_type = "proceedings" if key.startswith("conf/") else "book"
|
|
489
|
+
elif key.startswith("phd/"):
|
|
490
|
+
entry_type = "phdthesis"
|
|
491
|
+
elif record_type == "parts in books or collections":
|
|
492
|
+
entry_type = "incollection"
|
|
493
|
+
elif record_type == "books and theses":
|
|
494
|
+
entry_type = "book"
|
|
495
|
+
else:
|
|
496
|
+
entry_type = (
|
|
497
|
+
"inproceedings"
|
|
498
|
+
if "conference" in record_type or key.startswith("conf/")
|
|
499
|
+
else "article"
|
|
500
|
+
)
|
|
501
|
+
fields = {
|
|
502
|
+
"ID": key,
|
|
503
|
+
"ENTRYTYPE": entry_type,
|
|
504
|
+
"author": " and ".join(_dblp_hit_authors(info)),
|
|
505
|
+
"title": clean_title(html.unescape(str(info.get("title", "")))),
|
|
506
|
+
"year": str(info.get("year", "")),
|
|
507
|
+
"pages": str(info.get("pages", "")),
|
|
508
|
+
"volume": str(info.get("volume", "")),
|
|
509
|
+
"number": str(info.get("number", "")),
|
|
510
|
+
"doi": str(info.get("doi", "")),
|
|
511
|
+
"url": str(info.get("ee") or info.get("url") or ""),
|
|
512
|
+
"publisher": str(info.get("publisher", "")),
|
|
513
|
+
"editor": str(info.get("editor", "")),
|
|
514
|
+
"isbn": str(info.get("isbn", "")),
|
|
515
|
+
}
|
|
516
|
+
if entry_type in {"inproceedings", "incollection"}:
|
|
517
|
+
fields["booktitle"] = venue
|
|
518
|
+
elif entry_type == "article":
|
|
519
|
+
fields["journal"] = venue
|
|
520
|
+
if record_type == "editorship":
|
|
521
|
+
fields["editor"] = fields["editor"] or fields["author"]
|
|
522
|
+
fields.pop("author")
|
|
523
|
+
return entry_to_bibtex(fields)
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def _dblp_match(info: dict, source: str) -> Match:
|
|
527
|
+
venue = info["venue"]
|
|
528
|
+
if isinstance(venue, list):
|
|
529
|
+
venue = venue[0]
|
|
530
|
+
title = clean_title(html.unescape(info.get("title", "")))
|
|
531
|
+
return Match(
|
|
532
|
+
source=source,
|
|
533
|
+
venue=str(venue),
|
|
534
|
+
title=title,
|
|
535
|
+
year=str(info.get("year", "")),
|
|
536
|
+
authors=_dblp_hit_authors(info),
|
|
537
|
+
doi=info.get("doi", ""),
|
|
538
|
+
bibtex=_dblp_bibtex(info, str(venue)),
|
|
539
|
+
url=info.get("ee", "") or info.get("url", ""),
|
|
540
|
+
)
|
|
366
541
|
|
|
367
542
|
|
|
368
543
|
def try_dblp(title: str, author_hint: str = "") -> Match | None:
|
|
369
|
-
"""
|
|
370
|
-
ranking, so when we know the first author we query with their last name
|
|
371
|
-
first, then fall back to the bare title."""
|
|
372
|
-
queries = []
|
|
373
|
-
if author_hint:
|
|
374
|
-
queries.append(f"{title} {author_hint}")
|
|
375
|
-
queries.append(title)
|
|
544
|
+
"""One SPARQL title lookup; failures never start another network attempt."""
|
|
376
545
|
with _client() as c:
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
pass # keep the match; construct from fields
|
|
400
|
-
_log(f"[dblp] match: {venue} {info.get('year', '')}")
|
|
401
|
-
return Match(
|
|
402
|
-
source="dblp",
|
|
403
|
-
venue=str(venue),
|
|
404
|
-
title=clean_title(html.unescape(info.get("title", ""))),
|
|
405
|
-
year=str(info.get("year", "")),
|
|
406
|
-
doi=info.get("doi", ""),
|
|
407
|
-
bibtex=bibtex,
|
|
408
|
-
url=info.get("ee", "") or info.get("url", ""),
|
|
409
|
-
)
|
|
546
|
+
hits = _dblp_title_search(c, title)
|
|
547
|
+
# Prefer the original conference publication over later journal extensions.
|
|
548
|
+
hits.sort(key=lambda h: int(h.get("info", {}).get("year", 9999)))
|
|
549
|
+
ref = norm_title(title)
|
|
550
|
+
for hit in hits:
|
|
551
|
+
info = hit.get("info", {})
|
|
552
|
+
if norm_title(html.unescape(info.get("title", ""))) != ref:
|
|
553
|
+
continue
|
|
554
|
+
if not info.get("type") or info["type"] == "Informal and Other Publications":
|
|
555
|
+
continue
|
|
556
|
+
if not _is_published_venue(str(info.get("venue", ""))):
|
|
557
|
+
continue
|
|
558
|
+
authors = _dblp_hit_authors(info)
|
|
559
|
+
if author_hint and not any(
|
|
560
|
+
mini_hash(author_hint) == first_author_last_name(name) for name in authors
|
|
561
|
+
):
|
|
562
|
+
continue
|
|
563
|
+
match = _dblp_match(info, "dblp")
|
|
564
|
+
_log(f"[dblp] match: {match.venue} {info.get('year', '')}")
|
|
565
|
+
return match
|
|
566
|
+
if len(hits) >= 101:
|
|
567
|
+
raise TransientSourceError("dblp title candidate window exceeded")
|
|
410
568
|
return None
|
|
411
569
|
|
|
412
570
|
|
|
@@ -414,62 +572,58 @@ def _dblp_hit_authors(info: dict) -> list[str]:
|
|
|
414
572
|
authors = (info.get("authors") or {}).get("author") or []
|
|
415
573
|
if isinstance(authors, dict):
|
|
416
574
|
authors = [authors]
|
|
417
|
-
return [
|
|
575
|
+
return [
|
|
576
|
+
re.sub(r"\s+\d{4}$", "", a.get("text", ""))
|
|
577
|
+
for a in authors
|
|
578
|
+
if isinstance(a, dict) and a.get("text")
|
|
579
|
+
]
|
|
418
580
|
|
|
419
581
|
|
|
420
582
|
def try_dblp_fuzzy(title: str, author_hint: str, year: str = "") -> Match | None:
|
|
421
583
|
"""Title-drift fallback: camera-ready titles often differ from the arXiv
|
|
422
584
|
ones ("Information-Theoretic" -> "Information Theory"), and DBLP's
|
|
423
|
-
token-AND search then misses entirely. Query
|
|
424
|
-
|
|
585
|
+
token-AND search then misses entirely. Query the most distinctive
|
|
586
|
+
title tokens instead, and accept token-Jaccard-similar
|
|
425
587
|
titles — guarded by author and year so different papers can't sneak in.
|
|
426
588
|
"""
|
|
427
589
|
if not author_hint:
|
|
428
590
|
return None
|
|
429
|
-
tokens = sorted(sig_tokens(title), key=len,
|
|
591
|
+
tokens = sorted(sig_tokens(title), key=lambda t: (-len(t), t))[:3]
|
|
430
592
|
if not tokens:
|
|
431
593
|
return None
|
|
432
|
-
q = " ".join(
|
|
594
|
+
q = " ".join(tokens)
|
|
433
595
|
with _client() as c:
|
|
434
|
-
hits =
|
|
596
|
+
hits = _dblp_title_search(c, q)
|
|
435
597
|
hits.sort(key=lambda h: int(h.get("info", {}).get("year", 9999)))
|
|
436
598
|
for hit in hits:
|
|
437
599
|
info = hit.get("info", {})
|
|
438
600
|
hit_title = clean_title(html.unescape(info.get("title", "")))
|
|
439
|
-
if
|
|
601
|
+
if (
|
|
602
|
+
not info.get("type")
|
|
603
|
+
or info["type"] == "Informal and Other Publications"
|
|
604
|
+
):
|
|
605
|
+
continue
|
|
606
|
+
if not _is_published_venue(str(info.get("venue", ""))):
|
|
440
607
|
continue
|
|
441
608
|
if not titles_similar(hit_title, title):
|
|
442
609
|
continue
|
|
443
610
|
if year and info.get("year"):
|
|
444
611
|
if abs(int(info["year"]) - int(year)) > 2:
|
|
445
612
|
continue
|
|
446
|
-
|
|
447
|
-
|
|
613
|
+
if not any(
|
|
614
|
+
mini_hash(author_hint) == first_author_last_name(name)
|
|
615
|
+
for name in _dblp_hit_authors(info)
|
|
616
|
+
):
|
|
448
617
|
continue
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
venue = venue[0]
|
|
452
|
-
bibtex = ""
|
|
453
|
-
if info.get("url"):
|
|
454
|
-
try:
|
|
455
|
-
br = _dblp_get(c, info["url"] + ".bib")
|
|
456
|
-
if br.status_code == 200:
|
|
457
|
-
bibtex = br.text
|
|
458
|
-
except SourceUnavailable:
|
|
459
|
-
pass # keep the match; construct from fields
|
|
618
|
+
match = _dblp_match(info, "dblp-fuzzy")
|
|
619
|
+
venue = match.venue
|
|
460
620
|
_log(
|
|
461
621
|
f"[dblp-fuzzy] match with title drift: '{hit_title}' "
|
|
462
622
|
f"@ {venue} {info.get('year', '')}"
|
|
463
623
|
)
|
|
464
|
-
return
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
title=hit_title,
|
|
468
|
-
year=str(info.get("year", "")),
|
|
469
|
-
doi=info.get("doi", ""),
|
|
470
|
-
bibtex=bibtex,
|
|
471
|
-
url=info.get("ee", "") or info.get("url", ""),
|
|
472
|
-
)
|
|
624
|
+
return match
|
|
625
|
+
if len(hits) >= 101:
|
|
626
|
+
raise TransientSourceError("dblp title candidate window exceeded")
|
|
473
627
|
return None
|
|
474
628
|
|
|
475
629
|
|
|
@@ -615,9 +769,8 @@ def try_crossref(title: str) -> Match | None:
|
|
|
615
769
|
if r.status_code == 429:
|
|
616
770
|
raise SourceUnavailable("CrossRef rate-limited (429)")
|
|
617
771
|
if r.status_code >= 500:
|
|
618
|
-
# A dead endpoint
|
|
619
|
-
#
|
|
620
|
-
# to every remaining query.
|
|
772
|
+
# A dead endpoint gets benched for the run instead of adding
|
|
773
|
+
# latency and noise to every remaining query.
|
|
621
774
|
raise SourceUnavailable(f"CrossRef server error ({r.status_code})")
|
|
622
775
|
r.raise_for_status()
|
|
623
776
|
payload = r.json()
|
|
@@ -660,52 +813,6 @@ def try_crossref(title: str) -> Match | None:
|
|
|
660
813
|
return None
|
|
661
814
|
|
|
662
815
|
|
|
663
|
-
# ---------------------------------------------------------------------------
|
|
664
|
-
# Unpaywall
|
|
665
|
-
# ---------------------------------------------------------------------------
|
|
666
|
-
|
|
667
|
-
def try_unpaywall(title: str) -> Match | None:
|
|
668
|
-
with _client() as c:
|
|
669
|
-
r = _get(
|
|
670
|
-
c,
|
|
671
|
-
"https://api.unpaywall.org/v2/search",
|
|
672
|
-
params={"query": title, "is_oa": "true", "email": _mailto()},
|
|
673
|
-
)
|
|
674
|
-
if r.status_code == 429:
|
|
675
|
-
raise SourceUnavailable("Unpaywall rate-limited (429)")
|
|
676
|
-
if r.status_code >= 500:
|
|
677
|
-
# A dead endpoint (Unpaywall search 500s for days at a time)
|
|
678
|
-
# gets benched for the run instead of adding latency + noise
|
|
679
|
-
# to every remaining query.
|
|
680
|
-
raise SourceUnavailable(f"Unpaywall server error ({r.status_code})")
|
|
681
|
-
r.raise_for_status()
|
|
682
|
-
ref = norm_title(title)
|
|
683
|
-
for res in r.json().get("results") or []:
|
|
684
|
-
resp = res.get("response", {})
|
|
685
|
-
if norm_title(resp.get("title", "")) != ref:
|
|
686
|
-
continue
|
|
687
|
-
venue = (resp.get("journal_name") or "").strip()
|
|
688
|
-
if not _is_published_venue(venue):
|
|
689
|
-
continue
|
|
690
|
-
doi = resp.get("doi", "")
|
|
691
|
-
if ARXIV_DOI.match(doi):
|
|
692
|
-
continue
|
|
693
|
-
authors = [
|
|
694
|
-
" ".join(filter(None, [a.get("given"), a.get("family")]))
|
|
695
|
-
for a in resp.get("z_authors") or []
|
|
696
|
-
]
|
|
697
|
-
_log(f"[unpaywall] match: {venue} {resp.get('year', '')}")
|
|
698
|
-
return Match(
|
|
699
|
-
source="unpaywall",
|
|
700
|
-
venue=venue,
|
|
701
|
-
title=clean_title(resp.get("title", "")),
|
|
702
|
-
year=str(resp.get("year") or ""),
|
|
703
|
-
authors=[a for a in authors if a],
|
|
704
|
-
doi=doi,
|
|
705
|
-
)
|
|
706
|
-
return None
|
|
707
|
-
|
|
708
|
-
|
|
709
816
|
# ---------------------------------------------------------------------------
|
|
710
817
|
# OpenAlex (not in PaperMemory; unauthenticated with generous rate limits, so
|
|
711
818
|
# it doubles as the metadata fallback when the arXiv API / S2 are throttled)
|
|
@@ -722,9 +829,8 @@ def openalex_search(title: str) -> dict | None:
|
|
|
722
829
|
if r.status_code == 429:
|
|
723
830
|
raise SourceUnavailable("OpenAlex rate-limited (429)")
|
|
724
831
|
if r.status_code >= 500:
|
|
725
|
-
# A dead endpoint
|
|
726
|
-
#
|
|
727
|
-
# to every remaining query.
|
|
832
|
+
# A dead endpoint gets benched for the run instead of adding
|
|
833
|
+
# latency and noise to every remaining query.
|
|
728
834
|
raise SourceUnavailable(f"OpenAlex server error ({r.status_code})")
|
|
729
835
|
r.raise_for_status()
|
|
730
836
|
ref = norm_title(title)
|
|
@@ -829,7 +935,6 @@ CASCADE = (
|
|
|
829
935
|
("dblp", lambda t, y, a, au: try_dblp(t, au)),
|
|
830
936
|
("semanticscholar", lambda t, y, a, au: try_semantic_scholar(t, y, a)),
|
|
831
937
|
("crossref", lambda t, y, a, au: try_crossref(t)),
|
|
832
|
-
("unpaywall", lambda t, y, a, au: try_unpaywall(t)),
|
|
833
938
|
("openalex", lambda t, y, a, au: try_openalex(t)),
|
|
834
939
|
)
|
|
835
940
|
|
|
@@ -840,8 +945,7 @@ CASCADE = (
|
|
|
840
945
|
_DISABLED: dict[str, str] = {}
|
|
841
946
|
|
|
842
947
|
# Only these sources are authoritative enough that losing one taints a miss
|
|
843
|
-
# into "incomplete".
|
|
844
|
-
# "not_found" from ever being trustworthy.
|
|
948
|
+
# into "incomplete".
|
|
845
949
|
# Override with BIBCITE_CORE_SOURCES="dblp,semanticscholar" if one of these
|
|
846
950
|
# is down for days and keeps every verdict incomplete.
|
|
847
951
|
CORE_SOURCES = frozenset(
|
|
@@ -880,7 +984,9 @@ def find_published(
|
|
|
880
984
|
# version (the common case) misses everywhere, and used to pay the *sum* of
|
|
881
985
|
# each source's latency; now the wall-clock is the slowest single source.
|
|
882
986
|
# The first verified hit by CASCADE priority still wins.
|
|
883
|
-
|
|
987
|
+
started = time.monotonic()
|
|
988
|
+
deadline = started + PUBLICATION_TIMEOUT
|
|
989
|
+
dblp_deadline = min(deadline, started + 4.0)
|
|
884
990
|
active = [(name, fn) for name, fn in CASCADE if name not in _DISABLED]
|
|
885
991
|
outcomes: dict[str, tuple] = {}
|
|
886
992
|
if active:
|
|
@@ -888,7 +994,7 @@ def find_published(
|
|
|
888
994
|
futures = {
|
|
889
995
|
pool.submit(
|
|
890
996
|
_with_deadline,
|
|
891
|
-
deadline,
|
|
997
|
+
dblp_deadline if name == "dblp" else deadline,
|
|
892
998
|
fn,
|
|
893
999
|
title,
|
|
894
1000
|
year,
|
|
@@ -940,23 +1046,28 @@ def find_published(
|
|
|
940
1046
|
author_hint
|
|
941
1047
|
and dblp_outcome is not None
|
|
942
1048
|
and dblp_outcome[0] == "miss"
|
|
943
|
-
and time.monotonic() < deadline
|
|
944
1049
|
):
|
|
945
|
-
|
|
946
|
-
m = _with_deadline(deadline, try_dblp_fuzzy, title, author_hint, year)
|
|
947
|
-
if m:
|
|
948
|
-
cache.put(cache_key, m.__dict__)
|
|
949
|
-
return m, "found"
|
|
950
|
-
clean_misses += 1
|
|
951
|
-
except TransientSourceError as e:
|
|
952
|
-
incomplete = True
|
|
953
|
-
_log(f"[dblp-fuzzy] transient failure for this entry: {e}")
|
|
954
|
-
except SourceUnavailable as e:
|
|
955
|
-
_DISABLED["dblp"] = str(e)
|
|
1050
|
+
if time.monotonic() >= dblp_deadline:
|
|
956
1051
|
incomplete = True
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
1052
|
+
_log("[dblp-fuzzy] skipped: DBLP lookup budget exhausted")
|
|
1053
|
+
else:
|
|
1054
|
+
try:
|
|
1055
|
+
m = _with_deadline(
|
|
1056
|
+
dblp_deadline, try_dblp_fuzzy, title, author_hint, year
|
|
1057
|
+
)
|
|
1058
|
+
if m:
|
|
1059
|
+
cache.put(cache_key, m.__dict__)
|
|
1060
|
+
return m, "found"
|
|
1061
|
+
clean_misses += 1
|
|
1062
|
+
except TransientSourceError as e:
|
|
1063
|
+
incomplete = True
|
|
1064
|
+
_log(f"[dblp-fuzzy] transient failure for this entry: {e}")
|
|
1065
|
+
except SourceUnavailable as e:
|
|
1066
|
+
_DISABLED["dblp"] = str(e)
|
|
1067
|
+
incomplete = True
|
|
1068
|
+
except Exception as e:
|
|
1069
|
+
incomplete = True
|
|
1070
|
+
_log(f"[dblp-fuzzy] error: {type(e).__name__}: {e}")
|
|
960
1071
|
if not clean_misses:
|
|
961
1072
|
return None, "unavailable"
|
|
962
1073
|
return None, ("incomplete" if incomplete else "not_found")
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Parses the vendored ``data/strings.bib`` @string table (journals /
|
|
4
4
|
conferences / workshops) and maps venue strings returned by DBLP, Semantic
|
|
5
|
-
Scholar, CrossRef,
|
|
5
|
+
Scholar, CrossRef, OpenAlex, etc. onto the canonical names.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
import re
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import httpx
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
import bibcite.sources as sources
|
|
5
|
+
from bibcite.bibfile import parse_bibtex_entry
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.fixture(autouse=True)
|
|
9
|
+
def isolated(monkeypatch):
|
|
10
|
+
monkeypatch.setattr(sources, "_LAST_REQUEST", {})
|
|
11
|
+
monkeypatch.setattr(sources.time, "sleep", lambda _: None)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _hit(title="A Published Paper"):
|
|
15
|
+
return {
|
|
16
|
+
"info": {
|
|
17
|
+
"authors": {
|
|
18
|
+
"author": [
|
|
19
|
+
{"text": "Ada Lovelace 0001"},
|
|
20
|
+
{"text": "Grace Hopper"},
|
|
21
|
+
]
|
|
22
|
+
},
|
|
23
|
+
"title": title,
|
|
24
|
+
"venue": "Test Conference",
|
|
25
|
+
"year": "2025",
|
|
26
|
+
"pages": "12-24",
|
|
27
|
+
"volume": "42",
|
|
28
|
+
"number": "3",
|
|
29
|
+
"doi": "10.1000/test",
|
|
30
|
+
"ee": "https://doi.org/10.1000/test",
|
|
31
|
+
"url": "https://dblp.org/rec/conf/test/Lovelace25",
|
|
32
|
+
"key": "conf/test/Lovelace25",
|
|
33
|
+
"type": "Conference and Workshop Papers",
|
|
34
|
+
"publisher": "Example Press",
|
|
35
|
+
"editor": "Test Editor",
|
|
36
|
+
"isbn": "978-0-00-000000-0",
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@pytest.mark.parametrize("fuzzy", [False, True])
|
|
42
|
+
def test_dblp_match_uses_search_json_without_bib_download(monkeypatch, fuzzy):
|
|
43
|
+
calls = []
|
|
44
|
+
query_title = (
|
|
45
|
+
"An Information-Theoretic Perspective on Variance-Invariance-Covariance Regularization"
|
|
46
|
+
if fuzzy
|
|
47
|
+
else "A Published Paper"
|
|
48
|
+
)
|
|
49
|
+
hit = _hit(
|
|
50
|
+
"An Information Theory Perspective on Variance-Invariance-Covariance Regularization"
|
|
51
|
+
if fuzzy
|
|
52
|
+
else query_title
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
def get(client, url, params=None):
|
|
56
|
+
calls.append((url, params))
|
|
57
|
+
return httpx.Response(
|
|
58
|
+
200,
|
|
59
|
+
request=httpx.Request("GET", url),
|
|
60
|
+
json={"result": {"hits": {"hit": [hit]}}},
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
monkeypatch.setattr(sources, "_dblp_get", get)
|
|
64
|
+
monkeypatch.setattr(sources, "_dblp_title_search", lambda *args: [hit])
|
|
65
|
+
match = (
|
|
66
|
+
sources.try_dblp_fuzzy(query_title, "lovelace", "2024")
|
|
67
|
+
if fuzzy
|
|
68
|
+
else sources.try_dblp(query_title)
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
assert match is not None
|
|
72
|
+
assert calls == []
|
|
73
|
+
entry = parse_bibtex_entry(match.bibtex)
|
|
74
|
+
assert entry["ENTRYTYPE"] == "inproceedings"
|
|
75
|
+
assert entry["author"] == "Ada Lovelace and Grace Hopper"
|
|
76
|
+
assert entry["title"] == hit["info"]["title"]
|
|
77
|
+
assert entry["booktitle"] == "Test Conference"
|
|
78
|
+
for field in (
|
|
79
|
+
"year",
|
|
80
|
+
"pages",
|
|
81
|
+
"volume",
|
|
82
|
+
"number",
|
|
83
|
+
"doi",
|
|
84
|
+
"url",
|
|
85
|
+
"publisher",
|
|
86
|
+
"editor",
|
|
87
|
+
"isbn",
|
|
88
|
+
):
|
|
89
|
+
expected = hit["info"]["ee"] if field == "url" else hit["info"][field]
|
|
90
|
+
assert entry[field] == expected
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_dblp_json_type_distinguishes_journal_articles():
|
|
94
|
+
info = _hit()["info"]
|
|
95
|
+
info.update(
|
|
96
|
+
{
|
|
97
|
+
"type": "Journal Articles",
|
|
98
|
+
"key": "journals/test/Lovelace25",
|
|
99
|
+
"venue": "Test Journal",
|
|
100
|
+
}
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
entry = parse_bibtex_entry(sources._dblp_match(info, "dblp").bibtex)
|
|
104
|
+
|
|
105
|
+
assert entry["ENTRYTYPE"] == "article"
|
|
106
|
+
assert entry["journal"] == "Test Journal"
|
|
107
|
+
assert "booktitle" not in entry
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@pytest.mark.parametrize(
|
|
111
|
+
("record_type", "key", "expected"),
|
|
112
|
+
[
|
|
113
|
+
("Books and Theses", "books/example/Test", "book"),
|
|
114
|
+
("Books and Theses", "phd/example/Test", "phdthesis"),
|
|
115
|
+
("Parts in Books or Collections", "books/example/TestChapter", "incollection"),
|
|
116
|
+
("Editorship", "conf/example/2025", "proceedings"),
|
|
117
|
+
],
|
|
118
|
+
)
|
|
119
|
+
def test_search_json_preserves_other_record_types(record_type, key, expected):
|
|
120
|
+
info = _hit()["info"]
|
|
121
|
+
info.update(type=record_type, key=key)
|
|
122
|
+
entry = parse_bibtex_entry(sources._dblp_match(info, "dblp").bibtex)
|
|
123
|
+
assert entry["ENTRYTYPE"] == expected
|
|
124
|
+
assert "journal" not in entry
|
|
125
|
+
if expected == "proceedings":
|
|
126
|
+
assert entry["editor"] == "Test Editor"
|
|
127
|
+
assert "author" not in entry
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def test_dblp_exact_still_rejects_a_different_title(monkeypatch):
|
|
131
|
+
monkeypatch.setattr(
|
|
132
|
+
sources, "_dblp_title_search", lambda *args: [_hit("Other Paper")]
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
assert sources.try_dblp("A Published Paper") is None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@pytest.mark.parametrize(
|
|
139
|
+
("author", "year"),
|
|
140
|
+
[("different", "2024"), ("lovelace", "2010")],
|
|
141
|
+
)
|
|
142
|
+
def test_dblp_fuzzy_still_requires_author_and_plausible_year(monkeypatch, author, year):
|
|
143
|
+
drifted = _hit(
|
|
144
|
+
"An Information Theory Perspective on Variance-Invariance-Covariance Regularization"
|
|
145
|
+
)
|
|
146
|
+
monkeypatch.setattr(sources, "_dblp_title_search", lambda *args: [drifted])
|
|
147
|
+
|
|
148
|
+
match = sources.try_dblp_fuzzy(
|
|
149
|
+
"An Information-Theoretic Perspective on Variance-Invariance-Covariance Regularization",
|
|
150
|
+
author,
|
|
151
|
+
year,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
assert match is None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _triples(key="conf/test/Lovelace25", title="A Published Paper."):
|
|
158
|
+
prefix = "https://dblp.org/rdf/schema#"
|
|
159
|
+
url = "https://dblp.org/rec/" + key
|
|
160
|
+
triples = [
|
|
161
|
+
(url, "title", title),
|
|
162
|
+
(url, "yearOfPublication", "2025"),
|
|
163
|
+
(url, "publishedIn", "Test Conference"),
|
|
164
|
+
(url, "type", prefix + "Inproceedings"),
|
|
165
|
+
(url, "type", prefix + "Publication"),
|
|
166
|
+
(url, "pagination", "12-24"),
|
|
167
|
+
(url, "primaryDocumentPage", "https://example.org/paper"),
|
|
168
|
+
(url, "publishedInJournalVolume", "42"),
|
|
169
|
+
(url, "publishedInJournalVolumeIssue", "3"),
|
|
170
|
+
(url, "publishedBy", "Example Press"),
|
|
171
|
+
(url, "doi", "https://doi.org/10.1000/test"),
|
|
172
|
+
(url, "hasSignature", "s2"),
|
|
173
|
+
(url, "hasSignature", "s1"),
|
|
174
|
+
("s2", "signatureDblpName", "Grace Hopper"),
|
|
175
|
+
("s2", "signatureOrdinal", "2"),
|
|
176
|
+
("s2", "type", prefix + "AuthorSignature"),
|
|
177
|
+
("s1", "signatureDblpName", "Ada Lovelace 0001"),
|
|
178
|
+
("s1", "signatureOrdinal", "1"),
|
|
179
|
+
("s1", "type", prefix + "AuthorSignature"),
|
|
180
|
+
]
|
|
181
|
+
return [
|
|
182
|
+
{
|
|
183
|
+
k: {"type": "literal", "value": v}
|
|
184
|
+
for k, v in {
|
|
185
|
+
"publ": url,
|
|
186
|
+
"subject": subject,
|
|
187
|
+
"predicate": prefix + predicate,
|
|
188
|
+
"value": value,
|
|
189
|
+
}.items()
|
|
190
|
+
}
|
|
191
|
+
for subject, predicate, value in triples
|
|
192
|
+
]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def test_sparql_returns_complete_ordered_authors_in_one_request(monkeypatch):
|
|
196
|
+
calls = []
|
|
197
|
+
|
|
198
|
+
def get(client, url, params=None):
|
|
199
|
+
calls.append((url, params))
|
|
200
|
+
return httpx.Response(
|
|
201
|
+
200,
|
|
202
|
+
request=httpx.Request("GET", url),
|
|
203
|
+
json={"results": {"bindings": _triples()}},
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
monkeypatch.setattr(sources, "_dblp_get", get)
|
|
207
|
+
match = sources.try_dblp("A Published Paper", "lovelace")
|
|
208
|
+
assert match is not None
|
|
209
|
+
assert match.authors == ["Ada Lovelace", "Grace Hopper"]
|
|
210
|
+
assert match.doi == "10.1000/test"
|
|
211
|
+
entry = parse_bibtex_entry(match.bibtex)
|
|
212
|
+
assert entry["pages"] == "12-24"
|
|
213
|
+
assert entry["url"] == "https://example.org/paper"
|
|
214
|
+
assert entry["volume"] == "42"
|
|
215
|
+
assert entry["number"] == "3"
|
|
216
|
+
assert entry["publisher"] == "Example Press"
|
|
217
|
+
assert entry["ENTRYTYPE"] == "inproceedings"
|
|
218
|
+
assert len(calls) == 1
|
|
219
|
+
assert calls[0][0] == "https://sparql.dblp.org/sparql"
|
|
220
|
+
assert "LIMIT 101" in calls[0][1]["query"]
|
|
221
|
+
assert "dblp:hasSignature?" in calls[0][1]["query"]
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@pytest.mark.parametrize("author", ["different", "lace"])
|
|
225
|
+
def test_sparql_exact_checks_author_not_substring(monkeypatch, author):
|
|
226
|
+
monkeypatch.setattr(sources, "_dblp_title_search", lambda *args: [_hit()])
|
|
227
|
+
assert sources.try_dblp("A Published Paper", author) is None
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def test_sparql_overflow_is_incomplete_not_a_clean_miss(monkeypatch):
|
|
231
|
+
monkeypatch.setattr(
|
|
232
|
+
sources, "_dblp_title_search", lambda *args: [_hit("Other")] * 101
|
|
233
|
+
)
|
|
234
|
+
with pytest.raises(sources.TransientSourceError, match="window exceeded"):
|
|
235
|
+
sources.try_dblp("A Published Paper")
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@pytest.mark.parametrize(
|
|
239
|
+
"payload",
|
|
240
|
+
[{}, {"results": {}}, {"results": {"bindings": [{"publ": {"value": "bad"}}]}}],
|
|
241
|
+
)
|
|
242
|
+
def test_sparql_malformed_data_does_not_become_a_clean_miss(monkeypatch, payload):
|
|
243
|
+
monkeypatch.setattr(
|
|
244
|
+
sources,
|
|
245
|
+
"_dblp_get",
|
|
246
|
+
lambda c, url, params=None: httpx.Response(
|
|
247
|
+
200,
|
|
248
|
+
request=httpx.Request("GET", url),
|
|
249
|
+
json=payload,
|
|
250
|
+
),
|
|
251
|
+
)
|
|
252
|
+
with pytest.raises((KeyError, sources.TransientSourceError)):
|
|
253
|
+
sources.try_dblp("A Published Paper")
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def test_sparql_timeout_does_not_retry_search_endpoint(monkeypatch):
|
|
257
|
+
calls = []
|
|
258
|
+
|
|
259
|
+
def get(client, url, params=None):
|
|
260
|
+
calls.append(url)
|
|
261
|
+
raise sources.TransientSourceError("timeout")
|
|
262
|
+
|
|
263
|
+
monkeypatch.setattr(sources, "_dblp_get", get)
|
|
264
|
+
with pytest.raises(sources.TransientSourceError):
|
|
265
|
+
sources.try_dblp("A Published Paper")
|
|
266
|
+
assert calls == ["https://sparql.dblp.org/sparql"]
|
|
@@ -12,7 +12,7 @@ resolver = importlib.import_module("bibcite.resolve")
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
@pytest.mark.parametrize("operation", ["get", "add", "upgrade"])
|
|
15
|
-
def
|
|
15
|
+
def test_cli_never_queries_removed_sources(operation, monkeypatch, tmp_path, capsys):
|
|
16
16
|
monkeypatch.setattr(cache, "DISABLED", True)
|
|
17
17
|
monkeypatch.setattr(sources, "_DISABLED", {})
|
|
18
18
|
monkeypatch.setattr(
|
|
@@ -27,7 +27,7 @@ def test_cli_never_queries_google_scholar(operation, monkeypatch, tmp_path, caps
|
|
|
27
27
|
),
|
|
28
28
|
)
|
|
29
29
|
visited = []
|
|
30
|
-
for name in ("dblp", "semantic_scholar", "crossref", "
|
|
30
|
+
for name in ("dblp", "semantic_scholar", "crossref", "openalex"):
|
|
31
31
|
|
|
32
32
|
def miss(*args, source=name):
|
|
33
33
|
visited.append(source)
|
|
@@ -59,11 +59,7 @@ def test_cli_never_queries_google_scholar(operation, monkeypatch, tmp_path, caps
|
|
|
59
59
|
args = ["upgrade", str(path), "--dry-run"]
|
|
60
60
|
cli.main(args)
|
|
61
61
|
json.loads(capsys.readouterr().out)
|
|
62
|
-
assert set(visited) == {
|
|
63
|
-
"dblp",
|
|
64
|
-
"semantic_scholar",
|
|
65
|
-
"crossref",
|
|
66
|
-
"unpaywall",
|
|
67
|
-
"openalex",
|
|
68
|
-
}
|
|
62
|
+
assert set(visited) == {"dblp", "semantic_scholar", "crossref", "openalex"}
|
|
69
63
|
assert requests == []
|
|
64
|
+
assert not hasattr(sources, "try_unpaywall")
|
|
65
|
+
assert all("scholar.google" not in url and "unpaywall" not in url for url in requests)
|
|
@@ -86,9 +86,22 @@ def test_scrub_leaves_clean_files_alone(tmp_path: Path):
|
|
|
86
86
|
assert bib.read_text() == original # untouched, not even rewritten
|
|
87
87
|
|
|
88
88
|
|
|
89
|
-
def
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
89
|
+
def test_dblp_query_does_not_interpret_title_as_sparql(monkeypatch):
|
|
90
|
+
import httpx
|
|
91
|
+
from bibcite import sources
|
|
92
|
+
|
|
93
|
+
queries = []
|
|
94
|
+
|
|
95
|
+
def get(client, url, params=None):
|
|
96
|
+
queries.append(params["query"])
|
|
97
|
+
return httpx.Response(
|
|
98
|
+
200, request=httpx.Request("GET", url), json={"results": {"bindings": []}}
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
monkeypatch.setattr(sources, "_dblp_get", get)
|
|
102
|
+
with sources._client() as client:
|
|
103
|
+
sources._dblp_title_search(
|
|
104
|
+
client, 'LeJEPA: Provable? Self-Supervised (Learning) " } #'
|
|
105
|
+
)
|
|
106
|
+
assert '"supervised learning provable lejepa self"' in queries[0]
|
|
107
|
+
assert '#"' not in queries[0]
|
|
@@ -35,7 +35,7 @@ class _ReadErrorClient:
|
|
|
35
35
|
return httpx.Response(200, request=request, json={})
|
|
36
36
|
|
|
37
37
|
|
|
38
|
-
def
|
|
38
|
+
def test_paced_get_retries_a_single_read_error_for_other_sources():
|
|
39
39
|
client = _ReadErrorClient(failures=1)
|
|
40
40
|
|
|
41
41
|
response = sources._paced_get(client, "https://dblp.org/test", "dblp", 0)
|
|
@@ -44,15 +44,46 @@ def test_paced_get_retries_a_single_read_error():
|
|
|
44
44
|
assert client.calls == 2
|
|
45
45
|
|
|
46
46
|
|
|
47
|
+
@pytest.mark.parametrize("failure", ["timeout", 500, 429])
|
|
48
|
+
def test_dblp_get_never_retries(failure):
|
|
49
|
+
class Client:
|
|
50
|
+
calls = 0
|
|
51
|
+
|
|
52
|
+
def get(self, url, params=None, timeout=None):
|
|
53
|
+
self.calls += 1
|
|
54
|
+
request = httpx.Request("GET", url)
|
|
55
|
+
if failure == "timeout":
|
|
56
|
+
raise httpx.ReadTimeout("slow", request=request)
|
|
57
|
+
return httpx.Response(failure, request=request)
|
|
58
|
+
|
|
59
|
+
client = Client()
|
|
60
|
+
expected = SourceUnavailable if failure == 429 else TransientSourceError
|
|
61
|
+
with pytest.raises(expected):
|
|
62
|
+
sources._dblp_get(client, "https://dblp.org/test")
|
|
63
|
+
assert client.calls == 1
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_dblp_timeout_has_short_phase_caps(monkeypatch):
|
|
67
|
+
observed = None
|
|
68
|
+
|
|
69
|
+
class Client:
|
|
70
|
+
def get(self, url, params=None, timeout=None):
|
|
71
|
+
nonlocal observed
|
|
72
|
+
observed = timeout
|
|
73
|
+
return httpx.Response(200, request=httpx.Request("GET", url))
|
|
74
|
+
|
|
75
|
+
sources._dblp_get(Client(), "https://dblp.org/test")
|
|
76
|
+
assert isinstance(observed, httpx.Timeout)
|
|
77
|
+
assert observed.connect == 1.5
|
|
78
|
+
assert observed.read == 2.5
|
|
79
|
+
assert observed.write == 1.0
|
|
80
|
+
assert observed.pool == 1.0
|
|
81
|
+
|
|
82
|
+
|
|
47
83
|
def test_dblp_read_failures_do_not_disable_later_batch_entries(monkeypatch):
|
|
48
84
|
def dblp(title, year, arxiv_id, author_hint):
|
|
49
85
|
if title == "First paper":
|
|
50
|
-
return sources.
|
|
51
|
-
_ReadErrorClient(failures=3),
|
|
52
|
-
"https://dblp.org/test",
|
|
53
|
-
"dblp",
|
|
54
|
-
0,
|
|
55
|
-
)
|
|
86
|
+
return sources._dblp_get(_ReadErrorClient(failures=3), "https://dblp.org/test")
|
|
56
87
|
return Match(source="dblp", venue="TMLR", title=title, year="2025")
|
|
57
88
|
|
|
58
89
|
monkeypatch.setattr(
|
|
@@ -110,6 +141,44 @@ def test_requests_use_only_the_remaining_publication_budget(monkeypatch):
|
|
|
110
141
|
del sources._REQUEST_DEADLINE.value
|
|
111
142
|
|
|
112
143
|
|
|
144
|
+
def test_exact_and_fuzzy_share_one_four_second_dblp_budget(monkeypatch):
|
|
145
|
+
deadlines = []
|
|
146
|
+
real_with_deadline = sources._with_deadline
|
|
147
|
+
|
|
148
|
+
def capture(deadline, fn, *args):
|
|
149
|
+
if fn in {exact, fuzzy}:
|
|
150
|
+
deadlines.append(deadline)
|
|
151
|
+
return real_with_deadline(deadline, fn, *args)
|
|
152
|
+
|
|
153
|
+
def exact(*args):
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
def fuzzy(*args):
|
|
157
|
+
return None
|
|
158
|
+
|
|
159
|
+
started = sources.time.monotonic()
|
|
160
|
+
monkeypatch.setattr(sources, "CASCADE", (("dblp", exact),))
|
|
161
|
+
monkeypatch.setattr(sources, "try_dblp_fuzzy", fuzzy)
|
|
162
|
+
monkeypatch.setattr(sources, "_with_deadline", capture)
|
|
163
|
+
|
|
164
|
+
match, status = find_published("A paper", author_hint="author")
|
|
165
|
+
|
|
166
|
+
assert (match, status) == (None, "not_found")
|
|
167
|
+
assert len(deadlines) == 2
|
|
168
|
+
assert deadlines[0] == deadlines[1]
|
|
169
|
+
assert deadlines[0] <= started + 4.01
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def test_expired_budget_prevents_author_fuzzy_from_claiming_not_found(monkeypatch):
|
|
173
|
+
clock = iter((0.0, 4.0))
|
|
174
|
+
monkeypatch.setattr(sources.time, "monotonic", lambda: next(clock, 4.0))
|
|
175
|
+
monkeypatch.setattr(sources, "CASCADE", (("dblp", lambda *args: None),))
|
|
176
|
+
|
|
177
|
+
match, status = find_published("A paper", author_hint="author")
|
|
178
|
+
|
|
179
|
+
assert (match, status) == (None, "incomplete")
|
|
180
|
+
|
|
181
|
+
|
|
113
182
|
def test_upgrade_retries_dblp_after_previous_entry_read_failures(
|
|
114
183
|
tmp_path, monkeypatch
|
|
115
184
|
):
|
|
@@ -133,12 +202,7 @@ def test_upgrade_retries_dblp_after_previous_entry_read_failures(
|
|
|
133
202
|
|
|
134
203
|
def dblp(title, year, arxiv_id, author_hint):
|
|
135
204
|
if title == "First paper":
|
|
136
|
-
return sources.
|
|
137
|
-
_ReadErrorClient(failures=3),
|
|
138
|
-
"https://dblp.org/test",
|
|
139
|
-
"dblp",
|
|
140
|
-
0,
|
|
141
|
-
)
|
|
205
|
+
return sources._dblp_get(_ReadErrorClient(failures=3), "https://dblp.org/test")
|
|
142
206
|
return Match(source="dblp", venue="TMLR", title=title, year="2025")
|
|
143
207
|
|
|
144
208
|
monkeypatch.setattr(
|
|
@@ -43,7 +43,7 @@ def test_core_source_429_taints_verdict(monkeypatch):
|
|
|
43
43
|
monkeypatch.setattr(
|
|
44
44
|
sources,
|
|
45
45
|
"CASCADE",
|
|
46
|
-
_cascade(dblp="raise",
|
|
46
|
+
_cascade(dblp="raise", openalex=None, crossref=None),
|
|
47
47
|
)
|
|
48
48
|
match, status = find_published("Some Title", author_hint="smith")
|
|
49
49
|
assert (match, status) == (None, "incomplete")
|
|
@@ -59,17 +59,6 @@ def test_previously_disabled_core_source_taints_next_queries(monkeypatch):
|
|
|
59
59
|
assert (match, status) == (None, "incomplete")
|
|
60
60
|
|
|
61
61
|
|
|
62
|
-
def test_noncore_outage_does_not_taint(monkeypatch):
|
|
63
|
-
# Unpaywall outages are routine; a miss stays trustworthy.
|
|
64
|
-
monkeypatch.setattr(
|
|
65
|
-
sources,
|
|
66
|
-
"CASCADE",
|
|
67
|
-
_cascade(dblp=None, unpaywall="raise", crossref=None),
|
|
68
|
-
)
|
|
69
|
-
match, status = find_published("Some Title", author_hint="smith")
|
|
70
|
-
assert (match, status) == (None, "not_found")
|
|
71
|
-
|
|
72
|
-
|
|
73
62
|
def test_everything_down_is_unavailable(monkeypatch):
|
|
74
63
|
monkeypatch.setattr(
|
|
75
64
|
sources, "CASCADE", _cascade(dblp="raise", semanticscholar="raise")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|