doi2bib3 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- doi2bib3/__init__.py +5 -0
- doi2bib3/backend.py +371 -0
- doi2bib3/utils.py +156 -0
- doi2bib3-0.3.1.dist-info/METADATA +146 -0
- doi2bib3-0.3.1.dist-info/RECORD +9 -0
- doi2bib3-0.3.1.dist-info/WHEEL +5 -0
- doi2bib3-0.3.1.dist-info/entry_points.txt +2 -0
- doi2bib3-0.3.1.dist-info/licenses/LICENSE +674 -0
- doi2bib3-0.3.1.dist-info/top_level.txt +1 -0
doi2bib3/__init__.py
ADDED
doi2bib3/backend.py
ADDED
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
# Copyright (c) 2025 Archisman Panigrahi <apandada1ATgmail.com>
|
|
2
|
+
#
|
|
3
|
+
# This program is free software: you can redistribute it and/or modify
|
|
4
|
+
# it under the terms of the GNU General Public License as published by
|
|
5
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
6
|
+
# (at your option) any later version.
|
|
7
|
+
#
|
|
8
|
+
# This program is distributed in the hope that it will be useful,
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11
|
+
# GNU General Public License for more details.
|
|
12
|
+
#
|
|
13
|
+
# You should have received a copy of the GNU General Public License
|
|
14
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
15
|
+
|
|
16
|
+
from typing import Optional
|
|
17
|
+
import re
|
|
18
|
+
import requests
|
|
19
|
+
from urllib.parse import urlparse, unquote, quote
|
|
20
|
+
from .utils import normalize_bibtex
|
|
21
|
+
|
|
22
|
+
DOI_REGEX = re.compile(r"^10\..+/.+$")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def normalize_doi(doi_input: str) -> str:
|
|
26
|
+
s = doi_input.strip()
|
|
27
|
+
if s.lower().startswith('doi:'):
|
|
28
|
+
s = s[4:]
|
|
29
|
+
if s.lower().startswith('http://') or s.lower().startswith('https://'):
|
|
30
|
+
parsed = urlparse(s)
|
|
31
|
+
s = parsed.path.lstrip('/')
|
|
32
|
+
s = unquote(s)
|
|
33
|
+
if DOI_REGEX.match(s):
|
|
34
|
+
return s
|
|
35
|
+
raise DOIError(f"Invalid DOI: {doi_input}")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class DOIError(Exception):
|
|
39
|
+
pass
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def get_bibtex_from_doi(doi: str, timeout: int = 15) -> str:
|
|
43
|
+
# First, detect if the input is an arXiv id or arXiv URL. If so,
|
|
44
|
+
# resolve it using the arXiv API (arxiv_to_doi) and continue with the
|
|
45
|
+
# DOI fetch flow. This ensures arXiv links (abs/pdf/html) don't get
|
|
46
|
+
# mis-resolved by Crossref.
|
|
47
|
+
arxiv_id = _extract_arxiv_id(doi)
|
|
48
|
+
if arxiv_id:
|
|
49
|
+
found_doi = arxiv_to_doi(arxiv_id, timeout=timeout)
|
|
50
|
+
if not found_doi:
|
|
51
|
+
# Many arXiv entries (especially unpublished ones) are indexed in
|
|
52
|
+
# Crossref/DataCite with a DOI of the form 10.48550/arXiv.<id>
|
|
53
|
+
# (without version). Try that pattern before giving up.
|
|
54
|
+
arxiv_core = re.sub(r'v\d+$', '', arxiv_id)
|
|
55
|
+
candidate = f'10.48550/arXiv.{arxiv_core}'
|
|
56
|
+
try:
|
|
57
|
+
# normalize to ensure it's a valid DOI string
|
|
58
|
+
candidate = normalize_doi(candidate)
|
|
59
|
+
doi = candidate
|
|
60
|
+
except DOIError:
|
|
61
|
+
raise DOIError(f"No DOI found for arXiv id: {arxiv_id}")
|
|
62
|
+
else:
|
|
63
|
+
doi = found_doi
|
|
64
|
+
else:
|
|
65
|
+
# Try to normalize input to a DOI; if that fails, fall back to
|
|
66
|
+
# Crossref search which can accept publisher URLs or free-form queries.
|
|
67
|
+
try:
|
|
68
|
+
doi = normalize_doi(doi)
|
|
69
|
+
except DOIError:
|
|
70
|
+
found = crossref_search_for_doi(doi, timeout=timeout)
|
|
71
|
+
if not found:
|
|
72
|
+
raise DOIError(f"Invalid DOI and Crossref lookup failed for: {doi}")
|
|
73
|
+
doi = found
|
|
74
|
+
|
|
75
|
+
headers = {
|
|
76
|
+
'Accept': 'application/x-bibtex; charset=utf-8',
|
|
77
|
+
'User-Agent': 'doi2bib-python/1.0'
|
|
78
|
+
}
|
|
79
|
+
# try doi.org first
|
|
80
|
+
url = f'https://doi.org/{doi}'
|
|
81
|
+
resp = requests.get(url, headers=headers, timeout=timeout)
|
|
82
|
+
if resp.status_code == 200:
|
|
83
|
+
# Some providers mislabel encodings; prefer UTF-8 and fall back to
|
|
84
|
+
# the apparent encoding, replacing invalid bytes. This avoids
|
|
85
|
+
# mojibake like – for en-dash when requests guesses the wrong codec.
|
|
86
|
+
try:
|
|
87
|
+
return resp.content.decode('utf-8')
|
|
88
|
+
except Exception:
|
|
89
|
+
enc = resp.apparent_encoding or resp.encoding or 'utf-8'
|
|
90
|
+
return resp.content.decode(enc, errors='replace')
|
|
91
|
+
|
|
92
|
+
# fallback to Crossref transform endpoint
|
|
93
|
+
doi_quoted = quote(doi, safe='')
|
|
94
|
+
xurl = f'https://api.crossref.org/works/{doi_quoted}/transform/application/x-bibtex'
|
|
95
|
+
resp2 = requests.get(xurl, headers=headers, timeout=timeout)
|
|
96
|
+
if resp2.status_code == 200:
|
|
97
|
+
try:
|
|
98
|
+
return resp2.content.decode('utf-8')
|
|
99
|
+
except Exception:
|
|
100
|
+
enc2 = resp2.apparent_encoding or resp2.encoding or 'utf-8'
|
|
101
|
+
return resp2.content.decode(enc2, errors='replace')
|
|
102
|
+
|
|
103
|
+
raise DOIError(f"Failed to fetch DOI {doi}: doi.org HTTP {resp.status_code}, crossref HTTP {resp2.status_code}")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _extract_arxiv_id(s: str) -> Optional[str]:
|
|
107
|
+
"""Return an arXiv id if `s` is an arXiv URL or id, else None.
|
|
108
|
+
|
|
109
|
+
Accepts forms like:
|
|
110
|
+
- https://arxiv.org/abs/2411.08091
|
|
111
|
+
- https://arxiv.org/pdf/2411.08091.pdf
|
|
112
|
+
- https://arxiv.org/html/2411.08091
|
|
113
|
+
- arXiv:2411.08091
|
|
114
|
+
- 2411.08091
|
|
115
|
+
Also handles older IDs like hep-th/9901001.
|
|
116
|
+
"""
|
|
117
|
+
if not s:
|
|
118
|
+
return None
|
|
119
|
+
t = s.strip()
|
|
120
|
+
# arXiv: prefix
|
|
121
|
+
if t.lower().startswith('arxiv:'):
|
|
122
|
+
return t.split(':', 1)[1].strip()
|
|
123
|
+
|
|
124
|
+
# URL forms
|
|
125
|
+
if t.lower().startswith('http://') or t.lower().startswith('https://'):
|
|
126
|
+
try:
|
|
127
|
+
parsed = urlparse(t)
|
|
128
|
+
net = parsed.netloc.lower()
|
|
129
|
+
if 'arxiv.org' in net:
|
|
130
|
+
path = parsed.path.lstrip('/')
|
|
131
|
+
# match abs/, pdf/, html/ etc.
|
|
132
|
+
m = re.match(r'^(?:abs|pdf|html)/(?P<id>.+)$', path)
|
|
133
|
+
if m:
|
|
134
|
+
aid = m.group('id')
|
|
135
|
+
# strip .pdf extension when present
|
|
136
|
+
aid = re.sub(r'\.pdf$', '', aid, flags=re.I)
|
|
137
|
+
return aid
|
|
138
|
+
except Exception:
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
# bare modern arXiv id: YYYY.NNNNN or with vN
|
|
142
|
+
if re.match(r'^\d{4}\.\d+(v\d+)?$', t):
|
|
143
|
+
return t
|
|
144
|
+
|
|
145
|
+
# legacy arXiv id like hep-th/9901001
|
|
146
|
+
if re.match(r'^[a-z\-]+/\d{7}$', t, flags=re.I):
|
|
147
|
+
return t
|
|
148
|
+
|
|
149
|
+
return None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# pmid_to_doi is disabled — PubMed/PMID support commented out per project config
|
|
153
|
+
## def pmid_to_doi(pmid: str, timeout: int = 15) -> Optional[str]:
|
|
154
|
+
## pmid = pmid.strip()
|
|
155
|
+
## if not re.match(r"^\d+$|^PMC\d+(\.\d+)?$", pmid):
|
|
156
|
+
## raise ValueError("Invalid PMID")
|
|
157
|
+
##
|
|
158
|
+
## url = f'http://www.pubmedcentral.nih.gov/utils/idconv/v1.0/?format=json&ids={pmid}'
|
|
159
|
+
## resp = requests.get(url, timeout=timeout)
|
|
160
|
+
## if resp.status_code != 200:
|
|
161
|
+
## raise DOIError(f"PubMed ID conversion failed: HTTP {resp.status_code}")
|
|
162
|
+
## data = resp.json()
|
|
163
|
+
## records = data.get('records')
|
|
164
|
+
## if not records or not records[0]:
|
|
165
|
+
## return None
|
|
166
|
+
## return records[0].get('doi')
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def arxiv_to_doi(arxivid: str, timeout: int = 15) -> Optional[str]:
|
|
170
|
+
arxivid = arxivid.strip()
|
|
171
|
+
if arxivid.lower().startswith('arxiv:'):
|
|
172
|
+
arxivid = arxivid.split(':', 1)[1].strip()
|
|
173
|
+
|
|
174
|
+
# Accept modern arXiv IDs (YYYY.NNNNN with optional vN) and legacy
|
|
175
|
+
# subject-class IDs like hep-th/9901001 (optionally with vN).
|
|
176
|
+
if not re.match(r"^(?:\d{4}\.\d+(v\d+)?|[A-Za-z\-]+/\d{7}(v\d+)?)$", arxivid):
|
|
177
|
+
raise ValueError("Invalid arXiv ID")
|
|
178
|
+
|
|
179
|
+
url = f'https://export.arxiv.org/api/query?id_list={arxivid}'
|
|
180
|
+
resp = requests.get(url, timeout=timeout)
|
|
181
|
+
if resp.status_code != 200:
|
|
182
|
+
raise DOIError(f"arXiv query failed: HTTP {resp.status_code}")
|
|
183
|
+
text = resp.text
|
|
184
|
+
m = re.search(r"<arxiv:doi\b[^>]*>([^<]+)</arxiv:doi>", text)
|
|
185
|
+
if m:
|
|
186
|
+
return m.group(1).strip()
|
|
187
|
+
m = re.search(r"<doi\b[^>]*>([^<]+)</doi>", text)
|
|
188
|
+
if m:
|
|
189
|
+
return m.group(1).strip()
|
|
190
|
+
m = re.search(r'href=["\']https?://(?:dx\.)?doi\.org/([^"\']+)["\']', text)
|
|
191
|
+
if m:
|
|
192
|
+
return unquote(m.group(1).strip())
|
|
193
|
+
return None
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def crossref_search_for_doi(query: str, timeout: int = 15) -> Optional[str]:
|
|
197
|
+
q = query.strip()
|
|
198
|
+
if not q:
|
|
199
|
+
return None
|
|
200
|
+
|
|
201
|
+
headers = {
|
|
202
|
+
'User-Agent': 'doi2bib-python/1.0'
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
# If the query is a URL, try to extract a DOI-like substring from the path
|
|
206
|
+
# or from the publisher page HTML (meta tags, canonical/doi links). This
|
|
207
|
+
# avoids sending the full publisher URL as a free-form Crossref query
|
|
208
|
+
# which can produce unrelated matches.
|
|
209
|
+
if q.lower().startswith('http://') or q.lower().startswith('https://'):
|
|
210
|
+
try:
|
|
211
|
+
parsed = urlparse(q)
|
|
212
|
+
path = unquote(parsed.path or '')
|
|
213
|
+
m = re.search(r"10\.\d{4,9}/[^\s'\"<>]+", path)
|
|
214
|
+
if m:
|
|
215
|
+
candidate = m.group(0)
|
|
216
|
+
try:
|
|
217
|
+
return normalize_doi(candidate)
|
|
218
|
+
except DOIError:
|
|
219
|
+
pass
|
|
220
|
+
except Exception:
|
|
221
|
+
pass
|
|
222
|
+
|
|
223
|
+
# Try to fetch the publisher page and look for common DOI metadata
|
|
224
|
+
try:
|
|
225
|
+
doi_from_page = _extract_doi_from_url(q, timeout=timeout)
|
|
226
|
+
if doi_from_page:
|
|
227
|
+
return doi_from_page
|
|
228
|
+
except Exception:
|
|
229
|
+
# Don't fail hard on page parsing — fall back to Crossref search
|
|
230
|
+
pass
|
|
231
|
+
|
|
232
|
+
# Ask Crossref for a handful of candidates and pick the best match.
|
|
233
|
+
try:
|
|
234
|
+
url = f'https://api.crossref.org/works?query.bibliographic={quote(q)}&rows=5'
|
|
235
|
+
resp = requests.get(url, headers=headers, timeout=timeout)
|
|
236
|
+
if resp.status_code != 200:
|
|
237
|
+
return None
|
|
238
|
+
data = resp.json()
|
|
239
|
+
items = data.get('message', {}).get('items', [])
|
|
240
|
+
if not items:
|
|
241
|
+
return None
|
|
242
|
+
|
|
243
|
+
# If query was a URL, prefer items whose URL contains the same netloc
|
|
244
|
+
if q.lower().startswith('http://') or q.lower().startswith('https://'):
|
|
245
|
+
try:
|
|
246
|
+
parsed_q = urlparse(q)
|
|
247
|
+
q_netloc = parsed_q.netloc.lower()
|
|
248
|
+
for it in items:
|
|
249
|
+
it_url = (it.get('URL') or '')
|
|
250
|
+
if it_url and q_netloc in it_url.lower():
|
|
251
|
+
if it.get('DOI'):
|
|
252
|
+
return it.get('DOI')
|
|
253
|
+
except Exception:
|
|
254
|
+
pass
|
|
255
|
+
|
|
256
|
+
# Fallback: choose highest score returned by Crossref
|
|
257
|
+
items_sorted = sorted(items, key=lambda x: x.get('score', 0), reverse=True)
|
|
258
|
+
top = items_sorted[0]
|
|
259
|
+
return top.get('DOI')
|
|
260
|
+
except Exception:
|
|
261
|
+
return None
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _extract_doi_from_url(url: str, timeout: int = 10) -> Optional[str]:
|
|
265
|
+
"""Fetch a publisher URL and try to extract a DOI from common meta tags
|
|
266
|
+
and links.
|
|
267
|
+
|
|
268
|
+
Heuristics checked (in order):
|
|
269
|
+
- meta[name=citation_doi]
|
|
270
|
+
- meta[name=dc.Identifier] / meta[name=DC.identifier]
|
|
271
|
+
- meta[name=DC.identifier] with scheme DOI
|
|
272
|
+
- link[href] pointing to dx.doi.org or doi.org
|
|
273
|
+
- any href/src containing /10.xxxx/ pattern
|
|
274
|
+
Returns a normalized DOI string or None.
|
|
275
|
+
"""
|
|
276
|
+
# Try a polite bot UA first; some sites block unknown agents. If we get a
|
|
277
|
+
# non-200 (commonly 403), retry once with a common browser User-Agent and
|
|
278
|
+
# a Referer header to improve chances of acceptance.
|
|
279
|
+
ua_bot = 'doi2bib3-python/1.0'
|
|
280
|
+
ua_browser = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36'
|
|
281
|
+
headers = {'User-Agent': ua_bot}
|
|
282
|
+
try:
|
|
283
|
+
resp = requests.get(url, headers=headers, timeout=timeout)
|
|
284
|
+
if resp.status_code != 200 or not resp.text:
|
|
285
|
+
# retry with browser UA and Referer
|
|
286
|
+
try:
|
|
287
|
+
headers = {'User-Agent': ua_browser, 'Referer': url}
|
|
288
|
+
resp = requests.get(url, headers=headers, timeout=timeout)
|
|
289
|
+
except Exception:
|
|
290
|
+
return None
|
|
291
|
+
if resp.status_code != 200 or not resp.text:
|
|
292
|
+
return None
|
|
293
|
+
html = resp.text
|
|
294
|
+
except Exception:
|
|
295
|
+
return None
|
|
296
|
+
|
|
297
|
+
# meta tags: citation_doi is common (used by many publishers)
|
|
298
|
+
m = re.search(r'<meta[^>]+name=["\']citation_doi["\'][^>]*content=["\']([^"\']+)["\']', html, flags=re.I)
|
|
299
|
+
if m:
|
|
300
|
+
try:
|
|
301
|
+
return normalize_doi(m.group(1).strip())
|
|
302
|
+
except DOIError:
|
|
303
|
+
pass
|
|
304
|
+
|
|
305
|
+
# dc.identifier or DCTERMS.identifier
|
|
306
|
+
m = re.search(r'<meta[^>]+name=["\'](?:dc\.|DCTERMS\.)?identifier["\'][^>]*content=["\']([^"\']+)["\']', html, flags=re.I)
|
|
307
|
+
if m:
|
|
308
|
+
val = m.group(1).strip()
|
|
309
|
+
# sometimes comes as 'doi:10.xxx' or full URL
|
|
310
|
+
if val.lower().startswith('doi:'):
|
|
311
|
+
val = val.split(':', 1)[1]
|
|
312
|
+
if val.lower().startswith('http://') or val.lower().startswith('https://'):
|
|
313
|
+
try:
|
|
314
|
+
parsed = urlparse(val)
|
|
315
|
+
v = unquote(parsed.path.lstrip('/'))
|
|
316
|
+
return normalize_doi(v)
|
|
317
|
+
except Exception:
|
|
318
|
+
pass
|
|
319
|
+
try:
|
|
320
|
+
return normalize_doi(val)
|
|
321
|
+
except DOIError:
|
|
322
|
+
pass
|
|
323
|
+
|
|
324
|
+
# link tags and explicit DOI hrefs
|
|
325
|
+
m = re.search(r'href=["\']https?://(?:dx\.)?doi\.org/([^"\']+)["\']', html, flags=re.I)
|
|
326
|
+
if m:
|
|
327
|
+
try:
|
|
328
|
+
return normalize_doi(unquote(m.group(1).strip()))
|
|
329
|
+
except DOIError:
|
|
330
|
+
pass
|
|
331
|
+
|
|
332
|
+
# Any /10.xxx/ pattern in href/src attributes
|
|
333
|
+
m = re.search(r'(?:href|src)=["\'][^"\']*(10\.\d{4,9}/[^"\']+)["\']', html, flags=re.I)
|
|
334
|
+
if m:
|
|
335
|
+
try:
|
|
336
|
+
return normalize_doi(m.group(1).strip())
|
|
337
|
+
except DOIError:
|
|
338
|
+
pass
|
|
339
|
+
|
|
340
|
+
# last resort: any DOI-like substring in the page
|
|
341
|
+
m = re.search(r'10\.\d{4,9}/[^\s"\'"<>]+', html)
|
|
342
|
+
if m:
|
|
343
|
+
try:
|
|
344
|
+
return normalize_doi(m.group(0))
|
|
345
|
+
except DOIError:
|
|
346
|
+
pass
|
|
347
|
+
|
|
348
|
+
return None
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def fetch_bibtex(identifier: str, timeout: int = 15, normalize: bool = True) -> str:
|
|
352
|
+
"""Convenience wrapper for programmatic use.
|
|
353
|
+
|
|
354
|
+
- identifier: DOI, DOI URL, arXiv id/URL, or publisher URL (same as CLI)
|
|
355
|
+
- timeout: network timeout in seconds
|
|
356
|
+
- normalize: if True (default), pass the fetched BibTeX through
|
|
357
|
+
`doi2bib3.utils.normalize_bibtex` before returning. If False, the
|
|
358
|
+
raw text from doi.org / Crossref is returned.
|
|
359
|
+
|
|
360
|
+
This keeps `get_bibtex_from_doi` behaviour intact for callers that
|
|
361
|
+
expect the raw provider output, while giving a convenient API for
|
|
362
|
+
library users who want the nicely formatted BibTeX the CLI prints.
|
|
363
|
+
"""
|
|
364
|
+
raw = get_bibtex_from_doi(identifier, timeout=timeout)
|
|
365
|
+
if normalize:
|
|
366
|
+
try:
|
|
367
|
+
return normalize_bibtex(raw)
|
|
368
|
+
except Exception:
|
|
369
|
+
# If normalization fails for any reason, fall back to raw text
|
|
370
|
+
return raw
|
|
371
|
+
return raw
|
doi2bib3/utils.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# Copyright (c) 2025 Archisman Panigrahi <apandada1ATgmail.com>
|
|
2
|
+
#
|
|
3
|
+
# This program is free software: you can redistribute it and/or modify
|
|
4
|
+
# it under the terms of the GNU General Public License as published by
|
|
5
|
+
# the Free Software Foundation, either version 3 of the License, or
|
|
6
|
+
# (at your option) any later version.
|
|
7
|
+
#
|
|
8
|
+
# This program is distributed in the hope that it will be useful,
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11
|
+
# GNU General Public License for more details.
|
|
12
|
+
#
|
|
13
|
+
# You should have received a copy of the GNU General Public License
|
|
14
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
15
|
+
|
|
16
|
+
"""Utilities for bib normalization and IO inside the package."""
|
|
17
|
+
from typing import Optional
|
|
18
|
+
import re
|
|
19
|
+
import urllib.parse
|
|
20
|
+
import bibtexparser
|
|
21
|
+
import os
|
|
22
|
+
|
|
23
|
+
SPECIAL_CHARS = {
|
|
24
|
+
'a\u0300': "\\`a",
|
|
25
|
+
'\u00f4': "\\^o",
|
|
26
|
+
'\u00ea': "\\^e",
|
|
27
|
+
'\u00e2': "\\^a",
|
|
28
|
+
'\u00ae': '{\\textregistered}',
|
|
29
|
+
'\u00e7': "\\c{c}",
|
|
30
|
+
'\u00f6': "\\\"{o}",
|
|
31
|
+
'\u00e4': "\\\"{a}",
|
|
32
|
+
'\u00fc': "\\\"{u}",
|
|
33
|
+
'\u00d6': "\\\"{O}",
|
|
34
|
+
'\u00c4': "\\\"{A}",
|
|
35
|
+
'\u00dc': "\\\"{U}"
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
VAR_RE = re.compile(r"(\\{)(\\var[A-Z]?[a-z]*)(\\})")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def insert_dollars(title: str) -> str:
|
|
43
|
+
return VAR_RE.sub(r"\\1$\\2$\\3", title)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def encode_special_chars(value: str) -> str:
|
|
47
|
+
for k, v in SPECIAL_CHARS.items():
|
|
48
|
+
value = value.replace(k, v)
|
|
49
|
+
return value
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def normalize_bibtex(bib_str: str) -> str:
|
|
53
|
+
bib_db = bibtexparser.loads(bib_str)
|
|
54
|
+
for entry in bib_db.entries:
|
|
55
|
+
if 'ID' in entry:
|
|
56
|
+
entry['ID'] = entry['ID'].replace('_', '')
|
|
57
|
+
pages = entry.get('pages')
|
|
58
|
+
if pages:
|
|
59
|
+
# Normalize common N/A variants to remove the field entirely
|
|
60
|
+
norm = pages.strip().lower()
|
|
61
|
+
if norm in ('n/a-n/a', 'na-na', 'n/a', 'na'):
|
|
62
|
+
entry.pop('pages', None)
|
|
63
|
+
else:
|
|
64
|
+
p = pages
|
|
65
|
+
# Convert unicode en-dash/em-dash to ASCII double-hyphen
|
|
66
|
+
p = p.replace('\u2013', '--').replace('\u2014', '--')
|
|
67
|
+
# Replace en/em characters themselves if present
|
|
68
|
+
p = p.replace('\u2013', '--').replace('\u2014', '--')
|
|
69
|
+
# Replace any literal en-dash/em-dash characters too
|
|
70
|
+
p = p.replace('\u2013', '--').replace('\u2014', '--')
|
|
71
|
+
p = p.replace('–', '--').replace('—', '--')
|
|
72
|
+
# Replace single hyphen between digits (with optional spaces)
|
|
73
|
+
# e.g. '1932-1938', '1932 - 1938', '1932-1938.e3' -> '1932--1938' or '1932--1938.e3'
|
|
74
|
+
p = re.sub(r'(?<=\d)\s*-[\u2013\u2014-]?\s*(?=\d)', '--', p)
|
|
75
|
+
# If no double-dash already, ensure we don't inadvertently
|
|
76
|
+
# convert word hyphens — only numeric ranges should be changed
|
|
77
|
+
entry['pages'] = p
|
|
78
|
+
if 'url' in entry:
|
|
79
|
+
entry['url'] = urllib.parse.unquote(entry['url'])
|
|
80
|
+
if 'title' in entry:
|
|
81
|
+
entry['title'] = insert_dollars(entry['title'])
|
|
82
|
+
if 'month' in entry:
|
|
83
|
+
entry['month'] = entry['month'].strip()
|
|
84
|
+
if entry['month'].startswith('{') and entry['month'].endswith('}'):
|
|
85
|
+
entry['month'] = entry['month'][1:-1]
|
|
86
|
+
for key in list(entry.keys()):
|
|
87
|
+
if key in ['title', 'journal', 'booktitle']:
|
|
88
|
+
entry[key] = encode_special_chars(entry[key])
|
|
89
|
+
|
|
90
|
+
return bibtexparser.dumps(bib_db)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def save_bibtex_to_file(bib_str: str, path: str, append: bool = False) -> None:
|
|
94
|
+
if not append:
|
|
95
|
+
with open(path, 'w', encoding='utf-8') as f:
|
|
96
|
+
f.write(bib_str)
|
|
97
|
+
return
|
|
98
|
+
|
|
99
|
+
prefix = ''
|
|
100
|
+
try:
|
|
101
|
+
if os.path.exists(path) and os.path.getsize(path) > 0:
|
|
102
|
+
with open(path, 'rb') as fh:
|
|
103
|
+
fh.seek(-1, os.SEEK_END)
|
|
104
|
+
last = fh.read(1)
|
|
105
|
+
if last != b"\n":
|
|
106
|
+
prefix = "\n"
|
|
107
|
+
except OSError:
|
|
108
|
+
prefix = "\n"
|
|
109
|
+
|
|
110
|
+
with open(path, 'a', encoding='utf-8') as f:
|
|
111
|
+
if prefix:
|
|
112
|
+
f.write(prefix)
|
|
113
|
+
f.write(bib_str)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def cli_doi2bib3(argv=None):
|
|
117
|
+
"""A thin CLI wrapper to mirror the main.py behavior (entry point).
|
|
118
|
+
|
|
119
|
+
This function is intended to be callable programmatically with an argv
|
|
120
|
+
list (like sys.argv[1:]) and also used as the console script entry point.
|
|
121
|
+
"""
|
|
122
|
+
import argparse
|
|
123
|
+
import sys
|
|
124
|
+
from .backend import get_bibtex_from_doi
|
|
125
|
+
|
|
126
|
+
p = argparse.ArgumentParser(
|
|
127
|
+
description='Fetch BibTeX by DOI, DOI URL, arXiv id or arXiv URL'
|
|
128
|
+
)
|
|
129
|
+
p.add_argument('identifier', nargs='?', help='DOI, DOI URL, arXiv id/URL, or publisher URL')
|
|
130
|
+
p.add_argument('-o', '--out', help='Write .bib file to this path')
|
|
131
|
+
|
|
132
|
+
args = p.parse_args(argv)
|
|
133
|
+
|
|
134
|
+
if not args.identifier:
|
|
135
|
+
p.print_help()
|
|
136
|
+
sys.exit(2)
|
|
137
|
+
|
|
138
|
+
ident = args.identifier
|
|
139
|
+
out = args.out
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
bib = get_bibtex_from_doi(ident)
|
|
143
|
+
except Exception as e:
|
|
144
|
+
print('Error:', e, file=sys.stderr)
|
|
145
|
+
sys.exit(1)
|
|
146
|
+
|
|
147
|
+
bib = normalize_bibtex(bib)
|
|
148
|
+
if out:
|
|
149
|
+
save_bibtex_to_file(bib, out, append=True)
|
|
150
|
+
print('Wrote', out)
|
|
151
|
+
else:
|
|
152
|
+
print(bib)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
if __name__ == '__main__':
|
|
156
|
+
cli_doi2bib3()
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: doi2bib3
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Fetch BibTeX from DOI, or resolve arXiv IDs to DOI and fetch BibTeX
|
|
5
|
+
Author-email: Archisman Panigrahi <apandada1@gmail.com>
|
|
6
|
+
Maintainer-email: Archisman Panigrahi <apandada1@gmail.com>
|
|
7
|
+
License-Expression: GPL-3.0-only
|
|
8
|
+
Project-URL: Homepage, https://github.com/archisman-panigrahi/doi2bib3
|
|
9
|
+
Project-URL: Source, https://github.com/archisman-panigrahi/doi2bib3
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/archisman-panigrahi/doi2bib3/issues
|
|
11
|
+
Keywords: doi,bibtex,arxiv
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Requires-Python: >=3.8
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: requests
|
|
17
|
+
Requires-Dist: bibtexparser
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# doi2bib3
|
|
21
|
+
|
|
22
|
+
doi2bib3 is a small Python utility to fetch BibTeX metadata for a DOI or to
|
|
23
|
+
resolve arXiv identifiers to DOIs and fetch their BibTeX entries. It accepts
|
|
24
|
+
DOI inputs, DOI URLs, arXiv IDs/URLs (modern and legacy), publisher landing
|
|
25
|
+
pages, and uses a sequence of resolution strategies to return a BibTeX string.
|
|
26
|
+
This tool combines the features of [doi2bib](https://github.com/bibcure/doi2bib/) and [doi2bib2](https://github.com/davidagraf/doi2bib2).
|
|
27
|
+
|
|
28
|
+
**It will be submitted to PyPI soon.**
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
Key behaviors
|
|
32
|
+
- Provides bibtex entry for DOI and arXiv links.
|
|
33
|
+
- Automatically detects arXiv inputs (e.g. `2411.08091`, `arXiv:2411.08091`, or `https://arxiv.org/abs/2411.08091`) and queries the arXiv API for a DOI.
|
|
34
|
+
- For non-arXiv inputs: attempts DOI normalization, content negotiation at doi.org, Crossref transform, and as a last resort a Crossref bibliographic search.
|
|
35
|
+
|
|
36
|
+
A GUI frontend is available: Check out [QuickBib](https://archisman-panigrahi.github.io/QuickBib).
|
|
37
|
+
|
|
38
|
+
Installation
|
|
39
|
+
------------
|
|
40
|
+
|
|
41
|
+
Create a virtual environment and install runtime dependencies:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
python -m venv .venv
|
|
45
|
+
source .venv/bin/activate
|
|
46
|
+
pip install -r requirements.txt
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Install the package for local development:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install -e .
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
<a href="https://repology.org/project/doi2bib3/versions">
|
|
56
|
+
<img src="https://repology.org/badge/vertical-allrepos/doi2bib3.svg" alt="Packaging status" align="right">
|
|
57
|
+
</a>
|
|
58
|
+
|
|
59
|
+
### Arch Linux
|
|
60
|
+
In Arch Linux you can install it from the AUR with the command `yay -S doi2bib3`.
|
|
61
|
+
|
|
62
|
+
### Ubuntu
|
|
63
|
+
You can use our [official PPA](https://code.launchpad.net/~apandada1/+archive/ubuntu/quickbib)
|
|
64
|
+
```
|
|
65
|
+
sudo add-apt-repository ppa:apandada1/quickbib
|
|
66
|
+
sudo apt update
|
|
67
|
+
sudo apt install python3-doi2bib3
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
CLI usage
|
|
71
|
+
---------
|
|
72
|
+
|
|
73
|
+
The CLI accepts a single positional identifier and an optional `-o/--out`
|
|
74
|
+
path to save the BibTeX output. When installed, the package installs a console
|
|
75
|
+
script named `doi2bib3` (configured in `pyproject.toml`). From the repository
|
|
76
|
+
root you can also run the provided `main.py` shim.
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
# using the local shim
|
|
80
|
+
python main.py <identifier> [-o OUT]
|
|
81
|
+
|
|
82
|
+
# or when installed as console script
|
|
83
|
+
doi2bib3 <identifier> -o references.bib
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Examples
|
|
87
|
+
--------
|
|
88
|
+
|
|
89
|
+
Fetch by DOI (bare DOI or DOI URL):
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
doi2bib3 10.1038/nphys1170
|
|
93
|
+
doi2bib3 https://doi.org/10.1038/nphys1170
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
ArXiv inputs (detected automatically):
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
doi2bib3 https://arxiv.org/abs/2411.08091
|
|
100
|
+
doi2bib3 arXiv:2411.08091
|
|
101
|
+
doi2bib3 2411.08091
|
|
102
|
+
doi2bib3 hep-th/9901001
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Save to a file:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
doi2bib3 https://doi.org/10.1038/nphys1170 -o paper.bib
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Note: If the tool is not installed, you can run it with `python main.py https://doi.org/10.1038/nphys1170` and so on.
|
|
112
|
+
|
|
113
|
+
Programmatic usage
|
|
114
|
+
------------------
|
|
115
|
+
|
|
116
|
+
The package exposes a small programmatic API so you can use doi2bib3 from
|
|
117
|
+
Python code. The most convenient entry point is the package-level
|
|
118
|
+
`fetch_bibtex` function which mirrors the CLI behavior and returns normalized
|
|
119
|
+
BibTeX by default:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
from doi2bib3 import fetch_bibtex
|
|
123
|
+
|
|
124
|
+
# Get normalized BibTeX (default)
|
|
125
|
+
bib = fetch_bibtex('https://www.pnas.org/doi/10.1073/pnas.2305943120')
|
|
126
|
+
print(bib)
|
|
127
|
+
|
|
128
|
+
# Get the raw provider output without normalization
|
|
129
|
+
raw = fetch_bibtex('10.1073/pnas.2305943120', normalize=False)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
You can also invoke the thin CLI wrapper from `utils` when writing tests or
|
|
133
|
+
automation:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from doi2bib3.utils import cli_doi2bib3
|
|
137
|
+
cli_doi2bib3(['https://arxiv.org/abs/2411.08091', '--out', 'paper.bib'])
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
License
|
|
141
|
+
-------
|
|
142
|
+
This project is distributed under the GNU General Public License v3 (GPL-3.0-only).
|
|
143
|
+
|
|
144
|
+
Acknowledgements
|
|
145
|
+
---------------
|
|
146
|
+
Parts of the code and documentation were assisted by copilot.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
doi2bib3/__init__.py,sha256=RCO63-XQipWZjLue4Kvh6n3dOnH-sWR2qeDlI0-eaRA,92
|
|
2
|
+
doi2bib3/backend.py,sha256=PZM_Eh8goR2J_QitUvpaSG_n6NgKR3q3G0m6418NyaE,13977
|
|
3
|
+
doi2bib3/utils.py,sha256=hsLB7K5HWv56x0X5tynwIwVpU7pePcY4pUXJ4hj5frU,5230
|
|
4
|
+
doi2bib3-0.3.1.dist-info/licenses/LICENSE,sha256=YF6QR6Vjxcg5b_sYIyqkME7FZYau5TfEUGTG-0JeRK0,35129
|
|
5
|
+
doi2bib3-0.3.1.dist-info/METADATA,sha256=LKeB8bi65OpheygMwEnIkpsGgFeDGPXOl4PJqsV7Q74,4388
|
|
6
|
+
doi2bib3-0.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
7
|
+
doi2bib3-0.3.1.dist-info/entry_points.txt,sha256=JnhRCJT2LzIyPq3fg2OGqWZA2XJlhJL-zaxultI4I7M,57
|
|
8
|
+
doi2bib3-0.3.1.dist-info/top_level.txt,sha256=_IEyijwBNGHQnWejbC_DfE-LtlGf-VmJrJNEOP94fws,9
|
|
9
|
+
doi2bib3-0.3.1.dist-info/RECORD,,
|