litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""LibGen source with multi-mirror failover."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
import urllib.parse
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from ..network import fetch, polite_delay
|
|
12
|
+
from ..download.pdf import download_pdf
|
|
13
|
+
|
|
14
|
+
LIBGEN_MIRRORS = [
|
|
15
|
+
"https://libgen.li",
|
|
16
|
+
"https://libgen.bz",
|
|
17
|
+
"https://libgen.gs",
|
|
18
|
+
"https://libgen.rs",
|
|
19
|
+
"https://libgen.st",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def try_libgen(doi: str, output_path: Path, config: dict[str, Any], use_tor: bool = False) -> dict[str, Any] | None:
|
|
24
|
+
q = urllib.parse.quote(doi, safe="")
|
|
25
|
+
for mirror in LIBGEN_MIRRORS:
|
|
26
|
+
result = _try_libgen_mirror(doi, q, mirror, output_path, config, use_tor=use_tor)
|
|
27
|
+
if result:
|
|
28
|
+
return result
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _try_libgen_mirror(
|
|
33
|
+
doi: str, q: str, mirror: str, output_path: Path, config: dict[str, Any], use_tor: bool = False
|
|
34
|
+
) -> dict[str, Any] | None:
|
|
35
|
+
url = f"{mirror}/ads.php?doi={q}"
|
|
36
|
+
# Slow-CDN lane: floor the timeouts (users run 3/7s for source racing —
|
|
37
|
+
# the booksdl CDN drips and 503s) and allow a longer stream deadline.
|
|
38
|
+
config = {
|
|
39
|
+
**config,
|
|
40
|
+
"connect_timeout": max(int(config.get("connect_timeout", 3)), 10),
|
|
41
|
+
"read_timeout": max(int(config.get("read_timeout", 7)), 30),
|
|
42
|
+
"download_deadline_seconds": max(int(config.get("download_deadline_seconds", 60)), 180),
|
|
43
|
+
}
|
|
44
|
+
# The li/bz frontends serve EMPTY 200 responses (Content-Length: 0) to
|
|
45
|
+
# cookieless one-shot requests — the flow then finds no get.php link and
|
|
46
|
+
# silently misses. Warm a session on the homepage first and hit ads.php
|
|
47
|
+
# with its cookies + Referer; the page then renders normally
|
|
48
|
+
# (field-verified 2026-09: li via proxy, bz also direct).
|
|
49
|
+
try:
|
|
50
|
+
import requests
|
|
51
|
+
|
|
52
|
+
from ..network import USER_AGENT
|
|
53
|
+
|
|
54
|
+
session = requests.Session()
|
|
55
|
+
session.trust_env = False
|
|
56
|
+
session.headers.update({"User-Agent": USER_AGENT})
|
|
57
|
+
from ..network import select_proxy_for_url
|
|
58
|
+
proxy = select_proxy_for_url(mirror, config) # scihub_via_proxy / network_proxy
|
|
59
|
+
if proxy:
|
|
60
|
+
session.proxies = {"http": proxy, "https": proxy}
|
|
61
|
+
try:
|
|
62
|
+
session.get(f"{mirror}/", timeout=(10, 15))
|
|
63
|
+
except Exception:
|
|
64
|
+
pass # warm-up is best effort — ads.php sometimes works bare
|
|
65
|
+
resp = session.get(url, headers={"Referer": f"{mirror}/"},
|
|
66
|
+
timeout=(10, 25))
|
|
67
|
+
|
|
68
|
+
# Fallback to browser on Cloudflare/403
|
|
69
|
+
if resp.status_code in (403, 503):
|
|
70
|
+
from ..browser.engine import solve_url, is_available
|
|
71
|
+
if config.get("scihub_use_browser", True) and is_available(config):
|
|
72
|
+
result = solve_url(url, config)
|
|
73
|
+
if result:
|
|
74
|
+
solution = result.get("solution", {})
|
|
75
|
+
if solution.get("status", 0) < 400:
|
|
76
|
+
html = solution.get("response", "")
|
|
77
|
+
else:
|
|
78
|
+
return None
|
|
79
|
+
else:
|
|
80
|
+
return None
|
|
81
|
+
else:
|
|
82
|
+
return None
|
|
83
|
+
else:
|
|
84
|
+
if resp.status_code >= 400:
|
|
85
|
+
return None
|
|
86
|
+
html = resp.text
|
|
87
|
+
# booksdl CDN rotates nodes and 503s most requests (field 2026-09:
|
|
88
|
+
# ~1-in-4 serves the PDF) — retry the same link a few times.
|
|
89
|
+
for match in re.finditer(r'''href=["']([^"']*get\.php[^"']+)["']''', html, re.I):
|
|
90
|
+
dl_path = match.group(1)
|
|
91
|
+
dl_url = urllib.parse.urljoin(url, dl_path)
|
|
92
|
+
result = None
|
|
93
|
+
for attempt in range(4):
|
|
94
|
+
polite_delay(config)
|
|
95
|
+
result = download_pdf(dl_url, output_path, config, "LibGen",
|
|
96
|
+
require_pdf_like_url=False, use_tor=use_tor)
|
|
97
|
+
if result:
|
|
98
|
+
break
|
|
99
|
+
time.sleep(2 + 2 * attempt)
|
|
100
|
+
if result:
|
|
101
|
+
result["doi"] = doi
|
|
102
|
+
result["identifier"] = doi
|
|
103
|
+
return result
|
|
104
|
+
for match in re.finditer(r'''href=["']([^"']*\.pdf[^"']*)["']''', html, re.I):
|
|
105
|
+
dl_path = match.group(1)
|
|
106
|
+
if "get.php" in dl_path or dl_path.endswith(".pdf"):
|
|
107
|
+
dl_url = urllib.parse.urljoin(url, dl_path)
|
|
108
|
+
polite_delay(config)
|
|
109
|
+
result = download_pdf(dl_url, output_path, config, "LibGen", require_pdf_like_url=False, use_tor=use_tor)
|
|
110
|
+
if result:
|
|
111
|
+
result["doi"] = doi
|
|
112
|
+
result["identifier"] = doi
|
|
113
|
+
return result
|
|
114
|
+
except Exception:
|
|
115
|
+
pass
|
|
116
|
+
return None
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Nature.com direct PDF download."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ..network import fetch
|
|
10
|
+
from ..download.pdf import download_pdf
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def try_nature_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
14
|
+
if not doi.startswith("10.1038/"):
|
|
15
|
+
return None
|
|
16
|
+
article_id = doi.replace("10.1038/", "")
|
|
17
|
+
pdf_url = f"https://www.nature.com/articles/{article_id}.pdf"
|
|
18
|
+
result = download_pdf(pdf_url, output_path, config, "NatureDirect", require_pdf_like_url=False)
|
|
19
|
+
if result:
|
|
20
|
+
result["doi"] = doi
|
|
21
|
+
result["identifier"] = doi
|
|
22
|
+
return result
|
|
23
|
+
|
|
24
|
+
article_url = f"https://www.nature.com/articles/{article_id}"
|
|
25
|
+
try:
|
|
26
|
+
resp = fetch(article_url, config)
|
|
27
|
+
if resp.status_code < 400:
|
|
28
|
+
html = resp.text
|
|
29
|
+
pdf_pattern = re.compile(r'href=["\'](/articles/[^"\' ]*\.pdf[^"\' ]*)["\']', re.I)
|
|
30
|
+
for match in pdf_pattern.finditer(html):
|
|
31
|
+
dl_url = f"https://www.nature.com{match.group(1)}"
|
|
32
|
+
result = download_pdf(dl_url, output_path, config, "NatureDirect", require_pdf_like_url=False)
|
|
33
|
+
if result:
|
|
34
|
+
result["doi"] = doi
|
|
35
|
+
result["identifier"] = doi
|
|
36
|
+
return result
|
|
37
|
+
except Exception:
|
|
38
|
+
pass
|
|
39
|
+
return None
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""OA discovery sources: OpenAIRE, DOAJ.
|
|
2
|
+
|
|
3
|
+
These APIs find OA versions of papers that may not be indexed by Unpaywall/OpenAlex.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import urllib.parse
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import requests
|
|
13
|
+
|
|
14
|
+
from ..network import polite_delay, USER_AGENT, request_timeout, proxy_dict, select_proxy_for_url
|
|
15
|
+
from ..download.pdf import download_pdf, is_plausible_pdf_url
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def try_openaire(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
19
|
+
"""Try OpenAIRE API for OA PDF.
|
|
20
|
+
|
|
21
|
+
OpenAIRE aggregates European open access publications.
|
|
22
|
+
Extracts PDF URLs from children records (DOAJ, PMC, etc.).
|
|
23
|
+
"""
|
|
24
|
+
q = urllib.parse.quote(doi, safe="")
|
|
25
|
+
url = f"https://api.openaire.eu/search/publications?doi={q}&format=json"
|
|
26
|
+
try:
|
|
27
|
+
polite_delay(config)
|
|
28
|
+
s = requests.Session()
|
|
29
|
+
s.trust_env = False
|
|
30
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
31
|
+
resp = s.get(url, timeout=request_timeout(config),
|
|
32
|
+
proxies=proxy_dict(select_proxy_for_url(url, config)))
|
|
33
|
+
if resp.status_code >= 400:
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
data = resp.json()
|
|
37
|
+
results = data.get("response", {}).get("results", {})
|
|
38
|
+
if not results:
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
publications = results.get("result", [])
|
|
42
|
+
if isinstance(publications, dict):
|
|
43
|
+
publications = [publications]
|
|
44
|
+
|
|
45
|
+
for pub in publications:
|
|
46
|
+
main = pub.get("metadata", {}).get("oaf:entity", {}).get("oaf:result", {})
|
|
47
|
+
if not main:
|
|
48
|
+
continue
|
|
49
|
+
|
|
50
|
+
# Check access right
|
|
51
|
+
access_right = main.get("bestaccessright", {})
|
|
52
|
+
if isinstance(access_right, dict):
|
|
53
|
+
if access_right.get("classid", "") != "OPEN":
|
|
54
|
+
continue
|
|
55
|
+
|
|
56
|
+
# Extract URLs from children records
|
|
57
|
+
children = main.get("children", {})
|
|
58
|
+
child_results = children.get("result", [])
|
|
59
|
+
if isinstance(child_results, dict):
|
|
60
|
+
child_results = [child_results]
|
|
61
|
+
|
|
62
|
+
for child in child_results:
|
|
63
|
+
instance = child.get("instance", {})
|
|
64
|
+
# Check for direct URL
|
|
65
|
+
url_obj = instance.get("url", {})
|
|
66
|
+
child_url = url_obj.get("$", "") if isinstance(url_obj, dict) else ""
|
|
67
|
+
if child_url and child_url.startswith("http"):
|
|
68
|
+
polite_delay(config)
|
|
69
|
+
result = download_pdf(child_url, output_path, config, "OpenAIRE",
|
|
70
|
+
require_pdf_like_url=False)
|
|
71
|
+
if result:
|
|
72
|
+
result["doi"] = doi
|
|
73
|
+
result["identifier"] = doi
|
|
74
|
+
return result
|
|
75
|
+
|
|
76
|
+
# Check webresource URL
|
|
77
|
+
webres = instance.get("webresource", {})
|
|
78
|
+
if isinstance(webres, dict):
|
|
79
|
+
wr_url = webres.get("url", {})
|
|
80
|
+
child_url = wr_url.get("$", "") if isinstance(wr_url, dict) else ""
|
|
81
|
+
if child_url and child_url.startswith("http"):
|
|
82
|
+
polite_delay(config)
|
|
83
|
+
result = download_pdf(child_url, output_path, config, "OpenAIRE",
|
|
84
|
+
require_pdf_like_url=False)
|
|
85
|
+
if result:
|
|
86
|
+
result["doi"] = doi
|
|
87
|
+
result["identifier"] = doi
|
|
88
|
+
return result
|
|
89
|
+
except Exception:
|
|
90
|
+
pass
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def try_doaj(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
95
|
+
"""Try DOAJ (Directory of Open Access Journals) for OA PDF.
|
|
96
|
+
|
|
97
|
+
DOAJ indexes 20,000+ OA journals.
|
|
98
|
+
API: https://doaj.org/api/v2/search/articles/doi:{doi}
|
|
99
|
+
"""
|
|
100
|
+
url = f"https://doaj.org/api/v2/search/articles/doi:{doi}"
|
|
101
|
+
try:
|
|
102
|
+
polite_delay(config)
|
|
103
|
+
s = requests.Session()
|
|
104
|
+
s.trust_env = False
|
|
105
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
106
|
+
resp = s.get(url, timeout=request_timeout(config),
|
|
107
|
+
proxies=proxy_dict(select_proxy_for_url(url, config)))
|
|
108
|
+
if resp.status_code >= 400:
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
data = resp.json()
|
|
112
|
+
results = data.get("results", [])
|
|
113
|
+
if not results:
|
|
114
|
+
return None
|
|
115
|
+
|
|
116
|
+
for item in results:
|
|
117
|
+
bibjson = item.get("bibjson", {})
|
|
118
|
+
|
|
119
|
+
# Check link field for fulltext URLs
|
|
120
|
+
links = bibjson.get("link", [])
|
|
121
|
+
for link in links:
|
|
122
|
+
if isinstance(link, dict):
|
|
123
|
+
link_url = link.get("url", "")
|
|
124
|
+
link_type = link.get("type", "")
|
|
125
|
+
if link_type == "fulltext" and link_url and link_url.startswith("http"):
|
|
126
|
+
polite_delay(config)
|
|
127
|
+
dl_result = download_pdf(link_url, output_path, config, "DOAJ",
|
|
128
|
+
require_pdf_like_url=False)
|
|
129
|
+
if dl_result:
|
|
130
|
+
dl_result["doi"] = doi
|
|
131
|
+
dl_result["identifier"] = doi
|
|
132
|
+
return dl_result
|
|
133
|
+
except Exception:
|
|
134
|
+
pass
|
|
135
|
+
return None
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""OpenAIRE aggregator source: green OA / repository copies via api.openaire.eu.
|
|
2
|
+
|
|
3
|
+
OpenAIRE aggregates repository copies (green OA, institutional repositories,
|
|
4
|
+
EU-funded deposits) that Unpaywall's index sometimes misses — its webresource
|
|
5
|
+
URLs are nested at varying depths, so extraction walks the payload generically.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from typing import Any
|
|
12
|
+
from urllib.parse import quote
|
|
13
|
+
|
|
14
|
+
from ..network import fetch_json, polite_delay
|
|
15
|
+
from ..download.pdf import download_pdf
|
|
16
|
+
|
|
17
|
+
_WEBRESOURCE_RE = re.compile(r"<webresource>\s*<url>([^<]+)</url>", re.I)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def extract_openaire_fulltext_urls(payload: Any) -> list[str]:
|
|
21
|
+
"""Recursively collect fulltext URLs from an OpenAIRE payload, PDF-ish first."""
|
|
22
|
+
urls: list[str] = []
|
|
23
|
+
|
|
24
|
+
def walk(node: Any) -> None:
|
|
25
|
+
if isinstance(node, dict):
|
|
26
|
+
for key, value in node.items():
|
|
27
|
+
if key in ("url", "fulltext") and isinstance(value, str) and value.startswith("http"):
|
|
28
|
+
urls.append(value)
|
|
29
|
+
else:
|
|
30
|
+
walk(value)
|
|
31
|
+
elif isinstance(node, list):
|
|
32
|
+
for item in node:
|
|
33
|
+
walk(item)
|
|
34
|
+
|
|
35
|
+
walk(payload)
|
|
36
|
+
seen: set[str] = set()
|
|
37
|
+
ordered: list[str] = []
|
|
38
|
+
for u in sorted(urls, key=lambda u: 0 if ".pdf" in u.lower() else 1):
|
|
39
|
+
if u not in seen:
|
|
40
|
+
seen.add(u)
|
|
41
|
+
ordered.append(u)
|
|
42
|
+
return ordered
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _extract_from_xml(xml: str) -> list[str]:
|
|
46
|
+
seen: set[str] = set()
|
|
47
|
+
ordered: list[str] = []
|
|
48
|
+
from html import unescape
|
|
49
|
+
|
|
50
|
+
for u in _WEBRESOURCE_RE.findall(xml):
|
|
51
|
+
u = unescape(u).strip()
|
|
52
|
+
if u.startswith("http") and u not in seen:
|
|
53
|
+
seen.add(u)
|
|
54
|
+
ordered.append(u)
|
|
55
|
+
return ordered
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def try_openaire(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
59
|
+
"""Race source: download an OA copy listed by the OpenAIRE aggregator."""
|
|
60
|
+
# XML is the proven shape: webresource URLs nest under results. The
|
|
61
|
+
# format=json variant does not expose url keys in a walkable way.
|
|
62
|
+
candidates = []
|
|
63
|
+
try:
|
|
64
|
+
import requests
|
|
65
|
+
|
|
66
|
+
from ..network import USER_AGENT
|
|
67
|
+
|
|
68
|
+
sess = requests.Session()
|
|
69
|
+
sess.trust_env = False
|
|
70
|
+
proxy = (config or {}).get("network_proxy", "")
|
|
71
|
+
r = sess.get(
|
|
72
|
+
f"https://api.openaire.eu/search/publications?doi={quote(doi, safe='')}",
|
|
73
|
+
headers={"User-Agent": USER_AGENT},
|
|
74
|
+
proxies={"http": proxy, "https": proxy} if proxy else None,
|
|
75
|
+
timeout=20,
|
|
76
|
+
)
|
|
77
|
+
if r.status_code == 200:
|
|
78
|
+
candidates = _extract_from_xml(r.text)
|
|
79
|
+
except Exception:
|
|
80
|
+
pass
|
|
81
|
+
|
|
82
|
+
if not candidates:
|
|
83
|
+
# Last resort: the format=json response (structure varies) — walk it.
|
|
84
|
+
payload = fetch_json(
|
|
85
|
+
f"https://api.openaire.eu/search/publications?doi={quote(doi, safe='')}&format=json",
|
|
86
|
+
config,
|
|
87
|
+
)
|
|
88
|
+
if payload:
|
|
89
|
+
candidates = extract_openaire_fulltext_urls(payload)
|
|
90
|
+
if not candidates:
|
|
91
|
+
# The legacy endpoint sometimes answers JSON requests with XML —
|
|
92
|
+
# parse that too before giving up.
|
|
93
|
+
try:
|
|
94
|
+
import requests
|
|
95
|
+
|
|
96
|
+
from ..network import USER_AGENT
|
|
97
|
+
|
|
98
|
+
s = requests.Session()
|
|
99
|
+
s.trust_env = False
|
|
100
|
+
proxy = (config or {}).get("network_proxy", "")
|
|
101
|
+
r = s.get(
|
|
102
|
+
f"https://api.openaire.eu/search/publications?doi={quote(doi, safe='')}",
|
|
103
|
+
headers={"User-Agent": USER_AGENT},
|
|
104
|
+
proxies={"http": proxy, "https": proxy} if proxy else None,
|
|
105
|
+
timeout=20,
|
|
106
|
+
)
|
|
107
|
+
if r.status_code == 200:
|
|
108
|
+
candidates = _extract_from_xml(r.text)
|
|
109
|
+
except Exception:
|
|
110
|
+
pass
|
|
111
|
+
|
|
112
|
+
for url in candidates[:5]:
|
|
113
|
+
polite_delay(config)
|
|
114
|
+
# require_pdf_like_url=False: OpenAIRE repository URLs often have no
|
|
115
|
+
# file extension (PLOS file?id=...); the %PDF check inside
|
|
116
|
+
# download_pdf is the real validation.
|
|
117
|
+
result = download_pdf(url, output_path, config, "OpenAIRE", require_pdf_like_url=False)
|
|
118
|
+
if result:
|
|
119
|
+
result["doi"] = doi
|
|
120
|
+
result["identifier"] = doi
|
|
121
|
+
return result
|
|
122
|
+
return None
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""OpenAlex OA direct link source + Content API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import urllib.parse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
from ..network import fetch_json, polite_delay, USER_AGENT, request_timeout, proxy_dict, select_proxy_for_url
|
|
12
|
+
from ..download.pdf import download_pdf, is_plausible_pdf_url, is_pdf_file, success, _response_looks_pdf
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def try_openalex_oa(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
16
|
+
"""Try OpenAlex best_oa_location for OA PDF link."""
|
|
17
|
+
q = urllib.parse.quote(doi, safe="")
|
|
18
|
+
email = config.get("email", "")
|
|
19
|
+
mailto = f"?mailto={email}" if email else ""
|
|
20
|
+
url = f"https://api.openalex.org/works/doi:{q}{mailto}"
|
|
21
|
+
try:
|
|
22
|
+
payload = fetch_json(url, config)
|
|
23
|
+
if not payload:
|
|
24
|
+
return None
|
|
25
|
+
oa = payload.get("open_access", {})
|
|
26
|
+
if not oa.get("is_oa"):
|
|
27
|
+
return None
|
|
28
|
+
pdf_url = oa.get("oa_url") or ""
|
|
29
|
+
if not pdf_url:
|
|
30
|
+
primary = payload.get("primary_location", {})
|
|
31
|
+
if isinstance(primary, dict):
|
|
32
|
+
pdf_url = primary.get("pdf_url") or ""
|
|
33
|
+
if pdf_url and is_plausible_pdf_url(pdf_url):
|
|
34
|
+
polite_delay(config)
|
|
35
|
+
result = download_pdf(pdf_url, output_path, config, "OpenAlexOA")
|
|
36
|
+
if result:
|
|
37
|
+
result["doi"] = doi
|
|
38
|
+
result["identifier"] = doi
|
|
39
|
+
return result
|
|
40
|
+
except Exception:
|
|
41
|
+
pass
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def try_openalex_content_api(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
46
|
+
"""Try OpenAlex Content API for direct PDF download.
|
|
47
|
+
|
|
48
|
+
Uses content.openalex.org which returns a 302 redirect to a signed
|
|
49
|
+
Cloudflare R2 URL containing the actual PDF.
|
|
50
|
+
Requires a free API key from https://openalex.org/users (set openalex_api_key in config).
|
|
51
|
+
"""
|
|
52
|
+
api_key = config.get("openalex_api_key", "")
|
|
53
|
+
if not api_key:
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
q = urllib.parse.quote(doi, safe="")
|
|
57
|
+
meta_url = f"https://api.openalex.org/works/doi:{q}?api_key={api_key}" # was sent as mailto
|
|
58
|
+
try:
|
|
59
|
+
payload = fetch_json(meta_url, config)
|
|
60
|
+
if not payload:
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
work_id = payload.get("id", "")
|
|
64
|
+
if not work_id:
|
|
65
|
+
return None
|
|
66
|
+
if "/" in work_id:
|
|
67
|
+
work_id = work_id.rsplit("/", 1)[-1]
|
|
68
|
+
|
|
69
|
+
oa = payload.get("open_access", {})
|
|
70
|
+
if not oa.get("is_oa"):
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
content_url = f"https://content.openalex.org/works/{work_id}.pdf?api_key={api_key}"
|
|
74
|
+
polite_delay(config)
|
|
75
|
+
|
|
76
|
+
s = requests.Session()
|
|
77
|
+
s.trust_env = False
|
|
78
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
79
|
+
|
|
80
|
+
resp = s.get(content_url, timeout=request_timeout(config),
|
|
81
|
+
proxies=proxy_dict(select_proxy_for_url(content_url, config)),
|
|
82
|
+
allow_redirects=True, stream=True)
|
|
83
|
+
|
|
84
|
+
if resp.status_code >= 400:
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
88
|
+
first = next(iterator, b"")
|
|
89
|
+
if not _response_looks_pdf(resp, first):
|
|
90
|
+
return None
|
|
91
|
+
|
|
92
|
+
from .publishers import _write_pdf_atomic
|
|
93
|
+
if not _write_pdf_atomic(output_path, first, iterator):
|
|
94
|
+
return None
|
|
95
|
+
|
|
96
|
+
if is_pdf_file(output_path):
|
|
97
|
+
return success(doi, output_path, "OpenAlexContent")
|
|
98
|
+
except Exception:
|
|
99
|
+
pass
|
|
100
|
+
return None
|