litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,116 @@
1
+ """LibGen source with multi-mirror failover."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ import time
7
+ import urllib.parse
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from ..network import fetch, polite_delay
12
+ from ..download.pdf import download_pdf
13
+
14
+ LIBGEN_MIRRORS = [
15
+ "https://libgen.li",
16
+ "https://libgen.bz",
17
+ "https://libgen.gs",
18
+ "https://libgen.rs",
19
+ "https://libgen.st",
20
+ ]
21
+
22
+
23
+ def try_libgen(doi: str, output_path: Path, config: dict[str, Any], use_tor: bool = False) -> dict[str, Any] | None:
24
+ q = urllib.parse.quote(doi, safe="")
25
+ for mirror in LIBGEN_MIRRORS:
26
+ result = _try_libgen_mirror(doi, q, mirror, output_path, config, use_tor=use_tor)
27
+ if result:
28
+ return result
29
+ return None
30
+
31
+
32
+ def _try_libgen_mirror(
33
+ doi: str, q: str, mirror: str, output_path: Path, config: dict[str, Any], use_tor: bool = False
34
+ ) -> dict[str, Any] | None:
35
+ url = f"{mirror}/ads.php?doi={q}"
36
+ # Slow-CDN lane: floor the timeouts (users run 3/7s for source racing —
37
+ # the booksdl CDN drips and 503s) and allow a longer stream deadline.
38
+ config = {
39
+ **config,
40
+ "connect_timeout": max(int(config.get("connect_timeout", 3)), 10),
41
+ "read_timeout": max(int(config.get("read_timeout", 7)), 30),
42
+ "download_deadline_seconds": max(int(config.get("download_deadline_seconds", 60)), 180),
43
+ }
44
+ # The li/bz frontends serve EMPTY 200 responses (Content-Length: 0) to
45
+ # cookieless one-shot requests — the flow then finds no get.php link and
46
+ # silently misses. Warm a session on the homepage first and hit ads.php
47
+ # with its cookies + Referer; the page then renders normally
48
+ # (field-verified 2026-09: li via proxy, bz also direct).
49
+ try:
50
+ import requests
51
+
52
+ from ..network import USER_AGENT
53
+
54
+ session = requests.Session()
55
+ session.trust_env = False
56
+ session.headers.update({"User-Agent": USER_AGENT})
57
+ from ..network import select_proxy_for_url
58
+ proxy = select_proxy_for_url(mirror, config) # scihub_via_proxy / network_proxy
59
+ if proxy:
60
+ session.proxies = {"http": proxy, "https": proxy}
61
+ try:
62
+ session.get(f"{mirror}/", timeout=(10, 15))
63
+ except Exception:
64
+ pass # warm-up is best effort — ads.php sometimes works bare
65
+ resp = session.get(url, headers={"Referer": f"{mirror}/"},
66
+ timeout=(10, 25))
67
+
68
+ # Fallback to browser on Cloudflare/403
69
+ if resp.status_code in (403, 503):
70
+ from ..browser.engine import solve_url, is_available
71
+ if config.get("scihub_use_browser", True) and is_available(config):
72
+ result = solve_url(url, config)
73
+ if result:
74
+ solution = result.get("solution", {})
75
+ if solution.get("status", 0) < 400:
76
+ html = solution.get("response", "")
77
+ else:
78
+ return None
79
+ else:
80
+ return None
81
+ else:
82
+ return None
83
+ else:
84
+ if resp.status_code >= 400:
85
+ return None
86
+ html = resp.text
87
+ # booksdl CDN rotates nodes and 503s most requests (field 2026-09:
88
+ # ~1-in-4 serves the PDF) — retry the same link a few times.
89
+ for match in re.finditer(r'''href=["']([^"']*get\.php[^"']+)["']''', html, re.I):
90
+ dl_path = match.group(1)
91
+ dl_url = urllib.parse.urljoin(url, dl_path)
92
+ result = None
93
+ for attempt in range(4):
94
+ polite_delay(config)
95
+ result = download_pdf(dl_url, output_path, config, "LibGen",
96
+ require_pdf_like_url=False, use_tor=use_tor)
97
+ if result:
98
+ break
99
+ time.sleep(2 + 2 * attempt)
100
+ if result:
101
+ result["doi"] = doi
102
+ result["identifier"] = doi
103
+ return result
104
+ for match in re.finditer(r'''href=["']([^"']*\.pdf[^"']*)["']''', html, re.I):
105
+ dl_path = match.group(1)
106
+ if "get.php" in dl_path or dl_path.endswith(".pdf"):
107
+ dl_url = urllib.parse.urljoin(url, dl_path)
108
+ polite_delay(config)
109
+ result = download_pdf(dl_url, output_path, config, "LibGen", require_pdf_like_url=False, use_tor=use_tor)
110
+ if result:
111
+ result["doi"] = doi
112
+ result["identifier"] = doi
113
+ return result
114
+ except Exception:
115
+ pass
116
+ return None
@@ -0,0 +1,39 @@
1
+ """Nature.com direct PDF download."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from ..network import fetch
10
+ from ..download.pdf import download_pdf
11
+
12
+
13
+ def try_nature_direct(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
14
+ if not doi.startswith("10.1038/"):
15
+ return None
16
+ article_id = doi.replace("10.1038/", "")
17
+ pdf_url = f"https://www.nature.com/articles/{article_id}.pdf"
18
+ result = download_pdf(pdf_url, output_path, config, "NatureDirect", require_pdf_like_url=False)
19
+ if result:
20
+ result["doi"] = doi
21
+ result["identifier"] = doi
22
+ return result
23
+
24
+ article_url = f"https://www.nature.com/articles/{article_id}"
25
+ try:
26
+ resp = fetch(article_url, config)
27
+ if resp.status_code < 400:
28
+ html = resp.text
29
+ pdf_pattern = re.compile(r'href=["\'](/articles/[^"\' ]*\.pdf[^"\' ]*)["\']', re.I)
30
+ for match in pdf_pattern.finditer(html):
31
+ dl_url = f"https://www.nature.com{match.group(1)}"
32
+ result = download_pdf(dl_url, output_path, config, "NatureDirect", require_pdf_like_url=False)
33
+ if result:
34
+ result["doi"] = doi
35
+ result["identifier"] = doi
36
+ return result
37
+ except Exception:
38
+ pass
39
+ return None
@@ -0,0 +1,135 @@
1
+ """OA discovery sources: OpenAIRE, DOAJ.
2
+
3
+ These APIs find OA versions of papers that may not be indexed by Unpaywall/OpenAlex.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import urllib.parse
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import requests
13
+
14
+ from ..network import polite_delay, USER_AGENT, request_timeout, proxy_dict, select_proxy_for_url
15
+ from ..download.pdf import download_pdf, is_plausible_pdf_url
16
+
17
+
18
+ def try_openaire(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
19
+ """Try OpenAIRE API for OA PDF.
20
+
21
+ OpenAIRE aggregates European open access publications.
22
+ Extracts PDF URLs from children records (DOAJ, PMC, etc.).
23
+ """
24
+ q = urllib.parse.quote(doi, safe="")
25
+ url = f"https://api.openaire.eu/search/publications?doi={q}&format=json"
26
+ try:
27
+ polite_delay(config)
28
+ s = requests.Session()
29
+ s.trust_env = False
30
+ s.headers.update({"User-Agent": USER_AGENT})
31
+ resp = s.get(url, timeout=request_timeout(config),
32
+ proxies=proxy_dict(select_proxy_for_url(url, config)))
33
+ if resp.status_code >= 400:
34
+ return None
35
+
36
+ data = resp.json()
37
+ results = data.get("response", {}).get("results", {})
38
+ if not results:
39
+ return None
40
+
41
+ publications = results.get("result", [])
42
+ if isinstance(publications, dict):
43
+ publications = [publications]
44
+
45
+ for pub in publications:
46
+ main = pub.get("metadata", {}).get("oaf:entity", {}).get("oaf:result", {})
47
+ if not main:
48
+ continue
49
+
50
+ # Check access right
51
+ access_right = main.get("bestaccessright", {})
52
+ if isinstance(access_right, dict):
53
+ if access_right.get("classid", "") != "OPEN":
54
+ continue
55
+
56
+ # Extract URLs from children records
57
+ children = main.get("children", {})
58
+ child_results = children.get("result", [])
59
+ if isinstance(child_results, dict):
60
+ child_results = [child_results]
61
+
62
+ for child in child_results:
63
+ instance = child.get("instance", {})
64
+ # Check for direct URL
65
+ url_obj = instance.get("url", {})
66
+ child_url = url_obj.get("$", "") if isinstance(url_obj, dict) else ""
67
+ if child_url and child_url.startswith("http"):
68
+ polite_delay(config)
69
+ result = download_pdf(child_url, output_path, config, "OpenAIRE",
70
+ require_pdf_like_url=False)
71
+ if result:
72
+ result["doi"] = doi
73
+ result["identifier"] = doi
74
+ return result
75
+
76
+ # Check webresource URL
77
+ webres = instance.get("webresource", {})
78
+ if isinstance(webres, dict):
79
+ wr_url = webres.get("url", {})
80
+ child_url = wr_url.get("$", "") if isinstance(wr_url, dict) else ""
81
+ if child_url and child_url.startswith("http"):
82
+ polite_delay(config)
83
+ result = download_pdf(child_url, output_path, config, "OpenAIRE",
84
+ require_pdf_like_url=False)
85
+ if result:
86
+ result["doi"] = doi
87
+ result["identifier"] = doi
88
+ return result
89
+ except Exception:
90
+ pass
91
+ return None
92
+
93
+
94
+ def try_doaj(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
95
+ """Try DOAJ (Directory of Open Access Journals) for OA PDF.
96
+
97
+ DOAJ indexes 20,000+ OA journals.
98
+ API: https://doaj.org/api/v2/search/articles/doi:{doi}
99
+ """
100
+ url = f"https://doaj.org/api/v2/search/articles/doi:{doi}"
101
+ try:
102
+ polite_delay(config)
103
+ s = requests.Session()
104
+ s.trust_env = False
105
+ s.headers.update({"User-Agent": USER_AGENT})
106
+ resp = s.get(url, timeout=request_timeout(config),
107
+ proxies=proxy_dict(select_proxy_for_url(url, config)))
108
+ if resp.status_code >= 400:
109
+ return None
110
+
111
+ data = resp.json()
112
+ results = data.get("results", [])
113
+ if not results:
114
+ return None
115
+
116
+ for item in results:
117
+ bibjson = item.get("bibjson", {})
118
+
119
+ # Check link field for fulltext URLs
120
+ links = bibjson.get("link", [])
121
+ for link in links:
122
+ if isinstance(link, dict):
123
+ link_url = link.get("url", "")
124
+ link_type = link.get("type", "")
125
+ if link_type == "fulltext" and link_url and link_url.startswith("http"):
126
+ polite_delay(config)
127
+ dl_result = download_pdf(link_url, output_path, config, "DOAJ",
128
+ require_pdf_like_url=False)
129
+ if dl_result:
130
+ dl_result["doi"] = doi
131
+ dl_result["identifier"] = doi
132
+ return dl_result
133
+ except Exception:
134
+ pass
135
+ return None
@@ -0,0 +1,122 @@
1
+ """OpenAIRE aggregator source: green OA / repository copies via api.openaire.eu.
2
+
3
+ OpenAIRE aggregates repository copies (green OA, institutional repositories,
4
+ EU-funded deposits) that Unpaywall's index sometimes misses — its webresource
5
+ URLs are nested at varying depths, so extraction walks the payload generically.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ from typing import Any
12
+ from urllib.parse import quote
13
+
14
+ from ..network import fetch_json, polite_delay
15
+ from ..download.pdf import download_pdf
16
+
17
+ _WEBRESOURCE_RE = re.compile(r"<webresource>\s*<url>([^<]+)</url>", re.I)
18
+
19
+
20
+ def extract_openaire_fulltext_urls(payload: Any) -> list[str]:
21
+ """Recursively collect fulltext URLs from an OpenAIRE payload, PDF-ish first."""
22
+ urls: list[str] = []
23
+
24
+ def walk(node: Any) -> None:
25
+ if isinstance(node, dict):
26
+ for key, value in node.items():
27
+ if key in ("url", "fulltext") and isinstance(value, str) and value.startswith("http"):
28
+ urls.append(value)
29
+ else:
30
+ walk(value)
31
+ elif isinstance(node, list):
32
+ for item in node:
33
+ walk(item)
34
+
35
+ walk(payload)
36
+ seen: set[str] = set()
37
+ ordered: list[str] = []
38
+ for u in sorted(urls, key=lambda u: 0 if ".pdf" in u.lower() else 1):
39
+ if u not in seen:
40
+ seen.add(u)
41
+ ordered.append(u)
42
+ return ordered
43
+
44
+
45
+ def _extract_from_xml(xml: str) -> list[str]:
46
+ seen: set[str] = set()
47
+ ordered: list[str] = []
48
+ from html import unescape
49
+
50
+ for u in _WEBRESOURCE_RE.findall(xml):
51
+ u = unescape(u).strip()
52
+ if u.startswith("http") and u not in seen:
53
+ seen.add(u)
54
+ ordered.append(u)
55
+ return ordered
56
+
57
+
58
+ def try_openaire(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
59
+ """Race source: download an OA copy listed by the OpenAIRE aggregator."""
60
+ # XML is the proven shape: webresource URLs nest under results. The
61
+ # format=json variant does not expose url keys in a walkable way.
62
+ candidates = []
63
+ try:
64
+ import requests
65
+
66
+ from ..network import USER_AGENT
67
+
68
+ sess = requests.Session()
69
+ sess.trust_env = False
70
+ proxy = (config or {}).get("network_proxy", "")
71
+ r = sess.get(
72
+ f"https://api.openaire.eu/search/publications?doi={quote(doi, safe='')}",
73
+ headers={"User-Agent": USER_AGENT},
74
+ proxies={"http": proxy, "https": proxy} if proxy else None,
75
+ timeout=20,
76
+ )
77
+ if r.status_code == 200:
78
+ candidates = _extract_from_xml(r.text)
79
+ except Exception:
80
+ pass
81
+
82
+ if not candidates:
83
+ # Last resort: the format=json response (structure varies) — walk it.
84
+ payload = fetch_json(
85
+ f"https://api.openaire.eu/search/publications?doi={quote(doi, safe='')}&format=json",
86
+ config,
87
+ )
88
+ if payload:
89
+ candidates = extract_openaire_fulltext_urls(payload)
90
+ if not candidates:
91
+ # The legacy endpoint sometimes answers JSON requests with XML —
92
+ # parse that too before giving up.
93
+ try:
94
+ import requests
95
+
96
+ from ..network import USER_AGENT
97
+
98
+ s = requests.Session()
99
+ s.trust_env = False
100
+ proxy = (config or {}).get("network_proxy", "")
101
+ r = s.get(
102
+ f"https://api.openaire.eu/search/publications?doi={quote(doi, safe='')}",
103
+ headers={"User-Agent": USER_AGENT},
104
+ proxies={"http": proxy, "https": proxy} if proxy else None,
105
+ timeout=20,
106
+ )
107
+ if r.status_code == 200:
108
+ candidates = _extract_from_xml(r.text)
109
+ except Exception:
110
+ pass
111
+
112
+ for url in candidates[:5]:
113
+ polite_delay(config)
114
+ # require_pdf_like_url=False: OpenAIRE repository URLs often have no
115
+ # file extension (PLOS file?id=...); the %PDF check inside
116
+ # download_pdf is the real validation.
117
+ result = download_pdf(url, output_path, config, "OpenAIRE", require_pdf_like_url=False)
118
+ if result:
119
+ result["doi"] = doi
120
+ result["identifier"] = doi
121
+ return result
122
+ return None
@@ -0,0 +1,100 @@
1
+ """OpenAlex OA direct link source + Content API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import urllib.parse
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ import requests
10
+
11
+ from ..network import fetch_json, polite_delay, USER_AGENT, request_timeout, proxy_dict, select_proxy_for_url
12
+ from ..download.pdf import download_pdf, is_plausible_pdf_url, is_pdf_file, success, _response_looks_pdf
13
+
14
+
15
+ def try_openalex_oa(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
16
+ """Try OpenAlex best_oa_location for OA PDF link."""
17
+ q = urllib.parse.quote(doi, safe="")
18
+ email = config.get("email", "")
19
+ mailto = f"?mailto={email}" if email else ""
20
+ url = f"https://api.openalex.org/works/doi:{q}{mailto}"
21
+ try:
22
+ payload = fetch_json(url, config)
23
+ if not payload:
24
+ return None
25
+ oa = payload.get("open_access", {})
26
+ if not oa.get("is_oa"):
27
+ return None
28
+ pdf_url = oa.get("oa_url") or ""
29
+ if not pdf_url:
30
+ primary = payload.get("primary_location", {})
31
+ if isinstance(primary, dict):
32
+ pdf_url = primary.get("pdf_url") or ""
33
+ if pdf_url and is_plausible_pdf_url(pdf_url):
34
+ polite_delay(config)
35
+ result = download_pdf(pdf_url, output_path, config, "OpenAlexOA")
36
+ if result:
37
+ result["doi"] = doi
38
+ result["identifier"] = doi
39
+ return result
40
+ except Exception:
41
+ pass
42
+ return None
43
+
44
+
45
+ def try_openalex_content_api(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
46
+ """Try OpenAlex Content API for direct PDF download.
47
+
48
+ Uses content.openalex.org which returns a 302 redirect to a signed
49
+ Cloudflare R2 URL containing the actual PDF.
50
+ Requires a free API key from https://openalex.org/users (set openalex_api_key in config).
51
+ """
52
+ api_key = config.get("openalex_api_key", "")
53
+ if not api_key:
54
+ return None
55
+
56
+ q = urllib.parse.quote(doi, safe="")
57
+ meta_url = f"https://api.openalex.org/works/doi:{q}?api_key={api_key}" # was sent as mailto
58
+ try:
59
+ payload = fetch_json(meta_url, config)
60
+ if not payload:
61
+ return None
62
+
63
+ work_id = payload.get("id", "")
64
+ if not work_id:
65
+ return None
66
+ if "/" in work_id:
67
+ work_id = work_id.rsplit("/", 1)[-1]
68
+
69
+ oa = payload.get("open_access", {})
70
+ if not oa.get("is_oa"):
71
+ return None
72
+
73
+ content_url = f"https://content.openalex.org/works/{work_id}.pdf?api_key={api_key}"
74
+ polite_delay(config)
75
+
76
+ s = requests.Session()
77
+ s.trust_env = False
78
+ s.headers.update({"User-Agent": USER_AGENT})
79
+
80
+ resp = s.get(content_url, timeout=request_timeout(config),
81
+ proxies=proxy_dict(select_proxy_for_url(content_url, config)),
82
+ allow_redirects=True, stream=True)
83
+
84
+ if resp.status_code >= 400:
85
+ return None
86
+
87
+ iterator = resp.iter_content(chunk_size=8192)
88
+ first = next(iterator, b"")
89
+ if not _response_looks_pdf(resp, first):
90
+ return None
91
+
92
+ from .publishers import _write_pdf_atomic
93
+ if not _write_pdf_atomic(output_path, first, iterator):
94
+ return None
95
+
96
+ if is_pdf_file(output_path):
97
+ return success(doi, output_path, "OpenAlexContent")
98
+ except Exception:
99
+ pass
100
+ return None