litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,451 @@
1
+ """Publisher-owned PDF routing helpers.
2
+
3
+ This module keeps DOI/source-page PDF routes out of individual download
4
+ callers. It follows the same shape as provider catalogs: profile templates
5
+ first, then source URL derivation, then page-discovered links.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import html
11
+ import re
12
+ from collections.abc import Sequence
13
+ from urllib.parse import parse_qs, quote, urljoin, urlparse, urlunparse
14
+
15
+ from bs4 import BeautifulSoup
16
+
17
+ from .profiles import PublisherProfile
18
+
19
+ PDF_LINK_TEXT_MARKERS = ("pdf", "download pdf", "full text pdf", "view pdf")
20
+ PDF_URL_MARKERS = (".pdf", "/pdf", "/epdf", "/pdfdirect", "/pdfft", "download=true")
21
+ PDF_JS_DEFAULT_URL_RE = re.compile(
22
+ r"PDFViewerApplicationOptions\.set\(\s*['\"]defaultUrl['\"]\s*,\s*['\"]([^'\"]+)['\"]",
23
+ flags=re.IGNORECASE,
24
+ )
25
+ PLOS_JOURNAL_PATHS = {
26
+ "pbio": "plosbiology",
27
+ "pcbi": "ploscompbiol",
28
+ "pclm": "climate",
29
+ "pdig": "digitalhealth",
30
+ "pgen": "plosgenetics",
31
+ "pgph": "globalpublichealth",
32
+ "pmed": "plosmedicine",
33
+ "pntd": "plosntds",
34
+ "pone": "plosone",
35
+ "ppat": "plospathogens",
36
+ "pstr": "sustainabilitytransformation",
37
+ "pwat": "water",
38
+ }
39
+ PLOS_DOI_JOURNAL_RE = re.compile(r"^10\.1371/journal\.([a-z0-9]+)\.", flags=re.IGNORECASE)
40
+ COPERNICUS_DOI_RE = re.compile(
41
+ r"^10\.5194/(?P<journal>[a-z0-9]+)-(?P<volume>\d+)-(?P<page>.+)-(?P<year>\d{4})$",
42
+ flags=re.IGNORECASE,
43
+ )
44
+ MDPI_ISSN_JOURNAL_CODES = {
45
+ "1424-8220": "s",
46
+ "1660-4601": "ijerph",
47
+ "1996-1073": "en",
48
+ "2071-1050": "su",
49
+ "2072-4292": "rs",
50
+ "2073-4441": "w",
51
+ "2077-0375": "membranes",
52
+ "2227-7390": "math",
53
+ "2304-8158": "foods",
54
+ }
55
+ MDPI_JOURNAL_CODE_ISSNS = {
56
+ journal_code: issn
57
+ for issn, journal_code in MDPI_ISSN_JOURNAL_CODES.items()
58
+ }
59
+ MDPI_NUMERIC_PATH_RE = re.compile(r"/[0-9]{4}-[0-9Xx]{3,4}/[0-9]+/[0-9]+/[0-9]+$")
60
+
61
+
62
+ def extract_elsevier_pii(url: str) -> str:
63
+ """Extract an Elsevier PII from article, retrieve, query, or signed asset URLs."""
64
+ for pattern in (
65
+ r"/pii/([A-Z0-9]+)",
66
+ r"/retrieve/pii/([A-Z0-9]+)",
67
+ r"[?&]pii=([A-Z0-9]+)",
68
+ r"1-s2\.0-([A-Z0-9]+)",
69
+ ):
70
+ match = re.search(pattern, url, flags=re.IGNORECASE)
71
+ if match:
72
+ return match.group(1)
73
+ return ""
74
+
75
+
76
+ def extract_ieee_article_number(url: str) -> str:
77
+ """Extract an IEEE Xplore article number from document or query URLs."""
78
+ for pattern in (
79
+ r"/document/(\d+)",
80
+ r"[?&]arnumber=(\d+)",
81
+ ):
82
+ match = re.search(pattern, url, flags=re.IGNORECASE)
83
+ if match:
84
+ return match.group(1)
85
+ return ""
86
+
87
+
88
+ def discover_pdf_candidates_from_html(html_text: str, source_url: str) -> list[str]:
89
+ """Extract likely PDF URLs from publisher HTML without committing to a provider."""
90
+ candidates: list[str] = []
91
+ soup = BeautifulSoup(html_text, "lxml")
92
+
93
+ for meta in soup.find_all("meta"):
94
+ key = _clean(meta.get("name") or meta.get("property") or meta.get("itemprop")).lower()
95
+ content = _clean(meta.get("content"))
96
+ if content and ("citation_pdf_url" in key or key.endswith("pdf_url") or key == "pdf_url"):
97
+ _append_candidate(candidates, content, source_url=source_url)
98
+ _append_query_pdf_candidates(candidates, content, source_url=source_url)
99
+
100
+ for node in soup.find_all(["a", "link", "iframe", "embed", "object"]):
101
+ target = _clean(node.get("href") or node.get("src") or node.get("data"))
102
+ if not target:
103
+ continue
104
+ label = _clean(
105
+ " ".join(
106
+ filter(
107
+ None,
108
+ [node.get_text(" ", strip=True), node.get("title"), node.get("aria-label")],
109
+ )
110
+ )
111
+ ).lower()
112
+ content_type = _clean(node.get("type")).lower()
113
+ lower_target = target.lower()
114
+ if (
115
+ any(marker in lower_target for marker in PDF_URL_MARKERS)
116
+ or any(marker in label for marker in PDF_LINK_TEXT_MARKERS)
117
+ or "pdf" in content_type
118
+ ):
119
+ _append_candidate(candidates, target, source_url=source_url)
120
+ _append_query_pdf_candidates(candidates, target, source_url=source_url)
121
+
122
+ for script in soup.find_all("script"):
123
+ text = script.string if script.string is not None else script.get_text(" ", strip=False)
124
+ for match in PDF_JS_DEFAULT_URL_RE.finditer(str(text or "")):
125
+ target = html.unescape(match.group(1))
126
+ if any(marker in target.lower() for marker in PDF_URL_MARKERS):
127
+ _append_candidate(candidates, target, source_url=source_url)
128
+ _append_query_pdf_candidates(candidates, target, source_url=source_url)
129
+
130
+ return candidates
131
+
132
+
133
+ def build_pdf_candidates(
134
+ profile: PublisherProfile,
135
+ doi: str,
136
+ *,
137
+ source_url: str = "",
138
+ discovered_urls: Sequence[str] = (),
139
+ ) -> list[str]:
140
+ """Build ordered, de-duplicated official PDF candidates for a publisher profile."""
141
+ candidates: list[str] = []
142
+ pii = extract_elsevier_pii(source_url)
143
+ if profile.name.lower() == "elsevier" and pii:
144
+ _append_candidate(candidates, f"https://www.sciencedirect.com/science/article/pii/{pii}/pdfft")
145
+ arnumber = extract_ieee_article_number(source_url)
146
+ if profile.name.lower() == "ieee" and arnumber:
147
+ _append_candidate(
148
+ candidates,
149
+ f"https://ieeexplore.ieee.org/stampPDF/getPDF.jsp?tp=&isnumber=&arnumber={arnumber}",
150
+ )
151
+ _append_candidate(
152
+ candidates,
153
+ f"https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber={arnumber}",
154
+ )
155
+ if profile.name.lower() == "aps":
156
+ _append_aps_candidates(candidates, source_url)
157
+ if profile.name.lower() == "ams":
158
+ _append_ams_candidates(candidates, source_url)
159
+ if profile.name.lower() == "plos":
160
+ _append_plos_candidates(candidates, doi, source_url)
161
+ if profile.name.lower() == "copernicus":
162
+ _append_copernicus_candidates(candidates, doi, source_url)
163
+ if profile.name.lower() == "mdpi":
164
+ _append_mdpi_candidates(candidates, doi, source_url)
165
+ if profile.name.lower() == "oxford academic":
166
+ _append_oxford_candidates(candidates, doi, source_url)
167
+
168
+ _append_source_path_candidates(candidates, profile, doi, source_url)
169
+
170
+ for url in profile.pdf_urls(doi):
171
+ _append_candidate(candidates, url)
172
+
173
+ for url in discovered_urls:
174
+ if not _clean(url):
175
+ continue
176
+ if is_supplementary_url(profile, url):
177
+ continue
178
+ if not is_pdf_candidate_url(profile, url):
179
+ continue
180
+ if not belongs_to_current_article(profile, url, doi=doi, source_url=source_url):
181
+ continue
182
+ _append_candidate(candidates, url, source_url=source_url)
183
+
184
+ return candidates
185
+
186
+
187
+ def is_pdf_candidate_url(profile: PublisherProfile, url: str) -> bool:
188
+ lower = url.lower()
189
+ return lower.endswith(".pdf") or any(marker.lower() in lower for marker in profile.pdf_url_markers)
190
+
191
+
192
+ def is_supplementary_url(profile: PublisherProfile, url: str) -> bool:
193
+ lower = url.lower()
194
+ return any(marker.lower() in lower for marker in profile.supplementary_url_markers)
195
+
196
+
197
+ def belongs_to_current_article(profile: PublisherProfile, url: str, *, doi: str, source_url: str = "") -> bool:
198
+ if profile.name.lower() == "aps":
199
+ lower = url.lower()
200
+ host = (urlparse(url).hostname or "").lower()
201
+ if host.endswith("journals.aps.org") or host.endswith("link.aps.org"):
202
+ if "/accepted" in lower:
203
+ return False
204
+ doi_lower = doi.lower()
205
+ doi_escaped = doi_lower.replace("/", "%2f")
206
+ if "/pdf/" in lower or "/doi/" in lower:
207
+ return doi_lower in lower or doi_escaped in lower
208
+ return True
209
+ if profile.name.lower() != "elsevier":
210
+ return True
211
+ source_pii = extract_elsevier_pii(source_url)
212
+ candidate_pii = extract_elsevier_pii(url)
213
+ if source_pii and candidate_pii:
214
+ return source_pii.lower() == candidate_pii.lower()
215
+ lower = url.lower()
216
+ doi_lower = doi.lower()
217
+ doi_escaped = doi_lower.replace("/", "%2f")
218
+ if source_pii:
219
+ return source_pii.lower() in lower or doi_lower in lower or doi_escaped in lower
220
+ return True
221
+
222
+
223
+ def _append_source_path_candidates(
224
+ candidates: list[str],
225
+ profile: PublisherProfile,
226
+ doi: str,
227
+ source_url: str,
228
+ ) -> None:
229
+ normalized = _clean(source_url)
230
+ if not normalized:
231
+ return
232
+ parsed = urlparse(normalized)
233
+ host = (parsed.hostname or "").lower()
234
+ source_path = parsed.path.rstrip("/")
235
+ if parsed.scheme not in {"http", "https"} or not host or not source_path:
236
+ return
237
+ for template in profile.pdf_source_path_templates:
238
+ domain = template.domain.lower()
239
+ if host != domain and not host.endswith(f".{domain}"):
240
+ continue
241
+ path_prefix = _clean(template.path_prefix)
242
+ if path_prefix and not source_path.startswith(path_prefix):
243
+ continue
244
+ candidate_path = _format_template(
245
+ template.path_template,
246
+ doi=doi,
247
+ source_path=source_path,
248
+ source_path_after_prefix=source_path[len(path_prefix):].lstrip("/") if path_prefix else source_path.lstrip("/"),
249
+ )
250
+ if not candidate_path:
251
+ continue
252
+ parsed_candidate = urlparse(candidate_path)
253
+ if parsed_candidate.scheme in {"http", "https"}:
254
+ _append_candidate(candidates, candidate_path)
255
+ continue
256
+ if not candidate_path.startswith("/"):
257
+ candidate_path = f"/{candidate_path}"
258
+ _append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, candidate_path, "", "", "")))
259
+
260
+
261
+ def _append_aps_candidates(candidates: list[str], source_url: str) -> None:
262
+ normalized = _clean(source_url)
263
+ if not normalized:
264
+ return
265
+ parsed = urlparse(normalized)
266
+ host = (parsed.hostname or "").lower()
267
+ source_path = parsed.path.rstrip("/")
268
+ if parsed.scheme not in {"http", "https"} or not host or not source_path:
269
+ return
270
+ if host.endswith("link.aps.org") and source_path.startswith("/doi/"):
271
+ pdf_path = source_path.replace("/doi/", "/pdf/", 1)
272
+ _append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
273
+ return
274
+ if not host.endswith("journals.aps.org"):
275
+ return
276
+ if "/abstract/" in source_path:
277
+ pdf_path = source_path.replace("/abstract/", "/pdf/", 1)
278
+ elif source_path.endswith("/pdf"):
279
+ pdf_path = source_path
280
+ else:
281
+ return
282
+ _append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
283
+
284
+
285
+ def _append_ams_candidates(candidates: list[str], source_url: str) -> None:
286
+ normalized = _clean(source_url)
287
+ if not normalized:
288
+ return
289
+ parsed = urlparse(normalized)
290
+ host = (parsed.hostname or "").lower()
291
+ source_path = parsed.path.rstrip("/")
292
+ if parsed.scheme not in {"http", "https"} or not host.endswith("journals.ametsoc.org"):
293
+ return
294
+ if not source_path.startswith("/view/") or not source_path.lower().endswith(".xml"):
295
+ return
296
+ pdf_path = f"/downloadpdf{source_path[:-4]}.pdf"
297
+ _append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
298
+
299
+
300
+ def _append_plos_candidates(candidates: list[str], doi: str, source_url: str) -> None:
301
+ journal_path = _plos_journal_path_from_source(source_url) or _plos_journal_path_from_doi(doi)
302
+ if not journal_path:
303
+ return
304
+ _append_candidate(
305
+ candidates,
306
+ f"https://journals.plos.org/{journal_path}/article/file?id={doi}&type=printable",
307
+ )
308
+
309
+
310
+ def _plos_journal_path_from_source(source_url: str) -> str:
311
+ parsed = urlparse(_clean(source_url))
312
+ host = (parsed.hostname or "").lower()
313
+ if host != "journals.plos.org":
314
+ return ""
315
+ parts = [part for part in parsed.path.split("/") if part]
316
+ if not parts or parts[0] == "article":
317
+ return ""
318
+ return parts[0]
319
+
320
+
321
+ def _plos_journal_path_from_doi(doi: str) -> str:
322
+ match = PLOS_DOI_JOURNAL_RE.match(doi.strip())
323
+ if not match:
324
+ return ""
325
+ return PLOS_JOURNAL_PATHS.get(match.group(1).lower(), "")
326
+
327
+
328
+ def _append_copernicus_candidates(candidates: list[str], doi: str, source_url: str) -> None:
329
+ match = COPERNICUS_DOI_RE.match(doi.strip().lower())
330
+ if not match:
331
+ return
332
+ parsed = urlparse(_clean(source_url))
333
+ host = (parsed.hostname or "").lower()
334
+ if not host.endswith("copernicus.org"):
335
+ host = f"{match.group('journal').lower()}.copernicus.org"
336
+ suffix = doi.split("/", 1)[-1].lower()
337
+ path = f"/articles/{match.group('volume')}/{match.group('page')}/{match.group('year')}/{suffix}.pdf"
338
+ _append_candidate(candidates, f"https://{host}{path}")
339
+
340
+
341
+ def _append_mdpi_candidates(candidates: list[str], doi: str, source_url: str) -> None:
342
+ source_pdf = _mdpi_pdf_url_from_landing_url(source_url)
343
+ if source_pdf:
344
+ _append_candidate(candidates, source_pdf)
345
+ derived_landing = _mdpi_landing_url_from_doi(doi)
346
+ derived_pdf = _mdpi_pdf_url_from_landing_url(derived_landing)
347
+ if derived_pdf:
348
+ _append_candidate(candidates, derived_pdf)
349
+
350
+
351
+ def _mdpi_pdf_url_from_landing_url(url: str | None) -> str:
352
+ candidate = _clean(url)
353
+ if not candidate:
354
+ return ""
355
+ parsed = urlparse(candidate)
356
+ host = (parsed.hostname or "").lower()
357
+ if host not in {"www.mdpi.com", "mdpi.com"}:
358
+ return ""
359
+ path = parsed.path.rstrip("/")
360
+ if not path:
361
+ return ""
362
+ if path.endswith("/pdf") or "/pdf/" in path:
363
+ return urlunparse(parsed._replace(path=path, query=""))
364
+ if not MDPI_NUMERIC_PATH_RE.fullmatch(path):
365
+ return ""
366
+ return urlunparse(parsed._replace(path=f"{path}/pdf", query=""))
367
+
368
+
369
+ def _mdpi_landing_url_from_doi(doi: str) -> str:
370
+ normalized = doi.strip().lower()
371
+ if not normalized.startswith("10.3390/") or "/" not in normalized:
372
+ return ""
373
+ suffix = normalized.split("/", 1)[1]
374
+ for journal_code in sorted(MDPI_JOURNAL_CODE_ISSNS, key=len, reverse=True):
375
+ if not suffix.startswith(journal_code):
376
+ continue
377
+ encoded = suffix[len(journal_code):]
378
+ if len(encoded) < 7 or not encoded.isdigit():
379
+ continue
380
+ volume = encoded[:-6]
381
+ issue = encoded[-6:-4]
382
+ article_number = encoded[-4:]
383
+ if not volume:
384
+ continue
385
+ return (
386
+ f"https://www.mdpi.com/{MDPI_JOURNAL_CODE_ISSNS[journal_code]}/"
387
+ f"{int(volume)}/{int(issue)}/{int(article_number)}"
388
+ )
389
+ return ""
390
+
391
+
392
+ def _append_oxford_candidates(candidates: list[str], doi: str, source_url: str) -> None:
393
+ parsed = urlparse(_clean(source_url))
394
+ host = (parsed.hostname or "").lower()
395
+ source_path = parsed.path.rstrip("/")
396
+ if parsed.scheme not in {"http", "https"} or not host.endswith("academic.oup.com"):
397
+ return
398
+ match = re.match(r"^/(?P<journal>[^/]+)/article/(?P<rest>.+)$", source_path, flags=re.IGNORECASE)
399
+ if not match:
400
+ return
401
+ suffix = doi.rsplit("/", 1)[-1] if "/" in doi else doi
402
+ pdf_path = f"/{match.group('journal')}/article-pdf/{match.group('rest')}/{suffix}.pdf"
403
+ _append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
404
+
405
+
406
+ def _format_template(
407
+ template: str,
408
+ *,
409
+ doi: str,
410
+ source_path: str = "",
411
+ source_path_after_prefix: str = "",
412
+ ) -> str:
413
+ doi_suffix = doi.split("/", 1)[-1] if "/" in doi else doi
414
+ return template.format(
415
+ doi=doi,
416
+ doi_quoted=quote(doi, safe=""),
417
+ doi_suffix=doi_suffix,
418
+ doi_suffix_quoted=quote(doi_suffix, safe=""),
419
+ source_path=source_path,
420
+ source_path_quoted=quote(source_path, safe="/"),
421
+ source_path_after_prefix=source_path_after_prefix,
422
+ source_path_after_prefix_quoted=quote(source_path_after_prefix, safe="/"),
423
+ )
424
+
425
+
426
+ def _append_candidate(candidates: list[str], candidate: str | None, *, source_url: str = "") -> None:
427
+ normalized = _clean(candidate)
428
+ if not normalized:
429
+ return
430
+ if source_url:
431
+ normalized = urljoin(source_url, normalized)
432
+ parsed = urlparse(normalized)
433
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
434
+ return
435
+ if normalized not in candidates:
436
+ candidates.append(normalized)
437
+
438
+
439
+ def _append_query_pdf_candidates(candidates: list[str], candidate: str, *, source_url: str) -> None:
440
+ absolute = urljoin(source_url, _clean(candidate))
441
+ parsed = urlparse(absolute)
442
+ for key, values in parse_qs(parsed.query, keep_blank_values=True).items():
443
+ if _clean(key).lower() not in {"file", "pdf", "src", "url"}:
444
+ continue
445
+ for value in values:
446
+ if any(marker in value.lower() for marker in PDF_URL_MARKERS):
447
+ _append_candidate(candidates, value, source_url=absolute)
448
+
449
+
450
+ def _clean(value: object) -> str:
451
+ return re.sub(r"\s+", " ", str(value or "")).strip()