litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
"""Publisher-owned PDF routing helpers.
|
|
2
|
+
|
|
3
|
+
This module keeps DOI/source-page PDF routes out of individual download
|
|
4
|
+
callers. It follows the same shape as provider catalogs: profile templates
|
|
5
|
+
first, then source URL derivation, then page-discovered links.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import html
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from urllib.parse import parse_qs, quote, urljoin, urlparse, urlunparse
|
|
14
|
+
|
|
15
|
+
from bs4 import BeautifulSoup
|
|
16
|
+
|
|
17
|
+
from .profiles import PublisherProfile
|
|
18
|
+
|
|
19
|
+
PDF_LINK_TEXT_MARKERS = ("pdf", "download pdf", "full text pdf", "view pdf")
|
|
20
|
+
PDF_URL_MARKERS = (".pdf", "/pdf", "/epdf", "/pdfdirect", "/pdfft", "download=true")
|
|
21
|
+
PDF_JS_DEFAULT_URL_RE = re.compile(
|
|
22
|
+
r"PDFViewerApplicationOptions\.set\(\s*['\"]defaultUrl['\"]\s*,\s*['\"]([^'\"]+)['\"]",
|
|
23
|
+
flags=re.IGNORECASE,
|
|
24
|
+
)
|
|
25
|
+
PLOS_JOURNAL_PATHS = {
|
|
26
|
+
"pbio": "plosbiology",
|
|
27
|
+
"pcbi": "ploscompbiol",
|
|
28
|
+
"pclm": "climate",
|
|
29
|
+
"pdig": "digitalhealth",
|
|
30
|
+
"pgen": "plosgenetics",
|
|
31
|
+
"pgph": "globalpublichealth",
|
|
32
|
+
"pmed": "plosmedicine",
|
|
33
|
+
"pntd": "plosntds",
|
|
34
|
+
"pone": "plosone",
|
|
35
|
+
"ppat": "plospathogens",
|
|
36
|
+
"pstr": "sustainabilitytransformation",
|
|
37
|
+
"pwat": "water",
|
|
38
|
+
}
|
|
39
|
+
PLOS_DOI_JOURNAL_RE = re.compile(r"^10\.1371/journal\.([a-z0-9]+)\.", flags=re.IGNORECASE)
|
|
40
|
+
COPERNICUS_DOI_RE = re.compile(
|
|
41
|
+
r"^10\.5194/(?P<journal>[a-z0-9]+)-(?P<volume>\d+)-(?P<page>.+)-(?P<year>\d{4})$",
|
|
42
|
+
flags=re.IGNORECASE,
|
|
43
|
+
)
|
|
44
|
+
MDPI_ISSN_JOURNAL_CODES = {
|
|
45
|
+
"1424-8220": "s",
|
|
46
|
+
"1660-4601": "ijerph",
|
|
47
|
+
"1996-1073": "en",
|
|
48
|
+
"2071-1050": "su",
|
|
49
|
+
"2072-4292": "rs",
|
|
50
|
+
"2073-4441": "w",
|
|
51
|
+
"2077-0375": "membranes",
|
|
52
|
+
"2227-7390": "math",
|
|
53
|
+
"2304-8158": "foods",
|
|
54
|
+
}
|
|
55
|
+
MDPI_JOURNAL_CODE_ISSNS = {
|
|
56
|
+
journal_code: issn
|
|
57
|
+
for issn, journal_code in MDPI_ISSN_JOURNAL_CODES.items()
|
|
58
|
+
}
|
|
59
|
+
MDPI_NUMERIC_PATH_RE = re.compile(r"/[0-9]{4}-[0-9Xx]{3,4}/[0-9]+/[0-9]+/[0-9]+$")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def extract_elsevier_pii(url: str) -> str:
|
|
63
|
+
"""Extract an Elsevier PII from article, retrieve, query, or signed asset URLs."""
|
|
64
|
+
for pattern in (
|
|
65
|
+
r"/pii/([A-Z0-9]+)",
|
|
66
|
+
r"/retrieve/pii/([A-Z0-9]+)",
|
|
67
|
+
r"[?&]pii=([A-Z0-9]+)",
|
|
68
|
+
r"1-s2\.0-([A-Z0-9]+)",
|
|
69
|
+
):
|
|
70
|
+
match = re.search(pattern, url, flags=re.IGNORECASE)
|
|
71
|
+
if match:
|
|
72
|
+
return match.group(1)
|
|
73
|
+
return ""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def extract_ieee_article_number(url: str) -> str:
|
|
77
|
+
"""Extract an IEEE Xplore article number from document or query URLs."""
|
|
78
|
+
for pattern in (
|
|
79
|
+
r"/document/(\d+)",
|
|
80
|
+
r"[?&]arnumber=(\d+)",
|
|
81
|
+
):
|
|
82
|
+
match = re.search(pattern, url, flags=re.IGNORECASE)
|
|
83
|
+
if match:
|
|
84
|
+
return match.group(1)
|
|
85
|
+
return ""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def discover_pdf_candidates_from_html(html_text: str, source_url: str) -> list[str]:
|
|
89
|
+
"""Extract likely PDF URLs from publisher HTML without committing to a provider."""
|
|
90
|
+
candidates: list[str] = []
|
|
91
|
+
soup = BeautifulSoup(html_text, "lxml")
|
|
92
|
+
|
|
93
|
+
for meta in soup.find_all("meta"):
|
|
94
|
+
key = _clean(meta.get("name") or meta.get("property") or meta.get("itemprop")).lower()
|
|
95
|
+
content = _clean(meta.get("content"))
|
|
96
|
+
if content and ("citation_pdf_url" in key or key.endswith("pdf_url") or key == "pdf_url"):
|
|
97
|
+
_append_candidate(candidates, content, source_url=source_url)
|
|
98
|
+
_append_query_pdf_candidates(candidates, content, source_url=source_url)
|
|
99
|
+
|
|
100
|
+
for node in soup.find_all(["a", "link", "iframe", "embed", "object"]):
|
|
101
|
+
target = _clean(node.get("href") or node.get("src") or node.get("data"))
|
|
102
|
+
if not target:
|
|
103
|
+
continue
|
|
104
|
+
label = _clean(
|
|
105
|
+
" ".join(
|
|
106
|
+
filter(
|
|
107
|
+
None,
|
|
108
|
+
[node.get_text(" ", strip=True), node.get("title"), node.get("aria-label")],
|
|
109
|
+
)
|
|
110
|
+
)
|
|
111
|
+
).lower()
|
|
112
|
+
content_type = _clean(node.get("type")).lower()
|
|
113
|
+
lower_target = target.lower()
|
|
114
|
+
if (
|
|
115
|
+
any(marker in lower_target for marker in PDF_URL_MARKERS)
|
|
116
|
+
or any(marker in label for marker in PDF_LINK_TEXT_MARKERS)
|
|
117
|
+
or "pdf" in content_type
|
|
118
|
+
):
|
|
119
|
+
_append_candidate(candidates, target, source_url=source_url)
|
|
120
|
+
_append_query_pdf_candidates(candidates, target, source_url=source_url)
|
|
121
|
+
|
|
122
|
+
for script in soup.find_all("script"):
|
|
123
|
+
text = script.string if script.string is not None else script.get_text(" ", strip=False)
|
|
124
|
+
for match in PDF_JS_DEFAULT_URL_RE.finditer(str(text or "")):
|
|
125
|
+
target = html.unescape(match.group(1))
|
|
126
|
+
if any(marker in target.lower() for marker in PDF_URL_MARKERS):
|
|
127
|
+
_append_candidate(candidates, target, source_url=source_url)
|
|
128
|
+
_append_query_pdf_candidates(candidates, target, source_url=source_url)
|
|
129
|
+
|
|
130
|
+
return candidates
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def build_pdf_candidates(
|
|
134
|
+
profile: PublisherProfile,
|
|
135
|
+
doi: str,
|
|
136
|
+
*,
|
|
137
|
+
source_url: str = "",
|
|
138
|
+
discovered_urls: Sequence[str] = (),
|
|
139
|
+
) -> list[str]:
|
|
140
|
+
"""Build ordered, de-duplicated official PDF candidates for a publisher profile."""
|
|
141
|
+
candidates: list[str] = []
|
|
142
|
+
pii = extract_elsevier_pii(source_url)
|
|
143
|
+
if profile.name.lower() == "elsevier" and pii:
|
|
144
|
+
_append_candidate(candidates, f"https://www.sciencedirect.com/science/article/pii/{pii}/pdfft")
|
|
145
|
+
arnumber = extract_ieee_article_number(source_url)
|
|
146
|
+
if profile.name.lower() == "ieee" and arnumber:
|
|
147
|
+
_append_candidate(
|
|
148
|
+
candidates,
|
|
149
|
+
f"https://ieeexplore.ieee.org/stampPDF/getPDF.jsp?tp=&isnumber=&arnumber={arnumber}",
|
|
150
|
+
)
|
|
151
|
+
_append_candidate(
|
|
152
|
+
candidates,
|
|
153
|
+
f"https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber={arnumber}",
|
|
154
|
+
)
|
|
155
|
+
if profile.name.lower() == "aps":
|
|
156
|
+
_append_aps_candidates(candidates, source_url)
|
|
157
|
+
if profile.name.lower() == "ams":
|
|
158
|
+
_append_ams_candidates(candidates, source_url)
|
|
159
|
+
if profile.name.lower() == "plos":
|
|
160
|
+
_append_plos_candidates(candidates, doi, source_url)
|
|
161
|
+
if profile.name.lower() == "copernicus":
|
|
162
|
+
_append_copernicus_candidates(candidates, doi, source_url)
|
|
163
|
+
if profile.name.lower() == "mdpi":
|
|
164
|
+
_append_mdpi_candidates(candidates, doi, source_url)
|
|
165
|
+
if profile.name.lower() == "oxford academic":
|
|
166
|
+
_append_oxford_candidates(candidates, doi, source_url)
|
|
167
|
+
|
|
168
|
+
_append_source_path_candidates(candidates, profile, doi, source_url)
|
|
169
|
+
|
|
170
|
+
for url in profile.pdf_urls(doi):
|
|
171
|
+
_append_candidate(candidates, url)
|
|
172
|
+
|
|
173
|
+
for url in discovered_urls:
|
|
174
|
+
if not _clean(url):
|
|
175
|
+
continue
|
|
176
|
+
if is_supplementary_url(profile, url):
|
|
177
|
+
continue
|
|
178
|
+
if not is_pdf_candidate_url(profile, url):
|
|
179
|
+
continue
|
|
180
|
+
if not belongs_to_current_article(profile, url, doi=doi, source_url=source_url):
|
|
181
|
+
continue
|
|
182
|
+
_append_candidate(candidates, url, source_url=source_url)
|
|
183
|
+
|
|
184
|
+
return candidates
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def is_pdf_candidate_url(profile: PublisherProfile, url: str) -> bool:
|
|
188
|
+
lower = url.lower()
|
|
189
|
+
return lower.endswith(".pdf") or any(marker.lower() in lower for marker in profile.pdf_url_markers)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def is_supplementary_url(profile: PublisherProfile, url: str) -> bool:
|
|
193
|
+
lower = url.lower()
|
|
194
|
+
return any(marker.lower() in lower for marker in profile.supplementary_url_markers)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def belongs_to_current_article(profile: PublisherProfile, url: str, *, doi: str, source_url: str = "") -> bool:
|
|
198
|
+
if profile.name.lower() == "aps":
|
|
199
|
+
lower = url.lower()
|
|
200
|
+
host = (urlparse(url).hostname or "").lower()
|
|
201
|
+
if host.endswith("journals.aps.org") or host.endswith("link.aps.org"):
|
|
202
|
+
if "/accepted" in lower:
|
|
203
|
+
return False
|
|
204
|
+
doi_lower = doi.lower()
|
|
205
|
+
doi_escaped = doi_lower.replace("/", "%2f")
|
|
206
|
+
if "/pdf/" in lower or "/doi/" in lower:
|
|
207
|
+
return doi_lower in lower or doi_escaped in lower
|
|
208
|
+
return True
|
|
209
|
+
if profile.name.lower() != "elsevier":
|
|
210
|
+
return True
|
|
211
|
+
source_pii = extract_elsevier_pii(source_url)
|
|
212
|
+
candidate_pii = extract_elsevier_pii(url)
|
|
213
|
+
if source_pii and candidate_pii:
|
|
214
|
+
return source_pii.lower() == candidate_pii.lower()
|
|
215
|
+
lower = url.lower()
|
|
216
|
+
doi_lower = doi.lower()
|
|
217
|
+
doi_escaped = doi_lower.replace("/", "%2f")
|
|
218
|
+
if source_pii:
|
|
219
|
+
return source_pii.lower() in lower or doi_lower in lower or doi_escaped in lower
|
|
220
|
+
return True
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _append_source_path_candidates(
|
|
224
|
+
candidates: list[str],
|
|
225
|
+
profile: PublisherProfile,
|
|
226
|
+
doi: str,
|
|
227
|
+
source_url: str,
|
|
228
|
+
) -> None:
|
|
229
|
+
normalized = _clean(source_url)
|
|
230
|
+
if not normalized:
|
|
231
|
+
return
|
|
232
|
+
parsed = urlparse(normalized)
|
|
233
|
+
host = (parsed.hostname or "").lower()
|
|
234
|
+
source_path = parsed.path.rstrip("/")
|
|
235
|
+
if parsed.scheme not in {"http", "https"} or not host or not source_path:
|
|
236
|
+
return
|
|
237
|
+
for template in profile.pdf_source_path_templates:
|
|
238
|
+
domain = template.domain.lower()
|
|
239
|
+
if host != domain and not host.endswith(f".{domain}"):
|
|
240
|
+
continue
|
|
241
|
+
path_prefix = _clean(template.path_prefix)
|
|
242
|
+
if path_prefix and not source_path.startswith(path_prefix):
|
|
243
|
+
continue
|
|
244
|
+
candidate_path = _format_template(
|
|
245
|
+
template.path_template,
|
|
246
|
+
doi=doi,
|
|
247
|
+
source_path=source_path,
|
|
248
|
+
source_path_after_prefix=source_path[len(path_prefix):].lstrip("/") if path_prefix else source_path.lstrip("/"),
|
|
249
|
+
)
|
|
250
|
+
if not candidate_path:
|
|
251
|
+
continue
|
|
252
|
+
parsed_candidate = urlparse(candidate_path)
|
|
253
|
+
if parsed_candidate.scheme in {"http", "https"}:
|
|
254
|
+
_append_candidate(candidates, candidate_path)
|
|
255
|
+
continue
|
|
256
|
+
if not candidate_path.startswith("/"):
|
|
257
|
+
candidate_path = f"/{candidate_path}"
|
|
258
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, candidate_path, "", "", "")))
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _append_aps_candidates(candidates: list[str], source_url: str) -> None:
|
|
262
|
+
normalized = _clean(source_url)
|
|
263
|
+
if not normalized:
|
|
264
|
+
return
|
|
265
|
+
parsed = urlparse(normalized)
|
|
266
|
+
host = (parsed.hostname or "").lower()
|
|
267
|
+
source_path = parsed.path.rstrip("/")
|
|
268
|
+
if parsed.scheme not in {"http", "https"} or not host or not source_path:
|
|
269
|
+
return
|
|
270
|
+
if host.endswith("link.aps.org") and source_path.startswith("/doi/"):
|
|
271
|
+
pdf_path = source_path.replace("/doi/", "/pdf/", 1)
|
|
272
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
|
|
273
|
+
return
|
|
274
|
+
if not host.endswith("journals.aps.org"):
|
|
275
|
+
return
|
|
276
|
+
if "/abstract/" in source_path:
|
|
277
|
+
pdf_path = source_path.replace("/abstract/", "/pdf/", 1)
|
|
278
|
+
elif source_path.endswith("/pdf"):
|
|
279
|
+
pdf_path = source_path
|
|
280
|
+
else:
|
|
281
|
+
return
|
|
282
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _append_ams_candidates(candidates: list[str], source_url: str) -> None:
|
|
286
|
+
normalized = _clean(source_url)
|
|
287
|
+
if not normalized:
|
|
288
|
+
return
|
|
289
|
+
parsed = urlparse(normalized)
|
|
290
|
+
host = (parsed.hostname or "").lower()
|
|
291
|
+
source_path = parsed.path.rstrip("/")
|
|
292
|
+
if parsed.scheme not in {"http", "https"} or not host.endswith("journals.ametsoc.org"):
|
|
293
|
+
return
|
|
294
|
+
if not source_path.startswith("/view/") or not source_path.lower().endswith(".xml"):
|
|
295
|
+
return
|
|
296
|
+
pdf_path = f"/downloadpdf{source_path[:-4]}.pdf"
|
|
297
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _append_plos_candidates(candidates: list[str], doi: str, source_url: str) -> None:
|
|
301
|
+
journal_path = _plos_journal_path_from_source(source_url) or _plos_journal_path_from_doi(doi)
|
|
302
|
+
if not journal_path:
|
|
303
|
+
return
|
|
304
|
+
_append_candidate(
|
|
305
|
+
candidates,
|
|
306
|
+
f"https://journals.plos.org/{journal_path}/article/file?id={doi}&type=printable",
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _plos_journal_path_from_source(source_url: str) -> str:
|
|
311
|
+
parsed = urlparse(_clean(source_url))
|
|
312
|
+
host = (parsed.hostname or "").lower()
|
|
313
|
+
if host != "journals.plos.org":
|
|
314
|
+
return ""
|
|
315
|
+
parts = [part for part in parsed.path.split("/") if part]
|
|
316
|
+
if not parts or parts[0] == "article":
|
|
317
|
+
return ""
|
|
318
|
+
return parts[0]
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _plos_journal_path_from_doi(doi: str) -> str:
|
|
322
|
+
match = PLOS_DOI_JOURNAL_RE.match(doi.strip())
|
|
323
|
+
if not match:
|
|
324
|
+
return ""
|
|
325
|
+
return PLOS_JOURNAL_PATHS.get(match.group(1).lower(), "")
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _append_copernicus_candidates(candidates: list[str], doi: str, source_url: str) -> None:
|
|
329
|
+
match = COPERNICUS_DOI_RE.match(doi.strip().lower())
|
|
330
|
+
if not match:
|
|
331
|
+
return
|
|
332
|
+
parsed = urlparse(_clean(source_url))
|
|
333
|
+
host = (parsed.hostname or "").lower()
|
|
334
|
+
if not host.endswith("copernicus.org"):
|
|
335
|
+
host = f"{match.group('journal').lower()}.copernicus.org"
|
|
336
|
+
suffix = doi.split("/", 1)[-1].lower()
|
|
337
|
+
path = f"/articles/{match.group('volume')}/{match.group('page')}/{match.group('year')}/{suffix}.pdf"
|
|
338
|
+
_append_candidate(candidates, f"https://{host}{path}")
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _append_mdpi_candidates(candidates: list[str], doi: str, source_url: str) -> None:
|
|
342
|
+
source_pdf = _mdpi_pdf_url_from_landing_url(source_url)
|
|
343
|
+
if source_pdf:
|
|
344
|
+
_append_candidate(candidates, source_pdf)
|
|
345
|
+
derived_landing = _mdpi_landing_url_from_doi(doi)
|
|
346
|
+
derived_pdf = _mdpi_pdf_url_from_landing_url(derived_landing)
|
|
347
|
+
if derived_pdf:
|
|
348
|
+
_append_candidate(candidates, derived_pdf)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _mdpi_pdf_url_from_landing_url(url: str | None) -> str:
|
|
352
|
+
candidate = _clean(url)
|
|
353
|
+
if not candidate:
|
|
354
|
+
return ""
|
|
355
|
+
parsed = urlparse(candidate)
|
|
356
|
+
host = (parsed.hostname or "").lower()
|
|
357
|
+
if host not in {"www.mdpi.com", "mdpi.com"}:
|
|
358
|
+
return ""
|
|
359
|
+
path = parsed.path.rstrip("/")
|
|
360
|
+
if not path:
|
|
361
|
+
return ""
|
|
362
|
+
if path.endswith("/pdf") or "/pdf/" in path:
|
|
363
|
+
return urlunparse(parsed._replace(path=path, query=""))
|
|
364
|
+
if not MDPI_NUMERIC_PATH_RE.fullmatch(path):
|
|
365
|
+
return ""
|
|
366
|
+
return urlunparse(parsed._replace(path=f"{path}/pdf", query=""))
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _mdpi_landing_url_from_doi(doi: str) -> str:
|
|
370
|
+
normalized = doi.strip().lower()
|
|
371
|
+
if not normalized.startswith("10.3390/") or "/" not in normalized:
|
|
372
|
+
return ""
|
|
373
|
+
suffix = normalized.split("/", 1)[1]
|
|
374
|
+
for journal_code in sorted(MDPI_JOURNAL_CODE_ISSNS, key=len, reverse=True):
|
|
375
|
+
if not suffix.startswith(journal_code):
|
|
376
|
+
continue
|
|
377
|
+
encoded = suffix[len(journal_code):]
|
|
378
|
+
if len(encoded) < 7 or not encoded.isdigit():
|
|
379
|
+
continue
|
|
380
|
+
volume = encoded[:-6]
|
|
381
|
+
issue = encoded[-6:-4]
|
|
382
|
+
article_number = encoded[-4:]
|
|
383
|
+
if not volume:
|
|
384
|
+
continue
|
|
385
|
+
return (
|
|
386
|
+
f"https://www.mdpi.com/{MDPI_JOURNAL_CODE_ISSNS[journal_code]}/"
|
|
387
|
+
f"{int(volume)}/{int(issue)}/{int(article_number)}"
|
|
388
|
+
)
|
|
389
|
+
return ""
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _append_oxford_candidates(candidates: list[str], doi: str, source_url: str) -> None:
|
|
393
|
+
parsed = urlparse(_clean(source_url))
|
|
394
|
+
host = (parsed.hostname or "").lower()
|
|
395
|
+
source_path = parsed.path.rstrip("/")
|
|
396
|
+
if parsed.scheme not in {"http", "https"} or not host.endswith("academic.oup.com"):
|
|
397
|
+
return
|
|
398
|
+
match = re.match(r"^/(?P<journal>[^/]+)/article/(?P<rest>.+)$", source_path, flags=re.IGNORECASE)
|
|
399
|
+
if not match:
|
|
400
|
+
return
|
|
401
|
+
suffix = doi.rsplit("/", 1)[-1] if "/" in doi else doi
|
|
402
|
+
pdf_path = f"/{match.group('journal')}/article-pdf/{match.group('rest')}/{suffix}.pdf"
|
|
403
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _format_template(
|
|
407
|
+
template: str,
|
|
408
|
+
*,
|
|
409
|
+
doi: str,
|
|
410
|
+
source_path: str = "",
|
|
411
|
+
source_path_after_prefix: str = "",
|
|
412
|
+
) -> str:
|
|
413
|
+
doi_suffix = doi.split("/", 1)[-1] if "/" in doi else doi
|
|
414
|
+
return template.format(
|
|
415
|
+
doi=doi,
|
|
416
|
+
doi_quoted=quote(doi, safe=""),
|
|
417
|
+
doi_suffix=doi_suffix,
|
|
418
|
+
doi_suffix_quoted=quote(doi_suffix, safe=""),
|
|
419
|
+
source_path=source_path,
|
|
420
|
+
source_path_quoted=quote(source_path, safe="/"),
|
|
421
|
+
source_path_after_prefix=source_path_after_prefix,
|
|
422
|
+
source_path_after_prefix_quoted=quote(source_path_after_prefix, safe="/"),
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _append_candidate(candidates: list[str], candidate: str | None, *, source_url: str = "") -> None:
|
|
427
|
+
normalized = _clean(candidate)
|
|
428
|
+
if not normalized:
|
|
429
|
+
return
|
|
430
|
+
if source_url:
|
|
431
|
+
normalized = urljoin(source_url, normalized)
|
|
432
|
+
parsed = urlparse(normalized)
|
|
433
|
+
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
434
|
+
return
|
|
435
|
+
if normalized not in candidates:
|
|
436
|
+
candidates.append(normalized)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _append_query_pdf_candidates(candidates: list[str], candidate: str, *, source_url: str) -> None:
|
|
440
|
+
absolute = urljoin(source_url, _clean(candidate))
|
|
441
|
+
parsed = urlparse(absolute)
|
|
442
|
+
for key, values in parse_qs(parsed.query, keep_blank_values=True).items():
|
|
443
|
+
if _clean(key).lower() not in {"file", "pdf", "src", "url"}:
|
|
444
|
+
continue
|
|
445
|
+
for value in values:
|
|
446
|
+
if any(marker in value.lower() for marker in PDF_URL_MARKERS):
|
|
447
|
+
_append_candidate(candidates, value, source_url=absolute)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _clean(value: object) -> str:
|
|
451
|
+
return re.sub(r"\s+", " ", str(value or "")).strip()
|