litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""One object per paper being downloaded.
|
|
2
|
+
|
|
3
|
+
The engine's state used to live in separate process-wide sets and thread-locals
|
|
4
|
+
(in flight, settled, skipped, the cancel generation, user-wait heartbeats), each
|
|
5
|
+
looked up by DOI and kept in step by hand. A ``DownloadAttempt`` holds what one
|
|
6
|
+
paper's download knows about itself; the registry maps the paper's key (its
|
|
7
|
+
normalized DOI or arXiv ID) to its attempt while the download runs.
|
|
8
|
+
|
|
9
|
+
What it records:
|
|
10
|
+
- ``gen``: the cancel generation the download started under (a cancel bumps it);
|
|
11
|
+
- ``user_wait_at``: the last heartbeat of a source waiting for the user;
|
|
12
|
+
- ``steps``: every source tried — route, label, seconds, outcome, error type and
|
|
13
|
+
reason — so the app can say which way was tried and why it failed;
|
|
14
|
+
- ``last_error``: the latest (error_type, action) a browser strategy of this
|
|
15
|
+
paper reported, for the final failure message and its advice.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import threading
|
|
21
|
+
import time
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class DownloadAttempt:
|
|
28
|
+
key: str
|
|
29
|
+
gen: int
|
|
30
|
+
started: float = field(default_factory=time.perf_counter) # browser_backend's clock
|
|
31
|
+
user_wait_at: float = 0.0
|
|
32
|
+
steps: list[dict[str, Any]] = field(default_factory=list)
|
|
33
|
+
last_error: tuple[str, str] = ("", "") # the most telling of ``errors`` (worst_error)
|
|
34
|
+
errors: list[tuple[str, str]] = field(default_factory=list)
|
|
35
|
+
_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
|
36
|
+
|
|
37
|
+
def add_step(self, route: str, label: str, seconds: float, result: dict[str, Any] | None,
|
|
38
|
+
error: tuple[str, str] = ("", "")) -> None:
|
|
39
|
+
"""``error``: the (error_type, action) the source's browser strategy reported,
|
|
40
|
+
for sources that fail by returning None."""
|
|
41
|
+
from ..desktop.error_text import raw_for
|
|
42
|
+
ok = bool(result and result.get("success"))
|
|
43
|
+
step = {"route": route, "source": label, "seconds": round(seconds, 1), "ok": ok}
|
|
44
|
+
if not ok:
|
|
45
|
+
if result and (result.get("reason") or result.get("error")):
|
|
46
|
+
step["error_type"] = str(result.get("error_type") or "")
|
|
47
|
+
step["reason"] = str(result.get("reason") or result.get("error") or "")[:200]
|
|
48
|
+
elif error[0]:
|
|
49
|
+
step["error_type"] = error[0]
|
|
50
|
+
step["reason"] = raw_for(*error)
|
|
51
|
+
with self._lock:
|
|
52
|
+
self.steps.append(step)
|
|
53
|
+
if not ok and error[0]:
|
|
54
|
+
self.errors.append(error)
|
|
55
|
+
self.last_error = self.worst_error()
|
|
56
|
+
|
|
57
|
+
def worst_error(self) -> tuple[str, str]:
|
|
58
|
+
"""The browser error that explains this paper best (error_text._FAILURE_ORDER),
|
|
59
|
+
not merely the last lane to fail (DeepSeek review 2026-10-19)."""
|
|
60
|
+
from ..desktop.error_text import _FAILURE_ORDER
|
|
61
|
+
rank = {t: i for i, t in enumerate(_FAILURE_ORDER)}
|
|
62
|
+
best = min(self.errors, key=lambda e: rank.get(e[0], len(rank)), default=("", ""))
|
|
63
|
+
return best
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
_LOCK = threading.Lock()
|
|
67
|
+
_ACTIVE: dict[str, DownloadAttempt] = {}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def begin(key: str, gen: int) -> DownloadAttempt:
|
|
71
|
+
with _LOCK:
|
|
72
|
+
att = DownloadAttempt(key=key, gen=gen)
|
|
73
|
+
_ACTIVE[key] = att
|
|
74
|
+
return att
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def end(key: str) -> DownloadAttempt | None:
|
|
78
|
+
with _LOCK:
|
|
79
|
+
return _ACTIVE.pop(key, None)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def active_keys() -> list[str]:
|
|
83
|
+
with _LOCK:
|
|
84
|
+
return list(_ACTIVE)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def get(key: str | None) -> DownloadAttempt | None:
|
|
88
|
+
if not key:
|
|
89
|
+
return None
|
|
90
|
+
with _LOCK:
|
|
91
|
+
return _ACTIVE.get(key)
|
litloop/sources/carsi.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Standalone CARSI download source — decoupled from WebVPN.
|
|
2
|
+
|
|
3
|
+
CARSI can now be used independently: set carsi_enabled=True and carsi_idp_name,
|
|
4
|
+
and it will be tried in the download tier system without requiring instsci_enabled.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from ..log import get_logger
|
|
13
|
+
|
|
14
|
+
log = get_logger()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def try_carsi(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
18
|
+
"""Try downloading via CARSI federated auth without WebVPN dependency.
|
|
19
|
+
|
|
20
|
+
Returns a result dict on success, None on failure.
|
|
21
|
+
"""
|
|
22
|
+
if not config.get("carsi_enabled", False):
|
|
23
|
+
return None
|
|
24
|
+
|
|
25
|
+
idp_name = config.get("carsi_idp_name", "").strip()
|
|
26
|
+
if not idp_name:
|
|
27
|
+
return None
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
from ..institutional.carsi_login import CARSIClient, detect_publisher
|
|
31
|
+
from .instsci import _resolve_doi_url
|
|
32
|
+
|
|
33
|
+
resolved_url = _resolve_doi_url(doi)
|
|
34
|
+
if not resolved_url:
|
|
35
|
+
resolved_url = f"https://doi.org/{doi}"
|
|
36
|
+
|
|
37
|
+
publisher = detect_publisher(resolved_url)
|
|
38
|
+
if not publisher:
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
log.info(f" [CARSI] Trying {publisher} via {idp_name} for {doi}")
|
|
42
|
+
client = CARSIClient(config)
|
|
43
|
+
|
|
44
|
+
# If resolved URL is on a mirror/proxy domain, rebuild using primary domain
|
|
45
|
+
from urllib.parse import urlparse
|
|
46
|
+
cfg = client._publisher_configs.get(publisher)
|
|
47
|
+
# Elsevier DOIs resolve to linkinghub.elsevier.com/retrieve/pii/<PII>; that path does
|
|
48
|
+
# not exist on sciencedirect.com, so map it to the real article page.
|
|
49
|
+
_lh = re.search(r"linkinghub\.elsevier\.com/retrieve/pii/([A-Za-z0-9]+)", resolved_url)
|
|
50
|
+
if _lh:
|
|
51
|
+
resolved_url = f"https://www.sciencedirect.com/science/article/pii/{_lh.group(1)}"
|
|
52
|
+
if cfg:
|
|
53
|
+
resolved_host = urlparse(resolved_url).hostname or ""
|
|
54
|
+
primary_domain = cfg.domains[0]
|
|
55
|
+
# Only a host outside every known domain is a mirror: nature.com is
|
|
56
|
+
# one of Springer's own domains, and moving a Nature article onto
|
|
57
|
+
# link.springer.com gives "Page Unavailable".
|
|
58
|
+
if resolved_host and not any(d in resolved_host for d in cfg.domains):
|
|
59
|
+
# Reconstruct URL using primary domain + same path
|
|
60
|
+
from urllib.parse import urlunparse
|
|
61
|
+
parsed = urlparse(resolved_url)
|
|
62
|
+
resolved_url = urlunparse(parsed._replace(
|
|
63
|
+
scheme="https", netloc=primary_domain))
|
|
64
|
+
log.info(f" [CARSI] Redirected to primary domain: {resolved_url[:80]}")
|
|
65
|
+
|
|
66
|
+
# Try browser download (CloakBrowser first, Selenium fallback)
|
|
67
|
+
result = client.download_via_browser(doi, resolved_url, output_path)
|
|
68
|
+
if result:
|
|
69
|
+
return result
|
|
70
|
+
except ImportError:
|
|
71
|
+
return None
|
|
72
|
+
except Exception as e:
|
|
73
|
+
log.info(f" [CARSI] {e}")
|
|
74
|
+
return None
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""CORE API source for institutional repository papers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import urllib.parse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ..network import fetch_json, polite_delay
|
|
10
|
+
from ..download.pdf import download_pdf, is_plausible_pdf_url, dedupe, iter_urls
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def is_core_pdf_candidate(url: str) -> bool:
|
|
14
|
+
if not is_plausible_pdf_url(url):
|
|
15
|
+
return False
|
|
16
|
+
parsed = urllib.parse.urlparse(url)
|
|
17
|
+
host = (parsed.hostname or "").lower()
|
|
18
|
+
path = parsed.path.lower()
|
|
19
|
+
if "core.ac.uk" in host and "/data-providers/" in path:
|
|
20
|
+
return False
|
|
21
|
+
return True
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def extract_core_pdf_candidates(payload: dict[str, Any]) -> list[str]:
|
|
25
|
+
return dedupe(url for url in iter_urls(payload) if is_core_pdf_candidate(url))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def try_core(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
29
|
+
q = urllib.parse.quote(f'doi:"{doi}"')
|
|
30
|
+
url = f"https://api.core.ac.uk/v3/search/works?q={q}&limit=5"
|
|
31
|
+
headers = {}
|
|
32
|
+
if config.get("core_api_key"):
|
|
33
|
+
headers["Authorization"] = f"Bearer {config['core_api_key']}"
|
|
34
|
+
payload = fetch_json(url, config, headers=headers)
|
|
35
|
+
if not payload:
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
candidates = extract_core_pdf_candidates(payload)
|
|
39
|
+
for pdf_url in candidates[: int(config.get("max_core_candidates", 3))]:
|
|
40
|
+
polite_delay(config)
|
|
41
|
+
result = download_pdf(pdf_url, output_path, config, "CORE")
|
|
42
|
+
if result:
|
|
43
|
+
result["doi"] = doi
|
|
44
|
+
result["identifier"] = doi
|
|
45
|
+
return result
|
|
46
|
+
return None
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Crossref API source for publisher PDF links + page scraping."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import urllib.parse
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import requests
|
|
11
|
+
|
|
12
|
+
from ..network import fetch_json, polite_delay, USER_AGENT, request_timeout, proxy_dict, select_proxy_for_url
|
|
13
|
+
from ..download.pdf import download_pdf, is_pdf_file, success, _response_looks_pdf, is_plausible_pdf_url
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def try_crossref(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
17
|
+
"""Try Crossref API link field for direct PDF."""
|
|
18
|
+
q = urllib.parse.quote(doi, safe="")
|
|
19
|
+
url = f"https://api.crossref.org/works/{q}"
|
|
20
|
+
try:
|
|
21
|
+
payload = fetch_json(url, config, headers={"Accept": "application/json"})
|
|
22
|
+
if not payload:
|
|
23
|
+
return None
|
|
24
|
+
message = payload.get("message", {})
|
|
25
|
+
links = message.get("link", [])
|
|
26
|
+
if not isinstance(links, list):
|
|
27
|
+
links = [links]
|
|
28
|
+
for link in links:
|
|
29
|
+
if not isinstance(link, dict):
|
|
30
|
+
continue
|
|
31
|
+
content_type = link.get("content-type", "")
|
|
32
|
+
link_url = link.get("URL", "")
|
|
33
|
+
if "application/pdf" in content_type and link_url:
|
|
34
|
+
polite_delay(config)
|
|
35
|
+
result = download_pdf(link_url, output_path, config, "Crossref")
|
|
36
|
+
if result:
|
|
37
|
+
result["doi"] = doi
|
|
38
|
+
result["identifier"] = doi
|
|
39
|
+
return result
|
|
40
|
+
except Exception:
|
|
41
|
+
pass
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def try_crossref_page_scrape(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
46
|
+
"""Resolve publisher landing page and scrape for PDF links.
|
|
47
|
+
|
|
48
|
+
Inspired by PyPaperRetriever's approach:
|
|
49
|
+
1. Resolve DOI → publisher page via doi.org redirect
|
|
50
|
+
2. Scan <a> tags for .pdf hrefs
|
|
51
|
+
3. Regex for JS-based redirects (window.open, href=, location=)
|
|
52
|
+
4. Check data-pdf-url attributes
|
|
53
|
+
"""
|
|
54
|
+
try:
|
|
55
|
+
# Step 1: Resolve DOI to publisher URL
|
|
56
|
+
polite_delay(config)
|
|
57
|
+
s = requests.Session()
|
|
58
|
+
s.trust_env = False
|
|
59
|
+
s.headers.update({"User-Agent": USER_AGENT})
|
|
60
|
+
from ..network import apply_host_proxy
|
|
61
|
+
apply_host_proxy(s, config)
|
|
62
|
+
doi_resp = s.head(f"https://doi.org/{doi}", allow_redirects=True,
|
|
63
|
+
timeout=10)
|
|
64
|
+
if doi_resp.status_code >= 400:
|
|
65
|
+
return None
|
|
66
|
+
publisher_url = doi_resp.url
|
|
67
|
+
|
|
68
|
+
# Step 2: Fetch publisher page
|
|
69
|
+
polite_delay(config)
|
|
70
|
+
page_resp = s.get(publisher_url, timeout=15, allow_redirects=True)
|
|
71
|
+
if page_resp.status_code >= 400:
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
html = page_resp.text
|
|
75
|
+
base_url = page_resp.url
|
|
76
|
+
|
|
77
|
+
# Step 3: Extract PDF URLs using multiple strategies
|
|
78
|
+
pdf_candidates = _extract_pdf_urls(html, base_url)
|
|
79
|
+
|
|
80
|
+
# Step 4: Try each candidate
|
|
81
|
+
for pdf_url in pdf_candidates:
|
|
82
|
+
try:
|
|
83
|
+
polite_delay(config)
|
|
84
|
+
result = download_pdf(pdf_url, output_path, config, "CrossrefPage",
|
|
85
|
+
require_pdf_like_url=False)
|
|
86
|
+
if result:
|
|
87
|
+
result["doi"] = doi
|
|
88
|
+
result["identifier"] = doi
|
|
89
|
+
return result
|
|
90
|
+
except Exception:
|
|
91
|
+
continue
|
|
92
|
+
except Exception:
|
|
93
|
+
pass
|
|
94
|
+
return None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _extract_pdf_urls(html: str, base_url: str) -> list[str]:
|
|
98
|
+
"""Extract candidate PDF URLs from publisher page HTML."""
|
|
99
|
+
candidates: list[str] = []
|
|
100
|
+
|
|
101
|
+
# Strategy 1: <a> tags with href ending in .pdf
|
|
102
|
+
for match in re.finditer(r'href=["\']([^"\']*?\.pdf[^"\']*?)["\']', html, re.IGNORECASE):
|
|
103
|
+
url = _resolve_url(match.group(1), base_url)
|
|
104
|
+
if url:
|
|
105
|
+
candidates.append(url)
|
|
106
|
+
|
|
107
|
+
# Strategy 2: <a> tags with /pdf in path
|
|
108
|
+
for match in re.finditer(r'href=["\']([^"\']*?/pdf[^"\']*?)["\']', html, re.IGNORECASE):
|
|
109
|
+
url = _resolve_url(match.group(1), base_url)
|
|
110
|
+
if url and url not in candidates:
|
|
111
|
+
candidates.append(url)
|
|
112
|
+
|
|
113
|
+
# Strategy 3: /article/file patterns (PLOS, Frontiers, etc.)
|
|
114
|
+
for match in re.finditer(r'href=["\']([^"\']*?/article/file[^"\']*?)["\']', html, re.IGNORECASE):
|
|
115
|
+
url = _resolve_url(match.group(1), base_url)
|
|
116
|
+
if url and url not in candidates:
|
|
117
|
+
candidates.append(url)
|
|
118
|
+
|
|
119
|
+
# Strategy 4: type=printable or format=pdf in query
|
|
120
|
+
for match in re.finditer(r'href=["\']([^"\']*?(?:type=printable|format=pdf)[^"\']*?)["\']', html, re.IGNORECASE):
|
|
121
|
+
url = _resolve_url(match.group(1), base_url)
|
|
122
|
+
if url and url not in candidates:
|
|
123
|
+
candidates.append(url)
|
|
124
|
+
|
|
125
|
+
# Strategy 5: data-pdf-url attribute
|
|
126
|
+
for match in re.finditer(r'data-pdf[^=]*=["\']([^"\']+?)["\']', html, re.IGNORECASE):
|
|
127
|
+
url = _resolve_url(match.group(1), base_url)
|
|
128
|
+
if url and url not in candidates:
|
|
129
|
+
candidates.append(url)
|
|
130
|
+
|
|
131
|
+
# Strategy 6: JS-based redirects (window.open, location=, href=)
|
|
132
|
+
js_patterns = [
|
|
133
|
+
r'window\.open\(["\']([^"\']*?\.pdf[^"\']*?)["\']',
|
|
134
|
+
r'(?:location|href)\s*=\s*["\']([^"\']*?\.pdf[^"\']*?)["\']',
|
|
135
|
+
]
|
|
136
|
+
for pattern in js_patterns:
|
|
137
|
+
for match in re.finditer(pattern, html, re.IGNORECASE):
|
|
138
|
+
url = _resolve_url(match.group(1), base_url)
|
|
139
|
+
if url and url not in candidates and is_plausible_pdf_url(url):
|
|
140
|
+
candidates.append(url)
|
|
141
|
+
|
|
142
|
+
# Strategy 7: <embed> or <iframe> with PDF src
|
|
143
|
+
for match in re.finditer(r'<(?:embed|iframe)[^>]*src=["\']([^"\']*?\.pdf[^"\']*?)["\']', html, re.IGNORECASE):
|
|
144
|
+
url = _resolve_url(match.group(1), base_url)
|
|
145
|
+
if url and url not in candidates:
|
|
146
|
+
candidates.append(url)
|
|
147
|
+
|
|
148
|
+
return candidates
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _resolve_url(url: str, base_url: str) -> str | None:
|
|
152
|
+
"""Resolve relative URL to absolute, validate scheme."""
|
|
153
|
+
if not url:
|
|
154
|
+
return None
|
|
155
|
+
try:
|
|
156
|
+
resolved = urllib.parse.urljoin(base_url, url)
|
|
157
|
+
parsed = urllib.parse.urlparse(resolved)
|
|
158
|
+
if parsed.scheme not in ("http", "https"):
|
|
159
|
+
return None
|
|
160
|
+
return resolved
|
|
161
|
+
except Exception:
|
|
162
|
+
return None
|