litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,569 @@
|
|
|
1
|
+
"""CARSI (Shibboleth/SAML) federated authentication for publisher access.
|
|
2
|
+
|
|
3
|
+
Provides institutional login through CARSI federation, supporting
|
|
4
|
+
publishers like Elsevier, Springer Nature, Wiley, ACS, etc.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import threading
|
|
13
|
+
import time
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
from urllib.parse import urlparse
|
|
18
|
+
|
|
19
|
+
import requests
|
|
20
|
+
|
|
21
|
+
from ..browser.pdf import capture_pdf_via_cdp, find_page_pdf_link
|
|
22
|
+
from ..config import DATA_DIR
|
|
23
|
+
from ..log import get_logger
|
|
24
|
+
from ..publishers.strategies import (
|
|
25
|
+
_IDP_MAP,
|
|
26
|
+
_AUTH_KEYWORDS,
|
|
27
|
+
_AUTH_TITLES,
|
|
28
|
+
_INSTITUTION_SEARCH_SELECTORS,
|
|
29
|
+
_SSO_LINK_FINDER_JS,
|
|
30
|
+
_INSTITUTION_CLICK_JS,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
log = get_logger()
|
|
34
|
+
|
|
35
|
+
_PUBLISHER_CONFIGS_FILE = DATA_DIR / "publisher_carsi.json"
|
|
36
|
+
_PKG_DATA_DIR = Path(__file__).resolve().parent.parent / "data"
|
|
37
|
+
_PKG_PUBLISHER_CONFIGS_FILE = _PKG_DATA_DIR / "publisher_carsi.json"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class PublisherCARSIConfig:
|
|
42
|
+
name: str
|
|
43
|
+
domains: list[str]
|
|
44
|
+
login_url: str
|
|
45
|
+
search_selector: str
|
|
46
|
+
result_selector: str
|
|
47
|
+
success_url_pattern: str
|
|
48
|
+
pdf_pattern: str
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _load_publisher_configs() -> dict[str, PublisherCARSIConfig]:
|
|
52
|
+
# Try package data first, then user data dir
|
|
53
|
+
config_file = _PKG_PUBLISHER_CONFIGS_FILE if _PKG_PUBLISHER_CONFIGS_FILE.exists() else _PUBLISHER_CONFIGS_FILE
|
|
54
|
+
if not config_file.exists():
|
|
55
|
+
return {}
|
|
56
|
+
data = json.loads(config_file.read_text(encoding="utf-8"))
|
|
57
|
+
configs = {}
|
|
58
|
+
for key, val in data.items():
|
|
59
|
+
configs[key] = PublisherCARSIConfig(**val)
|
|
60
|
+
return configs
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def detect_publisher(url: str) -> str | None:
|
|
64
|
+
"""Detect publisher key from a URL."""
|
|
65
|
+
hostname = urlparse(url).hostname or ""
|
|
66
|
+
configs = _load_publisher_configs()
|
|
67
|
+
for key, cfg in configs.items():
|
|
68
|
+
for domain in cfg.domains:
|
|
69
|
+
if domain in hostname:
|
|
70
|
+
return key
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class CARSIClient:
|
|
75
|
+
"""Manages CARSI/Shibboleth federated authentication with academic publishers."""
|
|
76
|
+
|
|
77
|
+
_login_lock = threading.Lock()
|
|
78
|
+
|
|
79
|
+
def __init__(self, config: dict[str, Any]):
|
|
80
|
+
self.config = config
|
|
81
|
+
self._sessions: dict[str, requests.Session] = {}
|
|
82
|
+
self._publisher_configs = _load_publisher_configs()
|
|
83
|
+
self._cookie_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache"))) / "carsi_cookies"
|
|
84
|
+
self._cookie_dir.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
|
|
86
|
+
def _cookie_path(self, publisher: str) -> Path:
|
|
87
|
+
return self._cookie_dir / f"{publisher}.json"
|
|
88
|
+
|
|
89
|
+
def _get_session(self, publisher: str) -> requests.Session:
|
|
90
|
+
if publisher not in self._sessions:
|
|
91
|
+
sess = requests.Session()
|
|
92
|
+
sess.trust_env = False
|
|
93
|
+
sess.headers.update({
|
|
94
|
+
"User-Agent": (
|
|
95
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
96
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
97
|
+
"Chrome/120.0.0.0 Safari/537.36"
|
|
98
|
+
)
|
|
99
|
+
})
|
|
100
|
+
self._sessions[publisher] = sess
|
|
101
|
+
return self._sessions[publisher]
|
|
102
|
+
|
|
103
|
+
def login(self, publisher: str, force: bool = False) -> bool:
|
|
104
|
+
"""Ensure we have a valid CARSI session for the given publisher."""
|
|
105
|
+
with self._login_lock:
|
|
106
|
+
if not force and self._try_load_cookies(publisher):
|
|
107
|
+
log.info(f" [CARSI] Loaded saved cookies for {publisher}")
|
|
108
|
+
return True
|
|
109
|
+
log.info(f" [CARSI] No valid session for {publisher}. Opening browser...")
|
|
110
|
+
return self._browser_login(publisher)
|
|
111
|
+
|
|
112
|
+
def fetch(self, url: str, **kwargs) -> requests.Response | None:
|
|
113
|
+
"""Fetch a URL using CARSI-authenticated session."""
|
|
114
|
+
publisher = detect_publisher(url)
|
|
115
|
+
if not publisher:
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
if not self.login(publisher):
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
sess = self._get_session(publisher)
|
|
122
|
+
kwargs.setdefault("timeout", 30)
|
|
123
|
+
kwargs.setdefault("allow_redirects", True)
|
|
124
|
+
try:
|
|
125
|
+
return sess.get(url, **kwargs)
|
|
126
|
+
except requests.RequestException as e:
|
|
127
|
+
log.warning(f" [CARSI] Fetch failed: {e}")
|
|
128
|
+
return None
|
|
129
|
+
|
|
130
|
+
def download_via_browser(self, doi: str, article_url: str, output_path: Path) -> dict[str, Any] | None:
|
|
131
|
+
"""Download PDF via CloakBrowser with CARSI auth."""
|
|
132
|
+
return self._download_via_cloakbrowser(doi, article_url, output_path)
|
|
133
|
+
|
|
134
|
+
def _download_via_cloakbrowser(self, doi: str, article_url: str, output_path: Path) -> dict[str, Any] | None:
|
|
135
|
+
"""Download PDF via CloakBrowser with CARSI auth. Single session: login + download."""
|
|
136
|
+
publisher = detect_publisher(article_url)
|
|
137
|
+
if not publisher:
|
|
138
|
+
return None
|
|
139
|
+
cfg = self._publisher_configs.get(publisher)
|
|
140
|
+
if not cfg:
|
|
141
|
+
return None
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
from ..browser.backend import launch # noqa: F401
|
|
145
|
+
from ..browser.backend import is_available as _browser_backend_available
|
|
146
|
+
if not _browser_backend_available():
|
|
147
|
+
log.info(" [CARSI-Browser] no browser backend available")
|
|
148
|
+
return None
|
|
149
|
+
except ImportError:
|
|
150
|
+
log.info(" [CARSI-Browser] no browser backend available")
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
idp_name = self.config.get("carsi_idp_name", "")
|
|
154
|
+
if not idp_name:
|
|
155
|
+
log.info(" [CARSI-Browser] No carsi_idp_name configured")
|
|
156
|
+
return None
|
|
157
|
+
|
|
158
|
+
idp_en = _IDP_MAP.get(idp_name, idp_name)
|
|
159
|
+
|
|
160
|
+
from ..download.pdf import is_pdf_file, success as _success
|
|
161
|
+
|
|
162
|
+
# Serialize browser opens across threads — only one browser at a time
|
|
163
|
+
with self._login_lock:
|
|
164
|
+
log.info(f" [CARSI-Browser] Opening browser for {publisher}...")
|
|
165
|
+
try:
|
|
166
|
+
from ..publishers.strategies import _visible_browser, _save_all_cookie_formats
|
|
167
|
+
|
|
168
|
+
with _visible_browser(self.config, publisher, viewport=None) as (context, page):
|
|
169
|
+
def _try_save_captured() -> dict[str, Any] | None:
|
|
170
|
+
"""If a PDF was captured, save and validate it."""
|
|
171
|
+
if captured_pdf:
|
|
172
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
173
|
+
output_path.write_bytes(captured_pdf[-1])
|
|
174
|
+
if is_pdf_file(output_path):
|
|
175
|
+
return _success(doi, output_path, "CARSI-Browser")
|
|
176
|
+
return None
|
|
177
|
+
|
|
178
|
+
def _try_fetch_pdf(url: str) -> dict[str, Any] | None:
|
|
179
|
+
"""Open a PDF URL in the page and save the PDF the browser receives."""
|
|
180
|
+
body = capture_pdf_via_cdp(context, page, url)
|
|
181
|
+
if body:
|
|
182
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
183
|
+
output_path.write_bytes(body)
|
|
184
|
+
if is_pdf_file(output_path):
|
|
185
|
+
return _success(doi, output_path, "CARSI-Browser")
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
# Restore saved cookies if any (supplements persistent profile)
|
|
189
|
+
cookie_file = self._cookie_path(publisher)
|
|
190
|
+
if cookie_file.exists():
|
|
191
|
+
try:
|
|
192
|
+
saved = json.loads(cookie_file.read_text(encoding="utf-8"))
|
|
193
|
+
pw_cookies = []
|
|
194
|
+
for c in saved:
|
|
195
|
+
pw_c = {"name": c["name"], "value": c["value"], "domain": c.get("domain", ""), "path": c.get("path", "/")}
|
|
196
|
+
if pw_c["domain"]:
|
|
197
|
+
pw_cookies.append(pw_c)
|
|
198
|
+
if pw_cookies:
|
|
199
|
+
context.add_cookies(pw_cookies)
|
|
200
|
+
log.info(f" [CARSI-Browser] Restored {len(pw_cookies)} cookies from file")
|
|
201
|
+
except Exception:
|
|
202
|
+
pass
|
|
203
|
+
|
|
204
|
+
# Capture PDF from network
|
|
205
|
+
captured_pdf = []
|
|
206
|
+
def on_response(response):
|
|
207
|
+
try:
|
|
208
|
+
ct = response.headers.get("content-type", "")
|
|
209
|
+
url = response.url
|
|
210
|
+
is_pdf_ct = "pdf" in ct.lower() or "octet-stream" in ct.lower()
|
|
211
|
+
is_pdf_url = url.lower().endswith(".pdf") or "/pdfdirect/" in url or "/doi/pdf/" in url
|
|
212
|
+
if not (is_pdf_ct or is_pdf_url):
|
|
213
|
+
return
|
|
214
|
+
if response.status >= 400:
|
|
215
|
+
return
|
|
216
|
+
body = response.body()
|
|
217
|
+
if len(body) > 5000 and body[:4] == b"%PDF-":
|
|
218
|
+
captured_pdf.append(body)
|
|
219
|
+
log.info(f" [CARSI-Browser] PDF captured: {len(body)} bytes")
|
|
220
|
+
except Exception:
|
|
221
|
+
pass
|
|
222
|
+
page.on("response", on_response)
|
|
223
|
+
|
|
224
|
+
# Step 1: Navigate to article page first (gets Cloudflare clearance)
|
|
225
|
+
log.info(f" [CARSI-Browser] Loading article: {article_url[:60]}")
|
|
226
|
+
try:
|
|
227
|
+
page.goto(article_url, wait_until="domcontentloaded", timeout=60000)
|
|
228
|
+
time.sleep(5)
|
|
229
|
+
except Exception:
|
|
230
|
+
pass
|
|
231
|
+
|
|
232
|
+
title = page.title()
|
|
233
|
+
url = page.url
|
|
234
|
+
log.info(f" [CARSI-Browser] Page: '{title[:40]}' {url[:60]}")
|
|
235
|
+
if url.startswith("chrome-error://"):
|
|
236
|
+
# The article did not load (network): that says nothing
|
|
237
|
+
# about access, so no login is asked for. One more try.
|
|
238
|
+
try:
|
|
239
|
+
page.goto(article_url, wait_until="domcontentloaded", timeout=60000)
|
|
240
|
+
time.sleep(3)
|
|
241
|
+
except Exception:
|
|
242
|
+
pass
|
|
243
|
+
if page.url.startswith("chrome-error://"):
|
|
244
|
+
log.info(" [CARSI-Browser] the article page did not load (network); not asking for a login")
|
|
245
|
+
return None
|
|
246
|
+
|
|
247
|
+
# Wait for Cloudflare challenge to resolve (visible stealth browser can pass it)
|
|
248
|
+
from ..network import is_cloudflare_challenge
|
|
249
|
+
from ..sources import note_user_waiting
|
|
250
|
+
for _cf_wait in range(60): # up to 5 min: the user may have to tick it
|
|
251
|
+
if is_cloudflare_challenge(page.title() or ""):
|
|
252
|
+
if _cf_wait % 6 == 0:
|
|
253
|
+
log.info(f" [CARSI-Browser] Cloudflare challenge detected, waiting for it or the user... ({_cf_wait * 5}s)")
|
|
254
|
+
note_user_waiting("verify")
|
|
255
|
+
try:
|
|
256
|
+
from ..publishers.strategies._core import _is_challenge_dead_end
|
|
257
|
+
if _is_challenge_dead_end(page.content()):
|
|
258
|
+
log.info(" [CARSI-Browser] the check cannot run in this browser; giving up")
|
|
259
|
+
break
|
|
260
|
+
except Exception:
|
|
261
|
+
pass
|
|
262
|
+
time.sleep(5)
|
|
263
|
+
else:
|
|
264
|
+
break
|
|
265
|
+
else:
|
|
266
|
+
log.info(" [CARSI-Browser] Cloudflare challenge did not resolve")
|
|
267
|
+
|
|
268
|
+
# Step 1b: Check if restored cookies already grant access
|
|
269
|
+
needs_login = False
|
|
270
|
+
try:
|
|
271
|
+
needs_login = page.evaluate("""
|
|
272
|
+
() => {
|
|
273
|
+
if (!document.body) return true;
|
|
274
|
+
const body = (document.body.innerText || '').toLowerCase();
|
|
275
|
+
const hasPaywall = body.includes('purchase') || body.includes('subscribe')
|
|
276
|
+
|| body.includes('access through your institution')
|
|
277
|
+
|| body.includes('sign in to access')
|
|
278
|
+
|| body.includes('buy this article');
|
|
279
|
+
const hasPdf = !!document.querySelector('a[href*="pdf"], a[href*="download"], iframe[src*="pdf"]');
|
|
280
|
+
return hasPaywall && !hasPdf;
|
|
281
|
+
}
|
|
282
|
+
""")
|
|
283
|
+
except Exception as _e:
|
|
284
|
+
log.info(f" [CARSI-Browser] paywall check error (likely Cloudflare): {_e}")
|
|
285
|
+
needs_login = True
|
|
286
|
+
|
|
287
|
+
# Decide whether access is already granted from the page itself.
|
|
288
|
+
# (A fetch() of /pdfft cannot prove it: ScienceDirect's PDF link needs
|
|
289
|
+
# page-issued md5/pid params and redirects cross-origin, so such a
|
|
290
|
+
# probe fails even with valid entitlement.)
|
|
291
|
+
cookies_valid = False
|
|
292
|
+
if not needs_login:
|
|
293
|
+
try:
|
|
294
|
+
cookies_valid = bool(page.evaluate("""
|
|
295
|
+
() => {
|
|
296
|
+
if (!document.body) return false;
|
|
297
|
+
const body = (document.body.innerText || '').toLowerCase();
|
|
298
|
+
if (body.includes('get access') || body.includes('purchase pdf')
|
|
299
|
+
|| body.includes('access through your institution')) return false;
|
|
300
|
+
for (const a of document.querySelectorAll('a')) {
|
|
301
|
+
const href = (a.getAttribute('href') || '').toLowerCase();
|
|
302
|
+
const text = (a.innerText || '').toLowerCase();
|
|
303
|
+
if (href.includes('pdfft') || href.includes('/doi/pdf')
|
|
304
|
+
|| href.includes('/pdfdirect/') || text.includes('view pdf')
|
|
305
|
+
|| text.includes('download pdf')) return true;
|
|
306
|
+
}
|
|
307
|
+
return false;
|
|
308
|
+
}
|
|
309
|
+
"""))
|
|
310
|
+
except Exception as _e:
|
|
311
|
+
log.info(f" [CARSI-Browser] access check error: {_e}")
|
|
312
|
+
if cookies_valid:
|
|
313
|
+
log.info(" [CARSI-Browser] Full-text access detected, skipping login")
|
|
314
|
+
else:
|
|
315
|
+
log.info(" [CARSI-Browser] No full-text access on page, login needed")
|
|
316
|
+
|
|
317
|
+
if not cookies_valid:
|
|
318
|
+
# Step 2: Navigate to "Institutional login" link on article page
|
|
319
|
+
sso_href = page.evaluate(_SSO_LINK_FINDER_JS)
|
|
320
|
+
if sso_href:
|
|
321
|
+
log.info(f" [CARSI-Browser] Navigating to SSO: {sso_href[:80]}")
|
|
322
|
+
try:
|
|
323
|
+
page.goto(sso_href, wait_until="domcontentloaded", timeout=30000)
|
|
324
|
+
except Exception:
|
|
325
|
+
pass
|
|
326
|
+
else:
|
|
327
|
+
log.info(" [CARSI-Browser] No SSO link found, trying direct login URL...")
|
|
328
|
+
try:
|
|
329
|
+
page.goto(cfg.login_url, wait_until="domcontentloaded", timeout=30000)
|
|
330
|
+
except Exception:
|
|
331
|
+
pass
|
|
332
|
+
|
|
333
|
+
time.sleep(8)
|
|
334
|
+
|
|
335
|
+
# Step 3: Search for institution in the WAYF page
|
|
336
|
+
search_input = page.query_selector('#searchInstitution')
|
|
337
|
+
if not search_input:
|
|
338
|
+
for sel in _INSTITUTION_SEARCH_SELECTORS[1:]: # skip #searchInstitution (already tried)
|
|
339
|
+
search_input = page.query_selector(sel)
|
|
340
|
+
if search_input:
|
|
341
|
+
break
|
|
342
|
+
|
|
343
|
+
if search_input:
|
|
344
|
+
search_input.fill(idp_en)
|
|
345
|
+
time.sleep(3)
|
|
346
|
+
log.info(f" [CARSI-Browser] Searched for '{idp_en}'")
|
|
347
|
+
|
|
348
|
+
# Click matching institution
|
|
349
|
+
clicked = page.evaluate(_INSTITUTION_CLICK_JS, idp_en)
|
|
350
|
+
if clicked:
|
|
351
|
+
log.info(f" [CARSI-Browser] Selected: {clicked}")
|
|
352
|
+
time.sleep(5)
|
|
353
|
+
else:
|
|
354
|
+
search_input.press("Enter")
|
|
355
|
+
time.sleep(3)
|
|
356
|
+
else:
|
|
357
|
+
log.info(" [CARSI-Browser] No institution search box found")
|
|
358
|
+
|
|
359
|
+
# Step 4: Wait for CAS login
|
|
360
|
+
_ak = _AUTH_KEYWORDS
|
|
361
|
+
_at = _AUTH_TITLES
|
|
362
|
+
|
|
363
|
+
url = page.url
|
|
364
|
+
title = page.title()
|
|
365
|
+
if any(x in url.lower() for x in _ak) or any(x in title for x in _at):
|
|
366
|
+
log.info(" [CARSI-Browser] CAS login required. Please log in...")
|
|
367
|
+
from ..sources import note_user_waiting
|
|
368
|
+
for i in range(100):
|
|
369
|
+
note_user_waiting("login") # keep the race from closing this window mid-login
|
|
370
|
+
time.sleep(3)
|
|
371
|
+
try:
|
|
372
|
+
title = page.title()
|
|
373
|
+
url = page.url
|
|
374
|
+
except Exception:
|
|
375
|
+
return None
|
|
376
|
+
is_auth = any(x in title for x in _at)
|
|
377
|
+
is_auth_url = any(x in url.lower() for x in _ak)
|
|
378
|
+
if not is_auth and not is_auth_url:
|
|
379
|
+
# Login success - save cookies in all formats + bridge to CloakBrowser
|
|
380
|
+
try:
|
|
381
|
+
cookies = context.cookies()
|
|
382
|
+
_save_all_cookie_formats(cookies, publisher, self.config)
|
|
383
|
+
except Exception:
|
|
384
|
+
pass
|
|
385
|
+
break
|
|
386
|
+
else:
|
|
387
|
+
log.info(" [CARSI-Browser] Login timed out")
|
|
388
|
+
return None
|
|
389
|
+
else:
|
|
390
|
+
log.info(" [CARSI-Browser] Already authenticated")
|
|
391
|
+
|
|
392
|
+
# Step 5: Navigate to article (with CARSI auth now)
|
|
393
|
+
time.sleep(2)
|
|
394
|
+
log.info(f" [CARSI-Browser] Navigating to article: {article_url[:60]}")
|
|
395
|
+
try:
|
|
396
|
+
page.goto(article_url, wait_until="domcontentloaded", timeout=30000)
|
|
397
|
+
time.sleep(5)
|
|
398
|
+
except Exception:
|
|
399
|
+
pass
|
|
400
|
+
|
|
401
|
+
# Check for PDF via network capture
|
|
402
|
+
saved = _try_save_captured()
|
|
403
|
+
if saved:
|
|
404
|
+
return saved
|
|
405
|
+
|
|
406
|
+
# Step 5b: Fetch the page's own PDF link (carries page-issued tokens,
|
|
407
|
+
# e.g. ScienceDirect's /pdfft?md5=...&pid=...)
|
|
408
|
+
page_pdf_href = find_page_pdf_link(page, doi=doi)
|
|
409
|
+
if page_pdf_href:
|
|
410
|
+
log.info(f" [CARSI-Browser] Page PDF link: {page_pdf_href[:80]}")
|
|
411
|
+
saved = _try_fetch_pdf(page_pdf_href)
|
|
412
|
+
if saved:
|
|
413
|
+
return saved
|
|
414
|
+
|
|
415
|
+
# Step 6: Try direct PDF URL
|
|
416
|
+
pii_match = re.search(r"pii/([A-Z0-9]+)", page.url)
|
|
417
|
+
pii_value = pii_match.group(1) if pii_match else ""
|
|
418
|
+
art_match = re.search(r"/document/(\d+)", page.url)
|
|
419
|
+
article_id = art_match.group(1) if art_match else ""
|
|
420
|
+
doi_suffix = doi.split("/", 1)[1] if "/" in doi else ""
|
|
421
|
+
pdf_pattern = (
|
|
422
|
+
cfg.pdf_pattern.replace("{doi}", doi)
|
|
423
|
+
.replace("{pii}", pii_value)
|
|
424
|
+
.replace("{article_id}", article_id)
|
|
425
|
+
.replace("{doi_suffix}", doi_suffix)
|
|
426
|
+
)
|
|
427
|
+
if pdf_pattern and not pdf_pattern.startswith("http"):
|
|
428
|
+
pdf_url = f"https://{cfg.domains[0]}{pdf_pattern}"
|
|
429
|
+
else:
|
|
430
|
+
pdf_url = pdf_pattern
|
|
431
|
+
|
|
432
|
+
# A leftover "{...}" means a placeholder we could not fill
|
|
433
|
+
# (e.g. {article_id} on a non-IEEE page) — the URL would be
|
|
434
|
+
# sent with literal braces and fail downstream (#54).
|
|
435
|
+
if pdf_url and "{" not in pdf_url:
|
|
436
|
+
log.info(f" [CARSI-Browser] Trying PDF: {pdf_url[:80]}")
|
|
437
|
+
captured_pdf.clear()
|
|
438
|
+
try:
|
|
439
|
+
page.goto(pdf_url, wait_until="commit", timeout=30000)
|
|
440
|
+
time.sleep(5)
|
|
441
|
+
except Exception:
|
|
442
|
+
pass
|
|
443
|
+
saved = _try_save_captured() or _try_fetch_pdf(page.url)
|
|
444
|
+
if saved:
|
|
445
|
+
return saved
|
|
446
|
+
|
|
447
|
+
# Step 7: Find PDF link in HTML
|
|
448
|
+
from ..download.pdf import extract_pdf_url_from_html
|
|
449
|
+
html = page.content()
|
|
450
|
+
found_pdf = extract_pdf_url_from_html(html, page.url)
|
|
451
|
+
if found_pdf:
|
|
452
|
+
log.info(f" [CARSI-Browser] Found PDF link: {found_pdf[:80]}")
|
|
453
|
+
captured_pdf.clear()
|
|
454
|
+
try:
|
|
455
|
+
page.goto(found_pdf, wait_until="commit", timeout=30000)
|
|
456
|
+
time.sleep(5)
|
|
457
|
+
except Exception:
|
|
458
|
+
pass
|
|
459
|
+
saved = _try_save_captured() or _try_fetch_pdf(page.url)
|
|
460
|
+
if saved:
|
|
461
|
+
return saved
|
|
462
|
+
|
|
463
|
+
# Step 8: Click PDF button
|
|
464
|
+
click_result = page.evaluate("""
|
|
465
|
+
() => {
|
|
466
|
+
const links = document.querySelectorAll('a');
|
|
467
|
+
for (const a of links) {
|
|
468
|
+
const href = (a.getAttribute('href') || '').toLowerCase();
|
|
469
|
+
const text = (a.innerText || '').toLowerCase();
|
|
470
|
+
if ((href.includes('pdf') || href.includes('download')) && !href.includes('supplement')) {
|
|
471
|
+
if (text.includes('pdf') || text.includes('download')) {
|
|
472
|
+
a.click();
|
|
473
|
+
return a.href;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
return null;
|
|
478
|
+
}
|
|
479
|
+
""")
|
|
480
|
+
if click_result:
|
|
481
|
+
log.info(f" [CARSI-Browser] Clicked: {str(click_result)[:80]}")
|
|
482
|
+
time.sleep(8)
|
|
483
|
+
saved = _try_save_captured() or _try_fetch_pdf(page.url)
|
|
484
|
+
if saved:
|
|
485
|
+
return saved
|
|
486
|
+
|
|
487
|
+
log.info(f" [CARSI-Browser] No PDF found. Title: {page.title()[:40]} URL: {page.url[:60]}")
|
|
488
|
+
return None
|
|
489
|
+
|
|
490
|
+
except Exception as e:
|
|
491
|
+
log.info(f" [CARSI-Browser] Error: {e}")
|
|
492
|
+
return None
|
|
493
|
+
|
|
494
|
+
def _try_load_cookies(self, publisher: str) -> bool:
|
|
495
|
+
cookie_file = self._cookie_path(publisher)
|
|
496
|
+
if not cookie_file.exists():
|
|
497
|
+
return False
|
|
498
|
+
try:
|
|
499
|
+
cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
|
|
500
|
+
except (json.JSONDecodeError, OSError):
|
|
501
|
+
return False
|
|
502
|
+
|
|
503
|
+
sess = self._get_session(publisher)
|
|
504
|
+
for cookie in cookies:
|
|
505
|
+
sess.cookies.set(
|
|
506
|
+
cookie["name"],
|
|
507
|
+
cookie["value"],
|
|
508
|
+
domain=cookie.get("domain", ""),
|
|
509
|
+
path=cookie.get("path", "/"),
|
|
510
|
+
)
|
|
511
|
+
return self._validate_session(publisher)
|
|
512
|
+
|
|
513
|
+
def _validate_session(self, publisher: str) -> bool:
|
|
514
|
+
cfg = self._publisher_configs.get(publisher)
|
|
515
|
+
if not cfg:
|
|
516
|
+
return False
|
|
517
|
+
sess = self._get_session(publisher)
|
|
518
|
+
|
|
519
|
+
# Check cookie file freshness — accept if < 24h old
|
|
520
|
+
cookie_file = self._cookie_path(publisher)
|
|
521
|
+
try:
|
|
522
|
+
age_hours = (time.time() - os.path.getmtime(cookie_file)) / 3600
|
|
523
|
+
if age_hours > 24:
|
|
524
|
+
log.info(f" [CARSI] Cookies for {publisher} expired ({age_hours:.1f}h old)")
|
|
525
|
+
return False
|
|
526
|
+
except OSError:
|
|
527
|
+
return False
|
|
528
|
+
|
|
529
|
+
# Validate by hitting a publisher page that requires auth
|
|
530
|
+
# Use the main domain, not login_url (which always contains "login")
|
|
531
|
+
try:
|
|
532
|
+
test_url = f"https://{cfg.domains[0]}/"
|
|
533
|
+
resp = sess.get(test_url, timeout=15, allow_redirects=True)
|
|
534
|
+
# If we get redirected to a SSO/CAS/WAYF page, session is invalid
|
|
535
|
+
url_lower = resp.url.lower()
|
|
536
|
+
sso_keywords = ("wayf", "shibboleth", "saml", "idp.bayern", "passport")
|
|
537
|
+
if any(k in url_lower for k in sso_keywords):
|
|
538
|
+
return False
|
|
539
|
+
# Cloudflare-fronted publishers (ScienceDirect, Wiley, ...) challenge
|
|
540
|
+
# every non-browser request, so this probe cannot tell a live session
|
|
541
|
+
# from a dead one. Trust the freshness check above; the browser flow
|
|
542
|
+
# verifies access on the article page itself.
|
|
543
|
+
server = resp.headers.get("server", "").lower()
|
|
544
|
+
if resp.headers.get("cf-mitigated") == "challenge" or (
|
|
545
|
+
resp.status_code in (403, 503) and "cloudflare" in server):
|
|
546
|
+
log.info(f" [CARSI] {publisher}: Cloudflare blocks the HTTP probe; using fresh cookies")
|
|
547
|
+
return True
|
|
548
|
+
return resp.status_code == 200
|
|
549
|
+
except requests.RequestException:
|
|
550
|
+
return False
|
|
551
|
+
|
|
552
|
+
def _browser_login(self, publisher: str) -> bool:
|
|
553
|
+
"""Login via CARSI using CloakBrowser."""
|
|
554
|
+
cfg = self._publisher_configs.get(publisher)
|
|
555
|
+
if not cfg:
|
|
556
|
+
log.error(f" [CARSI] Unknown publisher: {publisher}")
|
|
557
|
+
return False
|
|
558
|
+
|
|
559
|
+
try:
|
|
560
|
+
from ..browser.login import carsi_login
|
|
561
|
+
return carsi_login(publisher, self.config, login_url=cfg.login_url, domains=cfg.domains)
|
|
562
|
+
except Exception as exc:
|
|
563
|
+
log.error(f" [CARSI] CloakBrowser login failed: {exc}")
|
|
564
|
+
return False
|
|
565
|
+
|
|
566
|
+
def close(self):
|
|
567
|
+
for sess in self._sessions.values():
|
|
568
|
+
sess.close()
|
|
569
|
+
self._sessions.clear()
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Config adapter wrapping the flat config dict for institutional workflows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ConfigAdapter:
|
|
9
|
+
"""Dict-compatible wrapper around litloop config.
|
|
10
|
+
|
|
11
|
+
Provides ``.load()`` as the canonical entry point and exposes
|
|
12
|
+
``._config`` for direct dict access when needed (e.g. setting
|
|
13
|
+
``output_dir``). Implements enough of the ``dict`` protocol so
|
|
14
|
+
that downstream consumers (``PaperFetcher``, ``EZProxyAuth``,
|
|
15
|
+
``WebVPNAuth``, ``CARSIClient``, ...) can treat it as a plain
|
|
16
|
+
config dict.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self) -> None:
|
|
20
|
+
from ..config import load_config
|
|
21
|
+
|
|
22
|
+
self._config: dict[str, Any] = load_config()
|
|
23
|
+
|
|
24
|
+
@classmethod
|
|
25
|
+
def load(cls) -> ConfigAdapter:
|
|
26
|
+
"""Return a fully initialised adapter (same as ``ConfigAdapter()``)."""
|
|
27
|
+
return cls()
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def chrome_profile_dir(self) -> str:
|
|
31
|
+
"""Path to the Chrome/Chromium profile directory."""
|
|
32
|
+
return self._config.get("chrome_profile_dir", "")
|
|
33
|
+
|
|
34
|
+
# dict protocol – makes the adapter transparent to downstream code
|
|
35
|
+
def get(self, key: str, default: Any = None) -> Any:
|
|
36
|
+
return self._config.get(key, default)
|
|
37
|
+
|
|
38
|
+
def __getitem__(self, key: str) -> Any:
|
|
39
|
+
return self._config[key]
|
|
40
|
+
|
|
41
|
+
def __setitem__(self, key: str, value: Any) -> None:
|
|
42
|
+
self._config[key] = value
|
|
43
|
+
|
|
44
|
+
def __contains__(self, key: str) -> bool: # type: ignore[override]
|
|
45
|
+
return key in self._config
|
|
46
|
+
|
|
47
|
+
def __iter__(self): # type: ignore[override]
|
|
48
|
+
return iter(self._config)
|
|
49
|
+
|
|
50
|
+
def keys(self):
|
|
51
|
+
return self._config.keys()
|
|
52
|
+
|
|
53
|
+
def items(self):
|
|
54
|
+
return self._config.items()
|
|
55
|
+
|
|
56
|
+
def values(self):
|
|
57
|
+
return self._config.values()
|
|
58
|
+
|
|
59
|
+
def __len__(self) -> int:
|
|
60
|
+
return len(self._config)
|
|
61
|
+
|
|
62
|
+
def __repr__(self) -> str:
|
|
63
|
+
return f"ConfigAdapter({self._config!r})"
|