litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,576 @@
|
|
|
1
|
+
"""Campus institutional access authentication management using CloakBrowser."""
|
|
2
|
+
|
|
3
|
+
import binascii
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
from urllib.parse import urlparse
|
|
10
|
+
|
|
11
|
+
import requests
|
|
12
|
+
from Crypto.Cipher import AES
|
|
13
|
+
|
|
14
|
+
from .session_store import CookieStore
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
from ..browser.backend import launch
|
|
18
|
+
from ..browser.backend import BACKEND_CAMOUFOX, BACKEND_CLOAKBROWSER, BACKEND_PATCHRIGHT
|
|
19
|
+
from ..browser.backend import is_available as _browser_backend_available
|
|
20
|
+
_HAS_CLOAKBROWSER = any(
|
|
21
|
+
_browser_backend_available(b)
|
|
22
|
+
for b in (BACKEND_PATCHRIGHT, BACKEND_CLOAKBROWSER, BACKEND_CAMOUFOX)
|
|
23
|
+
)
|
|
24
|
+
except ImportError:
|
|
25
|
+
launch = None # type: ignore[assignment]
|
|
26
|
+
_HAS_CLOAKBROWSER = False
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
TEST_URL = "https://www.nature.com"
|
|
31
|
+
WEBVPN_DEFAULT_KEY = b"wrdvpnisthebest!"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _get_cookie_path(config: dict) -> Path:
|
|
35
|
+
"""Resolve cookie file path from config."""
|
|
36
|
+
explicit = config.get("instsci_cookie_file", "")
|
|
37
|
+
if explicit:
|
|
38
|
+
return Path(explicit)
|
|
39
|
+
data_dir = Path(os.environ.get("LITLOOP_DATA_DIR", str(Path.home() / ".litloop")))
|
|
40
|
+
return data_dir / "cookies" / "webvpn-cookies.json"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _get_profile_dir(config: dict) -> Path:
|
|
44
|
+
"""Resolve Chrome profile directory from config."""
|
|
45
|
+
explicit = config.get("chrome_profile_dir", "")
|
|
46
|
+
if explicit:
|
|
47
|
+
return Path(explicit)
|
|
48
|
+
data_dir = Path(os.environ.get("LITLOOP_DATA_DIR", str(Path.home() / ".litloop")))
|
|
49
|
+
return data_dir / "browser_profiles" / "webvpn"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _seed_saved_cookies(cookie_path: Path, context: Any) -> int:
|
|
53
|
+
"""Restore cookies saved to disk into the freshly launched browser.
|
|
54
|
+
|
|
55
|
+
Gateway session cookies (no Expires) never survive a browser close
|
|
56
|
+
inside the persistent profile — Chrome keeps them in memory only —
|
|
57
|
+
so the profile itself cannot restore the session. CookieStore keeps
|
|
58
|
+
them as valid on disk; inject them before navigation so the gateway
|
|
59
|
+
accepts the restored session instead of forcing a manual re-login.
|
|
60
|
+
"""
|
|
61
|
+
cookies = CookieStore(cookie_path).load()
|
|
62
|
+
if not cookies:
|
|
63
|
+
return 0
|
|
64
|
+
for cookie in cookies:
|
|
65
|
+
if cookie.get("expires", 0) <= 0:
|
|
66
|
+
cookie.pop("expires", None) # Playwright treats 0 as already expired
|
|
67
|
+
try:
|
|
68
|
+
context.add_cookies(cookies)
|
|
69
|
+
except Exception as exc:
|
|
70
|
+
logger.warning("Failed to seed saved cookies into browser: %s", exc)
|
|
71
|
+
return 0
|
|
72
|
+
logger.info("Restored %d saved cookies into browser session", len(cookies))
|
|
73
|
+
return len(cookies)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class WebVPNAuth:
|
|
77
|
+
"""Manages campus access authentication and URL conversion.
|
|
78
|
+
|
|
79
|
+
Supports Chinese university campus gateway systems.
|
|
80
|
+
URL conversion uses AES-CFB encryption on the hostname.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def __init__(
|
|
84
|
+
self,
|
|
85
|
+
config: dict,
|
|
86
|
+
key: bytes | None = None,
|
|
87
|
+
iv: bytes | None = None,
|
|
88
|
+
base_url: str = "",
|
|
89
|
+
):
|
|
90
|
+
self.config = config
|
|
91
|
+
self._encrypt_key = key or WEBVPN_DEFAULT_KEY
|
|
92
|
+
self._encrypt_iv = iv or self._encrypt_key
|
|
93
|
+
self._session: requests.Session | None = None
|
|
94
|
+
self._browser = None
|
|
95
|
+
self._context = None
|
|
96
|
+
self._page = None
|
|
97
|
+
# Explicit config wins over the school-database fallback: users may
|
|
98
|
+
# have customized their gateway URL after `setup`.
|
|
99
|
+
base = (config.get("instsci_base_url", "")
|
|
100
|
+
or config.get("vpnsci_base_url", "")
|
|
101
|
+
or base_url)
|
|
102
|
+
self._webvpn_base = base.rstrip("/") if base else ""
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def browser_context(self):
|
|
106
|
+
return self._context
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def browser_page(self):
|
|
110
|
+
return self._page
|
|
111
|
+
|
|
112
|
+
def _browser_launch_args(self) -> list[str]:
|
|
113
|
+
return [
|
|
114
|
+
"--no-proxy-server",
|
|
115
|
+
"--disable-features=CrossOriginOpenerPolicy",
|
|
116
|
+
]
|
|
117
|
+
|
|
118
|
+
def _close_browser_quietly(self) -> None:
|
|
119
|
+
"""Close a launched persistent context and release its Playwright loop.
|
|
120
|
+
|
|
121
|
+
Skipping this leaks the sync dispatcher loop on the current thread;
|
|
122
|
+
every later launch in the same thread then fails with "Playwright Sync
|
|
123
|
+
API inside the asyncio loop" (seen in CLI fetch/batch cascades).
|
|
124
|
+
"""
|
|
125
|
+
for attr in ("_page", "_context", "_browser"):
|
|
126
|
+
obj = getattr(self, attr, None)
|
|
127
|
+
if obj is None:
|
|
128
|
+
continue
|
|
129
|
+
try:
|
|
130
|
+
obj.close()
|
|
131
|
+
except Exception:
|
|
132
|
+
pass
|
|
133
|
+
setattr(self, attr, None)
|
|
134
|
+
|
|
135
|
+
@property
|
|
136
|
+
def session(self) -> requests.Session:
|
|
137
|
+
if self._session is None:
|
|
138
|
+
self._session = requests.Session()
|
|
139
|
+
self._session.headers.update({
|
|
140
|
+
"User-Agent": (
|
|
141
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
142
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
143
|
+
"Chrome/120.0.0.0 Safari/537.36"
|
|
144
|
+
)
|
|
145
|
+
})
|
|
146
|
+
if os.environ.get("HTTP_PROXY") or os.environ.get("HTTPS_PROXY") or \
|
|
147
|
+
os.environ.get("http_proxy") or os.environ.get("https_proxy"):
|
|
148
|
+
self._session.verify = False
|
|
149
|
+
proxy = self.config.get("network_proxy", "")
|
|
150
|
+
if proxy:
|
|
151
|
+
self._session.proxies = {
|
|
152
|
+
"http": proxy,
|
|
153
|
+
"https": proxy,
|
|
154
|
+
}
|
|
155
|
+
logger.info("Using connector: %s", proxy)
|
|
156
|
+
return self._session
|
|
157
|
+
|
|
158
|
+
def convert_url(self, url: str) -> str:
|
|
159
|
+
"""Convert a regular URL to a campus gateway URL using AES-CFB encryption."""
|
|
160
|
+
parsed = urlparse(url)
|
|
161
|
+
scheme = parsed.scheme.lower()
|
|
162
|
+
hostname = parsed.hostname
|
|
163
|
+
port = parsed.port
|
|
164
|
+
path = parsed.path
|
|
165
|
+
query = parsed.query
|
|
166
|
+
|
|
167
|
+
if not hostname:
|
|
168
|
+
return url
|
|
169
|
+
if not self._webvpn_base:
|
|
170
|
+
# Never return a relative gateway URL — callers would navigate to
|
|
171
|
+
# "/https/77726476..." which is always an invalid URL.
|
|
172
|
+
raise ValueError(
|
|
173
|
+
"WebVPN base URL is empty — run `litloop setup <学校全名>` first"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
cipher = AES.new(self._encrypt_key, AES.MODE_CFB, self._encrypt_iv, segment_size=128)
|
|
177
|
+
encrypted = cipher.encrypt(hostname.encode("utf-8"))
|
|
178
|
+
|
|
179
|
+
encrypted_hex = binascii.hexlify(self._encrypt_iv).decode() + binascii.hexlify(encrypted).decode()
|
|
180
|
+
|
|
181
|
+
scheme_part = scheme
|
|
182
|
+
if port:
|
|
183
|
+
scheme_part = f"{scheme}-{port}"
|
|
184
|
+
|
|
185
|
+
result = f"{self._webvpn_base}/{scheme_part}/{encrypted_hex}{path}"
|
|
186
|
+
if query:
|
|
187
|
+
result += f"?{query}"
|
|
188
|
+
return result
|
|
189
|
+
|
|
190
|
+
def login(self, force: bool = False) -> bool:
|
|
191
|
+
"""Ensure we have a valid session."""
|
|
192
|
+
proxy = self.config.get("network_proxy", "")
|
|
193
|
+
if proxy:
|
|
194
|
+
# Connector/Campus-tunnel assumption: the configured proxy carries
|
|
195
|
+
# campus egress, so IP-based entitlement should work without SSO.
|
|
196
|
+
# Still try saved cookies first, and make the assumption LOUD —
|
|
197
|
+
# with a personal tunnel (Clash etc.) this assumption silently
|
|
198
|
+
# starves every institutional request behind auth_required.
|
|
199
|
+
if self._try_load_cookies():
|
|
200
|
+
logger.info("Loaded saved cookies - session is valid.")
|
|
201
|
+
return True
|
|
202
|
+
logger.warning(
|
|
203
|
+
"network_proxy is set — assuming connector/campus-tunnel mode "
|
|
204
|
+
"and skipping institutional login. If your proxy is a personal "
|
|
205
|
+
"tunnel, institutional access WILL fail: clear network_proxy "
|
|
206
|
+
"and run `litloop login` to establish an SSO session."
|
|
207
|
+
)
|
|
208
|
+
return True
|
|
209
|
+
|
|
210
|
+
if not force and self._try_load_cookies():
|
|
211
|
+
logger.info("Loaded saved cookies - session is valid.")
|
|
212
|
+
return True
|
|
213
|
+
|
|
214
|
+
logger.info("No valid session found. Opening browser for login...")
|
|
215
|
+
return self._browser_login()
|
|
216
|
+
|
|
217
|
+
def _try_load_cookies(self) -> bool:
|
|
218
|
+
cookie_path = _get_cookie_path(self.config)
|
|
219
|
+
if not CookieStore(cookie_path).load_into(self.session):
|
|
220
|
+
logger.info("All saved cookies have expired.")
|
|
221
|
+
return False
|
|
222
|
+
return self._validate_session()
|
|
223
|
+
|
|
224
|
+
def _validate_session(self) -> bool:
|
|
225
|
+
proxy = self.config.get("network_proxy", "")
|
|
226
|
+
if proxy:
|
|
227
|
+
test_url = TEST_URL
|
|
228
|
+
else:
|
|
229
|
+
if not self._webvpn_base:
|
|
230
|
+
return False
|
|
231
|
+
test_url = self.convert_url(TEST_URL)
|
|
232
|
+
try:
|
|
233
|
+
resp = self.session.get(test_url, timeout=15, allow_redirects=True)
|
|
234
|
+
if "cas" in resp.url.lower() or "login" in resp.url.lower():
|
|
235
|
+
logger.info("Session expired - redirected to login page.")
|
|
236
|
+
return False
|
|
237
|
+
if resp.status_code == 200:
|
|
238
|
+
return True
|
|
239
|
+
except requests.RequestException as e:
|
|
240
|
+
logger.warning("Session validation failed: %s", e)
|
|
241
|
+
return False
|
|
242
|
+
|
|
243
|
+
def _browser_login(self) -> bool:
|
|
244
|
+
# Fail BEFORE launching any browser: launching and then bailing out
|
|
245
|
+
# leaks the sync Playwright dispatcher loop on this thread, and every
|
|
246
|
+
# later launch then dies with "Playwright Sync API inside the asyncio
|
|
247
|
+
# loop" for the rest of the process (seen in CLI fetch/batch).
|
|
248
|
+
if not self._webvpn_base:
|
|
249
|
+
logger.error(
|
|
250
|
+
"WebVPN base URL is empty — school gateway unknown. "
|
|
251
|
+
"Run `litloop setup <学校全名>` first and check `litloop schools`."
|
|
252
|
+
)
|
|
253
|
+
return False
|
|
254
|
+
if not _HAS_CLOAKBROWSER:
|
|
255
|
+
logger.error("no browser backend available. Run: pip install patchright (or pip install cloakbrowser)")
|
|
256
|
+
return False
|
|
257
|
+
|
|
258
|
+
try:
|
|
259
|
+
from ..browser.backend import launch_persistent_context
|
|
260
|
+
profile_dir = _get_profile_dir(self.config)
|
|
261
|
+
profile_dir.mkdir(parents=True, exist_ok=True)
|
|
262
|
+
self._context = launch_persistent_context(
|
|
263
|
+
user_data_dir=str(profile_dir),
|
|
264
|
+
headless=False, humanize=True,
|
|
265
|
+
args=self._browser_launch_args(),
|
|
266
|
+
)
|
|
267
|
+
self._browser = None
|
|
268
|
+
self._page = self._context.new_page()
|
|
269
|
+
except Exception as e:
|
|
270
|
+
logger.error("Failed to start CloakBrowser: %s", e)
|
|
271
|
+
self._close_browser_quietly()
|
|
272
|
+
return False
|
|
273
|
+
|
|
274
|
+
_seed_saved_cookies(_get_cookie_path(self.config), self._context)
|
|
275
|
+
|
|
276
|
+
self._page.goto(self._webvpn_base, wait_until="networkidle", timeout=30000)
|
|
277
|
+
current_url = self._page.url
|
|
278
|
+
logger.info("Session test: navigated to campus gateway, landed on %s", current_url[:80])
|
|
279
|
+
|
|
280
|
+
parsed = urlparse(current_url)
|
|
281
|
+
url_host = (parsed.hostname or "").lower()
|
|
282
|
+
on_login_page = (
|
|
283
|
+
"cas" in current_url.lower()
|
|
284
|
+
or "login" in current_url.lower()
|
|
285
|
+
or "/oauth/" in current_url.lower()
|
|
286
|
+
or "/sso/" in current_url.lower()
|
|
287
|
+
or "/wayf" in current_url.lower()
|
|
288
|
+
or "/shibboleth" in current_url.lower()
|
|
289
|
+
)
|
|
290
|
+
is_idp = "id.tsinghua" in url_host or "idp." in url_host or "auth." in url_host
|
|
291
|
+
|
|
292
|
+
if not on_login_page and not is_idp:
|
|
293
|
+
logger.info("Persistent context has valid session! URL=%s", current_url[:60])
|
|
294
|
+
self._save_browser_cookies()
|
|
295
|
+
return True
|
|
296
|
+
|
|
297
|
+
self._page.goto(self._webvpn_base, wait_until="domcontentloaded")
|
|
298
|
+
|
|
299
|
+
print("\n" + "=" * 60)
|
|
300
|
+
print(f" Please log in at {self._webvpn_base}")
|
|
301
|
+
print(" in the browser window that just opened.")
|
|
302
|
+
print(" The tool will detect when login is complete.")
|
|
303
|
+
print("=" * 60 + "\n")
|
|
304
|
+
|
|
305
|
+
max_wait = 600
|
|
306
|
+
poll_interval = 3
|
|
307
|
+
elapsed = 0
|
|
308
|
+
last_url = ""
|
|
309
|
+
|
|
310
|
+
while elapsed < max_wait:
|
|
311
|
+
time.sleep(poll_interval)
|
|
312
|
+
elapsed += poll_interval
|
|
313
|
+
|
|
314
|
+
try:
|
|
315
|
+
if not self._context.pages:
|
|
316
|
+
logger.info("Browser closed by user.")
|
|
317
|
+
self._browser = None
|
|
318
|
+
self._context = None
|
|
319
|
+
self._page = None
|
|
320
|
+
return False
|
|
321
|
+
|
|
322
|
+
current_url = self._page.url
|
|
323
|
+
|
|
324
|
+
if current_url != last_url:
|
|
325
|
+
logger.info("Browser URL: %s", current_url)
|
|
326
|
+
last_url = current_url
|
|
327
|
+
|
|
328
|
+
cookies = self._context.cookies()
|
|
329
|
+
vpn_cookies = [
|
|
330
|
+
c for c in cookies
|
|
331
|
+
if "webvpn" in c.get("domain", "").lower()
|
|
332
|
+
and c.get("name", "").startswith("wengine_vpn_ticket")
|
|
333
|
+
]
|
|
334
|
+
if vpn_cookies:
|
|
335
|
+
parsed_url = urlparse(current_url)
|
|
336
|
+
url_host = (parsed_url.hostname or "").lower()
|
|
337
|
+
on_webvpn = url_host and "webvpn" in url_host
|
|
338
|
+
if on_webvpn:
|
|
339
|
+
logger.info("Login confirmed: campus session cookie and gateway URL. URL=%s", current_url[:60])
|
|
340
|
+
self._save_browser_cookies()
|
|
341
|
+
print("\n Login successful! Cookies saved. Browser kept alive for PDF download.\n")
|
|
342
|
+
return True
|
|
343
|
+
|
|
344
|
+
on_login_page = (
|
|
345
|
+
"/login" in current_url.lower()
|
|
346
|
+
or "cas" in current_url.lower()
|
|
347
|
+
or "/oauth/" in current_url.lower()
|
|
348
|
+
or "/sso/" in current_url.lower()
|
|
349
|
+
or "/wayf" in current_url.lower()
|
|
350
|
+
or "/shibboleth" in current_url.lower()
|
|
351
|
+
)
|
|
352
|
+
if not on_login_page:
|
|
353
|
+
is_gateway = (
|
|
354
|
+
self._webvpn_base in current_url
|
|
355
|
+
or "otrust" in current_url.lower()
|
|
356
|
+
or "/portal/" in current_url.lower()
|
|
357
|
+
)
|
|
358
|
+
if is_gateway:
|
|
359
|
+
logger.info("Login detected via URL! (url=%s)", current_url[:60])
|
|
360
|
+
self._save_browser_cookies()
|
|
361
|
+
print("\n Login successful! Cookies saved. Browser kept alive for PDF download.\n")
|
|
362
|
+
return True
|
|
363
|
+
|
|
364
|
+
except Exception:
|
|
365
|
+
logger.warning("Browser connection lost.")
|
|
366
|
+
self._browser = None
|
|
367
|
+
self._context = None
|
|
368
|
+
self._page = None
|
|
369
|
+
return False
|
|
370
|
+
|
|
371
|
+
print("\n Login timed out after 10 minutes.\n")
|
|
372
|
+
self._close_browser()
|
|
373
|
+
return False
|
|
374
|
+
|
|
375
|
+
def _save_browser_cookies(self):
|
|
376
|
+
if not self._context:
|
|
377
|
+
return
|
|
378
|
+
cookie_path = _get_cookie_path(self.config)
|
|
379
|
+
store = CookieStore(cookie_path)
|
|
380
|
+
cookies = store.save(self._context.cookies())
|
|
381
|
+
logger.info("Saved %d cookies to %s", len(cookies), store.path)
|
|
382
|
+
store.apply_to_session(self.session, cookies)
|
|
383
|
+
|
|
384
|
+
def _close_browser(self):
|
|
385
|
+
if self._context:
|
|
386
|
+
try:
|
|
387
|
+
self._context.close()
|
|
388
|
+
except Exception:
|
|
389
|
+
pass
|
|
390
|
+
if self._browser:
|
|
391
|
+
try:
|
|
392
|
+
self._browser.close()
|
|
393
|
+
except Exception:
|
|
394
|
+
pass
|
|
395
|
+
self._browser = None
|
|
396
|
+
self._context = None
|
|
397
|
+
self._page = None
|
|
398
|
+
|
|
399
|
+
def fetch(self, url: str, **kwargs) -> requests.Response:
|
|
400
|
+
"""Fetch a URL through the campus, EasyConnect, or connector session."""
|
|
401
|
+
kwargs.setdefault("timeout", 30)
|
|
402
|
+
kwargs.setdefault("allow_redirects", True)
|
|
403
|
+
|
|
404
|
+
proxy = self.config.get("network_proxy", "")
|
|
405
|
+
if proxy:
|
|
406
|
+
return self.session.get(url, **kwargs)
|
|
407
|
+
|
|
408
|
+
if self._webvpn_base in url:
|
|
409
|
+
proxied = url
|
|
410
|
+
else:
|
|
411
|
+
proxied = self.convert_url(url)
|
|
412
|
+
|
|
413
|
+
return self.session.get(proxied, **kwargs)
|
|
414
|
+
|
|
415
|
+
def close(self):
|
|
416
|
+
self._close_browser()
|
|
417
|
+
if self._session:
|
|
418
|
+
self._session.close()
|
|
419
|
+
self._session = None
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
class EZProxyAuth:
|
|
423
|
+
"""Manages EZproxy authentication and URL proxying."""
|
|
424
|
+
|
|
425
|
+
def __init__(
|
|
426
|
+
self,
|
|
427
|
+
config: dict,
|
|
428
|
+
proxy_base: str = "",
|
|
429
|
+
):
|
|
430
|
+
self.config = config
|
|
431
|
+
self._proxy_base = proxy_base or config.get("ezproxy_login_url", "")
|
|
432
|
+
self._session: requests.Session | None = None
|
|
433
|
+
self._browser = None
|
|
434
|
+
self._context = None
|
|
435
|
+
self._page = None
|
|
436
|
+
|
|
437
|
+
@property
|
|
438
|
+
def browser_context(self):
|
|
439
|
+
return self._context
|
|
440
|
+
|
|
441
|
+
@property
|
|
442
|
+
def session(self) -> requests.Session:
|
|
443
|
+
if self._session is None:
|
|
444
|
+
self._session = requests.Session()
|
|
445
|
+
self._session.headers.update({
|
|
446
|
+
"User-Agent": (
|
|
447
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
448
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
449
|
+
"Chrome/120.0.0.0 Safari/537.36"
|
|
450
|
+
)
|
|
451
|
+
})
|
|
452
|
+
return self._session
|
|
453
|
+
|
|
454
|
+
def login(self, force: bool = False) -> bool:
|
|
455
|
+
if not force and self._try_load_cookies():
|
|
456
|
+
logger.info("Loaded saved EZproxy cookies.")
|
|
457
|
+
return True
|
|
458
|
+
logger.info("No valid EZproxy session. Opening browser for login...")
|
|
459
|
+
return self._browser_login()
|
|
460
|
+
|
|
461
|
+
def _try_load_cookies(self) -> bool:
|
|
462
|
+
cookie_path = _get_cookie_path(self.config)
|
|
463
|
+
if not CookieStore(cookie_path).load_into(self.session):
|
|
464
|
+
return False
|
|
465
|
+
return self._validate_session()
|
|
466
|
+
|
|
467
|
+
def _validate_session(self) -> bool:
|
|
468
|
+
try:
|
|
469
|
+
resp = self.session.get(self._proxy_base + TEST_URL, timeout=15, allow_redirects=True)
|
|
470
|
+
if "login" in resp.url.lower() or "cas" in resp.url.lower():
|
|
471
|
+
return False
|
|
472
|
+
return resp.status_code == 200
|
|
473
|
+
except requests.RequestException:
|
|
474
|
+
return False
|
|
475
|
+
|
|
476
|
+
def _browser_login(self) -> bool:
|
|
477
|
+
if not _HAS_CLOAKBROWSER:
|
|
478
|
+
logger.error("no browser backend available. Run: pip install patchright (or pip install cloakbrowser)")
|
|
479
|
+
return False
|
|
480
|
+
|
|
481
|
+
try:
|
|
482
|
+
self._browser = launch(
|
|
483
|
+
headless=False, humanize=True,
|
|
484
|
+
args=self._browser_launch_args(),
|
|
485
|
+
)
|
|
486
|
+
self._context = self._browser.new_context()
|
|
487
|
+
self._page = self._context.new_page()
|
|
488
|
+
except Exception as e:
|
|
489
|
+
logger.error("Failed to start CloakBrowser: %s", e)
|
|
490
|
+
return False
|
|
491
|
+
|
|
492
|
+
_seed_saved_cookies(_get_cookie_path(self.config), self._context)
|
|
493
|
+
|
|
494
|
+
self._page.goto(self._proxy_base + TEST_URL, wait_until="domcontentloaded")
|
|
495
|
+
|
|
496
|
+
print("\n" + "=" * 60)
|
|
497
|
+
print(f" Please log in at the EZproxy page.")
|
|
498
|
+
print(" The tool will detect when login is complete.")
|
|
499
|
+
print("=" * 60 + "\n")
|
|
500
|
+
|
|
501
|
+
max_wait = 600
|
|
502
|
+
poll_interval = 3
|
|
503
|
+
elapsed = 0
|
|
504
|
+
last_url = ""
|
|
505
|
+
|
|
506
|
+
while elapsed < max_wait:
|
|
507
|
+
time.sleep(poll_interval)
|
|
508
|
+
elapsed += poll_interval
|
|
509
|
+
|
|
510
|
+
try:
|
|
511
|
+
if not self._context.pages:
|
|
512
|
+
logger.info("Browser closed by user.")
|
|
513
|
+
self._browser = None
|
|
514
|
+
self._context = None
|
|
515
|
+
self._page = None
|
|
516
|
+
return False
|
|
517
|
+
|
|
518
|
+
current_url = self._page.url
|
|
519
|
+
|
|
520
|
+
if current_url != last_url:
|
|
521
|
+
logger.info("Browser URL: %s", current_url)
|
|
522
|
+
last_url = current_url
|
|
523
|
+
|
|
524
|
+
on_login = "login" in current_url.lower() or "cas" in current_url.lower()
|
|
525
|
+
if not on_login and self._proxy_base not in current_url:
|
|
526
|
+
logger.info("EZproxy login detected! URL: %s", current_url)
|
|
527
|
+
self._save_browser_cookies()
|
|
528
|
+
print("\n Login successful! Cookies saved. Browser kept alive for PDF download.\n")
|
|
529
|
+
return True
|
|
530
|
+
|
|
531
|
+
except Exception:
|
|
532
|
+
logger.warning("Browser connection lost.")
|
|
533
|
+
self._browser = None
|
|
534
|
+
self._context = None
|
|
535
|
+
self._page = None
|
|
536
|
+
return False
|
|
537
|
+
|
|
538
|
+
print("\n Login timed out after 10 minutes.\n")
|
|
539
|
+
self._close_browser()
|
|
540
|
+
return False
|
|
541
|
+
|
|
542
|
+
def _save_browser_cookies(self):
|
|
543
|
+
if not self._context:
|
|
544
|
+
return
|
|
545
|
+
cookie_path = _get_cookie_path(self.config)
|
|
546
|
+
store = CookieStore(cookie_path)
|
|
547
|
+
cookies = store.save(self._context.cookies())
|
|
548
|
+
logger.info("Saved %d cookies to %s", len(cookies), store.path)
|
|
549
|
+
store.apply_to_session(self.session, cookies)
|
|
550
|
+
|
|
551
|
+
def _close_browser(self):
|
|
552
|
+
if self._browser:
|
|
553
|
+
try:
|
|
554
|
+
self._browser.close()
|
|
555
|
+
except Exception:
|
|
556
|
+
pass
|
|
557
|
+
self._browser = None
|
|
558
|
+
self._context = None
|
|
559
|
+
self._page = None
|
|
560
|
+
|
|
561
|
+
def get_proxied_url(self, url: str) -> str:
|
|
562
|
+
if self._proxy_base and self._proxy_base.rstrip("/").split("//")[-1].split("/")[0] in url:
|
|
563
|
+
return url
|
|
564
|
+
return self._proxy_base + url
|
|
565
|
+
|
|
566
|
+
def fetch(self, url: str, **kwargs) -> requests.Response:
|
|
567
|
+
kwargs.setdefault("timeout", 30)
|
|
568
|
+
kwargs.setdefault("allow_redirects", True)
|
|
569
|
+
proxied = self.get_proxied_url(url)
|
|
570
|
+
return self.session.get(proxied, **kwargs)
|
|
571
|
+
|
|
572
|
+
def close(self):
|
|
573
|
+
self._close_browser()
|
|
574
|
+
if self._session:
|
|
575
|
+
self._session.close()
|
|
576
|
+
self._session = None
|