litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/browser/login.py
ADDED
|
@@ -0,0 +1,636 @@
|
|
|
1
|
+
"""Institutional login via stealth browser. Replaces Selenium for WebVPN/CARSI/EZProxy login."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import time
|
|
7
|
+
import atexit
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
from urllib.parse import urlparse
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
from .backend import launch
|
|
14
|
+
from .backend import BACKEND_CAMOUFOX, BACKEND_CLOAKBROWSER, BACKEND_PATCHRIGHT
|
|
15
|
+
from .backend import is_available as _browser_backend_available
|
|
16
|
+
_HAS_CLOAKBROWSER = any(
|
|
17
|
+
_browser_backend_available(b)
|
|
18
|
+
for b in (BACKEND_PATCHRIGHT, BACKEND_CLOAKBROWSER, BACKEND_CAMOUFOX)
|
|
19
|
+
)
|
|
20
|
+
except ImportError:
|
|
21
|
+
launch = None # type: ignore[assignment]
|
|
22
|
+
_HAS_CLOAKBROWSER = False
|
|
23
|
+
from ..log import get_logger
|
|
24
|
+
|
|
25
|
+
log = get_logger()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class PersistentBrowser:
|
|
29
|
+
"""Keeps a stealth browser alive across multiple operations.
|
|
30
|
+
|
|
31
|
+
Login once, reuse the same browser for all subsequent downloads.
|
|
32
|
+
The WebVPN session stays valid because the browser instance never closes.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(self):
|
|
36
|
+
self._browser = None
|
|
37
|
+
self._context = None
|
|
38
|
+
self._page = None
|
|
39
|
+
self._cookies_saved = False
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def is_alive(self) -> bool:
|
|
43
|
+
if self._browser is None:
|
|
44
|
+
return False
|
|
45
|
+
try:
|
|
46
|
+
self._page.url # noqa: B018
|
|
47
|
+
return True
|
|
48
|
+
except Exception:
|
|
49
|
+
self._cleanup()
|
|
50
|
+
return False
|
|
51
|
+
|
|
52
|
+
def get_page(self, config: dict[str, Any] | None = None):
|
|
53
|
+
"""Get or create the browser page. Returns (context, page)."""
|
|
54
|
+
if self.is_alive:
|
|
55
|
+
return self._context, self._page
|
|
56
|
+
return self._start(config)
|
|
57
|
+
|
|
58
|
+
def _start(self, config: dict[str, Any] | None = None):
|
|
59
|
+
"""Start a new browser instance. Restores saved state if available."""
|
|
60
|
+
if not _HAS_CLOAKBROWSER:
|
|
61
|
+
raise RuntimeError("cloakbrowser not installed. Run: pip install cloakbrowser")
|
|
62
|
+
log.info(" [browser] Starting persistent browser...")
|
|
63
|
+
self._browser = launch(
|
|
64
|
+
headless=False, humanize=True,
|
|
65
|
+
args=["--disable-features=CrossOriginOpenerPolicy"],
|
|
66
|
+
config=config,
|
|
67
|
+
)
|
|
68
|
+
self._context = self._browser.new_context()
|
|
69
|
+
self._page = self._context.new_page()
|
|
70
|
+
|
|
71
|
+
if config:
|
|
72
|
+
self._restore_state(config)
|
|
73
|
+
|
|
74
|
+
return self._context, self._page
|
|
75
|
+
|
|
76
|
+
def _restore_state(self, config: dict[str, Any]):
|
|
77
|
+
"""Restore saved cookies and localStorage into the browser."""
|
|
78
|
+
from ..config import DATA_DIR
|
|
79
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
80
|
+
state_file = cache_dir / "browser_state.json"
|
|
81
|
+
if not state_file.exists():
|
|
82
|
+
return
|
|
83
|
+
try:
|
|
84
|
+
state = json.loads(state_file.read_text(encoding="utf-8"))
|
|
85
|
+
except Exception:
|
|
86
|
+
log.info(" [browser] browser_state.json corrupted, starting fresh")
|
|
87
|
+
return
|
|
88
|
+
|
|
89
|
+
cookies = state.get("cookies", [])
|
|
90
|
+
if cookies:
|
|
91
|
+
try:
|
|
92
|
+
self._context.add_cookies(cookies)
|
|
93
|
+
log.info(f" [browser] Restored {len(cookies)} cookies")
|
|
94
|
+
except Exception as e:
|
|
95
|
+
log.info(f" [browser] Cookie restore warning: {e}")
|
|
96
|
+
|
|
97
|
+
storage = state.get("localStorage", {})
|
|
98
|
+
for origin, items in storage.items():
|
|
99
|
+
try:
|
|
100
|
+
self._page.goto(origin, wait_until="commit", timeout=10000)
|
|
101
|
+
for key, value in items.items():
|
|
102
|
+
self._page.evaluate(f"localStorage.setItem({json.dumps(key)}, {json.dumps(value)})")
|
|
103
|
+
except Exception as e:
|
|
104
|
+
log.info(f" [browser] localStorage restore failed for {origin}: {e}")
|
|
105
|
+
|
|
106
|
+
log.info(" [browser] Browser state restored")
|
|
107
|
+
|
|
108
|
+
def save_cookies(self, config: dict[str, Any]):
|
|
109
|
+
"""Save current browser state (cookies + localStorage) to disk."""
|
|
110
|
+
if not self._context:
|
|
111
|
+
return
|
|
112
|
+
try:
|
|
113
|
+
from ..config import DATA_DIR
|
|
114
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
115
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
116
|
+
|
|
117
|
+
cookies = self._context.cookies()
|
|
118
|
+
|
|
119
|
+
localStorage = {}
|
|
120
|
+
for page in self._context.pages:
|
|
121
|
+
try:
|
|
122
|
+
url = page.url
|
|
123
|
+
if url.startswith("http"):
|
|
124
|
+
from urllib.parse import urlparse
|
|
125
|
+
origin = f"{urlparse(url).scheme}://{urlparse(url).hostname}"
|
|
126
|
+
items = page.evaluate("""
|
|
127
|
+
(() => {
|
|
128
|
+
const items = {};
|
|
129
|
+
for (let i = 0; i < localStorage.length; i++) {
|
|
130
|
+
const key = localStorage.key(i);
|
|
131
|
+
items[key] = localStorage.getItem(key);
|
|
132
|
+
}
|
|
133
|
+
return items;
|
|
134
|
+
})()
|
|
135
|
+
""")
|
|
136
|
+
if items:
|
|
137
|
+
localStorage[origin] = items
|
|
138
|
+
except Exception:
|
|
139
|
+
pass
|
|
140
|
+
|
|
141
|
+
state = {"cookies": cookies, "localStorage": localStorage}
|
|
142
|
+
state_file = cache_dir / "browser_state.json"
|
|
143
|
+
from ..institutional.session_store import keep_private
|
|
144
|
+
state_file.write_text(json.dumps(state, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
145
|
+
keep_private(state_file)
|
|
146
|
+
|
|
147
|
+
cookie_file = cache_dir / "instsci-cookies.json"
|
|
148
|
+
cookie_data = [
|
|
149
|
+
{"name": c["name"], "value": c["value"], "domain": c.get("domain", ""), "path": c.get("path", "/")}
|
|
150
|
+
for c in cookies
|
|
151
|
+
]
|
|
152
|
+
cookie_file.write_text(json.dumps(cookie_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
153
|
+
keep_private(cookie_file)
|
|
154
|
+
|
|
155
|
+
netscape_file = cache_dir / "instsci-cookies.txt"
|
|
156
|
+
from .cookies import cookies_to_netscape
|
|
157
|
+
netscape_file.write_text(cookies_to_netscape(cookies), encoding="utf-8")
|
|
158
|
+
keep_private(netscape_file)
|
|
159
|
+
|
|
160
|
+
self._cookies_saved = True
|
|
161
|
+
log.info(f" [browser] Saved {len(cookies)} cookies + {len(localStorage)} localStorage origins")
|
|
162
|
+
except Exception as e:
|
|
163
|
+
log.info(f" [browser] Failed to save state: {e}")
|
|
164
|
+
|
|
165
|
+
def _cleanup(self):
|
|
166
|
+
"""Close browser gracefully."""
|
|
167
|
+
try:
|
|
168
|
+
if self._browser:
|
|
169
|
+
self._browser.close()
|
|
170
|
+
except Exception:
|
|
171
|
+
pass
|
|
172
|
+
self._browser = None
|
|
173
|
+
self._context = None
|
|
174
|
+
self._page = None
|
|
175
|
+
|
|
176
|
+
def close(self):
|
|
177
|
+
"""Explicitly close the browser."""
|
|
178
|
+
self._cleanup()
|
|
179
|
+
log.info(" [browser] Persistent browser closed")
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# Module-level singleton
|
|
183
|
+
_browser = PersistentBrowser()
|
|
184
|
+
atexit.register(_browser.close)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _save_cookies_json(cookies: list[dict[str, Any]], cookie_file: Path) -> None:
|
|
188
|
+
"""Save cookies in JSON format (litloop compatible)."""
|
|
189
|
+
cookie_data = [
|
|
190
|
+
{"name": c["name"], "value": c["value"], "domain": c.get("domain", ""), "path": c.get("path", "/")}
|
|
191
|
+
for c in cookies
|
|
192
|
+
]
|
|
193
|
+
from ..institutional.session_store import keep_private
|
|
194
|
+
cookie_file.parent.mkdir(parents=True, exist_ok=True)
|
|
195
|
+
cookie_file.write_text(
|
|
196
|
+
json.dumps(cookie_data, indent=2, ensure_ascii=False),
|
|
197
|
+
encoding="utf-8",
|
|
198
|
+
)
|
|
199
|
+
keep_private(cookie_file)
|
|
200
|
+
# Also sync to legacy webvpn-cookies.json for PaperFetcher.auth compatibility
|
|
201
|
+
# — WebVPN logins only: CARSI/publisher logins save through here too and
|
|
202
|
+
# would overwrite the WebVPN session with unrelated cookies.
|
|
203
|
+
if cookie_file.name != "instsci-cookies.json":
|
|
204
|
+
return
|
|
205
|
+
try:
|
|
206
|
+
from ..institutional.auth import _get_cookie_path
|
|
207
|
+
auth_cookie_path = _get_cookie_path({})
|
|
208
|
+
auth_cookie_path.parent.mkdir(parents=True, exist_ok=True)
|
|
209
|
+
auth_cookie_path.write_text(
|
|
210
|
+
json.dumps(cookie_data, indent=2, ensure_ascii=False),
|
|
211
|
+
encoding="utf-8",
|
|
212
|
+
)
|
|
213
|
+
keep_private(auth_cookie_path)
|
|
214
|
+
except Exception:
|
|
215
|
+
pass
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _save_cookies_netscape(cookies: list[dict[str, Any]], cookie_file: Path) -> None:
|
|
220
|
+
"""Save cookies in Netscape format (CloakBrowser import compatible)."""
|
|
221
|
+
from .cookies import cookies_to_netscape
|
|
222
|
+
from ..institutional.session_store import keep_private
|
|
223
|
+
cookie_file.write_text(cookies_to_netscape(cookies), encoding="utf-8")
|
|
224
|
+
keep_private(cookie_file)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _import_to_browser(cookie_file: Path, config: dict[str, Any]) -> int:
|
|
228
|
+
"""Import cookies into CloakBrowser. Returns count imported."""
|
|
229
|
+
# The login window's sync Playwright still owns this thread's event loop,
|
|
230
|
+
# and the shared browser cannot start under it ("Sync API inside an asyncio
|
|
231
|
+
# loop"). Nothing is lost by skipping: download tabs re-import the saved
|
|
232
|
+
# cookie files when they open (_inject_cookies_to_tab).
|
|
233
|
+
try:
|
|
234
|
+
import asyncio
|
|
235
|
+
asyncio.get_running_loop()
|
|
236
|
+
log.info(" [browser] Cookies saved; the shared browser loads them on the next download")
|
|
237
|
+
return 0
|
|
238
|
+
except RuntimeError:
|
|
239
|
+
pass
|
|
240
|
+
try:
|
|
241
|
+
from .engine import import_cookies, is_available
|
|
242
|
+
if not is_available(config):
|
|
243
|
+
log.info(" [browser] CloakBrowser not running, skipping auto-import")
|
|
244
|
+
return 0
|
|
245
|
+
count = import_cookies(cookie_file, config)
|
|
246
|
+
log.info(f" [browser] Imported {count} cookies into CloakBrowser")
|
|
247
|
+
return count
|
|
248
|
+
except Exception as exc:
|
|
249
|
+
log.info(f" [browser] Could not auto-import to CloakBrowser: {exc}")
|
|
250
|
+
return 0
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def open_login_browser(
|
|
254
|
+
url: str,
|
|
255
|
+
config: dict[str, Any],
|
|
256
|
+
*,
|
|
257
|
+
cookie_file: Path,
|
|
258
|
+
detect_login: Any = None,
|
|
259
|
+
max_wait: int = 300,
|
|
260
|
+
auto_import: bool = True,
|
|
261
|
+
keep_alive: bool = False,
|
|
262
|
+
publisher: str = "",
|
|
263
|
+
) -> bool | tuple[bool, Any, Any, Any]:
|
|
264
|
+
"""Open a visible stealth browser for interactive login.
|
|
265
|
+
|
|
266
|
+
Args:
|
|
267
|
+
url: Login URL to open.
|
|
268
|
+
config: litloop config dict.
|
|
269
|
+
cookie_file: Path to save captured cookies (JSON).
|
|
270
|
+
detect_login: Optional callable(browser_context, page) -> bool for custom login detection.
|
|
271
|
+
max_wait: Max seconds to wait for login.
|
|
272
|
+
auto_import: Whether to auto-import cookies into CloakBrowser.
|
|
273
|
+
keep_alive: If True, return (True, context, page) without closing browser.
|
|
274
|
+
publisher: Publisher name for remote assist display.
|
|
275
|
+
|
|
276
|
+
Returns:
|
|
277
|
+
True if login succeeded, or (True, context, page) if keep_alive.
|
|
278
|
+
"""
|
|
279
|
+
log.info(f" [browser] Opening stealth browser: {url}")
|
|
280
|
+
print(f"\n 请在浏览器中登录 ({url})")
|
|
281
|
+
print(" 程序会自动检测登录完成...\n")
|
|
282
|
+
|
|
283
|
+
# Start remote assist if port is configured
|
|
284
|
+
remote = None
|
|
285
|
+
if int(config.get("remote_assist_port", 0)) > 0:
|
|
286
|
+
from ..institutional.remote_assist import RemoteAssist
|
|
287
|
+
remote = RemoteAssist(config, publisher=publisher)
|
|
288
|
+
remote.start()
|
|
289
|
+
remote.update_url(url)
|
|
290
|
+
|
|
291
|
+
if not _HAS_CLOAKBROWSER:
|
|
292
|
+
log.info(" [browser] cloakbrowser not installed")
|
|
293
|
+
return (False, None, None, None) if keep_alive else False
|
|
294
|
+
|
|
295
|
+
# Close any lingering background shared browser on this thread
|
|
296
|
+
try:
|
|
297
|
+
from .engine import close_shared_browser
|
|
298
|
+
close_shared_browser(config)
|
|
299
|
+
except Exception:
|
|
300
|
+
pass
|
|
301
|
+
|
|
302
|
+
browser = None
|
|
303
|
+
context = None
|
|
304
|
+
|
|
305
|
+
browser_args = [
|
|
306
|
+
"--disable-features=CrossOriginOpenerPolicy",
|
|
307
|
+
"--start-maximized",
|
|
308
|
+
]
|
|
309
|
+
|
|
310
|
+
try:
|
|
311
|
+
browser = launch(
|
|
312
|
+
headless=False,
|
|
313
|
+
humanize=True,
|
|
314
|
+
args=browser_args,
|
|
315
|
+
config=config,
|
|
316
|
+
)
|
|
317
|
+
context = browser.new_context()
|
|
318
|
+
except Exception as exc:
|
|
319
|
+
log.error(f" [browser] Failed to launch browser: {exc}")
|
|
320
|
+
return (False, None, None, None) if keep_alive else False
|
|
321
|
+
|
|
322
|
+
def _close_all():
|
|
323
|
+
if browser is not None:
|
|
324
|
+
try:
|
|
325
|
+
browser.close()
|
|
326
|
+
except Exception:
|
|
327
|
+
pass
|
|
328
|
+
|
|
329
|
+
try:
|
|
330
|
+
# Seed previously saved cookies into context if available
|
|
331
|
+
if cookie_file.exists():
|
|
332
|
+
try:
|
|
333
|
+
saved_raw = json.loads(cookie_file.read_text(encoding="utf-8"))
|
|
334
|
+
if saved_raw:
|
|
335
|
+
pw_cookies = []
|
|
336
|
+
for c in saved_raw:
|
|
337
|
+
c_dict = {
|
|
338
|
+
"name": c["name"],
|
|
339
|
+
"value": c["value"],
|
|
340
|
+
"domain": c.get("domain", ""),
|
|
341
|
+
"path": c.get("path", "/"),
|
|
342
|
+
}
|
|
343
|
+
if c.get("expires", 0) > 0:
|
|
344
|
+
c_dict["expires"] = c["expires"]
|
|
345
|
+
pw_cookies.append(c_dict)
|
|
346
|
+
context.add_cookies(pw_cookies)
|
|
347
|
+
except Exception:
|
|
348
|
+
pass
|
|
349
|
+
|
|
350
|
+
page = context.pages[0] if context.pages else context.new_page()
|
|
351
|
+
|
|
352
|
+
try:
|
|
353
|
+
page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
|
354
|
+
try:
|
|
355
|
+
page.bring_to_front()
|
|
356
|
+
except Exception:
|
|
357
|
+
pass
|
|
358
|
+
except Exception as exc:
|
|
359
|
+
log.info(f" [browser] Page load warning: {exc}")
|
|
360
|
+
print(" 页面加载超时,但仍可手动登录。")
|
|
361
|
+
|
|
362
|
+
elapsed = 0
|
|
363
|
+
while elapsed < max_wait:
|
|
364
|
+
# Wait via Playwright, not time.sleep: the sync API only processes
|
|
365
|
+
# browser events (navigations, new tabs) while a Playwright call is
|
|
366
|
+
# running, so after a bare sleep page.url still shows the login page
|
|
367
|
+
# the user has long since left.
|
|
368
|
+
try:
|
|
369
|
+
live = [p for p in context.pages if not p.is_closed()]
|
|
370
|
+
if live:
|
|
371
|
+
live[-1].title() # lets the driver deliver events; raises if the browser is gone
|
|
372
|
+
time.sleep(2) # not wait_for_timeout: it hangs on a page stuck loading
|
|
373
|
+
except Exception:
|
|
374
|
+
time.sleep(2)
|
|
375
|
+
elapsed += 2
|
|
376
|
+
# Tell the download this window waits for the user (the race keeps
|
|
377
|
+
# waiting, the app shows a notice); raises once the paper is settled.
|
|
378
|
+
from ..sources import note_user_waiting
|
|
379
|
+
note_user_waiting("login")
|
|
380
|
+
|
|
381
|
+
if elapsed % 10 == 0:
|
|
382
|
+
log.info(f" [browser] Waiting for user login... elapsed={elapsed}s, open_tabs={len(context.pages)}")
|
|
383
|
+
|
|
384
|
+
try:
|
|
385
|
+
pages = [p for p in context.pages if not p.is_closed()]
|
|
386
|
+
except Exception:
|
|
387
|
+
pages = []
|
|
388
|
+
|
|
389
|
+
if not pages:
|
|
390
|
+
log.info(" [browser] Browser closed by user.")
|
|
391
|
+
if remote:
|
|
392
|
+
remote.stop()
|
|
393
|
+
# Salvage cookies if user completed login and manually closed the window
|
|
394
|
+
try:
|
|
395
|
+
salvaged = context.cookies()
|
|
396
|
+
if salvaged and len(salvaged) >= 3:
|
|
397
|
+
if detect_login is None or detect_login(context, None):
|
|
398
|
+
_save_cookies_json(salvaged, cookie_file)
|
|
399
|
+
netscape_path = cookie_file.with_suffix(".txt")
|
|
400
|
+
_save_cookies_netscape(salvaged, netscape_path)
|
|
401
|
+
log.info(f" [browser] Salvaged {len(salvaged)} cookies upon window close.")
|
|
402
|
+
print(f" 登录成功!Cookie 已保存至 {cookie_file}")
|
|
403
|
+
if auto_import:
|
|
404
|
+
_import_to_browser(netscape_path, config)
|
|
405
|
+
if not keep_alive:
|
|
406
|
+
_close_all()
|
|
407
|
+
return True
|
|
408
|
+
except Exception:
|
|
409
|
+
pass
|
|
410
|
+
if not keep_alive:
|
|
411
|
+
_close_all()
|
|
412
|
+
return (False, None, None, None) if keep_alive else False
|
|
413
|
+
|
|
414
|
+
active_page = pages[-1]
|
|
415
|
+
try:
|
|
416
|
+
current_url = active_page.url
|
|
417
|
+
if remote:
|
|
418
|
+
remote.update_url(current_url)
|
|
419
|
+
except Exception:
|
|
420
|
+
pass
|
|
421
|
+
|
|
422
|
+
if detect_login is not None:
|
|
423
|
+
matched = False
|
|
424
|
+
for p in reversed(pages):
|
|
425
|
+
try:
|
|
426
|
+
if detect_login(context, p):
|
|
427
|
+
matched = True
|
|
428
|
+
active_page = p
|
|
429
|
+
break
|
|
430
|
+
except Exception:
|
|
431
|
+
continue
|
|
432
|
+
|
|
433
|
+
if matched:
|
|
434
|
+
cookies = context.cookies()
|
|
435
|
+
_save_cookies_json(cookies, cookie_file)
|
|
436
|
+
netscape_path = cookie_file.with_suffix(".txt")
|
|
437
|
+
_save_cookies_netscape(cookies, netscape_path)
|
|
438
|
+
log.info(f" [browser] Login successful! Saved {len(cookies)} cookies.")
|
|
439
|
+
print(f" 登录成功!Cookie 已保存至 {cookie_file}")
|
|
440
|
+
if auto_import:
|
|
441
|
+
_import_to_browser(netscape_path, config)
|
|
442
|
+
if remote:
|
|
443
|
+
remote.stop()
|
|
444
|
+
if keep_alive:
|
|
445
|
+
return True, context, active_page
|
|
446
|
+
time.sleep(0.5)
|
|
447
|
+
_close_all()
|
|
448
|
+
return True
|
|
449
|
+
continue
|
|
450
|
+
|
|
451
|
+
# Generic detection fallback (when detect_login is None)
|
|
452
|
+
url_lower = current_url.lower()
|
|
453
|
+
if "login" not in url_lower and "cas" not in url_lower and "sso" not in url_lower:
|
|
454
|
+
cookies = context.cookies()
|
|
455
|
+
if len(cookies) > 3:
|
|
456
|
+
_save_cookies_json(cookies, cookie_file)
|
|
457
|
+
netscape_path = cookie_file.with_suffix(".txt")
|
|
458
|
+
_save_cookies_netscape(cookies, netscape_path)
|
|
459
|
+
log.info(f" [browser] Login successful! Saved {len(cookies)} cookies.")
|
|
460
|
+
print(f" 登录成功!Cookie 已保存至 {cookie_file}")
|
|
461
|
+
if auto_import:
|
|
462
|
+
_import_to_browser(netscape_path, config)
|
|
463
|
+
if remote:
|
|
464
|
+
remote.stop()
|
|
465
|
+
if keep_alive:
|
|
466
|
+
return True, context, active_page
|
|
467
|
+
time.sleep(0.5)
|
|
468
|
+
_close_all()
|
|
469
|
+
return True
|
|
470
|
+
|
|
471
|
+
print(" 登录超时。")
|
|
472
|
+
if remote:
|
|
473
|
+
remote.stop()
|
|
474
|
+
if not keep_alive:
|
|
475
|
+
_close_all()
|
|
476
|
+
return (False, None, None, None) if keep_alive else False
|
|
477
|
+
|
|
478
|
+
except Exception as exc:
|
|
479
|
+
log.info(f" [browser] Login error: {exc}")
|
|
480
|
+
print(f" 登录出错: {exc}")
|
|
481
|
+
_close_all()
|
|
482
|
+
return (False, None, None, None) if keep_alive else False
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def webvpn_login(config: dict[str, Any]) -> bool:
|
|
486
|
+
"""Login to WebVPN via stealth browser."""
|
|
487
|
+
from ..sources.instsci import _get_webvpn_base
|
|
488
|
+
base = _get_webvpn_base(config)
|
|
489
|
+
if not base:
|
|
490
|
+
log.info(" [WebVPN] No base URL configured")
|
|
491
|
+
return False
|
|
492
|
+
|
|
493
|
+
from ..config import DATA_DIR
|
|
494
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
495
|
+
cookie_file = cache_dir / "instsci-cookies.json"
|
|
496
|
+
|
|
497
|
+
from urllib.parse import urlparse as _urlparse
|
|
498
|
+
base_root = base.rstrip("/") + "/"
|
|
499
|
+
base_host = _urlparse(base_root).netloc
|
|
500
|
+
state = {"last_log": 0.0}
|
|
501
|
+
|
|
502
|
+
def _probe(cookies: list[dict[str, Any]]) -> tuple[bool, str]:
|
|
503
|
+
"""Authoritative check: replay the browser's cookies against the gateway root.
|
|
504
|
+
|
|
505
|
+
An unauthenticated session is redirected to the gateway's /login (then
|
|
506
|
+
the school's SSO); an authenticated one stays on the portal. This holds
|
|
507
|
+
for any WebVPN gateway and needs no cookie-name or page-text guesses.
|
|
508
|
+
"""
|
|
509
|
+
import requests
|
|
510
|
+
from ..network import USER_AGENT
|
|
511
|
+
s = requests.Session()
|
|
512
|
+
s.trust_env = False
|
|
513
|
+
for c in cookies:
|
|
514
|
+
s.cookies.set(c["name"], c["value"], domain=c.get("domain", ""), path=c.get("path", "/"))
|
|
515
|
+
try:
|
|
516
|
+
resp = s.get(base_root, timeout=15, allow_redirects=True, headers={"User-Agent": USER_AGENT})
|
|
517
|
+
except Exception as exc:
|
|
518
|
+
return False, f"probe error: {exc}"
|
|
519
|
+
final = _urlparse(resp.url)
|
|
520
|
+
ok = resp.status_code < 400 and final.netloc == base_host and "login" not in final.path.lower()
|
|
521
|
+
return ok, resp.url
|
|
522
|
+
|
|
523
|
+
def _detect(context: Any, page: Any) -> bool:
|
|
524
|
+
try:
|
|
525
|
+
cookies = context.cookies()
|
|
526
|
+
url, title = "", ""
|
|
527
|
+
if page is not None:
|
|
528
|
+
try:
|
|
529
|
+
url = page.evaluate("location.href") or "" # live, never the cached page.url
|
|
530
|
+
title = page.title() or ""
|
|
531
|
+
except Exception:
|
|
532
|
+
pass
|
|
533
|
+
# Still on a login form: definitely not logged in yet (skip the probe).
|
|
534
|
+
low = (url + " " + title).lower()
|
|
535
|
+
if "统一身份认证" in title or "用户登录" in title or "cas login" in low or "/login" in url.lower():
|
|
536
|
+
_log_state(url, title, cookies, "on login page")
|
|
537
|
+
return False
|
|
538
|
+
ok, landed = _probe(cookies)
|
|
539
|
+
_log_state(url, title, cookies, landed, force=ok)
|
|
540
|
+
return ok
|
|
541
|
+
except Exception as exc:
|
|
542
|
+
log.info(f" [WebVPN] detect error: {exc}")
|
|
543
|
+
return False
|
|
544
|
+
|
|
545
|
+
def _log_state(url: str, title: str, cookies: list[dict[str, Any]], probe: str, force: bool = False) -> None:
|
|
546
|
+
now = time.monotonic()
|
|
547
|
+
if force or now - state["last_log"] >= 10:
|
|
548
|
+
state["last_log"] = now
|
|
549
|
+
names = sorted({c.get("name", "") for c in cookies if base_host in (c.get("domain") or "")})
|
|
550
|
+
log.info(f" [WebVPN] url={url[:70]} title={title[:30]!r} vpn_cookies={names} probe={probe[:70]}")
|
|
551
|
+
|
|
552
|
+
return open_login_browser(base, config, cookie_file=cookie_file, detect_login=_detect, max_wait=600)
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def carsi_login(publisher: str, config: dict[str, Any], *, login_url: str, domains: list[str]) -> bool:
|
|
556
|
+
"""Login to CARSI federated authentication via stealth browser."""
|
|
557
|
+
from ..config import DATA_DIR
|
|
558
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache"))) / "carsi_cookies"
|
|
559
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
560
|
+
cookie_file = cache_dir / f"{publisher}.json"
|
|
561
|
+
|
|
562
|
+
def _detect(context: Any, page: Any) -> bool:
|
|
563
|
+
try:
|
|
564
|
+
current_url = page.url
|
|
565
|
+
on_publisher = any(d in current_url for d in domains)
|
|
566
|
+
on_login = any(x in current_url.lower() for x in ("login", "institutional", "wayf", "saml", "cas", "idp"))
|
|
567
|
+
if not on_publisher or on_login:
|
|
568
|
+
return False
|
|
569
|
+
# The URL gate alone false-positives: some publishers bounce the
|
|
570
|
+
# unauthenticated visitor to their homepage, which passes the URL
|
|
571
|
+
# check before the user has seen the IdP (#29). Require a
|
|
572
|
+
# federated-auth artifact — completing CARSI always leaves a
|
|
573
|
+
# Shibboleth/SAML session cookie or an IdP-domain (.edu.cn)
|
|
574
|
+
# cookie in the context.
|
|
575
|
+
for c in context.cookies():
|
|
576
|
+
name = (c.get("name") or "").lower()
|
|
577
|
+
dom = (c.get("domain") or "").lower()
|
|
578
|
+
if "shibsession" in name or "saml" in name or "idp" in dom or dom.endswith(".edu.cn"):
|
|
579
|
+
return True
|
|
580
|
+
return False
|
|
581
|
+
except Exception:
|
|
582
|
+
return False
|
|
583
|
+
|
|
584
|
+
return open_login_browser(
|
|
585
|
+
login_url,
|
|
586
|
+
config,
|
|
587
|
+
cookie_file=cookie_file,
|
|
588
|
+
detect_login=_detect,
|
|
589
|
+
max_wait=600, # IdP + 2FA is human-paced; 180s wastes the run (#62)
|
|
590
|
+
)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def ezproxy_login(config: dict[str, Any]) -> bool:
|
|
594
|
+
"""Login to EZProxy via stealth browser."""
|
|
595
|
+
from ..sources.ezproxy import _ezproxy_origin, _is_proxy_domain
|
|
596
|
+
|
|
597
|
+
origin = _ezproxy_origin(config)
|
|
598
|
+
if not origin:
|
|
599
|
+
log.info(" [EZProxy] No ezproxy_login_url configured")
|
|
600
|
+
return False
|
|
601
|
+
|
|
602
|
+
from ..config import DATA_DIR
|
|
603
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
604
|
+
cookie_file = cache_dir / "ezproxy_cookies.json"
|
|
605
|
+
|
|
606
|
+
# Land on the bare /login endpoint: it is the only URL that reliably
|
|
607
|
+
# challenges. /login?url=<target> skips straight to the raw publisher on
|
|
608
|
+
# hostname-rewriting deployments and never authenticates (#62).
|
|
609
|
+
login_url = f"{origin}/login"
|
|
610
|
+
|
|
611
|
+
def _detect(context: Any, page: Any) -> bool:
|
|
612
|
+
try:
|
|
613
|
+
parsed = urlparse(page.url)
|
|
614
|
+
path_q = f"{parsed.path}?{parsed.query}".lower()
|
|
615
|
+
if "/login" in path_q or "wayf" in path_q or "shibboleth" in path_q:
|
|
616
|
+
return False
|
|
617
|
+
# The authentication signal is the proxy's own session cookie —
|
|
618
|
+
# not the URL shape. "URL is not a login page" used to pass the
|
|
619
|
+
# moment an unauthenticated visitor was redirected to the raw
|
|
620
|
+
# publisher, saving 17 publisher cookies and zero proxy
|
|
621
|
+
# cookies as a "successful" login (#62).
|
|
622
|
+
for c in context.cookies():
|
|
623
|
+
if _is_proxy_domain(c.get("domain") or ""):
|
|
624
|
+
return True
|
|
625
|
+
host = (parsed.hostname or "").lower()
|
|
626
|
+
return _is_proxy_domain(host)
|
|
627
|
+
except Exception:
|
|
628
|
+
return False
|
|
629
|
+
|
|
630
|
+
return open_login_browser(
|
|
631
|
+
login_url,
|
|
632
|
+
config,
|
|
633
|
+
cookie_file=cookie_file,
|
|
634
|
+
detect_login=_detect,
|
|
635
|
+
max_wait=600, # SSO + 2FA is human-paced; 180s wastes the run (#62)
|
|
636
|
+
)
|