litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,636 @@
1
+ """Institutional login via stealth browser. Replaces Selenium for WebVPN/CARSI/EZProxy login."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import time
7
+ import atexit
8
+ from pathlib import Path
9
+ from typing import Any
10
+ from urllib.parse import urlparse
11
+
12
+ try:
13
+ from .backend import launch
14
+ from .backend import BACKEND_CAMOUFOX, BACKEND_CLOAKBROWSER, BACKEND_PATCHRIGHT
15
+ from .backend import is_available as _browser_backend_available
16
+ _HAS_CLOAKBROWSER = any(
17
+ _browser_backend_available(b)
18
+ for b in (BACKEND_PATCHRIGHT, BACKEND_CLOAKBROWSER, BACKEND_CAMOUFOX)
19
+ )
20
+ except ImportError:
21
+ launch = None # type: ignore[assignment]
22
+ _HAS_CLOAKBROWSER = False
23
+ from ..log import get_logger
24
+
25
+ log = get_logger()
26
+
27
+
28
+ class PersistentBrowser:
29
+ """Keeps a stealth browser alive across multiple operations.
30
+
31
+ Login once, reuse the same browser for all subsequent downloads.
32
+ The WebVPN session stays valid because the browser instance never closes.
33
+ """
34
+
35
+ def __init__(self):
36
+ self._browser = None
37
+ self._context = None
38
+ self._page = None
39
+ self._cookies_saved = False
40
+
41
+ @property
42
+ def is_alive(self) -> bool:
43
+ if self._browser is None:
44
+ return False
45
+ try:
46
+ self._page.url # noqa: B018
47
+ return True
48
+ except Exception:
49
+ self._cleanup()
50
+ return False
51
+
52
+ def get_page(self, config: dict[str, Any] | None = None):
53
+ """Get or create the browser page. Returns (context, page)."""
54
+ if self.is_alive:
55
+ return self._context, self._page
56
+ return self._start(config)
57
+
58
+ def _start(self, config: dict[str, Any] | None = None):
59
+ """Start a new browser instance. Restores saved state if available."""
60
+ if not _HAS_CLOAKBROWSER:
61
+ raise RuntimeError("cloakbrowser not installed. Run: pip install cloakbrowser")
62
+ log.info(" [browser] Starting persistent browser...")
63
+ self._browser = launch(
64
+ headless=False, humanize=True,
65
+ args=["--disable-features=CrossOriginOpenerPolicy"],
66
+ config=config,
67
+ )
68
+ self._context = self._browser.new_context()
69
+ self._page = self._context.new_page()
70
+
71
+ if config:
72
+ self._restore_state(config)
73
+
74
+ return self._context, self._page
75
+
76
+ def _restore_state(self, config: dict[str, Any]):
77
+ """Restore saved cookies and localStorage into the browser."""
78
+ from ..config import DATA_DIR
79
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
80
+ state_file = cache_dir / "browser_state.json"
81
+ if not state_file.exists():
82
+ return
83
+ try:
84
+ state = json.loads(state_file.read_text(encoding="utf-8"))
85
+ except Exception:
86
+ log.info(" [browser] browser_state.json corrupted, starting fresh")
87
+ return
88
+
89
+ cookies = state.get("cookies", [])
90
+ if cookies:
91
+ try:
92
+ self._context.add_cookies(cookies)
93
+ log.info(f" [browser] Restored {len(cookies)} cookies")
94
+ except Exception as e:
95
+ log.info(f" [browser] Cookie restore warning: {e}")
96
+
97
+ storage = state.get("localStorage", {})
98
+ for origin, items in storage.items():
99
+ try:
100
+ self._page.goto(origin, wait_until="commit", timeout=10000)
101
+ for key, value in items.items():
102
+ self._page.evaluate(f"localStorage.setItem({json.dumps(key)}, {json.dumps(value)})")
103
+ except Exception as e:
104
+ log.info(f" [browser] localStorage restore failed for {origin}: {e}")
105
+
106
+ log.info(" [browser] Browser state restored")
107
+
108
+ def save_cookies(self, config: dict[str, Any]):
109
+ """Save current browser state (cookies + localStorage) to disk."""
110
+ if not self._context:
111
+ return
112
+ try:
113
+ from ..config import DATA_DIR
114
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
115
+ cache_dir.mkdir(parents=True, exist_ok=True)
116
+
117
+ cookies = self._context.cookies()
118
+
119
+ localStorage = {}
120
+ for page in self._context.pages:
121
+ try:
122
+ url = page.url
123
+ if url.startswith("http"):
124
+ from urllib.parse import urlparse
125
+ origin = f"{urlparse(url).scheme}://{urlparse(url).hostname}"
126
+ items = page.evaluate("""
127
+ (() => {
128
+ const items = {};
129
+ for (let i = 0; i < localStorage.length; i++) {
130
+ const key = localStorage.key(i);
131
+ items[key] = localStorage.getItem(key);
132
+ }
133
+ return items;
134
+ })()
135
+ """)
136
+ if items:
137
+ localStorage[origin] = items
138
+ except Exception:
139
+ pass
140
+
141
+ state = {"cookies": cookies, "localStorage": localStorage}
142
+ state_file = cache_dir / "browser_state.json"
143
+ from ..institutional.session_store import keep_private
144
+ state_file.write_text(json.dumps(state, indent=2, ensure_ascii=False), encoding="utf-8")
145
+ keep_private(state_file)
146
+
147
+ cookie_file = cache_dir / "instsci-cookies.json"
148
+ cookie_data = [
149
+ {"name": c["name"], "value": c["value"], "domain": c.get("domain", ""), "path": c.get("path", "/")}
150
+ for c in cookies
151
+ ]
152
+ cookie_file.write_text(json.dumps(cookie_data, indent=2, ensure_ascii=False), encoding="utf-8")
153
+ keep_private(cookie_file)
154
+
155
+ netscape_file = cache_dir / "instsci-cookies.txt"
156
+ from .cookies import cookies_to_netscape
157
+ netscape_file.write_text(cookies_to_netscape(cookies), encoding="utf-8")
158
+ keep_private(netscape_file)
159
+
160
+ self._cookies_saved = True
161
+ log.info(f" [browser] Saved {len(cookies)} cookies + {len(localStorage)} localStorage origins")
162
+ except Exception as e:
163
+ log.info(f" [browser] Failed to save state: {e}")
164
+
165
+ def _cleanup(self):
166
+ """Close browser gracefully."""
167
+ try:
168
+ if self._browser:
169
+ self._browser.close()
170
+ except Exception:
171
+ pass
172
+ self._browser = None
173
+ self._context = None
174
+ self._page = None
175
+
176
+ def close(self):
177
+ """Explicitly close the browser."""
178
+ self._cleanup()
179
+ log.info(" [browser] Persistent browser closed")
180
+
181
+
182
+ # Module-level singleton
183
+ _browser = PersistentBrowser()
184
+ atexit.register(_browser.close)
185
+
186
+
187
+ def _save_cookies_json(cookies: list[dict[str, Any]], cookie_file: Path) -> None:
188
+ """Save cookies in JSON format (litloop compatible)."""
189
+ cookie_data = [
190
+ {"name": c["name"], "value": c["value"], "domain": c.get("domain", ""), "path": c.get("path", "/")}
191
+ for c in cookies
192
+ ]
193
+ from ..institutional.session_store import keep_private
194
+ cookie_file.parent.mkdir(parents=True, exist_ok=True)
195
+ cookie_file.write_text(
196
+ json.dumps(cookie_data, indent=2, ensure_ascii=False),
197
+ encoding="utf-8",
198
+ )
199
+ keep_private(cookie_file)
200
+ # Also sync to legacy webvpn-cookies.json for PaperFetcher.auth compatibility
201
+ # — WebVPN logins only: CARSI/publisher logins save through here too and
202
+ # would overwrite the WebVPN session with unrelated cookies.
203
+ if cookie_file.name != "instsci-cookies.json":
204
+ return
205
+ try:
206
+ from ..institutional.auth import _get_cookie_path
207
+ auth_cookie_path = _get_cookie_path({})
208
+ auth_cookie_path.parent.mkdir(parents=True, exist_ok=True)
209
+ auth_cookie_path.write_text(
210
+ json.dumps(cookie_data, indent=2, ensure_ascii=False),
211
+ encoding="utf-8",
212
+ )
213
+ keep_private(auth_cookie_path)
214
+ except Exception:
215
+ pass
216
+
217
+
218
+
219
+ def _save_cookies_netscape(cookies: list[dict[str, Any]], cookie_file: Path) -> None:
220
+ """Save cookies in Netscape format (CloakBrowser import compatible)."""
221
+ from .cookies import cookies_to_netscape
222
+ from ..institutional.session_store import keep_private
223
+ cookie_file.write_text(cookies_to_netscape(cookies), encoding="utf-8")
224
+ keep_private(cookie_file)
225
+
226
+
227
+ def _import_to_browser(cookie_file: Path, config: dict[str, Any]) -> int:
228
+ """Import cookies into CloakBrowser. Returns count imported."""
229
+ # The login window's sync Playwright still owns this thread's event loop,
230
+ # and the shared browser cannot start under it ("Sync API inside an asyncio
231
+ # loop"). Nothing is lost by skipping: download tabs re-import the saved
232
+ # cookie files when they open (_inject_cookies_to_tab).
233
+ try:
234
+ import asyncio
235
+ asyncio.get_running_loop()
236
+ log.info(" [browser] Cookies saved; the shared browser loads them on the next download")
237
+ return 0
238
+ except RuntimeError:
239
+ pass
240
+ try:
241
+ from .engine import import_cookies, is_available
242
+ if not is_available(config):
243
+ log.info(" [browser] CloakBrowser not running, skipping auto-import")
244
+ return 0
245
+ count = import_cookies(cookie_file, config)
246
+ log.info(f" [browser] Imported {count} cookies into CloakBrowser")
247
+ return count
248
+ except Exception as exc:
249
+ log.info(f" [browser] Could not auto-import to CloakBrowser: {exc}")
250
+ return 0
251
+
252
+
253
+ def open_login_browser(
254
+ url: str,
255
+ config: dict[str, Any],
256
+ *,
257
+ cookie_file: Path,
258
+ detect_login: Any = None,
259
+ max_wait: int = 300,
260
+ auto_import: bool = True,
261
+ keep_alive: bool = False,
262
+ publisher: str = "",
263
+ ) -> bool | tuple[bool, Any, Any, Any]:
264
+ """Open a visible stealth browser for interactive login.
265
+
266
+ Args:
267
+ url: Login URL to open.
268
+ config: litloop config dict.
269
+ cookie_file: Path to save captured cookies (JSON).
270
+ detect_login: Optional callable(browser_context, page) -> bool for custom login detection.
271
+ max_wait: Max seconds to wait for login.
272
+ auto_import: Whether to auto-import cookies into CloakBrowser.
273
+ keep_alive: If True, return (True, context, page) without closing browser.
274
+ publisher: Publisher name for remote assist display.
275
+
276
+ Returns:
277
+ True if login succeeded, or (True, context, page) if keep_alive.
278
+ """
279
+ log.info(f" [browser] Opening stealth browser: {url}")
280
+ print(f"\n 请在浏览器中登录 ({url})")
281
+ print(" 程序会自动检测登录完成...\n")
282
+
283
+ # Start remote assist if port is configured
284
+ remote = None
285
+ if int(config.get("remote_assist_port", 0)) > 0:
286
+ from ..institutional.remote_assist import RemoteAssist
287
+ remote = RemoteAssist(config, publisher=publisher)
288
+ remote.start()
289
+ remote.update_url(url)
290
+
291
+ if not _HAS_CLOAKBROWSER:
292
+ log.info(" [browser] cloakbrowser not installed")
293
+ return (False, None, None, None) if keep_alive else False
294
+
295
+ # Close any lingering background shared browser on this thread
296
+ try:
297
+ from .engine import close_shared_browser
298
+ close_shared_browser(config)
299
+ except Exception:
300
+ pass
301
+
302
+ browser = None
303
+ context = None
304
+
305
+ browser_args = [
306
+ "--disable-features=CrossOriginOpenerPolicy",
307
+ "--start-maximized",
308
+ ]
309
+
310
+ try:
311
+ browser = launch(
312
+ headless=False,
313
+ humanize=True,
314
+ args=browser_args,
315
+ config=config,
316
+ )
317
+ context = browser.new_context()
318
+ except Exception as exc:
319
+ log.error(f" [browser] Failed to launch browser: {exc}")
320
+ return (False, None, None, None) if keep_alive else False
321
+
322
+ def _close_all():
323
+ if browser is not None:
324
+ try:
325
+ browser.close()
326
+ except Exception:
327
+ pass
328
+
329
+ try:
330
+ # Seed previously saved cookies into context if available
331
+ if cookie_file.exists():
332
+ try:
333
+ saved_raw = json.loads(cookie_file.read_text(encoding="utf-8"))
334
+ if saved_raw:
335
+ pw_cookies = []
336
+ for c in saved_raw:
337
+ c_dict = {
338
+ "name": c["name"],
339
+ "value": c["value"],
340
+ "domain": c.get("domain", ""),
341
+ "path": c.get("path", "/"),
342
+ }
343
+ if c.get("expires", 0) > 0:
344
+ c_dict["expires"] = c["expires"]
345
+ pw_cookies.append(c_dict)
346
+ context.add_cookies(pw_cookies)
347
+ except Exception:
348
+ pass
349
+
350
+ page = context.pages[0] if context.pages else context.new_page()
351
+
352
+ try:
353
+ page.goto(url, wait_until="domcontentloaded", timeout=60000)
354
+ try:
355
+ page.bring_to_front()
356
+ except Exception:
357
+ pass
358
+ except Exception as exc:
359
+ log.info(f" [browser] Page load warning: {exc}")
360
+ print(" 页面加载超时,但仍可手动登录。")
361
+
362
+ elapsed = 0
363
+ while elapsed < max_wait:
364
+ # Wait via Playwright, not time.sleep: the sync API only processes
365
+ # browser events (navigations, new tabs) while a Playwright call is
366
+ # running, so after a bare sleep page.url still shows the login page
367
+ # the user has long since left.
368
+ try:
369
+ live = [p for p in context.pages if not p.is_closed()]
370
+ if live:
371
+ live[-1].title() # lets the driver deliver events; raises if the browser is gone
372
+ time.sleep(2) # not wait_for_timeout: it hangs on a page stuck loading
373
+ except Exception:
374
+ time.sleep(2)
375
+ elapsed += 2
376
+ # Tell the download this window waits for the user (the race keeps
377
+ # waiting, the app shows a notice); raises once the paper is settled.
378
+ from ..sources import note_user_waiting
379
+ note_user_waiting("login")
380
+
381
+ if elapsed % 10 == 0:
382
+ log.info(f" [browser] Waiting for user login... elapsed={elapsed}s, open_tabs={len(context.pages)}")
383
+
384
+ try:
385
+ pages = [p for p in context.pages if not p.is_closed()]
386
+ except Exception:
387
+ pages = []
388
+
389
+ if not pages:
390
+ log.info(" [browser] Browser closed by user.")
391
+ if remote:
392
+ remote.stop()
393
+ # Salvage cookies if user completed login and manually closed the window
394
+ try:
395
+ salvaged = context.cookies()
396
+ if salvaged and len(salvaged) >= 3:
397
+ if detect_login is None or detect_login(context, None):
398
+ _save_cookies_json(salvaged, cookie_file)
399
+ netscape_path = cookie_file.with_suffix(".txt")
400
+ _save_cookies_netscape(salvaged, netscape_path)
401
+ log.info(f" [browser] Salvaged {len(salvaged)} cookies upon window close.")
402
+ print(f" 登录成功!Cookie 已保存至 {cookie_file}")
403
+ if auto_import:
404
+ _import_to_browser(netscape_path, config)
405
+ if not keep_alive:
406
+ _close_all()
407
+ return True
408
+ except Exception:
409
+ pass
410
+ if not keep_alive:
411
+ _close_all()
412
+ return (False, None, None, None) if keep_alive else False
413
+
414
+ active_page = pages[-1]
415
+ try:
416
+ current_url = active_page.url
417
+ if remote:
418
+ remote.update_url(current_url)
419
+ except Exception:
420
+ pass
421
+
422
+ if detect_login is not None:
423
+ matched = False
424
+ for p in reversed(pages):
425
+ try:
426
+ if detect_login(context, p):
427
+ matched = True
428
+ active_page = p
429
+ break
430
+ except Exception:
431
+ continue
432
+
433
+ if matched:
434
+ cookies = context.cookies()
435
+ _save_cookies_json(cookies, cookie_file)
436
+ netscape_path = cookie_file.with_suffix(".txt")
437
+ _save_cookies_netscape(cookies, netscape_path)
438
+ log.info(f" [browser] Login successful! Saved {len(cookies)} cookies.")
439
+ print(f" 登录成功!Cookie 已保存至 {cookie_file}")
440
+ if auto_import:
441
+ _import_to_browser(netscape_path, config)
442
+ if remote:
443
+ remote.stop()
444
+ if keep_alive:
445
+ return True, context, active_page
446
+ time.sleep(0.5)
447
+ _close_all()
448
+ return True
449
+ continue
450
+
451
+ # Generic detection fallback (when detect_login is None)
452
+ url_lower = current_url.lower()
453
+ if "login" not in url_lower and "cas" not in url_lower and "sso" not in url_lower:
454
+ cookies = context.cookies()
455
+ if len(cookies) > 3:
456
+ _save_cookies_json(cookies, cookie_file)
457
+ netscape_path = cookie_file.with_suffix(".txt")
458
+ _save_cookies_netscape(cookies, netscape_path)
459
+ log.info(f" [browser] Login successful! Saved {len(cookies)} cookies.")
460
+ print(f" 登录成功!Cookie 已保存至 {cookie_file}")
461
+ if auto_import:
462
+ _import_to_browser(netscape_path, config)
463
+ if remote:
464
+ remote.stop()
465
+ if keep_alive:
466
+ return True, context, active_page
467
+ time.sleep(0.5)
468
+ _close_all()
469
+ return True
470
+
471
+ print(" 登录超时。")
472
+ if remote:
473
+ remote.stop()
474
+ if not keep_alive:
475
+ _close_all()
476
+ return (False, None, None, None) if keep_alive else False
477
+
478
+ except Exception as exc:
479
+ log.info(f" [browser] Login error: {exc}")
480
+ print(f" 登录出错: {exc}")
481
+ _close_all()
482
+ return (False, None, None, None) if keep_alive else False
483
+
484
+
485
+ def webvpn_login(config: dict[str, Any]) -> bool:
486
+ """Login to WebVPN via stealth browser."""
487
+ from ..sources.instsci import _get_webvpn_base
488
+ base = _get_webvpn_base(config)
489
+ if not base:
490
+ log.info(" [WebVPN] No base URL configured")
491
+ return False
492
+
493
+ from ..config import DATA_DIR
494
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
495
+ cookie_file = cache_dir / "instsci-cookies.json"
496
+
497
+ from urllib.parse import urlparse as _urlparse
498
+ base_root = base.rstrip("/") + "/"
499
+ base_host = _urlparse(base_root).netloc
500
+ state = {"last_log": 0.0}
501
+
502
+ def _probe(cookies: list[dict[str, Any]]) -> tuple[bool, str]:
503
+ """Authoritative check: replay the browser's cookies against the gateway root.
504
+
505
+ An unauthenticated session is redirected to the gateway's /login (then
506
+ the school's SSO); an authenticated one stays on the portal. This holds
507
+ for any WebVPN gateway and needs no cookie-name or page-text guesses.
508
+ """
509
+ import requests
510
+ from ..network import USER_AGENT
511
+ s = requests.Session()
512
+ s.trust_env = False
513
+ for c in cookies:
514
+ s.cookies.set(c["name"], c["value"], domain=c.get("domain", ""), path=c.get("path", "/"))
515
+ try:
516
+ resp = s.get(base_root, timeout=15, allow_redirects=True, headers={"User-Agent": USER_AGENT})
517
+ except Exception as exc:
518
+ return False, f"probe error: {exc}"
519
+ final = _urlparse(resp.url)
520
+ ok = resp.status_code < 400 and final.netloc == base_host and "login" not in final.path.lower()
521
+ return ok, resp.url
522
+
523
+ def _detect(context: Any, page: Any) -> bool:
524
+ try:
525
+ cookies = context.cookies()
526
+ url, title = "", ""
527
+ if page is not None:
528
+ try:
529
+ url = page.evaluate("location.href") or "" # live, never the cached page.url
530
+ title = page.title() or ""
531
+ except Exception:
532
+ pass
533
+ # Still on a login form: definitely not logged in yet (skip the probe).
534
+ low = (url + " " + title).lower()
535
+ if "统一身份认证" in title or "用户登录" in title or "cas login" in low or "/login" in url.lower():
536
+ _log_state(url, title, cookies, "on login page")
537
+ return False
538
+ ok, landed = _probe(cookies)
539
+ _log_state(url, title, cookies, landed, force=ok)
540
+ return ok
541
+ except Exception as exc:
542
+ log.info(f" [WebVPN] detect error: {exc}")
543
+ return False
544
+
545
+ def _log_state(url: str, title: str, cookies: list[dict[str, Any]], probe: str, force: bool = False) -> None:
546
+ now = time.monotonic()
547
+ if force or now - state["last_log"] >= 10:
548
+ state["last_log"] = now
549
+ names = sorted({c.get("name", "") for c in cookies if base_host in (c.get("domain") or "")})
550
+ log.info(f" [WebVPN] url={url[:70]} title={title[:30]!r} vpn_cookies={names} probe={probe[:70]}")
551
+
552
+ return open_login_browser(base, config, cookie_file=cookie_file, detect_login=_detect, max_wait=600)
553
+
554
+
555
+ def carsi_login(publisher: str, config: dict[str, Any], *, login_url: str, domains: list[str]) -> bool:
556
+ """Login to CARSI federated authentication via stealth browser."""
557
+ from ..config import DATA_DIR
558
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache"))) / "carsi_cookies"
559
+ cache_dir.mkdir(parents=True, exist_ok=True)
560
+ cookie_file = cache_dir / f"{publisher}.json"
561
+
562
+ def _detect(context: Any, page: Any) -> bool:
563
+ try:
564
+ current_url = page.url
565
+ on_publisher = any(d in current_url for d in domains)
566
+ on_login = any(x in current_url.lower() for x in ("login", "institutional", "wayf", "saml", "cas", "idp"))
567
+ if not on_publisher or on_login:
568
+ return False
569
+ # The URL gate alone false-positives: some publishers bounce the
570
+ # unauthenticated visitor to their homepage, which passes the URL
571
+ # check before the user has seen the IdP (#29). Require a
572
+ # federated-auth artifact — completing CARSI always leaves a
573
+ # Shibboleth/SAML session cookie or an IdP-domain (.edu.cn)
574
+ # cookie in the context.
575
+ for c in context.cookies():
576
+ name = (c.get("name") or "").lower()
577
+ dom = (c.get("domain") or "").lower()
578
+ if "shibsession" in name or "saml" in name or "idp" in dom or dom.endswith(".edu.cn"):
579
+ return True
580
+ return False
581
+ except Exception:
582
+ return False
583
+
584
+ return open_login_browser(
585
+ login_url,
586
+ config,
587
+ cookie_file=cookie_file,
588
+ detect_login=_detect,
589
+ max_wait=600, # IdP + 2FA is human-paced; 180s wastes the run (#62)
590
+ )
591
+
592
+
593
+ def ezproxy_login(config: dict[str, Any]) -> bool:
594
+ """Login to EZProxy via stealth browser."""
595
+ from ..sources.ezproxy import _ezproxy_origin, _is_proxy_domain
596
+
597
+ origin = _ezproxy_origin(config)
598
+ if not origin:
599
+ log.info(" [EZProxy] No ezproxy_login_url configured")
600
+ return False
601
+
602
+ from ..config import DATA_DIR
603
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
604
+ cookie_file = cache_dir / "ezproxy_cookies.json"
605
+
606
+ # Land on the bare /login endpoint: it is the only URL that reliably
607
+ # challenges. /login?url=<target> skips straight to the raw publisher on
608
+ # hostname-rewriting deployments and never authenticates (#62).
609
+ login_url = f"{origin}/login"
610
+
611
+ def _detect(context: Any, page: Any) -> bool:
612
+ try:
613
+ parsed = urlparse(page.url)
614
+ path_q = f"{parsed.path}?{parsed.query}".lower()
615
+ if "/login" in path_q or "wayf" in path_q or "shibboleth" in path_q:
616
+ return False
617
+ # The authentication signal is the proxy's own session cookie —
618
+ # not the URL shape. "URL is not a login page" used to pass the
619
+ # moment an unauthenticated visitor was redirected to the raw
620
+ # publisher, saving 17 publisher cookies and zero proxy
621
+ # cookies as a "successful" login (#62).
622
+ for c in context.cookies():
623
+ if _is_proxy_domain(c.get("domain") or ""):
624
+ return True
625
+ host = (parsed.hostname or "").lower()
626
+ return _is_proxy_domain(host)
627
+ except Exception:
628
+ return False
629
+
630
+ return open_login_browser(
631
+ login_url,
632
+ config,
633
+ cookie_file=cookie_file,
634
+ detect_login=_detect,
635
+ max_wait=600, # SSO + 2FA is human-paced; 180s wastes the run (#62)
636
+ )