litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,965 @@
1
+ """Browser engine: CloakBrowser-based replacement for camofox daemon API.
2
+
3
+ Provides the same public API as the old camofox.py (is_available, solve_url,
4
+ get_cookies, get_html, import_cookies, evaluate_js, create_tab, close_tab,
5
+ navigate_tab, get_snapshot, download_pdf_via_browser, fetch_url,
6
+ get_captured_responses, close_all_tabs) but uses CloakBrowser's direct
7
+ Playwright API instead of HTTP calls to an external daemon.
8
+
9
+ Uses a single shared browser instance with multiple tabs (pages). Cookies,
10
+ login sessions, and Cloudflare bypass state are shared across all tabs.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import base64
16
+ import json
17
+ import logging
18
+ import os
19
+ import re
20
+ import shutil
21
+ import subprocess
22
+ import time
23
+ import uuid
24
+ from pathlib import Path
25
+ from typing import Any
26
+ from urllib.parse import urlparse
27
+
28
+ logger = logging.getLogger(__name__)
29
+
30
+ # ---------------------------------------------------------------------------
31
+ # Browser backend (patchright default, CloakBrowser opt-in fallback)
32
+ # ---------------------------------------------------------------------------
33
+ # patchright is the Apache-2.0 open-source Playwright fork that patches the
34
+ # detectable automation fingerprints and drives the local Chrome (channel).
35
+ # CloakBrowser's free tier is pinned to Chromium 146 (newer 148/150 are
36
+ # Pro-only), so it remains as an opt-in fallback via the `browser_backend`
37
+ # config key. Local browser detection / CLOAKBROWSER_BINARY_PATH logic for
38
+ # the CloakBrowser backend lives in browser_backend.py.
39
+ # ---------------------------------------------------------------------------
40
+
41
+ from .backend import ( # noqa: E402
42
+ BACKEND_CLOAKBROWSER,
43
+ BACKEND_PATCHRIGHT,
44
+ find_local_browser, # noqa: F401 (re-exported for CLI diagnostics)
45
+ launch,
46
+ launch_persistent_context,
47
+ resolve_backend,
48
+ resolve_browser_binary, # noqa: F401 (re-exported for CLI diagnostics)
49
+ )
50
+
51
+ # ---------------------------------------------------------------------------
52
+ # Browser backend availability check
53
+ # ---------------------------------------------------------------------------
54
+
55
+ _HAS_BROWSER_BACKEND: bool | None = None
56
+
57
+
58
+ def _check_browser_backend(config: dict[str, Any] | None = None) -> bool:
59
+ """Check whether the resolved backend is importable (cached per backend)."""
60
+ from .backend import (
61
+ BACKEND_CAMOUFOX,
62
+ BACKEND_CLOAKBROWSER,
63
+ BACKEND_PATCHRIGHT,
64
+ is_available,
65
+ resolve_backend,
66
+ )
67
+
68
+ backend = resolve_backend(config)
69
+ if backend == BACKEND_CLOAKBROWSER:
70
+ return is_available(BACKEND_CLOAKBROWSER)
71
+ if backend == BACKEND_CAMOUFOX:
72
+ # A missing camoufox must not be masked by (or masked as) a patchright
73
+ # check — resolve_backend already guarantees it is the real selection.
74
+ return is_available(BACKEND_CAMOUFOX)
75
+ global _HAS_BROWSER_BACKEND
76
+ if _HAS_BROWSER_BACKEND is None:
77
+ _HAS_BROWSER_BACKEND = is_available(BACKEND_PATCHRIGHT)
78
+ return _HAS_BROWSER_BACKEND
79
+
80
+
81
+ # ---------------------------------------------------------------------------
82
+ # Per-thread browser (one browser per thread, multiple tabs per browser)
83
+ # Playwright sync API uses greenlets bound to threads — cross-thread
84
+ # usage crashes. Each thread gets its own browser; within a thread,
85
+ # tabs share cookies/session/Cloudflare state.
86
+ # ---------------------------------------------------------------------------
87
+
88
+ import threading as _threading
89
+
90
+ _tls = _threading.local()
91
+
92
+ # All live browsers across threads. Playwright sync objects are thread-affine
93
+ # (greenlets), so a graceful cross-thread close at process exit may fail —
94
+ # the reaper is best-effort and the driver's own EOF kill is the last resort.
95
+ _LIVE_BROWSERS: set = set()
96
+ _LIVE_BROWSERS_LOCK = _threading.Lock()
97
+
98
+
99
+ def _register_browser(browser: Any) -> None:
100
+ with _LIVE_BROWSERS_LOCK:
101
+ _LIVE_BROWSERS.add(browser)
102
+
103
+
104
+ def _unregister_browser(browser: Any) -> None:
105
+ with _LIVE_BROWSERS_LOCK:
106
+ _LIVE_BROWSERS.discard(browser)
107
+
108
+
109
+ def _tree_kill(proc: Any) -> None:
110
+ """Force-kill a driver process and its whole child tree (Windows-safe).
111
+
112
+ Accepts both handle shapes the transport hands back: subprocess.Popen
113
+ (``poll()``) and multiprocessing.Process (``is_alive()`` — Playwright's
114
+ driver since 1.x). Assuming Popen made the tree-kill backstop raise
115
+ AttributeError inside its callers' ``except Exception: pass`` — the
116
+ backstop never ran and long-lived MCP servers accumulated Chromium
117
+ processes (issue #57).
118
+ """
119
+ if proc is None:
120
+ return
121
+ try:
122
+ if hasattr(proc, "poll"):
123
+ alive = proc.poll() is None
124
+ elif hasattr(proc, "is_alive"):
125
+ alive = proc.is_alive()
126
+ else:
127
+ alive = True
128
+ except Exception:
129
+ alive = True # can't tell — assume alive and force-kill
130
+ if not alive:
131
+ return
132
+ pid = getattr(proc, "pid", None)
133
+ if not pid:
134
+ return
135
+ if os.name == "nt":
136
+ subprocess.run(
137
+ ["taskkill", "/F", "/T", "/PID", str(pid)],
138
+ capture_output=True, timeout=15,
139
+ )
140
+ else:
141
+ try:
142
+ proc.kill()
143
+ except Exception:
144
+ try:
145
+ os.kill(pid, 9)
146
+ except Exception:
147
+ pass
148
+
149
+
150
+ def _reap_browsers_at_exit() -> None:
151
+ """Close every live browser when the process exits.
152
+
153
+ This is what allows callers to keep a browser alive across an entire
154
+ batch (no per-paper close churn) without leaking orphan Chromium
155
+ processes (issue #19).
156
+ """
157
+ for browser in list(_LIVE_BROWSERS):
158
+ _unregister_browser(browser)
159
+ proc = None
160
+ try:
161
+ proc = browser._impl_obj._connection._transport._proc # type: ignore[attr-defined]
162
+ except Exception:
163
+ proc = None
164
+ try:
165
+ browser.close()
166
+ except Exception:
167
+ pass
168
+ # Verify the driver process actually died. A "successful" close can
169
+ # still leave the node/chromium tree alive (Windows propagates no
170
+ # signal to children), and a failed close means the owning thread is
171
+ # gone. Either way, tree-kill is the definitive backstop — without it
172
+ # every batch leaks its pooled browsers (issue #19).
173
+ try:
174
+ _tree_kill(proc)
175
+ except Exception:
176
+ pass
177
+
178
+
179
+ import atexit as _atexit
180
+
181
+ _atexit.register(_reap_browsers_at_exit)
182
+
183
+
184
+ def _build_browser_args(config: dict[str, Any] | None = None) -> list[str]:
185
+ """Build Chromium launch args from config (proxy, flags, etc.)."""
186
+ args = ["--disable-features=CrossOriginOpenerPolicy"]
187
+ if config:
188
+ proxy = config.get("browser_static_proxy", "")
189
+ if proxy:
190
+ args.append(f"--proxy-server={proxy}")
191
+ return args
192
+
193
+
194
+ def close_shared_browser(config: dict[str, Any] | None = None) -> None:
195
+ """Tear down THIS thread's shared browser (driver + loop included).
196
+
197
+ Needed before opening a VISIBLE browser on the same thread: the shared
198
+ sync-API loop would otherwise reject the second launch ("Playwright Sync
199
+ API inside the asyncio loop"). The shared browser relaunches lazily on
200
+ the next headless use.
201
+ """
202
+ browser = getattr(_tls, "browser", None)
203
+ if browser is None:
204
+ return
205
+ try:
206
+ browser.close()
207
+ except Exception:
208
+ pass
209
+ try:
210
+ _unregister_browser(browser)
211
+ except Exception:
212
+ pass
213
+ _tls.browser = None
214
+ _tls.context = None
215
+
216
+
217
+ def _get_shared_browser(config: dict[str, Any] | None = None):
218
+ """Get or create a browser for the current thread. Returns (browser, context)."""
219
+ browser = getattr(_tls, "browser", None)
220
+ context = getattr(_tls, "context", None)
221
+ if browser is not None:
222
+ # The browser lives as long as the thread — possibly a whole batch.
223
+ # Replace a crashed one instead of failing every remaining paper.
224
+ try:
225
+ connected = browser.is_connected()
226
+ except Exception:
227
+ connected = False
228
+ if connected:
229
+ return browser, context
230
+ # Full teardown: browser.close() also stops the Playwright driver and
231
+ # its asyncio loop. A half-alive driver would otherwise break the
232
+ # fresh launch below ("Sync API inside an asyncio event loop").
233
+ try:
234
+ browser.close()
235
+ except Exception:
236
+ pass
237
+ _unregister_browser(browser)
238
+ _tls.browser = None
239
+ _tls.context = None
240
+
241
+ # Playwright's Sync API cannot serve a *user* asyncio loop, but the loop
242
+ # the sync API itself leaves running in our worker threads is fine — see
243
+ # is_playwright_owned_loop(). Bail out only for foreign loops.
244
+ try:
245
+ import asyncio
246
+ loop = asyncio.get_running_loop()
247
+ except RuntimeError:
248
+ loop = None
249
+ if loop is not None and not is_playwright_owned_loop(loop):
250
+ raise RuntimeError(
251
+ "Browser (Playwright Sync API) cannot run inside an asyncio "
252
+ "event loop. Use the HTTP download sources instead, or call from "
253
+ "a synchronous context."
254
+ )
255
+
256
+ if not _check_browser_backend(config):
257
+ raise RuntimeError(
258
+ "no browser backend available. Run: pip install patchright (or pip install cloakbrowser)"
259
+ )
260
+
261
+ backend = resolve_backend(config)
262
+
263
+ # Platform compat shim + kernel override only apply to CloakBrowser
264
+ if backend == BACKEND_CLOAKBROWSER:
265
+ try:
266
+ from .cloakbrowser_compat import ensure_cloakbrowser_platform_compatible
267
+ ensure_cloakbrowser_platform_compatible()
268
+ except Exception:
269
+ logger.debug("CloakBrowser platform shim not applied", exc_info=True)
270
+ binary = resolve_browser_binary(config)
271
+ if binary:
272
+ os.environ["CLOAKBROWSER_BINARY_PATH"] = binary
273
+
274
+ # scihub_browser_headless overrides browser_headless for THIS shared
275
+ # browser (the grey-source racing engine). It exists because the racing
276
+ # window otherwise flashes popup/redirect attempts from sci-hub mirrors
277
+ # at the taskbar; institutional logins keep their own visible windows
278
+ # via get_persistent_context / auth.py, which still honor browser_headless.
279
+ headless = False
280
+ humanize = True
281
+ if config:
282
+ headless = bool(
283
+ config.get("scihub_browser_headless", config.get("browser_headless", False))
284
+ )
285
+ humanize = config.get("browser_humanize", True)
286
+
287
+ args = _build_browser_args(config)
288
+ browser = launch(headless=headless, humanize=humanize, args=args, config=config)
289
+ _register_browser(browser)
290
+ context = browser.new_context()
291
+
292
+ # Launching the sync API leaves its dispatcher event loop "running" in
293
+ # this thread (asyncio.get_running_loop() now succeeds here). Register it
294
+ # so is_playwright_owned_loop() can tell OUR worker-thread loop apart from
295
+ # a user's asyncio loop — the latter is the only case sync API can't serve.
296
+ try:
297
+ import asyncio as _asyncio
298
+ _tls.owned_loop = _asyncio.get_running_loop()
299
+ except RuntimeError:
300
+ _tls.owned_loop = None
301
+ _tls.browser = browser
302
+ _tls.context = context
303
+ logger.info(f"browser_engine: browser ready for thread {_threading.current_thread().name}")
304
+ return browser, context
305
+
306
+
307
+ def get_persistent_context(
308
+ profile_dir: str | Path,
309
+ config: dict[str, Any] | None = None,
310
+ ):
311
+ """Get or create a persistent browser context for fingerprint consistency.
312
+
313
+ Unlike launch() + cookie restore, persistent context preserves:
314
+ - Browser fingerprint (canvas, WebGL, audio, fonts)
315
+ - Cookies and localStorage across restarts
316
+ - Login sessions without re-authentication
317
+
318
+ This is the recommended approach for publisher sessions that need
319
+ stable identity across multiple download runs.
320
+ """
321
+ if not _check_browser_backend(config):
322
+ raise RuntimeError(
323
+ "no browser backend available. Run: pip install patchright (or pip install cloakbrowser)"
324
+ )
325
+
326
+ backend = resolve_backend(config)
327
+
328
+ if backend == BACKEND_CLOAKBROWSER:
329
+ try:
330
+ from .cloakbrowser_compat import prepare_cloakbrowser_runtime
331
+ prepare_cloakbrowser_runtime()
332
+ except Exception:
333
+ pass
334
+ binary = resolve_browser_binary(config)
335
+ if binary:
336
+ os.environ["CLOAKBROWSER_BINARY_PATH"] = binary
337
+
338
+ headless = False
339
+ humanize = True
340
+ if config:
341
+ headless = config.get("browser_headless", False)
342
+ humanize = config.get("browser_humanize", True)
343
+
344
+ args = _build_browser_args(config)
345
+ profile_path = Path(profile_dir)
346
+ profile_path.mkdir(parents=True, exist_ok=True)
347
+
348
+ ctx = launch_persistent_context(
349
+ str(profile_path),
350
+ headless=headless,
351
+ humanize=humanize,
352
+ args=args,
353
+ config=config,
354
+ )
355
+ logger.info(f"browser_engine: persistent context ready at {profile_path}")
356
+ return ctx
357
+
358
+
359
+ def get_browser_page(config: dict[str, Any] | None = None):
360
+ """Get a new page from the shared browser (for custom browser interactions).
361
+
362
+ Returns a Playwright Page object that the caller must close after use.
363
+ Returns None if the browser is not available.
364
+ """
365
+ if not _check_browser_backend(config):
366
+ return None
367
+ try:
368
+ _browser, context = _get_shared_browser(config)
369
+ return context.new_page()
370
+ except Exception:
371
+ return None
372
+
373
+
374
+ def is_playwright_owned_loop(loop: Any) -> bool:
375
+ """True when `loop` is the sync-API dispatcher loop of THIS thread's browser.
376
+
377
+ A running asyncio loop normally means "user asyncio code — sync Playwright
378
+ forbidden". Exception: our persistent worker threads, where the loop was
379
+ created by the sync API itself and must not disqualify the thread.
380
+ """
381
+ return getattr(_tls, "owned_loop", None) is loop and loop is not None
382
+
383
+
384
+ def shutdown_shared_browser():
385
+ """Shut down the current thread's browser. Call on thread exit or process exit."""
386
+ browser = getattr(_tls, "browser", None)
387
+ if browser is not None:
388
+ try:
389
+ browser.close()
390
+ except Exception:
391
+ pass
392
+ _unregister_browser(browser)
393
+ _tls.browser = None
394
+ _tls.context = None
395
+ logger.info("browser_engine: browser shut down")
396
+
397
+
398
+ # ---------------------------------------------------------------------------
399
+ # Public API — drop-in replacements for camofox.py functions
400
+ # ---------------------------------------------------------------------------
401
+
402
+ def is_available(config: dict[str, Any] | None = None) -> bool:
403
+ """Check if the resolved browser backend is available (importable)."""
404
+ return _check_browser_backend(config)
405
+
406
+
407
+ # ---------------------------------------------------------------------------
408
+ # Tab registry — maps tab_id → (browser, context, page)
409
+ # Used by create_tab/evaluate_js/navigate_tab/close_tab for sequential
410
+ # tab-based workflows within a single operation (thread-safe: one thread per tab).
411
+ # ---------------------------------------------------------------------------
412
+
413
+ _tabs: dict[str, Any] = {} # tab_id → page
414
+ _captured: dict[str, list] = {} # tab_id → captured PDF responses
415
+
416
+
417
+ def _register_tab(browser, context, page) -> str:
418
+ tab_id = uuid.uuid4().hex[:12]
419
+ _tabs[tab_id] = page
420
+ _captured[tab_id] = []
421
+
422
+ # Listen for PDF responses
423
+ def _on_response(response):
424
+ try:
425
+ ct = response.headers.get("content-type", "")
426
+ if "pdf" in ct or "octet-stream" in ct:
427
+ try:
428
+ body = response.body()
429
+ if body[:5] == b"%PDF-":
430
+ _captured[tab_id].append({
431
+ "url": response.url,
432
+ "status": response.status,
433
+ "contentType": ct,
434
+ "dataBase64": base64.b64encode(body).decode(),
435
+ })
436
+ except Exception:
437
+ pass
438
+ except Exception:
439
+ pass
440
+
441
+ try:
442
+ page.on("response", _on_response)
443
+ except Exception:
444
+ pass
445
+
446
+ return tab_id
447
+
448
+
449
+ def _resolve_tab(tab_id: str):
450
+ """Look up page for a tab_id. Returns page or None."""
451
+ page = _tabs.get(tab_id)
452
+ return page
453
+
454
+
455
+ def solve_url(
456
+ url: str,
457
+ config: dict[str, Any],
458
+ *,
459
+ max_timeout: int = 60000,
460
+ ) -> dict[str, Any] | None:
461
+ """Fetch URL via CloakBrowser shared browser. Returns dict with status/solution keys."""
462
+ page = None
463
+ try:
464
+ _, context = _get_shared_browser(config)
465
+ page = context.new_page()
466
+ page.goto(url, wait_until="domcontentloaded", timeout=int(max_timeout))
467
+
468
+ raw_cookies = context.cookies()
469
+ cookies = [{"name": c["name"], "value": c.get("value", "")} for c in raw_cookies if c.get("name")]
470
+
471
+ html = ""
472
+ try:
473
+ html = page.content()
474
+ except Exception as e:
475
+ logger.info(f"browser_engine: HTML extraction failed: {e}")
476
+
477
+ final_url = page.url
478
+ logger.info(f"browser_engine: ok, final_url={final_url}, html_len={len(html)}")
479
+ return {
480
+ "status": "ok",
481
+ "solution": {
482
+ "url": final_url,
483
+ "status": 200,
484
+ "response": html,
485
+ "cookies": cookies,
486
+ },
487
+ }
488
+ except Exception as e:
489
+ logger.info(f"browser_engine: error - {e}")
490
+ return None
491
+ finally:
492
+ if page:
493
+ try:
494
+ page.close()
495
+ except Exception:
496
+ pass
497
+
498
+
499
+ def get_html(
500
+ url: str,
501
+ config: dict[str, Any],
502
+ *,
503
+ max_timeout: int = 60000,
504
+ ) -> str | None:
505
+ """Solve and return page HTML."""
506
+ result = solve_url(url, config, max_timeout=max_timeout)
507
+ if not result:
508
+ return None
509
+ return result.get("solution", {}).get("response")
510
+
511
+
512
+ def import_cookies(cookie_file: str | Path, config: dict[str, Any], *, domain_suffix: str | None = None) -> int:
513
+ """Import Netscape-format cookies into the shared browser context. Returns count imported."""
514
+ try:
515
+ text = Path(cookie_file).read_text(encoding="utf-8")
516
+ except Exception as e:
517
+ logger.info(f"browser_engine: failed to read cookie file: {e}")
518
+ return 0
519
+ cookies = _parse_netscape_cookies(text)
520
+ # Cloudflare clearance/bot cookies are bound to the browser fingerprint and IP
521
+ # they were issued to; replaying stale ones into another session triggers
522
+ # fresh challenges instead of skipping them.
523
+ cookies = [c for c in cookies if not (c.get("name", "") in ("cf_clearance", "__cf_bm", "_cfuvid")
524
+ or c.get("name", "").startswith("cf_chl"))]
525
+ if not cookies:
526
+ return 0
527
+ if domain_suffix:
528
+ cookies = [c for c in cookies if domain_suffix in c.get("domain", "")]
529
+ try:
530
+ _, ctx = _get_shared_browser(config)
531
+ ctx.add_cookies(cookies)
532
+ logger.info(f"browser_engine: imported {len(cookies)} cookies")
533
+ return len(cookies)
534
+ except Exception as e:
535
+ logger.info(f"browser_engine: import_cookies error: {e}")
536
+ return 0
537
+
538
+
539
+ def evaluate_js(
540
+ tab_id: str,
541
+ expression: str,
542
+ config: dict[str, Any],
543
+ *,
544
+ timeout: float = 15.0,
545
+ ) -> Any:
546
+ """Evaluate JavaScript expression in a browser page. Returns the JS result."""
547
+ page = _resolve_tab(tab_id)
548
+ if not page:
549
+ logger.info(f"browser_engine: evaluate_js - tab {tab_id} not found")
550
+ return None
551
+ try:
552
+ return page.evaluate(expression)
553
+ except Exception as e:
554
+ logger.info(f"browser_engine: evaluate_js error: {e}")
555
+ return None
556
+
557
+
558
+ def create_tab(url: str, config: dict[str, Any], *, timeout: float = 30.0) -> str | None:
559
+ """Create a new tab (page) in the shared browser and navigate to URL. Returns tab_id or None."""
560
+ try:
561
+ browser, context = _get_shared_browser(config)
562
+ page = context.new_page()
563
+ page.goto(url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
564
+ tab_id = _register_tab(browser, context, page)
565
+ return tab_id
566
+ except Exception as e:
567
+ logger.info(f"browser_engine: create_tab failed - {e}")
568
+ return None
569
+
570
+
571
+ def close_tab(tab_id: str, config: dict[str, Any]) -> None:
572
+ """Close a browser tab (page only, not the shared browser)."""
573
+ page = _resolve_tab(tab_id)
574
+ if page:
575
+ try:
576
+ page.close()
577
+ except Exception:
578
+ pass
579
+ _tabs.pop(tab_id, None)
580
+ _captured.pop(tab_id, None)
581
+
582
+
583
+ def navigate_tab(tab_id: str, url: str, config: dict[str, Any], *, timeout: float = 30.0) -> bool:
584
+ """Navigate an existing page to a new URL."""
585
+ page = _resolve_tab(tab_id)
586
+ if not page:
587
+ logger.info(f"browser_engine: navigate_tab - tab {tab_id} not found")
588
+ return False
589
+ for attempt in (1, 2):
590
+ try:
591
+ page.goto(url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
592
+ return True
593
+ except Exception as e:
594
+ # A connection that never came up (net::ERR_TIMED_OUT/RESET/...) is
595
+ # usually a transient network hiccup: retry once in this same tab
596
+ # rather than failing the source and opening a fallback window.
597
+ transient = any(code in str(e) for code in _TRANSIENT_NET_ERRORS)
598
+ logger.info(f"browser_engine: navigate failed - {str(e).splitlines()[0]}"
599
+ + (" (retrying)" if transient and attempt == 1 else ""))
600
+ if not (transient and attempt == 1):
601
+ return False
602
+ return False
603
+
604
+
605
+ _TRANSIENT_NET_ERRORS = (
606
+ "net::ERR_TIMED_OUT", "net::ERR_CONNECTION_TIMED_OUT", "net::ERR_CONNECTION_RESET",
607
+ "net::ERR_CONNECTION_CLOSED", "net::ERR_NETWORK_CHANGED", "net::ERR_EMPTY_RESPONSE",
608
+ )
609
+
610
+
611
+ def download_pdf_via_browser(
612
+ pdf_url: str,
613
+ output_path: Path,
614
+ config: dict[str, Any],
615
+ *,
616
+ timeout: float = 60.0,
617
+ ) -> bool:
618
+ """Download a PDF URL via CloakBrowser shared browser. Returns True on success.
619
+
620
+ 4-strategy cascade:
621
+ 1. Network response capture (from page.on("response"))
622
+ 2. In-browser fetch API with credentials
623
+ 3. PDF link discovery in page DOM
624
+ 4. Download button click
625
+ """
626
+ page = None
627
+ try:
628
+ # Fragments (#view=FitH...) are never sent to servers but they break
629
+ # .pdf suffix checks and _is_pdf_url — strip before anything else.
630
+ pdf_url = str(pdf_url).split("#", 1)[0]
631
+ _, context = _get_shared_browser(config)
632
+ page = context.new_page()
633
+
634
+ # Set up response listener for PDF captures
635
+ captured_responses: list[dict] = []
636
+
637
+ def _on_response(response):
638
+ try:
639
+ ct = response.headers.get("content-type", "")
640
+ if "pdf" in ct or "octet-stream" in ct:
641
+ try:
642
+ body = response.body()
643
+ if body[:5] == b"%PDF-":
644
+ captured_responses.append({
645
+ "url": response.url,
646
+ "dataBase64": base64.b64encode(body).decode(),
647
+ })
648
+ except Exception:
649
+ pass
650
+ except Exception:
651
+ pass
652
+
653
+ try:
654
+ page.on("response", _on_response)
655
+ except Exception:
656
+ pass
657
+
658
+ page.goto(pdf_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
659
+ time.sleep(3)
660
+
661
+ # Check for anti-bot challenges
662
+ html = ""
663
+ try:
664
+ html = page.content()
665
+ except Exception:
666
+ pass
667
+ lower_html = html.lower()
668
+ if any(sig in lower_html for sig in [
669
+ "cf-browser-verification", "challenge-platform",
670
+ "just a moment", "attention required",
671
+ "security check", "captcha",
672
+ "请稍候", "正在验证", "checking your browser",
673
+ # ALTCHA anti-bot verification (used by sci-hub.ru and other Sci-Hub mirrors)
674
+ "altcha", "你是机器人吗", "not a robot", "nope",
675
+ ]):
676
+ logger.info("browser_engine: anti-bot challenge detected, waiting...")
677
+ time.sleep(10)
678
+
679
+ current_url = page.url
680
+
681
+ # Strategy 0: Network response capture
682
+ for resp in captured_responses:
683
+ data = resp.get("dataBase64", "")
684
+ if data:
685
+ pdf_bytes = base64.b64decode(data)
686
+ if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
687
+ output_path.parent.mkdir(parents=True, exist_ok=True)
688
+ output_path.write_bytes(pdf_bytes)
689
+ logger.info(f"browser_engine: downloaded {len(pdf_bytes)} bytes via network capture")
690
+ return True
691
+
692
+ # Build candidate fetch paths
693
+ fetch_paths: list[str] = []
694
+ parsed = urlparse(str(current_url))
695
+ origin = f"{parsed.scheme}://{parsed.netloc}"
696
+
697
+ # Extract DOI from URL for pdfdirect construction
698
+ doi_from_url = None
699
+ path = parsed.path
700
+ # Longest first: "/doi/" also matches "/doi/pdfdirect/10.1111/x", which
701
+ # made "pdfdirect/10.1111/x" the DOI (2026-10-18, Wiley 403 x3).
702
+ for prefix in ["/doi/pdfdirect/", "/doi/pdf/", "/doi/epdf/", "/doi/", "/articles/"]:
703
+ if prefix in path:
704
+ doi_from_url = path.split(prefix)[-1].split("?")[0].split("#")[0]
705
+ break
706
+
707
+ if doi_from_url:
708
+ fetch_paths.append(f"/doi/pdfdirect/{doi_from_url}")
709
+ fetch_paths.append(f"/doi/pdf/{doi_from_url}")
710
+ fetch_paths.append(f"/content/pdf/{doi_from_url}.pdf")
711
+
712
+ if _is_pdf_url(pdf_url):
713
+ parsed_orig = urlparse(pdf_url)
714
+ if parsed_orig.netloc == parsed.netloc:
715
+ fetch_paths.append(parsed_orig.path)
716
+
717
+ # Strategy 1: In-browser fetch API
718
+ for fetch_path in fetch_paths:
719
+ logger.info(f"browser_engine: trying in-browser fetch {origin}{fetch_path[:60]}")
720
+ try:
721
+ pdf_b64 = page.evaluate(f"""
722
+ (async () => {{
723
+ try {{
724
+ const resp = await fetch('{fetch_path}', {{
725
+ credentials: 'include',
726
+ headers: {{'Accept': 'application/pdf,*/*'}}
727
+ }});
728
+ if (!resp.ok) return 'status:' + resp.status;
729
+ const ct = resp.headers.get('content-type') || '';
730
+ if (!ct.includes('pdf') && !ct.includes('octet')) return 'ct:' + ct;
731
+ const blob = await resp.blob();
732
+ return new Promise((resolve) => {{
733
+ const reader = new FileReader();
734
+ reader.onload = () => resolve(reader.result);
735
+ reader.readAsDataURL(blob);
736
+ }});
737
+ }} catch(e) {{
738
+ return 'error:' + e.message;
739
+ }}
740
+ }})()
741
+ """)
742
+
743
+ if isinstance(pdf_b64, str) and pdf_b64.startswith("data:"):
744
+ header, data = pdf_b64.split(",", 1)
745
+ pdf_bytes = base64.b64decode(data)
746
+ if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
747
+ output_path.parent.mkdir(parents=True, exist_ok=True)
748
+ output_path.write_bytes(pdf_bytes)
749
+ logger.info(f"browser_engine: downloaded {len(pdf_bytes)} bytes via in-browser fetch")
750
+ return True
751
+ else:
752
+ logger.info(f"browser_engine: fetch returned non-PDF ({len(pdf_bytes)} bytes)")
753
+ else:
754
+ logger.info(f"browser_engine: fetch result: {str(pdf_b64)[:80]}")
755
+ except Exception as e:
756
+ logger.info(f"browser_engine: fetch error: {e}")
757
+
758
+ # Strategy 2: PDF link discovery in DOM
759
+ try:
760
+ pdf_link = page.evaluate("""
761
+ (() => {
762
+ for (const el of document.querySelectorAll('iframe, embed, object')) {
763
+ const src = el.src || el.data || '';
764
+ if (src.includes('.pdf') && !src.includes('supplement') && !src.includes('Suppl')) return src;
765
+ }
766
+ const viewer = document.querySelector('#viewer, .pdfViewer, [data-l10n-id="download"]');
767
+ if (viewer) return window.location.href;
768
+ for (const a of document.querySelectorAll('a[href]')) {
769
+ const href = (a.href || '').toLowerCase();
770
+ const text = (a.innerText || '').toLowerCase();
771
+ if (href.includes('supplement') || href.includes('supporting') || href.includes('downloadsupplement') || href.includes('pb-assets')) continue;
772
+ if (text.includes('supplement') || text.includes('supporting info')) continue;
773
+ if (href.includes('.pdf') || href.includes('/pdf/') || href.includes('pdfdirect')) {
774
+ if (a.href.startsWith('http')) return a.href;
775
+ }
776
+ }
777
+ const sdPdf = document.querySelector('a[aria-label*="PDF"], a[aria-label*="pdf"], a.pdf-download-btn-link');
778
+ if (sdPdf) return sdPdf.href;
779
+ return null;
780
+ })()
781
+ """)
782
+
783
+ if pdf_link and isinstance(pdf_link, str) and pdf_link.startswith("http"):
784
+ logger.info(f"browser_engine: found PDF link: {pdf_link[:80]}")
785
+ parsed_link = urlparse(pdf_link)
786
+ link_path = parsed_link.path + ("?" + parsed_link.query if parsed_link.query else "")
787
+ if parsed_link.netloc == parsed.netloc:
788
+ try:
789
+ pdf_b64 = page.evaluate(f"""
790
+ (async () => {{
791
+ try {{
792
+ const resp = await fetch('{link_path}', {{
793
+ credentials: 'include',
794
+ headers: {{'Accept': 'application/pdf,*/*'}}
795
+ }});
796
+ if (!resp.ok) return 'status:' + resp.status;
797
+ const ct = resp.headers.get('content-type') || '';
798
+ if (!ct.includes('pdf') && !ct.includes('octet')) return 'ct:' + ct;
799
+ const blob = await resp.blob();
800
+ return new Promise((resolve) => {{
801
+ const reader = new FileReader();
802
+ reader.onload = () => resolve(reader.result);
803
+ reader.readAsDataURL(blob);
804
+ }});
805
+ }} catch(e) {{
806
+ return 'error:' + e.message;
807
+ }}
808
+ }})()
809
+ """)
810
+
811
+ if isinstance(pdf_b64, str) and pdf_b64.startswith("data:"):
812
+ header, data = pdf_b64.split(",", 1)
813
+ pdf_bytes = base64.b64decode(data)
814
+ if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
815
+ output_path.parent.mkdir(parents=True, exist_ok=True)
816
+ output_path.write_bytes(pdf_bytes)
817
+ logger.info(f"browser_engine: downloaded {len(pdf_bytes)} bytes via PDF link fetch")
818
+ return True
819
+ except Exception as e:
820
+ logger.info(f"browser_engine: PDF link fetch error: {e}")
821
+ except Exception as e:
822
+ logger.info(f"browser_engine: DOM scan error: {e}")
823
+
824
+ # Strategy 3: Click download button
825
+ try:
826
+ clicked = page.evaluate("""
827
+ (() => {
828
+ for (const btn of document.querySelectorAll(
829
+ '#download, [aria-label*="download" i], [aria-label*="PDF" i], .pdf-download-btn-link, a[data-aa-name="download-pdf"]'
830
+ )) {
831
+ if (btn.offsetParent !== null) { btn.click(); return true; }
832
+ }
833
+ return false;
834
+ })()
835
+ """)
836
+ if clicked:
837
+ logger.info("browser_engine: clicked download button, waiting...")
838
+ time.sleep(5)
839
+ except Exception as e:
840
+ logger.info(f"browser_engine: click error: {e}")
841
+
842
+ return False
843
+ except Exception as e:
844
+ logger.info(f"browser_engine: download_pdf_via_browser error: {e}")
845
+ return False
846
+ finally:
847
+ if page:
848
+ try:
849
+ page.close()
850
+ except Exception:
851
+ pass
852
+
853
+
854
+ # Backward-compat alias
855
+
856
+
857
+ def _is_pdf_url(url: str) -> bool:
858
+ """Check if a URL looks like a direct PDF link."""
859
+ lower = str(url).split("#", 1)[0].lower()
860
+ return (
861
+ lower.endswith(".pdf")
862
+ or "/pdf/" in lower
863
+ or "content/pdf" in lower
864
+ or "pdfdirect" in lower
865
+ or "/doi/pdf/" in lower
866
+ or "type=printable" in lower
867
+ )
868
+
869
+
870
+ def fetch_url(
871
+ tab_id: str,
872
+ url: str,
873
+ config: dict[str, Any],
874
+ *,
875
+ timeout: float = 30.0,
876
+ ) -> dict[str, Any] | None:
877
+ """Navigate tab to URL and capture any PDF responses from the network layer."""
878
+ page = _resolve_tab(tab_id)
879
+ if not page:
880
+ logger.info(f"browser_engine: fetch_url - tab {tab_id} not found")
881
+ return None
882
+
883
+ # The inline PDF viewer hides the body from the response listener; read it
884
+ # from the paused response via CDP while the browser itself makes the request.
885
+ from .pdf import capture_pdf_via_cdp
886
+ pdf_bytes = capture_pdf_via_cdp(page.context, page, url, timeout_s=max(timeout, 90.0))
887
+ if pdf_bytes:
888
+ return {"status": "ok", "bytes": len(pdf_bytes), "data": pdf_bytes}
889
+
890
+ try:
891
+ page.goto(url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
892
+ except Exception as e:
893
+ logger.info(f"browser_engine: fetch_url navigate error: {e}")
894
+
895
+ # Check captured responses
896
+ for resp in _captured.get(tab_id, []):
897
+ data = resp.get("dataBase64", "")
898
+ if data:
899
+ pdf_bytes = base64.b64decode(data)
900
+ if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
901
+ return {"status": "ok", "bytes": len(pdf_bytes), "data": pdf_bytes}
902
+
903
+ # Also check direct response body
904
+ try:
905
+ response = page.goto(url, wait_until="commit", timeout=int(timeout * 1000))
906
+ if response is not None:
907
+ body = response.body()
908
+ if body[:5] == b"%PDF-" and len(body) > 5000:
909
+ return {"status": "ok", "bytes": len(body), "data": body}
910
+ except Exception:
911
+ pass
912
+
913
+ return None
914
+
915
+
916
+ def get_captured_responses(
917
+ tab_id: str,
918
+ config: dict[str, Any],
919
+ *,
920
+ consume: bool = True,
921
+ ) -> list[dict[str, Any]]:
922
+ """Get captured PDF responses for a tab. Optionally consume (clear) them."""
923
+ captured = _captured.get(tab_id, [])
924
+ result = list(captured)
925
+ if consume:
926
+ _captured[tab_id] = []
927
+ return result
928
+
929
+
930
+ # ---------------------------------------------------------------------------
931
+ # Netscape cookie parser (shared utility)
932
+ # ---------------------------------------------------------------------------
933
+
934
+ def _parse_netscape_cookies(text: str) -> list[dict[str, Any]]:
935
+ """Parse Netscape-format cookie file into structured cookie objects."""
936
+ cookies: list[dict[str, Any]] = []
937
+ for line in text.splitlines():
938
+ line = line.strip()
939
+ if not line or (line.startswith("#") and not line.startswith("#HttpOnly_")):
940
+ continue
941
+ http_only = False
942
+ if line.startswith("#HttpOnly_"):
943
+ http_only = True
944
+ line = line.removeprefix("#HttpOnly_")
945
+ parts = line.split("\t")
946
+ if len(parts) < 7:
947
+ continue
948
+ domain = parts[0]
949
+ cookie_path = parts[2]
950
+ secure = parts[3].upper() == "TRUE"
951
+ try:
952
+ expires = int(parts[4])
953
+ except (ValueError, IndexError):
954
+ expires = 0
955
+ name = parts[5]
956
+ value = "\t".join(parts[6:])
957
+ cookie: dict[str, Any] = {
958
+ "name": name, "value": value,
959
+ "domain": domain, "path": cookie_path,
960
+ "secure": secure, "expires": expires,
961
+ }
962
+ if http_only:
963
+ cookie["httpOnly"] = True
964
+ cookies.append(cookie)
965
+ return cookies