litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,965 @@
|
|
|
1
|
+
"""Browser engine: CloakBrowser-based replacement for camofox daemon API.
|
|
2
|
+
|
|
3
|
+
Provides the same public API as the old camofox.py (is_available, solve_url,
|
|
4
|
+
get_cookies, get_html, import_cookies, evaluate_js, create_tab, close_tab,
|
|
5
|
+
navigate_tab, get_snapshot, download_pdf_via_browser, fetch_url,
|
|
6
|
+
get_captured_responses, close_all_tabs) but uses CloakBrowser's direct
|
|
7
|
+
Playwright API instead of HTTP calls to an external daemon.
|
|
8
|
+
|
|
9
|
+
Uses a single shared browser instance with multiple tabs (pages). Cookies,
|
|
10
|
+
login sessions, and Cloudflare bypass state are shared across all tabs.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import base64
|
|
16
|
+
import json
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import shutil
|
|
21
|
+
import subprocess
|
|
22
|
+
import time
|
|
23
|
+
import uuid
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
26
|
+
from urllib.parse import urlparse
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
# ---------------------------------------------------------------------------
|
|
31
|
+
# Browser backend (patchright default, CloakBrowser opt-in fallback)
|
|
32
|
+
# ---------------------------------------------------------------------------
|
|
33
|
+
# patchright is the Apache-2.0 open-source Playwright fork that patches the
|
|
34
|
+
# detectable automation fingerprints and drives the local Chrome (channel).
|
|
35
|
+
# CloakBrowser's free tier is pinned to Chromium 146 (newer 148/150 are
|
|
36
|
+
# Pro-only), so it remains as an opt-in fallback via the `browser_backend`
|
|
37
|
+
# config key. Local browser detection / CLOAKBROWSER_BINARY_PATH logic for
|
|
38
|
+
# the CloakBrowser backend lives in browser_backend.py.
|
|
39
|
+
# ---------------------------------------------------------------------------
|
|
40
|
+
|
|
41
|
+
from .backend import ( # noqa: E402
|
|
42
|
+
BACKEND_CLOAKBROWSER,
|
|
43
|
+
BACKEND_PATCHRIGHT,
|
|
44
|
+
find_local_browser, # noqa: F401 (re-exported for CLI diagnostics)
|
|
45
|
+
launch,
|
|
46
|
+
launch_persistent_context,
|
|
47
|
+
resolve_backend,
|
|
48
|
+
resolve_browser_binary, # noqa: F401 (re-exported for CLI diagnostics)
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
# ---------------------------------------------------------------------------
|
|
52
|
+
# Browser backend availability check
|
|
53
|
+
# ---------------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
_HAS_BROWSER_BACKEND: bool | None = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _check_browser_backend(config: dict[str, Any] | None = None) -> bool:
|
|
59
|
+
"""Check whether the resolved backend is importable (cached per backend)."""
|
|
60
|
+
from .backend import (
|
|
61
|
+
BACKEND_CAMOUFOX,
|
|
62
|
+
BACKEND_CLOAKBROWSER,
|
|
63
|
+
BACKEND_PATCHRIGHT,
|
|
64
|
+
is_available,
|
|
65
|
+
resolve_backend,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
backend = resolve_backend(config)
|
|
69
|
+
if backend == BACKEND_CLOAKBROWSER:
|
|
70
|
+
return is_available(BACKEND_CLOAKBROWSER)
|
|
71
|
+
if backend == BACKEND_CAMOUFOX:
|
|
72
|
+
# A missing camoufox must not be masked by (or masked as) a patchright
|
|
73
|
+
# check — resolve_backend already guarantees it is the real selection.
|
|
74
|
+
return is_available(BACKEND_CAMOUFOX)
|
|
75
|
+
global _HAS_BROWSER_BACKEND
|
|
76
|
+
if _HAS_BROWSER_BACKEND is None:
|
|
77
|
+
_HAS_BROWSER_BACKEND = is_available(BACKEND_PATCHRIGHT)
|
|
78
|
+
return _HAS_BROWSER_BACKEND
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ---------------------------------------------------------------------------
|
|
82
|
+
# Per-thread browser (one browser per thread, multiple tabs per browser)
|
|
83
|
+
# Playwright sync API uses greenlets bound to threads — cross-thread
|
|
84
|
+
# usage crashes. Each thread gets its own browser; within a thread,
|
|
85
|
+
# tabs share cookies/session/Cloudflare state.
|
|
86
|
+
# ---------------------------------------------------------------------------
|
|
87
|
+
|
|
88
|
+
import threading as _threading
|
|
89
|
+
|
|
90
|
+
_tls = _threading.local()
|
|
91
|
+
|
|
92
|
+
# All live browsers across threads. Playwright sync objects are thread-affine
|
|
93
|
+
# (greenlets), so a graceful cross-thread close at process exit may fail —
|
|
94
|
+
# the reaper is best-effort and the driver's own EOF kill is the last resort.
|
|
95
|
+
_LIVE_BROWSERS: set = set()
|
|
96
|
+
_LIVE_BROWSERS_LOCK = _threading.Lock()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _register_browser(browser: Any) -> None:
|
|
100
|
+
with _LIVE_BROWSERS_LOCK:
|
|
101
|
+
_LIVE_BROWSERS.add(browser)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _unregister_browser(browser: Any) -> None:
|
|
105
|
+
with _LIVE_BROWSERS_LOCK:
|
|
106
|
+
_LIVE_BROWSERS.discard(browser)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _tree_kill(proc: Any) -> None:
|
|
110
|
+
"""Force-kill a driver process and its whole child tree (Windows-safe).
|
|
111
|
+
|
|
112
|
+
Accepts both handle shapes the transport hands back: subprocess.Popen
|
|
113
|
+
(``poll()``) and multiprocessing.Process (``is_alive()`` — Playwright's
|
|
114
|
+
driver since 1.x). Assuming Popen made the tree-kill backstop raise
|
|
115
|
+
AttributeError inside its callers' ``except Exception: pass`` — the
|
|
116
|
+
backstop never ran and long-lived MCP servers accumulated Chromium
|
|
117
|
+
processes (issue #57).
|
|
118
|
+
"""
|
|
119
|
+
if proc is None:
|
|
120
|
+
return
|
|
121
|
+
try:
|
|
122
|
+
if hasattr(proc, "poll"):
|
|
123
|
+
alive = proc.poll() is None
|
|
124
|
+
elif hasattr(proc, "is_alive"):
|
|
125
|
+
alive = proc.is_alive()
|
|
126
|
+
else:
|
|
127
|
+
alive = True
|
|
128
|
+
except Exception:
|
|
129
|
+
alive = True # can't tell — assume alive and force-kill
|
|
130
|
+
if not alive:
|
|
131
|
+
return
|
|
132
|
+
pid = getattr(proc, "pid", None)
|
|
133
|
+
if not pid:
|
|
134
|
+
return
|
|
135
|
+
if os.name == "nt":
|
|
136
|
+
subprocess.run(
|
|
137
|
+
["taskkill", "/F", "/T", "/PID", str(pid)],
|
|
138
|
+
capture_output=True, timeout=15,
|
|
139
|
+
)
|
|
140
|
+
else:
|
|
141
|
+
try:
|
|
142
|
+
proc.kill()
|
|
143
|
+
except Exception:
|
|
144
|
+
try:
|
|
145
|
+
os.kill(pid, 9)
|
|
146
|
+
except Exception:
|
|
147
|
+
pass
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _reap_browsers_at_exit() -> None:
|
|
151
|
+
"""Close every live browser when the process exits.
|
|
152
|
+
|
|
153
|
+
This is what allows callers to keep a browser alive across an entire
|
|
154
|
+
batch (no per-paper close churn) without leaking orphan Chromium
|
|
155
|
+
processes (issue #19).
|
|
156
|
+
"""
|
|
157
|
+
for browser in list(_LIVE_BROWSERS):
|
|
158
|
+
_unregister_browser(browser)
|
|
159
|
+
proc = None
|
|
160
|
+
try:
|
|
161
|
+
proc = browser._impl_obj._connection._transport._proc # type: ignore[attr-defined]
|
|
162
|
+
except Exception:
|
|
163
|
+
proc = None
|
|
164
|
+
try:
|
|
165
|
+
browser.close()
|
|
166
|
+
except Exception:
|
|
167
|
+
pass
|
|
168
|
+
# Verify the driver process actually died. A "successful" close can
|
|
169
|
+
# still leave the node/chromium tree alive (Windows propagates no
|
|
170
|
+
# signal to children), and a failed close means the owning thread is
|
|
171
|
+
# gone. Either way, tree-kill is the definitive backstop — without it
|
|
172
|
+
# every batch leaks its pooled browsers (issue #19).
|
|
173
|
+
try:
|
|
174
|
+
_tree_kill(proc)
|
|
175
|
+
except Exception:
|
|
176
|
+
pass
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
import atexit as _atexit
|
|
180
|
+
|
|
181
|
+
_atexit.register(_reap_browsers_at_exit)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _build_browser_args(config: dict[str, Any] | None = None) -> list[str]:
|
|
185
|
+
"""Build Chromium launch args from config (proxy, flags, etc.)."""
|
|
186
|
+
args = ["--disable-features=CrossOriginOpenerPolicy"]
|
|
187
|
+
if config:
|
|
188
|
+
proxy = config.get("browser_static_proxy", "")
|
|
189
|
+
if proxy:
|
|
190
|
+
args.append(f"--proxy-server={proxy}")
|
|
191
|
+
return args
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def close_shared_browser(config: dict[str, Any] | None = None) -> None:
|
|
195
|
+
"""Tear down THIS thread's shared browser (driver + loop included).
|
|
196
|
+
|
|
197
|
+
Needed before opening a VISIBLE browser on the same thread: the shared
|
|
198
|
+
sync-API loop would otherwise reject the second launch ("Playwright Sync
|
|
199
|
+
API inside the asyncio loop"). The shared browser relaunches lazily on
|
|
200
|
+
the next headless use.
|
|
201
|
+
"""
|
|
202
|
+
browser = getattr(_tls, "browser", None)
|
|
203
|
+
if browser is None:
|
|
204
|
+
return
|
|
205
|
+
try:
|
|
206
|
+
browser.close()
|
|
207
|
+
except Exception:
|
|
208
|
+
pass
|
|
209
|
+
try:
|
|
210
|
+
_unregister_browser(browser)
|
|
211
|
+
except Exception:
|
|
212
|
+
pass
|
|
213
|
+
_tls.browser = None
|
|
214
|
+
_tls.context = None
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _get_shared_browser(config: dict[str, Any] | None = None):
|
|
218
|
+
"""Get or create a browser for the current thread. Returns (browser, context)."""
|
|
219
|
+
browser = getattr(_tls, "browser", None)
|
|
220
|
+
context = getattr(_tls, "context", None)
|
|
221
|
+
if browser is not None:
|
|
222
|
+
# The browser lives as long as the thread — possibly a whole batch.
|
|
223
|
+
# Replace a crashed one instead of failing every remaining paper.
|
|
224
|
+
try:
|
|
225
|
+
connected = browser.is_connected()
|
|
226
|
+
except Exception:
|
|
227
|
+
connected = False
|
|
228
|
+
if connected:
|
|
229
|
+
return browser, context
|
|
230
|
+
# Full teardown: browser.close() also stops the Playwright driver and
|
|
231
|
+
# its asyncio loop. A half-alive driver would otherwise break the
|
|
232
|
+
# fresh launch below ("Sync API inside an asyncio event loop").
|
|
233
|
+
try:
|
|
234
|
+
browser.close()
|
|
235
|
+
except Exception:
|
|
236
|
+
pass
|
|
237
|
+
_unregister_browser(browser)
|
|
238
|
+
_tls.browser = None
|
|
239
|
+
_tls.context = None
|
|
240
|
+
|
|
241
|
+
# Playwright's Sync API cannot serve a *user* asyncio loop, but the loop
|
|
242
|
+
# the sync API itself leaves running in our worker threads is fine — see
|
|
243
|
+
# is_playwright_owned_loop(). Bail out only for foreign loops.
|
|
244
|
+
try:
|
|
245
|
+
import asyncio
|
|
246
|
+
loop = asyncio.get_running_loop()
|
|
247
|
+
except RuntimeError:
|
|
248
|
+
loop = None
|
|
249
|
+
if loop is not None and not is_playwright_owned_loop(loop):
|
|
250
|
+
raise RuntimeError(
|
|
251
|
+
"Browser (Playwright Sync API) cannot run inside an asyncio "
|
|
252
|
+
"event loop. Use the HTTP download sources instead, or call from "
|
|
253
|
+
"a synchronous context."
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
if not _check_browser_backend(config):
|
|
257
|
+
raise RuntimeError(
|
|
258
|
+
"no browser backend available. Run: pip install patchright (or pip install cloakbrowser)"
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
backend = resolve_backend(config)
|
|
262
|
+
|
|
263
|
+
# Platform compat shim + kernel override only apply to CloakBrowser
|
|
264
|
+
if backend == BACKEND_CLOAKBROWSER:
|
|
265
|
+
try:
|
|
266
|
+
from .cloakbrowser_compat import ensure_cloakbrowser_platform_compatible
|
|
267
|
+
ensure_cloakbrowser_platform_compatible()
|
|
268
|
+
except Exception:
|
|
269
|
+
logger.debug("CloakBrowser platform shim not applied", exc_info=True)
|
|
270
|
+
binary = resolve_browser_binary(config)
|
|
271
|
+
if binary:
|
|
272
|
+
os.environ["CLOAKBROWSER_BINARY_PATH"] = binary
|
|
273
|
+
|
|
274
|
+
# scihub_browser_headless overrides browser_headless for THIS shared
|
|
275
|
+
# browser (the grey-source racing engine). It exists because the racing
|
|
276
|
+
# window otherwise flashes popup/redirect attempts from sci-hub mirrors
|
|
277
|
+
# at the taskbar; institutional logins keep their own visible windows
|
|
278
|
+
# via get_persistent_context / auth.py, which still honor browser_headless.
|
|
279
|
+
headless = False
|
|
280
|
+
humanize = True
|
|
281
|
+
if config:
|
|
282
|
+
headless = bool(
|
|
283
|
+
config.get("scihub_browser_headless", config.get("browser_headless", False))
|
|
284
|
+
)
|
|
285
|
+
humanize = config.get("browser_humanize", True)
|
|
286
|
+
|
|
287
|
+
args = _build_browser_args(config)
|
|
288
|
+
browser = launch(headless=headless, humanize=humanize, args=args, config=config)
|
|
289
|
+
_register_browser(browser)
|
|
290
|
+
context = browser.new_context()
|
|
291
|
+
|
|
292
|
+
# Launching the sync API leaves its dispatcher event loop "running" in
|
|
293
|
+
# this thread (asyncio.get_running_loop() now succeeds here). Register it
|
|
294
|
+
# so is_playwright_owned_loop() can tell OUR worker-thread loop apart from
|
|
295
|
+
# a user's asyncio loop — the latter is the only case sync API can't serve.
|
|
296
|
+
try:
|
|
297
|
+
import asyncio as _asyncio
|
|
298
|
+
_tls.owned_loop = _asyncio.get_running_loop()
|
|
299
|
+
except RuntimeError:
|
|
300
|
+
_tls.owned_loop = None
|
|
301
|
+
_tls.browser = browser
|
|
302
|
+
_tls.context = context
|
|
303
|
+
logger.info(f"browser_engine: browser ready for thread {_threading.current_thread().name}")
|
|
304
|
+
return browser, context
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def get_persistent_context(
|
|
308
|
+
profile_dir: str | Path,
|
|
309
|
+
config: dict[str, Any] | None = None,
|
|
310
|
+
):
|
|
311
|
+
"""Get or create a persistent browser context for fingerprint consistency.
|
|
312
|
+
|
|
313
|
+
Unlike launch() + cookie restore, persistent context preserves:
|
|
314
|
+
- Browser fingerprint (canvas, WebGL, audio, fonts)
|
|
315
|
+
- Cookies and localStorage across restarts
|
|
316
|
+
- Login sessions without re-authentication
|
|
317
|
+
|
|
318
|
+
This is the recommended approach for publisher sessions that need
|
|
319
|
+
stable identity across multiple download runs.
|
|
320
|
+
"""
|
|
321
|
+
if not _check_browser_backend(config):
|
|
322
|
+
raise RuntimeError(
|
|
323
|
+
"no browser backend available. Run: pip install patchright (or pip install cloakbrowser)"
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
backend = resolve_backend(config)
|
|
327
|
+
|
|
328
|
+
if backend == BACKEND_CLOAKBROWSER:
|
|
329
|
+
try:
|
|
330
|
+
from .cloakbrowser_compat import prepare_cloakbrowser_runtime
|
|
331
|
+
prepare_cloakbrowser_runtime()
|
|
332
|
+
except Exception:
|
|
333
|
+
pass
|
|
334
|
+
binary = resolve_browser_binary(config)
|
|
335
|
+
if binary:
|
|
336
|
+
os.environ["CLOAKBROWSER_BINARY_PATH"] = binary
|
|
337
|
+
|
|
338
|
+
headless = False
|
|
339
|
+
humanize = True
|
|
340
|
+
if config:
|
|
341
|
+
headless = config.get("browser_headless", False)
|
|
342
|
+
humanize = config.get("browser_humanize", True)
|
|
343
|
+
|
|
344
|
+
args = _build_browser_args(config)
|
|
345
|
+
profile_path = Path(profile_dir)
|
|
346
|
+
profile_path.mkdir(parents=True, exist_ok=True)
|
|
347
|
+
|
|
348
|
+
ctx = launch_persistent_context(
|
|
349
|
+
str(profile_path),
|
|
350
|
+
headless=headless,
|
|
351
|
+
humanize=humanize,
|
|
352
|
+
args=args,
|
|
353
|
+
config=config,
|
|
354
|
+
)
|
|
355
|
+
logger.info(f"browser_engine: persistent context ready at {profile_path}")
|
|
356
|
+
return ctx
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def get_browser_page(config: dict[str, Any] | None = None):
|
|
360
|
+
"""Get a new page from the shared browser (for custom browser interactions).
|
|
361
|
+
|
|
362
|
+
Returns a Playwright Page object that the caller must close after use.
|
|
363
|
+
Returns None if the browser is not available.
|
|
364
|
+
"""
|
|
365
|
+
if not _check_browser_backend(config):
|
|
366
|
+
return None
|
|
367
|
+
try:
|
|
368
|
+
_browser, context = _get_shared_browser(config)
|
|
369
|
+
return context.new_page()
|
|
370
|
+
except Exception:
|
|
371
|
+
return None
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def is_playwright_owned_loop(loop: Any) -> bool:
|
|
375
|
+
"""True when `loop` is the sync-API dispatcher loop of THIS thread's browser.
|
|
376
|
+
|
|
377
|
+
A running asyncio loop normally means "user asyncio code — sync Playwright
|
|
378
|
+
forbidden". Exception: our persistent worker threads, where the loop was
|
|
379
|
+
created by the sync API itself and must not disqualify the thread.
|
|
380
|
+
"""
|
|
381
|
+
return getattr(_tls, "owned_loop", None) is loop and loop is not None
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def shutdown_shared_browser():
|
|
385
|
+
"""Shut down the current thread's browser. Call on thread exit or process exit."""
|
|
386
|
+
browser = getattr(_tls, "browser", None)
|
|
387
|
+
if browser is not None:
|
|
388
|
+
try:
|
|
389
|
+
browser.close()
|
|
390
|
+
except Exception:
|
|
391
|
+
pass
|
|
392
|
+
_unregister_browser(browser)
|
|
393
|
+
_tls.browser = None
|
|
394
|
+
_tls.context = None
|
|
395
|
+
logger.info("browser_engine: browser shut down")
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
# ---------------------------------------------------------------------------
|
|
399
|
+
# Public API — drop-in replacements for camofox.py functions
|
|
400
|
+
# ---------------------------------------------------------------------------
|
|
401
|
+
|
|
402
|
+
def is_available(config: dict[str, Any] | None = None) -> bool:
|
|
403
|
+
"""Check if the resolved browser backend is available (importable)."""
|
|
404
|
+
return _check_browser_backend(config)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
# ---------------------------------------------------------------------------
|
|
408
|
+
# Tab registry — maps tab_id → (browser, context, page)
|
|
409
|
+
# Used by create_tab/evaluate_js/navigate_tab/close_tab for sequential
|
|
410
|
+
# tab-based workflows within a single operation (thread-safe: one thread per tab).
|
|
411
|
+
# ---------------------------------------------------------------------------
|
|
412
|
+
|
|
413
|
+
_tabs: dict[str, Any] = {} # tab_id → page
|
|
414
|
+
_captured: dict[str, list] = {} # tab_id → captured PDF responses
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _register_tab(browser, context, page) -> str:
|
|
418
|
+
tab_id = uuid.uuid4().hex[:12]
|
|
419
|
+
_tabs[tab_id] = page
|
|
420
|
+
_captured[tab_id] = []
|
|
421
|
+
|
|
422
|
+
# Listen for PDF responses
|
|
423
|
+
def _on_response(response):
|
|
424
|
+
try:
|
|
425
|
+
ct = response.headers.get("content-type", "")
|
|
426
|
+
if "pdf" in ct or "octet-stream" in ct:
|
|
427
|
+
try:
|
|
428
|
+
body = response.body()
|
|
429
|
+
if body[:5] == b"%PDF-":
|
|
430
|
+
_captured[tab_id].append({
|
|
431
|
+
"url": response.url,
|
|
432
|
+
"status": response.status,
|
|
433
|
+
"contentType": ct,
|
|
434
|
+
"dataBase64": base64.b64encode(body).decode(),
|
|
435
|
+
})
|
|
436
|
+
except Exception:
|
|
437
|
+
pass
|
|
438
|
+
except Exception:
|
|
439
|
+
pass
|
|
440
|
+
|
|
441
|
+
try:
|
|
442
|
+
page.on("response", _on_response)
|
|
443
|
+
except Exception:
|
|
444
|
+
pass
|
|
445
|
+
|
|
446
|
+
return tab_id
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _resolve_tab(tab_id: str):
|
|
450
|
+
"""Look up page for a tab_id. Returns page or None."""
|
|
451
|
+
page = _tabs.get(tab_id)
|
|
452
|
+
return page
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def solve_url(
|
|
456
|
+
url: str,
|
|
457
|
+
config: dict[str, Any],
|
|
458
|
+
*,
|
|
459
|
+
max_timeout: int = 60000,
|
|
460
|
+
) -> dict[str, Any] | None:
|
|
461
|
+
"""Fetch URL via CloakBrowser shared browser. Returns dict with status/solution keys."""
|
|
462
|
+
page = None
|
|
463
|
+
try:
|
|
464
|
+
_, context = _get_shared_browser(config)
|
|
465
|
+
page = context.new_page()
|
|
466
|
+
page.goto(url, wait_until="domcontentloaded", timeout=int(max_timeout))
|
|
467
|
+
|
|
468
|
+
raw_cookies = context.cookies()
|
|
469
|
+
cookies = [{"name": c["name"], "value": c.get("value", "")} for c in raw_cookies if c.get("name")]
|
|
470
|
+
|
|
471
|
+
html = ""
|
|
472
|
+
try:
|
|
473
|
+
html = page.content()
|
|
474
|
+
except Exception as e:
|
|
475
|
+
logger.info(f"browser_engine: HTML extraction failed: {e}")
|
|
476
|
+
|
|
477
|
+
final_url = page.url
|
|
478
|
+
logger.info(f"browser_engine: ok, final_url={final_url}, html_len={len(html)}")
|
|
479
|
+
return {
|
|
480
|
+
"status": "ok",
|
|
481
|
+
"solution": {
|
|
482
|
+
"url": final_url,
|
|
483
|
+
"status": 200,
|
|
484
|
+
"response": html,
|
|
485
|
+
"cookies": cookies,
|
|
486
|
+
},
|
|
487
|
+
}
|
|
488
|
+
except Exception as e:
|
|
489
|
+
logger.info(f"browser_engine: error - {e}")
|
|
490
|
+
return None
|
|
491
|
+
finally:
|
|
492
|
+
if page:
|
|
493
|
+
try:
|
|
494
|
+
page.close()
|
|
495
|
+
except Exception:
|
|
496
|
+
pass
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def get_html(
|
|
500
|
+
url: str,
|
|
501
|
+
config: dict[str, Any],
|
|
502
|
+
*,
|
|
503
|
+
max_timeout: int = 60000,
|
|
504
|
+
) -> str | None:
|
|
505
|
+
"""Solve and return page HTML."""
|
|
506
|
+
result = solve_url(url, config, max_timeout=max_timeout)
|
|
507
|
+
if not result:
|
|
508
|
+
return None
|
|
509
|
+
return result.get("solution", {}).get("response")
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def import_cookies(cookie_file: str | Path, config: dict[str, Any], *, domain_suffix: str | None = None) -> int:
|
|
513
|
+
"""Import Netscape-format cookies into the shared browser context. Returns count imported."""
|
|
514
|
+
try:
|
|
515
|
+
text = Path(cookie_file).read_text(encoding="utf-8")
|
|
516
|
+
except Exception as e:
|
|
517
|
+
logger.info(f"browser_engine: failed to read cookie file: {e}")
|
|
518
|
+
return 0
|
|
519
|
+
cookies = _parse_netscape_cookies(text)
|
|
520
|
+
# Cloudflare clearance/bot cookies are bound to the browser fingerprint and IP
|
|
521
|
+
# they were issued to; replaying stale ones into another session triggers
|
|
522
|
+
# fresh challenges instead of skipping them.
|
|
523
|
+
cookies = [c for c in cookies if not (c.get("name", "") in ("cf_clearance", "__cf_bm", "_cfuvid")
|
|
524
|
+
or c.get("name", "").startswith("cf_chl"))]
|
|
525
|
+
if not cookies:
|
|
526
|
+
return 0
|
|
527
|
+
if domain_suffix:
|
|
528
|
+
cookies = [c for c in cookies if domain_suffix in c.get("domain", "")]
|
|
529
|
+
try:
|
|
530
|
+
_, ctx = _get_shared_browser(config)
|
|
531
|
+
ctx.add_cookies(cookies)
|
|
532
|
+
logger.info(f"browser_engine: imported {len(cookies)} cookies")
|
|
533
|
+
return len(cookies)
|
|
534
|
+
except Exception as e:
|
|
535
|
+
logger.info(f"browser_engine: import_cookies error: {e}")
|
|
536
|
+
return 0
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def evaluate_js(
|
|
540
|
+
tab_id: str,
|
|
541
|
+
expression: str,
|
|
542
|
+
config: dict[str, Any],
|
|
543
|
+
*,
|
|
544
|
+
timeout: float = 15.0,
|
|
545
|
+
) -> Any:
|
|
546
|
+
"""Evaluate JavaScript expression in a browser page. Returns the JS result."""
|
|
547
|
+
page = _resolve_tab(tab_id)
|
|
548
|
+
if not page:
|
|
549
|
+
logger.info(f"browser_engine: evaluate_js - tab {tab_id} not found")
|
|
550
|
+
return None
|
|
551
|
+
try:
|
|
552
|
+
return page.evaluate(expression)
|
|
553
|
+
except Exception as e:
|
|
554
|
+
logger.info(f"browser_engine: evaluate_js error: {e}")
|
|
555
|
+
return None
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def create_tab(url: str, config: dict[str, Any], *, timeout: float = 30.0) -> str | None:
|
|
559
|
+
"""Create a new tab (page) in the shared browser and navigate to URL. Returns tab_id or None."""
|
|
560
|
+
try:
|
|
561
|
+
browser, context = _get_shared_browser(config)
|
|
562
|
+
page = context.new_page()
|
|
563
|
+
page.goto(url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
|
|
564
|
+
tab_id = _register_tab(browser, context, page)
|
|
565
|
+
return tab_id
|
|
566
|
+
except Exception as e:
|
|
567
|
+
logger.info(f"browser_engine: create_tab failed - {e}")
|
|
568
|
+
return None
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def close_tab(tab_id: str, config: dict[str, Any]) -> None:
|
|
572
|
+
"""Close a browser tab (page only, not the shared browser)."""
|
|
573
|
+
page = _resolve_tab(tab_id)
|
|
574
|
+
if page:
|
|
575
|
+
try:
|
|
576
|
+
page.close()
|
|
577
|
+
except Exception:
|
|
578
|
+
pass
|
|
579
|
+
_tabs.pop(tab_id, None)
|
|
580
|
+
_captured.pop(tab_id, None)
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def navigate_tab(tab_id: str, url: str, config: dict[str, Any], *, timeout: float = 30.0) -> bool:
|
|
584
|
+
"""Navigate an existing page to a new URL."""
|
|
585
|
+
page = _resolve_tab(tab_id)
|
|
586
|
+
if not page:
|
|
587
|
+
logger.info(f"browser_engine: navigate_tab - tab {tab_id} not found")
|
|
588
|
+
return False
|
|
589
|
+
for attempt in (1, 2):
|
|
590
|
+
try:
|
|
591
|
+
page.goto(url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
|
|
592
|
+
return True
|
|
593
|
+
except Exception as e:
|
|
594
|
+
# A connection that never came up (net::ERR_TIMED_OUT/RESET/...) is
|
|
595
|
+
# usually a transient network hiccup: retry once in this same tab
|
|
596
|
+
# rather than failing the source and opening a fallback window.
|
|
597
|
+
transient = any(code in str(e) for code in _TRANSIENT_NET_ERRORS)
|
|
598
|
+
logger.info(f"browser_engine: navigate failed - {str(e).splitlines()[0]}"
|
|
599
|
+
+ (" (retrying)" if transient and attempt == 1 else ""))
|
|
600
|
+
if not (transient and attempt == 1):
|
|
601
|
+
return False
|
|
602
|
+
return False
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
_TRANSIENT_NET_ERRORS = (
|
|
606
|
+
"net::ERR_TIMED_OUT", "net::ERR_CONNECTION_TIMED_OUT", "net::ERR_CONNECTION_RESET",
|
|
607
|
+
"net::ERR_CONNECTION_CLOSED", "net::ERR_NETWORK_CHANGED", "net::ERR_EMPTY_RESPONSE",
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def download_pdf_via_browser(
|
|
612
|
+
pdf_url: str,
|
|
613
|
+
output_path: Path,
|
|
614
|
+
config: dict[str, Any],
|
|
615
|
+
*,
|
|
616
|
+
timeout: float = 60.0,
|
|
617
|
+
) -> bool:
|
|
618
|
+
"""Download a PDF URL via CloakBrowser shared browser. Returns True on success.
|
|
619
|
+
|
|
620
|
+
4-strategy cascade:
|
|
621
|
+
1. Network response capture (from page.on("response"))
|
|
622
|
+
2. In-browser fetch API with credentials
|
|
623
|
+
3. PDF link discovery in page DOM
|
|
624
|
+
4. Download button click
|
|
625
|
+
"""
|
|
626
|
+
page = None
|
|
627
|
+
try:
|
|
628
|
+
# Fragments (#view=FitH...) are never sent to servers but they break
|
|
629
|
+
# .pdf suffix checks and _is_pdf_url — strip before anything else.
|
|
630
|
+
pdf_url = str(pdf_url).split("#", 1)[0]
|
|
631
|
+
_, context = _get_shared_browser(config)
|
|
632
|
+
page = context.new_page()
|
|
633
|
+
|
|
634
|
+
# Set up response listener for PDF captures
|
|
635
|
+
captured_responses: list[dict] = []
|
|
636
|
+
|
|
637
|
+
def _on_response(response):
|
|
638
|
+
try:
|
|
639
|
+
ct = response.headers.get("content-type", "")
|
|
640
|
+
if "pdf" in ct or "octet-stream" in ct:
|
|
641
|
+
try:
|
|
642
|
+
body = response.body()
|
|
643
|
+
if body[:5] == b"%PDF-":
|
|
644
|
+
captured_responses.append({
|
|
645
|
+
"url": response.url,
|
|
646
|
+
"dataBase64": base64.b64encode(body).decode(),
|
|
647
|
+
})
|
|
648
|
+
except Exception:
|
|
649
|
+
pass
|
|
650
|
+
except Exception:
|
|
651
|
+
pass
|
|
652
|
+
|
|
653
|
+
try:
|
|
654
|
+
page.on("response", _on_response)
|
|
655
|
+
except Exception:
|
|
656
|
+
pass
|
|
657
|
+
|
|
658
|
+
page.goto(pdf_url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
|
|
659
|
+
time.sleep(3)
|
|
660
|
+
|
|
661
|
+
# Check for anti-bot challenges
|
|
662
|
+
html = ""
|
|
663
|
+
try:
|
|
664
|
+
html = page.content()
|
|
665
|
+
except Exception:
|
|
666
|
+
pass
|
|
667
|
+
lower_html = html.lower()
|
|
668
|
+
if any(sig in lower_html for sig in [
|
|
669
|
+
"cf-browser-verification", "challenge-platform",
|
|
670
|
+
"just a moment", "attention required",
|
|
671
|
+
"security check", "captcha",
|
|
672
|
+
"请稍候", "正在验证", "checking your browser",
|
|
673
|
+
# ALTCHA anti-bot verification (used by sci-hub.ru and other Sci-Hub mirrors)
|
|
674
|
+
"altcha", "你是机器人吗", "not a robot", "nope",
|
|
675
|
+
]):
|
|
676
|
+
logger.info("browser_engine: anti-bot challenge detected, waiting...")
|
|
677
|
+
time.sleep(10)
|
|
678
|
+
|
|
679
|
+
current_url = page.url
|
|
680
|
+
|
|
681
|
+
# Strategy 0: Network response capture
|
|
682
|
+
for resp in captured_responses:
|
|
683
|
+
data = resp.get("dataBase64", "")
|
|
684
|
+
if data:
|
|
685
|
+
pdf_bytes = base64.b64decode(data)
|
|
686
|
+
if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
|
|
687
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
688
|
+
output_path.write_bytes(pdf_bytes)
|
|
689
|
+
logger.info(f"browser_engine: downloaded {len(pdf_bytes)} bytes via network capture")
|
|
690
|
+
return True
|
|
691
|
+
|
|
692
|
+
# Build candidate fetch paths
|
|
693
|
+
fetch_paths: list[str] = []
|
|
694
|
+
parsed = urlparse(str(current_url))
|
|
695
|
+
origin = f"{parsed.scheme}://{parsed.netloc}"
|
|
696
|
+
|
|
697
|
+
# Extract DOI from URL for pdfdirect construction
|
|
698
|
+
doi_from_url = None
|
|
699
|
+
path = parsed.path
|
|
700
|
+
# Longest first: "/doi/" also matches "/doi/pdfdirect/10.1111/x", which
|
|
701
|
+
# made "pdfdirect/10.1111/x" the DOI (2026-10-18, Wiley 403 x3).
|
|
702
|
+
for prefix in ["/doi/pdfdirect/", "/doi/pdf/", "/doi/epdf/", "/doi/", "/articles/"]:
|
|
703
|
+
if prefix in path:
|
|
704
|
+
doi_from_url = path.split(prefix)[-1].split("?")[0].split("#")[0]
|
|
705
|
+
break
|
|
706
|
+
|
|
707
|
+
if doi_from_url:
|
|
708
|
+
fetch_paths.append(f"/doi/pdfdirect/{doi_from_url}")
|
|
709
|
+
fetch_paths.append(f"/doi/pdf/{doi_from_url}")
|
|
710
|
+
fetch_paths.append(f"/content/pdf/{doi_from_url}.pdf")
|
|
711
|
+
|
|
712
|
+
if _is_pdf_url(pdf_url):
|
|
713
|
+
parsed_orig = urlparse(pdf_url)
|
|
714
|
+
if parsed_orig.netloc == parsed.netloc:
|
|
715
|
+
fetch_paths.append(parsed_orig.path)
|
|
716
|
+
|
|
717
|
+
# Strategy 1: In-browser fetch API
|
|
718
|
+
for fetch_path in fetch_paths:
|
|
719
|
+
logger.info(f"browser_engine: trying in-browser fetch {origin}{fetch_path[:60]}")
|
|
720
|
+
try:
|
|
721
|
+
pdf_b64 = page.evaluate(f"""
|
|
722
|
+
(async () => {{
|
|
723
|
+
try {{
|
|
724
|
+
const resp = await fetch('{fetch_path}', {{
|
|
725
|
+
credentials: 'include',
|
|
726
|
+
headers: {{'Accept': 'application/pdf,*/*'}}
|
|
727
|
+
}});
|
|
728
|
+
if (!resp.ok) return 'status:' + resp.status;
|
|
729
|
+
const ct = resp.headers.get('content-type') || '';
|
|
730
|
+
if (!ct.includes('pdf') && !ct.includes('octet')) return 'ct:' + ct;
|
|
731
|
+
const blob = await resp.blob();
|
|
732
|
+
return new Promise((resolve) => {{
|
|
733
|
+
const reader = new FileReader();
|
|
734
|
+
reader.onload = () => resolve(reader.result);
|
|
735
|
+
reader.readAsDataURL(blob);
|
|
736
|
+
}});
|
|
737
|
+
}} catch(e) {{
|
|
738
|
+
return 'error:' + e.message;
|
|
739
|
+
}}
|
|
740
|
+
}})()
|
|
741
|
+
""")
|
|
742
|
+
|
|
743
|
+
if isinstance(pdf_b64, str) and pdf_b64.startswith("data:"):
|
|
744
|
+
header, data = pdf_b64.split(",", 1)
|
|
745
|
+
pdf_bytes = base64.b64decode(data)
|
|
746
|
+
if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
|
|
747
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
748
|
+
output_path.write_bytes(pdf_bytes)
|
|
749
|
+
logger.info(f"browser_engine: downloaded {len(pdf_bytes)} bytes via in-browser fetch")
|
|
750
|
+
return True
|
|
751
|
+
else:
|
|
752
|
+
logger.info(f"browser_engine: fetch returned non-PDF ({len(pdf_bytes)} bytes)")
|
|
753
|
+
else:
|
|
754
|
+
logger.info(f"browser_engine: fetch result: {str(pdf_b64)[:80]}")
|
|
755
|
+
except Exception as e:
|
|
756
|
+
logger.info(f"browser_engine: fetch error: {e}")
|
|
757
|
+
|
|
758
|
+
# Strategy 2: PDF link discovery in DOM
|
|
759
|
+
try:
|
|
760
|
+
pdf_link = page.evaluate("""
|
|
761
|
+
(() => {
|
|
762
|
+
for (const el of document.querySelectorAll('iframe, embed, object')) {
|
|
763
|
+
const src = el.src || el.data || '';
|
|
764
|
+
if (src.includes('.pdf') && !src.includes('supplement') && !src.includes('Suppl')) return src;
|
|
765
|
+
}
|
|
766
|
+
const viewer = document.querySelector('#viewer, .pdfViewer, [data-l10n-id="download"]');
|
|
767
|
+
if (viewer) return window.location.href;
|
|
768
|
+
for (const a of document.querySelectorAll('a[href]')) {
|
|
769
|
+
const href = (a.href || '').toLowerCase();
|
|
770
|
+
const text = (a.innerText || '').toLowerCase();
|
|
771
|
+
if (href.includes('supplement') || href.includes('supporting') || href.includes('downloadsupplement') || href.includes('pb-assets')) continue;
|
|
772
|
+
if (text.includes('supplement') || text.includes('supporting info')) continue;
|
|
773
|
+
if (href.includes('.pdf') || href.includes('/pdf/') || href.includes('pdfdirect')) {
|
|
774
|
+
if (a.href.startsWith('http')) return a.href;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
const sdPdf = document.querySelector('a[aria-label*="PDF"], a[aria-label*="pdf"], a.pdf-download-btn-link');
|
|
778
|
+
if (sdPdf) return sdPdf.href;
|
|
779
|
+
return null;
|
|
780
|
+
})()
|
|
781
|
+
""")
|
|
782
|
+
|
|
783
|
+
if pdf_link and isinstance(pdf_link, str) and pdf_link.startswith("http"):
|
|
784
|
+
logger.info(f"browser_engine: found PDF link: {pdf_link[:80]}")
|
|
785
|
+
parsed_link = urlparse(pdf_link)
|
|
786
|
+
link_path = parsed_link.path + ("?" + parsed_link.query if parsed_link.query else "")
|
|
787
|
+
if parsed_link.netloc == parsed.netloc:
|
|
788
|
+
try:
|
|
789
|
+
pdf_b64 = page.evaluate(f"""
|
|
790
|
+
(async () => {{
|
|
791
|
+
try {{
|
|
792
|
+
const resp = await fetch('{link_path}', {{
|
|
793
|
+
credentials: 'include',
|
|
794
|
+
headers: {{'Accept': 'application/pdf,*/*'}}
|
|
795
|
+
}});
|
|
796
|
+
if (!resp.ok) return 'status:' + resp.status;
|
|
797
|
+
const ct = resp.headers.get('content-type') || '';
|
|
798
|
+
if (!ct.includes('pdf') && !ct.includes('octet')) return 'ct:' + ct;
|
|
799
|
+
const blob = await resp.blob();
|
|
800
|
+
return new Promise((resolve) => {{
|
|
801
|
+
const reader = new FileReader();
|
|
802
|
+
reader.onload = () => resolve(reader.result);
|
|
803
|
+
reader.readAsDataURL(blob);
|
|
804
|
+
}});
|
|
805
|
+
}} catch(e) {{
|
|
806
|
+
return 'error:' + e.message;
|
|
807
|
+
}}
|
|
808
|
+
}})()
|
|
809
|
+
""")
|
|
810
|
+
|
|
811
|
+
if isinstance(pdf_b64, str) and pdf_b64.startswith("data:"):
|
|
812
|
+
header, data = pdf_b64.split(",", 1)
|
|
813
|
+
pdf_bytes = base64.b64decode(data)
|
|
814
|
+
if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
|
|
815
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
816
|
+
output_path.write_bytes(pdf_bytes)
|
|
817
|
+
logger.info(f"browser_engine: downloaded {len(pdf_bytes)} bytes via PDF link fetch")
|
|
818
|
+
return True
|
|
819
|
+
except Exception as e:
|
|
820
|
+
logger.info(f"browser_engine: PDF link fetch error: {e}")
|
|
821
|
+
except Exception as e:
|
|
822
|
+
logger.info(f"browser_engine: DOM scan error: {e}")
|
|
823
|
+
|
|
824
|
+
# Strategy 3: Click download button
|
|
825
|
+
try:
|
|
826
|
+
clicked = page.evaluate("""
|
|
827
|
+
(() => {
|
|
828
|
+
for (const btn of document.querySelectorAll(
|
|
829
|
+
'#download, [aria-label*="download" i], [aria-label*="PDF" i], .pdf-download-btn-link, a[data-aa-name="download-pdf"]'
|
|
830
|
+
)) {
|
|
831
|
+
if (btn.offsetParent !== null) { btn.click(); return true; }
|
|
832
|
+
}
|
|
833
|
+
return false;
|
|
834
|
+
})()
|
|
835
|
+
""")
|
|
836
|
+
if clicked:
|
|
837
|
+
logger.info("browser_engine: clicked download button, waiting...")
|
|
838
|
+
time.sleep(5)
|
|
839
|
+
except Exception as e:
|
|
840
|
+
logger.info(f"browser_engine: click error: {e}")
|
|
841
|
+
|
|
842
|
+
return False
|
|
843
|
+
except Exception as e:
|
|
844
|
+
logger.info(f"browser_engine: download_pdf_via_browser error: {e}")
|
|
845
|
+
return False
|
|
846
|
+
finally:
|
|
847
|
+
if page:
|
|
848
|
+
try:
|
|
849
|
+
page.close()
|
|
850
|
+
except Exception:
|
|
851
|
+
pass
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
# Backward-compat alias
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def _is_pdf_url(url: str) -> bool:
|
|
858
|
+
"""Check if a URL looks like a direct PDF link."""
|
|
859
|
+
lower = str(url).split("#", 1)[0].lower()
|
|
860
|
+
return (
|
|
861
|
+
lower.endswith(".pdf")
|
|
862
|
+
or "/pdf/" in lower
|
|
863
|
+
or "content/pdf" in lower
|
|
864
|
+
or "pdfdirect" in lower
|
|
865
|
+
or "/doi/pdf/" in lower
|
|
866
|
+
or "type=printable" in lower
|
|
867
|
+
)
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
def fetch_url(
|
|
871
|
+
tab_id: str,
|
|
872
|
+
url: str,
|
|
873
|
+
config: dict[str, Any],
|
|
874
|
+
*,
|
|
875
|
+
timeout: float = 30.0,
|
|
876
|
+
) -> dict[str, Any] | None:
|
|
877
|
+
"""Navigate tab to URL and capture any PDF responses from the network layer."""
|
|
878
|
+
page = _resolve_tab(tab_id)
|
|
879
|
+
if not page:
|
|
880
|
+
logger.info(f"browser_engine: fetch_url - tab {tab_id} not found")
|
|
881
|
+
return None
|
|
882
|
+
|
|
883
|
+
# The inline PDF viewer hides the body from the response listener; read it
|
|
884
|
+
# from the paused response via CDP while the browser itself makes the request.
|
|
885
|
+
from .pdf import capture_pdf_via_cdp
|
|
886
|
+
pdf_bytes = capture_pdf_via_cdp(page.context, page, url, timeout_s=max(timeout, 90.0))
|
|
887
|
+
if pdf_bytes:
|
|
888
|
+
return {"status": "ok", "bytes": len(pdf_bytes), "data": pdf_bytes}
|
|
889
|
+
|
|
890
|
+
try:
|
|
891
|
+
page.goto(url, wait_until="domcontentloaded", timeout=int(timeout * 1000))
|
|
892
|
+
except Exception as e:
|
|
893
|
+
logger.info(f"browser_engine: fetch_url navigate error: {e}")
|
|
894
|
+
|
|
895
|
+
# Check captured responses
|
|
896
|
+
for resp in _captured.get(tab_id, []):
|
|
897
|
+
data = resp.get("dataBase64", "")
|
|
898
|
+
if data:
|
|
899
|
+
pdf_bytes = base64.b64decode(data)
|
|
900
|
+
if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
|
|
901
|
+
return {"status": "ok", "bytes": len(pdf_bytes), "data": pdf_bytes}
|
|
902
|
+
|
|
903
|
+
# Also check direct response body
|
|
904
|
+
try:
|
|
905
|
+
response = page.goto(url, wait_until="commit", timeout=int(timeout * 1000))
|
|
906
|
+
if response is not None:
|
|
907
|
+
body = response.body()
|
|
908
|
+
if body[:5] == b"%PDF-" and len(body) > 5000:
|
|
909
|
+
return {"status": "ok", "bytes": len(body), "data": body}
|
|
910
|
+
except Exception:
|
|
911
|
+
pass
|
|
912
|
+
|
|
913
|
+
return None
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
def get_captured_responses(
|
|
917
|
+
tab_id: str,
|
|
918
|
+
config: dict[str, Any],
|
|
919
|
+
*,
|
|
920
|
+
consume: bool = True,
|
|
921
|
+
) -> list[dict[str, Any]]:
|
|
922
|
+
"""Get captured PDF responses for a tab. Optionally consume (clear) them."""
|
|
923
|
+
captured = _captured.get(tab_id, [])
|
|
924
|
+
result = list(captured)
|
|
925
|
+
if consume:
|
|
926
|
+
_captured[tab_id] = []
|
|
927
|
+
return result
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
# ---------------------------------------------------------------------------
|
|
931
|
+
# Netscape cookie parser (shared utility)
|
|
932
|
+
# ---------------------------------------------------------------------------
|
|
933
|
+
|
|
934
|
+
def _parse_netscape_cookies(text: str) -> list[dict[str, Any]]:
|
|
935
|
+
"""Parse Netscape-format cookie file into structured cookie objects."""
|
|
936
|
+
cookies: list[dict[str, Any]] = []
|
|
937
|
+
for line in text.splitlines():
|
|
938
|
+
line = line.strip()
|
|
939
|
+
if not line or (line.startswith("#") and not line.startswith("#HttpOnly_")):
|
|
940
|
+
continue
|
|
941
|
+
http_only = False
|
|
942
|
+
if line.startswith("#HttpOnly_"):
|
|
943
|
+
http_only = True
|
|
944
|
+
line = line.removeprefix("#HttpOnly_")
|
|
945
|
+
parts = line.split("\t")
|
|
946
|
+
if len(parts) < 7:
|
|
947
|
+
continue
|
|
948
|
+
domain = parts[0]
|
|
949
|
+
cookie_path = parts[2]
|
|
950
|
+
secure = parts[3].upper() == "TRUE"
|
|
951
|
+
try:
|
|
952
|
+
expires = int(parts[4])
|
|
953
|
+
except (ValueError, IndexError):
|
|
954
|
+
expires = 0
|
|
955
|
+
name = parts[5]
|
|
956
|
+
value = "\t".join(parts[6:])
|
|
957
|
+
cookie: dict[str, Any] = {
|
|
958
|
+
"name": name, "value": value,
|
|
959
|
+
"domain": domain, "path": cookie_path,
|
|
960
|
+
"secure": secure, "expires": expires,
|
|
961
|
+
}
|
|
962
|
+
if http_only:
|
|
963
|
+
cookie["httpOnly"] = True
|
|
964
|
+
cookies.append(cookie)
|
|
965
|
+
return cookies
|