litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1009 @@
|
|
|
1
|
+
"""Unified browser backend: patchright (default), CloakBrowser, or Camoufox.
|
|
2
|
+
|
|
3
|
+
patchright is the Apache-2.0 open-source Playwright fork that patches the
|
|
4
|
+
detectable automation fingerprints (``--enable-automation`` removal,
|
|
5
|
+
``Runtime.enable`` leak, ``navigator.webdriver``, ...). Its documented best
|
|
6
|
+
practice is ``channel="chrome"`` + ``headless=False`` — i.e. driving the local
|
|
7
|
+
Google Chrome, whose kernel auto-updates. CloakBrowser's free tier is pinned
|
|
8
|
+
to Chromium 146 (newer 148/150 kernels are Pro-only), so it is kept as an
|
|
9
|
+
opt-in fallback. Camoufox is the last-resort fallback: an open-source
|
|
10
|
+
anti-detect Firefox (kernel Firefox 152), with a clean headless UA — no
|
|
11
|
+
``HeadlessChrome``/``HeadlessFirefox`` leak, verified at runtime.
|
|
12
|
+
|
|
13
|
+
All backends expose the same Playwright sync API surface (``launch``,
|
|
14
|
+
``launch_persistent_context``); the CloakBrowser-only ``humanize`` flag is
|
|
15
|
+
accepted everywhere and ignored on patchright.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import logging
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import shutil
|
|
24
|
+
import subprocess
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
BACKEND_PATCHRIGHT = "patchright"
|
|
31
|
+
BACKEND_CLOAKBROWSER = "cloakbrowser"
|
|
32
|
+
BACKEND_CAMOUFOX = "camoufox"
|
|
33
|
+
DEFAULT_BACKEND = BACKEND_PATCHRIGHT
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ---------------------------------------------------------------------------
|
|
37
|
+
# Availability / resolution
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
def is_available(name: str | None = None) -> bool:
|
|
41
|
+
"""Check whether a backend package is importable. None → resolved backend."""
|
|
42
|
+
if name is None:
|
|
43
|
+
name = DEFAULT_BACKEND
|
|
44
|
+
try:
|
|
45
|
+
if name == BACKEND_PATCHRIGHT:
|
|
46
|
+
import patchright # noqa: F401
|
|
47
|
+
return True
|
|
48
|
+
if name == BACKEND_CLOAKBROWSER:
|
|
49
|
+
import cloakbrowser # noqa: F401
|
|
50
|
+
return True
|
|
51
|
+
if name == BACKEND_CAMOUFOX:
|
|
52
|
+
import camoufox # noqa: F401
|
|
53
|
+
return True
|
|
54
|
+
except ImportError:
|
|
55
|
+
return False
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def resolve_backend(config: dict[str, Any] | None = None) -> str:
|
|
60
|
+
"""Resolve the effective backend from config, falling back gracefully.
|
|
61
|
+
|
|
62
|
+
``browser_backend`` config key (default "patchright"); an unknown value
|
|
63
|
+
warns and resets to the default. Fallback chain:
|
|
64
|
+
patchright → cloakbrowser → camoufox (last resort). A missing patchright
|
|
65
|
+
falls to cloakbrowser; a missing — or below the kernel floor — cloakbrowser
|
|
66
|
+
falls to camoufox when that is installed.
|
|
67
|
+
"""
|
|
68
|
+
cfg = config or {}
|
|
69
|
+
requested = str(cfg.get("browser_backend", DEFAULT_BACKEND) or DEFAULT_BACKEND).strip().lower()
|
|
70
|
+
if requested not in (BACKEND_PATCHRIGHT, BACKEND_CLOAKBROWSER, BACKEND_CAMOUFOX):
|
|
71
|
+
logger.warning("browser_backend: unknown backend '%s', using %s", requested, DEFAULT_BACKEND)
|
|
72
|
+
requested = DEFAULT_BACKEND
|
|
73
|
+
if requested == BACKEND_PATCHRIGHT and not is_available(BACKEND_PATCHRIGHT):
|
|
74
|
+
logger.warning(
|
|
75
|
+
"browser_backend: patchright not installed, falling back to cloakbrowser. "
|
|
76
|
+
"Run: pip install patchright"
|
|
77
|
+
)
|
|
78
|
+
requested = BACKEND_CLOAKBROWSER
|
|
79
|
+
if requested == BACKEND_CLOAKBROWSER and not is_available(BACKEND_CLOAKBROWSER):
|
|
80
|
+
logger.warning(
|
|
81
|
+
"browser_backend: cloakbrowser not installed, falling back to camoufox. "
|
|
82
|
+
"Run: pip install cloakbrowser (or pip install camoufox)"
|
|
83
|
+
)
|
|
84
|
+
requested = BACKEND_CAMOUFOX
|
|
85
|
+
if requested == BACKEND_CAMOUFOX and not is_available(BACKEND_CAMOUFOX):
|
|
86
|
+
logger.warning(
|
|
87
|
+
"browser_backend: camoufox not installed, falling back to cloakbrowser. "
|
|
88
|
+
"Run: pip install camoufox"
|
|
89
|
+
)
|
|
90
|
+
requested = BACKEND_CLOAKBROWSER if is_available(BACKEND_CLOAKBROWSER) else BACKEND_PATCHRIGHT
|
|
91
|
+
# A stale cloakbrowser kernel fails the hard gate at launch time; redirect
|
|
92
|
+
# to camoufox when available instead of erroring out (escape hatch kept).
|
|
93
|
+
if requested == BACKEND_CLOAKBROWSER and is_available(BACKEND_CAMOUFOX):
|
|
94
|
+
v = _cloakbrowser_dist_version()
|
|
95
|
+
if v is not None and v < CLOAKBROWSER_REQUIRED_MIN and not os.environ.get("LITLOOP_ALLOW_OLD_CLOAKBROWSER"):
|
|
96
|
+
logger.warning(
|
|
97
|
+
"browser_backend: cloakbrowser %s below required floor %s, using camoufox",
|
|
98
|
+
".".join(str(x) for x in v), ".".join(str(x) for x in CLOAKBROWSER_REQUIRED_MIN),
|
|
99
|
+
)
|
|
100
|
+
return BACKEND_CAMOUFOX
|
|
101
|
+
return requested
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# ---------------------------------------------------------------------------
|
|
105
|
+
# Local browser discovery (Chrome/Edge newer than bundled stealth Chromium 146)
|
|
106
|
+
# ---------------------------------------------------------------------------
|
|
107
|
+
# CloakBrowser's free tier is pinned to Chromium 146 (free builds stopped
|
|
108
|
+
# updating after 2026-05; newer 148/150 are Pro-only). Sites behind
|
|
109
|
+
# Cloudflare Turnstile repeatedly challenge the stale 146 fingerprint, so we
|
|
110
|
+
# prefer a local Chrome/Edge with a newer kernel.
|
|
111
|
+
# ---------------------------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
_BUILTIN_CHROMIUM_MIN = (146, 0) # bundled stealth Chromium; only swap when newer
|
|
114
|
+
|
|
115
|
+
_SYSTEM_BROWSER_PATHS_WIN = [
|
|
116
|
+
r"C:\Program Files\Google\Chrome\Application\chrome.exe",
|
|
117
|
+
r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
|
|
118
|
+
r"C:\Program Files\Microsoft\Edge\Application\msedge.exe",
|
|
119
|
+
r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
|
|
120
|
+
os.path.expandvars(r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe"),
|
|
121
|
+
]
|
|
122
|
+
|
|
123
|
+
_SYSTEM_BROWSER_NAMES_POSIX = [
|
|
124
|
+
"google-chrome", "google-chrome-stable", "chromium", "chromium-browser",
|
|
125
|
+
"microsoft-edge", "msedge",
|
|
126
|
+
]
|
|
127
|
+
|
|
128
|
+
_version_cache: dict[str, str] = {}
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _parse_version(v: str) -> tuple[int, ...]:
|
|
132
|
+
"""'150.0.7871.187' -> (150, 0, 7871, 187); (0,) when unparseable."""
|
|
133
|
+
nums = [int(n) for n in re.findall(r"\d+", v or "")]
|
|
134
|
+
return tuple(nums[:4]) if nums else (0,)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _probe_windows_versions() -> dict[str, str]:
|
|
138
|
+
"""One PowerShell call: {path: version} for every existing system browser."""
|
|
139
|
+
paths = ",".join(f"'{p}'" for p in _SYSTEM_BROWSER_PATHS_WIN)
|
|
140
|
+
script = (
|
|
141
|
+
f"$paths = @({paths}); "
|
|
142
|
+
"foreach ($p in $paths) { if (Test-Path $p) { "
|
|
143
|
+
'Write-Output ("$p`t" + (Get-Item $p).VersionInfo.ProductVersion) } }'
|
|
144
|
+
)
|
|
145
|
+
try:
|
|
146
|
+
out = subprocess.run(
|
|
147
|
+
["powershell", "-NoProfile", "-Command", script],
|
|
148
|
+
capture_output=True, text=True, timeout=10, errors="replace",
|
|
149
|
+
).stdout or ""
|
|
150
|
+
except Exception:
|
|
151
|
+
return {}
|
|
152
|
+
result: dict[str, str] = {}
|
|
153
|
+
for line in out.splitlines():
|
|
154
|
+
line = line.strip()
|
|
155
|
+
if "\t" in line:
|
|
156
|
+
path, version = line.split("\t", 1)
|
|
157
|
+
result[path.strip()] = version.strip()
|
|
158
|
+
_version_cache.update(result)
|
|
159
|
+
return result
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _probe_posix_versions() -> dict[str, str]:
|
|
163
|
+
result: dict[str, str] = {}
|
|
164
|
+
for name in _SYSTEM_BROWSER_NAMES_POSIX:
|
|
165
|
+
path = shutil.which(name)
|
|
166
|
+
if not path:
|
|
167
|
+
continue
|
|
168
|
+
try:
|
|
169
|
+
out = subprocess.run([path, "--version"], capture_output=True, text=True, timeout=5).stdout or ""
|
|
170
|
+
m = re.search(r"\d+\.\d+\.\d+\.\d+", out)
|
|
171
|
+
result[path] = m.group(0) if m else "999.0.0.0"
|
|
172
|
+
except Exception:
|
|
173
|
+
result[path] = "999.0.0.0"
|
|
174
|
+
return result
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def find_local_browser() -> str | None:
|
|
178
|
+
"""Find a local Chrome/Edge newer than the bundled stealth Chromium 146.
|
|
179
|
+
|
|
180
|
+
Returns the binary path, or None when the bundled Chromium should be kept.
|
|
181
|
+
"""
|
|
182
|
+
versions = _probe_windows_versions() if os.name == "nt" else _probe_posix_versions()
|
|
183
|
+
best: str | None = None
|
|
184
|
+
best_version = _BUILTIN_CHROMIUM_MIN
|
|
185
|
+
for path, version in versions.items():
|
|
186
|
+
parsed = _parse_version(version)
|
|
187
|
+
if parsed > best_version:
|
|
188
|
+
best, best_version = path, parsed
|
|
189
|
+
if best:
|
|
190
|
+
logger.info(
|
|
191
|
+
"browser_backend: using local browser %s (kernel %s) instead of bundled Chromium",
|
|
192
|
+
best, best_version,
|
|
193
|
+
)
|
|
194
|
+
return best
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# ---------------------------------------------------------------------------
|
|
198
|
+
# CloakBrowser kernel override (CLOAKBROWSER_BINARY_PATH protocol)
|
|
199
|
+
# ---------------------------------------------------------------------------
|
|
200
|
+
|
|
201
|
+
def resolve_browser_binary(config: dict[str, Any] | None = None) -> str | None:
|
|
202
|
+
"""Decide which browser binary CloakBrowser should launch.
|
|
203
|
+
|
|
204
|
+
Priority: explicit config ``browser_executable`` > an externally pinned
|
|
205
|
+
``CLOAKBROWSER_BINARY_PATH`` env var (left untouched) > auto-detected
|
|
206
|
+
local Chrome/Edge when ``browser_auto_upgrade`` is on (default).
|
|
207
|
+
|
|
208
|
+
Returns a path to set as CLOAKBROWSER_BINARY_PATH, or None to keep the
|
|
209
|
+
bundled stealth Chromium.
|
|
210
|
+
"""
|
|
211
|
+
cfg = config or {}
|
|
212
|
+
explicit = str(cfg.get("browser_executable", "") or "").strip()
|
|
213
|
+
if explicit:
|
|
214
|
+
if Path(explicit).exists():
|
|
215
|
+
logger.info("browser_backend: using configured browser_executable: %s", explicit)
|
|
216
|
+
return explicit
|
|
217
|
+
logger.warning("browser_backend: browser_executable '%s' not found, falling back", explicit)
|
|
218
|
+
env_path = os.environ.get("CLOAKBROWSER_BINARY_PATH", "").strip()
|
|
219
|
+
if env_path and Path(env_path).exists():
|
|
220
|
+
return None # externally pinned — leave as-is
|
|
221
|
+
if cfg.get("browser_auto_upgrade", True):
|
|
222
|
+
return find_local_browser()
|
|
223
|
+
return None
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
# ---------------------------------------------------------------------------
|
|
227
|
+
# patchright launch helpers
|
|
228
|
+
# ---------------------------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
def _patchright_browser_kwargs(config: dict[str, Any] | None) -> dict[str, Any]:
|
|
231
|
+
"""Browser selection for the patchright backend.
|
|
232
|
+
|
|
233
|
+
Explicit ``browser_executable`` wins; otherwise channel="chrome" (local
|
|
234
|
+
Google Chrome, official best practice). Returns kwargs for launch().
|
|
235
|
+
"""
|
|
236
|
+
cfg = config or {}
|
|
237
|
+
explicit = str(cfg.get("browser_executable", "") or "").strip()
|
|
238
|
+
if explicit:
|
|
239
|
+
if Path(explicit).exists():
|
|
240
|
+
# A dedicated browser (e.g. Chrome for Testing) runs under the stable
|
|
241
|
+
# litloop-chrome.exe name; the system Chrome shares its name with the
|
|
242
|
+
# user's own browser and cannot be singled out anyway.
|
|
243
|
+
dedicated = "program files" not in explicit.lower()
|
|
244
|
+
if dedicated and cfg.get("browser_copy_follow_system", True):
|
|
245
|
+
# A copy of the installed Chrome falls behind when Chrome updates.
|
|
246
|
+
from .chrome_copy import refresh_chrome_copy
|
|
247
|
+
refresh_chrome_copy(explicit)
|
|
248
|
+
return {"executable_path": stable_browser_alias(explicit) if dedicated else explicit}
|
|
249
|
+
logger.warning("browser_backend: browser_executable '%s' not found, using channel=chrome", explicit)
|
|
250
|
+
return {"channel": "chrome"}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
_DESKTOP_UA_TEMPLATE = (
|
|
254
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
255
|
+
"(KHTML, like Gecko) Chrome/{major}.0.0.0 Safari/537.36"
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _clean_patchright_ua(browser: Any, headless: bool) -> str | None:
|
|
260
|
+
"""Drop the ``HeadlessChrome/...`` UA prefix that headless Chrome sends.
|
|
261
|
+
|
|
262
|
+
Returns a desktop-UA string for the launched kernel's real major version
|
|
263
|
+
(so the UA always matches the actual binary), or None when the version
|
|
264
|
+
cannot be read. Only headless launches need this: non-headless Chrome
|
|
265
|
+
already sends the clean desktop UA.
|
|
266
|
+
"""
|
|
267
|
+
if not headless:
|
|
268
|
+
return None
|
|
269
|
+
try:
|
|
270
|
+
major = str(browser.version).split(".")[0]
|
|
271
|
+
if not major.isdigit():
|
|
272
|
+
return None
|
|
273
|
+
return _DESKTOP_UA_TEMPLATE.format(major=major)
|
|
274
|
+
except Exception:
|
|
275
|
+
return None
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _launch_patchright(
|
|
279
|
+
config: dict[str, Any] | None,
|
|
280
|
+
headless: bool,
|
|
281
|
+
proxy: Any,
|
|
282
|
+
args: list[str] | None,
|
|
283
|
+
**kwargs: Any,
|
|
284
|
+
) -> Any:
|
|
285
|
+
"""Launch via patchright (Playwright sync API, patched driver).
|
|
286
|
+
|
|
287
|
+
Tries channel=chrome / executable_path first; if the local Chrome is
|
|
288
|
+
missing, retries with the bundled chromium build (patchright install
|
|
289
|
+
chromium). ``humanize`` is a no-op here.
|
|
290
|
+
"""
|
|
291
|
+
from patchright.sync_api import sync_playwright
|
|
292
|
+
|
|
293
|
+
browser_kwargs = _patchright_browser_kwargs(config)
|
|
294
|
+
pw = sync_playwright().start()
|
|
295
|
+
last_error: Exception | None = None
|
|
296
|
+
for attempt in (browser_kwargs, {}): # preferred → bundled chromium fallback
|
|
297
|
+
try:
|
|
298
|
+
browser = pw.chromium.launch(
|
|
299
|
+
headless=headless,
|
|
300
|
+
args=args or [],
|
|
301
|
+
proxy=proxy,
|
|
302
|
+
**attempt,
|
|
303
|
+
**kwargs,
|
|
304
|
+
)
|
|
305
|
+
break
|
|
306
|
+
except Exception as exc: # noqa: BLE001 - try the fallback binary
|
|
307
|
+
last_error = exc
|
|
308
|
+
else:
|
|
309
|
+
pw.stop()
|
|
310
|
+
raise RuntimeError(
|
|
311
|
+
f"patchright could not launch any browser: {last_error}. "
|
|
312
|
+
"Install Google Chrome or run: patchright install chromium"
|
|
313
|
+
) from last_error
|
|
314
|
+
|
|
315
|
+
# Headless Chrome's UA advertises ``HeadlessChrome``, a clear automation
|
|
316
|
+
# signal to anti-bot checks. Inject a clean desktop UA at the context level
|
|
317
|
+
# (renderer-level override — page-level JS UA spoofing is itself detected,
|
|
318
|
+
# verified via sannysoft: JS override trips HEADCHR_UA, context-level UA is
|
|
319
|
+
# "ok"). setdefault keeps an explicit caller-supplied user_agent.
|
|
320
|
+
clean_ua = _clean_patchright_ua(browser, headless)
|
|
321
|
+
if clean_ua:
|
|
322
|
+
_original_new_context = browser.new_context
|
|
323
|
+
|
|
324
|
+
def _new_context_with_clean_ua(*a: Any, **kw: Any) -> Any:
|
|
325
|
+
kw.setdefault("user_agent", clean_ua)
|
|
326
|
+
return _original_new_context(*a, **kw)
|
|
327
|
+
|
|
328
|
+
browser.new_context = _new_context_with_clean_ua # type: ignore[method-assign]
|
|
329
|
+
|
|
330
|
+
# Stop the Playwright driver when the browser closes (mirrors CloakBrowser).
|
|
331
|
+
original_close = browser.close
|
|
332
|
+
|
|
333
|
+
def _close_with_cleanup() -> None:
|
|
334
|
+
try:
|
|
335
|
+
if not _driver_killed(original_close):
|
|
336
|
+
original_close()
|
|
337
|
+
finally:
|
|
338
|
+
try:
|
|
339
|
+
pw.stop()
|
|
340
|
+
except Exception:
|
|
341
|
+
pass
|
|
342
|
+
|
|
343
|
+
browser.close = _close_with_cleanup # type: ignore[method-assign]
|
|
344
|
+
return browser
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _launch_patchright_persistent(
|
|
348
|
+
config: dict[str, Any] | None,
|
|
349
|
+
user_data_dir: str,
|
|
350
|
+
headless: bool,
|
|
351
|
+
proxy: Any,
|
|
352
|
+
args: list[str] | None,
|
|
353
|
+
**kwargs: Any,
|
|
354
|
+
) -> Any:
|
|
355
|
+
"""Persistent context via patchright, with the same binary fallbacks."""
|
|
356
|
+
from patchright.sync_api import sync_playwright
|
|
357
|
+
|
|
358
|
+
browser_kwargs = _patchright_browser_kwargs(config)
|
|
359
|
+
pw = sync_playwright().start()
|
|
360
|
+
last_error: Exception | None = None
|
|
361
|
+
for attempt in (browser_kwargs, {}):
|
|
362
|
+
try:
|
|
363
|
+
context = pw.chromium.launch_persistent_context(
|
|
364
|
+
user_data_dir=user_data_dir,
|
|
365
|
+
headless=headless,
|
|
366
|
+
args=args or [],
|
|
367
|
+
proxy=proxy,
|
|
368
|
+
**attempt,
|
|
369
|
+
**kwargs,
|
|
370
|
+
)
|
|
371
|
+
break
|
|
372
|
+
except Exception as exc: # noqa: BLE001 - try the fallback binary
|
|
373
|
+
last_error = exc
|
|
374
|
+
else:
|
|
375
|
+
pw.stop()
|
|
376
|
+
raise RuntimeError(
|
|
377
|
+
f"patchright could not launch any browser: {last_error}. "
|
|
378
|
+
"Install Google Chrome or run: patchright install chromium"
|
|
379
|
+
) from last_error
|
|
380
|
+
|
|
381
|
+
original_close = context.close
|
|
382
|
+
|
|
383
|
+
def _close_with_cleanup() -> None:
|
|
384
|
+
try:
|
|
385
|
+
if not _driver_killed(original_close):
|
|
386
|
+
original_close()
|
|
387
|
+
finally:
|
|
388
|
+
try:
|
|
389
|
+
pw.stop()
|
|
390
|
+
except Exception:
|
|
391
|
+
pass
|
|
392
|
+
|
|
393
|
+
context.close = _close_with_cleanup # type: ignore[method-assign]
|
|
394
|
+
return context
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
CLOAKBROWSER_REQUIRED_MIN = (0, 5, 9) # keep in sync with the pyproject extra floor
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _cloakbrowser_dist_version() -> tuple[int, ...] | None:
|
|
401
|
+
try:
|
|
402
|
+
from importlib.metadata import version as _dist_version
|
|
403
|
+
|
|
404
|
+
raw = _dist_version("cloakbrowser")
|
|
405
|
+
except Exception:
|
|
406
|
+
return None
|
|
407
|
+
parts: list[int] = []
|
|
408
|
+
for token in raw.split("."):
|
|
409
|
+
digits = "".join(ch for ch in token if ch.isdigit())
|
|
410
|
+
parts.append(int(digits) if digits else 0)
|
|
411
|
+
return tuple(parts)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _enforce_cloakbrowser_floor() -> None:
|
|
415
|
+
"""Hard gate: stale stealth kernels get flagged by publishers, so browser
|
|
416
|
+
launches refuse to run on an outdated cloakbrowser. Escape hatch for
|
|
417
|
+
offline/emergency use: LITLOOP_ALLOW_OLD_CLOAKBROWSER=1."""
|
|
418
|
+
import os
|
|
419
|
+
|
|
420
|
+
if os.environ.get("LITLOOP_ALLOW_OLD_CLOAKBROWSER"):
|
|
421
|
+
return
|
|
422
|
+
raw = None
|
|
423
|
+
try:
|
|
424
|
+
from importlib.metadata import version as _dist_version
|
|
425
|
+
|
|
426
|
+
raw = _dist_version("cloakbrowser")
|
|
427
|
+
except Exception:
|
|
428
|
+
pass
|
|
429
|
+
v = _cloakbrowser_dist_version()
|
|
430
|
+
if v is None or v >= CLOAKBROWSER_REQUIRED_MIN:
|
|
431
|
+
return
|
|
432
|
+
required = ".".join(str(x) for x in CLOAKBROWSER_REQUIRED_MIN)
|
|
433
|
+
raise RuntimeError(
|
|
434
|
+
f"cloakbrowser {raw} is below the required minimum {required} — stale stealth "
|
|
435
|
+
"kernels get flagged by publishers. Run: pip install -U cloakbrowser "
|
|
436
|
+
"(escape hatch: set LITLOOP_ALLOW_OLD_CLOAKBROWSER=1)"
|
|
437
|
+
)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
STABLE_BROWSER_NAME = "litloop-chrome.exe"
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def stable_browser_alias(binary: str | Path) -> str:
|
|
444
|
+
"""Return a hard link named litloop-chrome.exe next to a dedicated Chromium binary.
|
|
445
|
+
|
|
446
|
+
Gives every Chromium process the plugin launches a stable, unique process
|
|
447
|
+
name, so proxy/TUN bypass rules can target it by name: versioned install
|
|
448
|
+
directories break per-path rules on upgrade, and a bare "chrome.exe" rule
|
|
449
|
+
would also catch the user's own Chrome. Recreated automatically when the
|
|
450
|
+
binary is replaced. Windows only; returns the original path on failure.
|
|
451
|
+
"""
|
|
452
|
+
original = Path(binary)
|
|
453
|
+
if os.name != "nt" or original.name.lower() == STABLE_BROWSER_NAME:
|
|
454
|
+
return str(original)
|
|
455
|
+
try:
|
|
456
|
+
alias = original.with_name(STABLE_BROWSER_NAME)
|
|
457
|
+
if not alias.exists() or alias.stat().st_size != original.stat().st_size or alias.stat().st_mtime < original.stat().st_mtime:
|
|
458
|
+
if alias.exists():
|
|
459
|
+
alias.unlink()
|
|
460
|
+
try:
|
|
461
|
+
os.link(original, alias)
|
|
462
|
+
except OSError:
|
|
463
|
+
shutil.copy2(original, alias)
|
|
464
|
+
logger.info("browser_backend: created %s -> %s", alias, original.name)
|
|
465
|
+
return str(alias)
|
|
466
|
+
except Exception as exc: # never block a launch over the alias
|
|
467
|
+
logger.info("browser_backend: stable browser name unavailable (%s)", exc)
|
|
468
|
+
return str(original)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _use_stable_cloak_binary() -> None:
|
|
472
|
+
"""Point CloakBrowser at a litloop-chrome.exe alias of its bundled Chromium."""
|
|
473
|
+
if os.name != "nt" or os.environ.get("CLOAKBROWSER_BINARY_PATH"):
|
|
474
|
+
return # non-Windows, or a binary was pinned explicitly (local Chrome, user build)
|
|
475
|
+
try:
|
|
476
|
+
from cloakbrowser.download import ensure_binary
|
|
477
|
+
os.environ["CLOAKBROWSER_BINARY_PATH"] = stable_browser_alias(ensure_binary())
|
|
478
|
+
except Exception as exc:
|
|
479
|
+
logger.info("browser_backend: stable browser name unavailable (%s)", exc)
|
|
480
|
+
|
|
481
|
+
def _launch_cloakbrowser(
|
|
482
|
+
headless: bool,
|
|
483
|
+
proxy: Any,
|
|
484
|
+
args: list[str] | None,
|
|
485
|
+
humanize: bool,
|
|
486
|
+
**kwargs: Any,
|
|
487
|
+
) -> Any:
|
|
488
|
+
_enforce_cloakbrowser_floor()
|
|
489
|
+
_use_stable_cloak_binary()
|
|
490
|
+
from cloakbrowser import launch
|
|
491
|
+
|
|
492
|
+
return launch(headless=headless, humanize=humanize, args=args, proxy=proxy, **kwargs)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _launch_cloakbrowser_persistent(
|
|
496
|
+
user_data_dir: str,
|
|
497
|
+
headless: bool,
|
|
498
|
+
proxy: Any,
|
|
499
|
+
args: list[str] | None,
|
|
500
|
+
humanize: bool,
|
|
501
|
+
**kwargs: Any,
|
|
502
|
+
) -> Any:
|
|
503
|
+
_enforce_cloakbrowser_floor()
|
|
504
|
+
_use_stable_cloak_binary()
|
|
505
|
+
from cloakbrowser import launch_persistent_context
|
|
506
|
+
|
|
507
|
+
return launch_persistent_context(
|
|
508
|
+
user_data_dir=user_data_dir,
|
|
509
|
+
headless=headless,
|
|
510
|
+
humanize=humanize,
|
|
511
|
+
args=args,
|
|
512
|
+
proxy=proxy,
|
|
513
|
+
**kwargs,
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _launch_camoufox(
|
|
518
|
+
headless: bool,
|
|
519
|
+
proxy: Any,
|
|
520
|
+
args: list[str] | None,
|
|
521
|
+
humanize: bool,
|
|
522
|
+
**kwargs: Any,
|
|
523
|
+
) -> Any:
|
|
524
|
+
"""Launch via Camoufox (anti-detect Firefox, Playwright API).
|
|
525
|
+
|
|
526
|
+
Returns a vanilla ``playwright.sync_api.Browser`` so the rest of the stack
|
|
527
|
+
(contexts, cookies, close) works unchanged. Handles its own Playwright
|
|
528
|
+
driver lifecycle, mirroring the other backends.
|
|
529
|
+
|
|
530
|
+
Callers pass Chromium-style flags (--disable-blink-features=..., ...);
|
|
531
|
+
Camoufox is a Firefox fork that configures its own anti-fingerprinting,
|
|
532
|
+
so caller args are dropped rather than handed to the Firefox binary
|
|
533
|
+
(unknown flags end up treated as open-URL arguments there).
|
|
534
|
+
"""
|
|
535
|
+
if args:
|
|
536
|
+
logger.debug("browser_backend: camoufox ignores %d caller arg(s) (Chromium flags)", len(args))
|
|
537
|
+
from camoufox import DefaultAddons, NewBrowser
|
|
538
|
+
from playwright.sync_api import sync_playwright
|
|
539
|
+
|
|
540
|
+
opts = dict(kwargs)
|
|
541
|
+
# The bundled uBlock addon downloads at build time and fails (or is
|
|
542
|
+
# rate-limited) -> exclude it, it is not needed for download sessions.
|
|
543
|
+
opts.setdefault("exclude_addons", [DefaultAddons.UBO])
|
|
544
|
+
pw = sync_playwright().start()
|
|
545
|
+
try:
|
|
546
|
+
browser = NewBrowser(
|
|
547
|
+
pw,
|
|
548
|
+
headless=headless,
|
|
549
|
+
humanize=humanize,
|
|
550
|
+
args=[], # chromium flags dropped — see docstring
|
|
551
|
+
proxy=proxy,
|
|
552
|
+
**opts,
|
|
553
|
+
)
|
|
554
|
+
except Exception:
|
|
555
|
+
pw.stop()
|
|
556
|
+
raise
|
|
557
|
+
|
|
558
|
+
original_close = browser.close
|
|
559
|
+
|
|
560
|
+
def _close_with_cleanup() -> None:
|
|
561
|
+
try:
|
|
562
|
+
if not _driver_killed(original_close):
|
|
563
|
+
original_close()
|
|
564
|
+
finally:
|
|
565
|
+
try:
|
|
566
|
+
pw.stop()
|
|
567
|
+
except Exception:
|
|
568
|
+
pass
|
|
569
|
+
|
|
570
|
+
browser.close = _close_with_cleanup # type: ignore[method-assign]
|
|
571
|
+
return browser
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def _launch_camoufox_persistent(
|
|
575
|
+
user_data_dir: str,
|
|
576
|
+
headless: bool,
|
|
577
|
+
proxy: Any,
|
|
578
|
+
args: list[str] | None,
|
|
579
|
+
humanize: bool,
|
|
580
|
+
**kwargs: Any,
|
|
581
|
+
) -> Any:
|
|
582
|
+
"""Persistent context via Camoufox (returns a BrowserContext).
|
|
583
|
+
|
|
584
|
+
Caller args (Chromium flags) are dropped — see _launch_camoufox.
|
|
585
|
+
"""
|
|
586
|
+
if args:
|
|
587
|
+
logger.debug("browser_backend: camoufox ignores %d caller arg(s) (Chromium flags)", len(args))
|
|
588
|
+
from camoufox import DefaultAddons, NewBrowser
|
|
589
|
+
from playwright.sync_api import sync_playwright
|
|
590
|
+
|
|
591
|
+
opts = dict(kwargs)
|
|
592
|
+
opts.setdefault("exclude_addons", [DefaultAddons.UBO])
|
|
593
|
+
pw = sync_playwright().start()
|
|
594
|
+
try:
|
|
595
|
+
context = NewBrowser(
|
|
596
|
+
pw,
|
|
597
|
+
persistent_context=True,
|
|
598
|
+
user_data_dir=user_data_dir,
|
|
599
|
+
headless=headless,
|
|
600
|
+
humanize=humanize,
|
|
601
|
+
args=[], # chromium flags dropped — see _launch_camoufox
|
|
602
|
+
proxy=proxy,
|
|
603
|
+
**opts,
|
|
604
|
+
)
|
|
605
|
+
except Exception:
|
|
606
|
+
pw.stop()
|
|
607
|
+
raise
|
|
608
|
+
|
|
609
|
+
original_close = context.close
|
|
610
|
+
|
|
611
|
+
def _close_with_cleanup() -> None:
|
|
612
|
+
try:
|
|
613
|
+
if not _driver_killed(original_close):
|
|
614
|
+
original_close()
|
|
615
|
+
finally:
|
|
616
|
+
try:
|
|
617
|
+
pw.stop()
|
|
618
|
+
except Exception:
|
|
619
|
+
pass
|
|
620
|
+
|
|
621
|
+
context.close = _close_with_cleanup # type: ignore[method-assign]
|
|
622
|
+
return context
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
# ---------------------------------------------------------------------------
|
|
626
|
+
# Public entry points
|
|
627
|
+
# ---------------------------------------------------------------------------
|
|
628
|
+
|
|
629
|
+
def _proxy_from_config(config: dict[str, Any] | None) -> Any:
|
|
630
|
+
"""Playwright proxy dict from config, or None.
|
|
631
|
+
|
|
632
|
+
Camoufox (Firefox) ignores Chromium's --proxy-server launch arg, so the
|
|
633
|
+
configured egress (browser_static_proxy, else network_proxy) must be
|
|
634
|
+
passed explicitly — otherwise camoufox always goes direct and every
|
|
635
|
+
proxy-required host times out.
|
|
636
|
+
"""
|
|
637
|
+
cfg = config or {}
|
|
638
|
+
proxy = (str(cfg.get("browser_static_proxy", "") or "").strip()
|
|
639
|
+
or str(cfg.get("network_proxy", "") or "").strip())
|
|
640
|
+
return {"server": proxy} if proxy else None
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def _resolve_launch_config(config: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
644
|
+
"""Fall back to the saved config when a caller passes none, so ``browser_backend``
|
|
645
|
+
and ``browser_auto_upgrade`` are honored on every launch path."""
|
|
646
|
+
if config is not None:
|
|
647
|
+
return config
|
|
648
|
+
try:
|
|
649
|
+
from ..config import load_config
|
|
650
|
+
return load_config()
|
|
651
|
+
except Exception:
|
|
652
|
+
return None
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def _direct_args(backend: str, proxy: Any, args: list[str] | None,
|
|
656
|
+
config: dict[str, Any] | None = None) -> list[str] | None:
|
|
657
|
+
"""Chromium backends bypass the OS/system proxy unless an explicit proxy is given.
|
|
658
|
+
|
|
659
|
+
With a per-host proxy exception configured (host_proxy_url/_hosts), a PAC
|
|
660
|
+
script sends only those domains through it — falling back to DIRECT if the
|
|
661
|
+
proxy goes away mid-session — and everything else stays direct.
|
|
662
|
+
"""
|
|
663
|
+
if proxy is not None or backend == BACKEND_CAMOUFOX:
|
|
664
|
+
return args
|
|
665
|
+
args = list(args or [])
|
|
666
|
+
if not any(a.startswith(("--no-proxy-server", "--proxy-server", "--proxy-pac-url")) for a in args):
|
|
667
|
+
from ..network import grey_hosts, grey_proxy, host_proxy
|
|
668
|
+
url, hosts = host_proxy(config)
|
|
669
|
+
grey = grey_proxy(config)
|
|
670
|
+
if grey: # Sci-Hub / LibGen pages opened in the browser go through it too
|
|
671
|
+
hosts = (tuple(hosts) if url == grey else ()) + grey_hosts(config)
|
|
672
|
+
url = grey
|
|
673
|
+
args.append(_pac_arg(url, hosts) if url else "--no-proxy-server")
|
|
674
|
+
return args
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _pac_arg(proxy_url: str, hosts: tuple[str, ...]) -> str:
|
|
678
|
+
import base64
|
|
679
|
+
from urllib.parse import urlparse
|
|
680
|
+
u = urlparse(proxy_url)
|
|
681
|
+
kind = "SOCKS5" if (u.scheme or "").startswith("socks") else "PROXY"
|
|
682
|
+
cond = " || ".join(f'host == "{h}" || dnsDomainIs(host, ".{h}")' for h in hosts)
|
|
683
|
+
pac = (f"function FindProxyForURL(url, host) {{ if ({cond}) "
|
|
684
|
+
f'return "{kind} {u.hostname}:{u.port}; DIRECT"; return "DIRECT"; }}')
|
|
685
|
+
return "--proxy-pac-url=data:application/x-ns-proxy-autoconfig;base64," + base64.b64encode(pac.encode()).decode()
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _launch_unregistered(
|
|
689
|
+
*,
|
|
690
|
+
headless: bool = True,
|
|
691
|
+
proxy: Any = None,
|
|
692
|
+
args: list[str] | None = None,
|
|
693
|
+
humanize: bool = True,
|
|
694
|
+
config: dict[str, Any] | None = None,
|
|
695
|
+
**kwargs: Any,
|
|
696
|
+
) -> Any:
|
|
697
|
+
"""Launch the resolved backend's stealth browser (Playwright sync API).
|
|
698
|
+
|
|
699
|
+
``humanize`` is accepted for CloakBrowser compatibility and ignored on
|
|
700
|
+
the patchright backend. Returns the same object as
|
|
701
|
+
``playwright.chromium.launch()`` / ``cloakbrowser.launch()``.
|
|
702
|
+
"""
|
|
703
|
+
config = _resolve_launch_config(config)
|
|
704
|
+
backend = resolve_backend(config)
|
|
705
|
+
if backend == BACKEND_CAMOUFOX and proxy is None:
|
|
706
|
+
proxy = _proxy_from_config(config)
|
|
707
|
+
args = _direct_args(backend, proxy, args, config)
|
|
708
|
+
if backend == BACKEND_PATCHRIGHT:
|
|
709
|
+
if humanize:
|
|
710
|
+
logger.debug("browser_backend: humanize not supported by patchright, ignored")
|
|
711
|
+
return _launch_patchright(config, headless=headless, proxy=proxy, args=args, **kwargs)
|
|
712
|
+
if backend == BACKEND_CAMOUFOX:
|
|
713
|
+
return _launch_camoufox(headless=headless, proxy=proxy, args=args, humanize=humanize, **kwargs)
|
|
714
|
+
return _launch_cloakbrowser(headless=headless, proxy=proxy, args=args, humanize=humanize, **kwargs)
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
def _launch_persistent_unregistered(
|
|
718
|
+
user_data_dir: str,
|
|
719
|
+
*,
|
|
720
|
+
headless: bool = True,
|
|
721
|
+
proxy: Any = None,
|
|
722
|
+
args: list[str] | None = None,
|
|
723
|
+
humanize: bool = True,
|
|
724
|
+
config: dict[str, Any] | None = None,
|
|
725
|
+
**kwargs: Any,
|
|
726
|
+
) -> Any:
|
|
727
|
+
"""Launch the resolved backend with a persistent profile.
|
|
728
|
+
|
|
729
|
+
Same contract as ``playwright.chromium.launch_persistent_context()``.
|
|
730
|
+
"""
|
|
731
|
+
config = _resolve_launch_config(config)
|
|
732
|
+
backend = resolve_backend(config)
|
|
733
|
+
if backend == BACKEND_CAMOUFOX and proxy is None:
|
|
734
|
+
proxy = _proxy_from_config(config)
|
|
735
|
+
args = _direct_args(backend, proxy, args, config)
|
|
736
|
+
if backend != BACKEND_CAMOUFOX:
|
|
737
|
+
# A profile whose browser was killed (the app restarted mid-download)
|
|
738
|
+
# would greet the user with Chrome's "Restore pages?" bubble.
|
|
739
|
+
args = [*(args or []), "--hide-crash-restore-bubble"]
|
|
740
|
+
if backend == BACKEND_PATCHRIGHT:
|
|
741
|
+
if humanize:
|
|
742
|
+
logger.debug("browser_backend: humanize not supported by patchright, ignored")
|
|
743
|
+
return _launch_patchright_persistent(
|
|
744
|
+
config, user_data_dir, headless=headless, proxy=proxy, args=args, **kwargs
|
|
745
|
+
)
|
|
746
|
+
if backend == BACKEND_CAMOUFOX:
|
|
747
|
+
return _launch_camoufox_persistent(
|
|
748
|
+
user_data_dir, headless=headless, proxy=proxy, args=args, humanize=humanize, **kwargs
|
|
749
|
+
)
|
|
750
|
+
return _launch_cloakbrowser_persistent(
|
|
751
|
+
user_data_dir, headless=headless, proxy=proxy, args=args, humanize=humanize, **kwargs
|
|
752
|
+
)
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def browser_info(config: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
756
|
+
"""Human-readable info about the current backend for diagnostics.
|
|
757
|
+
|
|
758
|
+
Returns {"backend": ..., "binary": ..., "version": ...}.
|
|
759
|
+
"""
|
|
760
|
+
cfg = config or {}
|
|
761
|
+
backend = resolve_backend(cfg)
|
|
762
|
+
info: dict[str, Any] = {"backend": backend, "binary": "", "version": ""}
|
|
763
|
+
if backend == BACKEND_PATCHRIGHT:
|
|
764
|
+
binary = ""
|
|
765
|
+
explicit = str(cfg.get("browser_executable", "") or "").strip()
|
|
766
|
+
if explicit and Path(explicit).exists():
|
|
767
|
+
binary = explicit
|
|
768
|
+
else:
|
|
769
|
+
versions = _probe_windows_versions() if os.name == "nt" else _probe_posix_versions()
|
|
770
|
+
for path, version in versions.items():
|
|
771
|
+
if "chrome" in path.lower():
|
|
772
|
+
binary, info["version"] = path, version
|
|
773
|
+
break
|
|
774
|
+
else:
|
|
775
|
+
binary = "channel=chrome"
|
|
776
|
+
info["binary"] = binary
|
|
777
|
+
return info
|
|
778
|
+
if backend == BACKEND_CAMOUFOX:
|
|
779
|
+
try:
|
|
780
|
+
from camoufox.pkgman import INSTALL_DIR, Version
|
|
781
|
+
installed = Version.from_path()
|
|
782
|
+
info["binary"] = str(INSTALL_DIR / "camoufox.exe") if os.name == "nt" else str(INSTALL_DIR)
|
|
783
|
+
info["version"] = installed.full_string if installed.is_supported() else "unsupported"
|
|
784
|
+
except Exception:
|
|
785
|
+
info["binary"] = "camoufox"
|
|
786
|
+
info["version"] = "?"
|
|
787
|
+
return info
|
|
788
|
+
# CloakBrowser backend
|
|
789
|
+
binary = resolve_browser_binary(cfg)
|
|
790
|
+
try:
|
|
791
|
+
from cloakbrowser.config import get_chromium_version
|
|
792
|
+
bundled = get_chromium_version()
|
|
793
|
+
except Exception:
|
|
794
|
+
bundled = "?"
|
|
795
|
+
if binary:
|
|
796
|
+
info["binary"] = binary
|
|
797
|
+
info["version"] = f"local (bundled {bundled})"
|
|
798
|
+
else:
|
|
799
|
+
info["binary"] = "bundled stealth Chromium"
|
|
800
|
+
info["version"] = bundled
|
|
801
|
+
return info
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
# ---------------------------------------------------------------------------
|
|
805
|
+
# Launch registry: lets a finished download close the windows its still-running
|
|
806
|
+
# (waived) sources opened. Racing returns on the first success, but the other
|
|
807
|
+
# source threads keep their visible browsers open until they time out.
|
|
808
|
+
# ---------------------------------------------------------------------------
|
|
809
|
+
|
|
810
|
+
import threading as _threading
|
|
811
|
+
import time as _time
|
|
812
|
+
|
|
813
|
+
_LAUNCHED: list[tuple[float, Any, str | None]] = [] # (launched at, browser, owner)
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _guard_dead_driver() -> None:
|
|
817
|
+
"""Make every sync browser call fail once its driver is gone.
|
|
818
|
+
|
|
819
|
+
patchright's sync wrapper waits for an answer by switching to the
|
|
820
|
+
dispatcher greenlet until the task is done; after the driver dies (killed
|
|
821
|
+
by close_browsers_since on cancel, skip or the school-route watchdog) that
|
|
822
|
+
greenlet is dead, the switch returns at once and the wait spins forever
|
|
823
|
+
(2026-10-17: a cancelled download stuck in browser.close() at 100% CPU).
|
|
824
|
+
"""
|
|
825
|
+
try:
|
|
826
|
+
from patchright._impl import _sync_base
|
|
827
|
+
except Exception:
|
|
828
|
+
return
|
|
829
|
+
if getattr(_sync_base.SyncBase._sync, "_litloop_guarded", False):
|
|
830
|
+
return
|
|
831
|
+
import asyncio
|
|
832
|
+
import greenlet
|
|
833
|
+
from patchright._impl._errors import Error
|
|
834
|
+
|
|
835
|
+
def _sync(self: Any, coro: Any) -> Any:
|
|
836
|
+
if self._loop.is_closed():
|
|
837
|
+
coro.close()
|
|
838
|
+
raise Error("Event loop is closed! Is Playwright already stopped?")
|
|
839
|
+
g_self = greenlet.getcurrent()
|
|
840
|
+
task = self._loop.create_task(coro)
|
|
841
|
+
task.add_done_callback(lambda _: g_self.switch())
|
|
842
|
+
while not task.done():
|
|
843
|
+
if self._dispatcher_fiber.dead:
|
|
844
|
+
task.cancel()
|
|
845
|
+
raise Error("the browser driver is gone")
|
|
846
|
+
self._dispatcher_fiber.switch()
|
|
847
|
+
asyncio._set_running_loop(self._loop)
|
|
848
|
+
return task.result()
|
|
849
|
+
|
|
850
|
+
_sync._litloop_guarded = True # type: ignore[attr-defined]
|
|
851
|
+
_sync_base.SyncBase._sync = _sync # type: ignore[method-assign]
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
_guard_dead_driver()
|
|
855
|
+
_LAUNCHED_LOCK = _threading.Lock()
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
# Set by the download orchestrator: returns the paper (DOI key) the calling
|
|
859
|
+
# thread works for, so a paper's skip, end or watchdog closes only its own
|
|
860
|
+
# browsers, not another paper's.
|
|
861
|
+
OWNER: Any = None
|
|
862
|
+
# The owner a pool thread inherited from the thread that submitted its task
|
|
863
|
+
# (see carry_owner): pool threads have none of the orchestrator's thread-locals.
|
|
864
|
+
_OWNER_TLS = _threading.local()
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def _owner() -> str | None:
|
|
868
|
+
try:
|
|
869
|
+
owner = OWNER() if OWNER is not None else None
|
|
870
|
+
except Exception:
|
|
871
|
+
owner = None
|
|
872
|
+
return owner or getattr(_OWNER_TLS, "owner", None)
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
def current_owner() -> str | None:
|
|
876
|
+
"""The paper the calling thread works for, if any."""
|
|
877
|
+
return _owner()
|
|
878
|
+
|
|
879
|
+
|
|
880
|
+
def carry_owner(fn: Any) -> Any:
|
|
881
|
+
"""Wrap ``fn`` for another thread (a pool worker): browsers it launches belong
|
|
882
|
+
to the paper the submitting thread works for, so that paper's skip or end
|
|
883
|
+
closes them (2026-10-18: Sci-Hub's mirror race opened windows nobody owned)."""
|
|
884
|
+
owner = _owner()
|
|
885
|
+
|
|
886
|
+
def run(*args: Any, **kwargs: Any) -> Any:
|
|
887
|
+
prev = getattr(_OWNER_TLS, "owner", None)
|
|
888
|
+
_OWNER_TLS.owner = owner
|
|
889
|
+
try:
|
|
890
|
+
return fn(*args, **kwargs)
|
|
891
|
+
finally:
|
|
892
|
+
_OWNER_TLS.owner = prev
|
|
893
|
+
return run
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _gone(obj: Any) -> bool:
|
|
897
|
+
"""Whether a registered browser is already dead (killed, or its driver exited)."""
|
|
898
|
+
conn = _connection_of(obj)
|
|
899
|
+
if conn is None:
|
|
900
|
+
return False
|
|
901
|
+
if getattr(conn, "_litloop_killed", False):
|
|
902
|
+
return True
|
|
903
|
+
try:
|
|
904
|
+
return conn._transport._proc.returncode is not None # type: ignore[union-attr]
|
|
905
|
+
except Exception:
|
|
906
|
+
return False
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
# Owner of a browser that several papers take turns on (a Sci-Hub pool thread's
|
|
910
|
+
# browser, a publisher group's window, a fetcher's browser): no paper's skip or
|
|
911
|
+
# end closes it, since another paper may be using it; a cancel (owner=None) does.
|
|
912
|
+
SHARED = "<shared>"
|
|
913
|
+
_SHARED_TLS = _threading.local()
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
class shared:
|
|
917
|
+
"""``with shared():`` browsers launched inside are SHARED (2026-10-19, DeepSeek
|
|
918
|
+
review: a Sci-Hub worker reuses its browser, so skipping the paper that
|
|
919
|
+
happened to launch it killed it under the next paper)."""
|
|
920
|
+
|
|
921
|
+
def __enter__(self) -> None:
|
|
922
|
+
self._prev = getattr(_SHARED_TLS, "on", False)
|
|
923
|
+
_SHARED_TLS.on = True
|
|
924
|
+
|
|
925
|
+
def __exit__(self, *exc: Any) -> None:
|
|
926
|
+
_SHARED_TLS.on = self._prev
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def _remember(obj: Any) -> Any:
|
|
930
|
+
owner = SHARED if getattr(_SHARED_TLS, "on", False) else _owner()
|
|
931
|
+
with _LAUNCHED_LOCK:
|
|
932
|
+
# Drop the dead ones, so a browser whose owner never closed it (closed by
|
|
933
|
+
# its own code, or by a crash) does not stay referenced for ever.
|
|
934
|
+
_LAUNCHED[:] = [e for e in _LAUNCHED if not _gone(e[1])]
|
|
935
|
+
_LAUNCHED.append((_time.perf_counter(), obj, owner))
|
|
936
|
+
return obj
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
# Set by the download orchestrator: returns False when the calling thread works
|
|
940
|
+
# for a download that has already finished (a waived racing source). Such
|
|
941
|
+
# threads must not (re)launch browsers -- the shared browser would otherwise be
|
|
942
|
+
# relaunched after the finished download closed it.
|
|
943
|
+
LAUNCH_GUARD: Any = None
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def _check_launch_guard() -> None:
|
|
947
|
+
if LAUNCH_GUARD is not None and not LAUNCH_GUARD():
|
|
948
|
+
raise RuntimeError("download already finished; not launching a browser for a waived source")
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def launch(**kwargs: Any) -> Any:
|
|
952
|
+
"""Launch the resolved backend's stealth browser and register it (see ``close_browsers_since``)."""
|
|
953
|
+
_check_launch_guard()
|
|
954
|
+
return _remember(_launch_unregistered(**kwargs))
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def launch_persistent_context(user_data_dir: str, **kwargs: Any) -> Any:
|
|
958
|
+
"""Launch a persistent-profile context and register it (see ``close_browsers_since``)."""
|
|
959
|
+
_check_launch_guard()
|
|
960
|
+
return _remember(_launch_persistent_unregistered(user_data_dir, **kwargs))
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
def _connection_of(obj: Any) -> Any:
|
|
964
|
+
try:
|
|
965
|
+
return obj._impl_obj._connection # type: ignore[attr-defined]
|
|
966
|
+
except Exception:
|
|
967
|
+
return None
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def _driver_killed(bound_close: Any) -> bool:
|
|
971
|
+
"""Whether close_browsers_since killed this object's driver: close() would
|
|
972
|
+
then wait forever for an answer (2026-10-17: a cancelled download spun in
|
|
973
|
+
browser.close() after its driver was killed)."""
|
|
974
|
+
conn = _connection_of(getattr(bound_close, "__self__", None))
|
|
975
|
+
return bool(conn is not None and getattr(conn, "_litloop_killed", False))
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
def close_browsers_since(t0: float = 0.0, owner: str | None = None) -> int:
|
|
979
|
+
"""Force-close every browser launched at or after ``time.perf_counter()`` value ``t0``
|
|
980
|
+
(only those launched for ``owner``, a paper's DOI key, when it is given).
|
|
981
|
+
|
|
982
|
+
Sync-API objects are bound to the thread that created them, so they cannot
|
|
983
|
+
be closed from here; kill the driver process tree instead. The owning
|
|
984
|
+
threads then fail fast, and the shared per-thread browser is relaunched on
|
|
985
|
+
next use (browser_engine checks ``is_connected``).
|
|
986
|
+
"""
|
|
987
|
+
from .engine import _tree_kill
|
|
988
|
+
|
|
989
|
+
def hit(ts: float, who: str | None) -> bool:
|
|
990
|
+
return ts >= t0 and (owner is None or who == owner)
|
|
991
|
+
|
|
992
|
+
with _LAUNCHED_LOCK:
|
|
993
|
+
targets = [obj for ts, obj, who in _LAUNCHED if hit(ts, who)]
|
|
994
|
+
_LAUNCHED[:] = [e for e in _LAUNCHED if not hit(e[0], e[2])]
|
|
995
|
+
closed = 0
|
|
996
|
+
for obj in targets:
|
|
997
|
+
conn = _connection_of(obj)
|
|
998
|
+
try:
|
|
999
|
+
proc = conn._transport._proc # type: ignore[union-attr]
|
|
1000
|
+
except Exception:
|
|
1001
|
+
proc = None
|
|
1002
|
+
if conn is not None:
|
|
1003
|
+
conn._litloop_killed = True # its close() must not wait for the dead driver
|
|
1004
|
+
try:
|
|
1005
|
+
_tree_kill(proc)
|
|
1006
|
+
closed += 1
|
|
1007
|
+
except Exception:
|
|
1008
|
+
pass
|
|
1009
|
+
return closed
|