dala 2.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dala/core/browser.py ADDED
@@ -0,0 +1,356 @@
1
+ import asyncio
2
+ import base64
3
+ import importlib.util
4
+ import os
5
+ import shutil
6
+ import tempfile
7
+ from dataclasses import dataclass, field
8
+ from pathlib import Path
9
+ from typing import Dict, Iterable, Optional
10
+ from urllib.parse import urlparse
11
+
12
+
13
+ class BrowserFetchError(RuntimeError):
14
+ """Raised when browser-backed source acquisition cannot run."""
15
+
16
+
17
+ class BrowserChallengeError(BrowserFetchError):
18
+ """Raised when browser-backed source acquisition reaches an interactive challenge."""
19
+
20
+ def __init__(self, url: str, marker: str):
21
+ self.url = url
22
+ self.marker = marker
23
+ super().__init__(
24
+ "Browser was served an anti-bot/challenge page "
25
+ f"({marker}) for {url}. Verification is required in the server browser."
26
+ )
27
+
28
+
29
+ @dataclass
30
+ class BrowserFetchOptions:
31
+ extension_path: Optional[str] = None
32
+ profile_dir: Optional[str] = None
33
+ executable_path: Optional[str] = None
34
+ headed: bool = False
35
+ timeout_ms: int = 30000
36
+ wait_until: str = "load"
37
+ settle_ms: int = 1000
38
+ challenge_action: str = "archive"
39
+
40
+
41
+ @dataclass
42
+ class BrowserFetchResult:
43
+ url: str
44
+ html: str
45
+ cookies: Dict[str, str]
46
+ assets: list = field(default_factory=list)
47
+
48
+
49
+ DEFAULT_BPC_EXTENSION_PATH = (
50
+ Path(__file__).resolve().parents[2]
51
+ / "config"
52
+ / "bpc"
53
+ / "chrome-unpacked"
54
+ / "bypass-paywalls-chrome-clean-master"
55
+ )
56
+ DEFAULT_BROWSER_PROFILE_DIR = os.path.expanduser(
57
+ os.getenv("DALA_BROWSER_PROFILE_DIR", "~/.local/share/dala/browser-profile")
58
+ )
59
+
60
+ DEFAULT_BROWSER_USER_AGENT = (
61
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
62
+ "(KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36"
63
+ )
64
+
65
+
66
+ def is_playwright_available() -> bool:
67
+ return (
68
+ importlib.util.find_spec("playwright") is not None
69
+ and importlib.util.find_spec("playwright.async_api") is not None
70
+ )
71
+
72
+
73
+ def browser_executable_exists(executable_path: Optional[str]) -> bool:
74
+ if not executable_path:
75
+ return False
76
+ expanded = Path(executable_path).expanduser()
77
+ return bool(shutil.which(executable_path) or expanded.is_file())
78
+
79
+
80
+ def browser_executable_candidates(env: Optional[Dict[str, str]] = None) -> Iterable[str]:
81
+ env = env or os.environ
82
+ names = [
83
+ "chromium",
84
+ "chromium-browser",
85
+ "google-chrome",
86
+ "google-chrome-stable",
87
+ "microsoft-edge",
88
+ "microsoft-edge-stable",
89
+ "brave-browser",
90
+ ]
91
+ yield from names
92
+
93
+ for path in [
94
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
95
+ "/Applications/Chromium.app/Contents/MacOS/Chromium",
96
+ "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
97
+ "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
98
+ ]:
99
+ yield path
100
+
101
+ windows_roots = [
102
+ env.get("PROGRAMFILES"),
103
+ env.get("PROGRAMFILES(X86)"),
104
+ env.get("LOCALAPPDATA"),
105
+ ]
106
+ windows_relatives = [
107
+ ("Google", "Chrome", "Application", "chrome.exe"),
108
+ ("Microsoft", "Edge", "Application", "msedge.exe"),
109
+ ("BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
110
+ ]
111
+ for root in windows_roots:
112
+ if not root:
113
+ continue
114
+ for parts in windows_relatives:
115
+ yield str(Path(root, *parts))
116
+
117
+
118
+ def resolve_browser_executable(requested_path: Optional[str] = None) -> Optional[str]:
119
+ candidate = requested_path or os.getenv("DALA_BROWSER_EXECUTABLE")
120
+ if candidate:
121
+ expanded = Path(candidate).expanduser()
122
+ found = shutil.which(candidate)
123
+ if found:
124
+ return found
125
+ if expanded.is_file():
126
+ return str(expanded)
127
+ return None
128
+ for candidate in browser_executable_candidates():
129
+ found = shutil.which(candidate)
130
+ if found:
131
+ return found
132
+ expanded = Path(candidate).expanduser()
133
+ if expanded.is_file():
134
+ return str(expanded)
135
+ return None
136
+
137
+
138
+ BROWSER_CHALLENGE_MARKERS = (
139
+ "geo.captcha-delivery.com",
140
+ "datadome",
141
+ "you have been blocked from the new york times",
142
+ "to continue to the new york times, please confirm that you are human",
143
+ "verification required",
144
+ "slide right to secure your access",
145
+ "please enable js and disable any ad blocker",
146
+ "enable javascript and cookies to continue",
147
+ "checking your browser before accessing",
148
+ )
149
+
150
+
151
+ def detect_browser_challenge(html: str) -> Optional[str]:
152
+ lowered = (html or "").lower()
153
+ for marker in BROWSER_CHALLENGE_MARKERS:
154
+ if marker in lowered:
155
+ return marker
156
+ return None
157
+
158
+
159
+ def resolve_browser_extension_path(requested_path: Optional[str] = None) -> Optional[str]:
160
+ path = (
161
+ requested_path
162
+ or os.getenv("DALA_BPC_EXTENSION_PATH")
163
+ or os.getenv("DALA_BROWSER_EXTENSION_PATH")
164
+ )
165
+ if path:
166
+ return path
167
+ if (DEFAULT_BPC_EXTENSION_PATH / "manifest.json").is_file():
168
+ return str(DEFAULT_BPC_EXTENSION_PATH)
169
+ return None
170
+
171
+
172
+ def validate_browser_options(options: BrowserFetchOptions) -> None:
173
+ if options.extension_path:
174
+ extension = Path(options.extension_path)
175
+ if not extension.is_dir():
176
+ raise BrowserFetchError(f"Browser extension path does not exist or is not a directory: {options.extension_path}")
177
+ if not (extension / "manifest.json").is_file():
178
+ raise BrowserFetchError(f"Browser extension path does not contain manifest.json: {options.extension_path}")
179
+ if options.profile_dir:
180
+ profile = Path(options.profile_dir)
181
+ if profile.exists() and not profile.is_dir():
182
+ raise BrowserFetchError(f"Browser profile path is not a directory: {options.profile_dir}")
183
+ if options.executable_path and not browser_executable_exists(options.executable_path):
184
+ raise BrowserFetchError(f"Browser executable path does not exist or is not a file: {options.executable_path}")
185
+ if options.wait_until not in {"domcontentloaded", "load", "networkidle"}:
186
+ raise BrowserFetchError(f"Unsupported browser wait mode: {options.wait_until}")
187
+ if options.challenge_action not in {"archive", "user_browser", "warm", "error"}:
188
+ raise BrowserFetchError(f"Unsupported browser challenge action: {options.challenge_action}")
189
+ if options.timeout_ms <= 0:
190
+ raise BrowserFetchError("--browser-timeout-ms must be greater than 0")
191
+ if options.settle_ms < 0:
192
+ raise BrowserFetchError("--browser-settle-ms must be 0 or greater")
193
+
194
+
195
+ def _cookie_applies_to_host(cookie: dict, host: str) -> bool:
196
+ domain = (cookie.get("domain") or "").lstrip(".").lower()
197
+ if not domain:
198
+ return False
199
+ return host == domain or host.endswith(f".{domain}")
200
+
201
+
202
+ def _cookies_for_url(cookies: list, url: str) -> Dict[str, str]:
203
+ host = urlparse(url).hostname or ""
204
+ host = host.lower()
205
+ jar = {}
206
+ for cookie in cookies:
207
+ if _cookie_applies_to_host(cookie, host):
208
+ name = cookie.get("name")
209
+ value = cookie.get("value")
210
+ if name is not None and value is not None:
211
+ jar[str(name)] = str(value)
212
+ return jar
213
+
214
+
215
+ async def fetch_rendered_source(url: str, options: BrowserFetchOptions) -> BrowserFetchResult:
216
+ validate_browser_options(options)
217
+
218
+ try:
219
+ from playwright.async_api import Error as PlaywrightError
220
+ from playwright.async_api import TimeoutError as PlaywrightTimeoutError
221
+ from playwright.async_api import async_playwright
222
+ except ImportError as exc:
223
+ raise BrowserFetchError(
224
+ "Browser mode requires Playwright. Install it with `uv sync --extra browser` "
225
+ "and then run `uv run playwright install chromium`."
226
+ ) from exc
227
+
228
+ launch_args = [
229
+ "--disable-blink-features=AutomationControlled",
230
+ "--disable-infobars",
231
+ "--lang=en-US",
232
+ ]
233
+ if options.extension_path:
234
+ ext_path = str(Path(options.extension_path).resolve())
235
+ launch_args.extend([
236
+ f"--disable-extensions-except={ext_path}",
237
+ f"--load-extension={ext_path}",
238
+ ])
239
+
240
+ temp_profile = None
241
+ user_data_dir = options.profile_dir
242
+ if not user_data_dir:
243
+ temp_profile = tempfile.TemporaryDirectory(prefix="dala-browser-")
244
+ user_data_dir = temp_profile.name
245
+
246
+ async def launch_context(playwright, executable_path: Optional[str] = None):
247
+ kwargs = {
248
+ "headless": not options.headed,
249
+ "args": launch_args,
250
+ "locale": os.getenv("DALA_BROWSER_LOCALE", "en-US"),
251
+ "timezone_id": os.getenv("DALA_BROWSER_TIMEZONE", "America/New_York"),
252
+ "user_agent": os.getenv("DALA_BROWSER_USER_AGENT", DEFAULT_BROWSER_USER_AGENT),
253
+ "viewport": {"width": 1365, "height": 900},
254
+ }
255
+ if executable_path:
256
+ kwargs["executable_path"] = executable_path
257
+ else:
258
+ kwargs["channel"] = "chromium"
259
+ return await playwright.chromium.launch_persistent_context(user_data_dir, **kwargs)
260
+
261
+ try:
262
+ async with async_playwright() as p:
263
+ try:
264
+ context = await launch_context(p, options.executable_path)
265
+ except PlaywrightError as exc:
266
+ fallback = None
267
+ if not options.executable_path:
268
+ fallback = resolve_browser_executable()
269
+ if fallback:
270
+ try:
271
+ context = await launch_context(p, fallback)
272
+ except PlaywrightError as fallback_exc:
273
+ raise BrowserFetchError(
274
+ "Failed to launch browser mode using both Playwright-managed Chromium "
275
+ f"and the detected Chromium-compatible browser at {fallback}. "
276
+ "Run `uv run playwright install chromium`, or pass --browser-executable "
277
+ "with a working Chrome, Edge, Brave, or Chromium binary."
278
+ ) from fallback_exc
279
+ else:
280
+ raise BrowserFetchError(
281
+ "Failed to launch browser mode. Run `uv run playwright install chromium`, "
282
+ "install Chrome/Edge/Brave/Chromium, or pass --browser-executable with a working binary."
283
+ ) from exc
284
+
285
+ try:
286
+ await context.add_init_script(
287
+ """
288
+ Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
289
+ Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
290
+ Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
291
+ """
292
+ )
293
+ page = context.pages[0] if context.pages else await context.new_page()
294
+ page.set_default_timeout(options.timeout_ms)
295
+ captured_assets = []
296
+ capture_tasks = []
297
+ seen_asset_urls = set()
298
+ max_assets = int(os.getenv("DALA_BROWSER_CAPTURE_MAX_IMAGES", "80"))
299
+ max_asset_bytes = int(float(os.getenv("DALA_BROWSER_CAPTURE_MAX_MB", "24")) * 1024 * 1024)
300
+ captured_bytes = 0
301
+
302
+ async def capture_image_response(response):
303
+ nonlocal captured_bytes
304
+ if len(captured_assets) >= max_assets:
305
+ return
306
+ asset_url = response.url
307
+ if not asset_url or asset_url in seen_asset_urls or not asset_url.startswith(("http://", "https://")):
308
+ return
309
+ request = response.request
310
+ content_type = (response.headers.get("content-type") or "").split(";", 1)[0].strip().lower()
311
+ if request.resource_type != "image" and not content_type.startswith("image/"):
312
+ return
313
+ try:
314
+ body = await response.body()
315
+ except Exception:
316
+ return
317
+ if not body:
318
+ return
319
+ if captured_bytes + len(body) > max_asset_bytes:
320
+ return
321
+ seen_asset_urls.add(asset_url)
322
+ captured_bytes += len(body)
323
+ filename_hint = os.path.basename(urlparse(asset_url).path) or "image"
324
+ captured_assets.append({
325
+ "original_url": asset_url,
326
+ "canonical_url": asset_url.split("?", 1)[0],
327
+ "filename_hint": filename_hint,
328
+ "content_type": content_type or "image/jpeg",
329
+ "content": base64.b64encode(body).decode("ascii"),
330
+ })
331
+
332
+ def on_response(response):
333
+ capture_tasks.append(asyncio.create_task(capture_image_response(response)))
334
+
335
+ page.on("response", on_response)
336
+ response = await page.goto(url, wait_until=options.wait_until, timeout=options.timeout_ms)
337
+ if options.settle_ms:
338
+ await page.wait_for_timeout(options.settle_ms)
339
+ if capture_tasks:
340
+ await asyncio.gather(*capture_tasks, return_exceptions=True)
341
+ html = await page.content()
342
+ challenge_marker = detect_browser_challenge(html)
343
+ if challenge_marker:
344
+ raise BrowserChallengeError(url, challenge_marker)
345
+ if response and response.status >= 400:
346
+ raise BrowserFetchError(f"Browser navigation returned HTTP {response.status} for {url}")
347
+ final_url = page.url
348
+ cookies = _cookies_for_url(await context.cookies(final_url), final_url)
349
+ return BrowserFetchResult(url=final_url, html=html, cookies=cookies, assets=captured_assets)
350
+ except PlaywrightTimeoutError as exc:
351
+ raise BrowserFetchError(f"Browser navigation timed out after {options.timeout_ms}ms for {url}") from exc
352
+ finally:
353
+ await context.close()
354
+ finally:
355
+ if temp_profile:
356
+ await asyncio.to_thread(temp_profile.cleanup)