dala 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dala/__init__.py +0 -0
- dala/cli.py +500 -0
- dala/core/__init__.py +0 -0
- dala/core/browser.py +356 -0
- dala/core/discovery.py +583 -0
- dala/core/dispatcher.py +55 -0
- dala/core/extractor.py +678 -0
- dala/core/forum_image_processor.py +611 -0
- dala/core/image_budget.py +198 -0
- dala/core/image_processor.py +1662 -0
- dala/core/profiles.py +56 -0
- dala/core/server_jobs.py +161 -0
- dala/core/session.py +99 -0
- dala/core/translation.py +817 -0
- dala/core/writer.py +1052 -0
- dala/data/__init__.py +1 -0
- dala/data/sites.yaml +28 -0
- dala/drivers/__init__.py +0 -0
- dala/drivers/base.py +8 -0
- dala/drivers/forum.py +461 -0
- dala/drivers/generic.py +361 -0
- dala/drivers/hn.py +170 -0
- dala/drivers/reddit.py +199 -0
- dala/drivers/substack.py +255 -0
- dala/drivers/wordpress.py +172 -0
- dala/drivers/youtube.py +427 -0
- dala/models.py +211 -0
- dala/server.py +1616 -0
- dala/setup_browser.py +98 -0
- dala/utils/__init__.py +0 -0
- dala/utils/formatting.py +98 -0
- dala/utils/llm.py +213 -0
- dala-2.4.0.dist-info/METADATA +746 -0
- dala-2.4.0.dist-info/RECORD +37 -0
- dala-2.4.0.dist-info/WHEEL +4 -0
- dala-2.4.0.dist-info/entry_points.txt +4 -0
- dala-2.4.0.dist-info/licenses/LICENSE +21 -0
dala/core/browser.py
ADDED
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import base64
|
|
3
|
+
import importlib.util
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import tempfile
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Dict, Iterable, Optional
|
|
10
|
+
from urllib.parse import urlparse
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BrowserFetchError(RuntimeError):
|
|
14
|
+
"""Raised when browser-backed source acquisition cannot run."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class BrowserChallengeError(BrowserFetchError):
|
|
18
|
+
"""Raised when browser-backed source acquisition reaches an interactive challenge."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, url: str, marker: str):
|
|
21
|
+
self.url = url
|
|
22
|
+
self.marker = marker
|
|
23
|
+
super().__init__(
|
|
24
|
+
"Browser was served an anti-bot/challenge page "
|
|
25
|
+
f"({marker}) for {url}. Verification is required in the server browser."
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class BrowserFetchOptions:
|
|
31
|
+
extension_path: Optional[str] = None
|
|
32
|
+
profile_dir: Optional[str] = None
|
|
33
|
+
executable_path: Optional[str] = None
|
|
34
|
+
headed: bool = False
|
|
35
|
+
timeout_ms: int = 30000
|
|
36
|
+
wait_until: str = "load"
|
|
37
|
+
settle_ms: int = 1000
|
|
38
|
+
challenge_action: str = "archive"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class BrowserFetchResult:
|
|
43
|
+
url: str
|
|
44
|
+
html: str
|
|
45
|
+
cookies: Dict[str, str]
|
|
46
|
+
assets: list = field(default_factory=list)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
DEFAULT_BPC_EXTENSION_PATH = (
|
|
50
|
+
Path(__file__).resolve().parents[2]
|
|
51
|
+
/ "config"
|
|
52
|
+
/ "bpc"
|
|
53
|
+
/ "chrome-unpacked"
|
|
54
|
+
/ "bypass-paywalls-chrome-clean-master"
|
|
55
|
+
)
|
|
56
|
+
DEFAULT_BROWSER_PROFILE_DIR = os.path.expanduser(
|
|
57
|
+
os.getenv("DALA_BROWSER_PROFILE_DIR", "~/.local/share/dala/browser-profile")
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
DEFAULT_BROWSER_USER_AGENT = (
|
|
61
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
|
62
|
+
"(KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def is_playwright_available() -> bool:
|
|
67
|
+
return (
|
|
68
|
+
importlib.util.find_spec("playwright") is not None
|
|
69
|
+
and importlib.util.find_spec("playwright.async_api") is not None
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def browser_executable_exists(executable_path: Optional[str]) -> bool:
|
|
74
|
+
if not executable_path:
|
|
75
|
+
return False
|
|
76
|
+
expanded = Path(executable_path).expanduser()
|
|
77
|
+
return bool(shutil.which(executable_path) or expanded.is_file())
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def browser_executable_candidates(env: Optional[Dict[str, str]] = None) -> Iterable[str]:
|
|
81
|
+
env = env or os.environ
|
|
82
|
+
names = [
|
|
83
|
+
"chromium",
|
|
84
|
+
"chromium-browser",
|
|
85
|
+
"google-chrome",
|
|
86
|
+
"google-chrome-stable",
|
|
87
|
+
"microsoft-edge",
|
|
88
|
+
"microsoft-edge-stable",
|
|
89
|
+
"brave-browser",
|
|
90
|
+
]
|
|
91
|
+
yield from names
|
|
92
|
+
|
|
93
|
+
for path in [
|
|
94
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
95
|
+
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
|
96
|
+
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
|
97
|
+
"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
|
|
98
|
+
]:
|
|
99
|
+
yield path
|
|
100
|
+
|
|
101
|
+
windows_roots = [
|
|
102
|
+
env.get("PROGRAMFILES"),
|
|
103
|
+
env.get("PROGRAMFILES(X86)"),
|
|
104
|
+
env.get("LOCALAPPDATA"),
|
|
105
|
+
]
|
|
106
|
+
windows_relatives = [
|
|
107
|
+
("Google", "Chrome", "Application", "chrome.exe"),
|
|
108
|
+
("Microsoft", "Edge", "Application", "msedge.exe"),
|
|
109
|
+
("BraveSoftware", "Brave-Browser", "Application", "brave.exe"),
|
|
110
|
+
]
|
|
111
|
+
for root in windows_roots:
|
|
112
|
+
if not root:
|
|
113
|
+
continue
|
|
114
|
+
for parts in windows_relatives:
|
|
115
|
+
yield str(Path(root, *parts))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def resolve_browser_executable(requested_path: Optional[str] = None) -> Optional[str]:
|
|
119
|
+
candidate = requested_path or os.getenv("DALA_BROWSER_EXECUTABLE")
|
|
120
|
+
if candidate:
|
|
121
|
+
expanded = Path(candidate).expanduser()
|
|
122
|
+
found = shutil.which(candidate)
|
|
123
|
+
if found:
|
|
124
|
+
return found
|
|
125
|
+
if expanded.is_file():
|
|
126
|
+
return str(expanded)
|
|
127
|
+
return None
|
|
128
|
+
for candidate in browser_executable_candidates():
|
|
129
|
+
found = shutil.which(candidate)
|
|
130
|
+
if found:
|
|
131
|
+
return found
|
|
132
|
+
expanded = Path(candidate).expanduser()
|
|
133
|
+
if expanded.is_file():
|
|
134
|
+
return str(expanded)
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
BROWSER_CHALLENGE_MARKERS = (
|
|
139
|
+
"geo.captcha-delivery.com",
|
|
140
|
+
"datadome",
|
|
141
|
+
"you have been blocked from the new york times",
|
|
142
|
+
"to continue to the new york times, please confirm that you are human",
|
|
143
|
+
"verification required",
|
|
144
|
+
"slide right to secure your access",
|
|
145
|
+
"please enable js and disable any ad blocker",
|
|
146
|
+
"enable javascript and cookies to continue",
|
|
147
|
+
"checking your browser before accessing",
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def detect_browser_challenge(html: str) -> Optional[str]:
|
|
152
|
+
lowered = (html or "").lower()
|
|
153
|
+
for marker in BROWSER_CHALLENGE_MARKERS:
|
|
154
|
+
if marker in lowered:
|
|
155
|
+
return marker
|
|
156
|
+
return None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def resolve_browser_extension_path(requested_path: Optional[str] = None) -> Optional[str]:
|
|
160
|
+
path = (
|
|
161
|
+
requested_path
|
|
162
|
+
or os.getenv("DALA_BPC_EXTENSION_PATH")
|
|
163
|
+
or os.getenv("DALA_BROWSER_EXTENSION_PATH")
|
|
164
|
+
)
|
|
165
|
+
if path:
|
|
166
|
+
return path
|
|
167
|
+
if (DEFAULT_BPC_EXTENSION_PATH / "manifest.json").is_file():
|
|
168
|
+
return str(DEFAULT_BPC_EXTENSION_PATH)
|
|
169
|
+
return None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def validate_browser_options(options: BrowserFetchOptions) -> None:
|
|
173
|
+
if options.extension_path:
|
|
174
|
+
extension = Path(options.extension_path)
|
|
175
|
+
if not extension.is_dir():
|
|
176
|
+
raise BrowserFetchError(f"Browser extension path does not exist or is not a directory: {options.extension_path}")
|
|
177
|
+
if not (extension / "manifest.json").is_file():
|
|
178
|
+
raise BrowserFetchError(f"Browser extension path does not contain manifest.json: {options.extension_path}")
|
|
179
|
+
if options.profile_dir:
|
|
180
|
+
profile = Path(options.profile_dir)
|
|
181
|
+
if profile.exists() and not profile.is_dir():
|
|
182
|
+
raise BrowserFetchError(f"Browser profile path is not a directory: {options.profile_dir}")
|
|
183
|
+
if options.executable_path and not browser_executable_exists(options.executable_path):
|
|
184
|
+
raise BrowserFetchError(f"Browser executable path does not exist or is not a file: {options.executable_path}")
|
|
185
|
+
if options.wait_until not in {"domcontentloaded", "load", "networkidle"}:
|
|
186
|
+
raise BrowserFetchError(f"Unsupported browser wait mode: {options.wait_until}")
|
|
187
|
+
if options.challenge_action not in {"archive", "user_browser", "warm", "error"}:
|
|
188
|
+
raise BrowserFetchError(f"Unsupported browser challenge action: {options.challenge_action}")
|
|
189
|
+
if options.timeout_ms <= 0:
|
|
190
|
+
raise BrowserFetchError("--browser-timeout-ms must be greater than 0")
|
|
191
|
+
if options.settle_ms < 0:
|
|
192
|
+
raise BrowserFetchError("--browser-settle-ms must be 0 or greater")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _cookie_applies_to_host(cookie: dict, host: str) -> bool:
|
|
196
|
+
domain = (cookie.get("domain") or "").lstrip(".").lower()
|
|
197
|
+
if not domain:
|
|
198
|
+
return False
|
|
199
|
+
return host == domain or host.endswith(f".{domain}")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _cookies_for_url(cookies: list, url: str) -> Dict[str, str]:
|
|
203
|
+
host = urlparse(url).hostname or ""
|
|
204
|
+
host = host.lower()
|
|
205
|
+
jar = {}
|
|
206
|
+
for cookie in cookies:
|
|
207
|
+
if _cookie_applies_to_host(cookie, host):
|
|
208
|
+
name = cookie.get("name")
|
|
209
|
+
value = cookie.get("value")
|
|
210
|
+
if name is not None and value is not None:
|
|
211
|
+
jar[str(name)] = str(value)
|
|
212
|
+
return jar
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
async def fetch_rendered_source(url: str, options: BrowserFetchOptions) -> BrowserFetchResult:
|
|
216
|
+
validate_browser_options(options)
|
|
217
|
+
|
|
218
|
+
try:
|
|
219
|
+
from playwright.async_api import Error as PlaywrightError
|
|
220
|
+
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
|
221
|
+
from playwright.async_api import async_playwright
|
|
222
|
+
except ImportError as exc:
|
|
223
|
+
raise BrowserFetchError(
|
|
224
|
+
"Browser mode requires Playwright. Install it with `uv sync --extra browser` "
|
|
225
|
+
"and then run `uv run playwright install chromium`."
|
|
226
|
+
) from exc
|
|
227
|
+
|
|
228
|
+
launch_args = [
|
|
229
|
+
"--disable-blink-features=AutomationControlled",
|
|
230
|
+
"--disable-infobars",
|
|
231
|
+
"--lang=en-US",
|
|
232
|
+
]
|
|
233
|
+
if options.extension_path:
|
|
234
|
+
ext_path = str(Path(options.extension_path).resolve())
|
|
235
|
+
launch_args.extend([
|
|
236
|
+
f"--disable-extensions-except={ext_path}",
|
|
237
|
+
f"--load-extension={ext_path}",
|
|
238
|
+
])
|
|
239
|
+
|
|
240
|
+
temp_profile = None
|
|
241
|
+
user_data_dir = options.profile_dir
|
|
242
|
+
if not user_data_dir:
|
|
243
|
+
temp_profile = tempfile.TemporaryDirectory(prefix="dala-browser-")
|
|
244
|
+
user_data_dir = temp_profile.name
|
|
245
|
+
|
|
246
|
+
async def launch_context(playwright, executable_path: Optional[str] = None):
|
|
247
|
+
kwargs = {
|
|
248
|
+
"headless": not options.headed,
|
|
249
|
+
"args": launch_args,
|
|
250
|
+
"locale": os.getenv("DALA_BROWSER_LOCALE", "en-US"),
|
|
251
|
+
"timezone_id": os.getenv("DALA_BROWSER_TIMEZONE", "America/New_York"),
|
|
252
|
+
"user_agent": os.getenv("DALA_BROWSER_USER_AGENT", DEFAULT_BROWSER_USER_AGENT),
|
|
253
|
+
"viewport": {"width": 1365, "height": 900},
|
|
254
|
+
}
|
|
255
|
+
if executable_path:
|
|
256
|
+
kwargs["executable_path"] = executable_path
|
|
257
|
+
else:
|
|
258
|
+
kwargs["channel"] = "chromium"
|
|
259
|
+
return await playwright.chromium.launch_persistent_context(user_data_dir, **kwargs)
|
|
260
|
+
|
|
261
|
+
try:
|
|
262
|
+
async with async_playwright() as p:
|
|
263
|
+
try:
|
|
264
|
+
context = await launch_context(p, options.executable_path)
|
|
265
|
+
except PlaywrightError as exc:
|
|
266
|
+
fallback = None
|
|
267
|
+
if not options.executable_path:
|
|
268
|
+
fallback = resolve_browser_executable()
|
|
269
|
+
if fallback:
|
|
270
|
+
try:
|
|
271
|
+
context = await launch_context(p, fallback)
|
|
272
|
+
except PlaywrightError as fallback_exc:
|
|
273
|
+
raise BrowserFetchError(
|
|
274
|
+
"Failed to launch browser mode using both Playwright-managed Chromium "
|
|
275
|
+
f"and the detected Chromium-compatible browser at {fallback}. "
|
|
276
|
+
"Run `uv run playwright install chromium`, or pass --browser-executable "
|
|
277
|
+
"with a working Chrome, Edge, Brave, or Chromium binary."
|
|
278
|
+
) from fallback_exc
|
|
279
|
+
else:
|
|
280
|
+
raise BrowserFetchError(
|
|
281
|
+
"Failed to launch browser mode. Run `uv run playwright install chromium`, "
|
|
282
|
+
"install Chrome/Edge/Brave/Chromium, or pass --browser-executable with a working binary."
|
|
283
|
+
) from exc
|
|
284
|
+
|
|
285
|
+
try:
|
|
286
|
+
await context.add_init_script(
|
|
287
|
+
"""
|
|
288
|
+
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
|
289
|
+
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
|
290
|
+
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
|
|
291
|
+
"""
|
|
292
|
+
)
|
|
293
|
+
page = context.pages[0] if context.pages else await context.new_page()
|
|
294
|
+
page.set_default_timeout(options.timeout_ms)
|
|
295
|
+
captured_assets = []
|
|
296
|
+
capture_tasks = []
|
|
297
|
+
seen_asset_urls = set()
|
|
298
|
+
max_assets = int(os.getenv("DALA_BROWSER_CAPTURE_MAX_IMAGES", "80"))
|
|
299
|
+
max_asset_bytes = int(float(os.getenv("DALA_BROWSER_CAPTURE_MAX_MB", "24")) * 1024 * 1024)
|
|
300
|
+
captured_bytes = 0
|
|
301
|
+
|
|
302
|
+
async def capture_image_response(response):
|
|
303
|
+
nonlocal captured_bytes
|
|
304
|
+
if len(captured_assets) >= max_assets:
|
|
305
|
+
return
|
|
306
|
+
asset_url = response.url
|
|
307
|
+
if not asset_url or asset_url in seen_asset_urls or not asset_url.startswith(("http://", "https://")):
|
|
308
|
+
return
|
|
309
|
+
request = response.request
|
|
310
|
+
content_type = (response.headers.get("content-type") or "").split(";", 1)[0].strip().lower()
|
|
311
|
+
if request.resource_type != "image" and not content_type.startswith("image/"):
|
|
312
|
+
return
|
|
313
|
+
try:
|
|
314
|
+
body = await response.body()
|
|
315
|
+
except Exception:
|
|
316
|
+
return
|
|
317
|
+
if not body:
|
|
318
|
+
return
|
|
319
|
+
if captured_bytes + len(body) > max_asset_bytes:
|
|
320
|
+
return
|
|
321
|
+
seen_asset_urls.add(asset_url)
|
|
322
|
+
captured_bytes += len(body)
|
|
323
|
+
filename_hint = os.path.basename(urlparse(asset_url).path) or "image"
|
|
324
|
+
captured_assets.append({
|
|
325
|
+
"original_url": asset_url,
|
|
326
|
+
"canonical_url": asset_url.split("?", 1)[0],
|
|
327
|
+
"filename_hint": filename_hint,
|
|
328
|
+
"content_type": content_type or "image/jpeg",
|
|
329
|
+
"content": base64.b64encode(body).decode("ascii"),
|
|
330
|
+
})
|
|
331
|
+
|
|
332
|
+
def on_response(response):
|
|
333
|
+
capture_tasks.append(asyncio.create_task(capture_image_response(response)))
|
|
334
|
+
|
|
335
|
+
page.on("response", on_response)
|
|
336
|
+
response = await page.goto(url, wait_until=options.wait_until, timeout=options.timeout_ms)
|
|
337
|
+
if options.settle_ms:
|
|
338
|
+
await page.wait_for_timeout(options.settle_ms)
|
|
339
|
+
if capture_tasks:
|
|
340
|
+
await asyncio.gather(*capture_tasks, return_exceptions=True)
|
|
341
|
+
html = await page.content()
|
|
342
|
+
challenge_marker = detect_browser_challenge(html)
|
|
343
|
+
if challenge_marker:
|
|
344
|
+
raise BrowserChallengeError(url, challenge_marker)
|
|
345
|
+
if response and response.status >= 400:
|
|
346
|
+
raise BrowserFetchError(f"Browser navigation returned HTTP {response.status} for {url}")
|
|
347
|
+
final_url = page.url
|
|
348
|
+
cookies = _cookies_for_url(await context.cookies(final_url), final_url)
|
|
349
|
+
return BrowserFetchResult(url=final_url, html=html, cookies=cookies, assets=captured_assets)
|
|
350
|
+
except PlaywrightTimeoutError as exc:
|
|
351
|
+
raise BrowserFetchError(f"Browser navigation timed out after {options.timeout_ms}ms for {url}") from exc
|
|
352
|
+
finally:
|
|
353
|
+
await context.close()
|
|
354
|
+
finally:
|
|
355
|
+
if temp_profile:
|
|
356
|
+
await asyncio.to_thread(temp_profile.cleanup)
|