dala 2.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dala/__init__.py ADDED
File without changes
dala/cli.py ADDED
@@ -0,0 +1,500 @@
1
+ import argparse
2
+ import sys
3
+ import os
4
+ import asyncio
5
+ import aiohttp
6
+ import socket
7
+ import time
8
+ from datetime import datetime
9
+ from urllib.parse import urlparse
10
+ from inspect import isawaitable
11
+ from typing import Any, Callable, Dict, List, Optional
12
+ from tqdm.asyncio import tqdm_asyncio
13
+ from ebooklib import epub
14
+ from dotenv import load_dotenv
15
+
16
+ load_dotenv()
17
+
18
+ from dala.models import (
19
+ log, Source, ConversionOptions, BookData, ConversionContext,
20
+ GLOBAL_SEMAPHORE, REQUEST_TIMEOUT, normalize_image_preset, sanitize_filename, parse_page_spec
21
+ )
22
+ from dala.core.profiles import ProfileManager
23
+ from dala.core.dispatcher import DriverDispatcher
24
+ from dala.core.session import get_session, load_cookie_file
25
+ from dala.core.writer import OutputWriteError, default_output_filename, ensure_output_extension, write_output_book
26
+ from dala.core.browser import BrowserChallengeError, BrowserFetchError, BrowserFetchOptions, fetch_rendered_source, validate_browser_options
27
+ from dala.core.image_budget import ImageBudgetExceeded, assert_image_budget, prepare_books_for_bundle
28
+ from dala.core.discovery import DiscoveryError, discover_posts_for_sources
29
+ from dala.core.translation import TranslationCache, TranslationProcessor, TranslationError
30
+
31
+ async def process_urls(
32
+ sources: List[Source],
33
+ options: ConversionOptions,
34
+ session,
35
+ progress_callback: Optional[Callable[[int, int, Optional[str]], Any]] = None,
36
+ source_timing_callback: Optional[Callable[[str, int, bool, Optional[str]], Any]] = None,
37
+ ) -> List[BookData]:
38
+ processed_books = []
39
+ progress_lock = asyncio.Lock()
40
+ completed = 0
41
+ total = len(sources)
42
+
43
+ async def emit_progress(processed: int, current_url: Optional[str]) -> None:
44
+ if not progress_callback:
45
+ return
46
+ maybe = progress_callback(processed, total, current_url)
47
+ if isawaitable(maybe):
48
+ await maybe
49
+
50
+ async def emit_source_timing(
51
+ url: str,
52
+ duration_ms: int,
53
+ success: bool,
54
+ error_type: Optional[str],
55
+ ) -> None:
56
+ if not source_timing_callback:
57
+ return
58
+ maybe = source_timing_callback(url, duration_ms, success, error_type)
59
+ if isawaitable(maybe):
60
+ await maybe
61
+
62
+ async def safe_process(source):
63
+ nonlocal completed
64
+ started_at = time.perf_counter()
65
+ success = False
66
+ error_type = None
67
+ async with GLOBAL_SEMAPHORE:
68
+ profile = ProfileManager.get_instance().get_profile(source.url)
69
+ driver = DriverDispatcher.get_driver(source, profile)
70
+
71
+ local_session = session
72
+ if source.cookies:
73
+ connector = aiohttp.TCPConnector(
74
+ resolver=aiohttp.resolver.ThreadedResolver(),
75
+ ttl_dns_cache=300,
76
+ family=socket.AF_INET
77
+ )
78
+ local_session = aiohttp.ClientSession(timeout=REQUEST_TIMEOUT, cookies=source.cookies, connector=connector)
79
+ setattr(local_session, "_extra_cookies", source.cookies)
80
+
81
+ try:
82
+ context = ConversionContext(session=local_session, options=options, profile=profile)
83
+ book = await driver.prepare_book_data(context, source)
84
+ if book is None and source.is_forum:
85
+ from dala.drivers.forum import ForumDriver
86
+ from dala.drivers.generic import GenericDriver
87
+
88
+ if isinstance(driver, ForumDriver):
89
+ log.warning(f"Forum driver found no posts for {source.url}; retrying generic extraction.")
90
+ generic_source = Source(
91
+ url=source.url,
92
+ html=source.html,
93
+ page_htmls=source.page_htmls,
94
+ cookies=source.cookies,
95
+ assets=source.assets,
96
+ is_forum=False,
97
+ )
98
+ book = await GenericDriver().prepare_book_data(context, generic_source)
99
+ success = book is not None
100
+ return book
101
+ except BrowserChallengeError:
102
+ raise
103
+ except Exception as e:
104
+ error_type = type(e).__name__
105
+ log.exception(f"Failed to process {source.url}: {e}")
106
+ return None
107
+ finally:
108
+ if local_session is not session:
109
+ await local_session.close()
110
+ duration_ms = int((time.perf_counter() - started_at) * 1000)
111
+ await emit_source_timing(source.url, duration_ms, success, error_type)
112
+ async with progress_lock:
113
+ completed += 1
114
+ await emit_progress(completed, source.url)
115
+
116
+ results = await tqdm_asyncio.gather(*[safe_process(s) for s in sources], desc="Processing URLs")
117
+ return [b for b in results if b]
118
+
119
+ def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
120
+ master_uid = f"urn:bundle:{abs(hash(title))}"
121
+ master_chapters = []
122
+ master_images = []
123
+ master_toc = []
124
+ prepared_books, image_stats = prepare_books_for_bundle(books)
125
+ if image_stats.duplicate_count or image_stats.remapped_count:
126
+ log.info(
127
+ f"Bundle image prep: deduped={image_stats.duplicate_count}, "
128
+ f"renamed={image_stats.remapped_count}"
129
+ )
130
+
131
+ for book in prepared_books:
132
+ for img in book.images:
133
+ if img not in master_images: master_images.append(img)
134
+
135
+ article_chap = None
136
+ comments_chap = None
137
+
138
+ for chap in book.chapters:
139
+ chap.filename = f"doc_{abs(hash(book.source_url))}_{chap.filename}"
140
+ master_chapters.append(chap)
141
+ if chap.is_article: article_chap = chap
142
+ elif chap.is_comments: comments_chap = chap
143
+
144
+ if article_chap:
145
+ if comments_chap:
146
+ master_toc.append( (epub.Link(article_chap.filename, book.title, article_chap.uid), [epub.Link(comments_chap.filename, "Comments", comments_chap.uid)]) )
147
+ else:
148
+ master_toc.append(epub.Link(article_chap.filename, book.title, article_chap.uid))
149
+
150
+ return BookData(title=title, author=author, uid=master_uid, language="en", description=f"Bundle of {len(books)} articles.", source_url="", chapters=master_chapters, images=master_images, toc_structure=master_toc)
151
+
152
+ def parse_args():
153
+ parser = argparse.ArgumentParser(description="Universal Web to EPUB Downloader")
154
+ parser.add_argument("url", nargs='*', help="URL(s) to process")
155
+ parser.add_argument("-i", "--input-file", help="File with URLs")
156
+ parser.add_argument("-o", "--output", help="Output filename")
157
+ parser.add_argument("--format", choices=["epub", "pdf"], default="epub", dest="output_format", help="Output file format")
158
+ parser.add_argument("--pdf-preset", choices=["document", "ereader"], default="document", help="PDF layout preset")
159
+ parser.add_argument("--pdf-page-size", choices=["letter", "a4", "kobo_clara"], default="letter", help="PDF page size")
160
+ parser.add_argument("--no-article", action="store_true")
161
+ parser.add_argument("--no-images", action="store_true")
162
+ parser.add_argument("-a", "--archive", action="store_true")
163
+ parser.add_argument("--no-comments", action="store_true", help="Skip downloading comments entirely")
164
+ parser.add_argument("--bundle", action="store_true", help="Combine all URLs into a single EPUB")
165
+ parser.add_argument("--bundle-title", help="Custom title for the bundle")
166
+ parser.add_argument("--bundle-author", help="Custom author for the bundle")
167
+ parser.add_argument("--max-depth", type=int, default=None)
168
+ parser.add_argument("--max-pages", type=int, default=None, help="Max forum pages to fetch")
169
+ parser.add_argument("--max-posts", type=int, default=None, help="Max forum posts to fetch")
170
+ parser.add_argument("--pages", help="Specific pages to fetch (e.g. '1,3-5')")
171
+ parser.add_argument("--css", help="Custom CSS file to inject")
172
+ parser.add_argument("--llm", action="store_true", dest="llm_format", help="Format transcript using LLM (requires GEMINI_API_KEY)")
173
+ parser.add_argument("--llm-provider", choices=["auto", "gemini", "openrouter", "openai"], default=os.getenv("LLM_PROVIDER", "auto"), help="LLM provider selection")
174
+ parser.add_argument("--llm-model", help="Specify LLM model (default: gemini-3.1-flash-lite for Gemini or deepseek/deepseek-v4-flash for OpenRouter)")
175
+ parser.add_argument("--api-key", help="API Key for LLM (overrides env vars)")
176
+ parser.add_argument("--summary", action="store_true", help="Generate AI summary at start of article")
177
+ parser.add_argument("--translate", dest="translation_target_lang", help="Translate article text to this target language")
178
+ parser.add_argument("--translation-provider", choices=["llm", "google"], default="llm", help="Translation provider")
179
+ parser.add_argument("--translation-source", default="auto", help="Source language for translation (default: auto)")
180
+ parser.add_argument(
181
+ "--translation-display",
182
+ choices=["underneath", "side-by-side", "popup-footnote", "replace"],
183
+ default="underneath",
184
+ help="How to display translated text",
185
+ )
186
+ parser.add_argument(
187
+ "--translation-scope",
188
+ choices=["article", "article-captions", "all-readable"],
189
+ default="article-captions",
190
+ help="Text blocks to translate",
191
+ )
192
+ parser.add_argument("--translation-glossary", help="Glossary file with source=target lines")
193
+ parser.add_argument("--no-translation-cache", action="store_true", help="Disable persistent translation cache")
194
+ parser.add_argument("--clear-translation-cache", action="store_true", help="Remove the persistent translation cache and exit if no URL is provided")
195
+ parser.add_argument("--test-translation-provider", metavar="TEXT", help="Translate a short text and print the result without downloading")
196
+ parser.add_argument("--forum", action="store_true", help="Enable forum driver for URLs (autodetected usually)")
197
+ parser.add_argument("--cookie-file", help="Netscape cookie file for gated content")
198
+ parser.add_argument("--compact-comments", action="store_true", help="Use compact comment layout")
199
+ parser.add_argument("--yt-lang", default="en", help="Languages for YouTube transcripts (comma-separated, default: en)")
200
+ parser.add_argument("--yt-auto", action="store_true", help="Prefer auto-generated YouTube captions")
201
+ parser.add_argument("--thumbnails", action="store_true", help="Embed periodic thumbnails (YouTube only)")
202
+ parser.add_argument("--yt-max-comments", type=int, default=25, help="Max YouTube comments to fetch")
203
+ parser.add_argument("--yt-sort", choices=["top", "new"], default="top", help="YouTube comment sort order")
204
+ parser.add_argument("--browser", action="store_true", help="Fetch URLs with a headless Playwright Chromium browser")
205
+ parser.add_argument("--browser-extension", help="Unpacked Chromium extension directory to load with --browser")
206
+ parser.add_argument("--browser-profile", help="Chromium user data directory to reuse with --browser")
207
+ parser.add_argument("--browser-executable", help="Chromium-compatible executable path for --browser")
208
+ parser.add_argument("--headed", action="store_true", help="Show the browser window for --browser")
209
+ parser.add_argument("--browser-timeout-ms", type=int, default=30000, help="Browser navigation timeout in milliseconds")
210
+ parser.add_argument(
211
+ "--browser-wait-until",
212
+ choices=["domcontentloaded", "load", "networkidle"],
213
+ default="load",
214
+ help="Playwright page.goto wait condition for --browser",
215
+ )
216
+ parser.add_argument("--browser-settle-ms", type=int, default=1000, help="Extra delay after navigation before capturing HTML")
217
+ parser.add_argument(
218
+ "--browser-challenge-action",
219
+ choices=["archive", "user_browser", "warm", "error"],
220
+ default="archive",
221
+ help="What to do when browser fallback hits a challenge",
222
+ )
223
+ parser.add_argument("--image-preset", choices=["compact", "balanced", "full"], default="balanced", help="Image optimization/budget preset")
224
+ parser.add_argument("--image-color", choices=["color", "grayscale"], default="color", help="Image color mode")
225
+ parser.add_argument("--max-bundle-images", type=int, default=None, help="Override maximum images allowed before write")
226
+ parser.add_argument("--max-image-bytes-mb", type=int, default=None, help="Override maximum optimized image MB allowed before write")
227
+ parser.add_argument("--start-date", help="Discover and include posts on/after this date (YYYY, YYYY-MM, or YYYY-MM-DD)")
228
+ parser.add_argument("--end-date", help="Discover and include posts on/before this date (YYYY, YYYY-MM, or YYYY-MM-DD)")
229
+ parser.add_argument("--date-fallback", choices=["auto", "shallow", "metadata", "full"], default="auto", help="How hard to work to find post dates during discovery")
230
+ parser.add_argument("--include-undated", action="store_true", help="Include discovered posts with no date")
231
+ parser.add_argument("--max-discovery-pages", type=int, default=20, help="Maximum listing/archive pages to scan")
232
+ parser.add_argument("--max-discovered-posts", type=int, default=200, help="Maximum post candidates to discover")
233
+ return parser.parse_args()
234
+
235
+ async def acquire_browser_sources(sources: List[Source], options: BrowserFetchOptions) -> List[Source]:
236
+ validate_browser_options(options)
237
+ captured = []
238
+ for source in sources:
239
+ log.info(f"Fetching rendered page with browser: {source.url}")
240
+ try:
241
+ result = await fetch_rendered_source(source.url, options)
242
+ except BrowserChallengeError:
243
+ if options.challenge_action in {"user_browser", "warm", "error"}:
244
+ raise
245
+ log.warning(f"Browser challenge for {source.url}; continuing with normal fetch/archive fallback.")
246
+ captured.append(source)
247
+ continue
248
+ except BrowserFetchError as e:
249
+ log.warning(f"Browser fetch failed for {source.url}; continuing with normal fetch/archive fallback: {e}")
250
+ captured.append(source)
251
+ continue
252
+ cookies = dict(source.cookies or {})
253
+ cookies.update(result.cookies)
254
+ assets = list(source.assets or [])
255
+ for asset in result.assets or []:
256
+ asset_url = asset.get("original_url")
257
+ if asset_url and not any(existing.get("original_url") == asset_url for existing in assets):
258
+ assets.append(asset)
259
+ captured.append(Source(
260
+ url=result.url or source.url,
261
+ html=result.html,
262
+ cookies=cookies or None,
263
+ assets=assets or None,
264
+ is_forum=source.is_forum,
265
+ ))
266
+ return captured
267
+
268
+ def cookies_for_source_url(cookie_entries: List[Dict[str, str]], u: str) -> Optional[Dict[str, str]]:
269
+ if not cookie_entries:
270
+ return None
271
+ host = (urlparse(u).hostname or "").lower()
272
+ jar = {}
273
+ for c in cookie_entries:
274
+ dom = c.get("domain", "").lower()
275
+ if dom and (host == dom or host.endswith(f".{dom}")):
276
+ jar[c["name"]] = c["value"]
277
+ return jar or None
278
+
279
+ def discovery_bundle_title(sources: List[Source], options: ConversionOptions) -> str:
280
+ domains = {urlparse(s.url).netloc.replace("www.", "") for s in sources if s.url}
281
+ domain = sorted(domains)[0] if len(domains) == 1 else "Discovered_Posts"
282
+ if options.start_date and options.end_date:
283
+ return f"{domain}_{options.start_date}_to_{options.end_date}"
284
+ if options.start_date:
285
+ return f"{domain}_from_{options.start_date}"
286
+ if options.end_date:
287
+ return f"{domain}_until_{options.end_date}"
288
+ return f"{domain}_discovered"
289
+
290
+ async def async_main():
291
+ args = parse_args()
292
+ translation_glossary = None
293
+ if args.translation_glossary:
294
+ with open(args.translation_glossary, "r", encoding="utf-8") as f:
295
+ translation_glossary = f.read()
296
+
297
+ if args.clear_translation_cache:
298
+ cleared = TranslationCache().clear()
299
+ print("Translation cache cleared." if cleared else "Translation cache was already empty.")
300
+
301
+ if args.test_translation_provider:
302
+ if not args.translation_target_lang:
303
+ print("--test-translation-provider requires --translate LANG.", file=sys.stderr)
304
+ sys.exit(2)
305
+ test_options = ConversionOptions(
306
+ llm_model=args.llm_model,
307
+ llm_provider=args.llm_provider,
308
+ llm_api_key=args.api_key,
309
+ translation_enabled=True,
310
+ translation_provider=args.translation_provider,
311
+ translation_target_lang=args.translation_target_lang,
312
+ translation_source_lang=args.translation_source,
313
+ translation_display=args.translation_display,
314
+ translation_scope=args.translation_scope,
315
+ translation_glossary=translation_glossary,
316
+ translation_cache=not args.no_translation_cache,
317
+ )
318
+ try:
319
+ print(await TranslationProcessor.test_provider(args.test_translation_provider, test_options))
320
+ except TranslationError as exc:
321
+ print(str(exc), file=sys.stderr)
322
+ sys.exit(1)
323
+ if not args.url and not args.input_file:
324
+ return
325
+ elif args.clear_translation_cache and not args.url and not args.input_file:
326
+ return
327
+
328
+ urls = []
329
+ if args.input_file:
330
+ with open(args.input_file, 'r') as f:
331
+ urls.extend([line.strip() for line in f if line.strip() and not line.startswith('#')])
332
+ urls.extend(args.url)
333
+
334
+ if not urls:
335
+ print("No URLs provided.")
336
+ return
337
+
338
+ options = ConversionOptions(
339
+ no_article=args.no_article,
340
+ no_comments=args.no_comments,
341
+ no_images=args.no_images,
342
+ archive=args.archive,
343
+ compact_comments=args.compact_comments,
344
+ max_depth=args.max_depth,
345
+ max_pages=args.max_pages,
346
+ max_posts=args.max_posts,
347
+ page_spec=parse_page_spec(args.pages),
348
+ llm_format=args.llm_format,
349
+ llm_provider=args.llm_provider,
350
+ llm_model=args.llm_model,
351
+ llm_api_key=args.api_key,
352
+ summary=args.summary,
353
+ translation_enabled=bool(args.translation_target_lang),
354
+ translation_provider=args.translation_provider,
355
+ translation_target_lang=args.translation_target_lang,
356
+ translation_source_lang=args.translation_source,
357
+ translation_display=args.translation_display,
358
+ translation_scope=args.translation_scope,
359
+ translation_glossary=translation_glossary,
360
+ translation_cache=not args.no_translation_cache,
361
+ youtube_lang=args.yt_lang,
362
+ youtube_prefer_auto=args.yt_auto,
363
+ thumbnails=args.thumbnails,
364
+ youtube_max_comments=args.yt_max_comments,
365
+ youtube_comment_sort=args.yt_sort,
366
+ image_preset=normalize_image_preset(args.image_preset),
367
+ image_color=args.image_color,
368
+ max_bundle_images=args.max_bundle_images,
369
+ max_image_bytes_mb=args.max_image_bytes_mb,
370
+ output_format=args.output_format,
371
+ pdf_preset=args.pdf_preset,
372
+ pdf_page_size=args.pdf_page_size,
373
+ start_date=args.start_date,
374
+ end_date=args.end_date,
375
+ date_fallback=args.date_fallback,
376
+ include_undated=args.include_undated,
377
+ max_discovery_pages=args.max_discovery_pages,
378
+ max_discovered_posts=args.max_discovered_posts,
379
+ browser_challenge_action=args.browser_challenge_action,
380
+ )
381
+
382
+ cookie_entries = load_cookie_file(args.cookie_file) if args.cookie_file else []
383
+
384
+ sources = []
385
+ for u in urls:
386
+ sources.append(Source(
387
+ url=u,
388
+ html=None,
389
+ cookies=cookies_for_source_url(cookie_entries, u),
390
+ assets=None,
391
+ is_forum=args.forum
392
+ ))
393
+
394
+ if args.browser:
395
+ browser_options = BrowserFetchOptions(
396
+ extension_path=args.browser_extension,
397
+ profile_dir=args.browser_profile,
398
+ executable_path=args.browser_executable,
399
+ headed=args.headed,
400
+ timeout_ms=args.browser_timeout_ms,
401
+ wait_until=args.browser_wait_until,
402
+ settle_ms=args.browser_settle_ms,
403
+ challenge_action=args.browser_challenge_action,
404
+ )
405
+ try:
406
+ sources = await acquire_browser_sources(sources, browser_options)
407
+ except BrowserChallengeError as e:
408
+ if args.browser_challenge_action == "user_browser":
409
+ log.error(
410
+ "Browser challenge detected. Open this URL in your browser and rerun Dala from that tab: %s",
411
+ e.url,
412
+ )
413
+ else:
414
+ log.error(str(e))
415
+ sys.exit(1)
416
+ except BrowserFetchError as e:
417
+ log.error(str(e))
418
+ sys.exit(1)
419
+ elif args.browser_extension or args.browser_profile or args.browser_executable or args.headed:
420
+ log.error("--browser-extension, --browser-profile, --browser-executable, and --headed require --browser")
421
+ sys.exit(1)
422
+
423
+ discovery_title = None
424
+ async with get_session() as session:
425
+ if options.date_range_active:
426
+ try:
427
+ sources = await discover_posts_for_sources(session, sources, options)
428
+ discovery_title = discovery_bundle_title(sources, options)
429
+ except DiscoveryError as e:
430
+ log.error(str(e))
431
+ sys.exit(1)
432
+ try:
433
+ processed_books = await process_urls(sources, options, session)
434
+ except BrowserChallengeError as e:
435
+ if options.browser_challenge_action == "user_browser":
436
+ log.error(
437
+ "Browser challenge detected. Open this URL in your browser and rerun Dala from that tab: %s",
438
+ e.url,
439
+ )
440
+ else:
441
+ log.error(str(e))
442
+ sys.exit(1)
443
+
444
+ if not processed_books:
445
+ log.error("No content was successfully fetched.")
446
+ sys.exit(1)
447
+
448
+ css_content = None
449
+ if args.css:
450
+ with open(args.css) as f: css_content = f.read()
451
+
452
+ should_bundle = args.bundle or (options.date_range_active and len(processed_books) > 1)
453
+ if should_bundle:
454
+ master_title = args.bundle_title
455
+ if not master_title:
456
+ if discovery_title:
457
+ master_title = discovery_title
458
+ else:
459
+ date_str = datetime.now().strftime("%b_%d_%Y")
460
+ domains = {urlparse(b.source_url).netloc.replace("www.", "") for b in processed_books if b.source_url}
461
+ if len(domains) == 1: master_title = f"{list(domains)[0]}_{date_str}"
462
+ else: master_title = f"Anthology_{date_str}"
463
+
464
+ master_author = args.bundle_author or "Various Authors"
465
+ master_book = create_bundle(processed_books, master_title, master_author)
466
+ fname = ensure_output_extension(args.output, options.output_format) if args.output else default_output_filename(master_title, options.output_format)
467
+ try:
468
+ assert_image_budget(master_book, options)
469
+ except ImageBudgetExceeded as e:
470
+ log.error(str(e))
471
+ sys.exit(1)
472
+ try:
473
+ await write_output_book(master_book, fname, options, css_content)
474
+ except OutputWriteError as e:
475
+ log.error(str(e))
476
+ sys.exit(1)
477
+ else:
478
+ for book in processed_books:
479
+ fname = default_output_filename(book.title, options.output_format)
480
+ if args.output:
481
+ if len(urls) == 1:
482
+ fname = ensure_output_extension(args.output, options.output_format)
483
+ else:
484
+ fname = ensure_output_extension(f"{sanitize_filename(book.title)}_{args.output}", options.output_format)
485
+ try:
486
+ assert_image_budget(book, options)
487
+ except ImageBudgetExceeded as e:
488
+ log.error(str(e))
489
+ sys.exit(1)
490
+ try:
491
+ await write_output_book(book, fname, options, css_content)
492
+ except OutputWriteError as e:
493
+ log.error(str(e))
494
+ sys.exit(1)
495
+
496
+ def main():
497
+ asyncio.run(async_main())
498
+
499
+ if __name__ == "__main__":
500
+ main()
dala/core/__init__.py ADDED
File without changes