dala 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dala/__init__.py +0 -0
- dala/cli.py +500 -0
- dala/core/__init__.py +0 -0
- dala/core/browser.py +356 -0
- dala/core/discovery.py +583 -0
- dala/core/dispatcher.py +55 -0
- dala/core/extractor.py +678 -0
- dala/core/forum_image_processor.py +611 -0
- dala/core/image_budget.py +198 -0
- dala/core/image_processor.py +1662 -0
- dala/core/profiles.py +56 -0
- dala/core/server_jobs.py +161 -0
- dala/core/session.py +99 -0
- dala/core/translation.py +817 -0
- dala/core/writer.py +1052 -0
- dala/data/__init__.py +1 -0
- dala/data/sites.yaml +28 -0
- dala/drivers/__init__.py +0 -0
- dala/drivers/base.py +8 -0
- dala/drivers/forum.py +461 -0
- dala/drivers/generic.py +361 -0
- dala/drivers/hn.py +170 -0
- dala/drivers/reddit.py +199 -0
- dala/drivers/substack.py +255 -0
- dala/drivers/wordpress.py +172 -0
- dala/drivers/youtube.py +427 -0
- dala/models.py +211 -0
- dala/server.py +1616 -0
- dala/setup_browser.py +98 -0
- dala/utils/__init__.py +0 -0
- dala/utils/formatting.py +98 -0
- dala/utils/llm.py +213 -0
- dala-2.4.0.dist-info/METADATA +746 -0
- dala-2.4.0.dist-info/RECORD +37 -0
- dala-2.4.0.dist-info/WHEEL +4 -0
- dala-2.4.0.dist-info/entry_points.txt +4 -0
- dala-2.4.0.dist-info/licenses/LICENSE +21 -0
dala/__init__.py
ADDED
|
File without changes
|
dala/cli.py
ADDED
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import sys
|
|
3
|
+
import os
|
|
4
|
+
import asyncio
|
|
5
|
+
import aiohttp
|
|
6
|
+
import socket
|
|
7
|
+
import time
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from urllib.parse import urlparse
|
|
10
|
+
from inspect import isawaitable
|
|
11
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
12
|
+
from tqdm.asyncio import tqdm_asyncio
|
|
13
|
+
from ebooklib import epub
|
|
14
|
+
from dotenv import load_dotenv
|
|
15
|
+
|
|
16
|
+
load_dotenv()
|
|
17
|
+
|
|
18
|
+
from dala.models import (
|
|
19
|
+
log, Source, ConversionOptions, BookData, ConversionContext,
|
|
20
|
+
GLOBAL_SEMAPHORE, REQUEST_TIMEOUT, normalize_image_preset, sanitize_filename, parse_page_spec
|
|
21
|
+
)
|
|
22
|
+
from dala.core.profiles import ProfileManager
|
|
23
|
+
from dala.core.dispatcher import DriverDispatcher
|
|
24
|
+
from dala.core.session import get_session, load_cookie_file
|
|
25
|
+
from dala.core.writer import OutputWriteError, default_output_filename, ensure_output_extension, write_output_book
|
|
26
|
+
from dala.core.browser import BrowserChallengeError, BrowserFetchError, BrowserFetchOptions, fetch_rendered_source, validate_browser_options
|
|
27
|
+
from dala.core.image_budget import ImageBudgetExceeded, assert_image_budget, prepare_books_for_bundle
|
|
28
|
+
from dala.core.discovery import DiscoveryError, discover_posts_for_sources
|
|
29
|
+
from dala.core.translation import TranslationCache, TranslationProcessor, TranslationError
|
|
30
|
+
|
|
31
|
+
async def process_urls(
|
|
32
|
+
sources: List[Source],
|
|
33
|
+
options: ConversionOptions,
|
|
34
|
+
session,
|
|
35
|
+
progress_callback: Optional[Callable[[int, int, Optional[str]], Any]] = None,
|
|
36
|
+
source_timing_callback: Optional[Callable[[str, int, bool, Optional[str]], Any]] = None,
|
|
37
|
+
) -> List[BookData]:
|
|
38
|
+
processed_books = []
|
|
39
|
+
progress_lock = asyncio.Lock()
|
|
40
|
+
completed = 0
|
|
41
|
+
total = len(sources)
|
|
42
|
+
|
|
43
|
+
async def emit_progress(processed: int, current_url: Optional[str]) -> None:
|
|
44
|
+
if not progress_callback:
|
|
45
|
+
return
|
|
46
|
+
maybe = progress_callback(processed, total, current_url)
|
|
47
|
+
if isawaitable(maybe):
|
|
48
|
+
await maybe
|
|
49
|
+
|
|
50
|
+
async def emit_source_timing(
|
|
51
|
+
url: str,
|
|
52
|
+
duration_ms: int,
|
|
53
|
+
success: bool,
|
|
54
|
+
error_type: Optional[str],
|
|
55
|
+
) -> None:
|
|
56
|
+
if not source_timing_callback:
|
|
57
|
+
return
|
|
58
|
+
maybe = source_timing_callback(url, duration_ms, success, error_type)
|
|
59
|
+
if isawaitable(maybe):
|
|
60
|
+
await maybe
|
|
61
|
+
|
|
62
|
+
async def safe_process(source):
|
|
63
|
+
nonlocal completed
|
|
64
|
+
started_at = time.perf_counter()
|
|
65
|
+
success = False
|
|
66
|
+
error_type = None
|
|
67
|
+
async with GLOBAL_SEMAPHORE:
|
|
68
|
+
profile = ProfileManager.get_instance().get_profile(source.url)
|
|
69
|
+
driver = DriverDispatcher.get_driver(source, profile)
|
|
70
|
+
|
|
71
|
+
local_session = session
|
|
72
|
+
if source.cookies:
|
|
73
|
+
connector = aiohttp.TCPConnector(
|
|
74
|
+
resolver=aiohttp.resolver.ThreadedResolver(),
|
|
75
|
+
ttl_dns_cache=300,
|
|
76
|
+
family=socket.AF_INET
|
|
77
|
+
)
|
|
78
|
+
local_session = aiohttp.ClientSession(timeout=REQUEST_TIMEOUT, cookies=source.cookies, connector=connector)
|
|
79
|
+
setattr(local_session, "_extra_cookies", source.cookies)
|
|
80
|
+
|
|
81
|
+
try:
|
|
82
|
+
context = ConversionContext(session=local_session, options=options, profile=profile)
|
|
83
|
+
book = await driver.prepare_book_data(context, source)
|
|
84
|
+
if book is None and source.is_forum:
|
|
85
|
+
from dala.drivers.forum import ForumDriver
|
|
86
|
+
from dala.drivers.generic import GenericDriver
|
|
87
|
+
|
|
88
|
+
if isinstance(driver, ForumDriver):
|
|
89
|
+
log.warning(f"Forum driver found no posts for {source.url}; retrying generic extraction.")
|
|
90
|
+
generic_source = Source(
|
|
91
|
+
url=source.url,
|
|
92
|
+
html=source.html,
|
|
93
|
+
page_htmls=source.page_htmls,
|
|
94
|
+
cookies=source.cookies,
|
|
95
|
+
assets=source.assets,
|
|
96
|
+
is_forum=False,
|
|
97
|
+
)
|
|
98
|
+
book = await GenericDriver().prepare_book_data(context, generic_source)
|
|
99
|
+
success = book is not None
|
|
100
|
+
return book
|
|
101
|
+
except BrowserChallengeError:
|
|
102
|
+
raise
|
|
103
|
+
except Exception as e:
|
|
104
|
+
error_type = type(e).__name__
|
|
105
|
+
log.exception(f"Failed to process {source.url}: {e}")
|
|
106
|
+
return None
|
|
107
|
+
finally:
|
|
108
|
+
if local_session is not session:
|
|
109
|
+
await local_session.close()
|
|
110
|
+
duration_ms = int((time.perf_counter() - started_at) * 1000)
|
|
111
|
+
await emit_source_timing(source.url, duration_ms, success, error_type)
|
|
112
|
+
async with progress_lock:
|
|
113
|
+
completed += 1
|
|
114
|
+
await emit_progress(completed, source.url)
|
|
115
|
+
|
|
116
|
+
results = await tqdm_asyncio.gather(*[safe_process(s) for s in sources], desc="Processing URLs")
|
|
117
|
+
return [b for b in results if b]
|
|
118
|
+
|
|
119
|
+
def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
|
|
120
|
+
master_uid = f"urn:bundle:{abs(hash(title))}"
|
|
121
|
+
master_chapters = []
|
|
122
|
+
master_images = []
|
|
123
|
+
master_toc = []
|
|
124
|
+
prepared_books, image_stats = prepare_books_for_bundle(books)
|
|
125
|
+
if image_stats.duplicate_count or image_stats.remapped_count:
|
|
126
|
+
log.info(
|
|
127
|
+
f"Bundle image prep: deduped={image_stats.duplicate_count}, "
|
|
128
|
+
f"renamed={image_stats.remapped_count}"
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
for book in prepared_books:
|
|
132
|
+
for img in book.images:
|
|
133
|
+
if img not in master_images: master_images.append(img)
|
|
134
|
+
|
|
135
|
+
article_chap = None
|
|
136
|
+
comments_chap = None
|
|
137
|
+
|
|
138
|
+
for chap in book.chapters:
|
|
139
|
+
chap.filename = f"doc_{abs(hash(book.source_url))}_{chap.filename}"
|
|
140
|
+
master_chapters.append(chap)
|
|
141
|
+
if chap.is_article: article_chap = chap
|
|
142
|
+
elif chap.is_comments: comments_chap = chap
|
|
143
|
+
|
|
144
|
+
if article_chap:
|
|
145
|
+
if comments_chap:
|
|
146
|
+
master_toc.append( (epub.Link(article_chap.filename, book.title, article_chap.uid), [epub.Link(comments_chap.filename, "Comments", comments_chap.uid)]) )
|
|
147
|
+
else:
|
|
148
|
+
master_toc.append(epub.Link(article_chap.filename, book.title, article_chap.uid))
|
|
149
|
+
|
|
150
|
+
return BookData(title=title, author=author, uid=master_uid, language="en", description=f"Bundle of {len(books)} articles.", source_url="", chapters=master_chapters, images=master_images, toc_structure=master_toc)
|
|
151
|
+
|
|
152
|
+
def parse_args():
|
|
153
|
+
parser = argparse.ArgumentParser(description="Universal Web to EPUB Downloader")
|
|
154
|
+
parser.add_argument("url", nargs='*', help="URL(s) to process")
|
|
155
|
+
parser.add_argument("-i", "--input-file", help="File with URLs")
|
|
156
|
+
parser.add_argument("-o", "--output", help="Output filename")
|
|
157
|
+
parser.add_argument("--format", choices=["epub", "pdf"], default="epub", dest="output_format", help="Output file format")
|
|
158
|
+
parser.add_argument("--pdf-preset", choices=["document", "ereader"], default="document", help="PDF layout preset")
|
|
159
|
+
parser.add_argument("--pdf-page-size", choices=["letter", "a4", "kobo_clara"], default="letter", help="PDF page size")
|
|
160
|
+
parser.add_argument("--no-article", action="store_true")
|
|
161
|
+
parser.add_argument("--no-images", action="store_true")
|
|
162
|
+
parser.add_argument("-a", "--archive", action="store_true")
|
|
163
|
+
parser.add_argument("--no-comments", action="store_true", help="Skip downloading comments entirely")
|
|
164
|
+
parser.add_argument("--bundle", action="store_true", help="Combine all URLs into a single EPUB")
|
|
165
|
+
parser.add_argument("--bundle-title", help="Custom title for the bundle")
|
|
166
|
+
parser.add_argument("--bundle-author", help="Custom author for the bundle")
|
|
167
|
+
parser.add_argument("--max-depth", type=int, default=None)
|
|
168
|
+
parser.add_argument("--max-pages", type=int, default=None, help="Max forum pages to fetch")
|
|
169
|
+
parser.add_argument("--max-posts", type=int, default=None, help="Max forum posts to fetch")
|
|
170
|
+
parser.add_argument("--pages", help="Specific pages to fetch (e.g. '1,3-5')")
|
|
171
|
+
parser.add_argument("--css", help="Custom CSS file to inject")
|
|
172
|
+
parser.add_argument("--llm", action="store_true", dest="llm_format", help="Format transcript using LLM (requires GEMINI_API_KEY)")
|
|
173
|
+
parser.add_argument("--llm-provider", choices=["auto", "gemini", "openrouter", "openai"], default=os.getenv("LLM_PROVIDER", "auto"), help="LLM provider selection")
|
|
174
|
+
parser.add_argument("--llm-model", help="Specify LLM model (default: gemini-3.1-flash-lite for Gemini or deepseek/deepseek-v4-flash for OpenRouter)")
|
|
175
|
+
parser.add_argument("--api-key", help="API Key for LLM (overrides env vars)")
|
|
176
|
+
parser.add_argument("--summary", action="store_true", help="Generate AI summary at start of article")
|
|
177
|
+
parser.add_argument("--translate", dest="translation_target_lang", help="Translate article text to this target language")
|
|
178
|
+
parser.add_argument("--translation-provider", choices=["llm", "google"], default="llm", help="Translation provider")
|
|
179
|
+
parser.add_argument("--translation-source", default="auto", help="Source language for translation (default: auto)")
|
|
180
|
+
parser.add_argument(
|
|
181
|
+
"--translation-display",
|
|
182
|
+
choices=["underneath", "side-by-side", "popup-footnote", "replace"],
|
|
183
|
+
default="underneath",
|
|
184
|
+
help="How to display translated text",
|
|
185
|
+
)
|
|
186
|
+
parser.add_argument(
|
|
187
|
+
"--translation-scope",
|
|
188
|
+
choices=["article", "article-captions", "all-readable"],
|
|
189
|
+
default="article-captions",
|
|
190
|
+
help="Text blocks to translate",
|
|
191
|
+
)
|
|
192
|
+
parser.add_argument("--translation-glossary", help="Glossary file with source=target lines")
|
|
193
|
+
parser.add_argument("--no-translation-cache", action="store_true", help="Disable persistent translation cache")
|
|
194
|
+
parser.add_argument("--clear-translation-cache", action="store_true", help="Remove the persistent translation cache and exit if no URL is provided")
|
|
195
|
+
parser.add_argument("--test-translation-provider", metavar="TEXT", help="Translate a short text and print the result without downloading")
|
|
196
|
+
parser.add_argument("--forum", action="store_true", help="Enable forum driver for URLs (autodetected usually)")
|
|
197
|
+
parser.add_argument("--cookie-file", help="Netscape cookie file for gated content")
|
|
198
|
+
parser.add_argument("--compact-comments", action="store_true", help="Use compact comment layout")
|
|
199
|
+
parser.add_argument("--yt-lang", default="en", help="Languages for YouTube transcripts (comma-separated, default: en)")
|
|
200
|
+
parser.add_argument("--yt-auto", action="store_true", help="Prefer auto-generated YouTube captions")
|
|
201
|
+
parser.add_argument("--thumbnails", action="store_true", help="Embed periodic thumbnails (YouTube only)")
|
|
202
|
+
parser.add_argument("--yt-max-comments", type=int, default=25, help="Max YouTube comments to fetch")
|
|
203
|
+
parser.add_argument("--yt-sort", choices=["top", "new"], default="top", help="YouTube comment sort order")
|
|
204
|
+
parser.add_argument("--browser", action="store_true", help="Fetch URLs with a headless Playwright Chromium browser")
|
|
205
|
+
parser.add_argument("--browser-extension", help="Unpacked Chromium extension directory to load with --browser")
|
|
206
|
+
parser.add_argument("--browser-profile", help="Chromium user data directory to reuse with --browser")
|
|
207
|
+
parser.add_argument("--browser-executable", help="Chromium-compatible executable path for --browser")
|
|
208
|
+
parser.add_argument("--headed", action="store_true", help="Show the browser window for --browser")
|
|
209
|
+
parser.add_argument("--browser-timeout-ms", type=int, default=30000, help="Browser navigation timeout in milliseconds")
|
|
210
|
+
parser.add_argument(
|
|
211
|
+
"--browser-wait-until",
|
|
212
|
+
choices=["domcontentloaded", "load", "networkidle"],
|
|
213
|
+
default="load",
|
|
214
|
+
help="Playwright page.goto wait condition for --browser",
|
|
215
|
+
)
|
|
216
|
+
parser.add_argument("--browser-settle-ms", type=int, default=1000, help="Extra delay after navigation before capturing HTML")
|
|
217
|
+
parser.add_argument(
|
|
218
|
+
"--browser-challenge-action",
|
|
219
|
+
choices=["archive", "user_browser", "warm", "error"],
|
|
220
|
+
default="archive",
|
|
221
|
+
help="What to do when browser fallback hits a challenge",
|
|
222
|
+
)
|
|
223
|
+
parser.add_argument("--image-preset", choices=["compact", "balanced", "full"], default="balanced", help="Image optimization/budget preset")
|
|
224
|
+
parser.add_argument("--image-color", choices=["color", "grayscale"], default="color", help="Image color mode")
|
|
225
|
+
parser.add_argument("--max-bundle-images", type=int, default=None, help="Override maximum images allowed before write")
|
|
226
|
+
parser.add_argument("--max-image-bytes-mb", type=int, default=None, help="Override maximum optimized image MB allowed before write")
|
|
227
|
+
parser.add_argument("--start-date", help="Discover and include posts on/after this date (YYYY, YYYY-MM, or YYYY-MM-DD)")
|
|
228
|
+
parser.add_argument("--end-date", help="Discover and include posts on/before this date (YYYY, YYYY-MM, or YYYY-MM-DD)")
|
|
229
|
+
parser.add_argument("--date-fallback", choices=["auto", "shallow", "metadata", "full"], default="auto", help="How hard to work to find post dates during discovery")
|
|
230
|
+
parser.add_argument("--include-undated", action="store_true", help="Include discovered posts with no date")
|
|
231
|
+
parser.add_argument("--max-discovery-pages", type=int, default=20, help="Maximum listing/archive pages to scan")
|
|
232
|
+
parser.add_argument("--max-discovered-posts", type=int, default=200, help="Maximum post candidates to discover")
|
|
233
|
+
return parser.parse_args()
|
|
234
|
+
|
|
235
|
+
async def acquire_browser_sources(sources: List[Source], options: BrowserFetchOptions) -> List[Source]:
|
|
236
|
+
validate_browser_options(options)
|
|
237
|
+
captured = []
|
|
238
|
+
for source in sources:
|
|
239
|
+
log.info(f"Fetching rendered page with browser: {source.url}")
|
|
240
|
+
try:
|
|
241
|
+
result = await fetch_rendered_source(source.url, options)
|
|
242
|
+
except BrowserChallengeError:
|
|
243
|
+
if options.challenge_action in {"user_browser", "warm", "error"}:
|
|
244
|
+
raise
|
|
245
|
+
log.warning(f"Browser challenge for {source.url}; continuing with normal fetch/archive fallback.")
|
|
246
|
+
captured.append(source)
|
|
247
|
+
continue
|
|
248
|
+
except BrowserFetchError as e:
|
|
249
|
+
log.warning(f"Browser fetch failed for {source.url}; continuing with normal fetch/archive fallback: {e}")
|
|
250
|
+
captured.append(source)
|
|
251
|
+
continue
|
|
252
|
+
cookies = dict(source.cookies or {})
|
|
253
|
+
cookies.update(result.cookies)
|
|
254
|
+
assets = list(source.assets or [])
|
|
255
|
+
for asset in result.assets or []:
|
|
256
|
+
asset_url = asset.get("original_url")
|
|
257
|
+
if asset_url and not any(existing.get("original_url") == asset_url for existing in assets):
|
|
258
|
+
assets.append(asset)
|
|
259
|
+
captured.append(Source(
|
|
260
|
+
url=result.url or source.url,
|
|
261
|
+
html=result.html,
|
|
262
|
+
cookies=cookies or None,
|
|
263
|
+
assets=assets or None,
|
|
264
|
+
is_forum=source.is_forum,
|
|
265
|
+
))
|
|
266
|
+
return captured
|
|
267
|
+
|
|
268
|
+
def cookies_for_source_url(cookie_entries: List[Dict[str, str]], u: str) -> Optional[Dict[str, str]]:
|
|
269
|
+
if not cookie_entries:
|
|
270
|
+
return None
|
|
271
|
+
host = (urlparse(u).hostname or "").lower()
|
|
272
|
+
jar = {}
|
|
273
|
+
for c in cookie_entries:
|
|
274
|
+
dom = c.get("domain", "").lower()
|
|
275
|
+
if dom and (host == dom or host.endswith(f".{dom}")):
|
|
276
|
+
jar[c["name"]] = c["value"]
|
|
277
|
+
return jar or None
|
|
278
|
+
|
|
279
|
+
def discovery_bundle_title(sources: List[Source], options: ConversionOptions) -> str:
|
|
280
|
+
domains = {urlparse(s.url).netloc.replace("www.", "") for s in sources if s.url}
|
|
281
|
+
domain = sorted(domains)[0] if len(domains) == 1 else "Discovered_Posts"
|
|
282
|
+
if options.start_date and options.end_date:
|
|
283
|
+
return f"{domain}_{options.start_date}_to_{options.end_date}"
|
|
284
|
+
if options.start_date:
|
|
285
|
+
return f"{domain}_from_{options.start_date}"
|
|
286
|
+
if options.end_date:
|
|
287
|
+
return f"{domain}_until_{options.end_date}"
|
|
288
|
+
return f"{domain}_discovered"
|
|
289
|
+
|
|
290
|
+
async def async_main():
|
|
291
|
+
args = parse_args()
|
|
292
|
+
translation_glossary = None
|
|
293
|
+
if args.translation_glossary:
|
|
294
|
+
with open(args.translation_glossary, "r", encoding="utf-8") as f:
|
|
295
|
+
translation_glossary = f.read()
|
|
296
|
+
|
|
297
|
+
if args.clear_translation_cache:
|
|
298
|
+
cleared = TranslationCache().clear()
|
|
299
|
+
print("Translation cache cleared." if cleared else "Translation cache was already empty.")
|
|
300
|
+
|
|
301
|
+
if args.test_translation_provider:
|
|
302
|
+
if not args.translation_target_lang:
|
|
303
|
+
print("--test-translation-provider requires --translate LANG.", file=sys.stderr)
|
|
304
|
+
sys.exit(2)
|
|
305
|
+
test_options = ConversionOptions(
|
|
306
|
+
llm_model=args.llm_model,
|
|
307
|
+
llm_provider=args.llm_provider,
|
|
308
|
+
llm_api_key=args.api_key,
|
|
309
|
+
translation_enabled=True,
|
|
310
|
+
translation_provider=args.translation_provider,
|
|
311
|
+
translation_target_lang=args.translation_target_lang,
|
|
312
|
+
translation_source_lang=args.translation_source,
|
|
313
|
+
translation_display=args.translation_display,
|
|
314
|
+
translation_scope=args.translation_scope,
|
|
315
|
+
translation_glossary=translation_glossary,
|
|
316
|
+
translation_cache=not args.no_translation_cache,
|
|
317
|
+
)
|
|
318
|
+
try:
|
|
319
|
+
print(await TranslationProcessor.test_provider(args.test_translation_provider, test_options))
|
|
320
|
+
except TranslationError as exc:
|
|
321
|
+
print(str(exc), file=sys.stderr)
|
|
322
|
+
sys.exit(1)
|
|
323
|
+
if not args.url and not args.input_file:
|
|
324
|
+
return
|
|
325
|
+
elif args.clear_translation_cache and not args.url and not args.input_file:
|
|
326
|
+
return
|
|
327
|
+
|
|
328
|
+
urls = []
|
|
329
|
+
if args.input_file:
|
|
330
|
+
with open(args.input_file, 'r') as f:
|
|
331
|
+
urls.extend([line.strip() for line in f if line.strip() and not line.startswith('#')])
|
|
332
|
+
urls.extend(args.url)
|
|
333
|
+
|
|
334
|
+
if not urls:
|
|
335
|
+
print("No URLs provided.")
|
|
336
|
+
return
|
|
337
|
+
|
|
338
|
+
options = ConversionOptions(
|
|
339
|
+
no_article=args.no_article,
|
|
340
|
+
no_comments=args.no_comments,
|
|
341
|
+
no_images=args.no_images,
|
|
342
|
+
archive=args.archive,
|
|
343
|
+
compact_comments=args.compact_comments,
|
|
344
|
+
max_depth=args.max_depth,
|
|
345
|
+
max_pages=args.max_pages,
|
|
346
|
+
max_posts=args.max_posts,
|
|
347
|
+
page_spec=parse_page_spec(args.pages),
|
|
348
|
+
llm_format=args.llm_format,
|
|
349
|
+
llm_provider=args.llm_provider,
|
|
350
|
+
llm_model=args.llm_model,
|
|
351
|
+
llm_api_key=args.api_key,
|
|
352
|
+
summary=args.summary,
|
|
353
|
+
translation_enabled=bool(args.translation_target_lang),
|
|
354
|
+
translation_provider=args.translation_provider,
|
|
355
|
+
translation_target_lang=args.translation_target_lang,
|
|
356
|
+
translation_source_lang=args.translation_source,
|
|
357
|
+
translation_display=args.translation_display,
|
|
358
|
+
translation_scope=args.translation_scope,
|
|
359
|
+
translation_glossary=translation_glossary,
|
|
360
|
+
translation_cache=not args.no_translation_cache,
|
|
361
|
+
youtube_lang=args.yt_lang,
|
|
362
|
+
youtube_prefer_auto=args.yt_auto,
|
|
363
|
+
thumbnails=args.thumbnails,
|
|
364
|
+
youtube_max_comments=args.yt_max_comments,
|
|
365
|
+
youtube_comment_sort=args.yt_sort,
|
|
366
|
+
image_preset=normalize_image_preset(args.image_preset),
|
|
367
|
+
image_color=args.image_color,
|
|
368
|
+
max_bundle_images=args.max_bundle_images,
|
|
369
|
+
max_image_bytes_mb=args.max_image_bytes_mb,
|
|
370
|
+
output_format=args.output_format,
|
|
371
|
+
pdf_preset=args.pdf_preset,
|
|
372
|
+
pdf_page_size=args.pdf_page_size,
|
|
373
|
+
start_date=args.start_date,
|
|
374
|
+
end_date=args.end_date,
|
|
375
|
+
date_fallback=args.date_fallback,
|
|
376
|
+
include_undated=args.include_undated,
|
|
377
|
+
max_discovery_pages=args.max_discovery_pages,
|
|
378
|
+
max_discovered_posts=args.max_discovered_posts,
|
|
379
|
+
browser_challenge_action=args.browser_challenge_action,
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
cookie_entries = load_cookie_file(args.cookie_file) if args.cookie_file else []
|
|
383
|
+
|
|
384
|
+
sources = []
|
|
385
|
+
for u in urls:
|
|
386
|
+
sources.append(Source(
|
|
387
|
+
url=u,
|
|
388
|
+
html=None,
|
|
389
|
+
cookies=cookies_for_source_url(cookie_entries, u),
|
|
390
|
+
assets=None,
|
|
391
|
+
is_forum=args.forum
|
|
392
|
+
))
|
|
393
|
+
|
|
394
|
+
if args.browser:
|
|
395
|
+
browser_options = BrowserFetchOptions(
|
|
396
|
+
extension_path=args.browser_extension,
|
|
397
|
+
profile_dir=args.browser_profile,
|
|
398
|
+
executable_path=args.browser_executable,
|
|
399
|
+
headed=args.headed,
|
|
400
|
+
timeout_ms=args.browser_timeout_ms,
|
|
401
|
+
wait_until=args.browser_wait_until,
|
|
402
|
+
settle_ms=args.browser_settle_ms,
|
|
403
|
+
challenge_action=args.browser_challenge_action,
|
|
404
|
+
)
|
|
405
|
+
try:
|
|
406
|
+
sources = await acquire_browser_sources(sources, browser_options)
|
|
407
|
+
except BrowserChallengeError as e:
|
|
408
|
+
if args.browser_challenge_action == "user_browser":
|
|
409
|
+
log.error(
|
|
410
|
+
"Browser challenge detected. Open this URL in your browser and rerun Dala from that tab: %s",
|
|
411
|
+
e.url,
|
|
412
|
+
)
|
|
413
|
+
else:
|
|
414
|
+
log.error(str(e))
|
|
415
|
+
sys.exit(1)
|
|
416
|
+
except BrowserFetchError as e:
|
|
417
|
+
log.error(str(e))
|
|
418
|
+
sys.exit(1)
|
|
419
|
+
elif args.browser_extension or args.browser_profile or args.browser_executable or args.headed:
|
|
420
|
+
log.error("--browser-extension, --browser-profile, --browser-executable, and --headed require --browser")
|
|
421
|
+
sys.exit(1)
|
|
422
|
+
|
|
423
|
+
discovery_title = None
|
|
424
|
+
async with get_session() as session:
|
|
425
|
+
if options.date_range_active:
|
|
426
|
+
try:
|
|
427
|
+
sources = await discover_posts_for_sources(session, sources, options)
|
|
428
|
+
discovery_title = discovery_bundle_title(sources, options)
|
|
429
|
+
except DiscoveryError as e:
|
|
430
|
+
log.error(str(e))
|
|
431
|
+
sys.exit(1)
|
|
432
|
+
try:
|
|
433
|
+
processed_books = await process_urls(sources, options, session)
|
|
434
|
+
except BrowserChallengeError as e:
|
|
435
|
+
if options.browser_challenge_action == "user_browser":
|
|
436
|
+
log.error(
|
|
437
|
+
"Browser challenge detected. Open this URL in your browser and rerun Dala from that tab: %s",
|
|
438
|
+
e.url,
|
|
439
|
+
)
|
|
440
|
+
else:
|
|
441
|
+
log.error(str(e))
|
|
442
|
+
sys.exit(1)
|
|
443
|
+
|
|
444
|
+
if not processed_books:
|
|
445
|
+
log.error("No content was successfully fetched.")
|
|
446
|
+
sys.exit(1)
|
|
447
|
+
|
|
448
|
+
css_content = None
|
|
449
|
+
if args.css:
|
|
450
|
+
with open(args.css) as f: css_content = f.read()
|
|
451
|
+
|
|
452
|
+
should_bundle = args.bundle or (options.date_range_active and len(processed_books) > 1)
|
|
453
|
+
if should_bundle:
|
|
454
|
+
master_title = args.bundle_title
|
|
455
|
+
if not master_title:
|
|
456
|
+
if discovery_title:
|
|
457
|
+
master_title = discovery_title
|
|
458
|
+
else:
|
|
459
|
+
date_str = datetime.now().strftime("%b_%d_%Y")
|
|
460
|
+
domains = {urlparse(b.source_url).netloc.replace("www.", "") for b in processed_books if b.source_url}
|
|
461
|
+
if len(domains) == 1: master_title = f"{list(domains)[0]}_{date_str}"
|
|
462
|
+
else: master_title = f"Anthology_{date_str}"
|
|
463
|
+
|
|
464
|
+
master_author = args.bundle_author or "Various Authors"
|
|
465
|
+
master_book = create_bundle(processed_books, master_title, master_author)
|
|
466
|
+
fname = ensure_output_extension(args.output, options.output_format) if args.output else default_output_filename(master_title, options.output_format)
|
|
467
|
+
try:
|
|
468
|
+
assert_image_budget(master_book, options)
|
|
469
|
+
except ImageBudgetExceeded as e:
|
|
470
|
+
log.error(str(e))
|
|
471
|
+
sys.exit(1)
|
|
472
|
+
try:
|
|
473
|
+
await write_output_book(master_book, fname, options, css_content)
|
|
474
|
+
except OutputWriteError as e:
|
|
475
|
+
log.error(str(e))
|
|
476
|
+
sys.exit(1)
|
|
477
|
+
else:
|
|
478
|
+
for book in processed_books:
|
|
479
|
+
fname = default_output_filename(book.title, options.output_format)
|
|
480
|
+
if args.output:
|
|
481
|
+
if len(urls) == 1:
|
|
482
|
+
fname = ensure_output_extension(args.output, options.output_format)
|
|
483
|
+
else:
|
|
484
|
+
fname = ensure_output_extension(f"{sanitize_filename(book.title)}_{args.output}", options.output_format)
|
|
485
|
+
try:
|
|
486
|
+
assert_image_budget(book, options)
|
|
487
|
+
except ImageBudgetExceeded as e:
|
|
488
|
+
log.error(str(e))
|
|
489
|
+
sys.exit(1)
|
|
490
|
+
try:
|
|
491
|
+
await write_output_book(book, fname, options, css_content)
|
|
492
|
+
except OutputWriteError as e:
|
|
493
|
+
log.error(str(e))
|
|
494
|
+
sys.exit(1)
|
|
495
|
+
|
|
496
|
+
def main():
|
|
497
|
+
asyncio.run(async_main())
|
|
498
|
+
|
|
499
|
+
if __name__ == "__main__":
|
|
500
|
+
main()
|
dala/core/__init__.py
ADDED
|
File without changes
|