dala 2.4.2__tar.gz → 2.4.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. {dala-2.4.2 → dala-2.4.10}/CHANGELOG.md +66 -0
  2. {dala-2.4.2 → dala-2.4.10}/PKG-INFO +49 -10
  3. {dala-2.4.2 → dala-2.4.10}/dala/cli.py +236 -39
  4. {dala-2.4.2 → dala-2.4.10}/dala/core/browser.py +60 -6
  5. dala-2.4.10/dala/core/debug_capture.py +35 -0
  6. dala-2.4.10/dala/core/dispatcher.py +71 -0
  7. {dala-2.4.2 → dala-2.4.10}/dala/core/extractor.py +266 -26
  8. {dala-2.4.2 → dala-2.4.10}/dala/core/image_budget.py +1 -21
  9. dala-2.4.10/dala/core/image_layout.py +236 -0
  10. {dala-2.4.2 → dala-2.4.10}/dala/core/image_processor.py +55 -18
  11. dala-2.4.10/dala/core/job_store.py +84 -0
  12. dala-2.4.10/dala/core/pdf_columns.py +57 -0
  13. dala-2.4.10/dala/core/pdf_footnotes.py +282 -0
  14. dala-2.4.10/dala/core/pdf_input.py +96 -0
  15. dala-2.4.10/dala/core/pdf_layout.py +321 -0
  16. dala-2.4.10/dala/core/pdf_links.py +291 -0
  17. dala-2.4.10/dala/core/runtime.py +61 -0
  18. {dala-2.4.2 → dala-2.4.10}/dala/core/server_jobs.py +83 -1
  19. {dala-2.4.2 → dala-2.4.10}/dala/core/session.py +7 -0
  20. {dala-2.4.2 → dala-2.4.10}/dala/core/writer.py +173 -23
  21. {dala-2.4.2 → dala-2.4.10}/dala/drivers/forum.py +65 -80
  22. {dala-2.4.2 → dala-2.4.10}/dala/drivers/generic.py +19 -15
  23. {dala-2.4.2 → dala-2.4.10}/dala/drivers/hn.py +139 -27
  24. {dala-2.4.2 → dala-2.4.10}/dala/drivers/reddit.py +34 -12
  25. {dala-2.4.2 → dala-2.4.10}/dala/drivers/substack.py +37 -33
  26. {dala-2.4.2 → dala-2.4.10}/dala/drivers/wordpress.py +15 -12
  27. {dala-2.4.2 → dala-2.4.10}/dala/drivers/youtube.py +90 -17
  28. {dala-2.4.2 → dala-2.4.10}/dala/models.py +20 -0
  29. {dala-2.4.2 → dala-2.4.10}/dala/server.py +238 -71
  30. {dala-2.4.2 → dala-2.4.10}/dala/utils/llm.py +70 -74
  31. {dala-2.4.2 → dala-2.4.10}/installers/README.md +1 -1
  32. {dala-2.4.2 → dala-2.4.10}/pyproject.toml +2 -1
  33. {dala-2.4.2 → dala-2.4.10}/readme.md +46 -8
  34. dala-2.4.10/tests/bundle_recovery_extension.cjs +79 -0
  35. dala-2.4.10/tests/conftest.py +6 -0
  36. dala-2.4.10/tests/context_menus_extension.cjs +134 -0
  37. {dala-2.4.2 → dala-2.4.10}/tests/test_browser.py +80 -0
  38. dala-2.4.10/tests/test_bundle_recovery.py +402 -0
  39. {dala-2.4.2 → dala-2.4.10}/tests/test_chrome_extension.py +131 -4
  40. dala-2.4.10/tests/test_context_menus.py +20 -0
  41. dala-2.4.10/tests/test_debug_capture.py +84 -0
  42. dala-2.4.10/tests/test_driver_urls.py +197 -0
  43. {dala-2.4.2 → dala-2.4.10}/tests/test_drivers.py +341 -7
  44. dala-2.4.10/tests/test_ft_extraction.py +194 -0
  45. {dala-2.4.2 → dala-2.4.10}/tests/test_hn_integration.py +59 -0
  46. {dala-2.4.2 → dala-2.4.10}/tests/test_image_budget.py +13 -0
  47. dala-2.4.10/tests/test_image_layout.py +266 -0
  48. dala-2.4.10/tests/test_pdf_footnotes.py +177 -0
  49. dala-2.4.10/tests/test_pdf_input.py +202 -0
  50. dala-2.4.10/tests/test_pdf_layout.py +174 -0
  51. dala-2.4.10/tests/test_pdf_links.py +278 -0
  52. dala-2.4.10/tests/test_queue_ui.py +177 -0
  53. {dala-2.4.2 → dala-2.4.10}/tests/test_server.py +110 -1
  54. {dala-2.4.2 → dala-2.4.10}/tests/test_translation.py +45 -1
  55. {dala-2.4.2 → dala-2.4.10}/tests/test_units.py +135 -1
  56. {dala-2.4.2 → dala-2.4.10}/tests/test_writer.py +93 -5
  57. dala-2.4.2/dala/core/dispatcher.py +0 -55
  58. {dala-2.4.2 → dala-2.4.10}/.gitignore +0 -0
  59. {dala-2.4.2 → dala-2.4.10}/LICENSE +0 -0
  60. {dala-2.4.2 → dala-2.4.10}/PRIVACY.md +0 -0
  61. {dala-2.4.2 → dala-2.4.10}/dala/__init__.py +0 -0
  62. {dala-2.4.2 → dala-2.4.10}/dala/core/__init__.py +0 -0
  63. {dala-2.4.2 → dala-2.4.10}/dala/core/discovery.py +0 -0
  64. {dala-2.4.2 → dala-2.4.10}/dala/core/forum_image_processor.py +0 -0
  65. {dala-2.4.2 → dala-2.4.10}/dala/core/profiles.py +0 -0
  66. {dala-2.4.2 → dala-2.4.10}/dala/core/translation.py +0 -0
  67. {dala-2.4.2 → dala-2.4.10}/dala/data/__init__.py +0 -0
  68. {dala-2.4.2 → dala-2.4.10}/dala/data/sites.json +0 -0
  69. {dala-2.4.2 → dala-2.4.10}/dala/data/sites.yaml +0 -0
  70. {dala-2.4.2 → dala-2.4.10}/dala/drivers/__init__.py +0 -0
  71. {dala-2.4.2 → dala-2.4.10}/dala/drivers/base.py +0 -0
  72. {dala-2.4.2 → dala-2.4.10}/dala/setup_browser.py +0 -0
  73. {dala-2.4.2 → dala-2.4.10}/dala/utils/__init__.py +0 -0
  74. {dala-2.4.2 → dala-2.4.10}/dala/utils/formatting.py +0 -0
  75. {dala-2.4.2 → dala-2.4.10}/installers/Install or Update Dala.bat +0 -0
  76. {dala-2.4.2 → dala-2.4.10}/installers/Install or Update Dala.command +0 -0
  77. {dala-2.4.2 → dala-2.4.10}/installers/Install or Update Dala.ps1 +0 -0
  78. {dala-2.4.2 → dala-2.4.10}/installers/Install or Update Dala.sh +0 -0
  79. {dala-2.4.2 → dala-2.4.10}/launchers/Start Dala Server.bat +0 -0
  80. {dala-2.4.2 → dala-2.4.10}/launchers/Start Dala Server.command +0 -0
  81. {dala-2.4.2 → dala-2.4.10}/launchers/dala-server.desktop +0 -0
  82. {dala-2.4.2 → dala-2.4.10}/main.py +0 -0
  83. {dala-2.4.2 → dala-2.4.10}/scripts/install-dala.ps1 +0 -0
  84. {dala-2.4.2 → dala-2.4.10}/scripts/install-dala.sh +0 -0
  85. {dala-2.4.2 → dala-2.4.10}/server.py +0 -0
  86. {dala-2.4.2 → dala-2.4.10}/sites.yaml +0 -0
  87. {dala-2.4.2 → dala-2.4.10}/tests/helpers.py +0 -0
  88. {dala-2.4.2 → dala-2.4.10}/tests/test_discovery.py +0 -0
  89. {dala-2.4.2 → dala-2.4.10}/tests/test_server_saving.py +0 -0
  90. {dala-2.4.2 → dala-2.4.10}/tests/test_setup_browser.py +0 -0
  91. {dala-2.4.2 → dala-2.4.10}/web_to_epub.py +0 -0
@@ -1,5 +1,71 @@
1
1
  # Changelog
2
2
 
3
+ ## v2.4.10
4
+
5
+ ### LLM settings
6
+
7
+ - Explain Auto's provider priority. With blank model and key fields, Dala uses the first configured provider in this order. Gemini, OpenRouter, then OpenAI.
8
+ - Refresh the model presets and defaults for all three providers. Keep custom model IDs available.
9
+ - Save model selections correctly and route direct OpenAI model IDs and key overrides to their provider.
10
+
11
+ ### Browser extensions
12
+
13
+ - Stop the popup from scrolling into blank space beyond its top and bottom edges in Chromium and Firefox.
14
+ - Align the Python package and both extensions at 2.4.10. Release assets include the Chrome ZIP, AMO-signed Firefox XPI, and installer bundle.
15
+
16
+ ## v2.4.9
17
+
18
+ ### YouTube
19
+
20
+ - Convert YouTube Shorts and livestream links directly.
21
+ - Add optional timestamp links to the first caption in each minute in EPUB and PDF exports. Enable them with `--yt-timestamps` or **Link Transcript Timestamps** under **YouTube Settings**.
22
+
23
+ ### Extraction and browser extensions
24
+
25
+ - Resolve Reddit share links and preserve thread query parameters when requesting JSON. Reddit may still block unauthenticated JSON requests.
26
+ - Follow advertised forum pagination URLs and keep browser captures on their original page numbers.
27
+ - Improve driver selection, recover comments from custom-domain WordPress posts and redirected Substack posts, and apply linked-site profiles to Hacker News articles.
28
+ - Fix Chrome download preparation on newer Chromium builds and keep saved forum settings from changing YouTube and Hacker News downloads.
29
+
30
+ ### Release assets
31
+
32
+ - Align the Python package and both browser extensions at 2.4.9. Release assets include the Chrome ZIP, AMO-signed Firefox XPI, and installer bundle.
33
+
34
+ ## v2.4.7
35
+
36
+ ### PDF sources
37
+ - Convert direct PDF URLs and PDFs linked from Hacker News, including redirects and extensionless download URLs.
38
+ - Reflow paragraphs and headings, embed photographs, vector figures and tables, and retain captions and image settings.
39
+ - Move page-bottom footnotes out of running prose; preserve resolvable references and return links across numeric, Roman, alphabetic, symbolic, column-local and endnote structures.
40
+ - Preserve annotated citations, internal cross-references and external links, including links associated with rasterized figures.
41
+ - Fall back to plain text when layout extraction fails; scans remain image-based without OCR.
42
+
43
+ ### Browser extensions
44
+ - Show saved page titles in compact queue rows, with individual selection, select-all and bulk deletion. Retain titles when editing URLs and keep concurrent queue additions.
45
+ - Restore right-click queue and download actions on background/browser startup, supporting page, link, selection, image, media and editable contexts across Firefox and Chromium menu APIs.
46
+ - Remove development-only debug capture controls and scripts from the public extensions. Local server diagnostic endpoints remain available for development.
47
+
48
+ ### Extraction and packaging
49
+ - Improve Financial Times web/app capture cleanup and preserve photographs inside image controls, responsive picture elements and inline placeholders.
50
+ - Keep PDF anchors and caption links during ebook cleanup; use URL-safe image filenames and surface EPUB writing errors.
51
+ - Add `pdfplumber` and its PDFium renderer to the base dependencies for PDF input. PDF output still requires optional browser support.
52
+ - Keep the Python package at 2.4.7; replace this release’s browser assets with extension version 2.4.8 to remove debug controls. Rebuild Chrome, AMO-signed Firefox and installer assets.
53
+
54
+ ## v2.4.6
55
+
56
+ ### Bundle reliability
57
+ - Continue processing when individual sources fail or stall, with bounded source and browser recovery deadlines.
58
+ - Reuse already-open challenge tabs and accept successful captures during the recovery window.
59
+ - Checkpoint completed articles for retries after server restarts; expose progress, finish-available, and retry-skipped controls without additional toasts.
60
+
61
+ ### Figures and offline media
62
+ - Preserve inline figures, captions, and figure anchors using explicit page relationships across publishers.
63
+ - Reuse captured responsive and lazy-loaded image variants before network recovery, retaining distinct image identities.
64
+ - Mark unavailable images only after recovery fails and report media warnings in the existing status area.
65
+
66
+ ### Release assets
67
+ - Align Python and both browser extensions at 2.4.6; rebuild Chrome, AMO-signed Firefox, and installer assets.
68
+
3
69
  ## v2.4.2
4
70
 
5
71
  ### Android / Termux install
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: dala
3
- Version: 2.4.2
3
+ Version: 2.4.10
4
4
  Summary: Web to ebook downloader
5
5
  Project-URL: Homepage, https://github.com/anaxonda/dala
6
6
  Project-URL: Repository, https://github.com/anaxonda/dala
@@ -29,6 +29,7 @@ Requires-Dist: deep-translator>=1.11.4
29
29
  Requires-Dist: ebooklib>=0.18
30
30
  Requires-Dist: fastapi>=0.99.0
31
31
  Requires-Dist: lxml[html-clean]>=4.9.0
32
+ Requires-Dist: pdfplumber>=0.11.7
32
33
  Requires-Dist: pillow>=9.0.0
33
34
  Requires-Dist: pygments>=2.14.0
34
35
  Requires-Dist: pypdf>=5.9.0
@@ -49,7 +50,7 @@ Description-Content-Type: text/markdown
49
50
 
50
51
  **dala** turns articles, comment threads, forum posts, Substack pages, and YouTube transcripts into clean EPUBs optimized for e-ink readers such as Kobo/KOReader. It works from a browser extension or CLI, can bundle many links into one anthology, and can preserve logged-in images or attachments when you explicitly share your browser session with the local backend. Translation support built in.
51
52
 
52
- It is built for the "57 tabs I'll read later" problem: make an book out of it.
53
+ It is built for the "57 tabs I'll read later" problem: make a book out of them.
53
54
 
54
55
  > Everything mostly usable, i'm sure something doesn't work.
55
56
 
@@ -145,6 +146,24 @@ Headless browser support lets the Dala server control Chrome/Chromium in the bac
145
146
  | Translation | LLM or Google Translate; underneath, side-by-side inspired by [Bitextual](https://github.com/wydengyre/bitextual), EPUB footnote, or replace modes |
146
147
  | Summaries | Optional LLM-generated summaries for long articles, discussions, forums, and transcripts |
147
148
 
149
+ ### Reading PDF sources
150
+
151
+ Give Dala a PDF URL or a Hacker News story linking to one:
152
+
153
+ ```bash
154
+ dala "https://example.com/paper.pdf"
155
+ dala "https://news.ycombinator.com/item?id=50516245"
156
+ ```
157
+
158
+ PDF input works with the base install. Generating PDF output requires optional headless browser support. Dala detects PDFs by response type or file signature, including redirects and download URLs without a `.pdf` suffix. HN comments remain included when enabled.
159
+
160
+ - Prose reflows into paragraphs and headings with a contents list. Paragraphs split across pages are rejoined, and page-number footers are removed.
161
+ - Photographs, vector charts, and tables are embedded in reading order. Captions and table notes remain beside their graphics; tables use images to retain columns and mathematical notation. Images respect the image preset, grayscale, and `--no-images` settings.
162
+ - Page-bottom footnotes move to a Footnotes section, with links from the text and return links where references can be resolved. Numeric, Roman, alphabetic, and symbolic labels, column-local notes, and separate endnote sections are supported. Repeated numbering requires destination evidence to disambiguate it.
163
+ - PDF annotations retain external links, author-year citations, and internal cross-references. Internal jumps follow PDF destinations and text positions. Links inside rasterized figures appear beside the image; rotated destinations fall back to a page-level jump.
164
+
165
+ Layout extraction uses `pdfplumber` and PDFium. If it fails, Dala falls back to Poppler (`pdftotext`) when installed, then `pypdf`. Scanned pages can be included as images; OCR is not included. Password-protected or unreadable PDFs retain an explanatory source link. Layout and unannotated footnote detection are heuristic: complex columns, scans, and ambiguous references still need review. Unmatched or ambiguous numbered notes are reported in the conversion log.
166
+
148
167
  ## Headless Browser Support
149
168
 
150
169
  *Basic EPUB downloads do not need headless browser support. This section is about Dala's optional Python support for controlling Chrome/Chromium in the background, not the browser extension.*
@@ -267,12 +286,22 @@ Only load extensions you trust, and only when running a local server you control
267
286
 
268
287
  </details>
269
288
 
270
- When a site serves an interactive bot challenge, Dala defaults to archive fallback. If **Open challenged pages in my browser** is enabled, the extension opens the original article URL and asks you to run Dala again from the readable tab.
289
+ When a site serves an interactive bot challenge, Dala defaults to archive fallback. If **Open challenged pages in my browser** is enabled for a bundle, the extension checks existing tabs first and opens a new tab only if none matches. Resolve the challenge in that tab; Dala checks every three seconds for a readable capture for up to two minutes. An accepted capture gets a separate conversion deadline. Other sources continue processing, and unresolved or failed sources are skipped. Single-page downloads ask you to run Dala again from the readable tab. The server API can adjust verification time with `browser_recovery_timeout_seconds` (0–600; default 120, 0 skips recovery).
290
+
291
+ Images are matched using captured URL aliases and explicit page relationships, including responsive/lazy-load candidates and links from previews to full-size display copies. This works across publishers without hostname or filename rules. Existing captures are reused before network recovery, captions and figure links are retained, and redundant display copies are removed only when their content has been preserved. Distinct image query parameters remain distinct. If recovery is exhausted, the export retains an unavailable-image marker, caption, and source link; image warnings appear in the existing status area and logs without additional toast notifications.
292
+
293
+ The popup status area shows saved and skipped counts, the active stage and site, and elapsed time. **Finish available** builds a bundle from completed articles and stops unfinished work. **Retry skipped** reuses completed articles and retries the remaining sources. Browser queueing does not consume article processing slots. Browser image capture and cleanup have bounded waits; each source conversion has a 180-second overall deadline, configurable through `source_timeout_seconds` in the API or `--source-timeout-seconds` in the CLI.
294
+
295
+ Completed articles and job metadata are checkpointed under `~/.local/state/dala/jobs` (override with `DALA_JOB_STATE_DIR`). After a server restart, interrupted jobs can be retried from these checkpoints. Browser cookies, captured page HTML, and API keys are not stored in job metadata; retries after restart may need a fresh browser capture or credentials from the environment. Extracted article content and images are stored privately until the job retention period expires (two hours by default). This does not change localhost binding or the existing CORS policy.
271
296
 
272
297
  ## Extension Options
273
298
 
274
299
  The extension has a compact popup for common choices and an Options page for server, diagnostics, and advanced behavior.
275
300
 
301
+ The Queue tab uses tightly spaced rows with a compact page title and URL for each item. Select individual rows or use **All**, then **Delete selected** to remove a batch; each row also has a remove button. Titles are saved when adding current/selected/all tabs or using queue shortcuts/context menus. Older and pasted URLs use an open tab’s title when available, otherwise a readable URL label. **Paste or edit URLs** opens the automatic-saving raw URL editor; editing retained URLs preserves their saved title and timestamp. Queue selection only controls deletion; **Download Bundle** includes every remaining item.
302
+
303
+ Right-click a page or link for **Add to Dala Queue**, **Add selected tabs to Dala Queue**, or **Download with Dala** (the browser may group these under Dala). These actions also appear over selected text, images, media, and editable fields. A link targets its destination; other contexts use the current page. Menu entries are restored when the browser or extension background starts.
304
+
276
305
  <details>
277
306
  <summary>Extension options reference</summary>
278
307
 
@@ -405,6 +434,7 @@ uv run dala --translate es --test-translation-provider "Hello world"
405
434
  | `--test-translation-provider TEXT` | Translate a short text and print the result without downloading. |
406
435
  | `--yt-lang en,es` | Preferred YouTube transcript languages. |
407
436
  | `--yt-auto` | Prefer auto-generated YouTube captions. |
437
+ | `--yt-timestamps` | Link the first caption in each minute to its position on YouTube. In the browser extension, enable **Link Transcript Timestamps** under **YouTube Settings**. |
408
438
  | `--thumbnails` | Embed periodic YouTube thumbnails. |
409
439
  | `--yt-max-comments N` | Maximum YouTube comments. |
410
440
  | `--yt-sort top\|new` | YouTube comment sort order. |
@@ -415,11 +445,12 @@ uv run dala --translate es --test-translation-provider "Hello world"
415
445
 
416
446
  ### LLM configuration precedence
417
447
 
418
- LLM and translation settings are resolved in this order:
448
+ **Use LLM for YouTube Transcripts** edits the formatting of existing captions.
449
+ The same provider, model, and key settings apply to AI summaries and to translation when its provider is set to LLM.
419
450
 
420
- 1. Extension settings.
421
- 2. CLI flags.
422
- 3. Environment variables or `.env`.
451
+ With the extension set to **Auto** and the model and key fields left blank, Dala uses the first configured provider in this order. Gemini, OpenRouter, then OpenAI. Each request goes to a single provider. A specific model ID can direct Auto to its provider. An API key entered in the extension replaces the server keys for that request.
452
+
453
+ Use **Custom** to enter an exact model ID. The extension uses the selected provider's built-in default when the model is blank. On the CLI, use `--llm-provider` and `--llm-model` to choose explicitly.
423
454
 
424
455
  ### Environment variables
425
456
 
@@ -431,9 +462,8 @@ GEMINI_API_KEY=your-gemini-api-key
431
462
  OPENROUTER_API_KEY=your-openrouter-api-key
432
463
  OPENAI_API_KEY=your-openai-api-key
433
464
 
434
- # Default LLM provider/model
465
+ # Default CLI provider
435
466
  LLM_PROVIDER=auto
436
- LLM_MODEL=gemini-3.1-flash-lite
437
467
 
438
468
  # Headless browser fallback
439
469
  DALA_BROWSER_EXECUTABLE=/path/to/chrome-or-chromium
@@ -477,6 +507,14 @@ The GitHub release asset `dala-installers-vVERSION.zip` contains:
477
507
 
478
508
  The installers install or update the Python server from PyPI, install `uv` only if it is missing, optionally add headless browser support, and do not install the browser extension.
479
509
 
510
+ If a GitHub release includes a Python wheel such as `dala-VERSION-py3-none-any.whl`, it is mainly an exact-version/offline testing artifact. On desktop platforms you can install it with:
511
+
512
+ ```bash
513
+ uv tool install --force ./dala-VERSION-py3-none-any.whl
514
+ ```
515
+
516
+ The wheel installs the base CLI/server. For normal desktop installs, prefer `uv tool install dala` or the installer bundle. For Android/Termux, prefer the Termux installer because it handles native packages, constraints, storage setup, and known `uv` limitations.
517
+
480
518
  ### macOS
481
519
 
482
520
  Recommended: unzip `dala-installers-vVERSION.zip`, then right-click `installers/Install or Update Dala.command` and choose **Open**. The installer prompts for optional headless browser support and creates `Start Dala Server.command` on the Desktop when possible.
@@ -790,6 +828,7 @@ Expected GitHub release assets:
790
828
  dala-chrome-vEXTENSION_VERSION.zip
791
829
  dala-firefox-vEXTENSION_VERSION-signed.xpi
792
830
  dala-installers-vPYTHON_PACKAGE_VERSION.zip
831
+ dala-PYTHON_PACKAGE_VERSION-py3-none-any.whl # optional convenience artifact
793
832
  ```
794
833
 
795
834
  If only the Python package/installers changed, extension assets may keep the previous extension version. State that explicitly in the release notes.
@@ -5,11 +5,13 @@ import asyncio
5
5
  import aiohttp
6
6
  import socket
7
7
  import time
8
- from datetime import datetime
8
+ from http.cookies import CookieError, SimpleCookie
9
+ from datetime import datetime, timezone
10
+ from pathlib import Path
9
11
  from urllib.parse import urlparse
10
12
  from inspect import isawaitable
11
13
  from typing import Any, Callable, Dict, List, Optional
12
- from tqdm.asyncio import tqdm_asyncio
14
+ from tqdm import tqdm
13
15
  from ebooklib import epub
14
16
  from dotenv import load_dotenv
15
17
 
@@ -27,6 +29,89 @@ from dala.core.browser import BrowserChallengeError, BrowserFetchError, BrowserF
27
29
  from dala.core.image_budget import ImageBudgetExceeded, assert_image_budget, prepare_books_for_bundle
28
30
  from dala.core.discovery import DiscoveryError, discover_posts_for_sources
29
31
  from dala.core.translation import TranslationCache, TranslationProcessor, TranslationError
32
+ from dala.core.runtime import CURRENT_SOURCE, SourceRuntime
33
+ from dala.core.image_layout import validate_book_media
34
+
35
+ def _aiohttp_compatible_cookies(cookies: Dict[str, str]) -> Dict[str, str]:
36
+ """Return cookies that Python's SimpleCookie can place in an aiohttp jar.
37
+
38
+ Browser cookie stores permit names such as ``version`` that SimpleCookie
39
+ reserves for cookie attributes. Keep those cookies available to requests
40
+ fallbacks via ``_extra_cookies``, but do not let them abort the source (or
41
+ an entire bundle) while constructing an aiohttp session.
42
+ """
43
+ compatible = {}
44
+ for name, value in cookies.items():
45
+ try:
46
+ probe = SimpleCookie()
47
+ probe[name] = value
48
+ except (CookieError, TypeError, ValueError) as exc:
49
+ log.warning(f"Skipping browser cookie {name!r} in aiohttp cookie jar: {exc}")
50
+ continue
51
+ compatible[name] = value
52
+ return compatible
53
+
54
+
55
+ def _dedupe_comment_chapter_identity(book: BookData, chapter) -> None:
56
+ filenames = {chap.filename for chap in book.chapters}
57
+ uids = {chap.uid for chap in book.chapters}
58
+
59
+ if chapter.filename in filenames:
60
+ path = Path(chapter.filename)
61
+ stem = path.stem
62
+ suffix = path.suffix or ".xhtml"
63
+ idx = 2
64
+ while f"{stem}_{idx}{suffix}" in filenames:
65
+ idx += 1
66
+ chapter.filename = f"{stem}_{idx}{suffix}"
67
+
68
+ if chapter.uid in uids:
69
+ base = chapter.uid
70
+ idx = 2
71
+ while f"{base}_{idx}" in uids:
72
+ idx += 1
73
+ chapter.uid = f"{base}_{idx}"
74
+
75
+
76
+ def _append_comment_to_toc(book: BookData, chapter, title: str) -> None:
77
+ link = epub.Link(chapter.filename, title, chapter.uid)
78
+ article_chapter = next((chap for chap in book.chapters if chap.is_article), None)
79
+ if not article_chapter:
80
+ book.toc_structure.append(link)
81
+ return
82
+
83
+ for idx, item in enumerate(book.toc_structure):
84
+ if isinstance(item, tuple):
85
+ parent, children = item
86
+ if getattr(parent, "href", None) == article_chapter.filename:
87
+ if not any(getattr(child, "href", None) == chapter.filename for child in children):
88
+ children.append(link)
89
+ return
90
+ elif getattr(item, "href", None) == article_chapter.filename:
91
+ book.toc_structure[idx] = (item, [link])
92
+ return
93
+
94
+ article_title = article_chapter.toc_title or article_chapter.title or book.title
95
+ book.toc_structure.append((epub.Link(article_chapter.filename, article_title, article_chapter.uid), [link]))
96
+
97
+
98
+ async def _append_hn_comments_if_available(context: ConversionContext, source: Source, book: BookData) -> None:
99
+ if context.options.no_comments or not context.options.hn_comments:
100
+ return
101
+ try:
102
+ from dala.drivers.hn import HackerNewsDriver
103
+
104
+ chapter = await HackerNewsDriver.build_comments_chapter_for_url(context, source.url)
105
+ except Exception as exc:
106
+ log.warning(f"HN comment lookup failed for {source.url}: {exc}")
107
+ return
108
+ if not chapter:
109
+ return
110
+
111
+ _dedupe_comment_chapter_identity(book, chapter)
112
+ book.chapters.append(chapter)
113
+ _append_comment_to_toc(book, chapter, "HN Comments")
114
+ log.info(f"Added Hacker News comments for {source.url}")
30
115
 
31
116
  async def process_urls(
32
117
  sources: List[Source],
@@ -34,11 +119,17 @@ async def process_urls(
34
119
  session,
35
120
  progress_callback: Optional[Callable[[int, int, Optional[str]], Any]] = None,
36
121
  source_timing_callback: Optional[Callable[[str, int, bool, Optional[str]], Any]] = None,
122
+ challenge_callback: Optional[Callable[[Source, BrowserChallengeError], Any]] = None,
123
+ source_state_callback: Optional[Callable[[str, str], Any]] = None,
124
+ book_callback: Optional[Callable[[Source, BookData], Any]] = None,
125
+ cached_books: Optional[Dict[str, BookData]] = None,
126
+ finish_event: Optional[asyncio.Event] = None,
37
127
  ) -> List[BookData]:
38
- processed_books = []
128
+ completed_books = {}
39
129
  progress_lock = asyncio.Lock()
40
130
  completed = 0
41
131
  total = len(sources)
132
+ progress_bar = tqdm(total=total, desc="Processing URLs")
42
133
 
43
134
  async def emit_progress(processed: int, current_url: Optional[str]) -> None:
44
135
  if not progress_callback:
@@ -59,26 +150,37 @@ async def process_urls(
59
150
  if isawaitable(maybe):
60
151
  await maybe
61
152
 
62
- async def safe_process(source):
63
- nonlocal completed
64
- started_at = time.perf_counter()
65
- success = False
66
- error_type = None
67
- async with GLOBAL_SEMAPHORE:
153
+ def mark_book_saved_at(book: BookData, saved_at: str) -> None:
154
+ book.extra_metadata["saved_at"] = saved_at
155
+ has_article = any(chapter.is_article for chapter in book.chapters)
156
+ for chapter in book.chapters:
157
+ if chapter.is_article or not has_article:
158
+ chapter.saved_at = chapter.saved_at or saved_at
159
+ return
160
+
161
+ async def process_source(source):
162
+ runtime = SourceRuntime(source.url, GLOBAL_SEMAPHORE, source_state_callback)
163
+ token = CURRENT_SOURCE.set(runtime)
164
+ try:
165
+ await runtime.acquire()
68
166
  profile = ProfileManager.get_instance().get_profile(source.url)
69
167
  driver = DriverDispatcher.get_driver(source, profile)
70
168
 
71
169
  local_session = session
72
- if source.cookies:
73
- connector = aiohttp.TCPConnector(
74
- resolver=aiohttp.resolver.ThreadedResolver(),
75
- ttl_dns_cache=300,
76
- family=socket.AF_INET
77
- )
78
- local_session = aiohttp.ClientSession(timeout=REQUEST_TIMEOUT, cookies=source.cookies, connector=connector)
79
- setattr(local_session, "_extra_cookies", source.cookies)
80
-
81
170
  try:
171
+ if source.cookies:
172
+ connector = aiohttp.TCPConnector(
173
+ resolver=aiohttp.resolver.ThreadedResolver(),
174
+ ttl_dns_cache=300,
175
+ family=socket.AF_INET
176
+ )
177
+ local_session = aiohttp.ClientSession(
178
+ timeout=REQUEST_TIMEOUT,
179
+ cookies=_aiohttp_compatible_cookies(source.cookies),
180
+ connector=connector,
181
+ )
182
+ setattr(local_session, "_extra_cookies", source.cookies)
183
+
82
184
  context = ConversionContext(session=local_session, options=options, profile=profile)
83
185
  book = await driver.prepare_book_data(context, source)
84
186
  if book is None and source.is_forum:
@@ -95,29 +197,107 @@ async def process_urls(
95
197
  assets=source.assets,
96
198
  is_forum=False,
97
199
  published_date=source.published_date,
200
+ saved_at=source.saved_at,
98
201
  )
99
202
  book = await GenericDriver().prepare_book_data(context, generic_source)
100
- success = book is not None
203
+ if book is not None:
204
+ await runtime.report("comments")
205
+ await _append_hn_comments_if_available(context, source, book)
101
206
  if book is not None and source.published_date:
102
207
  book.extra_metadata["published_date"] = source.published_date
208
+ if book is not None and source.saved_at:
209
+ mark_book_saved_at(book, source.saved_at)
210
+ if book is not None:
211
+ validate_book_media(book, options)
103
212
  return book
104
- except BrowserChallengeError:
105
- raise
106
- except Exception as e:
107
- error_type = type(e).__name__
108
- log.exception(f"Failed to process {source.url}: {e}")
109
- return None
110
213
  finally:
111
214
  if local_session is not session:
112
215
  await local_session.close()
113
- duration_ms = int((time.perf_counter() - started_at) * 1000)
114
- await emit_source_timing(source.url, duration_ms, success, error_type)
115
- async with progress_lock:
116
- completed += 1
117
- await emit_progress(completed, source.url)
216
+ finally:
217
+ runtime.release()
218
+ CURRENT_SOURCE.reset(token)
118
219
 
119
- results = await tqdm_asyncio.gather(*[safe_process(s) for s in sources], desc="Processing URLs")
120
- return [b for b in results if b]
220
+ async def safe_process(source):
221
+ nonlocal completed
222
+ started_at = time.perf_counter()
223
+ success = False
224
+ error_type = None
225
+ try:
226
+ try:
227
+ if cached_books and source.url in cached_books:
228
+ book = cached_books[source.url]
229
+ else:
230
+ book = await asyncio.wait_for(process_source(source), timeout=options.source_timeout_seconds)
231
+ except BrowserChallengeError as exc:
232
+ error_type = type(exc).__name__
233
+ if challenge_callback:
234
+ # Recovery waits outside the source and browser semaphores.
235
+ book = await challenge_callback(source, exc)
236
+ elif total > 1:
237
+ log.warning(f"Skipping challenged bundle source {source.url}: {exc}")
238
+ book = None
239
+ else:
240
+ raise
241
+ success = book is not None
242
+ if success:
243
+ error_type = None
244
+ completed_books[source.url] = book
245
+ if book_callback:
246
+ maybe = book_callback(source, book)
247
+ if isawaitable(maybe):
248
+ await maybe
249
+ else:
250
+ error_type = error_type or "NoContent"
251
+ return book
252
+ except asyncio.CancelledError:
253
+ error_type = "FinishedEarly" if finish_event and finish_event.is_set() else "CancelledError"
254
+ raise
255
+ except BrowserChallengeError as exc:
256
+ error_type = type(exc).__name__
257
+ if total == 1:
258
+ raise
259
+ log.warning(f"Skipping challenged bundle source {source.url}: {exc}")
260
+ return None
261
+ except Exception as exc:
262
+ success = False
263
+ error_type = type(exc).__name__
264
+ log.exception(f"Failed to process {source.url}: {exc}")
265
+ return None
266
+ finally:
267
+ if source_state_callback:
268
+ stage = "completed" if success else "skipped"
269
+ maybe = source_state_callback(source.url, stage)
270
+ if isawaitable(maybe):
271
+ await maybe
272
+ duration_ms = int((time.perf_counter() - started_at) * 1000)
273
+ await emit_source_timing(source.url, duration_ms, success, error_type)
274
+ async with progress_lock:
275
+ completed += 1
276
+ progress_bar.update(1)
277
+ await emit_progress(completed, source.url)
278
+
279
+ tasks = [asyncio.create_task(safe_process(source)) for source in sources]
280
+ async def finish_available():
281
+ await finish_event.wait()
282
+ for task in tasks:
283
+ if not task.done():
284
+ task.cancel()
285
+
286
+ finisher = asyncio.create_task(finish_available()) if finish_event else None
287
+ try:
288
+ results = await asyncio.gather(*tasks, return_exceptions=True)
289
+ finally:
290
+ for task in tasks:
291
+ if not task.done():
292
+ task.cancel()
293
+ await asyncio.gather(*tasks, return_exceptions=True)
294
+ if finisher:
295
+ finisher.cancel()
296
+ await asyncio.gather(finisher, return_exceptions=True)
297
+ progress_bar.close()
298
+ if total == 1 and isinstance(results[0], BrowserChallengeError):
299
+ raise results[0]
300
+ return [completed_books[source.url] for source in sources if source.url in completed_books]
121
301
 
122
302
  def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
123
303
  master_uid = f"urn:bundle:{abs(hash(title))}"
@@ -136,13 +316,13 @@ def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
136
316
  if img not in master_images: master_images.append(img)
137
317
 
138
318
  article_chap = None
139
- comments_chap = None
319
+ comments_chapters = []
140
320
 
141
321
  for chap in book.chapters:
142
322
  chap.filename = f"doc_{abs(hash(book.source_url))}_{chap.filename}"
143
323
  master_chapters.append(chap)
144
324
  if chap.is_article: article_chap = chap
145
- elif chap.is_comments: comments_chap = chap
325
+ elif chap.is_comments: comments_chapters.append(chap)
146
326
 
147
327
  if article_chap:
148
328
  toc_title = book.title
@@ -150,15 +330,18 @@ def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
150
330
  if published_date:
151
331
  toc_title = f"{published_date} - {toc_title}"
152
332
  article_chap.toc_title = toc_title
153
- if comments_chap:
333
+ if comments_chapters:
154
334
  master_toc.append((
155
335
  epub.Link(article_chap.filename, toc_title, article_chap.uid),
156
- [epub.Link(comments_chap.filename, "Comments", comments_chap.uid)],
336
+ [
337
+ epub.Link(chap.filename, chap.title or "Comments", chap.uid)
338
+ for chap in comments_chapters
339
+ ],
157
340
  ))
158
341
  else:
159
342
  master_toc.append(epub.Link(article_chap.filename, toc_title, article_chap.uid))
160
343
 
161
- return BookData(title=title, author=author, uid=master_uid, language="en", description=f"Bundle of {len(books)} articles.", source_url="", chapters=master_chapters, images=master_images, toc_structure=master_toc)
344
+ return BookData(title=title, author=author, uid=master_uid, language="en", description=f"Bundle of {len(books)} articles.", source_url="", chapters=master_chapters, images=master_images, toc_structure=master_toc, media_warnings=[warning for book in books for warning in book.media_warnings])
162
345
 
163
346
 
164
347
  def bundle_filename_title(title: str, options: ConversionOptions) -> str:
@@ -187,6 +370,7 @@ def parse_args():
187
370
  parser.add_argument("--no-images", action="store_true")
188
371
  parser.add_argument("-a", "--archive", action="store_true")
189
372
  parser.add_argument("--no-comments", action="store_true", help="Skip downloading comments entirely")
373
+ parser.add_argument("--hn-comments", action="store_true", help="Add Hacker News comments when the article URL has a matching HN discussion")
190
374
  parser.add_argument("--bundle", action="store_true", help="Combine all URLs into a single EPUB")
191
375
  parser.add_argument("--bundle-title", help="Custom title for the bundle")
192
376
  parser.add_argument("--bundle-author", help="Custom author for the bundle")
@@ -197,7 +381,7 @@ def parse_args():
197
381
  parser.add_argument("--css", help="Custom CSS file to inject")
198
382
  parser.add_argument("--llm", action="store_true", dest="llm_format", help="Format transcript using LLM (requires GEMINI_API_KEY)")
199
383
  parser.add_argument("--llm-provider", choices=["auto", "gemini", "openrouter", "openai"], default=os.getenv("LLM_PROVIDER", "auto"), help="LLM provider selection")
200
- parser.add_argument("--llm-model", help="Specify LLM model (default: gemini-3.1-flash-lite for Gemini or deepseek/deepseek-v4-flash for OpenRouter)")
384
+ parser.add_argument("--llm-model", help="Specify an LLM model ID (default: the selected provider's model)")
201
385
  parser.add_argument("--api-key", help="API Key for LLM (overrides env vars)")
202
386
  parser.add_argument("--summary", action="store_true", help="Generate AI summary at start of article")
203
387
  parser.add_argument("--translate", dest="translation_target_lang", help="Translate article text to this target language")
@@ -224,6 +408,7 @@ def parse_args():
224
408
  parser.add_argument("--compact-comments", action="store_true", help="Use compact comment layout")
225
409
  parser.add_argument("--yt-lang", default="en", help="Languages for YouTube transcripts (comma-separated, default: en)")
226
410
  parser.add_argument("--yt-auto", action="store_true", help="Prefer auto-generated YouTube captions")
411
+ parser.add_argument("--yt-timestamps", action="store_true", help="Link the transcript to the first caption in each video minute")
227
412
  parser.add_argument("--thumbnails", action="store_true", help="Embed periodic thumbnails (YouTube only)")
228
413
  parser.add_argument("--yt-max-comments", type=int, default=25, help="Max YouTube comments to fetch")
229
414
  parser.add_argument("--yt-sort", choices=["top", "new"], default="top", help="YouTube comment sort order")
@@ -233,6 +418,7 @@ def parse_args():
233
418
  parser.add_argument("--browser-executable", help="Chromium-compatible executable path for --browser")
234
419
  parser.add_argument("--headed", action="store_true", help="Show the browser window for --browser")
235
420
  parser.add_argument("--browser-timeout-ms", type=int, default=30000, help="Browser navigation timeout in milliseconds")
421
+ parser.add_argument("--source-timeout-seconds", type=float, default=180, help="Overall deadline for each source conversion")
236
422
  parser.add_argument(
237
423
  "--browser-wait-until",
238
424
  choices=["domcontentloaded", "load", "networkidle"],
@@ -286,9 +472,12 @@ async def acquire_browser_sources(sources: List[Source], options: BrowserFetchOp
286
472
  captured.append(Source(
287
473
  url=result.url or source.url,
288
474
  html=result.html,
475
+ page_htmls=source.page_htmls,
289
476
  cookies=cookies or None,
290
477
  assets=assets or None,
291
478
  is_forum=source.is_forum,
479
+ published_date=source.published_date,
480
+ saved_at=source.saved_at,
292
481
  ))
293
482
  return captured
294
483
 
@@ -365,6 +554,7 @@ async def async_main():
365
554
  options = ConversionOptions(
366
555
  no_article=args.no_article,
367
556
  no_comments=args.no_comments,
557
+ hn_comments=args.hn_comments,
368
558
  no_images=args.no_images,
369
559
  archive=args.archive,
370
560
  compact_comments=args.compact_comments,
@@ -387,6 +577,7 @@ async def async_main():
387
577
  translation_cache=not args.no_translation_cache,
388
578
  youtube_lang=args.yt_lang,
389
579
  youtube_prefer_auto=args.yt_auto,
580
+ youtube_timestamps=args.yt_timestamps,
390
581
  thumbnails=args.thumbnails,
391
582
  youtube_max_comments=args.yt_max_comments,
392
583
  youtube_comment_sort=args.yt_sort,
@@ -405,18 +596,21 @@ async def async_main():
405
596
  max_discovery_pages=args.max_discovery_pages,
406
597
  max_discovered_posts=args.max_discovered_posts,
407
598
  browser_challenge_action=args.browser_challenge_action,
599
+ source_timeout_seconds=args.source_timeout_seconds,
408
600
  )
409
601
 
410
602
  cookie_entries = load_cookie_file(args.cookie_file) if args.cookie_file else []
411
603
 
412
604
  sources = []
605
+ saved_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
413
606
  for u in urls:
414
607
  sources.append(Source(
415
608
  url=u,
416
609
  html=None,
417
610
  cookies=cookies_for_source_url(cookie_entries, u),
418
611
  assets=None,
419
- is_forum=args.forum
612
+ is_forum=args.forum,
613
+ saved_at=saved_at,
420
614
  ))
421
615
 
422
616
  if args.browser:
@@ -453,6 +647,9 @@ async def async_main():
453
647
  if options.date_range_active:
454
648
  try:
455
649
  sources = await discover_posts_for_sources(session, sources, options)
650
+ for source in sources:
651
+ if not source.saved_at:
652
+ source.saved_at = saved_at
456
653
  discovery_title = discovery_bundle_title(sources, options)
457
654
  except DiscoveryError as e:
458
655
  log.error(str(e))