dala 2.4.2__tar.gz → 2.4.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dala-2.4.2 → dala-2.4.9}/CHANGELOG.md +53 -0
- {dala-2.4.2 → dala-2.4.9}/PKG-INFO +43 -4
- {dala-2.4.2 → dala-2.4.9}/dala/cli.py +235 -38
- {dala-2.4.2 → dala-2.4.9}/dala/core/browser.py +60 -6
- dala-2.4.9/dala/core/debug_capture.py +35 -0
- dala-2.4.9/dala/core/dispatcher.py +71 -0
- {dala-2.4.2 → dala-2.4.9}/dala/core/extractor.py +266 -26
- {dala-2.4.2 → dala-2.4.9}/dala/core/image_budget.py +1 -21
- dala-2.4.9/dala/core/image_layout.py +236 -0
- {dala-2.4.2 → dala-2.4.9}/dala/core/image_processor.py +55 -18
- dala-2.4.9/dala/core/job_store.py +84 -0
- dala-2.4.9/dala/core/pdf_columns.py +57 -0
- dala-2.4.9/dala/core/pdf_footnotes.py +282 -0
- dala-2.4.9/dala/core/pdf_input.py +96 -0
- dala-2.4.9/dala/core/pdf_layout.py +321 -0
- dala-2.4.9/dala/core/pdf_links.py +291 -0
- dala-2.4.9/dala/core/runtime.py +61 -0
- {dala-2.4.2 → dala-2.4.9}/dala/core/server_jobs.py +83 -1
- {dala-2.4.2 → dala-2.4.9}/dala/core/session.py +7 -0
- {dala-2.4.2 → dala-2.4.9}/dala/core/writer.py +173 -23
- {dala-2.4.2 → dala-2.4.9}/dala/drivers/forum.py +65 -80
- {dala-2.4.2 → dala-2.4.9}/dala/drivers/generic.py +19 -15
- {dala-2.4.2 → dala-2.4.9}/dala/drivers/hn.py +139 -27
- {dala-2.4.2 → dala-2.4.9}/dala/drivers/reddit.py +34 -12
- {dala-2.4.2 → dala-2.4.9}/dala/drivers/substack.py +37 -33
- {dala-2.4.2 → dala-2.4.9}/dala/drivers/wordpress.py +15 -12
- {dala-2.4.2 → dala-2.4.9}/dala/drivers/youtube.py +90 -17
- {dala-2.4.2 → dala-2.4.9}/dala/models.py +20 -0
- {dala-2.4.2 → dala-2.4.9}/dala/server.py +229 -38
- {dala-2.4.2 → dala-2.4.9}/installers/README.md +1 -1
- {dala-2.4.2 → dala-2.4.9}/pyproject.toml +2 -1
- {dala-2.4.2 → dala-2.4.9}/readme.md +40 -2
- dala-2.4.9/tests/bundle_recovery_extension.cjs +79 -0
- dala-2.4.9/tests/conftest.py +6 -0
- dala-2.4.9/tests/context_menus_extension.cjs +134 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_browser.py +80 -0
- dala-2.4.9/tests/test_bundle_recovery.py +402 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_chrome_extension.py +106 -4
- dala-2.4.9/tests/test_context_menus.py +20 -0
- dala-2.4.9/tests/test_debug_capture.py +84 -0
- dala-2.4.9/tests/test_driver_urls.py +197 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_drivers.py +341 -7
- dala-2.4.9/tests/test_ft_extraction.py +194 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_hn_integration.py +59 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_image_budget.py +13 -0
- dala-2.4.9/tests/test_image_layout.py +266 -0
- dala-2.4.9/tests/test_pdf_footnotes.py +177 -0
- dala-2.4.9/tests/test_pdf_input.py +202 -0
- dala-2.4.9/tests/test_pdf_layout.py +174 -0
- dala-2.4.9/tests/test_pdf_links.py +278 -0
- dala-2.4.9/tests/test_queue_ui.py +177 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_server.py +106 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_units.py +135 -1
- {dala-2.4.2 → dala-2.4.9}/tests/test_writer.py +93 -5
- dala-2.4.2/dala/core/dispatcher.py +0 -55
- {dala-2.4.2 → dala-2.4.9}/.gitignore +0 -0
- {dala-2.4.2 → dala-2.4.9}/LICENSE +0 -0
- {dala-2.4.2 → dala-2.4.9}/PRIVACY.md +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/__init__.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/core/__init__.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/core/discovery.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/core/forum_image_processor.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/core/profiles.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/core/translation.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/data/__init__.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/data/sites.json +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/data/sites.yaml +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/drivers/__init__.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/drivers/base.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/setup_browser.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/utils/__init__.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/utils/formatting.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/dala/utils/llm.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/installers/Install or Update Dala.bat +0 -0
- {dala-2.4.2 → dala-2.4.9}/installers/Install or Update Dala.command +0 -0
- {dala-2.4.2 → dala-2.4.9}/installers/Install or Update Dala.ps1 +0 -0
- {dala-2.4.2 → dala-2.4.9}/installers/Install or Update Dala.sh +0 -0
- {dala-2.4.2 → dala-2.4.9}/launchers/Start Dala Server.bat +0 -0
- {dala-2.4.2 → dala-2.4.9}/launchers/Start Dala Server.command +0 -0
- {dala-2.4.2 → dala-2.4.9}/launchers/dala-server.desktop +0 -0
- {dala-2.4.2 → dala-2.4.9}/main.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/scripts/install-dala.ps1 +0 -0
- {dala-2.4.2 → dala-2.4.9}/scripts/install-dala.sh +0 -0
- {dala-2.4.2 → dala-2.4.9}/server.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/sites.yaml +0 -0
- {dala-2.4.2 → dala-2.4.9}/tests/helpers.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_discovery.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_server_saving.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_setup_browser.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/tests/test_translation.py +0 -0
- {dala-2.4.2 → dala-2.4.9}/web_to_epub.py +0 -0
|
@@ -1,5 +1,58 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## v2.4.9
|
|
4
|
+
|
|
5
|
+
### YouTube
|
|
6
|
+
|
|
7
|
+
- Convert YouTube Shorts and livestream links directly.
|
|
8
|
+
- Add optional timestamp links to the first caption in each minute in EPUB and PDF exports. Enable them with `--yt-timestamps` or **Link Transcript Timestamps** under **YouTube Settings**.
|
|
9
|
+
|
|
10
|
+
### Extraction and browser extensions
|
|
11
|
+
|
|
12
|
+
- Resolve Reddit share links and preserve thread query parameters when requesting JSON. Reddit may still block unauthenticated JSON requests.
|
|
13
|
+
- Follow advertised forum pagination URLs and keep browser captures on their original page numbers.
|
|
14
|
+
- Improve driver selection, recover comments from custom-domain WordPress posts and redirected Substack posts, and apply linked-site profiles to Hacker News articles.
|
|
15
|
+
- Fix Chrome download preparation on newer Chromium builds and keep saved forum settings from changing YouTube and Hacker News downloads.
|
|
16
|
+
|
|
17
|
+
### Release assets
|
|
18
|
+
|
|
19
|
+
- Align the Python package and both browser extensions at 2.4.9. Release assets include the Chrome ZIP, AMO-signed Firefox XPI, and installer bundle.
|
|
20
|
+
|
|
21
|
+
## v2.4.7
|
|
22
|
+
|
|
23
|
+
### PDF sources
|
|
24
|
+
- Convert direct PDF URLs and PDFs linked from Hacker News, including redirects and extensionless download URLs.
|
|
25
|
+
- Reflow paragraphs and headings, embed photographs, vector figures and tables, and retain captions and image settings.
|
|
26
|
+
- Move page-bottom footnotes out of running prose; preserve resolvable references and return links across numeric, Roman, alphabetic, symbolic, column-local and endnote structures.
|
|
27
|
+
- Preserve annotated citations, internal cross-references and external links, including links associated with rasterized figures.
|
|
28
|
+
- Fall back to plain text when layout extraction fails; scans remain image-based without OCR.
|
|
29
|
+
|
|
30
|
+
### Browser extensions
|
|
31
|
+
- Show saved page titles in compact queue rows, with individual selection, select-all and bulk deletion. Retain titles when editing URLs and keep concurrent queue additions.
|
|
32
|
+
- Restore right-click queue and download actions on background/browser startup, supporting page, link, selection, image, media and editable contexts across Firefox and Chromium menu APIs.
|
|
33
|
+
- Remove development-only debug capture controls and scripts from the public extensions. Local server diagnostic endpoints remain available for development.
|
|
34
|
+
|
|
35
|
+
### Extraction and packaging
|
|
36
|
+
- Improve Financial Times web/app capture cleanup and preserve photographs inside image controls, responsive picture elements and inline placeholders.
|
|
37
|
+
- Keep PDF anchors and caption links during ebook cleanup; use URL-safe image filenames and surface EPUB writing errors.
|
|
38
|
+
- Add `pdfplumber` and its PDFium renderer to the base dependencies for PDF input. PDF output still requires optional browser support.
|
|
39
|
+
- Keep the Python package at 2.4.7; replace this release’s browser assets with extension version 2.4.8 to remove debug controls. Rebuild Chrome, AMO-signed Firefox and installer assets.
|
|
40
|
+
|
|
41
|
+
## v2.4.6
|
|
42
|
+
|
|
43
|
+
### Bundle reliability
|
|
44
|
+
- Continue processing when individual sources fail or stall, with bounded source and browser recovery deadlines.
|
|
45
|
+
- Reuse already-open challenge tabs and accept successful captures during the recovery window.
|
|
46
|
+
- Checkpoint completed articles for retries after server restarts; expose progress, finish-available, and retry-skipped controls without additional toasts.
|
|
47
|
+
|
|
48
|
+
### Figures and offline media
|
|
49
|
+
- Preserve inline figures, captions, and figure anchors using explicit page relationships across publishers.
|
|
50
|
+
- Reuse captured responsive and lazy-loaded image variants before network recovery, retaining distinct image identities.
|
|
51
|
+
- Mark unavailable images only after recovery fails and report media warnings in the existing status area.
|
|
52
|
+
|
|
53
|
+
### Release assets
|
|
54
|
+
- Align Python and both browser extensions at 2.4.6; rebuild Chrome, AMO-signed Firefox, and installer assets.
|
|
55
|
+
|
|
3
56
|
## v2.4.2
|
|
4
57
|
|
|
5
58
|
### Android / Termux install
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: dala
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.9
|
|
4
4
|
Summary: Web to ebook downloader
|
|
5
5
|
Project-URL: Homepage, https://github.com/anaxonda/dala
|
|
6
6
|
Project-URL: Repository, https://github.com/anaxonda/dala
|
|
@@ -29,6 +29,7 @@ Requires-Dist: deep-translator>=1.11.4
|
|
|
29
29
|
Requires-Dist: ebooklib>=0.18
|
|
30
30
|
Requires-Dist: fastapi>=0.99.0
|
|
31
31
|
Requires-Dist: lxml[html-clean]>=4.9.0
|
|
32
|
+
Requires-Dist: pdfplumber>=0.11.7
|
|
32
33
|
Requires-Dist: pillow>=9.0.0
|
|
33
34
|
Requires-Dist: pygments>=2.14.0
|
|
34
35
|
Requires-Dist: pypdf>=5.9.0
|
|
@@ -49,7 +50,7 @@ Description-Content-Type: text/markdown
|
|
|
49
50
|
|
|
50
51
|
**dala** turns articles, comment threads, forum posts, Substack pages, and YouTube transcripts into clean EPUBs optimized for e-ink readers such as Kobo/KOReader. It works from a browser extension or CLI, can bundle many links into one anthology, and can preserve logged-in images or attachments when you explicitly share your browser session with the local backend. Translation support built in.
|
|
51
52
|
|
|
52
|
-
It is built for the "57 tabs I'll read later" problem: make
|
|
53
|
+
It is built for the "57 tabs I'll read later" problem: make a book out of them.
|
|
53
54
|
|
|
54
55
|
> Everything mostly usable, i'm sure something doesn't work.
|
|
55
56
|
|
|
@@ -145,6 +146,24 @@ Headless browser support lets the Dala server control Chrome/Chromium in the bac
|
|
|
145
146
|
| Translation | LLM or Google Translate; underneath, side-by-side inspired by [Bitextual](https://github.com/wydengyre/bitextual), EPUB footnote, or replace modes |
|
|
146
147
|
| Summaries | Optional LLM-generated summaries for long articles, discussions, forums, and transcripts |
|
|
147
148
|
|
|
149
|
+
### Reading PDF sources
|
|
150
|
+
|
|
151
|
+
Give Dala a PDF URL or a Hacker News story linking to one:
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
dala "https://example.com/paper.pdf"
|
|
155
|
+
dala "https://news.ycombinator.com/item?id=50516245"
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
PDF input works with the base install. Generating PDF output requires optional headless browser support. Dala detects PDFs by response type or file signature, including redirects and download URLs without a `.pdf` suffix. HN comments remain included when enabled.
|
|
159
|
+
|
|
160
|
+
- Prose reflows into paragraphs and headings with a contents list. Paragraphs split across pages are rejoined, and page-number footers are removed.
|
|
161
|
+
- Photographs, vector charts, and tables are embedded in reading order. Captions and table notes remain beside their graphics; tables use images to retain columns and mathematical notation. Images respect the image preset, grayscale, and `--no-images` settings.
|
|
162
|
+
- Page-bottom footnotes move to a Footnotes section, with links from the text and return links where references can be resolved. Numeric, Roman, alphabetic, and symbolic labels, column-local notes, and separate endnote sections are supported. Repeated numbering requires destination evidence to disambiguate it.
|
|
163
|
+
- PDF annotations retain external links, author-year citations, and internal cross-references. Internal jumps follow PDF destinations and text positions. Links inside rasterized figures appear beside the image; rotated destinations fall back to a page-level jump.
|
|
164
|
+
|
|
165
|
+
Layout extraction uses `pdfplumber` and PDFium. If it fails, Dala falls back to Poppler (`pdftotext`) when installed, then `pypdf`. Scanned pages can be included as images; OCR is not included. Password-protected or unreadable PDFs retain an explanatory source link. Layout and unannotated footnote detection are heuristic: complex columns, scans, and ambiguous references still need review. Unmatched or ambiguous numbered notes are reported in the conversion log.
|
|
166
|
+
|
|
148
167
|
## Headless Browser Support
|
|
149
168
|
|
|
150
169
|
*Basic EPUB downloads do not need headless browser support. This section is about Dala's optional Python support for controlling Chrome/Chromium in the background, not the browser extension.*
|
|
@@ -267,12 +286,22 @@ Only load extensions you trust, and only when running a local server you control
|
|
|
267
286
|
|
|
268
287
|
</details>
|
|
269
288
|
|
|
270
|
-
When a site serves an interactive bot challenge, Dala defaults to archive fallback. If **Open challenged pages in my browser** is enabled, the extension opens the
|
|
289
|
+
When a site serves an interactive bot challenge, Dala defaults to archive fallback. If **Open challenged pages in my browser** is enabled for a bundle, the extension checks existing tabs first and opens a new tab only if none matches. Resolve the challenge in that tab; Dala checks every three seconds for a readable capture for up to two minutes. An accepted capture gets a separate conversion deadline. Other sources continue processing, and unresolved or failed sources are skipped. Single-page downloads ask you to run Dala again from the readable tab. The server API can adjust verification time with `browser_recovery_timeout_seconds` (0–600; default 120, 0 skips recovery).
|
|
290
|
+
|
|
291
|
+
Images are matched using captured URL aliases and explicit page relationships, including responsive/lazy-load candidates and links from previews to full-size display copies. This works across publishers without hostname or filename rules. Existing captures are reused before network recovery, captions and figure links are retained, and redundant display copies are removed only when their content has been preserved. Distinct image query parameters remain distinct. If recovery is exhausted, the export retains an unavailable-image marker, caption, and source link; image warnings appear in the existing status area and logs without additional toast notifications.
|
|
292
|
+
|
|
293
|
+
The popup status area shows saved and skipped counts, the active stage and site, and elapsed time. **Finish available** builds a bundle from completed articles and stops unfinished work. **Retry skipped** reuses completed articles and retries the remaining sources. Browser queueing does not consume article processing slots. Browser image capture and cleanup have bounded waits; each source conversion has a 180-second overall deadline, configurable through `source_timeout_seconds` in the API or `--source-timeout-seconds` in the CLI.
|
|
294
|
+
|
|
295
|
+
Completed articles and job metadata are checkpointed under `~/.local/state/dala/jobs` (override with `DALA_JOB_STATE_DIR`). After a server restart, interrupted jobs can be retried from these checkpoints. Browser cookies, captured page HTML, and API keys are not stored in job metadata; retries after restart may need a fresh browser capture or credentials from the environment. Extracted article content and images are stored privately until the job retention period expires (two hours by default). This does not change localhost binding or the existing CORS policy.
|
|
271
296
|
|
|
272
297
|
## Extension Options
|
|
273
298
|
|
|
274
299
|
The extension has a compact popup for common choices and an Options page for server, diagnostics, and advanced behavior.
|
|
275
300
|
|
|
301
|
+
The Queue tab uses tightly spaced rows with a compact page title and URL for each item. Select individual rows or use **All**, then **Delete selected** to remove a batch; each row also has a remove button. Titles are saved when adding current/selected/all tabs or using queue shortcuts/context menus. Older and pasted URLs use an open tab’s title when available, otherwise a readable URL label. **Paste or edit URLs** opens the automatic-saving raw URL editor; editing retained URLs preserves their saved title and timestamp. Queue selection only controls deletion; **Download Bundle** includes every remaining item.
|
|
302
|
+
|
|
303
|
+
Right-click a page or link for **Add to Dala Queue**, **Add selected tabs to Dala Queue**, or **Download with Dala** (the browser may group these under Dala). These actions also appear over selected text, images, media, and editable fields. A link targets its destination; other contexts use the current page. Menu entries are restored when the browser or extension background starts.
|
|
304
|
+
|
|
276
305
|
<details>
|
|
277
306
|
<summary>Extension options reference</summary>
|
|
278
307
|
|
|
@@ -405,6 +434,7 @@ uv run dala --translate es --test-translation-provider "Hello world"
|
|
|
405
434
|
| `--test-translation-provider TEXT` | Translate a short text and print the result without downloading. |
|
|
406
435
|
| `--yt-lang en,es` | Preferred YouTube transcript languages. |
|
|
407
436
|
| `--yt-auto` | Prefer auto-generated YouTube captions. |
|
|
437
|
+
| `--yt-timestamps` | Link the first caption in each minute to its position on YouTube. In the browser extension, enable **Link Transcript Timestamps** under **YouTube Settings**. |
|
|
408
438
|
| `--thumbnails` | Embed periodic YouTube thumbnails. |
|
|
409
439
|
| `--yt-max-comments N` | Maximum YouTube comments. |
|
|
410
440
|
| `--yt-sort top\|new` | YouTube comment sort order. |
|
|
@@ -477,6 +507,14 @@ The GitHub release asset `dala-installers-vVERSION.zip` contains:
|
|
|
477
507
|
|
|
478
508
|
The installers install or update the Python server from PyPI, install `uv` only if it is missing, optionally add headless browser support, and do not install the browser extension.
|
|
479
509
|
|
|
510
|
+
If a GitHub release includes a Python wheel such as `dala-VERSION-py3-none-any.whl`, it is mainly an exact-version/offline testing artifact. On desktop platforms you can install it with:
|
|
511
|
+
|
|
512
|
+
```bash
|
|
513
|
+
uv tool install --force ./dala-VERSION-py3-none-any.whl
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
The wheel installs the base CLI/server. For normal desktop installs, prefer `uv tool install dala` or the installer bundle. For Android/Termux, prefer the Termux installer because it handles native packages, constraints, storage setup, and known `uv` limitations.
|
|
517
|
+
|
|
480
518
|
### macOS
|
|
481
519
|
|
|
482
520
|
Recommended: unzip `dala-installers-vVERSION.zip`, then right-click `installers/Install or Update Dala.command` and choose **Open**. The installer prompts for optional headless browser support and creates `Start Dala Server.command` on the Desktop when possible.
|
|
@@ -790,6 +828,7 @@ Expected GitHub release assets:
|
|
|
790
828
|
dala-chrome-vEXTENSION_VERSION.zip
|
|
791
829
|
dala-firefox-vEXTENSION_VERSION-signed.xpi
|
|
792
830
|
dala-installers-vPYTHON_PACKAGE_VERSION.zip
|
|
831
|
+
dala-PYTHON_PACKAGE_VERSION-py3-none-any.whl # optional convenience artifact
|
|
793
832
|
```
|
|
794
833
|
|
|
795
834
|
If only the Python package/installers changed, extension assets may keep the previous extension version. State that explicitly in the release notes.
|
|
@@ -5,11 +5,13 @@ import asyncio
|
|
|
5
5
|
import aiohttp
|
|
6
6
|
import socket
|
|
7
7
|
import time
|
|
8
|
-
from
|
|
8
|
+
from http.cookies import CookieError, SimpleCookie
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from pathlib import Path
|
|
9
11
|
from urllib.parse import urlparse
|
|
10
12
|
from inspect import isawaitable
|
|
11
13
|
from typing import Any, Callable, Dict, List, Optional
|
|
12
|
-
from tqdm
|
|
14
|
+
from tqdm import tqdm
|
|
13
15
|
from ebooklib import epub
|
|
14
16
|
from dotenv import load_dotenv
|
|
15
17
|
|
|
@@ -27,6 +29,89 @@ from dala.core.browser import BrowserChallengeError, BrowserFetchError, BrowserF
|
|
|
27
29
|
from dala.core.image_budget import ImageBudgetExceeded, assert_image_budget, prepare_books_for_bundle
|
|
28
30
|
from dala.core.discovery import DiscoveryError, discover_posts_for_sources
|
|
29
31
|
from dala.core.translation import TranslationCache, TranslationProcessor, TranslationError
|
|
32
|
+
from dala.core.runtime import CURRENT_SOURCE, SourceRuntime
|
|
33
|
+
from dala.core.image_layout import validate_book_media
|
|
34
|
+
|
|
35
|
+
def _aiohttp_compatible_cookies(cookies: Dict[str, str]) -> Dict[str, str]:
|
|
36
|
+
"""Return cookies that Python's SimpleCookie can place in an aiohttp jar.
|
|
37
|
+
|
|
38
|
+
Browser cookie stores permit names such as ``version`` that SimpleCookie
|
|
39
|
+
reserves for cookie attributes. Keep those cookies available to requests
|
|
40
|
+
fallbacks via ``_extra_cookies``, but do not let them abort the source (or
|
|
41
|
+
an entire bundle) while constructing an aiohttp session.
|
|
42
|
+
"""
|
|
43
|
+
compatible = {}
|
|
44
|
+
for name, value in cookies.items():
|
|
45
|
+
try:
|
|
46
|
+
probe = SimpleCookie()
|
|
47
|
+
probe[name] = value
|
|
48
|
+
except (CookieError, TypeError, ValueError) as exc:
|
|
49
|
+
log.warning(f"Skipping browser cookie {name!r} in aiohttp cookie jar: {exc}")
|
|
50
|
+
continue
|
|
51
|
+
compatible[name] = value
|
|
52
|
+
return compatible
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _dedupe_comment_chapter_identity(book: BookData, chapter) -> None:
|
|
56
|
+
filenames = {chap.filename for chap in book.chapters}
|
|
57
|
+
uids = {chap.uid for chap in book.chapters}
|
|
58
|
+
|
|
59
|
+
if chapter.filename in filenames:
|
|
60
|
+
path = Path(chapter.filename)
|
|
61
|
+
stem = path.stem
|
|
62
|
+
suffix = path.suffix or ".xhtml"
|
|
63
|
+
idx = 2
|
|
64
|
+
while f"{stem}_{idx}{suffix}" in filenames:
|
|
65
|
+
idx += 1
|
|
66
|
+
chapter.filename = f"{stem}_{idx}{suffix}"
|
|
67
|
+
|
|
68
|
+
if chapter.uid in uids:
|
|
69
|
+
base = chapter.uid
|
|
70
|
+
idx = 2
|
|
71
|
+
while f"{base}_{idx}" in uids:
|
|
72
|
+
idx += 1
|
|
73
|
+
chapter.uid = f"{base}_{idx}"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _append_comment_to_toc(book: BookData, chapter, title: str) -> None:
|
|
77
|
+
link = epub.Link(chapter.filename, title, chapter.uid)
|
|
78
|
+
article_chapter = next((chap for chap in book.chapters if chap.is_article), None)
|
|
79
|
+
if not article_chapter:
|
|
80
|
+
book.toc_structure.append(link)
|
|
81
|
+
return
|
|
82
|
+
|
|
83
|
+
for idx, item in enumerate(book.toc_structure):
|
|
84
|
+
if isinstance(item, tuple):
|
|
85
|
+
parent, children = item
|
|
86
|
+
if getattr(parent, "href", None) == article_chapter.filename:
|
|
87
|
+
if not any(getattr(child, "href", None) == chapter.filename for child in children):
|
|
88
|
+
children.append(link)
|
|
89
|
+
return
|
|
90
|
+
elif getattr(item, "href", None) == article_chapter.filename:
|
|
91
|
+
book.toc_structure[idx] = (item, [link])
|
|
92
|
+
return
|
|
93
|
+
|
|
94
|
+
article_title = article_chapter.toc_title or article_chapter.title or book.title
|
|
95
|
+
book.toc_structure.append((epub.Link(article_chapter.filename, article_title, article_chapter.uid), [link]))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def _append_hn_comments_if_available(context: ConversionContext, source: Source, book: BookData) -> None:
|
|
99
|
+
if context.options.no_comments or not context.options.hn_comments:
|
|
100
|
+
return
|
|
101
|
+
try:
|
|
102
|
+
from dala.drivers.hn import HackerNewsDriver
|
|
103
|
+
|
|
104
|
+
chapter = await HackerNewsDriver.build_comments_chapter_for_url(context, source.url)
|
|
105
|
+
except Exception as exc:
|
|
106
|
+
log.warning(f"HN comment lookup failed for {source.url}: {exc}")
|
|
107
|
+
return
|
|
108
|
+
if not chapter:
|
|
109
|
+
return
|
|
110
|
+
|
|
111
|
+
_dedupe_comment_chapter_identity(book, chapter)
|
|
112
|
+
book.chapters.append(chapter)
|
|
113
|
+
_append_comment_to_toc(book, chapter, "HN Comments")
|
|
114
|
+
log.info(f"Added Hacker News comments for {source.url}")
|
|
30
115
|
|
|
31
116
|
async def process_urls(
|
|
32
117
|
sources: List[Source],
|
|
@@ -34,11 +119,17 @@ async def process_urls(
|
|
|
34
119
|
session,
|
|
35
120
|
progress_callback: Optional[Callable[[int, int, Optional[str]], Any]] = None,
|
|
36
121
|
source_timing_callback: Optional[Callable[[str, int, bool, Optional[str]], Any]] = None,
|
|
122
|
+
challenge_callback: Optional[Callable[[Source, BrowserChallengeError], Any]] = None,
|
|
123
|
+
source_state_callback: Optional[Callable[[str, str], Any]] = None,
|
|
124
|
+
book_callback: Optional[Callable[[Source, BookData], Any]] = None,
|
|
125
|
+
cached_books: Optional[Dict[str, BookData]] = None,
|
|
126
|
+
finish_event: Optional[asyncio.Event] = None,
|
|
37
127
|
) -> List[BookData]:
|
|
38
|
-
|
|
128
|
+
completed_books = {}
|
|
39
129
|
progress_lock = asyncio.Lock()
|
|
40
130
|
completed = 0
|
|
41
131
|
total = len(sources)
|
|
132
|
+
progress_bar = tqdm(total=total, desc="Processing URLs")
|
|
42
133
|
|
|
43
134
|
async def emit_progress(processed: int, current_url: Optional[str]) -> None:
|
|
44
135
|
if not progress_callback:
|
|
@@ -59,26 +150,37 @@ async def process_urls(
|
|
|
59
150
|
if isawaitable(maybe):
|
|
60
151
|
await maybe
|
|
61
152
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
153
|
+
def mark_book_saved_at(book: BookData, saved_at: str) -> None:
|
|
154
|
+
book.extra_metadata["saved_at"] = saved_at
|
|
155
|
+
has_article = any(chapter.is_article for chapter in book.chapters)
|
|
156
|
+
for chapter in book.chapters:
|
|
157
|
+
if chapter.is_article or not has_article:
|
|
158
|
+
chapter.saved_at = chapter.saved_at or saved_at
|
|
159
|
+
return
|
|
160
|
+
|
|
161
|
+
async def process_source(source):
|
|
162
|
+
runtime = SourceRuntime(source.url, GLOBAL_SEMAPHORE, source_state_callback)
|
|
163
|
+
token = CURRENT_SOURCE.set(runtime)
|
|
164
|
+
try:
|
|
165
|
+
await runtime.acquire()
|
|
68
166
|
profile = ProfileManager.get_instance().get_profile(source.url)
|
|
69
167
|
driver = DriverDispatcher.get_driver(source, profile)
|
|
70
168
|
|
|
71
169
|
local_session = session
|
|
72
|
-
if source.cookies:
|
|
73
|
-
connector = aiohttp.TCPConnector(
|
|
74
|
-
resolver=aiohttp.resolver.ThreadedResolver(),
|
|
75
|
-
ttl_dns_cache=300,
|
|
76
|
-
family=socket.AF_INET
|
|
77
|
-
)
|
|
78
|
-
local_session = aiohttp.ClientSession(timeout=REQUEST_TIMEOUT, cookies=source.cookies, connector=connector)
|
|
79
|
-
setattr(local_session, "_extra_cookies", source.cookies)
|
|
80
|
-
|
|
81
170
|
try:
|
|
171
|
+
if source.cookies:
|
|
172
|
+
connector = aiohttp.TCPConnector(
|
|
173
|
+
resolver=aiohttp.resolver.ThreadedResolver(),
|
|
174
|
+
ttl_dns_cache=300,
|
|
175
|
+
family=socket.AF_INET
|
|
176
|
+
)
|
|
177
|
+
local_session = aiohttp.ClientSession(
|
|
178
|
+
timeout=REQUEST_TIMEOUT,
|
|
179
|
+
cookies=_aiohttp_compatible_cookies(source.cookies),
|
|
180
|
+
connector=connector,
|
|
181
|
+
)
|
|
182
|
+
setattr(local_session, "_extra_cookies", source.cookies)
|
|
183
|
+
|
|
82
184
|
context = ConversionContext(session=local_session, options=options, profile=profile)
|
|
83
185
|
book = await driver.prepare_book_data(context, source)
|
|
84
186
|
if book is None and source.is_forum:
|
|
@@ -95,29 +197,107 @@ async def process_urls(
|
|
|
95
197
|
assets=source.assets,
|
|
96
198
|
is_forum=False,
|
|
97
199
|
published_date=source.published_date,
|
|
200
|
+
saved_at=source.saved_at,
|
|
98
201
|
)
|
|
99
202
|
book = await GenericDriver().prepare_book_data(context, generic_source)
|
|
100
|
-
|
|
203
|
+
if book is not None:
|
|
204
|
+
await runtime.report("comments")
|
|
205
|
+
await _append_hn_comments_if_available(context, source, book)
|
|
101
206
|
if book is not None and source.published_date:
|
|
102
207
|
book.extra_metadata["published_date"] = source.published_date
|
|
208
|
+
if book is not None and source.saved_at:
|
|
209
|
+
mark_book_saved_at(book, source.saved_at)
|
|
210
|
+
if book is not None:
|
|
211
|
+
validate_book_media(book, options)
|
|
103
212
|
return book
|
|
104
|
-
except BrowserChallengeError:
|
|
105
|
-
raise
|
|
106
|
-
except Exception as e:
|
|
107
|
-
error_type = type(e).__name__
|
|
108
|
-
log.exception(f"Failed to process {source.url}: {e}")
|
|
109
|
-
return None
|
|
110
213
|
finally:
|
|
111
214
|
if local_session is not session:
|
|
112
215
|
await local_session.close()
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
completed += 1
|
|
117
|
-
await emit_progress(completed, source.url)
|
|
216
|
+
finally:
|
|
217
|
+
runtime.release()
|
|
218
|
+
CURRENT_SOURCE.reset(token)
|
|
118
219
|
|
|
119
|
-
|
|
120
|
-
|
|
220
|
+
async def safe_process(source):
|
|
221
|
+
nonlocal completed
|
|
222
|
+
started_at = time.perf_counter()
|
|
223
|
+
success = False
|
|
224
|
+
error_type = None
|
|
225
|
+
try:
|
|
226
|
+
try:
|
|
227
|
+
if cached_books and source.url in cached_books:
|
|
228
|
+
book = cached_books[source.url]
|
|
229
|
+
else:
|
|
230
|
+
book = await asyncio.wait_for(process_source(source), timeout=options.source_timeout_seconds)
|
|
231
|
+
except BrowserChallengeError as exc:
|
|
232
|
+
error_type = type(exc).__name__
|
|
233
|
+
if challenge_callback:
|
|
234
|
+
# Recovery waits outside the source and browser semaphores.
|
|
235
|
+
book = await challenge_callback(source, exc)
|
|
236
|
+
elif total > 1:
|
|
237
|
+
log.warning(f"Skipping challenged bundle source {source.url}: {exc}")
|
|
238
|
+
book = None
|
|
239
|
+
else:
|
|
240
|
+
raise
|
|
241
|
+
success = book is not None
|
|
242
|
+
if success:
|
|
243
|
+
error_type = None
|
|
244
|
+
completed_books[source.url] = book
|
|
245
|
+
if book_callback:
|
|
246
|
+
maybe = book_callback(source, book)
|
|
247
|
+
if isawaitable(maybe):
|
|
248
|
+
await maybe
|
|
249
|
+
else:
|
|
250
|
+
error_type = error_type or "NoContent"
|
|
251
|
+
return book
|
|
252
|
+
except asyncio.CancelledError:
|
|
253
|
+
error_type = "FinishedEarly" if finish_event and finish_event.is_set() else "CancelledError"
|
|
254
|
+
raise
|
|
255
|
+
except BrowserChallengeError as exc:
|
|
256
|
+
error_type = type(exc).__name__
|
|
257
|
+
if total == 1:
|
|
258
|
+
raise
|
|
259
|
+
log.warning(f"Skipping challenged bundle source {source.url}: {exc}")
|
|
260
|
+
return None
|
|
261
|
+
except Exception as exc:
|
|
262
|
+
success = False
|
|
263
|
+
error_type = type(exc).__name__
|
|
264
|
+
log.exception(f"Failed to process {source.url}: {exc}")
|
|
265
|
+
return None
|
|
266
|
+
finally:
|
|
267
|
+
if source_state_callback:
|
|
268
|
+
stage = "completed" if success else "skipped"
|
|
269
|
+
maybe = source_state_callback(source.url, stage)
|
|
270
|
+
if isawaitable(maybe):
|
|
271
|
+
await maybe
|
|
272
|
+
duration_ms = int((time.perf_counter() - started_at) * 1000)
|
|
273
|
+
await emit_source_timing(source.url, duration_ms, success, error_type)
|
|
274
|
+
async with progress_lock:
|
|
275
|
+
completed += 1
|
|
276
|
+
progress_bar.update(1)
|
|
277
|
+
await emit_progress(completed, source.url)
|
|
278
|
+
|
|
279
|
+
tasks = [asyncio.create_task(safe_process(source)) for source in sources]
|
|
280
|
+
async def finish_available():
|
|
281
|
+
await finish_event.wait()
|
|
282
|
+
for task in tasks:
|
|
283
|
+
if not task.done():
|
|
284
|
+
task.cancel()
|
|
285
|
+
|
|
286
|
+
finisher = asyncio.create_task(finish_available()) if finish_event else None
|
|
287
|
+
try:
|
|
288
|
+
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
289
|
+
finally:
|
|
290
|
+
for task in tasks:
|
|
291
|
+
if not task.done():
|
|
292
|
+
task.cancel()
|
|
293
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
294
|
+
if finisher:
|
|
295
|
+
finisher.cancel()
|
|
296
|
+
await asyncio.gather(finisher, return_exceptions=True)
|
|
297
|
+
progress_bar.close()
|
|
298
|
+
if total == 1 and isinstance(results[0], BrowserChallengeError):
|
|
299
|
+
raise results[0]
|
|
300
|
+
return [completed_books[source.url] for source in sources if source.url in completed_books]
|
|
121
301
|
|
|
122
302
|
def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
|
|
123
303
|
master_uid = f"urn:bundle:{abs(hash(title))}"
|
|
@@ -136,13 +316,13 @@ def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
|
|
|
136
316
|
if img not in master_images: master_images.append(img)
|
|
137
317
|
|
|
138
318
|
article_chap = None
|
|
139
|
-
|
|
319
|
+
comments_chapters = []
|
|
140
320
|
|
|
141
321
|
for chap in book.chapters:
|
|
142
322
|
chap.filename = f"doc_{abs(hash(book.source_url))}_{chap.filename}"
|
|
143
323
|
master_chapters.append(chap)
|
|
144
324
|
if chap.is_article: article_chap = chap
|
|
145
|
-
elif chap.is_comments:
|
|
325
|
+
elif chap.is_comments: comments_chapters.append(chap)
|
|
146
326
|
|
|
147
327
|
if article_chap:
|
|
148
328
|
toc_title = book.title
|
|
@@ -150,15 +330,18 @@ def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
|
|
|
150
330
|
if published_date:
|
|
151
331
|
toc_title = f"{published_date} - {toc_title}"
|
|
152
332
|
article_chap.toc_title = toc_title
|
|
153
|
-
if
|
|
333
|
+
if comments_chapters:
|
|
154
334
|
master_toc.append((
|
|
155
335
|
epub.Link(article_chap.filename, toc_title, article_chap.uid),
|
|
156
|
-
[
|
|
336
|
+
[
|
|
337
|
+
epub.Link(chap.filename, chap.title or "Comments", chap.uid)
|
|
338
|
+
for chap in comments_chapters
|
|
339
|
+
],
|
|
157
340
|
))
|
|
158
341
|
else:
|
|
159
342
|
master_toc.append(epub.Link(article_chap.filename, toc_title, article_chap.uid))
|
|
160
343
|
|
|
161
|
-
return BookData(title=title, author=author, uid=master_uid, language="en", description=f"Bundle of {len(books)} articles.", source_url="", chapters=master_chapters, images=master_images, toc_structure=master_toc)
|
|
344
|
+
return BookData(title=title, author=author, uid=master_uid, language="en", description=f"Bundle of {len(books)} articles.", source_url="", chapters=master_chapters, images=master_images, toc_structure=master_toc, media_warnings=[warning for book in books for warning in book.media_warnings])
|
|
162
345
|
|
|
163
346
|
|
|
164
347
|
def bundle_filename_title(title: str, options: ConversionOptions) -> str:
|
|
@@ -187,6 +370,7 @@ def parse_args():
|
|
|
187
370
|
parser.add_argument("--no-images", action="store_true")
|
|
188
371
|
parser.add_argument("-a", "--archive", action="store_true")
|
|
189
372
|
parser.add_argument("--no-comments", action="store_true", help="Skip downloading comments entirely")
|
|
373
|
+
parser.add_argument("--hn-comments", action="store_true", help="Add Hacker News comments when the article URL has a matching HN discussion")
|
|
190
374
|
parser.add_argument("--bundle", action="store_true", help="Combine all URLs into a single EPUB")
|
|
191
375
|
parser.add_argument("--bundle-title", help="Custom title for the bundle")
|
|
192
376
|
parser.add_argument("--bundle-author", help="Custom author for the bundle")
|
|
@@ -224,6 +408,7 @@ def parse_args():
|
|
|
224
408
|
parser.add_argument("--compact-comments", action="store_true", help="Use compact comment layout")
|
|
225
409
|
parser.add_argument("--yt-lang", default="en", help="Languages for YouTube transcripts (comma-separated, default: en)")
|
|
226
410
|
parser.add_argument("--yt-auto", action="store_true", help="Prefer auto-generated YouTube captions")
|
|
411
|
+
parser.add_argument("--yt-timestamps", action="store_true", help="Link the transcript to the first caption in each video minute")
|
|
227
412
|
parser.add_argument("--thumbnails", action="store_true", help="Embed periodic thumbnails (YouTube only)")
|
|
228
413
|
parser.add_argument("--yt-max-comments", type=int, default=25, help="Max YouTube comments to fetch")
|
|
229
414
|
parser.add_argument("--yt-sort", choices=["top", "new"], default="top", help="YouTube comment sort order")
|
|
@@ -233,6 +418,7 @@ def parse_args():
|
|
|
233
418
|
parser.add_argument("--browser-executable", help="Chromium-compatible executable path for --browser")
|
|
234
419
|
parser.add_argument("--headed", action="store_true", help="Show the browser window for --browser")
|
|
235
420
|
parser.add_argument("--browser-timeout-ms", type=int, default=30000, help="Browser navigation timeout in milliseconds")
|
|
421
|
+
parser.add_argument("--source-timeout-seconds", type=float, default=180, help="Overall deadline for each source conversion")
|
|
236
422
|
parser.add_argument(
|
|
237
423
|
"--browser-wait-until",
|
|
238
424
|
choices=["domcontentloaded", "load", "networkidle"],
|
|
@@ -286,9 +472,12 @@ async def acquire_browser_sources(sources: List[Source], options: BrowserFetchOp
|
|
|
286
472
|
captured.append(Source(
|
|
287
473
|
url=result.url or source.url,
|
|
288
474
|
html=result.html,
|
|
475
|
+
page_htmls=source.page_htmls,
|
|
289
476
|
cookies=cookies or None,
|
|
290
477
|
assets=assets or None,
|
|
291
478
|
is_forum=source.is_forum,
|
|
479
|
+
published_date=source.published_date,
|
|
480
|
+
saved_at=source.saved_at,
|
|
292
481
|
))
|
|
293
482
|
return captured
|
|
294
483
|
|
|
@@ -365,6 +554,7 @@ async def async_main():
|
|
|
365
554
|
options = ConversionOptions(
|
|
366
555
|
no_article=args.no_article,
|
|
367
556
|
no_comments=args.no_comments,
|
|
557
|
+
hn_comments=args.hn_comments,
|
|
368
558
|
no_images=args.no_images,
|
|
369
559
|
archive=args.archive,
|
|
370
560
|
compact_comments=args.compact_comments,
|
|
@@ -387,6 +577,7 @@ async def async_main():
|
|
|
387
577
|
translation_cache=not args.no_translation_cache,
|
|
388
578
|
youtube_lang=args.yt_lang,
|
|
389
579
|
youtube_prefer_auto=args.yt_auto,
|
|
580
|
+
youtube_timestamps=args.yt_timestamps,
|
|
390
581
|
thumbnails=args.thumbnails,
|
|
391
582
|
youtube_max_comments=args.yt_max_comments,
|
|
392
583
|
youtube_comment_sort=args.yt_sort,
|
|
@@ -405,18 +596,21 @@ async def async_main():
|
|
|
405
596
|
max_discovery_pages=args.max_discovery_pages,
|
|
406
597
|
max_discovered_posts=args.max_discovered_posts,
|
|
407
598
|
browser_challenge_action=args.browser_challenge_action,
|
|
599
|
+
source_timeout_seconds=args.source_timeout_seconds,
|
|
408
600
|
)
|
|
409
601
|
|
|
410
602
|
cookie_entries = load_cookie_file(args.cookie_file) if args.cookie_file else []
|
|
411
603
|
|
|
412
604
|
sources = []
|
|
605
|
+
saved_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
413
606
|
for u in urls:
|
|
414
607
|
sources.append(Source(
|
|
415
608
|
url=u,
|
|
416
609
|
html=None,
|
|
417
610
|
cookies=cookies_for_source_url(cookie_entries, u),
|
|
418
611
|
assets=None,
|
|
419
|
-
is_forum=args.forum
|
|
612
|
+
is_forum=args.forum,
|
|
613
|
+
saved_at=saved_at,
|
|
420
614
|
))
|
|
421
615
|
|
|
422
616
|
if args.browser:
|
|
@@ -453,6 +647,9 @@ async def async_main():
|
|
|
453
647
|
if options.date_range_active:
|
|
454
648
|
try:
|
|
455
649
|
sources = await discover_posts_for_sources(session, sources, options)
|
|
650
|
+
for source in sources:
|
|
651
|
+
if not source.saved_at:
|
|
652
|
+
source.saved_at = saved_at
|
|
456
653
|
discovery_title = discovery_bundle_title(sources, options)
|
|
457
654
|
except DiscoveryError as e:
|
|
458
655
|
log.error(str(e))
|