dala 2.4.0__tar.gz → 2.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dala-2.4.0 → dala-2.4.1}/PKG-INFO +142 -84
- {dala-2.4.0 → dala-2.4.1}/dala/cli.py +34 -6
- {dala-2.4.0 → dala-2.4.1}/dala/core/discovery.py +17 -2
- {dala-2.4.0 → dala-2.4.1}/dala/core/translation.py +11 -1
- {dala-2.4.0 → dala-2.4.1}/dala/core/writer.py +1 -1
- {dala-2.4.0 → dala-2.4.1}/dala/models.py +3 -0
- {dala-2.4.0 → dala-2.4.1}/dala/server.py +13 -5
- {dala-2.4.0 → dala-2.4.1}/dala/setup_browser.py +7 -7
- dala-2.4.1/installers/Install or Update Dala.bat +3 -0
- dala-2.4.1/installers/Install or Update Dala.command +41 -0
- dala-2.4.1/installers/Install or Update Dala.ps1 +43 -0
- dala-2.4.1/installers/Install or Update Dala.sh +22 -0
- dala-2.4.1/installers/README.md +57 -0
- dala-2.4.1/launchers/Start Dala Server.bat +3 -0
- dala-2.4.1/launchers/Start Dala Server.command +2 -0
- {dala-2.4.0 → dala-2.4.1}/launchers/dala-server.desktop +1 -1
- {dala-2.4.0 → dala-2.4.1}/pyproject.toml +3 -2
- {dala-2.4.0 → dala-2.4.1}/readme.md +140 -82
- {dala-2.4.0 → dala-2.4.1}/scripts/install-dala.ps1 +11 -3
- {dala-2.4.0 → dala-2.4.1}/scripts/install-dala.sh +8 -3
- {dala-2.4.0 → dala-2.4.1}/tests/test_chrome_extension.py +14 -12
- {dala-2.4.0 → dala-2.4.1}/tests/test_discovery.py +40 -0
- {dala-2.4.0 → dala-2.4.1}/tests/test_image_budget.py +35 -0
- {dala-2.4.0 → dala-2.4.1}/tests/test_server.py +16 -2
- {dala-2.4.0 → dala-2.4.1}/tests/test_setup_browser.py +2 -2
- {dala-2.4.0 → dala-2.4.1}/tests/test_translation.py +61 -0
- {dala-2.4.0 → dala-2.4.1}/tests/test_units.py +3 -0
- {dala-2.4.0 → dala-2.4.1}/tests/test_writer.py +23 -0
- dala-2.4.0/launchers/Start Dala Server.bat +0 -3
- dala-2.4.0/launchers/Start Dala Server.command +0 -2
- {dala-2.4.0 → dala-2.4.1}/.gitignore +0 -0
- {dala-2.4.0 → dala-2.4.1}/CHANGELOG.md +0 -0
- {dala-2.4.0 → dala-2.4.1}/LICENSE +0 -0
- {dala-2.4.0 → dala-2.4.1}/PRIVACY.md +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/browser.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/dispatcher.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/extractor.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/forum_image_processor.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/image_budget.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/image_processor.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/profiles.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/server_jobs.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/core/session.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/data/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/data/sites.yaml +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/drivers/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/drivers/base.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/drivers/forum.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/drivers/generic.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/drivers/hn.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/drivers/reddit.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/drivers/substack.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/drivers/wordpress.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/drivers/youtube.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/utils/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/utils/formatting.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/dala/utils/llm.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/main.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/server.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/sites.yaml +0 -0
- {dala-2.4.0 → dala-2.4.1}/tests/helpers.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/tests/test_browser.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/tests/test_drivers.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/tests/test_hn_integration.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/tests/test_server_saving.py +0 -0
- {dala-2.4.0 → dala-2.4.1}/web_to_epub.py +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dala
|
|
3
|
-
Version: 2.4.
|
|
4
|
-
Summary: Web to
|
|
3
|
+
Version: 2.4.1
|
|
4
|
+
Summary: Web to ebook downloader
|
|
5
5
|
Project-URL: Homepage, https://github.com/anaxonda/dala
|
|
6
6
|
Project-URL: Repository, https://github.com/anaxonda/dala
|
|
7
7
|
Project-URL: Issues, https://github.com/anaxonda/dala/issues
|
|
@@ -82,7 +82,7 @@ Install the command-line tools:
|
|
|
82
82
|
|
|
83
83
|
```bash
|
|
84
84
|
uv tool install dala
|
|
85
|
-
dala-server
|
|
85
|
+
dala-server
|
|
86
86
|
```
|
|
87
87
|
|
|
88
88
|
For development from a source checkout:
|
|
@@ -90,7 +90,7 @@ For development from a source checkout:
|
|
|
90
90
|
```bash
|
|
91
91
|
git clone https://github.com/anaxonda/dala.git
|
|
92
92
|
cd dala
|
|
93
|
-
uv run dala-server
|
|
93
|
+
uv run dala-server
|
|
94
94
|
```
|
|
95
95
|
|
|
96
96
|
Install the browser extension:
|
|
@@ -100,7 +100,9 @@ Install the browser extension:
|
|
|
100
100
|
|
|
101
101
|
Then open a page, click the **dala** icon, and click **Download Page**.
|
|
102
102
|
|
|
103
|
-
|
|
103
|
+
Dala opens its local status page when `dala-server` starts. Use `dala-server --no-open` for background services, SSH sessions, or scripts that should not open a browser.
|
|
104
|
+
|
|
105
|
+
For PDF output, JavaScript-heavy pages without extension capture, or headless browser fallback, see [Headless Browser Support](#headless-browser-support).
|
|
104
106
|
|
|
105
107
|
## What Do I Install?
|
|
106
108
|
|
|
@@ -111,13 +113,13 @@ For PDF output, CLI/server rendering of JavaScript-heavy pages, or server-side b
|
|
|
111
113
|
| Save forum images or attachments behind login | Extension with **Use Site Cookies** | Usually none |
|
|
112
114
|
| Bundle browser links | Extension queue | None |
|
|
113
115
|
| Bundle URLs from a file | CLI | None |
|
|
114
|
-
| Render JavaScript-heavy pages without extension capture | CLI/
|
|
115
|
-
| Generate PDF | Extension or CLI |
|
|
116
|
+
| Render JavaScript-heavy pages without extension capture | CLI/headless browser rendering | Headless browser support |
|
|
117
|
+
| Generate PDF | Extension or CLI | Headless browser support |
|
|
116
118
|
| Download a YouTube transcript | CLI or extension | None |
|
|
117
119
|
| AI summaries or LLM translation | CLI or extension | API key |
|
|
118
120
|
| Google Translate translation | CLI or extension | No API key |
|
|
119
121
|
|
|
120
|
-
|
|
122
|
+
Headless browser support lets the Dala server control Chrome/Chromium in the background. It is needed for PDF output and some JavaScript-heavy pages, and is separate from the normal Dala browser extension.
|
|
121
123
|
|
|
122
124
|
## Source Support
|
|
123
125
|
|
|
@@ -136,22 +138,22 @@ Server-side browser setup uses Dala's optional Python browser-control support pl
|
|
|
136
138
|
| Feature | Supported |
|
|
137
139
|
| --- | --- |
|
|
138
140
|
| EPUB | E-reader-oriented typography, metadata, table of contents, image optimization |
|
|
139
|
-
| PDF | Document and e-reader presets through
|
|
141
|
+
| PDF | Document and e-reader presets through headless browser rendering |
|
|
140
142
|
| Bundles | Multiple pages combined into one anthology-style file |
|
|
141
143
|
| Images | Compact, Balanced, or Full presets; optional grayscale conversion |
|
|
142
144
|
| Translation | LLM or Google Translate; underneath, side-by-side inspired by [Bitextual](https://github.com/wydengyre/bitextual), EPUB footnote, or replace modes |
|
|
143
145
|
| Summaries | Optional LLM-generated summaries for long articles, discussions, forums, and transcripts |
|
|
144
146
|
|
|
145
|
-
##
|
|
147
|
+
## Headless Browser Support
|
|
146
148
|
|
|
147
|
-
*Basic EPUB downloads do not need
|
|
149
|
+
*Basic EPUB downloads do not need headless browser support. This section is about Dala's optional Python support for controlling Chrome/Chromium in the background, not the browser extension.*
|
|
148
150
|
|
|
149
|
-
Install browser
|
|
151
|
+
Install Dala with headless browser support only if you want PDF output, JavaScript-heavy page rendering without extension capture, or headless browser fallback:
|
|
150
152
|
|
|
151
153
|
```bash
|
|
152
154
|
uv tool install --force "dala[browser]"
|
|
153
155
|
dala-setup-browser
|
|
154
|
-
dala-server
|
|
156
|
+
dala-server
|
|
155
157
|
```
|
|
156
158
|
|
|
157
159
|
For development from a source checkout:
|
|
@@ -159,7 +161,7 @@ For development from a source checkout:
|
|
|
159
161
|
```bash
|
|
160
162
|
uv sync --extra browser
|
|
161
163
|
uv run dala-setup-browser
|
|
162
|
-
uv run dala-server
|
|
164
|
+
uv run dala-server
|
|
163
165
|
```
|
|
164
166
|
|
|
165
167
|
`dala-setup-browser` first tries to use an existing Chromium-compatible browser such as Chrome, Edge, Brave, or Chromium. If none is detected, it installs Playwright's managed Chromium.
|
|
@@ -167,24 +169,33 @@ uv run dala-server --open
|
|
|
167
169
|
<details>
|
|
168
170
|
<summary>Installer scripts and launchers</summary>
|
|
169
171
|
|
|
170
|
-
The `
|
|
172
|
+
The `installers/` directory has desktop-oriented installers and wrappers. Windows and macOS installers prompt for optional headless browser support and create a desktop launcher when possible; the Linux wrapper installs or updates from a terminal:
|
|
173
|
+
|
|
174
|
+
```text
|
|
175
|
+
installers/Install or Update Dala.bat # Windows double-click wrapper
|
|
176
|
+
installers/Install or Update Dala.ps1 # Windows PowerShell installer
|
|
177
|
+
installers/Install or Update Dala.command # macOS double-click installer
|
|
178
|
+
installers/Install or Update Dala.sh # Linux terminal installer
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
The `scripts/` directory has conservative lower-level installers that reuse an existing `uv` install, install `uv` only if missing, and make headless browser support opt-in:
|
|
171
182
|
|
|
172
183
|
```bash
|
|
173
184
|
# macOS/Linux
|
|
174
185
|
./scripts/install-dala.sh
|
|
175
|
-
./scripts/install-dala.sh --browser
|
|
186
|
+
./scripts/install-dala.sh --headless-browser
|
|
176
187
|
|
|
177
188
|
# Windows PowerShell
|
|
178
189
|
.\scripts\install-dala.ps1
|
|
179
|
-
.\scripts\install-dala.ps1 -
|
|
190
|
+
.\scripts\install-dala.ps1 -HeadlessBrowser
|
|
180
191
|
```
|
|
181
192
|
|
|
182
|
-
The `launchers/` directory has double-click templates that start `dala-server
|
|
193
|
+
The `launchers/` directory has double-click templates that start `dala-server` after Dala is installed.
|
|
183
194
|
|
|
184
195
|
</details>
|
|
185
196
|
|
|
186
197
|
<details>
|
|
187
|
-
<summary>CLI examples for
|
|
198
|
+
<summary>CLI examples for headless browser rendering</summary>
|
|
188
199
|
|
|
189
200
|
```bash
|
|
190
201
|
# Render with an auto-detected browser
|
|
@@ -238,28 +249,20 @@ Dala can process pages that are already readable in your browser, including page
|
|
|
238
249
|
|
|
239
250
|
The simplest workflow is normal browser capture: open the article until it is readable in your everyday browser, then click the Dala extension. If you use a browser extension to make pages readable, install it in the same browser where you run Dala.
|
|
240
251
|
|
|
241
|
-
|
|
252
|
+
Headless browser fallback is an advanced local fallback for automation and testing. The server uses a dedicated Dala browser profile at `~/.local/share/dala/browser-profile` by default; it does not automatically use your normal browser profile.
|
|
242
253
|
|
|
243
254
|
<details>
|
|
244
|
-
<summary>Advanced:
|
|
255
|
+
<summary>Advanced: headless browser extensions for local testing</summary>
|
|
245
256
|
|
|
246
|
-
|
|
257
|
+
Headless browser fallback can load an unpacked Chromium-compatible extension when you explicitly provide its path. This is useful for local automation tests where the rendered browser needs the same helper extension behavior you use interactively. Configure it with:
|
|
247
258
|
|
|
248
259
|
```bash
|
|
249
|
-
export
|
|
260
|
+
export DALA_BROWSER_EXTENSION_PATH=/path/to/unpacked-extension
|
|
250
261
|
export DALA_BROWSER_EXECUTABLE=/usr/bin/google-chrome
|
|
251
262
|
export DALA_BROWSER_PROFILE_DIR=/path/to/custom/dala-chromium-profile
|
|
252
263
|
```
|
|
253
264
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
```bash
|
|
257
|
-
git clone --depth 1 https://gitflic.ru/project/magnolia1234/bpc_uploads.git config/bpc/latest
|
|
258
|
-
mkdir -p config/bpc/chrome-unpacked
|
|
259
|
-
unzip -oq config/bpc/latest/bypass-paywalls-chrome-clean-master.zip -d config/bpc/chrome-unpacked
|
|
260
|
-
```
|
|
261
|
-
|
|
262
|
-
`config/bpc/` is ignored by git.
|
|
265
|
+
Only load extensions you trust, and only when running a local server you control.
|
|
263
266
|
|
|
264
267
|
</details>
|
|
265
268
|
|
|
@@ -285,12 +288,12 @@ The extension has a compact popup for common choices and an Options page for ser
|
|
|
285
288
|
| **Translation** | Translates article text with LLM or Google Translate. | Bilingual reading or translated-only output. |
|
|
286
289
|
| **Video Thumbnails** | Embeds periodic YouTube thumbnails. | Visual context for transcripts. |
|
|
287
290
|
| **Use Site Cookies** | Sends browser cookies to the backend. Defaults on only for local server URLs. | Forums, protected images, authenticated pages. |
|
|
288
|
-
| **
|
|
291
|
+
| **Headless Browser Fallback** | Lets the server retry failed extraction in a background Chromium-compatible browser. | JavaScript-heavy, blocked, or difficult pages. |
|
|
289
292
|
| **Force Forum Driver** | Uses forum multi-page scraping. | XenForo/vBulletin-style threads. |
|
|
290
293
|
| **Forum Pages** | Downloads specific pages such as `1,3-5`. | Partial forum thread downloads. |
|
|
291
294
|
| **Max Pages** | Limits sequential forum crawling. | Avoid unexpectedly large downloads. |
|
|
292
295
|
|
|
293
|
-
The Options page also includes diagnostics for server version, Playwright/browser/profile
|
|
296
|
+
The Options page also includes diagnostics for server version, Playwright/headless browser/profile status, PDF availability, retained jobs, cleanup retention, and the last conversion status/error. If PDF is unavailable, the extension disables PDF output and falls back to EPUB.
|
|
294
297
|
|
|
295
298
|
</details>
|
|
296
299
|
|
|
@@ -431,10 +434,10 @@ OPENAI_API_KEY=your-openai-api-key
|
|
|
431
434
|
LLM_PROVIDER=auto
|
|
432
435
|
LLM_MODEL=gemini-3.1-flash-lite
|
|
433
436
|
|
|
434
|
-
#
|
|
437
|
+
# Headless browser fallback
|
|
435
438
|
DALA_BROWSER_EXECUTABLE=/path/to/chrome-or-chromium
|
|
436
439
|
DALA_BROWSER_PROFILE_DIR=/path/to/dala-browser-profile
|
|
437
|
-
|
|
440
|
+
DALA_BROWSER_EXTENSION_PATH=/path/to/unpacked-extension
|
|
438
441
|
|
|
439
442
|
# Translation speed tuning
|
|
440
443
|
DALA_GOOGLE_TRANSLATE_CHUNK_SIZE=5
|
|
@@ -460,46 +463,46 @@ CLI and extension outputs use the same title-based naming helpers where possible
|
|
|
460
463
|
|
|
461
464
|
## Platform-Specific Installation
|
|
462
465
|
|
|
463
|
-
Dala is primarily tested on Linux and Android/Termux.
|
|
466
|
+
Dala is primarily tested on Linux and Android/Termux. Windows and macOS should work with the packaged installers or the manual `uv` commands below.
|
|
464
467
|
|
|
465
|
-
|
|
466
|
-
<summary>macOS, Windows, Linux, and Android setup</summary>
|
|
468
|
+
Git is not required when installing the published package with `uv tool install dala`. It is only needed for source-checkout development.
|
|
467
469
|
|
|
468
|
-
|
|
470
|
+
The GitHub release asset `dala-installers-vVERSION.zip` contains:
|
|
469
471
|
|
|
470
|
-
- `
|
|
471
|
-
-
|
|
472
|
+
- `installers/`: double-click or terminal installers for desktop users
|
|
473
|
+
- `scripts/`: lower-level install/update scripts
|
|
474
|
+
- `launchers/`: templates that start the installed `dala-server`
|
|
472
475
|
|
|
473
|
-
|
|
476
|
+
The installers install or update the Python server from PyPI, install `uv` only if it is missing, optionally add headless browser support, and do not install the browser extension.
|
|
474
477
|
|
|
475
|
-
|
|
476
|
-
# macOS
|
|
477
|
-
brew install uv
|
|
478
|
+
### macOS
|
|
478
479
|
|
|
479
|
-
|
|
480
|
-
powershell -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
480
|
+
Recommended: unzip `dala-installers-vVERSION.zip`, then right-click `installers/Install or Update Dala.command` and choose **Open**. The installer prompts for optional headless browser support and creates `Start Dala Server.command` on the Desktop when possible.
|
|
481
481
|
|
|
482
|
-
|
|
483
|
-
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
482
|
+
Manual install:
|
|
484
483
|
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
484
|
+
```bash
|
|
485
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
486
|
+
uv tool install dala
|
|
487
|
+
dala-server
|
|
488
488
|
```
|
|
489
489
|
|
|
490
|
-
|
|
490
|
+
Headless browser/PDF support:
|
|
491
491
|
|
|
492
492
|
```bash
|
|
493
|
-
|
|
494
|
-
dala-server --open
|
|
493
|
+
./scripts/install-dala.sh --headless-browser
|
|
495
494
|
```
|
|
496
495
|
|
|
497
|
-
macOS may prompt you to install Command Line Tools if Git is not installed.
|
|
498
|
-
|
|
499
496
|
<details>
|
|
500
497
|
<summary>Start Dala automatically with launchd</summary>
|
|
501
498
|
|
|
502
|
-
|
|
499
|
+
Find the installed server path:
|
|
500
|
+
|
|
501
|
+
```bash
|
|
502
|
+
command -v dala-server
|
|
503
|
+
```
|
|
504
|
+
|
|
505
|
+
Create `~/Library/LaunchAgents/com.dala.server.plist`, replacing `/Users/YOU/.local/bin/dala-server` with that path:
|
|
503
506
|
|
|
504
507
|
```xml
|
|
505
508
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
@@ -510,12 +513,9 @@ Create `~/Library/LaunchAgents/com.dala.server.plist`:
|
|
|
510
513
|
<string>com.dala.server</string>
|
|
511
514
|
<key>ProgramArguments</key>
|
|
512
515
|
<array>
|
|
513
|
-
<string>/
|
|
514
|
-
<string
|
|
515
|
-
<string>dala-server</string>
|
|
516
|
+
<string>/Users/YOU/.local/bin/dala-server</string>
|
|
517
|
+
<string>--no-open</string>
|
|
516
518
|
</array>
|
|
517
|
-
<key>WorkingDirectory</key>
|
|
518
|
-
<string>/path/to/dala</string>
|
|
519
519
|
<key>RunAtLoad</key>
|
|
520
520
|
<true/>
|
|
521
521
|
<key>KeepAlive</key>
|
|
@@ -538,11 +538,14 @@ launchctl load ~/Library/LaunchAgents/com.dala.server.plist
|
|
|
538
538
|
|
|
539
539
|
### Windows
|
|
540
540
|
|
|
541
|
-
|
|
541
|
+
Recommended: unzip `dala-installers-vVERSION.zip`, then double-click `installers\Install or Update Dala.bat`. It runs the PowerShell installer, prompts for optional headless browser support, and creates `Start Dala Server.bat` on the Desktop when possible.
|
|
542
|
+
|
|
543
|
+
Manual install from PowerShell:
|
|
542
544
|
|
|
543
545
|
```powershell
|
|
546
|
+
powershell -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
544
547
|
uv tool install dala
|
|
545
|
-
dala-server
|
|
548
|
+
dala-server
|
|
546
549
|
```
|
|
547
550
|
|
|
548
551
|
If you see an execution policy error, run:
|
|
@@ -551,6 +554,12 @@ If you see an execution policy error, run:
|
|
|
551
554
|
Set-ExecutionPolicy RemoteSigned -Scope CurrentUser
|
|
552
555
|
```
|
|
553
556
|
|
|
557
|
+
Headless browser/PDF support:
|
|
558
|
+
|
|
559
|
+
```powershell
|
|
560
|
+
.\scripts\install-dala.ps1 -HeadlessBrowser
|
|
561
|
+
```
|
|
562
|
+
|
|
554
563
|
<details>
|
|
555
564
|
<summary>Start Dala automatically from the Startup Folder</summary>
|
|
556
565
|
|
|
@@ -558,7 +567,7 @@ Use `launchers/Start Dala Server.bat`, or create `start_dala.bat`:
|
|
|
558
567
|
|
|
559
568
|
```bat
|
|
560
569
|
@echo off
|
|
561
|
-
dala-server
|
|
570
|
+
dala-server
|
|
562
571
|
```
|
|
563
572
|
|
|
564
573
|
Press `Win + R`, enter `shell:startup`, and add a shortcut to `start_dala.bat`.
|
|
@@ -567,21 +576,46 @@ Press `Win + R`, enter `shell:startup`, and add a shortcut to `start_dala.bat`.
|
|
|
567
576
|
|
|
568
577
|
### Linux
|
|
569
578
|
|
|
570
|
-
|
|
579
|
+
Recommended: unzip `dala-installers-vVERSION.zip`, then run:
|
|
580
|
+
|
|
581
|
+
```bash
|
|
582
|
+
sh "installers/Install or Update Dala.sh"
|
|
583
|
+
```
|
|
584
|
+
|
|
585
|
+
Manual install:
|
|
586
|
+
|
|
587
|
+
```bash
|
|
588
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
589
|
+
uv tool install dala
|
|
590
|
+
dala-server
|
|
591
|
+
```
|
|
592
|
+
|
|
593
|
+
Headless browser/PDF support:
|
|
594
|
+
|
|
595
|
+
```bash
|
|
596
|
+
./scripts/install-dala.sh --headless-browser
|
|
597
|
+
```
|
|
598
|
+
|
|
599
|
+
Dala can auto-detect `chromium`, `google-chrome`, `microsoft-edge`, or `brave-browser` from `PATH`. A `.desktop` launcher template is available at `launchers/dala-server.desktop`; if your desktop environment cannot find `dala-server`, edit `Exec=` to the absolute path from `command -v dala-server`.
|
|
571
600
|
|
|
572
601
|
<details>
|
|
573
602
|
<summary>Start Dala automatically with systemd</summary>
|
|
574
603
|
|
|
604
|
+
Find the installed server path:
|
|
605
|
+
|
|
606
|
+
```bash
|
|
607
|
+
command -v dala-server
|
|
608
|
+
```
|
|
609
|
+
|
|
575
610
|
Create `~/.config/systemd/user/epub_server.service`:
|
|
576
611
|
|
|
577
612
|
```ini
|
|
578
613
|
[Unit]
|
|
579
|
-
Description=Web to
|
|
614
|
+
Description=Web to ebook Python Server
|
|
580
615
|
After=network.target
|
|
581
616
|
|
|
582
617
|
[Service]
|
|
583
|
-
|
|
584
|
-
ExecStart=/path/to/dala/.venv/bin/dala-server
|
|
618
|
+
ExecStart=/home/YOU/.local/bin/dala-server --no-open
|
|
585
619
|
Restart=always
|
|
586
620
|
RestartSec=5
|
|
587
621
|
|
|
@@ -599,29 +633,30 @@ systemctl --user enable --now epub_server
|
|
|
599
633
|
|
|
600
634
|
### Android / Termux
|
|
601
635
|
|
|
602
|
-
|
|
636
|
+
Termux does not use the desktop installer bundle. Install from PyPI with Termux `uv`:
|
|
603
637
|
|
|
604
638
|
```bash
|
|
605
639
|
pkg update
|
|
606
|
-
pkg install
|
|
640
|
+
pkg install tur-repo
|
|
607
641
|
pkg install uv
|
|
608
|
-
|
|
609
|
-
|
|
642
|
+
uv tool install dala
|
|
643
|
+
dala-server --no-open
|
|
610
644
|
```
|
|
611
645
|
|
|
612
|
-
|
|
646
|
+
Open `http://127.0.0.1:8000/` manually, or point Firefox for Android plus the Dala extension at the local Termux server.
|
|
647
|
+
|
|
648
|
+
If `uv tool install` cache/linking gives trouble on Android, use a source checkout and virtualenv:
|
|
613
649
|
|
|
614
650
|
```bash
|
|
651
|
+
pkg install git python
|
|
652
|
+
git clone https://github.com/anaxonda/dala.git
|
|
653
|
+
cd dala
|
|
615
654
|
python -m venv .venv
|
|
616
655
|
source .venv/bin/activate
|
|
617
656
|
UV_LINK_MODE=copy UV_CACHE_DIR=$HOME/.cache/uv uv pip install -e .
|
|
618
|
-
uv run dala-server
|
|
657
|
+
uv run dala-server --no-open
|
|
619
658
|
```
|
|
620
659
|
|
|
621
|
-
Firefox for Android can use the extension against the local Termux server.
|
|
622
|
-
|
|
623
|
-
</details>
|
|
624
|
-
|
|
625
660
|
## sites.yaml Customization
|
|
626
661
|
|
|
627
662
|
You can define custom extraction rules for specific websites in `sites.yaml` at the project root.
|
|
@@ -660,7 +695,7 @@ Then open `http://127.0.0.1:8000/ping`.
|
|
|
660
695
|
|
|
661
696
|
### PDF option is disabled
|
|
662
697
|
|
|
663
|
-
Install
|
|
698
|
+
Install headless browser support:
|
|
664
699
|
|
|
665
700
|
```bash
|
|
666
701
|
uv tool install --force "dala[browser]"
|
|
@@ -669,7 +704,7 @@ dala-setup-browser
|
|
|
669
704
|
|
|
670
705
|
For a source checkout, use `uv sync --extra browser` and `uv run dala-setup-browser`. Restart the server and check `http://127.0.0.1:8000/`.
|
|
671
706
|
|
|
672
|
-
###
|
|
707
|
+
### Headless browser fallback fails
|
|
673
708
|
|
|
674
709
|
Use headed mode to debug login, bot challenges, or extension loading:
|
|
675
710
|
|
|
@@ -715,6 +750,29 @@ Browser download, CLI output, or server archive
|
|
|
715
750
|
- `server.py`: FastAPI backend used by the extensions and async job flow.
|
|
716
751
|
- `firefox_extension/` and `extension_chrome/`: browser clients for page capture, queueing, options, and downloads.
|
|
717
752
|
|
|
753
|
+
## Release Checklist
|
|
754
|
+
|
|
755
|
+
Dala releases now have two distribution channels:
|
|
756
|
+
|
|
757
|
+
- **PyPI:** publish the Python package so users can run `uv tool install dala`.
|
|
758
|
+
- **GitHub Releases:** attach browser extensions and the easy installer bundle.
|
|
759
|
+
|
|
760
|
+
Expected GitHub release assets:
|
|
761
|
+
|
|
762
|
+
```text
|
|
763
|
+
dala-chrome-vEXTENSION_VERSION.zip
|
|
764
|
+
dala-firefox-vEXTENSION_VERSION-signed.xpi
|
|
765
|
+
dala-installers-vPYTHON_PACKAGE_VERSION.zip
|
|
766
|
+
```
|
|
767
|
+
|
|
768
|
+
Build the unsigned extension packages and installer bundle with:
|
|
769
|
+
|
|
770
|
+
```bash
|
|
771
|
+
./package_extensions.sh
|
|
772
|
+
```
|
|
773
|
+
|
|
774
|
+
Then replace the unsigned Firefox XPI with the AMO-signed XPI before publishing the GitHub release. The installer bundle should be attached to the release page so nontechnical users can download the double-click installers without cloning the repository.
|
|
775
|
+
|
|
718
776
|
## Roadmap
|
|
719
777
|
|
|
720
778
|
### Planned
|
|
@@ -739,7 +797,7 @@ Browser download, CLI output, or server archive
|
|
|
739
797
|
- Only enable cookie sharing for a backend you control and trust.
|
|
740
798
|
- Do not expose the Dala server directly to the public internet.
|
|
741
799
|
- API keys in the extension or `.env` are secrets. Do not commit `.env`.
|
|
742
|
-
- Dala is intended to process pages you are allowed to access. Use authenticated capture and browser fallback responsibly.
|
|
800
|
+
- Dala is intended to process pages you are allowed to access. Use authenticated capture and headless browser fallback responsibly.
|
|
743
801
|
|
|
744
802
|
## License
|
|
745
803
|
|
|
@@ -94,9 +94,12 @@ async def process_urls(
|
|
|
94
94
|
cookies=source.cookies,
|
|
95
95
|
assets=source.assets,
|
|
96
96
|
is_forum=False,
|
|
97
|
+
published_date=source.published_date,
|
|
97
98
|
)
|
|
98
99
|
book = await GenericDriver().prepare_book_data(context, generic_source)
|
|
99
100
|
success = book is not None
|
|
101
|
+
if book is not None and source.published_date:
|
|
102
|
+
book.extra_metadata["published_date"] = source.published_date
|
|
100
103
|
return book
|
|
101
104
|
except BrowserChallengeError:
|
|
102
105
|
raise
|
|
@@ -142,15 +145,38 @@ def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
|
|
|
142
145
|
elif chap.is_comments: comments_chap = chap
|
|
143
146
|
|
|
144
147
|
if article_chap:
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
148
|
+
toc_title = book.title
|
|
149
|
+
published_date = (book.extra_metadata or {}).get("published_date")
|
|
150
|
+
if published_date:
|
|
151
|
+
toc_title = f"{published_date} - {toc_title}"
|
|
152
|
+
article_chap.toc_title = toc_title
|
|
153
|
+
if comments_chap:
|
|
154
|
+
master_toc.append((
|
|
155
|
+
epub.Link(article_chap.filename, toc_title, article_chap.uid),
|
|
156
|
+
[epub.Link(comments_chap.filename, "Comments", comments_chap.uid)],
|
|
157
|
+
))
|
|
158
|
+
else:
|
|
159
|
+
master_toc.append(epub.Link(article_chap.filename, toc_title, article_chap.uid))
|
|
149
160
|
|
|
150
161
|
return BookData(title=title, author=author, uid=master_uid, language="en", description=f"Bundle of {len(books)} articles.", source_url="", chapters=master_chapters, images=master_images, toc_structure=master_toc)
|
|
151
162
|
|
|
163
|
+
|
|
164
|
+
def bundle_filename_title(title: str, options: ConversionOptions) -> str:
|
|
165
|
+
if options.date_range_active:
|
|
166
|
+
if options.start_date and options.end_date:
|
|
167
|
+
suffix = f"{options.start_date}_to_{options.end_date}"
|
|
168
|
+
elif options.start_date:
|
|
169
|
+
suffix = f"from_{options.start_date}"
|
|
170
|
+
else:
|
|
171
|
+
suffix = f"until_{options.end_date}"
|
|
172
|
+
return title if suffix in title else f"{title}_{suffix}"
|
|
173
|
+
today = datetime.now().strftime("%Y-%m-%d")
|
|
174
|
+
if title.endswith(f"_{today}") or title.endswith(f"-{today}") or today in title:
|
|
175
|
+
return title
|
|
176
|
+
return f"{title}_{today}"
|
|
177
|
+
|
|
152
178
|
def parse_args():
|
|
153
|
-
parser = argparse.ArgumentParser(description="
|
|
179
|
+
parser = argparse.ArgumentParser(description="Web to ebook downloader")
|
|
154
180
|
parser.add_argument("url", nargs='*', help="URL(s) to process")
|
|
155
181
|
parser.add_argument("-i", "--input-file", help="File with URLs")
|
|
156
182
|
parser.add_argument("-o", "--output", help="Output filename")
|
|
@@ -226,6 +252,7 @@ def parse_args():
|
|
|
226
252
|
parser.add_argument("--max-image-bytes-mb", type=int, default=None, help="Override maximum optimized image MB allowed before write")
|
|
227
253
|
parser.add_argument("--start-date", help="Discover and include posts on/after this date (YYYY, YYYY-MM, or YYYY-MM-DD)")
|
|
228
254
|
parser.add_argument("--end-date", help="Discover and include posts on/before this date (YYYY, YYYY-MM, or YYYY-MM-DD)")
|
|
255
|
+
parser.add_argument("--date-sort", choices=["asc", "desc"], default="asc", help="Sort discovered date-range posts earliest-to-latest or latest-to-earliest")
|
|
229
256
|
parser.add_argument("--date-fallback", choices=["auto", "shallow", "metadata", "full"], default="auto", help="How hard to work to find post dates during discovery")
|
|
230
257
|
parser.add_argument("--include-undated", action="store_true", help="Include discovered posts with no date")
|
|
231
258
|
parser.add_argument("--max-discovery-pages", type=int, default=20, help="Maximum listing/archive pages to scan")
|
|
@@ -372,6 +399,7 @@ async def async_main():
|
|
|
372
399
|
pdf_page_size=args.pdf_page_size,
|
|
373
400
|
start_date=args.start_date,
|
|
374
401
|
end_date=args.end_date,
|
|
402
|
+
date_sort=args.date_sort,
|
|
375
403
|
date_fallback=args.date_fallback,
|
|
376
404
|
include_undated=args.include_undated,
|
|
377
405
|
max_discovery_pages=args.max_discovery_pages,
|
|
@@ -463,7 +491,7 @@ async def async_main():
|
|
|
463
491
|
|
|
464
492
|
master_author = args.bundle_author or "Various Authors"
|
|
465
493
|
master_book = create_bundle(processed_books, master_title, master_author)
|
|
466
|
-
fname = ensure_output_extension(args.output, options.output_format) if args.output else default_output_filename(master_title, options.output_format)
|
|
494
|
+
fname = ensure_output_extension(args.output, options.output_format) if args.output else default_output_filename(bundle_filename_title(master_title, options), options.output_format)
|
|
467
495
|
try:
|
|
468
496
|
assert_image_budget(master_book, options)
|
|
469
497
|
except ImageBudgetExceeded as e:
|
|
@@ -498,6 +498,18 @@ def in_range(value: Optional[date], start: Optional[date], end: Optional[date],
|
|
|
498
498
|
return True
|
|
499
499
|
|
|
500
500
|
|
|
501
|
+
def sorted_posts_for_output(posts: List[DiscoveredPost], options: ConversionOptions) -> List[DiscoveredPost]:
|
|
502
|
+
descending = (getattr(options, "date_sort", "asc") or "asc").lower() == "desc"
|
|
503
|
+
|
|
504
|
+
def key(post: DiscoveredPost):
|
|
505
|
+
if post.published_date is None:
|
|
506
|
+
return (1, 0)
|
|
507
|
+
ordinal = post.published_date.toordinal()
|
|
508
|
+
return (0, -ordinal if descending else ordinal)
|
|
509
|
+
|
|
510
|
+
return sorted(posts, key=key)
|
|
511
|
+
|
|
512
|
+
|
|
501
513
|
def default_year_from_bounds(start: Optional[date], end: Optional[date]) -> Optional[int]:
|
|
502
514
|
if start and end and start.year == end.year:
|
|
503
515
|
return start.year
|
|
@@ -570,12 +582,15 @@ async def discover_posts_for_sources(session, sources: Iterable[Source], options
|
|
|
570
582
|
]
|
|
571
583
|
deduped = []
|
|
572
584
|
seen = set()
|
|
573
|
-
for post in filtered:
|
|
585
|
+
for post in sorted_posts_for_output(filtered, options):
|
|
574
586
|
key = canonical_url(post.url)
|
|
575
587
|
if key in seen:
|
|
576
588
|
continue
|
|
577
589
|
seen.add(key)
|
|
578
|
-
deduped.append(Source(
|
|
590
|
+
deduped.append(Source(
|
|
591
|
+
url=post.url,
|
|
592
|
+
published_date=post.published_date.isoformat() if post.published_date else None,
|
|
593
|
+
))
|
|
579
594
|
|
|
580
595
|
if not deduped:
|
|
581
596
|
raise DiscoveryError("No posts found in date range.")
|
|
@@ -10,6 +10,7 @@ from pathlib import Path
|
|
|
10
10
|
from typing import Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
|
|
11
11
|
|
|
12
12
|
from bs4 import BeautifulSoup, Tag
|
|
13
|
+
from bs4.element import NavigableString
|
|
13
14
|
|
|
14
15
|
from ..models import BookData, Chapter, ConversionOptions, log
|
|
15
16
|
from ..utils.llm import LLMHelper
|
|
@@ -40,7 +41,6 @@ CONTENT_SCOPE_CLASSES = {
|
|
|
40
41
|
}
|
|
41
42
|
ALWAYS_SKIP_CLASSES = {
|
|
42
43
|
"post-meta",
|
|
43
|
-
"ai-summary",
|
|
44
44
|
"comment-header",
|
|
45
45
|
"forum-post-header",
|
|
46
46
|
"nav-bar",
|
|
@@ -507,7 +507,17 @@ def should_translate_tag(tag: Tag, scope: str) -> bool:
|
|
|
507
507
|
return True
|
|
508
508
|
|
|
509
509
|
|
|
510
|
+
def prepare_ai_summary_text_nodes(soup: BeautifulSoup) -> None:
|
|
511
|
+
for summary in soup.select(".ai-summary"):
|
|
512
|
+
for child in list(summary.children):
|
|
513
|
+
if isinstance(child, NavigableString) and normalized_source_text(str(child)):
|
|
514
|
+
p = soup.new_tag("p")
|
|
515
|
+
p.string = normalized_source_text(str(child))
|
|
516
|
+
child.replace_with(p)
|
|
517
|
+
|
|
518
|
+
|
|
510
519
|
def collect_translation_units(soup: BeautifulSoup, scope: str) -> List[TranslationUnit]:
|
|
520
|
+
prepare_ai_summary_text_nodes(soup)
|
|
511
521
|
units: List[TranslationUnit] = []
|
|
512
522
|
seen = set()
|
|
513
523
|
for tag in soup.find_all(list(TRANSLATABLE_TAGS)):
|
|
@@ -776,7 +776,7 @@ class PdfWriter:
|
|
|
776
776
|
for title, anchor in cls._chapter_subentries(chapter)
|
|
777
777
|
)
|
|
778
778
|
entries.append(TocEntry(
|
|
779
|
-
chapter.title or f"Chapter {idx}",
|
|
779
|
+
getattr(chapter, "toc_title", None) or chapter.title or f"Chapter {idx}",
|
|
780
780
|
chapter_anchor,
|
|
781
781
|
fallback_page,
|
|
782
782
|
children,
|
|
@@ -77,6 +77,7 @@ class ConversionOptions:
|
|
|
77
77
|
pdf_page_size: str = "letter"
|
|
78
78
|
start_date: Optional[str] = None
|
|
79
79
|
end_date: Optional[str] = None
|
|
80
|
+
date_sort: str = "asc"
|
|
80
81
|
date_fallback: str = "auto"
|
|
81
82
|
include_undated: bool = False
|
|
82
83
|
max_discovery_pages: int = 20
|
|
@@ -115,6 +116,7 @@ class Source:
|
|
|
115
116
|
cookies: Optional[Dict[str, str]] = None
|
|
116
117
|
assets: Optional[List[Dict[str, Any]]] = None
|
|
117
118
|
is_forum: bool = False
|
|
119
|
+
published_date: Optional[str] = None
|
|
118
120
|
|
|
119
121
|
@dataclass
|
|
120
122
|
class SiteProfile:
|
|
@@ -150,6 +152,7 @@ class Chapter:
|
|
|
150
152
|
uid: str
|
|
151
153
|
is_article: bool = False
|
|
152
154
|
is_comments: bool = False
|
|
155
|
+
toc_title: Optional[str] = None
|
|
153
156
|
|
|
154
157
|
@dataclass
|
|
155
158
|
class BookData:
|