dala 2.4.0__tar.gz → 2.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dala-2.4.0 → dala-2.4.2}/CHANGELOG.md +11 -0
- {dala-2.4.0 → dala-2.4.2}/PKG-INFO +180 -93
- {dala-2.4.0 → dala-2.4.2}/dala/cli.py +34 -6
- {dala-2.4.0 → dala-2.4.2}/dala/core/discovery.py +17 -2
- {dala-2.4.0 → dala-2.4.2}/dala/core/profiles.py +37 -16
- {dala-2.4.0 → dala-2.4.2}/dala/core/translation.py +11 -1
- {dala-2.4.0 → dala-2.4.2}/dala/core/writer.py +1 -1
- dala-2.4.2/dala/data/sites.json +37 -0
- {dala-2.4.0 → dala-2.4.2}/dala/models.py +3 -0
- {dala-2.4.0 → dala-2.4.2}/dala/server.py +13 -5
- {dala-2.4.0 → dala-2.4.2}/dala/setup_browser.py +7 -7
- dala-2.4.2/installers/Install or Update Dala.bat +3 -0
- dala-2.4.2/installers/Install or Update Dala.command +41 -0
- dala-2.4.2/installers/Install or Update Dala.ps1 +43 -0
- dala-2.4.2/installers/Install or Update Dala.sh +22 -0
- dala-2.4.2/installers/README.md +59 -0
- dala-2.4.2/launchers/Start Dala Server.bat +3 -0
- dala-2.4.2/launchers/Start Dala Server.command +2 -0
- {dala-2.4.0 → dala-2.4.2}/launchers/dala-server.desktop +1 -1
- {dala-2.4.0 → dala-2.4.2}/pyproject.toml +7 -4
- {dala-2.4.0 → dala-2.4.2}/readme.md +175 -89
- {dala-2.4.0 → dala-2.4.2}/scripts/install-dala.ps1 +11 -3
- dala-2.4.2/scripts/install-dala.sh +179 -0
- {dala-2.4.0 → dala-2.4.2}/tests/test_chrome_extension.py +14 -12
- {dala-2.4.0 → dala-2.4.2}/tests/test_discovery.py +40 -0
- {dala-2.4.0 → dala-2.4.2}/tests/test_image_budget.py +35 -0
- {dala-2.4.0 → dala-2.4.2}/tests/test_server.py +16 -2
- {dala-2.4.0 → dala-2.4.2}/tests/test_setup_browser.py +2 -2
- {dala-2.4.0 → dala-2.4.2}/tests/test_translation.py +61 -0
- {dala-2.4.0 → dala-2.4.2}/tests/test_units.py +44 -0
- {dala-2.4.0 → dala-2.4.2}/tests/test_writer.py +23 -0
- dala-2.4.0/launchers/Start Dala Server.bat +0 -3
- dala-2.4.0/launchers/Start Dala Server.command +0 -2
- dala-2.4.0/scripts/install-dala.sh +0 -52
- {dala-2.4.0 → dala-2.4.2}/.gitignore +0 -0
- {dala-2.4.0 → dala-2.4.2}/LICENSE +0 -0
- {dala-2.4.0 → dala-2.4.2}/PRIVACY.md +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/core/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/core/browser.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/core/dispatcher.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/core/extractor.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/core/forum_image_processor.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/core/image_budget.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/core/image_processor.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/core/server_jobs.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/core/session.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/data/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/data/sites.yaml +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/drivers/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/drivers/base.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/drivers/forum.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/drivers/generic.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/drivers/hn.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/drivers/reddit.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/drivers/substack.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/drivers/wordpress.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/drivers/youtube.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/utils/__init__.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/utils/formatting.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/dala/utils/llm.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/main.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/server.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/sites.yaml +0 -0
- {dala-2.4.0 → dala-2.4.2}/tests/helpers.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/tests/test_browser.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/tests/test_drivers.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/tests/test_hn_integration.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/tests/test_server_saving.py +0 -0
- {dala-2.4.0 → dala-2.4.2}/web_to_epub.py +0 -0
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## v2.4.2
|
|
4
|
+
|
|
5
|
+
### Android / Termux install
|
|
6
|
+
- Added a Termux-specific path to the shell installer that avoids `uv tool install`/`uv pip`, uses Termux Python with `pip --user`, preinstalls native `python-lxml` and `python-pillow`, and applies constraints for the compatible server stack.
|
|
7
|
+
- Re-running the Termux installer now skips package and pip work when Dala is already installed; use `--upgrade` for updates.
|
|
8
|
+
- Updated Termux:Widget start/stop/status scripts to manage the installed `dala-server` command instead of an old source checkout, and included the Android helpers in the installer release bundle.
|
|
9
|
+
|
|
10
|
+
### Site profile config
|
|
11
|
+
- Packaged site profiles now load from JSON so the base install does not require PyYAML.
|
|
12
|
+
- Custom `sites.yaml` remains supported when `dala[config]` is installed; custom `sites.json` is supported without extra dependencies.
|
|
13
|
+
|
|
3
14
|
## v2.3.6
|
|
4
15
|
- **Package:** Include missing Firefox extension XPI (v2.3.5) in the release.
|
|
5
16
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dala
|
|
3
|
-
Version: 2.4.
|
|
4
|
-
Summary: Web to
|
|
3
|
+
Version: 2.4.2
|
|
4
|
+
Summary: Web to ebook downloader
|
|
5
5
|
Project-URL: Homepage, https://github.com/anaxonda/dala
|
|
6
6
|
Project-URL: Repository, https://github.com/anaxonda/dala
|
|
7
7
|
Project-URL: Issues, https://github.com/anaxonda/dala/issues
|
|
@@ -27,13 +27,12 @@ Requires-Dist: aiohttp[speedups]>=3.9.0
|
|
|
27
27
|
Requires-Dist: beautifulsoup4>=4.11.0
|
|
28
28
|
Requires-Dist: deep-translator>=1.11.4
|
|
29
29
|
Requires-Dist: ebooklib>=0.18
|
|
30
|
-
Requires-Dist: fastapi>=0.
|
|
30
|
+
Requires-Dist: fastapi>=0.99.0
|
|
31
31
|
Requires-Dist: lxml[html-clean]>=4.9.0
|
|
32
32
|
Requires-Dist: pillow>=9.0.0
|
|
33
33
|
Requires-Dist: pygments>=2.14.0
|
|
34
34
|
Requires-Dist: pypdf>=5.9.0
|
|
35
35
|
Requires-Dist: python-dotenv>=1.0.0
|
|
36
|
-
Requires-Dist: pyyaml>=6.0.3
|
|
37
36
|
Requires-Dist: requests>=2.30.0
|
|
38
37
|
Requires-Dist: tqdm>=4.65.0
|
|
39
38
|
Requires-Dist: trafilatura>=1.6.0
|
|
@@ -42,6 +41,8 @@ Requires-Dist: youtube-comment-downloader>=0.1.0
|
|
|
42
41
|
Requires-Dist: youtube-transcript-api>=0.6.0
|
|
43
42
|
Provides-Extra: browser
|
|
44
43
|
Requires-Dist: playwright>=1.40.0; extra == 'browser'
|
|
44
|
+
Provides-Extra: config
|
|
45
|
+
Requires-Dist: pyyaml>=6.0.3; extra == 'config'
|
|
45
46
|
Description-Content-Type: text/markdown
|
|
46
47
|
|
|
47
48
|
# dala: Web pages and threads as books
|
|
@@ -82,7 +83,7 @@ Install the command-line tools:
|
|
|
82
83
|
|
|
83
84
|
```bash
|
|
84
85
|
uv tool install dala
|
|
85
|
-
dala-server
|
|
86
|
+
dala-server
|
|
86
87
|
```
|
|
87
88
|
|
|
88
89
|
For development from a source checkout:
|
|
@@ -90,7 +91,7 @@ For development from a source checkout:
|
|
|
90
91
|
```bash
|
|
91
92
|
git clone https://github.com/anaxonda/dala.git
|
|
92
93
|
cd dala
|
|
93
|
-
uv run dala-server
|
|
94
|
+
uv run dala-server
|
|
94
95
|
```
|
|
95
96
|
|
|
96
97
|
Install the browser extension:
|
|
@@ -100,7 +101,9 @@ Install the browser extension:
|
|
|
100
101
|
|
|
101
102
|
Then open a page, click the **dala** icon, and click **Download Page**.
|
|
102
103
|
|
|
103
|
-
|
|
104
|
+
Dala opens its local status page when `dala-server` starts. Use `dala-server --no-open` for background services, SSH sessions, or scripts that should not open a browser.
|
|
105
|
+
|
|
106
|
+
For PDF output, JavaScript-heavy pages without extension capture, or headless browser fallback, see [Headless Browser Support](#headless-browser-support).
|
|
104
107
|
|
|
105
108
|
## What Do I Install?
|
|
106
109
|
|
|
@@ -111,13 +114,13 @@ For PDF output, CLI/server rendering of JavaScript-heavy pages, or server-side b
|
|
|
111
114
|
| Save forum images or attachments behind login | Extension with **Use Site Cookies** | Usually none |
|
|
112
115
|
| Bundle browser links | Extension queue | None |
|
|
113
116
|
| Bundle URLs from a file | CLI | None |
|
|
114
|
-
| Render JavaScript-heavy pages without extension capture | CLI/
|
|
115
|
-
| Generate PDF | Extension or CLI |
|
|
117
|
+
| Render JavaScript-heavy pages without extension capture | CLI/headless browser rendering | Headless browser support |
|
|
118
|
+
| Generate PDF | Extension or CLI | Headless browser support |
|
|
116
119
|
| Download a YouTube transcript | CLI or extension | None |
|
|
117
120
|
| AI summaries or LLM translation | CLI or extension | API key |
|
|
118
121
|
| Google Translate translation | CLI or extension | No API key |
|
|
119
122
|
|
|
120
|
-
|
|
123
|
+
Headless browser support lets the Dala server control Chrome/Chromium in the background. It is needed for PDF output and some JavaScript-heavy pages, and is separate from the normal Dala browser extension.
|
|
121
124
|
|
|
122
125
|
## Source Support
|
|
123
126
|
|
|
@@ -136,22 +139,22 @@ Server-side browser setup uses Dala's optional Python browser-control support pl
|
|
|
136
139
|
| Feature | Supported |
|
|
137
140
|
| --- | --- |
|
|
138
141
|
| EPUB | E-reader-oriented typography, metadata, table of contents, image optimization |
|
|
139
|
-
| PDF | Document and e-reader presets through
|
|
142
|
+
| PDF | Document and e-reader presets through headless browser rendering |
|
|
140
143
|
| Bundles | Multiple pages combined into one anthology-style file |
|
|
141
144
|
| Images | Compact, Balanced, or Full presets; optional grayscale conversion |
|
|
142
145
|
| Translation | LLM or Google Translate; underneath, side-by-side inspired by [Bitextual](https://github.com/wydengyre/bitextual), EPUB footnote, or replace modes |
|
|
143
146
|
| Summaries | Optional LLM-generated summaries for long articles, discussions, forums, and transcripts |
|
|
144
147
|
|
|
145
|
-
##
|
|
148
|
+
## Headless Browser Support
|
|
146
149
|
|
|
147
|
-
*Basic EPUB downloads do not need
|
|
150
|
+
*Basic EPUB downloads do not need headless browser support. This section is about Dala's optional Python support for controlling Chrome/Chromium in the background, not the browser extension.*
|
|
148
151
|
|
|
149
|
-
Install browser
|
|
152
|
+
Install Dala with headless browser support only if you want PDF output, JavaScript-heavy page rendering without extension capture, or headless browser fallback:
|
|
150
153
|
|
|
151
154
|
```bash
|
|
152
155
|
uv tool install --force "dala[browser]"
|
|
153
156
|
dala-setup-browser
|
|
154
|
-
dala-server
|
|
157
|
+
dala-server
|
|
155
158
|
```
|
|
156
159
|
|
|
157
160
|
For development from a source checkout:
|
|
@@ -159,7 +162,7 @@ For development from a source checkout:
|
|
|
159
162
|
```bash
|
|
160
163
|
uv sync --extra browser
|
|
161
164
|
uv run dala-setup-browser
|
|
162
|
-
uv run dala-server
|
|
165
|
+
uv run dala-server
|
|
163
166
|
```
|
|
164
167
|
|
|
165
168
|
`dala-setup-browser` first tries to use an existing Chromium-compatible browser such as Chrome, Edge, Brave, or Chromium. If none is detected, it installs Playwright's managed Chromium.
|
|
@@ -167,24 +170,33 @@ uv run dala-server --open
|
|
|
167
170
|
<details>
|
|
168
171
|
<summary>Installer scripts and launchers</summary>
|
|
169
172
|
|
|
170
|
-
The `
|
|
173
|
+
The `installers/` directory has desktop-oriented installers and wrappers. Windows and macOS installers prompt for optional headless browser support and create a desktop launcher when possible; the Linux wrapper installs or updates from a terminal:
|
|
174
|
+
|
|
175
|
+
```text
|
|
176
|
+
installers/Install or Update Dala.bat # Windows double-click wrapper
|
|
177
|
+
installers/Install or Update Dala.ps1 # Windows PowerShell installer
|
|
178
|
+
installers/Install or Update Dala.command # macOS double-click installer
|
|
179
|
+
installers/Install or Update Dala.sh # Linux terminal installer
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
The `scripts/` directory has conservative lower-level installers that reuse an existing `uv` install, install `uv` only if missing, and make headless browser support opt-in:
|
|
171
183
|
|
|
172
184
|
```bash
|
|
173
185
|
# macOS/Linux
|
|
174
186
|
./scripts/install-dala.sh
|
|
175
|
-
./scripts/install-dala.sh --browser
|
|
187
|
+
./scripts/install-dala.sh --headless-browser
|
|
176
188
|
|
|
177
189
|
# Windows PowerShell
|
|
178
190
|
.\scripts\install-dala.ps1
|
|
179
|
-
.\scripts\install-dala.ps1 -
|
|
191
|
+
.\scripts\install-dala.ps1 -HeadlessBrowser
|
|
180
192
|
```
|
|
181
193
|
|
|
182
|
-
The `launchers/` directory has double-click templates that start `dala-server
|
|
194
|
+
The `launchers/` directory has double-click templates that start `dala-server` after Dala is installed.
|
|
183
195
|
|
|
184
196
|
</details>
|
|
185
197
|
|
|
186
198
|
<details>
|
|
187
|
-
<summary>CLI examples for
|
|
199
|
+
<summary>CLI examples for headless browser rendering</summary>
|
|
188
200
|
|
|
189
201
|
```bash
|
|
190
202
|
# Render with an auto-detected browser
|
|
@@ -238,28 +250,20 @@ Dala can process pages that are already readable in your browser, including page
|
|
|
238
250
|
|
|
239
251
|
The simplest workflow is normal browser capture: open the article until it is readable in your everyday browser, then click the Dala extension. If you use a browser extension to make pages readable, install it in the same browser where you run Dala.
|
|
240
252
|
|
|
241
|
-
|
|
253
|
+
Headless browser fallback is an advanced local fallback for automation and testing. The server uses a dedicated Dala browser profile at `~/.local/share/dala/browser-profile` by default; it does not automatically use your normal browser profile.
|
|
242
254
|
|
|
243
255
|
<details>
|
|
244
|
-
<summary>Advanced:
|
|
256
|
+
<summary>Advanced: headless browser extensions for local testing</summary>
|
|
245
257
|
|
|
246
|
-
|
|
258
|
+
Headless browser fallback can load an unpacked Chromium-compatible extension when you explicitly provide its path. This is useful for local automation tests where the rendered browser needs the same helper extension behavior you use interactively. Configure it with:
|
|
247
259
|
|
|
248
260
|
```bash
|
|
249
|
-
export
|
|
261
|
+
export DALA_BROWSER_EXTENSION_PATH=/path/to/unpacked-extension
|
|
250
262
|
export DALA_BROWSER_EXECUTABLE=/usr/bin/google-chrome
|
|
251
263
|
export DALA_BROWSER_PROFILE_DIR=/path/to/custom/dala-chromium-profile
|
|
252
264
|
```
|
|
253
265
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
```bash
|
|
257
|
-
git clone --depth 1 https://gitflic.ru/project/magnolia1234/bpc_uploads.git config/bpc/latest
|
|
258
|
-
mkdir -p config/bpc/chrome-unpacked
|
|
259
|
-
unzip -oq config/bpc/latest/bypass-paywalls-chrome-clean-master.zip -d config/bpc/chrome-unpacked
|
|
260
|
-
```
|
|
261
|
-
|
|
262
|
-
`config/bpc/` is ignored by git.
|
|
266
|
+
Only load extensions you trust, and only when running a local server you control.
|
|
263
267
|
|
|
264
268
|
</details>
|
|
265
269
|
|
|
@@ -285,12 +289,12 @@ The extension has a compact popup for common choices and an Options page for ser
|
|
|
285
289
|
| **Translation** | Translates article text with LLM or Google Translate. | Bilingual reading or translated-only output. |
|
|
286
290
|
| **Video Thumbnails** | Embeds periodic YouTube thumbnails. | Visual context for transcripts. |
|
|
287
291
|
| **Use Site Cookies** | Sends browser cookies to the backend. Defaults on only for local server URLs. | Forums, protected images, authenticated pages. |
|
|
288
|
-
| **
|
|
292
|
+
| **Headless Browser Fallback** | Lets the server retry failed extraction in a background Chromium-compatible browser. | JavaScript-heavy, blocked, or difficult pages. |
|
|
289
293
|
| **Force Forum Driver** | Uses forum multi-page scraping. | XenForo/vBulletin-style threads. |
|
|
290
294
|
| **Forum Pages** | Downloads specific pages such as `1,3-5`. | Partial forum thread downloads. |
|
|
291
295
|
| **Max Pages** | Limits sequential forum crawling. | Avoid unexpectedly large downloads. |
|
|
292
296
|
|
|
293
|
-
The Options page also includes diagnostics for server version, Playwright/browser/profile
|
|
297
|
+
The Options page also includes diagnostics for server version, Playwright/headless browser/profile status, PDF availability, retained jobs, cleanup retention, and the last conversion status/error. If PDF is unavailable, the extension disables PDF output and falls back to EPUB.
|
|
294
298
|
|
|
295
299
|
</details>
|
|
296
300
|
|
|
@@ -431,10 +435,10 @@ OPENAI_API_KEY=your-openai-api-key
|
|
|
431
435
|
LLM_PROVIDER=auto
|
|
432
436
|
LLM_MODEL=gemini-3.1-flash-lite
|
|
433
437
|
|
|
434
|
-
#
|
|
438
|
+
# Headless browser fallback
|
|
435
439
|
DALA_BROWSER_EXECUTABLE=/path/to/chrome-or-chromium
|
|
436
440
|
DALA_BROWSER_PROFILE_DIR=/path/to/dala-browser-profile
|
|
437
|
-
|
|
441
|
+
DALA_BROWSER_EXTENSION_PATH=/path/to/unpacked-extension
|
|
438
442
|
|
|
439
443
|
# Translation speed tuning
|
|
440
444
|
DALA_GOOGLE_TRANSLATE_CHUNK_SIZE=5
|
|
@@ -460,46 +464,47 @@ CLI and extension outputs use the same title-based naming helpers where possible
|
|
|
460
464
|
|
|
461
465
|
## Platform-Specific Installation
|
|
462
466
|
|
|
463
|
-
Dala is primarily tested on Linux and Android/Termux.
|
|
467
|
+
Dala is primarily tested on Linux and Android/Termux. Windows and macOS should work with the packaged installers or the manual `uv` commands below.
|
|
464
468
|
|
|
465
|
-
|
|
466
|
-
<summary>macOS, Windows, Linux, and Android setup</summary>
|
|
469
|
+
Git is not required when installing the published package with `uv tool install dala`. It is only needed for source-checkout development.
|
|
467
470
|
|
|
468
|
-
|
|
471
|
+
The GitHub release asset `dala-installers-vVERSION.zip` contains:
|
|
469
472
|
|
|
470
|
-
- `
|
|
471
|
-
-
|
|
473
|
+
- `installers/`: double-click or terminal installers for desktop users
|
|
474
|
+
- `scripts/`: lower-level install/update scripts
|
|
475
|
+
- `launchers/`: templates that start the installed `dala-server`
|
|
476
|
+
- `android/`: Termux notes and Termux:Widget start/stop/status shortcuts
|
|
472
477
|
|
|
473
|
-
|
|
478
|
+
The installers install or update the Python server from PyPI, install `uv` only if it is missing, optionally add headless browser support, and do not install the browser extension.
|
|
474
479
|
|
|
475
|
-
|
|
476
|
-
# macOS
|
|
477
|
-
brew install uv
|
|
480
|
+
### macOS
|
|
478
481
|
|
|
479
|
-
|
|
480
|
-
powershell -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
482
|
+
Recommended: unzip `dala-installers-vVERSION.zip`, then right-click `installers/Install or Update Dala.command` and choose **Open**. The installer prompts for optional headless browser support and creates `Start Dala Server.command` on the Desktop when possible.
|
|
481
483
|
|
|
482
|
-
|
|
483
|
-
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
484
|
+
Manual install:
|
|
484
485
|
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
486
|
+
```bash
|
|
487
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
488
|
+
uv tool install dala
|
|
489
|
+
dala-server
|
|
488
490
|
```
|
|
489
491
|
|
|
490
|
-
|
|
492
|
+
Headless browser/PDF support:
|
|
491
493
|
|
|
492
494
|
```bash
|
|
493
|
-
|
|
494
|
-
dala-server --open
|
|
495
|
+
./scripts/install-dala.sh --headless-browser
|
|
495
496
|
```
|
|
496
497
|
|
|
497
|
-
macOS may prompt you to install Command Line Tools if Git is not installed.
|
|
498
|
-
|
|
499
498
|
<details>
|
|
500
499
|
<summary>Start Dala automatically with launchd</summary>
|
|
501
500
|
|
|
502
|
-
|
|
501
|
+
Find the installed server path:
|
|
502
|
+
|
|
503
|
+
```bash
|
|
504
|
+
command -v dala-server
|
|
505
|
+
```
|
|
506
|
+
|
|
507
|
+
Create `~/Library/LaunchAgents/com.dala.server.plist`, replacing `/Users/YOU/.local/bin/dala-server` with that path:
|
|
503
508
|
|
|
504
509
|
```xml
|
|
505
510
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
@@ -510,12 +515,9 @@ Create `~/Library/LaunchAgents/com.dala.server.plist`:
|
|
|
510
515
|
<string>com.dala.server</string>
|
|
511
516
|
<key>ProgramArguments</key>
|
|
512
517
|
<array>
|
|
513
|
-
<string>/
|
|
514
|
-
<string
|
|
515
|
-
<string>dala-server</string>
|
|
518
|
+
<string>/Users/YOU/.local/bin/dala-server</string>
|
|
519
|
+
<string>--no-open</string>
|
|
516
520
|
</array>
|
|
517
|
-
<key>WorkingDirectory</key>
|
|
518
|
-
<string>/path/to/dala</string>
|
|
519
521
|
<key>RunAtLoad</key>
|
|
520
522
|
<true/>
|
|
521
523
|
<key>KeepAlive</key>
|
|
@@ -538,11 +540,14 @@ launchctl load ~/Library/LaunchAgents/com.dala.server.plist
|
|
|
538
540
|
|
|
539
541
|
### Windows
|
|
540
542
|
|
|
541
|
-
|
|
543
|
+
Recommended: unzip `dala-installers-vVERSION.zip`, then double-click `installers\Install or Update Dala.bat`. It runs the PowerShell installer, prompts for optional headless browser support, and creates `Start Dala Server.bat` on the Desktop when possible.
|
|
544
|
+
|
|
545
|
+
Manual install from PowerShell:
|
|
542
546
|
|
|
543
547
|
```powershell
|
|
548
|
+
powershell -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
544
549
|
uv tool install dala
|
|
545
|
-
dala-server
|
|
550
|
+
dala-server
|
|
546
551
|
```
|
|
547
552
|
|
|
548
553
|
If you see an execution policy error, run:
|
|
@@ -551,6 +556,12 @@ If you see an execution policy error, run:
|
|
|
551
556
|
Set-ExecutionPolicy RemoteSigned -Scope CurrentUser
|
|
552
557
|
```
|
|
553
558
|
|
|
559
|
+
Headless browser/PDF support:
|
|
560
|
+
|
|
561
|
+
```powershell
|
|
562
|
+
.\scripts\install-dala.ps1 -HeadlessBrowser
|
|
563
|
+
```
|
|
564
|
+
|
|
554
565
|
<details>
|
|
555
566
|
<summary>Start Dala automatically from the Startup Folder</summary>
|
|
556
567
|
|
|
@@ -558,7 +569,7 @@ Use `launchers/Start Dala Server.bat`, or create `start_dala.bat`:
|
|
|
558
569
|
|
|
559
570
|
```bat
|
|
560
571
|
@echo off
|
|
561
|
-
dala-server
|
|
572
|
+
dala-server
|
|
562
573
|
```
|
|
563
574
|
|
|
564
575
|
Press `Win + R`, enter `shell:startup`, and add a shortcut to `start_dala.bat`.
|
|
@@ -567,21 +578,46 @@ Press `Win + R`, enter `shell:startup`, and add a shortcut to `start_dala.bat`.
|
|
|
567
578
|
|
|
568
579
|
### Linux
|
|
569
580
|
|
|
570
|
-
|
|
581
|
+
Recommended: unzip `dala-installers-vVERSION.zip`, then run:
|
|
582
|
+
|
|
583
|
+
```bash
|
|
584
|
+
sh "installers/Install or Update Dala.sh"
|
|
585
|
+
```
|
|
586
|
+
|
|
587
|
+
Manual install:
|
|
588
|
+
|
|
589
|
+
```bash
|
|
590
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
591
|
+
uv tool install dala
|
|
592
|
+
dala-server
|
|
593
|
+
```
|
|
594
|
+
|
|
595
|
+
Headless browser/PDF support:
|
|
596
|
+
|
|
597
|
+
```bash
|
|
598
|
+
./scripts/install-dala.sh --headless-browser
|
|
599
|
+
```
|
|
600
|
+
|
|
601
|
+
Dala can auto-detect `chromium`, `google-chrome`, `microsoft-edge`, or `brave-browser` from `PATH`. A `.desktop` launcher template is available at `launchers/dala-server.desktop`; if your desktop environment cannot find `dala-server`, edit `Exec=` to the absolute path from `command -v dala-server`.
|
|
571
602
|
|
|
572
603
|
<details>
|
|
573
604
|
<summary>Start Dala automatically with systemd</summary>
|
|
574
605
|
|
|
606
|
+
Find the installed server path:
|
|
607
|
+
|
|
608
|
+
```bash
|
|
609
|
+
command -v dala-server
|
|
610
|
+
```
|
|
611
|
+
|
|
575
612
|
Create `~/.config/systemd/user/epub_server.service`:
|
|
576
613
|
|
|
577
614
|
```ini
|
|
578
615
|
[Unit]
|
|
579
|
-
Description=Web to
|
|
616
|
+
Description=Web to ebook Python Server
|
|
580
617
|
After=network.target
|
|
581
618
|
|
|
582
619
|
[Service]
|
|
583
|
-
|
|
584
|
-
ExecStart=/path/to/dala/.venv/bin/dala-server
|
|
620
|
+
ExecStart=/home/YOU/.local/bin/dala-server --no-open
|
|
585
621
|
Restart=always
|
|
586
622
|
RestartSec=5
|
|
587
623
|
|
|
@@ -599,32 +635,56 @@ systemctl --user enable --now epub_server
|
|
|
599
635
|
|
|
600
636
|
### Android / Termux
|
|
601
637
|
|
|
602
|
-
|
|
638
|
+
Termux does not use the desktop installer bundle. Use the shell installer; it detects Android/Termux and uses Termux Python plus `pip --user` instead of `uv tool install`:
|
|
603
639
|
|
|
604
640
|
```bash
|
|
605
641
|
pkg update
|
|
606
|
-
pkg install
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
642
|
+
pkg install python python-pip curl
|
|
643
|
+
curl -fsSLo install-dala.sh https://raw.githubusercontent.com/anaxonda/dala/main/scripts/install-dala.sh
|
|
644
|
+
sh install-dala.sh
|
|
645
|
+
dala-server --no-open
|
|
610
646
|
```
|
|
611
647
|
|
|
612
|
-
|
|
648
|
+
Open `http://127.0.0.1:8000/` manually, or point Firefox for Android plus the Dala extension at the local Termux server.
|
|
649
|
+
|
|
650
|
+
Optional Termux:Widget shortcuts are in `android/`; copy them into `~/.shortcuts/dala/` after installing Dala if you want home-screen start/stop/status buttons.
|
|
651
|
+
|
|
652
|
+
Why not `uv tool install` on Termux? Termux can package `uv`, but current `uv` releases can fail while inspecting Android Python with `Unknown operating system: android`. `uv pip` has the same limitation on affected devices. The installer avoids that path.
|
|
653
|
+
|
|
654
|
+
The installer also installs Termux `python-lxml` and `python-pillow` packages and uses a small constraints file for the server stack. Without that, PyPI may try to build heavy native dependencies such as `lxml`, `Pillow`, or `pydantic-core` from source on Android.
|
|
655
|
+
|
|
656
|
+
Re-running the installer is fast when Dala is already installed. To update Dala, run:
|
|
613
657
|
|
|
614
658
|
```bash
|
|
615
|
-
|
|
616
|
-
source .venv/bin/activate
|
|
617
|
-
UV_LINK_MODE=copy UV_CACHE_DIR=$HOME/.cache/uv uv pip install -e .
|
|
618
|
-
uv run dala-server
|
|
659
|
+
sh install-dala.sh --upgrade
|
|
619
660
|
```
|
|
620
661
|
|
|
621
|
-
|
|
662
|
+
If the installer reports broken `pip`, repair Termux Python packages and rerun it:
|
|
622
663
|
|
|
623
|
-
|
|
664
|
+
```bash
|
|
665
|
+
pkg update
|
|
666
|
+
pkg install python python-pip
|
|
667
|
+
pkg reinstall python-pip
|
|
668
|
+
sh install-dala.sh
|
|
669
|
+
```
|
|
670
|
+
|
|
671
|
+
If you want a source checkout on Android, install with `pip --user` from the checkout:
|
|
672
|
+
|
|
673
|
+
```bash
|
|
674
|
+
pkg install git python
|
|
675
|
+
git clone https://github.com/anaxonda/dala.git
|
|
676
|
+
cd dala
|
|
677
|
+
pip install --user -e .
|
|
678
|
+
python -m dala.server --no-open
|
|
679
|
+
```
|
|
624
680
|
|
|
625
681
|
## sites.yaml Customization
|
|
626
682
|
|
|
627
|
-
You can define custom extraction rules for specific websites in `sites.yaml` at the project root.
|
|
683
|
+
You can define custom extraction rules for specific websites in `sites.yaml` at the project root. YAML config support needs PyYAML; install `dala[config]` if you use custom YAML profiles. Packaged defaults do not need PyYAML, and JSON config files can be loaded without extra dependencies.
|
|
684
|
+
|
|
685
|
+
```bash
|
|
686
|
+
uv tool install --force "dala[config]"
|
|
687
|
+
```
|
|
628
688
|
|
|
629
689
|
```yaml
|
|
630
690
|
- name: "The New York Times"
|
|
@@ -660,7 +720,7 @@ Then open `http://127.0.0.1:8000/ping`.
|
|
|
660
720
|
|
|
661
721
|
### PDF option is disabled
|
|
662
722
|
|
|
663
|
-
Install
|
|
723
|
+
Install headless browser support:
|
|
664
724
|
|
|
665
725
|
```bash
|
|
666
726
|
uv tool install --force "dala[browser]"
|
|
@@ -669,7 +729,7 @@ dala-setup-browser
|
|
|
669
729
|
|
|
670
730
|
For a source checkout, use `uv sync --extra browser` and `uv run dala-setup-browser`. Restart the server and check `http://127.0.0.1:8000/`.
|
|
671
731
|
|
|
672
|
-
###
|
|
732
|
+
### Headless browser fallback fails
|
|
673
733
|
|
|
674
734
|
Use headed mode to debug login, bot challenges, or extension loading:
|
|
675
735
|
|
|
@@ -679,14 +739,16 @@ uv run dala --browser --headed "https://example.com/article"
|
|
|
679
739
|
|
|
680
740
|
If an interactive bot challenge appears, solve it in your normal browser and run the extension from the readable tab.
|
|
681
741
|
|
|
682
|
-
### Android / Termux
|
|
742
|
+
### Android / Termux install fails
|
|
683
743
|
|
|
684
744
|
```bash
|
|
685
|
-
pkg
|
|
686
|
-
pkg install
|
|
745
|
+
pkg update
|
|
746
|
+
pkg install python python-pip curl
|
|
747
|
+
pkg reinstall python-pip
|
|
748
|
+
sh install-dala.sh
|
|
687
749
|
```
|
|
688
750
|
|
|
689
|
-
|
|
751
|
+
Avoid `uv tool install` and `uv pip` on Termux if they fail with `Unknown operating system: android`; use the installer path from [Android / Termux](#android--termux).
|
|
690
752
|
|
|
691
753
|
### Translation fails
|
|
692
754
|
|
|
@@ -715,6 +777,31 @@ Browser download, CLI output, or server archive
|
|
|
715
777
|
- `server.py`: FastAPI backend used by the extensions and async job flow.
|
|
716
778
|
- `firefox_extension/` and `extension_chrome/`: browser clients for page capture, queueing, options, and downloads.
|
|
717
779
|
|
|
780
|
+
## Release Checklist
|
|
781
|
+
|
|
782
|
+
Dala releases now have two distribution channels:
|
|
783
|
+
|
|
784
|
+
- **PyPI:** publish the Python package so users can run `uv tool install dala`.
|
|
785
|
+
- **GitHub Releases:** attach browser extensions and the easy installer bundle.
|
|
786
|
+
|
|
787
|
+
Expected GitHub release assets:
|
|
788
|
+
|
|
789
|
+
```text
|
|
790
|
+
dala-chrome-vEXTENSION_VERSION.zip
|
|
791
|
+
dala-firefox-vEXTENSION_VERSION-signed.xpi
|
|
792
|
+
dala-installers-vPYTHON_PACKAGE_VERSION.zip
|
|
793
|
+
```
|
|
794
|
+
|
|
795
|
+
If only the Python package/installers changed, extension assets may keep the previous extension version. State that explicitly in the release notes.
|
|
796
|
+
|
|
797
|
+
Build the unsigned extension packages and installer bundle with:
|
|
798
|
+
|
|
799
|
+
```bash
|
|
800
|
+
./package_extensions.sh
|
|
801
|
+
```
|
|
802
|
+
|
|
803
|
+
Then replace the unsigned Firefox XPI with the AMO-signed XPI before publishing the GitHub release. The installer bundle should be attached to the release page so nontechnical users can download the double-click installers without cloning the repository.
|
|
804
|
+
|
|
718
805
|
## Roadmap
|
|
719
806
|
|
|
720
807
|
### Planned
|
|
@@ -739,7 +826,7 @@ Browser download, CLI output, or server archive
|
|
|
739
826
|
- Only enable cookie sharing for a backend you control and trust.
|
|
740
827
|
- Do not expose the Dala server directly to the public internet.
|
|
741
828
|
- API keys in the extension or `.env` are secrets. Do not commit `.env`.
|
|
742
|
-
- Dala is intended to process pages you are allowed to access. Use authenticated capture and browser fallback responsibly.
|
|
829
|
+
- Dala is intended to process pages you are allowed to access. Use authenticated capture and headless browser fallback responsibly.
|
|
743
830
|
|
|
744
831
|
## License
|
|
745
832
|
|
|
@@ -94,9 +94,12 @@ async def process_urls(
|
|
|
94
94
|
cookies=source.cookies,
|
|
95
95
|
assets=source.assets,
|
|
96
96
|
is_forum=False,
|
|
97
|
+
published_date=source.published_date,
|
|
97
98
|
)
|
|
98
99
|
book = await GenericDriver().prepare_book_data(context, generic_source)
|
|
99
100
|
success = book is not None
|
|
101
|
+
if book is not None and source.published_date:
|
|
102
|
+
book.extra_metadata["published_date"] = source.published_date
|
|
100
103
|
return book
|
|
101
104
|
except BrowserChallengeError:
|
|
102
105
|
raise
|
|
@@ -142,15 +145,38 @@ def create_bundle(books: List[BookData], title: str, author: str) -> BookData:
|
|
|
142
145
|
elif chap.is_comments: comments_chap = chap
|
|
143
146
|
|
|
144
147
|
if article_chap:
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
148
|
+
toc_title = book.title
|
|
149
|
+
published_date = (book.extra_metadata or {}).get("published_date")
|
|
150
|
+
if published_date:
|
|
151
|
+
toc_title = f"{published_date} - {toc_title}"
|
|
152
|
+
article_chap.toc_title = toc_title
|
|
153
|
+
if comments_chap:
|
|
154
|
+
master_toc.append((
|
|
155
|
+
epub.Link(article_chap.filename, toc_title, article_chap.uid),
|
|
156
|
+
[epub.Link(comments_chap.filename, "Comments", comments_chap.uid)],
|
|
157
|
+
))
|
|
158
|
+
else:
|
|
159
|
+
master_toc.append(epub.Link(article_chap.filename, toc_title, article_chap.uid))
|
|
149
160
|
|
|
150
161
|
return BookData(title=title, author=author, uid=master_uid, language="en", description=f"Bundle of {len(books)} articles.", source_url="", chapters=master_chapters, images=master_images, toc_structure=master_toc)
|
|
151
162
|
|
|
163
|
+
|
|
164
|
+
def bundle_filename_title(title: str, options: ConversionOptions) -> str:
|
|
165
|
+
if options.date_range_active:
|
|
166
|
+
if options.start_date and options.end_date:
|
|
167
|
+
suffix = f"{options.start_date}_to_{options.end_date}"
|
|
168
|
+
elif options.start_date:
|
|
169
|
+
suffix = f"from_{options.start_date}"
|
|
170
|
+
else:
|
|
171
|
+
suffix = f"until_{options.end_date}"
|
|
172
|
+
return title if suffix in title else f"{title}_{suffix}"
|
|
173
|
+
today = datetime.now().strftime("%Y-%m-%d")
|
|
174
|
+
if title.endswith(f"_{today}") or title.endswith(f"-{today}") or today in title:
|
|
175
|
+
return title
|
|
176
|
+
return f"{title}_{today}"
|
|
177
|
+
|
|
152
178
|
def parse_args():
|
|
153
|
-
parser = argparse.ArgumentParser(description="
|
|
179
|
+
parser = argparse.ArgumentParser(description="Web to ebook downloader")
|
|
154
180
|
parser.add_argument("url", nargs='*', help="URL(s) to process")
|
|
155
181
|
parser.add_argument("-i", "--input-file", help="File with URLs")
|
|
156
182
|
parser.add_argument("-o", "--output", help="Output filename")
|
|
@@ -226,6 +252,7 @@ def parse_args():
|
|
|
226
252
|
parser.add_argument("--max-image-bytes-mb", type=int, default=None, help="Override maximum optimized image MB allowed before write")
|
|
227
253
|
parser.add_argument("--start-date", help="Discover and include posts on/after this date (YYYY, YYYY-MM, or YYYY-MM-DD)")
|
|
228
254
|
parser.add_argument("--end-date", help="Discover and include posts on/before this date (YYYY, YYYY-MM, or YYYY-MM-DD)")
|
|
255
|
+
parser.add_argument("--date-sort", choices=["asc", "desc"], default="asc", help="Sort discovered date-range posts earliest-to-latest or latest-to-earliest")
|
|
229
256
|
parser.add_argument("--date-fallback", choices=["auto", "shallow", "metadata", "full"], default="auto", help="How hard to work to find post dates during discovery")
|
|
230
257
|
parser.add_argument("--include-undated", action="store_true", help="Include discovered posts with no date")
|
|
231
258
|
parser.add_argument("--max-discovery-pages", type=int, default=20, help="Maximum listing/archive pages to scan")
|
|
@@ -372,6 +399,7 @@ async def async_main():
|
|
|
372
399
|
pdf_page_size=args.pdf_page_size,
|
|
373
400
|
start_date=args.start_date,
|
|
374
401
|
end_date=args.end_date,
|
|
402
|
+
date_sort=args.date_sort,
|
|
375
403
|
date_fallback=args.date_fallback,
|
|
376
404
|
include_undated=args.include_undated,
|
|
377
405
|
max_discovery_pages=args.max_discovery_pages,
|
|
@@ -463,7 +491,7 @@ async def async_main():
|
|
|
463
491
|
|
|
464
492
|
master_author = args.bundle_author or "Various Authors"
|
|
465
493
|
master_book = create_bundle(processed_books, master_title, master_author)
|
|
466
|
-
fname = ensure_output_extension(args.output, options.output_format) if args.output else default_output_filename(master_title, options.output_format)
|
|
494
|
+
fname = ensure_output_extension(args.output, options.output_format) if args.output else default_output_filename(bundle_filename_title(master_title, options), options.output_format)
|
|
467
495
|
try:
|
|
468
496
|
assert_image_budget(master_book, options)
|
|
469
497
|
except ImageBudgetExceeded as e:
|