dala 2.4.1__tar.gz → 2.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dala-2.4.1 → dala-2.4.2}/CHANGELOG.md +11 -0
- {dala-2.4.1 → dala-2.4.2}/PKG-INFO +46 -17
- {dala-2.4.1 → dala-2.4.2}/dala/core/profiles.py +37 -16
- dala-2.4.2/dala/data/sites.json +37 -0
- {dala-2.4.1 → dala-2.4.2}/installers/README.md +2 -0
- {dala-2.4.1 → dala-2.4.2}/pyproject.toml +5 -3
- {dala-2.4.1 → dala-2.4.2}/readme.md +42 -14
- dala-2.4.2/scripts/install-dala.sh +179 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_units.py +41 -0
- dala-2.4.1/scripts/install-dala.sh +0 -57
- {dala-2.4.1 → dala-2.4.2}/.gitignore +0 -0
- {dala-2.4.1 → dala-2.4.2}/LICENSE +0 -0
- {dala-2.4.1 → dala-2.4.2}/PRIVACY.md +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/__init__.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/cli.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/__init__.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/browser.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/discovery.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/dispatcher.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/extractor.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/forum_image_processor.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/image_budget.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/image_processor.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/server_jobs.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/session.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/translation.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/core/writer.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/data/__init__.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/data/sites.yaml +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/drivers/__init__.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/drivers/base.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/drivers/forum.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/drivers/generic.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/drivers/hn.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/drivers/reddit.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/drivers/substack.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/drivers/wordpress.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/drivers/youtube.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/models.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/server.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/setup_browser.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/utils/__init__.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/utils/formatting.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/dala/utils/llm.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/installers/Install or Update Dala.bat +0 -0
- {dala-2.4.1 → dala-2.4.2}/installers/Install or Update Dala.command +0 -0
- {dala-2.4.1 → dala-2.4.2}/installers/Install or Update Dala.ps1 +0 -0
- {dala-2.4.1 → dala-2.4.2}/installers/Install or Update Dala.sh +0 -0
- {dala-2.4.1 → dala-2.4.2}/launchers/Start Dala Server.bat +0 -0
- {dala-2.4.1 → dala-2.4.2}/launchers/Start Dala Server.command +0 -0
- {dala-2.4.1 → dala-2.4.2}/launchers/dala-server.desktop +0 -0
- {dala-2.4.1 → dala-2.4.2}/main.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/scripts/install-dala.ps1 +0 -0
- {dala-2.4.1 → dala-2.4.2}/server.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/sites.yaml +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/helpers.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_browser.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_chrome_extension.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_discovery.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_drivers.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_hn_integration.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_image_budget.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_server.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_server_saving.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_setup_browser.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_translation.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/tests/test_writer.py +0 -0
- {dala-2.4.1 → dala-2.4.2}/web_to_epub.py +0 -0
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## v2.4.2
|
|
4
|
+
|
|
5
|
+
### Android / Termux install
|
|
6
|
+
- Added a Termux-specific path to the shell installer that avoids `uv tool install`/`uv pip`, uses Termux Python with `pip --user`, preinstalls native `python-lxml` and `python-pillow`, and applies constraints for the compatible server stack.
|
|
7
|
+
- Re-running the Termux installer now skips package and pip work when Dala is already installed; use `--upgrade` for updates.
|
|
8
|
+
- Updated Termux:Widget start/stop/status scripts to manage the installed `dala-server` command instead of an old source checkout, and included the Android helpers in the installer release bundle.
|
|
9
|
+
|
|
10
|
+
### Site profile config
|
|
11
|
+
- Packaged site profiles now load from JSON so the base install does not require PyYAML.
|
|
12
|
+
- Custom `sites.yaml` remains supported when `dala[config]` is installed; custom `sites.json` is supported without extra dependencies.
|
|
13
|
+
|
|
3
14
|
## v2.3.6
|
|
4
15
|
- **Package:** Include missing Firefox extension XPI (v2.3.5) in the release.
|
|
5
16
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dala
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.2
|
|
4
4
|
Summary: Web to ebook downloader
|
|
5
5
|
Project-URL: Homepage, https://github.com/anaxonda/dala
|
|
6
6
|
Project-URL: Repository, https://github.com/anaxonda/dala
|
|
@@ -27,13 +27,12 @@ Requires-Dist: aiohttp[speedups]>=3.9.0
|
|
|
27
27
|
Requires-Dist: beautifulsoup4>=4.11.0
|
|
28
28
|
Requires-Dist: deep-translator>=1.11.4
|
|
29
29
|
Requires-Dist: ebooklib>=0.18
|
|
30
|
-
Requires-Dist: fastapi>=0.
|
|
30
|
+
Requires-Dist: fastapi>=0.99.0
|
|
31
31
|
Requires-Dist: lxml[html-clean]>=4.9.0
|
|
32
32
|
Requires-Dist: pillow>=9.0.0
|
|
33
33
|
Requires-Dist: pygments>=2.14.0
|
|
34
34
|
Requires-Dist: pypdf>=5.9.0
|
|
35
35
|
Requires-Dist: python-dotenv>=1.0.0
|
|
36
|
-
Requires-Dist: pyyaml>=6.0.3
|
|
37
36
|
Requires-Dist: requests>=2.30.0
|
|
38
37
|
Requires-Dist: tqdm>=4.65.0
|
|
39
38
|
Requires-Dist: trafilatura>=1.6.0
|
|
@@ -42,6 +41,8 @@ Requires-Dist: youtube-comment-downloader>=0.1.0
|
|
|
42
41
|
Requires-Dist: youtube-transcript-api>=0.6.0
|
|
43
42
|
Provides-Extra: browser
|
|
44
43
|
Requires-Dist: playwright>=1.40.0; extra == 'browser'
|
|
44
|
+
Provides-Extra: config
|
|
45
|
+
Requires-Dist: pyyaml>=6.0.3; extra == 'config'
|
|
45
46
|
Description-Content-Type: text/markdown
|
|
46
47
|
|
|
47
48
|
# dala: Web pages and threads as books
|
|
@@ -472,6 +473,7 @@ The GitHub release asset `dala-installers-vVERSION.zip` contains:
|
|
|
472
473
|
- `installers/`: double-click or terminal installers for desktop users
|
|
473
474
|
- `scripts/`: lower-level install/update scripts
|
|
474
475
|
- `launchers/`: templates that start the installed `dala-server`
|
|
476
|
+
- `android/`: Termux notes and Termux:Widget start/stop/status shortcuts
|
|
475
477
|
|
|
476
478
|
The installers install or update the Python server from PyPI, install `uv` only if it is missing, optionally add headless browser support, and do not install the browser extension.
|
|
477
479
|
|
|
@@ -633,33 +635,56 @@ systemctl --user enable --now epub_server
|
|
|
633
635
|
|
|
634
636
|
### Android / Termux
|
|
635
637
|
|
|
636
|
-
Termux does not use the desktop installer bundle.
|
|
638
|
+
Termux does not use the desktop installer bundle. Use the shell installer; it detects Android/Termux and uses Termux Python plus `pip --user` instead of `uv tool install`:
|
|
637
639
|
|
|
638
640
|
```bash
|
|
639
641
|
pkg update
|
|
640
|
-
pkg install
|
|
641
|
-
|
|
642
|
-
|
|
642
|
+
pkg install python python-pip curl
|
|
643
|
+
curl -fsSLo install-dala.sh https://raw.githubusercontent.com/anaxonda/dala/main/scripts/install-dala.sh
|
|
644
|
+
sh install-dala.sh
|
|
643
645
|
dala-server --no-open
|
|
644
646
|
```
|
|
645
647
|
|
|
646
648
|
Open `http://127.0.0.1:8000/` manually, or point Firefox for Android plus the Dala extension at the local Termux server.
|
|
647
649
|
|
|
648
|
-
|
|
650
|
+
Optional Termux:Widget shortcuts are in `android/`; copy them into `~/.shortcuts/dala/` after installing Dala if you want home-screen start/stop/status buttons.
|
|
651
|
+
|
|
652
|
+
Why not `uv tool install` on Termux? Termux can package `uv`, but current `uv` releases can fail while inspecting Android Python with `Unknown operating system: android`. `uv pip` has the same limitation on affected devices. The installer avoids that path.
|
|
653
|
+
|
|
654
|
+
The installer also installs Termux `python-lxml` and `python-pillow` packages and uses a small constraints file for the server stack. Without that, PyPI may try to build heavy native dependencies such as `lxml`, `Pillow`, or `pydantic-core` from source on Android.
|
|
655
|
+
|
|
656
|
+
Re-running the installer is fast when Dala is already installed. To update Dala, run:
|
|
657
|
+
|
|
658
|
+
```bash
|
|
659
|
+
sh install-dala.sh --upgrade
|
|
660
|
+
```
|
|
661
|
+
|
|
662
|
+
If the installer reports broken `pip`, repair Termux Python packages and rerun it:
|
|
663
|
+
|
|
664
|
+
```bash
|
|
665
|
+
pkg update
|
|
666
|
+
pkg install python python-pip
|
|
667
|
+
pkg reinstall python-pip
|
|
668
|
+
sh install-dala.sh
|
|
669
|
+
```
|
|
670
|
+
|
|
671
|
+
If you want a source checkout on Android, install with `pip --user` from the checkout:
|
|
649
672
|
|
|
650
673
|
```bash
|
|
651
674
|
pkg install git python
|
|
652
675
|
git clone https://github.com/anaxonda/dala.git
|
|
653
676
|
cd dala
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
UV_LINK_MODE=copy UV_CACHE_DIR=$HOME/.cache/uv uv pip install -e .
|
|
657
|
-
uv run dala-server --no-open
|
|
677
|
+
pip install --user -e .
|
|
678
|
+
python -m dala.server --no-open
|
|
658
679
|
```
|
|
659
680
|
|
|
660
681
|
## sites.yaml Customization
|
|
661
682
|
|
|
662
|
-
You can define custom extraction rules for specific websites in `sites.yaml` at the project root.
|
|
683
|
+
You can define custom extraction rules for specific websites in `sites.yaml` at the project root. YAML config support needs PyYAML; install `dala[config]` if you use custom YAML profiles. Packaged defaults do not need PyYAML, and JSON config files can be loaded without extra dependencies.
|
|
684
|
+
|
|
685
|
+
```bash
|
|
686
|
+
uv tool install --force "dala[config]"
|
|
687
|
+
```
|
|
663
688
|
|
|
664
689
|
```yaml
|
|
665
690
|
- name: "The New York Times"
|
|
@@ -714,14 +739,16 @@ uv run dala --browser --headed "https://example.com/article"
|
|
|
714
739
|
|
|
715
740
|
If an interactive bot challenge appears, solve it in your normal browser and run the extension from the readable tab.
|
|
716
741
|
|
|
717
|
-
### Android / Termux
|
|
742
|
+
### Android / Termux install fails
|
|
718
743
|
|
|
719
744
|
```bash
|
|
720
|
-
pkg
|
|
721
|
-
pkg install
|
|
745
|
+
pkg update
|
|
746
|
+
pkg install python python-pip curl
|
|
747
|
+
pkg reinstall python-pip
|
|
748
|
+
sh install-dala.sh
|
|
722
749
|
```
|
|
723
750
|
|
|
724
|
-
|
|
751
|
+
Avoid `uv tool install` and `uv pip` on Termux if they fail with `Unknown operating system: android`; use the installer path from [Android / Termux](#android--termux).
|
|
725
752
|
|
|
726
753
|
### Translation fails
|
|
727
754
|
|
|
@@ -765,6 +792,8 @@ dala-firefox-vEXTENSION_VERSION-signed.xpi
|
|
|
765
792
|
dala-installers-vPYTHON_PACKAGE_VERSION.zip
|
|
766
793
|
```
|
|
767
794
|
|
|
795
|
+
If only the Python package/installers changed, extension assets may keep the previous extension version. State that explicitly in the release notes.
|
|
796
|
+
|
|
768
797
|
Build the unsigned extension packages and installer bundle with:
|
|
769
798
|
|
|
770
799
|
```bash
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import os
|
|
2
|
-
import
|
|
2
|
+
import json
|
|
3
3
|
import re
|
|
4
4
|
from importlib import resources
|
|
5
|
+
from pathlib import Path
|
|
5
6
|
from typing import List, Optional
|
|
6
7
|
from ..models import log, SiteProfile
|
|
7
8
|
|
|
@@ -17,33 +18,53 @@ class ProfileManager:
|
|
|
17
18
|
if not cls._instance:
|
|
18
19
|
cls._instance = cls()
|
|
19
20
|
cls._instance.load_packaged_defaults()
|
|
20
|
-
for path in [
|
|
21
|
+
for path in [
|
|
22
|
+
"sites.json",
|
|
23
|
+
"sites.yaml",
|
|
24
|
+
os.path.expanduser("~/.config/dala/sites.json"),
|
|
25
|
+
os.path.expanduser("~/.config/dala/sites.yaml"),
|
|
26
|
+
]:
|
|
21
27
|
cls._instance.load_config(path)
|
|
22
28
|
return cls._instance
|
|
23
29
|
|
|
24
30
|
def load_packaged_defaults(self):
|
|
25
31
|
try:
|
|
26
|
-
default_config = resources.files("dala.data").joinpath("sites.
|
|
32
|
+
default_config = resources.files("dala.data").joinpath("sites.json")
|
|
27
33
|
with resources.as_file(default_config) as path:
|
|
28
34
|
self.load_config(str(path))
|
|
29
35
|
except Exception as e:
|
|
30
36
|
log.warning(f"Failed to load packaged site profiles: {e}")
|
|
37
|
+
|
|
38
|
+
def _load_config_data(self, path: str):
|
|
39
|
+
suffix = Path(path).suffix.lower()
|
|
40
|
+
with open(path, 'r') as f:
|
|
41
|
+
if suffix == ".json":
|
|
42
|
+
return json.load(f)
|
|
43
|
+
|
|
44
|
+
try:
|
|
45
|
+
import yaml
|
|
46
|
+
except ImportError:
|
|
47
|
+
log.warning(
|
|
48
|
+
f"Skipping YAML config {path}: install PyYAML or use JSON config."
|
|
49
|
+
)
|
|
50
|
+
return None
|
|
51
|
+
return yaml.safe_load(f)
|
|
52
|
+
|
|
31
53
|
def load_config(self, path: str):
|
|
32
54
|
if not os.path.exists(path): return
|
|
33
55
|
try:
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
))
|
|
56
|
+
data = self._load_config_data(path)
|
|
57
|
+
if not data or not isinstance(data, list): return
|
|
58
|
+
for item in data:
|
|
59
|
+
self.profiles.append(SiteProfile(
|
|
60
|
+
name=item.get("name", "Unknown"),
|
|
61
|
+
domain_patterns=item.get("domains", []),
|
|
62
|
+
driver_alias=item.get("driver"),
|
|
63
|
+
content_selector=item.get("content_selector"),
|
|
64
|
+
remove_selectors=item.get("remove", []),
|
|
65
|
+
headers=item.get("headers", {}),
|
|
66
|
+
image_proxy_pattern=item.get("image_proxy_pattern")
|
|
67
|
+
))
|
|
47
68
|
log.info(f"Loaded {len(data)} profiles from {path}")
|
|
48
69
|
except Exception as e:
|
|
49
70
|
log.warning(f"Failed to load config {path}: {e}")
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"name": "Washington Post",
|
|
4
|
+
"domains": [
|
|
5
|
+
"washingtonpost.com"
|
|
6
|
+
],
|
|
7
|
+
"image_proxy_pattern": "imrs.php",
|
|
8
|
+
"remove": [
|
|
9
|
+
"[data-qa='collections-wrapper']",
|
|
10
|
+
"[data-qa='author-bio']",
|
|
11
|
+
"[data-qa='more-from-the-post']",
|
|
12
|
+
"ul[data-testid='collections-list']",
|
|
13
|
+
"footer",
|
|
14
|
+
"aside",
|
|
15
|
+
"nav",
|
|
16
|
+
".related-content",
|
|
17
|
+
".hide-for-print"
|
|
18
|
+
]
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"name": "The New York Times",
|
|
22
|
+
"domains": [
|
|
23
|
+
"nytimes.com"
|
|
24
|
+
],
|
|
25
|
+
"content_selector": "article#story",
|
|
26
|
+
"remove": [
|
|
27
|
+
"#top-wrapper",
|
|
28
|
+
"#sponsor-wrapper",
|
|
29
|
+
"#bottom-wrapper",
|
|
30
|
+
"[data-testid='recirculation']",
|
|
31
|
+
"#bottom-sheet-sensor",
|
|
32
|
+
"a[href^='#after-']",
|
|
33
|
+
"nav",
|
|
34
|
+
"header > div > ul"
|
|
35
|
+
]
|
|
36
|
+
}
|
|
37
|
+
]
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
These files install or update the Dala Python server from PyPI. They install `uv` only if it is missing, ask whether to add optional headless browser/PDF support, and do not install the browser extension.
|
|
4
4
|
|
|
5
|
+
These wrappers are for Windows, macOS, and desktop Linux. On Android/Termux, use `scripts/install-dala.sh`; it uses Termux Python plus `pip --user` instead of `uv tool install`. The release bundle also includes `android/` Termux:Widget shortcuts for starting, stopping, and checking the installed server.
|
|
6
|
+
|
|
5
7
|
After installation, start Dala with the Desktop launcher if one was created, or run:
|
|
6
8
|
|
|
7
9
|
```bash
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "dala"
|
|
7
|
-
version = "2.4.
|
|
7
|
+
version = "2.4.2"
|
|
8
8
|
description = "Web to ebook downloader"
|
|
9
9
|
readme = "readme.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -38,9 +38,8 @@ dependencies = [
|
|
|
38
38
|
"pygments>=2.14.0",
|
|
39
39
|
"tqdm>=4.65.0",
|
|
40
40
|
"Pillow>=9.0.0",
|
|
41
|
-
"fastapi>=0.
|
|
41
|
+
"fastapi>=0.99.0",
|
|
42
42
|
"uvicorn>=0.22.0",
|
|
43
|
-
"pyyaml>=6.0.3",
|
|
44
43
|
"youtube-transcript-api>=0.6.0",
|
|
45
44
|
"python-dotenv>=1.0.0",
|
|
46
45
|
"youtube-comment-downloader>=0.1.0",
|
|
@@ -52,6 +51,9 @@ dependencies = [
|
|
|
52
51
|
browser = [
|
|
53
52
|
"playwright>=1.40.0",
|
|
54
53
|
]
|
|
54
|
+
config = [
|
|
55
|
+
"pyyaml>=6.0.3",
|
|
56
|
+
]
|
|
55
57
|
|
|
56
58
|
[project.urls]
|
|
57
59
|
Homepage = "https://github.com/anaxonda/dala"
|
|
@@ -426,6 +426,7 @@ The GitHub release asset `dala-installers-vVERSION.zip` contains:
|
|
|
426
426
|
- `installers/`: double-click or terminal installers for desktop users
|
|
427
427
|
- `scripts/`: lower-level install/update scripts
|
|
428
428
|
- `launchers/`: templates that start the installed `dala-server`
|
|
429
|
+
- `android/`: Termux notes and Termux:Widget start/stop/status shortcuts
|
|
429
430
|
|
|
430
431
|
The installers install or update the Python server from PyPI, install `uv` only if it is missing, optionally add headless browser support, and do not install the browser extension.
|
|
431
432
|
|
|
@@ -587,33 +588,56 @@ systemctl --user enable --now epub_server
|
|
|
587
588
|
|
|
588
589
|
### Android / Termux
|
|
589
590
|
|
|
590
|
-
Termux does not use the desktop installer bundle.
|
|
591
|
+
Termux does not use the desktop installer bundle. Use the shell installer; it detects Android/Termux and uses Termux Python plus `pip --user` instead of `uv tool install`:
|
|
591
592
|
|
|
592
593
|
```bash
|
|
593
594
|
pkg update
|
|
594
|
-
pkg install
|
|
595
|
-
|
|
596
|
-
|
|
595
|
+
pkg install python python-pip curl
|
|
596
|
+
curl -fsSLo install-dala.sh https://raw.githubusercontent.com/anaxonda/dala/main/scripts/install-dala.sh
|
|
597
|
+
sh install-dala.sh
|
|
597
598
|
dala-server --no-open
|
|
598
599
|
```
|
|
599
600
|
|
|
600
601
|
Open `http://127.0.0.1:8000/` manually, or point Firefox for Android plus the Dala extension at the local Termux server.
|
|
601
602
|
|
|
602
|
-
|
|
603
|
+
Optional Termux:Widget shortcuts are in `android/`; copy them into `~/.shortcuts/dala/` after installing Dala if you want home-screen start/stop/status buttons.
|
|
604
|
+
|
|
605
|
+
Why not `uv tool install` on Termux? Termux can package `uv`, but current `uv` releases can fail while inspecting Android Python with `Unknown operating system: android`. `uv pip` has the same limitation on affected devices. The installer avoids that path.
|
|
606
|
+
|
|
607
|
+
The installer also installs Termux `python-lxml` and `python-pillow` packages and uses a small constraints file for the server stack. Without that, PyPI may try to build heavy native dependencies such as `lxml`, `Pillow`, or `pydantic-core` from source on Android.
|
|
608
|
+
|
|
609
|
+
Re-running the installer is fast when Dala is already installed. To update Dala, run:
|
|
610
|
+
|
|
611
|
+
```bash
|
|
612
|
+
sh install-dala.sh --upgrade
|
|
613
|
+
```
|
|
614
|
+
|
|
615
|
+
If the installer reports broken `pip`, repair Termux Python packages and rerun it:
|
|
616
|
+
|
|
617
|
+
```bash
|
|
618
|
+
pkg update
|
|
619
|
+
pkg install python python-pip
|
|
620
|
+
pkg reinstall python-pip
|
|
621
|
+
sh install-dala.sh
|
|
622
|
+
```
|
|
623
|
+
|
|
624
|
+
If you want a source checkout on Android, install with `pip --user` from the checkout:
|
|
603
625
|
|
|
604
626
|
```bash
|
|
605
627
|
pkg install git python
|
|
606
628
|
git clone https://github.com/anaxonda/dala.git
|
|
607
629
|
cd dala
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
UV_LINK_MODE=copy UV_CACHE_DIR=$HOME/.cache/uv uv pip install -e .
|
|
611
|
-
uv run dala-server --no-open
|
|
630
|
+
pip install --user -e .
|
|
631
|
+
python -m dala.server --no-open
|
|
612
632
|
```
|
|
613
633
|
|
|
614
634
|
## sites.yaml Customization
|
|
615
635
|
|
|
616
|
-
You can define custom extraction rules for specific websites in `sites.yaml` at the project root.
|
|
636
|
+
You can define custom extraction rules for specific websites in `sites.yaml` at the project root. YAML config support needs PyYAML; install `dala[config]` if you use custom YAML profiles. Packaged defaults do not need PyYAML, and JSON config files can be loaded without extra dependencies.
|
|
637
|
+
|
|
638
|
+
```bash
|
|
639
|
+
uv tool install --force "dala[config]"
|
|
640
|
+
```
|
|
617
641
|
|
|
618
642
|
```yaml
|
|
619
643
|
- name: "The New York Times"
|
|
@@ -668,14 +692,16 @@ uv run dala --browser --headed "https://example.com/article"
|
|
|
668
692
|
|
|
669
693
|
If an interactive bot challenge appears, solve it in your normal browser and run the extension from the readable tab.
|
|
670
694
|
|
|
671
|
-
### Android / Termux
|
|
695
|
+
### Android / Termux install fails
|
|
672
696
|
|
|
673
697
|
```bash
|
|
674
|
-
pkg
|
|
675
|
-
pkg install
|
|
698
|
+
pkg update
|
|
699
|
+
pkg install python python-pip curl
|
|
700
|
+
pkg reinstall python-pip
|
|
701
|
+
sh install-dala.sh
|
|
676
702
|
```
|
|
677
703
|
|
|
678
|
-
|
|
704
|
+
Avoid `uv tool install` and `uv pip` on Termux if they fail with `Unknown operating system: android`; use the installer path from [Android / Termux](#android--termux).
|
|
679
705
|
|
|
680
706
|
### Translation fails
|
|
681
707
|
|
|
@@ -719,6 +745,8 @@ dala-firefox-vEXTENSION_VERSION-signed.xpi
|
|
|
719
745
|
dala-installers-vPYTHON_PACKAGE_VERSION.zip
|
|
720
746
|
```
|
|
721
747
|
|
|
748
|
+
If only the Python package/installers changed, extension assets may keep the previous extension version. State that explicitly in the release notes.
|
|
749
|
+
|
|
722
750
|
Build the unsigned extension packages and installer bundle with:
|
|
723
751
|
|
|
724
752
|
```bash
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
set -eu
|
|
3
|
+
|
|
4
|
+
INSTALL_BROWSER=0
|
|
5
|
+
UPGRADE_DALA=0
|
|
6
|
+
UPGRADE_UV=0
|
|
7
|
+
PACKAGE_SPEC="${DALA_PACKAGE_SPEC:-dala}"
|
|
8
|
+
DALA_USER_BIN="${DALA_USER_BIN:-$HOME/.local/bin}"
|
|
9
|
+
DALA_TERMUX_PIP_CONSTRAINTS="${DALA_TERMUX_PIP_CONSTRAINTS:-fastapi<0.100 pydantic<2}"
|
|
10
|
+
|
|
11
|
+
for arg in "$@"; do
|
|
12
|
+
case "$arg" in
|
|
13
|
+
--headless-browser|--browser) INSTALL_BROWSER=1 ;;
|
|
14
|
+
--upgrade) UPGRADE_DALA=1 ;;
|
|
15
|
+
--upgrade-uv) UPGRADE_UV=1 ;;
|
|
16
|
+
*)
|
|
17
|
+
echo "Unknown option: $arg" >&2
|
|
18
|
+
echo "Usage: $0 [--headless-browser] [--upgrade] [--upgrade-uv]" >&2
|
|
19
|
+
exit 2
|
|
20
|
+
;;
|
|
21
|
+
esac
|
|
22
|
+
done
|
|
23
|
+
|
|
24
|
+
is_termux() {
|
|
25
|
+
[ -n "${TERMUX_VERSION:-}" ] || [ -d "/data/data/com.termux/files/usr" ]
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
write_termux_wrapper() {
|
|
29
|
+
name="$1"
|
|
30
|
+
module="$2"
|
|
31
|
+
mkdir -p "$DALA_USER_BIN"
|
|
32
|
+
{
|
|
33
|
+
echo "#!/bin/sh"
|
|
34
|
+
echo "exec python -m $module \"\$@\""
|
|
35
|
+
} > "$DALA_USER_BIN/$name"
|
|
36
|
+
chmod +x "$DALA_USER_BIN/$name"
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
termux_dala_installed() {
|
|
40
|
+
python -c 'import importlib.util; raise SystemExit(0 if importlib.util.find_spec("dala") else 1)' >/dev/null 2>&1
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
write_termux_constraints() {
|
|
44
|
+
constraints_dir="${TMPDIR:-$HOME/.cache/dala}"
|
|
45
|
+
mkdir -p "$constraints_dir"
|
|
46
|
+
constraints_file="$constraints_dir/termux-constraints.txt"
|
|
47
|
+
cat > "$constraints_file" <<'EOF'
|
|
48
|
+
aiohttp==3.14.1
|
|
49
|
+
anyio==4.14.1
|
|
50
|
+
dateparser==1.4.1
|
|
51
|
+
fastapi==0.99.1
|
|
52
|
+
htmldate==1.10.0
|
|
53
|
+
pydantic==1.10.26
|
|
54
|
+
regex==2026.5.9
|
|
55
|
+
starlette==0.27.0
|
|
56
|
+
trafilatura==2.1.0
|
|
57
|
+
uvicorn==0.49.0
|
|
58
|
+
EOF
|
|
59
|
+
echo "$constraints_file"
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
ensure_termux_pip() {
|
|
63
|
+
if command -v pip >/dev/null 2>&1 && pip --version >/dev/null 2>&1; then
|
|
64
|
+
return 0
|
|
65
|
+
fi
|
|
66
|
+
|
|
67
|
+
if command -v pkg >/dev/null 2>&1; then
|
|
68
|
+
echo "Installing or repairing Termux python-pip..."
|
|
69
|
+
pkg install -y python python-pip
|
|
70
|
+
fi
|
|
71
|
+
|
|
72
|
+
if command -v pip >/dev/null 2>&1 && pip --version >/dev/null 2>&1; then
|
|
73
|
+
return 0
|
|
74
|
+
fi
|
|
75
|
+
|
|
76
|
+
if command -v pkg >/dev/null 2>&1; then
|
|
77
|
+
echo "python-pip is present but not importable; reinstalling it..."
|
|
78
|
+
pkg reinstall -y python-pip
|
|
79
|
+
fi
|
|
80
|
+
|
|
81
|
+
if ! command -v pip >/dev/null 2>&1 || ! pip --version >/dev/null 2>&1; then
|
|
82
|
+
echo "Could not find a working Termux pip." >&2
|
|
83
|
+
echo "Run: pkg update && pkg install -y python python-pip" >&2
|
|
84
|
+
echo "If pip still fails, run: pkg reinstall -y python-pip" >&2
|
|
85
|
+
exit 1
|
|
86
|
+
fi
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
install_termux() {
|
|
90
|
+
if [ "$INSTALL_BROWSER" -eq 1 ]; then
|
|
91
|
+
echo "Headless browser/PDF support is not installed by the Termux installer."
|
|
92
|
+
echo "The base EPUB server will be installed. Use extension capture for browser-rendered pages."
|
|
93
|
+
fi
|
|
94
|
+
|
|
95
|
+
if ! command -v python >/dev/null 2>&1; then
|
|
96
|
+
if command -v pkg >/dev/null 2>&1; then
|
|
97
|
+
pkg install -y python
|
|
98
|
+
else
|
|
99
|
+
echo "Python is required. Install Python, then rerun this script." >&2
|
|
100
|
+
exit 1
|
|
101
|
+
fi
|
|
102
|
+
fi
|
|
103
|
+
|
|
104
|
+
if [ "$UPGRADE_DALA" -eq 0 ] && termux_dala_installed; then
|
|
105
|
+
write_termux_wrapper "dala" "dala.cli"
|
|
106
|
+
write_termux_wrapper "dala-server" "dala.server"
|
|
107
|
+
echo "Dala is already installed for Termux."
|
|
108
|
+
echo "Run '$0 --upgrade' to update it."
|
|
109
|
+
echo
|
|
110
|
+
echo "Start the server with:"
|
|
111
|
+
echo " dala-server --no-open"
|
|
112
|
+
return 0
|
|
113
|
+
fi
|
|
114
|
+
|
|
115
|
+
ensure_termux_pip
|
|
116
|
+
|
|
117
|
+
if command -v pkg >/dev/null 2>&1; then
|
|
118
|
+
echo "Installing Termux native packages for image and HTML dependencies..."
|
|
119
|
+
pkg install -y python-lxml python-pillow
|
|
120
|
+
fi
|
|
121
|
+
|
|
122
|
+
echo "Installing Dala for Termux with pip --user..."
|
|
123
|
+
termux_constraints_file="$(write_termux_constraints)"
|
|
124
|
+
# FastAPI 0.100+ pulls Pydantic 2/pydantic-core, which currently falls back
|
|
125
|
+
# to a Rust source build on Android. FastAPI 0.99 + Pydantic 1 are pure
|
|
126
|
+
# Python wheels and are sufficient for Dala's server API usage.
|
|
127
|
+
pip install --user --upgrade -c "$termux_constraints_file" $DALA_TERMUX_PIP_CONSTRAINTS "$PACKAGE_SPEC"
|
|
128
|
+
|
|
129
|
+
write_termux_wrapper "dala" "dala.cli"
|
|
130
|
+
write_termux_wrapper "dala-server" "dala.server"
|
|
131
|
+
|
|
132
|
+
echo
|
|
133
|
+
echo "Dala installed for Termux. Start the server with:"
|
|
134
|
+
echo " dala-server --no-open"
|
|
135
|
+
echo "If dala-server is not found, add this to your shell profile:"
|
|
136
|
+
echo " export PATH=\"\$HOME/.local/bin:\$PATH\""
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if is_termux; then
|
|
140
|
+
install_termux
|
|
141
|
+
exit 0
|
|
142
|
+
fi
|
|
143
|
+
|
|
144
|
+
if command -v uv >/dev/null 2>&1; then
|
|
145
|
+
echo "Found uv: $(uv --version)"
|
|
146
|
+
if [ "$UPGRADE_UV" -eq 1 ]; then
|
|
147
|
+
uv self update || echo "uv self update is not available for this install; continuing."
|
|
148
|
+
fi
|
|
149
|
+
else
|
|
150
|
+
echo "uv not found; installing uv..."
|
|
151
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
152
|
+
export PATH="$HOME/.local/bin:$PATH"
|
|
153
|
+
fi
|
|
154
|
+
|
|
155
|
+
INSTALL_ARGS=""
|
|
156
|
+
if [ "$UPGRADE_DALA" -eq 1 ] || [ "$INSTALL_BROWSER" -eq 1 ]; then
|
|
157
|
+
INSTALL_ARGS="--force"
|
|
158
|
+
fi
|
|
159
|
+
|
|
160
|
+
if [ "$INSTALL_BROWSER" -eq 1 ]; then
|
|
161
|
+
echo "Installing Dala with optional headless browser support..."
|
|
162
|
+
echo "This lets the Dala server control Chrome/Chromium in the background."
|
|
163
|
+
echo "It is needed for PDF output and some JavaScript-heavy pages."
|
|
164
|
+
echo "It is separate from the normal Dala browser extension."
|
|
165
|
+
uv tool install $INSTALL_ARGS --with playwright "$PACKAGE_SPEC"
|
|
166
|
+
if command -v dala-setup-browser >/dev/null 2>&1; then
|
|
167
|
+
dala-setup-browser
|
|
168
|
+
else
|
|
169
|
+
uv tool run --with playwright --from "$PACKAGE_SPEC" dala-setup-browser
|
|
170
|
+
fi
|
|
171
|
+
else
|
|
172
|
+
echo "Installing Dala server..."
|
|
173
|
+
uv tool install $INSTALL_ARGS "$PACKAGE_SPEC"
|
|
174
|
+
fi
|
|
175
|
+
|
|
176
|
+
echo
|
|
177
|
+
echo "Dala installed. Start the server with:"
|
|
178
|
+
echo " dala-server"
|
|
179
|
+
echo "If dala-server is not found, restart your terminal or run: uv tool update-shell"
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import builtins
|
|
1
2
|
import pytest
|
|
2
3
|
from bs4 import BeautifulSoup
|
|
3
4
|
import dala.cli as main
|
|
@@ -34,6 +35,46 @@ def test_profile_manager_loads_packaged_defaults(monkeypatch):
|
|
|
34
35
|
assert profile.content_selector == "article#story"
|
|
35
36
|
|
|
36
37
|
|
|
38
|
+
def test_profile_manager_loads_json_config(tmp_path):
|
|
39
|
+
config = tmp_path / "sites.json"
|
|
40
|
+
config.write_text(
|
|
41
|
+
"""
|
|
42
|
+
[
|
|
43
|
+
{
|
|
44
|
+
"name": "Example",
|
|
45
|
+
"domains": ["example\\\\.com"],
|
|
46
|
+
"content_selector": "main"
|
|
47
|
+
}
|
|
48
|
+
]
|
|
49
|
+
""".strip()
|
|
50
|
+
)
|
|
51
|
+
manager = ProfileManager()
|
|
52
|
+
manager.load_config(str(config))
|
|
53
|
+
|
|
54
|
+
profile = manager.get_profile("https://example.com/article")
|
|
55
|
+
|
|
56
|
+
assert profile is not None
|
|
57
|
+
assert profile.name == "Example"
|
|
58
|
+
assert profile.content_selector == "main"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_profile_manager_skips_yaml_without_pyyaml(tmp_path, monkeypatch):
|
|
62
|
+
config = tmp_path / "sites.yaml"
|
|
63
|
+
config.write_text('- name: "Example"\n domains:\n - "example.com"\n')
|
|
64
|
+
real_import = builtins.__import__
|
|
65
|
+
|
|
66
|
+
def block_yaml_import(name, *args, **kwargs):
|
|
67
|
+
if name == "yaml":
|
|
68
|
+
raise ImportError("blocked for test")
|
|
69
|
+
return real_import(name, *args, **kwargs)
|
|
70
|
+
|
|
71
|
+
monkeypatch.setattr(builtins, "__import__", block_yaml_import)
|
|
72
|
+
manager = ProfileManager()
|
|
73
|
+
manager.load_config(str(config))
|
|
74
|
+
|
|
75
|
+
assert manager.profiles == []
|
|
76
|
+
|
|
77
|
+
|
|
37
78
|
def test_driver_dispatch_explicit_forum():
|
|
38
79
|
src = Source(url="http://example.com", is_forum=True)
|
|
39
80
|
driver = DriverDispatcher.get_driver(src)
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
#!/bin/sh
|
|
2
|
-
set -eu
|
|
3
|
-
|
|
4
|
-
INSTALL_BROWSER=0
|
|
5
|
-
UPGRADE_DALA=0
|
|
6
|
-
UPGRADE_UV=0
|
|
7
|
-
PACKAGE_SPEC="${DALA_PACKAGE_SPEC:-dala}"
|
|
8
|
-
|
|
9
|
-
for arg in "$@"; do
|
|
10
|
-
case "$arg" in
|
|
11
|
-
--headless-browser|--browser) INSTALL_BROWSER=1 ;;
|
|
12
|
-
--upgrade) UPGRADE_DALA=1 ;;
|
|
13
|
-
--upgrade-uv) UPGRADE_UV=1 ;;
|
|
14
|
-
*)
|
|
15
|
-
echo "Unknown option: $arg" >&2
|
|
16
|
-
echo "Usage: $0 [--headless-browser] [--upgrade] [--upgrade-uv]" >&2
|
|
17
|
-
exit 2
|
|
18
|
-
;;
|
|
19
|
-
esac
|
|
20
|
-
done
|
|
21
|
-
|
|
22
|
-
if command -v uv >/dev/null 2>&1; then
|
|
23
|
-
echo "Found uv: $(uv --version)"
|
|
24
|
-
if [ "$UPGRADE_UV" -eq 1 ]; then
|
|
25
|
-
uv self update || echo "uv self update is not available for this install; continuing."
|
|
26
|
-
fi
|
|
27
|
-
else
|
|
28
|
-
echo "uv not found; installing uv..."
|
|
29
|
-
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
30
|
-
export PATH="$HOME/.local/bin:$PATH"
|
|
31
|
-
fi
|
|
32
|
-
|
|
33
|
-
INSTALL_ARGS=""
|
|
34
|
-
if [ "$UPGRADE_DALA" -eq 1 ] || [ "$INSTALL_BROWSER" -eq 1 ]; then
|
|
35
|
-
INSTALL_ARGS="--force"
|
|
36
|
-
fi
|
|
37
|
-
|
|
38
|
-
if [ "$INSTALL_BROWSER" -eq 1 ]; then
|
|
39
|
-
echo "Installing Dala with optional headless browser support..."
|
|
40
|
-
echo "This lets the Dala server control Chrome/Chromium in the background."
|
|
41
|
-
echo "It is needed for PDF output and some JavaScript-heavy pages."
|
|
42
|
-
echo "It is separate from the normal Dala browser extension."
|
|
43
|
-
uv tool install $INSTALL_ARGS --with playwright "$PACKAGE_SPEC"
|
|
44
|
-
if command -v dala-setup-browser >/dev/null 2>&1; then
|
|
45
|
-
dala-setup-browser
|
|
46
|
-
else
|
|
47
|
-
uv tool run --with playwright --from "$PACKAGE_SPEC" dala-setup-browser
|
|
48
|
-
fi
|
|
49
|
-
else
|
|
50
|
-
echo "Installing Dala server..."
|
|
51
|
-
uv tool install $INSTALL_ARGS "$PACKAGE_SPEC"
|
|
52
|
-
fi
|
|
53
|
-
|
|
54
|
-
echo
|
|
55
|
-
echo "Dala installed. Start the server with:"
|
|
56
|
-
echo " dala-server"
|
|
57
|
-
echo "If dala-server is not found, restart your terminal or run: uv tool update-shell"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|