visync 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of visync might be problematic. Click here for more details.
- visync-1.0.0/.gitignore +36 -0
- visync-1.0.0/LICENSE +21 -0
- visync-1.0.0/PKG-INFO +204 -0
- visync-1.0.0/README.md +166 -0
- visync-1.0.0/config.toml +259 -0
- visync-1.0.0/pyproject.toml +193 -0
- visync-1.0.0/test/ci_no_ventoy_drive.py +66 -0
- visync-1.0.0/test/conftest.py +3 -0
- visync-1.0.0/test/test_cli.py +1094 -0
- visync-1.0.0/test/test_download.py +1122 -0
- visync-1.0.0/test/test_finder.py +221 -0
- visync-1.0.0/test/test_identify.py +84 -0
- visync-1.0.0/test/test_metadata.py +572 -0
- visync-1.0.0/test/test_pm.py +582 -0
- visync-1.0.0/test/test_security_regressions.py +2434 -0
- visync-1.0.0/test/test_verify.py +476 -0
- visync-1.0.0/test/test_verify_cli.py +228 -0
- visync-1.0.0/uv.lock +353 -0
- visync-1.0.0/visync/__init__.py +3 -0
- visync-1.0.0/visync/download.py +1505 -0
- visync-1.0.0/visync/finder.py +699 -0
- visync-1.0.0/visync/main.py +1172 -0
- visync-1.0.0/visync/net.py +84 -0
- visync-1.0.0/visync/output.py +126 -0
- visync-1.0.0/visync/pm.py +133 -0
- visync-1.0.0/visync/verify.py +703 -0
visync-1.0.0/.gitignore
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.pyc
|
|
4
|
+
*.pyo
|
|
5
|
+
*.pyd
|
|
6
|
+
.Python
|
|
7
|
+
env/
|
|
8
|
+
venv/
|
|
9
|
+
ENV/
|
|
10
|
+
env.bak/
|
|
11
|
+
venv.bak/
|
|
12
|
+
.idea/
|
|
13
|
+
.vscode/
|
|
14
|
+
*.sublime-project
|
|
15
|
+
*.sublime-workspace
|
|
16
|
+
*.log
|
|
17
|
+
*.sqlite3
|
|
18
|
+
*.db
|
|
19
|
+
*.bak
|
|
20
|
+
*.backup
|
|
21
|
+
*.swp
|
|
22
|
+
.DS_Store
|
|
23
|
+
*.egg-info/
|
|
24
|
+
dist/
|
|
25
|
+
build/
|
|
26
|
+
*.egg
|
|
27
|
+
*.mo
|
|
28
|
+
*.pot
|
|
29
|
+
*.coverage
|
|
30
|
+
.coverage.*
|
|
31
|
+
.cache
|
|
32
|
+
nosetests.xml
|
|
33
|
+
coverage.xml
|
|
34
|
+
|
|
35
|
+
# 255 KB audit dump accidentally committed; see git history before 8bb78aa
|
|
36
|
+
audit.txt
|
visync-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hxmbl
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
visync-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: visync
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Ventoy ISO Synchronization Tool
|
|
5
|
+
Project-URL: Homepage, https://github.com/hxmbl/visync
|
|
6
|
+
Project-URL: Repository, https://github.com/hxmbl/visync
|
|
7
|
+
Project-URL: Issues, https://github.com/hxmbl/visync/issues
|
|
8
|
+
Author: hxmbl
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: checksum,iso,linux,package-manager,usb,ventoy
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
15
|
+
Classifier: Intended Audience :: System Administrators
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: MacOS
|
|
18
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
19
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
20
|
+
Classifier: Programming Language :: Python :: 3
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
25
|
+
Classifier: Topic :: System :: Archiving :: Backup
|
|
26
|
+
Classifier: Topic :: System :: Filesystems
|
|
27
|
+
Classifier: Topic :: Utilities
|
|
28
|
+
Requires-Python: <3.16,>=3.12
|
|
29
|
+
Requires-Dist: rich>=13.0
|
|
30
|
+
Requires-Dist: typer>=0.12
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
33
|
+
Requires-Dist: pyright>=1.1.380; extra == 'dev'
|
|
34
|
+
Requires-Dist: pytest-cov>=5; extra == 'dev'
|
|
35
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
36
|
+
Requires-Dist: ruff>=0.16; extra == 'dev'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# Visync
|
|
40
|
+
|
|
41
|
+
Ventoy Package Manager. Install, update, and manage Linux distros on your Ventoy drive.
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
Requires Python 3.12+ and a mounted Ventoy drive.
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install visync
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
For a development checkout:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install -e ".[dev]"
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Both layouts work out of the box: `config.toml` ships inside the installed package, so `visync search` finds the distro list from any directory.
|
|
58
|
+
|
|
59
|
+
## Quick Start
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
# See what's available
|
|
63
|
+
visync search
|
|
64
|
+
|
|
65
|
+
# Auto-detect distros already on the drive
|
|
66
|
+
visync autodetect
|
|
67
|
+
|
|
68
|
+
# Install a distro
|
|
69
|
+
visync install archlinux
|
|
70
|
+
visync install nixos-graphical
|
|
71
|
+
|
|
72
|
+
# Batch install from file
|
|
73
|
+
visync install -i packages.txt
|
|
74
|
+
|
|
75
|
+
# Update all installed distros
|
|
76
|
+
visync update
|
|
77
|
+
|
|
78
|
+
# Update a specific distro
|
|
79
|
+
visync update tails
|
|
80
|
+
|
|
81
|
+
# Check for old versions (dry-run)
|
|
82
|
+
visync sync
|
|
83
|
+
|
|
84
|
+
# Remove old versions
|
|
85
|
+
visync sync --clean
|
|
86
|
+
|
|
87
|
+
# Remove a distro
|
|
88
|
+
visync remove tails
|
|
89
|
+
|
|
90
|
+
# Show distro details
|
|
91
|
+
visync info archlinux
|
|
92
|
+
|
|
93
|
+
# Verify checksums
|
|
94
|
+
visync verify
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Commands
|
|
98
|
+
|
|
99
|
+
| Command | Description |
|
|
100
|
+
|---|---|
|
|
101
|
+
| `visync search [query]` | List available distros (filter by query) |
|
|
102
|
+
| `visync install <name>` | Download and register a distro |
|
|
103
|
+
| `visync install -i <file>` | Batch install from file (one name per line) |
|
|
104
|
+
| `visync remove <name>` | Delete from drive and unregister |
|
|
105
|
+
| `visync update [name]` [--no-staging] | Update installed distros (all if no name given) |
|
|
106
|
+
| `visync sync` [--no-staging] | Sync installed distros to latest |
|
|
107
|
+
| `visync sync --all` [--no-staging] | Sync all configured distros |
|
|
108
|
+
| `visync sync --clean` [--no-staging] | Remove old versions of same distro |
|
|
109
|
+
| `visync sync --reset-visync` | Allow the watchdog to wipe an over-budget `.visync/` |
|
|
110
|
+
| `visync list` | List ISOs on the drive |
|
|
111
|
+
| `visync autodetect` | Register existing ISOs as installed |
|
|
112
|
+
| `visync verify` | Verify checksums against upstream |
|
|
113
|
+
| `visync info <name>` | Show distro details |
|
|
114
|
+
|
|
115
|
+
## Batch Install File
|
|
116
|
+
|
|
117
|
+
Create a text file with one distro name per line:
|
|
118
|
+
|
|
119
|
+
```
|
|
120
|
+
# My Ventoy setup
|
|
121
|
+
archlinux
|
|
122
|
+
nixos-minimal
|
|
123
|
+
tails
|
|
124
|
+
ubuntu-desktop
|
|
125
|
+
ubuntu-server
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Then run:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
visync install -i packages.txt
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Blank lines and lines starting with `#` are ignored.
|
|
135
|
+
|
|
136
|
+
## Configuration
|
|
137
|
+
|
|
138
|
+
Edit `config.toml` to add or remove distros. Each distro entry defines a scraping strategy, mirror URL, and checksum verification method.
|
|
139
|
+
|
|
140
|
+
Config resolution order: explicit `--config` path, then `$VISYNC_CONFIG`, then `~/.config/visync/config.toml`, then the `config.toml` shipped inside the installed package. A `config.toml` in the current directory is only used as a last resort, so planted configs cannot hijack identification.
|
|
141
|
+
|
|
142
|
+
**Release selection:** `version_filter` restricts a scraper to one release family before the newest match is chosen. Ubuntu entries use `\d*[02468]\.04(\.\d+)*` so they track the current LTS (26.04.1) and ignore interim releases (25.10, 26.10). NixOS resolves the newest stable `YY.MM` channel from the release bucket at scrape time, so it advances to the next stable without a config edit; set `channel = "26.05"` to pin deliberately while that channel is current.
|
|
143
|
+
|
|
144
|
+
**Built-in strategies:**
|
|
145
|
+
|
|
146
|
+
| Strategy | Description | Example |
|
|
147
|
+
|---|---|---|
|
|
148
|
+
| `direct_match` | Flat index page | Arch Linux |
|
|
149
|
+
| `fedora_nested` | Version dirs + variant subdirs | Fedora, Fedora KDE |
|
|
150
|
+
| `ubuntu_nested` | Version dirs | Ubuntu, Parrot Security |
|
|
151
|
+
| `nixos_channel` | Stable-channel lookup + channel page parse | NixOS Minimal, NixOS Graphical |
|
|
152
|
+
| `popos_api` | JSON API | Pop!_OS |
|
|
153
|
+
| `tails_api` | JSON API | Tails |
|
|
154
|
+
|
|
155
|
+
**Checksum formats:** `gpg_checksum` (Fedora — inline `SHA256 (file) = hash`), `sha256sums` (Ubuntu, Arch, Parrot, NixOS, Tails), `json` (Tails)
|
|
156
|
+
|
|
157
|
+
Signature checking is configured separately via `signing_key_url` + `signing_key_fingerprint` and works with any signed layout, so Parrot's sectioned hash list gets both a signature check and a digest check.
|
|
158
|
+
|
|
159
|
+
## How it works
|
|
160
|
+
|
|
161
|
+
1. **Detect** — finds mounted Ventoy drives on Windows, macOS, or Linux (with udisksctl automount)
|
|
162
|
+
2. **Scrape** — concurrent mirror scraping with TCP pre-flight checks and watchdog timeouts. A distro whose mirror can't be read is reported with the reason and makes the command exit non-zero; it is never silently counted as up to date
|
|
163
|
+
3. **Select** — release-family filters keep scrapers on the intended series: Ubuntu tracks the current **LTS** (even-year `.04`, skipping interim releases and beta-only directories), and NixOS resolves the current **stable** channel from the release bucket instead of pinning a version string
|
|
164
|
+
4. **Compare** — version-aware comparison (semantic or date-based) against local ISOs
|
|
165
|
+
5. **Download** — streaming downloads with staging buffer (less drive wear), falls back to direct if staging full. Use `--no-staging` / `--no-buffer` to skip the staging buffer and download directly to the Ventoy drive.
|
|
166
|
+
6. **Verify** — optional checksum verification against published hashes
|
|
167
|
+
7. **Clean** — `--clean` removes deprecated ISOs of the same distro variant (dry-run by default)
|
|
168
|
+
8. **State** — tracks installed distros in `.visync/installed.json`
|
|
169
|
+
|
|
170
|
+
## Safety
|
|
171
|
+
|
|
172
|
+
- `--clean` is dry-run by default, and `--dry-run` never modifies the drive — `sync --clean --dry-run` deletes nothing and does not touch `.visync/`
|
|
173
|
+
- `remove` and `nuke-metadata` show exactly what will be deleted and ask for confirmation (`--yes` skips)
|
|
174
|
+
- Ambiguous distro queries are refused with candidate lists instead of acting on an arbitrary match
|
|
175
|
+
- Distro matching uses whole-token keywords (`pop` does not match `popcorn.iso`)
|
|
176
|
+
- `.visync/` watchdog enforces a 1 GiB ceiling by deep-cleaning orphaned metadata. Wiping `.visync/` entirely is **opt-in** via `sync --reset-visync`, because it destroys `installed.json` and every registration
|
|
177
|
+
- The watchdog never runs on `--dry-run`
|
|
178
|
+
- Guardrails prevent deletion of `.iso` or `.img` files under any circumstance — including inside `.visync/metadata`
|
|
179
|
+
- Downloads use parallel range requests with per-chunk HTTP 206 and byte-count validation; truncated or range-ignoring servers fail loudly instead of producing silent corruption
|
|
180
|
+
- Checksum mismatch deletes the download; an *unreachable* checksum source keeps the file and warns (`UNVERIFIED`)
|
|
181
|
+
- HTTPS is enforced for all mirrors and checksum sources (loopback exempt); https→http redirects are blocked
|
|
182
|
+
- GPG signature verification supports fingerprint pinning via `signing_key_fingerprint`, and applies to any signed checksum file regardless of its digest layout (Fedora's inline `SHA256 (file) = hash`, Parrot's sectioned md5/sha256/sha512 list)
|
|
183
|
+
- When multiple digests for the same file appear in one signed list, the configured `checksum_algo` selects the section — a first-match parser would compare an MD5 against a SHA-256 and delete a good download as corrupt
|
|
184
|
+
- Failed downloads clean up `.part` files automatically
|
|
185
|
+
- Install verifies file exists on drive before marking as installed
|
|
186
|
+
- State and metadata writes are atomic (tmp file + rename)
|
|
187
|
+
|
|
188
|
+
## Tests
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
python3 -m pytest test/ -v
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
CI installs from the committed `uv.lock` so a dependency release cannot break the build without a diff. `uv` is used for reproducible CI and builds only — installing visync itself with `pip` is fully supported.
|
|
195
|
+
|
|
196
|
+
## Debug
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
VISYNC_DEBUG=1 visync sync
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
## License
|
|
203
|
+
|
|
204
|
+
MIT — see [LICENSE](LICENSE).
|
visync-1.0.0/README.md
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# Visync
|
|
2
|
+
|
|
3
|
+
Ventoy Package Manager. Install, update, and manage Linux distros on your Ventoy drive.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
Requires Python 3.12+ and a mounted Ventoy drive.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install visync
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
For a development checkout:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install -e ".[dev]"
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Both layouts work out of the box: `config.toml` ships inside the installed package, so `visync search` finds the distro list from any directory.
|
|
20
|
+
|
|
21
|
+
## Quick Start
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
# See what's available
|
|
25
|
+
visync search
|
|
26
|
+
|
|
27
|
+
# Auto-detect distros already on the drive
|
|
28
|
+
visync autodetect
|
|
29
|
+
|
|
30
|
+
# Install a distro
|
|
31
|
+
visync install archlinux
|
|
32
|
+
visync install nixos-graphical
|
|
33
|
+
|
|
34
|
+
# Batch install from file
|
|
35
|
+
visync install -i packages.txt
|
|
36
|
+
|
|
37
|
+
# Update all installed distros
|
|
38
|
+
visync update
|
|
39
|
+
|
|
40
|
+
# Update a specific distro
|
|
41
|
+
visync update tails
|
|
42
|
+
|
|
43
|
+
# Check for old versions (dry-run)
|
|
44
|
+
visync sync
|
|
45
|
+
|
|
46
|
+
# Remove old versions
|
|
47
|
+
visync sync --clean
|
|
48
|
+
|
|
49
|
+
# Remove a distro
|
|
50
|
+
visync remove tails
|
|
51
|
+
|
|
52
|
+
# Show distro details
|
|
53
|
+
visync info archlinux
|
|
54
|
+
|
|
55
|
+
# Verify checksums
|
|
56
|
+
visync verify
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Commands
|
|
60
|
+
|
|
61
|
+
| Command | Description |
|
|
62
|
+
|---|---|
|
|
63
|
+
| `visync search [query]` | List available distros (filter by query) |
|
|
64
|
+
| `visync install <name>` | Download and register a distro |
|
|
65
|
+
| `visync install -i <file>` | Batch install from file (one name per line) |
|
|
66
|
+
| `visync remove <name>` | Delete from drive and unregister |
|
|
67
|
+
| `visync update [name]` [--no-staging] | Update installed distros (all if no name given) |
|
|
68
|
+
| `visync sync` [--no-staging] | Sync installed distros to latest |
|
|
69
|
+
| `visync sync --all` [--no-staging] | Sync all configured distros |
|
|
70
|
+
| `visync sync --clean` [--no-staging] | Remove old versions of same distro |
|
|
71
|
+
| `visync sync --reset-visync` | Allow the watchdog to wipe an over-budget `.visync/` |
|
|
72
|
+
| `visync list` | List ISOs on the drive |
|
|
73
|
+
| `visync autodetect` | Register existing ISOs as installed |
|
|
74
|
+
| `visync verify` | Verify checksums against upstream |
|
|
75
|
+
| `visync info <name>` | Show distro details |
|
|
76
|
+
|
|
77
|
+
## Batch Install File
|
|
78
|
+
|
|
79
|
+
Create a text file with one distro name per line:
|
|
80
|
+
|
|
81
|
+
```
|
|
82
|
+
# My Ventoy setup
|
|
83
|
+
archlinux
|
|
84
|
+
nixos-minimal
|
|
85
|
+
tails
|
|
86
|
+
ubuntu-desktop
|
|
87
|
+
ubuntu-server
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Then run:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
visync install -i packages.txt
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Blank lines and lines starting with `#` are ignored.
|
|
97
|
+
|
|
98
|
+
## Configuration
|
|
99
|
+
|
|
100
|
+
Edit `config.toml` to add or remove distros. Each distro entry defines a scraping strategy, mirror URL, and checksum verification method.
|
|
101
|
+
|
|
102
|
+
Config resolution order: explicit `--config` path, then `$VISYNC_CONFIG`, then `~/.config/visync/config.toml`, then the `config.toml` shipped inside the installed package. A `config.toml` in the current directory is only used as a last resort, so planted configs cannot hijack identification.
|
|
103
|
+
|
|
104
|
+
**Release selection:** `version_filter` restricts a scraper to one release family before the newest match is chosen. Ubuntu entries use `\d*[02468]\.04(\.\d+)*` so they track the current LTS (26.04.1) and ignore interim releases (25.10, 26.10). NixOS resolves the newest stable `YY.MM` channel from the release bucket at scrape time, so it advances to the next stable without a config edit; set `channel = "26.05"` to pin deliberately while that channel is current.
|
|
105
|
+
|
|
106
|
+
**Built-in strategies:**
|
|
107
|
+
|
|
108
|
+
| Strategy | Description | Example |
|
|
109
|
+
|---|---|---|
|
|
110
|
+
| `direct_match` | Flat index page | Arch Linux |
|
|
111
|
+
| `fedora_nested` | Version dirs + variant subdirs | Fedora, Fedora KDE |
|
|
112
|
+
| `ubuntu_nested` | Version dirs | Ubuntu, Parrot Security |
|
|
113
|
+
| `nixos_channel` | Stable-channel lookup + channel page parse | NixOS Minimal, NixOS Graphical |
|
|
114
|
+
| `popos_api` | JSON API | Pop!_OS |
|
|
115
|
+
| `tails_api` | JSON API | Tails |
|
|
116
|
+
|
|
117
|
+
**Checksum formats:** `gpg_checksum` (Fedora — inline `SHA256 (file) = hash`), `sha256sums` (Ubuntu, Arch, Parrot, NixOS, Tails), `json` (Tails)
|
|
118
|
+
|
|
119
|
+
Signature checking is configured separately via `signing_key_url` + `signing_key_fingerprint` and works with any signed layout, so Parrot's sectioned hash list gets both a signature check and a digest check.
|
|
120
|
+
|
|
121
|
+
## How it works
|
|
122
|
+
|
|
123
|
+
1. **Detect** — finds mounted Ventoy drives on Windows, macOS, or Linux (with udisksctl automount)
|
|
124
|
+
2. **Scrape** — concurrent mirror scraping with TCP pre-flight checks and watchdog timeouts. A distro whose mirror can't be read is reported with the reason and makes the command exit non-zero; it is never silently counted as up to date
|
|
125
|
+
3. **Select** — release-family filters keep scrapers on the intended series: Ubuntu tracks the current **LTS** (even-year `.04`, skipping interim releases and beta-only directories), and NixOS resolves the current **stable** channel from the release bucket instead of pinning a version string
|
|
126
|
+
4. **Compare** — version-aware comparison (semantic or date-based) against local ISOs
|
|
127
|
+
5. **Download** — streaming downloads with staging buffer (less drive wear), falls back to direct if staging full. Use `--no-staging` / `--no-buffer` to skip the staging buffer and download directly to the Ventoy drive.
|
|
128
|
+
6. **Verify** — optional checksum verification against published hashes
|
|
129
|
+
7. **Clean** — `--clean` removes deprecated ISOs of the same distro variant (dry-run by default)
|
|
130
|
+
8. **State** — tracks installed distros in `.visync/installed.json`
|
|
131
|
+
|
|
132
|
+
## Safety
|
|
133
|
+
|
|
134
|
+
- `--clean` is dry-run by default, and `--dry-run` never modifies the drive — `sync --clean --dry-run` deletes nothing and does not touch `.visync/`
|
|
135
|
+
- `remove` and `nuke-metadata` show exactly what will be deleted and ask for confirmation (`--yes` skips)
|
|
136
|
+
- Ambiguous distro queries are refused with candidate lists instead of acting on an arbitrary match
|
|
137
|
+
- Distro matching uses whole-token keywords (`pop` does not match `popcorn.iso`)
|
|
138
|
+
- `.visync/` watchdog enforces a 1 GiB ceiling by deep-cleaning orphaned metadata. Wiping `.visync/` entirely is **opt-in** via `sync --reset-visync`, because it destroys `installed.json` and every registration
|
|
139
|
+
- The watchdog never runs on `--dry-run`
|
|
140
|
+
- Guardrails prevent deletion of `.iso` or `.img` files under any circumstance — including inside `.visync/metadata`
|
|
141
|
+
- Downloads use parallel range requests with per-chunk HTTP 206 and byte-count validation; truncated or range-ignoring servers fail loudly instead of producing silent corruption
|
|
142
|
+
- Checksum mismatch deletes the download; an *unreachable* checksum source keeps the file and warns (`UNVERIFIED`)
|
|
143
|
+
- HTTPS is enforced for all mirrors and checksum sources (loopback exempt); https→http redirects are blocked
|
|
144
|
+
- GPG signature verification supports fingerprint pinning via `signing_key_fingerprint`, and applies to any signed checksum file regardless of its digest layout (Fedora's inline `SHA256 (file) = hash`, Parrot's sectioned md5/sha256/sha512 list)
|
|
145
|
+
- When multiple digests for the same file appear in one signed list, the configured `checksum_algo` selects the section — a first-match parser would compare an MD5 against a SHA-256 and delete a good download as corrupt
|
|
146
|
+
- Failed downloads clean up `.part` files automatically
|
|
147
|
+
- Install verifies file exists on drive before marking as installed
|
|
148
|
+
- State and metadata writes are atomic (tmp file + rename)
|
|
149
|
+
|
|
150
|
+
## Tests
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
python3 -m pytest test/ -v
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
CI installs from the committed `uv.lock` so a dependency release cannot break the build without a diff. `uv` is used for reproducible CI and builds only — installing visync itself with `pip` is fully supported.
|
|
157
|
+
|
|
158
|
+
## Debug
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
VISYNC_DEBUG=1 visync sync
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
## License
|
|
165
|
+
|
|
166
|
+
MIT — see [LICENSE](LICENSE).
|
visync-1.0.0/config.toml
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
#!/visync/config.toml
|
|
2
|
+
[iso]
|
|
3
|
+
download_dir = ""
|
|
4
|
+
# ^ We don't download straight to the ventoy drive as its faster to verify and acts as a buffer. --no-buffer flag can override this, but verify will still happen, just on the drive.
|
|
5
|
+
|
|
6
|
+
[checksums]
|
|
7
|
+
enabled = true
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
###################################
|
|
12
|
+
####### DISTRO RECOGNITION ########
|
|
13
|
+
###################################
|
|
14
|
+
[base_distros]
|
|
15
|
+
# Strict internal volume ID matches
|
|
16
|
+
arch = "Arch Linux"
|
|
17
|
+
fedora = "Fedora"
|
|
18
|
+
ubuntu = "Ubuntu"
|
|
19
|
+
debian = "Debian"
|
|
20
|
+
|
|
21
|
+
[fork_overrides]
|
|
22
|
+
# Key format: "parent_distro.keyword_in_filename" = "Clean Name"
|
|
23
|
+
"arch.manjaro" = "Manjaro"
|
|
24
|
+
"ubuntu.pop-os" = "Pop!_OS"
|
|
25
|
+
"ubuntu.pop_os" = "Pop!_OS"
|
|
26
|
+
"ubuntu.mint" = "Linux Mint"
|
|
27
|
+
"debian.kali" = "Kali Linux"
|
|
28
|
+
"debian.parrot" = "Parrot Security"
|
|
29
|
+
# Fedora's aarch64 images carry "Fedora" in the volume ID, so without this they
|
|
30
|
+
# are indistinguishable from x86_64 and install/remove cannot target one.
|
|
31
|
+
"fedora.aarch64" = "Fedora ARM"
|
|
32
|
+
|
|
33
|
+
[standalone_matches]
|
|
34
|
+
# Text matching targets for standalone distros, flavors, and tools
|
|
35
|
+
gparted = "GParted Live"
|
|
36
|
+
hbcd = "Hiren's BootCD"
|
|
37
|
+
hiren = "Hiren's BootCD"
|
|
38
|
+
kodachi = "Linux Kodachi"
|
|
39
|
+
nixos-minimal = "NixOS Minimal"
|
|
40
|
+
nixos-graphical = "NixOS Graphical"
|
|
41
|
+
nixos = "NixOS Minimal"
|
|
42
|
+
parrot = "Parrot Security"
|
|
43
|
+
pond = "Pond"
|
|
44
|
+
pop = "Pop!_OS"
|
|
45
|
+
systemrescue = "SystemRescue"
|
|
46
|
+
rescuezilla = "Rescuezilla"
|
|
47
|
+
tails = "Tails"
|
|
48
|
+
tiny11 = "Tiny11"
|
|
49
|
+
|
|
50
|
+
# Specific desktop flavor/version overrides so regex doesn't grab them poorly.
|
|
51
|
+
# These must mirror the [distros.*] clean_name exactly: install/remove match an
|
|
52
|
+
# identified distro against clean_name, so a missing entry here makes a
|
|
53
|
+
# configured distro impossible to remove (test_verify_consistency_enforces this).
|
|
54
|
+
fedora-kde = "Fedora KDE"
|
|
55
|
+
fedora-arm = "Fedora ARM"
|
|
56
|
+
ubuntu-desktop = "Ubuntu Desktop"
|
|
57
|
+
ubuntu-server = "Ubuntu Server"
|
|
58
|
+
|
|
59
|
+
###################################
|
|
60
|
+
######## DYNAMIC SCRAPERS #########
|
|
61
|
+
###################################
|
|
62
|
+
#
|
|
63
|
+
[distros.Fedora]
|
|
64
|
+
clean_name = "Fedora"
|
|
65
|
+
keyword = "fedora"
|
|
66
|
+
strategy = "fedora_nested"
|
|
67
|
+
# dl.fedoraproject.org serves an Anubis bot challenge to scrapers; kernel.org
|
|
68
|
+
# mirrors the same tree bot-free. Integrity is pinned via signing_key_fingerprint.
|
|
69
|
+
base_url = "https://mirrors.kernel.org/fedora/releases/"
|
|
70
|
+
version_regex = 'href="([0-9]+)\s*/"'
|
|
71
|
+
iso_regex = 'href="(Fedora-Workstation-Live-(?:x86_64-[0-9][0-9.\-]*\.iso|[0-9][0-9.\-]*\.x86_64\.iso))"'
|
|
72
|
+
checksum_url = "{base_url}{version}/{variant_dir}/{arch}/iso/{checksum_stem}-CHECKSUM"
|
|
73
|
+
checksum_format = "gpg_checksum"
|
|
74
|
+
signing_key_url = "https://fedoraproject.org/fedora.gpg"
|
|
75
|
+
# Release-signing keys from https://fedoraproject.org/fedora.gpg (refresh when
|
|
76
|
+
# Fedora publishes new keys). Matches VALIDSIG fingerprints via --status-fd.
|
|
77
|
+
signing_key_fingerprint = [
|
|
78
|
+
"C6E7F081CF80E13146676E88829B606631645531",
|
|
79
|
+
"36F612DCF27F7D1A48A835E4DBFCF71C6D9F90A6",
|
|
80
|
+
"4F50A6114CD5C6976A7F1179655A4B02F577861E",
|
|
81
|
+
"D924B10D3E810DABDD8B56B596E7E91491211FCE",
|
|
82
|
+
]
|
|
83
|
+
|
|
84
|
+
[distros.ArchLinux]
|
|
85
|
+
clean_name = "Arch Linux"
|
|
86
|
+
keyword = "archlinux"
|
|
87
|
+
strategy = "direct_match"
|
|
88
|
+
base_url = "https://geo.mirror.pkgbuild.com/iso/latest/"
|
|
89
|
+
iso_regex = 'href="(archlinux-[0-9\.]+-x86_64\.iso)"'
|
|
90
|
+
checksum_url = "{base_url}sha256sums.txt"
|
|
91
|
+
checksum_format = "sha256sums"
|
|
92
|
+
checksum_algo = "sha256"
|
|
93
|
+
|
|
94
|
+
[distros.UbuntuServer]
|
|
95
|
+
clean_name = "Ubuntu Server"
|
|
96
|
+
keyword = "live-server"
|
|
97
|
+
strategy = "ubuntu_nested"
|
|
98
|
+
base_url = "https://releases.ubuntu.com/"
|
|
99
|
+
version_regex = 'href="([0-9\.]+)/"'
|
|
100
|
+
# Current Ubuntu LTS only — see UbuntuDesktop for the rationale.
|
|
101
|
+
version_filter = '\d*[02468]\.04(\.\d+)*'
|
|
102
|
+
iso_regex = 'href="(ubuntu-[0-9\.-]+live-server-amd64\.iso)"'
|
|
103
|
+
checksum_url = "{base_url}{version}/SHA256SUMS"
|
|
104
|
+
checksum_format = "sha256sums"
|
|
105
|
+
checksum_algo = "sha256"
|
|
106
|
+
|
|
107
|
+
################
|
|
108
|
+
##### ARM ######
|
|
109
|
+
################
|
|
110
|
+
|
|
111
|
+
[distros.FedoraARM]
|
|
112
|
+
clean_name = "Fedora ARM"
|
|
113
|
+
keyword = "fedora-arm"
|
|
114
|
+
strategy = "fedora_nested"
|
|
115
|
+
# dl.fedoraproject.org serves an Anubis bot challenge to scrapers; kernel.org
|
|
116
|
+
# mirrors the same tree bot-free. Integrity is pinned via signing_key_fingerprint.
|
|
117
|
+
base_url = "https://mirrors.kernel.org/fedora/releases/"
|
|
118
|
+
variant_path = "Workstation/aarch64/iso"
|
|
119
|
+
version_regex = 'href="([0-9]+\s*/)"'
|
|
120
|
+
# Fedora ships Workstation aarch64 lives under the standard variant subtree
|
|
121
|
+
iso_regex = 'href="(Fedora-Workstation-Live-(?:aarch64-[0-9][0-9.\-]*\.iso|[0-9][0-9.\-]*\.aarch64\.iso))"'
|
|
122
|
+
checksum_url = "{base_url}{version}/{variant_dir}/{arch}/iso/{checksum_stem}-CHECKSUM"
|
|
123
|
+
checksum_format = "gpg_checksum"
|
|
124
|
+
signing_key_url = "https://fedoraproject.org/fedora.gpg"
|
|
125
|
+
signing_key_fingerprint = [
|
|
126
|
+
"C6E7F081CF80E13146676E88829B606631645531",
|
|
127
|
+
"36F612DCF27F7D1A48A835E4DBFCF71C6D9F90A6",
|
|
128
|
+
"4F50A6114CD5C6976A7F1179655A4B02F577861E",
|
|
129
|
+
"D924B10D3E810DABDD8B56B596E7E91491211FCE",
|
|
130
|
+
]
|
|
131
|
+
|
|
132
|
+
# NOTE: Arch Linux ARM was removed. os.archlinuxarm.org serves a TLS
|
|
133
|
+
# certificate that does not match its hostname (scraping fails closed), and
|
|
134
|
+
# its artifacts are per-device .tar.gz rootfs archives — not .iso/.img files
|
|
135
|
+
# Ventoy can boot or visync can track. Re-add here if upstream publishes a
|
|
136
|
+
# bootable image over a valid certificate.
|
|
137
|
+
|
|
138
|
+
#############################
|
|
139
|
+
###### ADDITIONAL DT #######
|
|
140
|
+
#############################
|
|
141
|
+
|
|
142
|
+
[distros.FedoraKDE]
|
|
143
|
+
clean_name = "Fedora KDE"
|
|
144
|
+
keyword = "fedora-kde"
|
|
145
|
+
strategy = "fedora_nested"
|
|
146
|
+
variant_path = "KDE/x86_64/iso"
|
|
147
|
+
# dl.fedoraproject.org serves an Anubis bot challenge to scrapers; kernel.org
|
|
148
|
+
# mirrors the same tree bot-free. Integrity is pinned via signing_key_fingerprint.
|
|
149
|
+
base_url = "https://mirrors.kernel.org/fedora/releases/"
|
|
150
|
+
version_regex = 'href="([0-9]+\s*/)"'
|
|
151
|
+
iso_regex = 'href="(Fedora-KDE-Desktop-Live-(?:x86_64-[0-9][0-9.\-]*\.iso|[0-9][0-9.\-]*\.x86_64\.iso))"'
|
|
152
|
+
checksum_url = "{base_url}{version}/{variant_dir}/{arch}/iso/{checksum_stem}-CHECKSUM"
|
|
153
|
+
checksum_format = "gpg_checksum"
|
|
154
|
+
signing_key_url = "https://fedoraproject.org/fedora.gpg"
|
|
155
|
+
signing_key_fingerprint = [
|
|
156
|
+
"C6E7F081CF80E13146676E88829B606631645531",
|
|
157
|
+
"36F612DCF27F7D1A48A835E4DBFCF71C6D9F90A6",
|
|
158
|
+
"4F50A6114CD5C6976A7F1179655A4B02F577861E",
|
|
159
|
+
"D924B10D3E810DABDD8B56B596E7E91491211FCE",
|
|
160
|
+
]
|
|
161
|
+
|
|
162
|
+
[distros.UbuntuDesktop]
|
|
163
|
+
clean_name = "Ubuntu Desktop"
|
|
164
|
+
keyword = "desktop"
|
|
165
|
+
strategy = "ubuntu_nested"
|
|
166
|
+
base_url = "https://releases.ubuntu.com/"
|
|
167
|
+
version_regex = 'href="([0-9\.]+)/"'
|
|
168
|
+
# Track the current Ubuntu LTS rather than the newest release of any kind.
|
|
169
|
+
# LTS is an even-numbered year with a .04 month; interim releases (26.10,
|
|
170
|
+
# 25.10, ...) are excluded. Point releases like 26.04.1 also match, which is
|
|
171
|
+
# what we want — those are the same LTS series.
|
|
172
|
+
version_filter = '\d*[02468]\.04(\.\d+)*'
|
|
173
|
+
iso_regex = 'href="(ubuntu-[0-9\.-]+desktop-amd64\.iso)"'
|
|
174
|
+
checksum_url = "{base_url}{version}/SHA256SUMS"
|
|
175
|
+
checksum_format = "sha256sums"
|
|
176
|
+
checksum_algo = "sha256"
|
|
177
|
+
|
|
178
|
+
[distros.ParrotSecurity]
|
|
179
|
+
clean_name = "Parrot Security"
|
|
180
|
+
keyword = "parrot"
|
|
181
|
+
strategy = "ubuntu_nested"
|
|
182
|
+
base_url = "https://deb.parrotsec.org/parrot/iso/"
|
|
183
|
+
version_regex = 'href="([0-9\.]+(?:-[0-9]+)?)/"'
|
|
184
|
+
iso_regex = 'href="(Parrot-security-[0-9\.]+_amd64\.iso)"'
|
|
185
|
+
# signed-hashes.txt is one clearsigned file listing md5, sha256 and sha512
|
|
186
|
+
# sections for every image. The parser selects the sha256 section by header
|
|
187
|
+
# and digest length — taking the first match would compare an md5 against a
|
|
188
|
+
# sha256 and delete a good download as "corrupt".
|
|
189
|
+
checksum_url = "{base_url}{version}/signed-hashes.txt"
|
|
190
|
+
checksum_format = "sha256sums"
|
|
191
|
+
checksum_algo = "sha256"
|
|
192
|
+
# Signing key published by Parrot and referenced from inside the signed
|
|
193
|
+
# content itself. Fingerprint confirmed via gpg --verify (VALIDSIG) against
|
|
194
|
+
# the untouched file, and matches the fingerprint in Parrot's own
|
|
195
|
+
# hash-and-key-verification documentation.
|
|
196
|
+
signing_key_url = "https://deb.parrot.sh/parrot/misc/archive.gpg"
|
|
197
|
+
signing_key_fingerprint = [
|
|
198
|
+
"B711822346552E4D92DA02DF7A8286AF0E81EE4A",
|
|
199
|
+
]
|
|
200
|
+
|
|
201
|
+
[distros.NixOS]
|
|
202
|
+
clean_name = "NixOS Minimal"
|
|
203
|
+
keyword = "nixos-minimal"
|
|
204
|
+
strategy = "nixos_channel"
|
|
205
|
+
# base_url is a prefix: the current stable channel (e.g. 26.05) is appended,
|
|
206
|
+
# giving https://channels.nixos.org/nixos-26.05, which redirects to the exact
|
|
207
|
+
# releases.nixos.org directory. NixOS publishes no nixos-stable alias, so the
|
|
208
|
+
# channel is resolved from the release bucket at scrape time — see
|
|
209
|
+
# nixos_channel in src/download.py.
|
|
210
|
+
base_url = "https://channels.nixos.org/nixos"
|
|
211
|
+
# Optional pin. If set and still a real channel it wins; otherwise the newest
|
|
212
|
+
# stable channel from the bucket is used. Omit to always track latest stable.
|
|
213
|
+
# channel = "26.05"
|
|
214
|
+
variant = "minimal"
|
|
215
|
+
# ISOs and .sha256 sidecars live under releases.nixos.org, NOT the channel host
|
|
216
|
+
# (channels.nixos.org serves no .sha256, which made `visync verify` fail with
|
|
217
|
+
# "source unreachable" for every NixOS ISO).
|
|
218
|
+
releases_base_url = "https://releases.nixos.org"
|
|
219
|
+
# Sidecars sit at /nixos/<channel>/nixos-<full release>/<iso>.sha256
|
|
220
|
+
checksum_url = "{release_base_url}/nixos/{version}/nixos-{release}/{iso_name}.sha256"
|
|
221
|
+
checksum_format = "sha256sums"
|
|
222
|
+
checksum_algo = "sha256"
|
|
223
|
+
|
|
224
|
+
[distros.NixOSGraphical]
|
|
225
|
+
clean_name = "NixOS Graphical"
|
|
226
|
+
keyword = "nixos-graphical"
|
|
227
|
+
strategy = "nixos_channel"
|
|
228
|
+
base_url = "https://channels.nixos.org/nixos"
|
|
229
|
+
variant = "graphical"
|
|
230
|
+
releases_base_url = "https://releases.nixos.org"
|
|
231
|
+
checksum_url = "{release_base_url}/nixos/{version}/nixos-{release}/{iso_name}.sha256"
|
|
232
|
+
checksum_format = "sha256sums"
|
|
233
|
+
checksum_algo = "sha256"
|
|
234
|
+
|
|
235
|
+
[distros.PopOS]
|
|
236
|
+
clean_name = "Pop!_OS"
|
|
237
|
+
keyword = "pop"
|
|
238
|
+
strategy = "popos_api"
|
|
239
|
+
api_url = "https://api.pop-os.org/builds"
|
|
240
|
+
variant = "generic"
|
|
241
|
+
release = "24.04"
|
|
242
|
+
checksum_format = "sha256sums"
|
|
243
|
+
checksum_algo = "sha256"
|
|
244
|
+
|
|
245
|
+
[distros.Tails]
|
|
246
|
+
clean_name = "Tails"
|
|
247
|
+
keyword = "tails"
|
|
248
|
+
strategy = "tails_api"
|
|
249
|
+
api_url = "https://tails.net/install/v2/Tails/amd64/stable/latest.json"
|
|
250
|
+
file_type = "img"
|
|
251
|
+
checksum_format = "sha256sums"
|
|
252
|
+
checksum_algo = "sha256"
|
|
253
|
+
|
|
254
|
+
[distros.Pond]
|
|
255
|
+
clean_name = "Pond"
|
|
256
|
+
keyword = "pond"
|
|
257
|
+
strategy = "content_disposition"
|
|
258
|
+
base_url = "https://releases.butterfly.so/iso/"
|
|
259
|
+
iso_regex = '^pond-installer-([0-9]+\.[0-9]+\.[0-9]+)-x86_64\.iso$'
|