airmate-navdata-sync 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airmate_navdata_sync-0.2.0.dist-info/METADATA +188 -0
- airmate_navdata_sync-0.2.0.dist-info/RECORD +14 -0
- airmate_navdata_sync-0.2.0.dist-info/WHEEL +4 -0
- airmate_navdata_sync-0.2.0.dist-info/entry_points.txt +2 -0
- airmate_navdata_sync-0.2.0.dist-info/licenses/LICENSE +21 -0
- navdata_sync/__init__.py +3 -0
- navdata_sync/__main__.py +3 -0
- navdata_sync/catalog.py +71 -0
- navdata_sync/cli.py +109 -0
- navdata_sync/config.example.toml +66 -0
- navdata_sync/config.py +210 -0
- navdata_sync/download.py +130 -0
- navdata_sync/prepare.py +55 -0
- navdata_sync/usb.py +314 -0
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: airmate-navdata-sync
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Download an Airmate AIRAC cycle and lay it out for a Dynon SkyView USB stick
|
|
5
|
+
Project-URL: Homepage, https://github.com/Natim/skydemon-airmate-navdata-sync
|
|
6
|
+
Project-URL: Source, https://github.com/Natim/skydemon-airmate-navdata-sync
|
|
7
|
+
Project-URL: Issues, https://github.com/Natim/skydemon-airmate-navdata-sync/issues
|
|
8
|
+
Author-email: Rémy Hubscher <remy@yampa.ai>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: airac,airmate,aviation,dynon,navdata,skyview,vfr
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
21
|
+
Classifier: Topic :: Utilities
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Requires-Dist: aiofile>=3.9
|
|
24
|
+
Requires-Dist: httpx[http2]>=0.28
|
|
25
|
+
Requires-Dist: rich>=13.9
|
|
26
|
+
Requires-Dist: tqdm>=4.67
|
|
27
|
+
Provides-Extra: mbtiles
|
|
28
|
+
Requires-Dist: pillow>=10.0; extra == 'mbtiles'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# skydemon-airmate-navdata-sync
|
|
32
|
+
|
|
33
|
+
Keeps the navigation data of a Dynon SkyView up to date: it downloads an
|
|
34
|
+
[Airmate](https://www.airmate.aero) AIRAC cycle, lays the files out exactly the
|
|
35
|
+
way the Dynon expects them, and copies the result onto every configured USB
|
|
36
|
+
stick without rewriting the gigabytes that did not change.
|
|
37
|
+
|
|
38
|
+
Everything specific to a subscription or a machine — the Airmate customer id,
|
|
39
|
+
the Dynon serial, the AIRAC cycle, the local paths — lives in `config.toml`,
|
|
40
|
+
which is git-ignored. The code contains no ids.
|
|
41
|
+
|
|
42
|
+
## Requirements
|
|
43
|
+
|
|
44
|
+
- Python 3.11 or newer (the config reader uses the standard-library `tomllib`)
|
|
45
|
+
- An active Airmate subscription bound to your Dynon serial
|
|
46
|
+
|
|
47
|
+
## Setup
|
|
48
|
+
|
|
49
|
+
Either install the release, which puts a `navdata-update` command on your PATH:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install airmate-navdata-sync
|
|
53
|
+
navdata-update --init-config # writes ~/.config/navdata-sync/config.toml
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
or work from a checkout, which keeps everything in one folder. `pip install -e .`
|
|
57
|
+
installs the dependencies from `pyproject.toml` and links the package in place,
|
|
58
|
+
so edits take effect without reinstalling:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
git clone https://github.com/Natim/skydemon-airmate-navdata-sync
|
|
62
|
+
cd skydemon-airmate-navdata-sync
|
|
63
|
+
python3 -m venv .venv && . .venv/bin/activate
|
|
64
|
+
pip install -e .
|
|
65
|
+
cp config.example.toml config.toml
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Then edit the config file. The two values you have to fill in are in the
|
|
69
|
+
`[airmate]` section:
|
|
70
|
+
|
|
71
|
+
- `id` — the customer id in your personal download URLs
|
|
72
|
+
(`https://www.airmate.aero/download/navdata/<id>/...`), visible on the Airmate
|
|
73
|
+
download page once logged in.
|
|
74
|
+
- `serial` — the serial of the Dynon the subscription is bound to. It is the
|
|
75
|
+
number embedded in the filenames Airmate offers you, as in
|
|
76
|
+
`airmate_av_data_eu_2608_<serial>.dup` and `CHARTS-<serial>.key`.
|
|
77
|
+
|
|
78
|
+
If you would rather not keep those on disk, leave the placeholders in place and
|
|
79
|
+
export them instead; the environment always wins over the file:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
export AIRMATE_ID=... AIRMATE_SERIAL=...
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Every other setting is documented inline in
|
|
86
|
+
[`config.example.toml`](config.example.toml).
|
|
87
|
+
|
|
88
|
+
Without `--config`, the first of these that exists is used: `./config.toml`,
|
|
89
|
+
then `~/.config/navdata-sync/config.toml` (honouring `XDG_CONFIG_HOME`), then
|
|
90
|
+
`config.toml` at the root of a checkout.
|
|
91
|
+
|
|
92
|
+
## Usage
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
navdata-update # download the configured cycle, then stage it
|
|
96
|
+
navdata-update --list # show which files the config resolves to
|
|
97
|
+
navdata-update --cycle 2609 # try the next cycle without editing the config
|
|
98
|
+
navdata-update --skip-download # rebuild the staging folder from the cache
|
|
99
|
+
navdata-update --sync # ...and copy it onto every configured USB stick
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
From a checkout without installing, `./navdata-update.py` and
|
|
103
|
+
`python -m navdata_sync` take the same arguments.
|
|
104
|
+
|
|
105
|
+
Without `--sync` the run stops after staging and prints the configured USB
|
|
106
|
+
mounts, so you can inspect the result first.
|
|
107
|
+
|
|
108
|
+
Downloads are resumable and run four at a time behind one global progress bar;
|
|
109
|
+
interrupting the script and running it again picks up where it left off. Files
|
|
110
|
+
that are already complete are skipped, with one deliberate exception: the
|
|
111
|
+
`CHARTS-*.key` file is always re-fetched, because its contents change every
|
|
112
|
+
cycle while its size stays identical.
|
|
113
|
+
|
|
114
|
+
## Keeping the configuration current
|
|
115
|
+
|
|
116
|
+
Two things drift and need a manual edit:
|
|
117
|
+
|
|
118
|
+
- **`data.cycle`** — the AIRAC cycle, as `<two-digit year><two-digit cycle>`
|
|
119
|
+
(`2608` is the 8th cycle of 2026). It changes every 28 days.
|
|
120
|
+
- **`data.raster`** — raster VFR charts are named after their *edition* date
|
|
121
|
+
(`VFR-FRANCE-OACI-16APR26.dcf`), which is unrelated to the AIRAC cycle. Copy
|
|
122
|
+
the new filenames from the Airmate download page when an edition is published.
|
|
123
|
+
|
|
124
|
+
`--list` is the quick way to check what the current configuration points at
|
|
125
|
+
before starting a multi-gigabyte download.
|
|
126
|
+
|
|
127
|
+
## What ends up on the stick
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
AIRMATE_AV_DATA_EU_<cycle>_<serial>.DUP navdata (uppercased, the Dynon is picky)
|
|
131
|
+
AIRMATE_OBSTACLE_DATA_EU_<cycle>_<serial>.DUP
|
|
132
|
+
CHARTS-<serial>.key unlocks the chart and raster layers
|
|
133
|
+
ChartData/Plates/... approach plates, from the region zips
|
|
134
|
+
Raster/VFR-*.dcf raster VFR charts
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
The copy checksums each file (blake2b) and writes only those whose bytes
|
|
138
|
+
actually changed, in 4 MiB sequential chunks via `aiofile`. The staging folder
|
|
139
|
+
is rebuilt from scratch every run, so timestamps are always new and a
|
|
140
|
+
size+mtime comparison would recopy everything. Full sequential writes of
|
|
141
|
+
changed files are easier on a FAT-32 stick than rsync's random block deltas,
|
|
142
|
+
and opening the destination in place avoids a temp copy on a nearly-full
|
|
143
|
+
volume. Several sticks (left and right SkyView) are updated in parallel, each
|
|
144
|
+
with its own sequential writer.
|
|
145
|
+
|
|
146
|
+
## Repository layout
|
|
147
|
+
|
|
148
|
+
```
|
|
149
|
+
pyproject.toml packaging metadata, distributed as airmate-navdata-sync
|
|
150
|
+
navdata-update.py entry point for a checkout, mirrors the installed command
|
|
151
|
+
config.example.toml documented template for the git-ignored config.toml
|
|
152
|
+
navdata_sync/
|
|
153
|
+
config.py finds and validates config.toml, applies env overrides
|
|
154
|
+
catalog.py turns the config into the list of URLs to fetch
|
|
155
|
+
download.py resumable parallel downloader
|
|
156
|
+
prepare.py staging into the Dynon layout
|
|
157
|
+
usb.py checksum-and-copy onto one or more USB sticks
|
|
158
|
+
cli.py argument parsing and the run sequence
|
|
159
|
+
tools/
|
|
160
|
+
mbtiles_to_dcf.py converts an MBTiles file into an Airmate-style .dcf
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
`tools/mbtiles_to_dcf.py` is a standalone experiment, not part of the update
|
|
164
|
+
flow, so it ships in the source distribution and the checkout rather than in the
|
|
165
|
+
wheel. It builds a `.dcf` raster layer from any MBTiles source; note that a layer
|
|
166
|
+
is only displayed if a matching entry exists in `CHARTS-<serial>.key`, which the
|
|
167
|
+
script cannot create. It needs Pillow, from the `mbtiles` extra
|
|
168
|
+
(`pip install -e '.[mbtiles]'` in a checkout); run
|
|
169
|
+
`tools/mbtiles_to_dcf.py --help` for the options.
|
|
170
|
+
|
|
171
|
+
## Releasing
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
pip install build twine
|
|
175
|
+
python -m build # writes dist/*.whl and dist/*.tar.gz
|
|
176
|
+
twine check dist/*
|
|
177
|
+
twine upload dist/*
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
The version lives in `navdata_sync/__init__.py` and is read from there by the
|
|
181
|
+
build backend, so bump it in that one place.
|
|
182
|
+
|
|
183
|
+
## A note on the data
|
|
184
|
+
|
|
185
|
+
The navdata is licensed per subscription and tied to your Dynon serial. Keep
|
|
186
|
+
`config.toml`, the downloads and the staging folder out of version control —
|
|
187
|
+
`.gitignore` already covers them. And, obviously: this is a personal convenience
|
|
188
|
+
script, so verify on the unit that the cycle you expect is the cycle it loaded.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
navdata_sync/__init__.py,sha256=hK23J6VmGu6ROH2_2ZtP6NmrHhYOn4Oviobpzs_yc1k,77
|
|
2
|
+
navdata_sync/__main__.py,sha256=SCuGk-L-bb1zLj15YeA4J9PvoYe60R0p2hiL5YDrvuo,51
|
|
3
|
+
navdata_sync/catalog.py,sha256=_Lbay5KooI8DTM2f0Yo2hbI4C-5Say130GxsjQ7lU-w,2471
|
|
4
|
+
navdata_sync/cli.py,sha256=2gv1jOf-DRjesh8RDWF2YQcfuw4zAJrSNcFL0FScYM4,3492
|
|
5
|
+
navdata_sync/config.py,sha256=64BeMisTUfXyGlLJigBV1mUCXCTzXrbBge6SIHVzGTA,7571
|
|
6
|
+
navdata_sync/download.py,sha256=Op8qM9fqRIvs7IGV4EP3xDqz320Bio9yuRpsB-4Zd9Q,4554
|
|
7
|
+
navdata_sync/prepare.py,sha256=mkSYjnq9zJ2E4-8DbeYIJJCd15YnByfu1UtLgRHIbXE,1817
|
|
8
|
+
navdata_sync/usb.py,sha256=fp-mzA6Gqh5GfvW3MuUj3qim5s475HF5v_4tfk4lcMY,10592
|
|
9
|
+
navdata_sync/config.example.toml,sha256=SA9zbOZKwybyePGH014dSfcsAipDtLuSTNuBFvEc-i8,2677
|
|
10
|
+
airmate_navdata_sync-0.2.0.dist-info/METADATA,sha256=T6oMKnTVlvWv7-d_CWUBybCnXrjzSIS2diClX473BWs,7944
|
|
11
|
+
airmate_navdata_sync-0.2.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
|
|
12
|
+
airmate_navdata_sync-0.2.0.dist-info/entry_points.txt,sha256=LN6qy6Jt94INAct2TYi-B_QAkKdxZVvje0MhwetKE7E,57
|
|
13
|
+
airmate_navdata_sync-0.2.0.dist-info/licenses/LICENSE,sha256=_bMIXIy67bjWeWdYt-iEMNjXF__qRwXIFhdn2s_Irzc,1071
|
|
14
|
+
airmate_navdata_sync-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Rémy Hubscher
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
navdata_sync/__init__.py
ADDED
navdata_sync/__main__.py
ADDED
navdata_sync/catalog.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Turns a Config into the list of files to fetch from Airmate.
|
|
2
|
+
|
|
3
|
+
Airmate publishes three kinds of URL:
|
|
4
|
+
|
|
5
|
+
<base>/<customer id>/... subscription-bound data, one file per serial
|
|
6
|
+
<base>/Plates/... approach plate bundles, one zip per region
|
|
7
|
+
<base>/Raster/... raster VFR charts, shared by everyone
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from enum import Enum
|
|
13
|
+
|
|
14
|
+
from .config import Config
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Kind(Enum):
|
|
18
|
+
"""What a file becomes on the USB stick, which decides how it is staged."""
|
|
19
|
+
|
|
20
|
+
DATA = "data" # .dup, navdata and obstacles, uppercased at the root
|
|
21
|
+
KEY = "key" # CHARTS-<serial>.key, unlocks the raster layers
|
|
22
|
+
PLATES = "plates" # zip expanded at the root, creates ChartData/
|
|
23
|
+
RASTER = "raster" # .dcf copied under Raster/
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class RemoteFile:
|
|
28
|
+
url: str
|
|
29
|
+
name: str
|
|
30
|
+
kind: Kind
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def always_refresh(self) -> bool:
|
|
34
|
+
"""Whether a byte-identical local copy still has to be re-downloaded.
|
|
35
|
+
|
|
36
|
+
The key file changes content every AIRAC cycle (chart filenames and auth
|
|
37
|
+
codes rotate) while keeping the exact same byte size, so a size-only
|
|
38
|
+
"already complete" check would keep a stale key on disk. The Dynon then
|
|
39
|
+
silently hides every layer whose filename or code no longer matches.
|
|
40
|
+
"""
|
|
41
|
+
return self.kind is Kind.KEY
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def build(config: Config) -> list[RemoteFile]:
|
|
45
|
+
account = f"{config.base_url}/{config.airmate_id}"
|
|
46
|
+
files = [
|
|
47
|
+
RemoteFile(
|
|
48
|
+
f"{account}/airmate_av_data_eu_{config.cycle}_{config.serial}.dup",
|
|
49
|
+
f"airmate_av_data_eu_{config.cycle}_{config.serial}.dup",
|
|
50
|
+
Kind.DATA,
|
|
51
|
+
),
|
|
52
|
+
RemoteFile(
|
|
53
|
+
f"{account}/airmate_obstacle_data_eu_{config.cycle}_{config.serial}.dup",
|
|
54
|
+
f"airmate_obstacle_data_eu_{config.cycle}_{config.serial}.dup",
|
|
55
|
+
Kind.DATA,
|
|
56
|
+
),
|
|
57
|
+
RemoteFile(
|
|
58
|
+
f"{account}/CHARTS-{config.serial}.key",
|
|
59
|
+
f"CHARTS-{config.serial}.key",
|
|
60
|
+
Kind.KEY,
|
|
61
|
+
),
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
for region in config.plates:
|
|
65
|
+
name = f"{region}-Plates-{config.cycle}.zip"
|
|
66
|
+
files.append(RemoteFile(f"{config.base_url}/Plates/{name}", name, Kind.PLATES))
|
|
67
|
+
|
|
68
|
+
for name in config.raster:
|
|
69
|
+
files.append(RemoteFile(f"{config.base_url}/Raster/{name}", name, Kind.RASTER))
|
|
70
|
+
|
|
71
|
+
return files
|
navdata_sync/cli.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Command line entry point: download, stage, optionally sync."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import asyncio
|
|
6
|
+
import dataclasses
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from . import catalog, download, prepare
|
|
10
|
+
from . import __version__
|
|
11
|
+
from .config import ConfigError, init_config, search_path
|
|
12
|
+
from .config import load as load_config
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
16
|
+
parser = argparse.ArgumentParser(
|
|
17
|
+
prog="navdata-update",
|
|
18
|
+
description="Download an Airmate AIRAC cycle and lay it out for a "
|
|
19
|
+
"Dynon SkyView USB stick.",
|
|
20
|
+
)
|
|
21
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
22
|
+
searched = " or ".join(str(candidate) for candidate in search_path())
|
|
23
|
+
parser.add_argument(
|
|
24
|
+
"-c",
|
|
25
|
+
"--config",
|
|
26
|
+
type=Path,
|
|
27
|
+
metavar="PATH",
|
|
28
|
+
help=f"configuration file (default: the first of {searched})",
|
|
29
|
+
)
|
|
30
|
+
parser.add_argument(
|
|
31
|
+
"--init-config",
|
|
32
|
+
action="store_true",
|
|
33
|
+
help="write a starter configuration to --config, or to the per-user "
|
|
34
|
+
"location, then exit",
|
|
35
|
+
)
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--cycle",
|
|
38
|
+
metavar="YYCC",
|
|
39
|
+
help="override data.cycle for this run, e.g. 2609",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"--list",
|
|
43
|
+
action="store_true",
|
|
44
|
+
help="list the files the configuration resolves to and exit",
|
|
45
|
+
)
|
|
46
|
+
parser.add_argument(
|
|
47
|
+
"--skip-download",
|
|
48
|
+
action="store_true",
|
|
49
|
+
help="rebuild the staging folder from the download cache only",
|
|
50
|
+
)
|
|
51
|
+
parser.add_argument(
|
|
52
|
+
"--sync",
|
|
53
|
+
action="store_true",
|
|
54
|
+
help="copy the staging folder onto every configured USB stick",
|
|
55
|
+
)
|
|
56
|
+
return parser
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def main(argv: list[str] | None = None) -> int:
|
|
60
|
+
args = build_parser().parse_args(argv)
|
|
61
|
+
|
|
62
|
+
if args.init_config:
|
|
63
|
+
try:
|
|
64
|
+
target = init_config(args.config)
|
|
65
|
+
except ConfigError as exc:
|
|
66
|
+
print(f"❌ {exc}")
|
|
67
|
+
return 2
|
|
68
|
+
print(
|
|
69
|
+
f"✅ Configuration créée: {target}\n"
|
|
70
|
+
" Renseignez airmate.id et airmate.serial avant le premier run."
|
|
71
|
+
)
|
|
72
|
+
return 0
|
|
73
|
+
|
|
74
|
+
try:
|
|
75
|
+
config = load_config(args.config)
|
|
76
|
+
except ConfigError as exc:
|
|
77
|
+
print(f"❌ {exc}")
|
|
78
|
+
return 2
|
|
79
|
+
|
|
80
|
+
if args.cycle:
|
|
81
|
+
config = dataclasses.replace(config, cycle=args.cycle)
|
|
82
|
+
|
|
83
|
+
# Checked before anything is downloaded or staged, since --sync is the whole
|
|
84
|
+
# point of the run when it is passed.
|
|
85
|
+
if args.sync and not config.usb_targets:
|
|
86
|
+
print("❌ --sync demande paths.usb_target ou paths.usb_targets dans la configuration")
|
|
87
|
+
return 2
|
|
88
|
+
|
|
89
|
+
files = catalog.build(config)
|
|
90
|
+
|
|
91
|
+
if args.list:
|
|
92
|
+
for file in files:
|
|
93
|
+
print(f"{file.kind.value:>7} {file.name}")
|
|
94
|
+
print(f"\ncycle {config.cycle} · {len(files)} fichiers · {config.download_dir}")
|
|
95
|
+
return 0
|
|
96
|
+
|
|
97
|
+
if not args.skip_download:
|
|
98
|
+
asyncio.run(download.run(files, config.download_dir))
|
|
99
|
+
|
|
100
|
+
missing = prepare.build(files, config.download_dir, config.prepared_dir)
|
|
101
|
+
|
|
102
|
+
if args.sync:
|
|
103
|
+
if missing:
|
|
104
|
+
print("❌ Synchronisation annulée: des fichiers manquent")
|
|
105
|
+
return 1
|
|
106
|
+
return prepare.sync(config.prepared_dir, config.usb_targets)
|
|
107
|
+
|
|
108
|
+
print(prepare.describe_sync(config.prepared_dir, config.usb_targets))
|
|
109
|
+
return 1 if missing else 0
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Copy this file to config.toml and fill in your own values.
|
|
2
|
+
#
|
|
3
|
+
# cp config.example.toml config.toml
|
|
4
|
+
#
|
|
5
|
+
# config.toml is git-ignored because it holds the two values that identify your
|
|
6
|
+
# Airmate subscription: the customer id and the Dynon serial it is bound to.
|
|
7
|
+
|
|
8
|
+
[airmate]
|
|
9
|
+
# The customer id that appears in your personal download URLs:
|
|
10
|
+
# https://www.airmate.aero/download/navdata/<id>/...
|
|
11
|
+
# It is shown on the Airmate download page once you are logged in.
|
|
12
|
+
# May also be supplied through the AIRMATE_ID environment variable.
|
|
13
|
+
id = "YOUR_AIRMATE_ID"
|
|
14
|
+
|
|
15
|
+
# Serial number of the Dynon unit the subscription is bound to. It is the
|
|
16
|
+
# number embedded in the data filenames:
|
|
17
|
+
# airmate_av_data_eu_<cycle>_<serial>.dup CHARTS-<serial>.key
|
|
18
|
+
# May also be supplied through the AIRMATE_SERIAL environment variable.
|
|
19
|
+
serial = "000000"
|
|
20
|
+
|
|
21
|
+
# Only change this if Airmate moves its download endpoint.
|
|
22
|
+
base_url = "https://www.airmate.aero/download/navdata"
|
|
23
|
+
|
|
24
|
+
[data]
|
|
25
|
+
# AIRAC cycle to download, as <two-digit year><two-digit cycle number>.
|
|
26
|
+
# 2608 is the 8th cycle of 2026. Bump this every 28 days.
|
|
27
|
+
cycle = "2608"
|
|
28
|
+
|
|
29
|
+
# Approach plate bundles, published as <region>-Plates-<cycle>.zip.
|
|
30
|
+
# "FR" alone is enough if you only fly in France; "Europe" is a large download.
|
|
31
|
+
plates = ["FR", "Europe"]
|
|
32
|
+
|
|
33
|
+
# Raster VFR charts, published under <base_url>/Raster/.
|
|
34
|
+
# The trailing date is the chart *edition*, not the AIRAC cycle, so these names
|
|
35
|
+
# have to be updated by hand whenever a new edition is published. Remove the
|
|
36
|
+
# ones you do not need: each entry is a few hundred MB on the USB stick.
|
|
37
|
+
raster = [
|
|
38
|
+
"VFR-FRANCE-OACI-16APR26.dcf",
|
|
39
|
+
"VFR-EUROPE-HIRES-14MAY26.dcf",
|
|
40
|
+
"VFR-DENMARK-NAVIAIR-14MAY26.dcf",
|
|
41
|
+
"VFR-IRELAND-01JAN24.dcf",
|
|
42
|
+
"VFR-UK-01JAN24.dcf",
|
|
43
|
+
"VFR-ITALY-16APR26.dcf",
|
|
44
|
+
"VFR-SWITZERLAND-19MAR26.dcf",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[paths]
|
|
48
|
+
# Relative paths below are resolved next to this config file, which keeps a git
|
|
49
|
+
# checkout self-contained. If this config lives in ~/.config, make them absolute
|
|
50
|
+
# instead: the AIRAC cycle is several gigabytes and does not belong there.
|
|
51
|
+
|
|
52
|
+
# Download cache. Partially downloaded files are resumed from here, so keeping
|
|
53
|
+
# it between runs saves bandwidth.
|
|
54
|
+
download_dir = "downloads"
|
|
55
|
+
|
|
56
|
+
# Staging area laid out exactly like the root of the Dynon USB stick. Rebuilt
|
|
57
|
+
# from scratch on every run.
|
|
58
|
+
prepared_dir = "downloads_prepared"
|
|
59
|
+
|
|
60
|
+
# Mount points of the Dynon USB sticks, used by --sync. One entry per unit
|
|
61
|
+
# (left and right SkyView). A single string still works. Leave the list empty
|
|
62
|
+
# to only build the staging directory.
|
|
63
|
+
usb_targets = [
|
|
64
|
+
"/run/media/YOUR_USER/LH D1000",
|
|
65
|
+
"/run/media/YOUR_USER/RH D1000",
|
|
66
|
+
]
|
navdata_sync/config.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Loading and validation of config.toml.
|
|
2
|
+
|
|
3
|
+
Everything that identifies a particular subscription or machine lives in the
|
|
4
|
+
config file, never in the code: the Airmate customer id, the Dynon serial, the
|
|
5
|
+
AIRAC cycle and the local paths. See config.example.toml for the reference.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import shutil
|
|
11
|
+
import tomllib
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
APP_NAME = "navdata-sync"
|
|
16
|
+
PACKAGE_DIR = Path(__file__).resolve().parent
|
|
17
|
+
CHECKOUT_ROOT = PACKAGE_DIR.parent # the repository root, in a git checkout
|
|
18
|
+
|
|
19
|
+
ID_ENV = "AIRMATE_ID"
|
|
20
|
+
SERIAL_ENV = "AIRMATE_SERIAL"
|
|
21
|
+
|
|
22
|
+
# Values shipped in config.example.toml. Refusing them keeps a half-filled copy
|
|
23
|
+
# of the example from producing a run of 404s.
|
|
24
|
+
PLACEHOLDERS = frozenset({
|
|
25
|
+
"YOUR_AIRMATE_ID",
|
|
26
|
+
"000000",
|
|
27
|
+
"/run/media/YOUR_USER/LH D1000",
|
|
28
|
+
"/run/media/YOUR_USER/RH D1000",
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ConfigError(Exception):
|
|
33
|
+
"""Raised when the config file is missing, unreadable or incomplete."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def user_config() -> Path:
|
|
37
|
+
"""The per-user configuration, following the XDG base directory spec."""
|
|
38
|
+
base = os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config"
|
|
39
|
+
return Path(base) / APP_NAME / "config.toml"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def search_path() -> list[Path]:
|
|
43
|
+
"""Where an invocation without --config looks, in order.
|
|
44
|
+
|
|
45
|
+
The current directory comes first so a checkout keeps working as a
|
|
46
|
+
self-contained folder, then the per-user location for an installed copy,
|
|
47
|
+
then the checkout root so cron jobs and shell aliases need no --config.
|
|
48
|
+
"""
|
|
49
|
+
candidates = [Path.cwd() / "config.toml", user_config()]
|
|
50
|
+
|
|
51
|
+
# Only a checkout has a sibling pyproject.toml; for an installed package
|
|
52
|
+
# CHECKOUT_ROOT is site-packages, which would be nonsense to suggest.
|
|
53
|
+
if (CHECKOUT_ROOT / "pyproject.toml").is_file():
|
|
54
|
+
candidates.append(CHECKOUT_ROOT / "config.toml")
|
|
55
|
+
|
|
56
|
+
return list(dict.fromkeys(candidates))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def example_config() -> Path | None:
|
|
60
|
+
"""The bundled template: beside the package, or inside it once installed."""
|
|
61
|
+
for candidate in (
|
|
62
|
+
CHECKOUT_ROOT / "config.example.toml",
|
|
63
|
+
PACKAGE_DIR / "config.example.toml",
|
|
64
|
+
):
|
|
65
|
+
if candidate.is_file():
|
|
66
|
+
return candidate
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def init_config(target: Path | None = None) -> Path:
|
|
71
|
+
"""Copy the bundled template to target, without ever clobbering a config."""
|
|
72
|
+
target = (target or user_config()).expanduser()
|
|
73
|
+
if target.exists():
|
|
74
|
+
raise ConfigError(f"{target} already exists, leaving it alone")
|
|
75
|
+
|
|
76
|
+
example = example_config()
|
|
77
|
+
if example is None:
|
|
78
|
+
raise ConfigError("this installation is missing its config.example.toml")
|
|
79
|
+
|
|
80
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
81
|
+
shutil.copyfile(example, target)
|
|
82
|
+
return target
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass(frozen=True)
|
|
86
|
+
class Config:
|
|
87
|
+
airmate_id: str
|
|
88
|
+
serial: str
|
|
89
|
+
base_url: str
|
|
90
|
+
cycle: str
|
|
91
|
+
plates: tuple[str, ...]
|
|
92
|
+
raster: tuple[str, ...]
|
|
93
|
+
download_dir: Path
|
|
94
|
+
prepared_dir: Path
|
|
95
|
+
usb_targets: tuple[Path, ...]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def load(path: Path | None = None) -> Config:
|
|
99
|
+
"""Read a config file, apply environment overrides and validate it.
|
|
100
|
+
|
|
101
|
+
Without an explicit path, the first file in search_path() wins.
|
|
102
|
+
"""
|
|
103
|
+
if path is None:
|
|
104
|
+
searched = search_path()
|
|
105
|
+
path = next((candidate for candidate in searched if candidate.is_file()), None)
|
|
106
|
+
if path is None:
|
|
107
|
+
locations = "".join(f" {candidate}\n" for candidate in searched)
|
|
108
|
+
raise ConfigError(
|
|
109
|
+
"no configuration file found, looked in:\n" + locations
|
|
110
|
+
+ " create one with: navdata-update --init-config"
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
path = path.expanduser()
|
|
114
|
+
if not path.is_file():
|
|
115
|
+
raise ConfigError(f"no config file at {path}")
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
with path.open("rb") as fh:
|
|
119
|
+
raw = tomllib.load(fh)
|
|
120
|
+
except tomllib.TOMLDecodeError as exc:
|
|
121
|
+
raise ConfigError(f"{path} is not valid TOML: {exc}") from exc
|
|
122
|
+
|
|
123
|
+
airmate = _section(raw, path, "airmate")
|
|
124
|
+
data = _section(raw, path, "data")
|
|
125
|
+
paths = _section(raw, path, "paths")
|
|
126
|
+
|
|
127
|
+
# The environment wins so the two identifying values can be kept in a
|
|
128
|
+
# password manager or a shell profile instead of on disk.
|
|
129
|
+
airmate_id = os.environ.get(ID_ENV) or _string(airmate, path, "airmate", "id")
|
|
130
|
+
serial = os.environ.get(SERIAL_ENV) or _string(airmate, path, "airmate", "serial")
|
|
131
|
+
|
|
132
|
+
for name, value, env in (("id", airmate_id, ID_ENV), ("serial", serial, SERIAL_ENV)):
|
|
133
|
+
if value in PLACEHOLDERS:
|
|
134
|
+
raise ConfigError(
|
|
135
|
+
f"airmate.{name} is still the example placeholder {value!r}; "
|
|
136
|
+
f"set it in {path} or export {env}"
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
# Relative paths follow the config file, so the script behaves the same from
|
|
140
|
+
# any working directory (cron, a shell alias, the repo itself).
|
|
141
|
+
anchor = path.parent
|
|
142
|
+
|
|
143
|
+
return Config(
|
|
144
|
+
airmate_id=airmate_id,
|
|
145
|
+
serial=serial,
|
|
146
|
+
base_url=_string(airmate, path, "airmate", "base_url").rstrip("/"),
|
|
147
|
+
cycle=_string(data, path, "data", "cycle"),
|
|
148
|
+
plates=_string_list(data, path, "data", "plates"),
|
|
149
|
+
raster=_string_list(data, path, "data", "raster"),
|
|
150
|
+
download_dir=_path(paths, path, "paths", "download_dir", anchor),
|
|
151
|
+
prepared_dir=_path(paths, path, "paths", "prepared_dir", anchor),
|
|
152
|
+
usb_targets=_usb_targets(paths, path),
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _section(raw: dict, path: Path, name: str) -> dict:
|
|
157
|
+
section = raw.get(name)
|
|
158
|
+
if not isinstance(section, dict):
|
|
159
|
+
raise ConfigError(f"{path} has no [{name}] section")
|
|
160
|
+
return section
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _string(section: dict, path: Path, name: str, key: str) -> str:
|
|
164
|
+
value = section.get(key)
|
|
165
|
+
if not isinstance(value, str) or not value:
|
|
166
|
+
raise ConfigError(f"{path} is missing a non-empty {name}.{key}")
|
|
167
|
+
return value
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _string_list(section: dict, path: Path, name: str, key: str) -> tuple[str, ...]:
|
|
171
|
+
value = section.get(key, [])
|
|
172
|
+
if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
|
|
173
|
+
raise ConfigError(f"{name}.{key} in {path} must be a list of strings")
|
|
174
|
+
return tuple(value)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _path(section: dict, path: Path, name: str, key: str, anchor: Path) -> Path:
|
|
178
|
+
value = Path(_string(section, path, name, key)).expanduser()
|
|
179
|
+
return value if value.is_absolute() else anchor / value
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _usb_targets(paths: dict, config_path: Path) -> tuple[Path, ...]:
|
|
183
|
+
"""Accept paths.usb_target (string or list) and/or paths.usb_targets (list)."""
|
|
184
|
+
collected: list[str] = []
|
|
185
|
+
for key in ("usb_target", "usb_targets"):
|
|
186
|
+
if key not in paths:
|
|
187
|
+
continue
|
|
188
|
+
value = paths[key]
|
|
189
|
+
if value in (None, "", []):
|
|
190
|
+
continue
|
|
191
|
+
if isinstance(value, str):
|
|
192
|
+
collected.append(value)
|
|
193
|
+
elif isinstance(value, list) and all(isinstance(item, str) for item in value):
|
|
194
|
+
collected.extend(value)
|
|
195
|
+
else:
|
|
196
|
+
raise ConfigError(
|
|
197
|
+
f"paths.{key} in {config_path} must be a string or a list of strings"
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
resolved: list[Path] = []
|
|
201
|
+
seen: set[Path] = set()
|
|
202
|
+
for raw in collected:
|
|
203
|
+
if not raw or raw in PLACEHOLDERS:
|
|
204
|
+
continue
|
|
205
|
+
target = Path(raw).expanduser()
|
|
206
|
+
if target in seen:
|
|
207
|
+
continue
|
|
208
|
+
seen.add(target)
|
|
209
|
+
resolved.append(target)
|
|
210
|
+
return tuple(resolved)
|
navdata_sync/download.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Resumable, parallel downloads behind a single global progress bar."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
from tqdm import tqdm
|
|
10
|
+
|
|
11
|
+
from .catalog import RemoteFile
|
|
12
|
+
|
|
13
|
+
CHUNK_SIZE = 1024 * 1024
|
|
14
|
+
MAX_CONNECTIONS = 4
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def human_size(n: float) -> str:
|
|
18
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
19
|
+
if n < 1024 or unit == "TB":
|
|
20
|
+
return f"{n:.1f} {unit}"
|
|
21
|
+
n /= 1024
|
|
22
|
+
raise AssertionError("unreachable")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class _Job:
|
|
27
|
+
"""A file to fetch, with what is already on disk resolved up front."""
|
|
28
|
+
|
|
29
|
+
file: RemoteFile
|
|
30
|
+
dest: Path
|
|
31
|
+
remote_size: int # 0 when the server does not advertise a content-length
|
|
32
|
+
local_size: int
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def complete(self) -> bool:
|
|
36
|
+
return bool(self.remote_size) and self.local_size >= self.remote_size
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
async def run(files: list[RemoteFile], download_dir: Path) -> None:
|
|
40
|
+
download_dir.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
limits = httpx.Limits(max_connections=MAX_CONNECTIONS)
|
|
42
|
+
|
|
43
|
+
async with httpx.AsyncClient(http2=True, timeout=None, limits=limits) as client:
|
|
44
|
+
jobs = await _plan(client, files, download_dir)
|
|
45
|
+
|
|
46
|
+
progress = tqdm(
|
|
47
|
+
total=sum(job.remote_size for job in jobs),
|
|
48
|
+
initial=sum(job.local_size for job in jobs),
|
|
49
|
+
unit="B",
|
|
50
|
+
unit_scale=True,
|
|
51
|
+
desc="Téléchargement global",
|
|
52
|
+
)
|
|
53
|
+
await asyncio.gather(*(_fetch(client, job, progress) for job in jobs))
|
|
54
|
+
progress.close()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def _plan(
|
|
58
|
+
client: httpx.AsyncClient, files: list[RemoteFile], download_dir: Path
|
|
59
|
+
) -> list[_Job]:
|
|
60
|
+
sizes = await asyncio.gather(*(_remote_size(client, f.url) for f in files))
|
|
61
|
+
|
|
62
|
+
jobs = []
|
|
63
|
+
for file, remote_size in zip(files, sizes):
|
|
64
|
+
dest = download_dir / file.name
|
|
65
|
+
if file.always_refresh:
|
|
66
|
+
dest.unlink(missing_ok=True)
|
|
67
|
+
local_size = dest.stat().st_size if dest.exists() else 0
|
|
68
|
+
jobs.append(_Job(file, dest, remote_size, local_size))
|
|
69
|
+
return jobs
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
async def _remote_size(client: httpx.AsyncClient, url: str) -> int:
|
|
73
|
+
try:
|
|
74
|
+
response = await client.head(url)
|
|
75
|
+
return int(response.headers.get("content-length", 0))
|
|
76
|
+
except (httpx.HTTPError, ValueError):
|
|
77
|
+
return 0
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
async def _fetch(client: httpx.AsyncClient, job: _Job, progress: tqdm) -> None:
|
|
81
|
+
if job.complete:
|
|
82
|
+
tqdm.write(f"⏭️ {job.dest.name} déjà complet ({human_size(job.local_size)})")
|
|
83
|
+
return
|
|
84
|
+
|
|
85
|
+
job.dest.parent.mkdir(parents=True, exist_ok=True)
|
|
86
|
+
headers = {}
|
|
87
|
+
|
|
88
|
+
if job.local_size:
|
|
89
|
+
headers["Range"] = f"bytes={job.local_size}-"
|
|
90
|
+
remaining = human_size(job.remote_size - job.local_size) if job.remote_size else "?"
|
|
91
|
+
tqdm.write(
|
|
92
|
+
f"↩️ Reprise {job.dest.name} "
|
|
93
|
+
f"({human_size(job.local_size)}/{human_size(job.remote_size)}, "
|
|
94
|
+
f"reste {remaining})"
|
|
95
|
+
)
|
|
96
|
+
else:
|
|
97
|
+
tqdm.write(f"⬇️ Téléchargement {job.dest.name} ({human_size(job.remote_size)})")
|
|
98
|
+
|
|
99
|
+
try:
|
|
100
|
+
resumed = await _stream(client, job, headers, progress)
|
|
101
|
+
|
|
102
|
+
# A 416 means the server disagrees with our idea of the file length, so
|
|
103
|
+
# the partial file is unusable: drop it, un-count its bytes and restart.
|
|
104
|
+
if not resumed:
|
|
105
|
+
tqdm.write(f"⚠️ {job.dest.name} mismatch → re-téléchargement complet")
|
|
106
|
+
job.dest.unlink(missing_ok=True)
|
|
107
|
+
progress.update(-job.local_size)
|
|
108
|
+
job.local_size = 0
|
|
109
|
+
await _stream(client, job, {}, progress)
|
|
110
|
+
|
|
111
|
+
tqdm.write(f"✅ {job.dest.name} terminé ({human_size(job.dest.stat().st_size)})")
|
|
112
|
+
|
|
113
|
+
except httpx.HTTPError as exc:
|
|
114
|
+
tqdm.write(f"❌ Erreur téléchargement {job.dest.name}: {exc}")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
async def _stream(
|
|
118
|
+
client: httpx.AsyncClient, job: _Job, headers: dict[str, str], progress: tqdm
|
|
119
|
+
) -> bool:
|
|
120
|
+
"""Append (or write) the response body to disk. False on a 416 rejection."""
|
|
121
|
+
async with client.stream("GET", job.file.url, headers=headers) as response:
|
|
122
|
+
if response.status_code == httpx.codes.REQUESTED_RANGE_NOT_SATISFIABLE:
|
|
123
|
+
return False
|
|
124
|
+
response.raise_for_status()
|
|
125
|
+
|
|
126
|
+
with open(job.dest, "ab" if headers.get("Range") else "wb") as fh:
|
|
127
|
+
async for chunk in response.aiter_bytes(CHUNK_SIZE):
|
|
128
|
+
fh.write(chunk)
|
|
129
|
+
progress.update(len(chunk))
|
|
130
|
+
return True
|
navdata_sync/prepare.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Staging of the downloaded files into the layout the Dynon expects.
|
|
2
|
+
|
|
3
|
+
The root of the USB stick looks like this:
|
|
4
|
+
|
|
5
|
+
AIRMATE_AV_DATA_EU_<cycle>_<serial>.DUP
|
|
6
|
+
AIRMATE_OBSTACLE_DATA_EU_<cycle>_<serial>.DUP
|
|
7
|
+
CHARTS-<serial>.key
|
|
8
|
+
ChartData/Plates/...
|
|
9
|
+
Raster/VFR-*.dcf
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import shutil
|
|
14
|
+
import zipfile
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from .catalog import Kind, RemoteFile
|
|
18
|
+
from .usb import describe_sync, sync
|
|
19
|
+
|
|
20
|
+
__all__ = ["build", "describe_sync", "sync"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def build(files: list[RemoteFile], download_dir: Path, prepared_dir: Path) -> list[str]:
|
|
24
|
+
"""Lay the downloads out under prepared_dir. Returns the names left out."""
|
|
25
|
+
print("📦 Construction du dossier final...")
|
|
26
|
+
|
|
27
|
+
if prepared_dir.exists():
|
|
28
|
+
shutil.rmtree(prepared_dir)
|
|
29
|
+
prepared_dir.mkdir(parents=True)
|
|
30
|
+
|
|
31
|
+
missing = []
|
|
32
|
+
for file in sorted(files, key=lambda f: f.name):
|
|
33
|
+
source = download_dir / file.name
|
|
34
|
+
if not source.is_file():
|
|
35
|
+
missing.append(file.name)
|
|
36
|
+
continue
|
|
37
|
+
|
|
38
|
+
match file.kind:
|
|
39
|
+
case Kind.DATA:
|
|
40
|
+
shutil.copyfile(source, prepared_dir / file.name.upper())
|
|
41
|
+
case Kind.KEY:
|
|
42
|
+
shutil.copyfile(source, prepared_dir / file.name)
|
|
43
|
+
case Kind.PLATES:
|
|
44
|
+
print(f"📂 Extraction {file.name}")
|
|
45
|
+
with zipfile.ZipFile(source) as archive:
|
|
46
|
+
archive.extractall(prepared_dir)
|
|
47
|
+
case Kind.RASTER:
|
|
48
|
+
raster_dir = prepared_dir / "Raster"
|
|
49
|
+
raster_dir.mkdir(exist_ok=True)
|
|
50
|
+
shutil.copyfile(source, raster_dir / file.name)
|
|
51
|
+
|
|
52
|
+
for name in missing:
|
|
53
|
+
print(f"⚠️ {name} absent du dossier de téléchargement, ignoré")
|
|
54
|
+
|
|
55
|
+
return missing
|
navdata_sync/usb.py
ADDED
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""Copy the staging folder onto one or more Dynon USB sticks.
|
|
2
|
+
|
|
3
|
+
rsync's default size+mtime check would recopy everything because the staging
|
|
4
|
+
folder is rebuilt from scratch on every run (fresh mtimes). The delta algorithm
|
|
5
|
+
(--no-whole-file) is also a poor fit for flash: it turns a changed file into
|
|
6
|
+
scattered block rewrites.
|
|
7
|
+
|
|
8
|
+
Instead we:
|
|
9
|
+
|
|
10
|
+
* decide by content (blake2b), so only files whose bytes actually changed
|
|
11
|
+
are written
|
|
12
|
+
* write changed files in full, sequentially, in large chunks — flash likes
|
|
13
|
+
that much more than random in-place deltas, and opening the dest with
|
|
14
|
+
"wb" is still in-place (no temp copy), which matters on a nearly-full
|
|
15
|
+
FAT-32 stick
|
|
16
|
+
* drop dest files that are no longer in the staging tree
|
|
17
|
+
* run one sequential writer per stick, in parallel, so LH and RH update
|
|
18
|
+
at the same time without randomizing I/O on either device
|
|
19
|
+
|
|
20
|
+
aiofile (caio thread backend) is used because vfat/USB does not reliably
|
|
21
|
+
support kernel AIO / O_DIRECT; the thread pool still lets the two sticks
|
|
22
|
+
overlap.
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import asyncio
|
|
27
|
+
import hashlib
|
|
28
|
+
import os
|
|
29
|
+
from collections.abc import Callable
|
|
30
|
+
from contextlib import AsyncExitStack
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
from aiofile import async_open
|
|
35
|
+
from caio.thread_aio_asyncio import AsyncioContext
|
|
36
|
+
from rich.progress import (
|
|
37
|
+
BarColumn,
|
|
38
|
+
DownloadColumn,
|
|
39
|
+
Progress,
|
|
40
|
+
TaskID,
|
|
41
|
+
TextColumn,
|
|
42
|
+
TimeRemainingColumn,
|
|
43
|
+
TransferSpeedColumn,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# 4 MiB sequential transfers: large enough to keep a USB controller busy,
|
|
47
|
+
# small enough to report progress and overlap two destinations.
|
|
48
|
+
CHUNK_SIZE = 4 * 1024 * 1024
|
|
49
|
+
HASH_NAME = "blake2b"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class _FileInfo:
|
|
54
|
+
rel: Path
|
|
55
|
+
size: int
|
|
56
|
+
digest: bytes
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class SyncReport:
|
|
61
|
+
target: Path
|
|
62
|
+
copied: int = 0
|
|
63
|
+
skipped: int = 0
|
|
64
|
+
deleted: int = 0
|
|
65
|
+
error: str | None = None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def relative_files(root: Path) -> list[Path]:
|
|
69
|
+
"""Files under root, as paths relative to it, in a stable order."""
|
|
70
|
+
if not root.is_dir():
|
|
71
|
+
return []
|
|
72
|
+
return sorted(
|
|
73
|
+
path.relative_to(root)
|
|
74
|
+
for path in root.rglob("*")
|
|
75
|
+
if path.is_file() and not path.is_symlink()
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def prune_extras(target: Path, wanted: set[Path]) -> list[Path]:
|
|
80
|
+
"""Delete dest files (and then empty dirs) that are not in the staging tree."""
|
|
81
|
+
extras = [rel for rel in relative_files(target) if rel not in wanted]
|
|
82
|
+
for rel in extras:
|
|
83
|
+
(target / rel).unlink(missing_ok=True)
|
|
84
|
+
for directory in sorted(
|
|
85
|
+
(path for path in target.rglob("*") if path.is_dir()),
|
|
86
|
+
key=lambda path: len(path.parts),
|
|
87
|
+
reverse=True,
|
|
88
|
+
):
|
|
89
|
+
try:
|
|
90
|
+
directory.rmdir()
|
|
91
|
+
except OSError:
|
|
92
|
+
pass
|
|
93
|
+
return extras
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def describe_sync(prepared_dir: Path, usb_targets: tuple[Path, ...]) -> str:
|
|
97
|
+
if not usb_targets:
|
|
98
|
+
return (
|
|
99
|
+
f"✅ Dossier prêt: {prepared_dir}\n"
|
|
100
|
+
" Renseignez paths.usb_targets dans config.toml pour la synchronisation."
|
|
101
|
+
)
|
|
102
|
+
lines = [f"✅ Dossier prêt: {prepared_dir}", " Cibles USB:"]
|
|
103
|
+
for target in usb_targets:
|
|
104
|
+
status = "montée" if target.is_dir() else "absente"
|
|
105
|
+
lines.append(f" • {target} ({status})")
|
|
106
|
+
lines.append(
|
|
107
|
+
" Relancez avec --sync pour copier "
|
|
108
|
+
"(fichiers identiques ignorés, sticks en parallèle)."
|
|
109
|
+
)
|
|
110
|
+
return "\n".join(lines)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def sync(prepared_dir: Path, usb_targets: tuple[Path, ...]) -> int:
|
|
114
|
+
"""Copy the staging folder onto every configured stick. Returns an exit code."""
|
|
115
|
+
return asyncio.run(_sync(prepared_dir, usb_targets))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
async def _sync(prepared_dir: Path, usb_targets: tuple[Path, ...]) -> int:
|
|
119
|
+
if not usb_targets:
|
|
120
|
+
print("❌ aucune cible USB dans la configuration")
|
|
121
|
+
return 1
|
|
122
|
+
|
|
123
|
+
mounted = [target for target in usb_targets if target.is_dir()]
|
|
124
|
+
missing = [target for target in usb_targets if not target.is_dir()]
|
|
125
|
+
for target in missing:
|
|
126
|
+
print(f"❌ {target} introuvable: la clé USB est-elle montée ?")
|
|
127
|
+
if not mounted:
|
|
128
|
+
return 1
|
|
129
|
+
|
|
130
|
+
source_files = relative_files(prepared_dir)
|
|
131
|
+
if not source_files:
|
|
132
|
+
print(f"❌ {prepared_dir} est vide, rien à copier")
|
|
133
|
+
return 1
|
|
134
|
+
|
|
135
|
+
async with AsyncioContext(max_requests=32) as ctx:
|
|
136
|
+
manifest = await _index_source(prepared_dir, source_files, ctx)
|
|
137
|
+
source_total = sum(info.size for info in manifest)
|
|
138
|
+
with _usb_progress() as progress:
|
|
139
|
+
task_ids = [
|
|
140
|
+
progress.add_task(
|
|
141
|
+
description=target.name or str(target),
|
|
142
|
+
total=max(source_total, 1),
|
|
143
|
+
label=target.name or str(target),
|
|
144
|
+
phase="Vérif.",
|
|
145
|
+
current="",
|
|
146
|
+
)
|
|
147
|
+
for target in mounted
|
|
148
|
+
]
|
|
149
|
+
reports = await asyncio.gather(*(
|
|
150
|
+
_sync_target(
|
|
151
|
+
prepared_dir, manifest, target, ctx, progress, task_id
|
|
152
|
+
)
|
|
153
|
+
for target, task_id in zip(mounted, task_ids)
|
|
154
|
+
))
|
|
155
|
+
print("💾 Vidage des caches disque...")
|
|
156
|
+
os.sync()
|
|
157
|
+
|
|
158
|
+
failed = False
|
|
159
|
+
for report in reports:
|
|
160
|
+
if report.error:
|
|
161
|
+
print(
|
|
162
|
+
f"❌ {report.target}: {report.error} "
|
|
163
|
+
f"({report.copied} copiés, {report.skipped} inchangés, "
|
|
164
|
+
f"{report.deleted} supprimés)"
|
|
165
|
+
)
|
|
166
|
+
failed = True
|
|
167
|
+
else:
|
|
168
|
+
print(
|
|
169
|
+
f"✅ {report.target}: "
|
|
170
|
+
f"{report.copied} copiés, {report.skipped} inchangés, "
|
|
171
|
+
f"{report.deleted} supprimés"
|
|
172
|
+
)
|
|
173
|
+
if missing:
|
|
174
|
+
failed = True
|
|
175
|
+
return 1 if failed else 0
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _usb_progress() -> Progress:
|
|
179
|
+
"""One live bar per USB stick (or a single bar for the source checksum)."""
|
|
180
|
+
return Progress(
|
|
181
|
+
TextColumn("[bold cyan]{task.fields[label]:<12}"),
|
|
182
|
+
TextColumn("{task.fields[phase]:<8}"),
|
|
183
|
+
BarColumn(bar_width=None),
|
|
184
|
+
DownloadColumn(),
|
|
185
|
+
TransferSpeedColumn(),
|
|
186
|
+
TimeRemainingColumn(),
|
|
187
|
+
TextColumn("{task.fields[current]}", style="dim", markup=False),
|
|
188
|
+
expand=True,
|
|
189
|
+
refresh_per_second=8,
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _advance(progress: Progress, task_id: TaskID, n: int) -> None:
|
|
194
|
+
progress.update(task_id, advance=n)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _finish(
|
|
198
|
+
progress: Progress, task_id: TaskID, phase: str, current: str = ""
|
|
199
|
+
) -> None:
|
|
200
|
+
total = 1.0
|
|
201
|
+
for task in progress.tasks:
|
|
202
|
+
if task.id == task_id:
|
|
203
|
+
total = task.total or 1
|
|
204
|
+
break
|
|
205
|
+
progress.update(task_id, phase=phase, current=current, completed=total)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
async def _index_source(
|
|
209
|
+
prepared_dir: Path, files: list[Path], ctx: AsyncioContext
|
|
210
|
+
) -> list[_FileInfo]:
|
|
211
|
+
total = sum((prepared_dir / rel).stat().st_size for rel in files)
|
|
212
|
+
with _usb_progress() as progress:
|
|
213
|
+
task_id = progress.add_task(
|
|
214
|
+
"Source",
|
|
215
|
+
total=max(total, 1),
|
|
216
|
+
label="Source",
|
|
217
|
+
phase="Hash",
|
|
218
|
+
current="",
|
|
219
|
+
)
|
|
220
|
+
manifest: list[_FileInfo] = []
|
|
221
|
+
for rel in files:
|
|
222
|
+
path = prepared_dir / rel
|
|
223
|
+
size = path.stat().st_size
|
|
224
|
+
progress.update(task_id, current=str(rel), phase="Hash")
|
|
225
|
+
digest = await _digest(path, ctx, lambda n: _advance(progress, task_id, n))
|
|
226
|
+
manifest.append(_FileInfo(rel, size, digest))
|
|
227
|
+
_finish(progress, task_id, "OK")
|
|
228
|
+
return manifest
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
async def _sync_target(
|
|
232
|
+
prepared_dir: Path,
|
|
233
|
+
manifest: list[_FileInfo],
|
|
234
|
+
target: Path,
|
|
235
|
+
ctx: AsyncioContext,
|
|
236
|
+
progress: Progress,
|
|
237
|
+
task_id: TaskID,
|
|
238
|
+
) -> SyncReport:
|
|
239
|
+
report = SyncReport(target)
|
|
240
|
+
label = target.name or str(target)
|
|
241
|
+
wanted = {info.rel for info in manifest}
|
|
242
|
+
|
|
243
|
+
try:
|
|
244
|
+
extras = prune_extras(target, wanted)
|
|
245
|
+
report.deleted = len(extras)
|
|
246
|
+
for rel in extras:
|
|
247
|
+
progress.log(f"🗑️ [{label}] {rel}")
|
|
248
|
+
|
|
249
|
+
to_copy: list[_FileInfo] = []
|
|
250
|
+
for info in manifest:
|
|
251
|
+
dest = target / info.rel
|
|
252
|
+
progress.update(task_id, phase="Vérif.", current=str(info.rel))
|
|
253
|
+
if dest.is_file() and dest.stat().st_size == info.size:
|
|
254
|
+
digest = await _digest(
|
|
255
|
+
dest, ctx, lambda n: _advance(progress, task_id, n)
|
|
256
|
+
)
|
|
257
|
+
if digest == info.digest:
|
|
258
|
+
report.skipped += 1
|
|
259
|
+
continue
|
|
260
|
+
else:
|
|
261
|
+
progress.update(task_id, advance=info.size)
|
|
262
|
+
to_copy.append(info)
|
|
263
|
+
|
|
264
|
+
if not to_copy:
|
|
265
|
+
_finish(progress, task_id, "OK")
|
|
266
|
+
return report
|
|
267
|
+
|
|
268
|
+
copy_total = sum(info.size for info in to_copy)
|
|
269
|
+
progress.reset(task_id, total=max(copy_total, 1))
|
|
270
|
+
progress.update(task_id, phase="Copie", current="")
|
|
271
|
+
for info in to_copy:
|
|
272
|
+
progress.update(task_id, phase="Copie", current=str(info.rel))
|
|
273
|
+
await _copy_file(
|
|
274
|
+
prepared_dir / info.rel,
|
|
275
|
+
target / info.rel,
|
|
276
|
+
ctx,
|
|
277
|
+
lambda n: _advance(progress, task_id, n),
|
|
278
|
+
)
|
|
279
|
+
report.copied += 1
|
|
280
|
+
_finish(progress, task_id, "OK")
|
|
281
|
+
except OSError as exc:
|
|
282
|
+
report.error = str(exc)
|
|
283
|
+
_finish(progress, task_id, "Erreur", str(exc))
|
|
284
|
+
return report
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
async def _digest(
|
|
288
|
+
path: Path,
|
|
289
|
+
ctx: AsyncioContext,
|
|
290
|
+
on_bytes: Callable[[int], None] | None = None,
|
|
291
|
+
) -> bytes:
|
|
292
|
+
digest = hashlib.new(HASH_NAME)
|
|
293
|
+
async with async_open(path, "rb", context=ctx) as fh:
|
|
294
|
+
async for chunk in fh.iter_chunked(CHUNK_SIZE):
|
|
295
|
+
digest.update(chunk)
|
|
296
|
+
if on_bytes is not None:
|
|
297
|
+
on_bytes(len(chunk))
|
|
298
|
+
return digest.digest()
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
async def _copy_file(
|
|
302
|
+
source: Path,
|
|
303
|
+
dest: Path,
|
|
304
|
+
ctx: AsyncioContext,
|
|
305
|
+
on_bytes: Callable[[int], None],
|
|
306
|
+
) -> None:
|
|
307
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
308
|
+
async with AsyncExitStack() as stack:
|
|
309
|
+
src = await stack.enter_async_context(async_open(source, "rb", context=ctx))
|
|
310
|
+
out = await stack.enter_async_context(async_open(dest, "wb", context=ctx))
|
|
311
|
+
async for chunk in src.iter_chunked(CHUNK_SIZE):
|
|
312
|
+
await out.write(chunk)
|
|
313
|
+
on_bytes(len(chunk))
|
|
314
|
+
await out.flush(sync_metadata=True)
|