blackvuesync 2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: blackvuesync
|
|
3
|
+
Version: 2.0
|
|
4
|
+
Summary: Synchronize recordings from BlackVue dashcams to a local directory
|
|
5
|
+
Author-email: Alessandro Colomba <angelo@colomba.org>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/acolomba/blackvuesync
|
|
8
|
+
Project-URL: Repository, https://github.com/acolomba/blackvuesync
|
|
9
|
+
Project-URL: Issues, https://github.com/acolomba/blackvuesync/issues
|
|
10
|
+
Keywords: blackvue,dashcam,sync,backup
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Topic :: System :: Archiving :: Backup
|
|
19
|
+
Classifier: Topic :: Utilities
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: COPYING
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: behave~=1.2; extra == "dev"
|
|
26
|
+
Requires-Dist: black~=25.11; extra == "dev"
|
|
27
|
+
Requires-Dist: Flask~=3.1; extra == "dev"
|
|
28
|
+
Requires-Dist: mypy~=1.14; extra == "dev"
|
|
29
|
+
Requires-Dist: pre-commit~=4.3; extra == "dev"
|
|
30
|
+
Requires-Dist: PyHamcrest~=2.1; extra == "dev"
|
|
31
|
+
Requires-Dist: pytest~=8.3; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest-cov~=7.0; extra == "dev"
|
|
33
|
+
Requires-Dist: python-lsp-server~=1.14; extra == "dev"
|
|
34
|
+
Requires-Dist: requests~=2.32; extra == "dev"
|
|
35
|
+
Requires-Dist: ruff~=0.9; extra == "dev"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# BlackVue Sync
|
|
39
|
+
|
|
40
|
+
[](https://github.com/acolomba/blackvuesync/actions/workflows/ci.yml)
|
|
41
|
+
[](https://github.com/acolomba/blackvuesync/actions/workflows/docker-build.yml)
|
|
42
|
+
[](https://sonarcloud.io/summary/overall?id=acolomba_blackvuesync)
|
|
43
|
+
[](https://sonarcloud.io/summary/overall?id=acolomba_blackvuesync)
|
|
44
|
+
[](https://sonarcloud.io/summary/overall?id=acolomba_blackvuesync)
|
|
45
|
+
[](https://sonarcloud.io/summary/overall?id=acolomba_blackvuesync)
|
|
46
|
+
[](https://sonarcloud.io/summary/overall?id=acolomba_blackvuesync)
|
|
47
|
+
[](https://sonarcloud.io/summary/overall?id=acolomba_blackvuesync)
|
|
48
|
+
[](https://sonarcloud.io/summary/overall?id=acolomba_blackvuesync)
|
|
49
|
+
|
|
50
|
+
Synchronizes recordings from a BlackVue dashcam with a local directory over a LAN.
|
|
51
|
+
|
|
52
|
+
BlackVue dashcams expose an HTTP server that can be used to download all recordings. This project downloads only recordings that are not already downloaded, optionally limiting downloads in a local directory to a date range.
|
|
53
|
+
|
|
54
|
+
A typical setup would be a periodic cron job or a Docker container running on a local server.
|
|
55
|
+
|
|
56
|
+
## Features
|
|
57
|
+
|
|
58
|
+
* **Portable runtimes:**
|
|
59
|
+
* A [single, self-contained Python script](https://github.com/acolomba/blackvuesync/blob/master/blackvuesync.py) with no third-party dependencies. It can be copied and run anywhere, either [manually](#manual-usage) or [periodically](#unattended-usage).
|
|
60
|
+
* A [docker image](#docker) that runs periodically via an internal cron job. Supports amd64 (Intel), arm64 (Apple Silicon, Raspberry Pi 4+) and armv7 (Raspberry Pi 2/3).
|
|
61
|
+
* **Smart**: Only downloads recordings that haven't already been downloaded.
|
|
62
|
+
* **Resilient**: If a download interrupts for whatever reason, the script resumes where it left off the next time it runs. This is especially useful for possibly unreliable Wi-Fi connections from a garage.
|
|
63
|
+
* **Hands-off**: Optionally retains recordings for a set amount of time. Outdated recordings are automatically removed.
|
|
64
|
+
* **Cron-friendly**: Only one process is allowed to run at any given time for a specific download destination.
|
|
65
|
+
* **Safe**: Stops executing if the destination disk is almost full.
|
|
66
|
+
* **Friendly error reporting**: Communicates a range of known error conditions with sensible verbosity.
|
|
67
|
+
|
|
68
|
+
## Prerequisites
|
|
69
|
+
|
|
70
|
+
### Software
|
|
71
|
+
|
|
72
|
+
* [Python](https://www.python.org/) 3.9+ or [Docker](https://docs.docker.com/).
|
|
73
|
+
* Sufficient disk space on a file system local to the script. Plan for about 5GB/hr per camera.
|
|
74
|
+
* [BlackVue Viewer](https://blackvue.com/kr/download/) or a media player to view the recordings.
|
|
75
|
+
|
|
76
|
+
### Hardware
|
|
77
|
+
|
|
78
|
+
A cloud-enabled [BlackVue](https://www.blackvue.com/) dashcam must be connected via Wi-Fi to the local network with a *static* IP address.
|
|
79
|
+
|
|
80
|
+
The dashcam must be kept powered for some time after the vehicle is turned off. BlackVue offers [hardwiring kits](https://blackvue.com/product-tag/battery/) and [batteries](https://blackvue.com/product-tag/battery/).
|
|
81
|
+
|
|
82
|
+
The camera should stay active for a period sufficiently long for recordings to be downloaded. Consult the dashcam manual for the bit rate for your chosen image quality, and compare it with the download speed reported by BlackVue Sync.
|
|
83
|
+
|
|
84
|
+
Example with a DR750S-2CH recording with two cameras at the highest quality setting and a good but conservative download speed:
|
|
85
|
+
|
|
86
|
+
```calca
|
|
87
|
+
# dashcam bitrates
|
|
88
|
+
dashcam_bitrate_front = 12Mbps
|
|
89
|
+
dashcam_bitrate_back = 10Mbps
|
|
90
|
+
dashcam_bitrate = dashcam_bitrate_front + dashcam_bitrate_back
|
|
91
|
+
|
|
92
|
+
download_speed = 20Mbps
|
|
93
|
+
|
|
94
|
+
# hours on the timer for every hour of recording
|
|
95
|
+
ratio = dashcam_bitrate / download_speed => 1.1
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### Verifying Connectivity
|
|
99
|
+
|
|
100
|
+
For illustration purposes, all examples assume that the camera is reachable at the `dashcam.example.net` address. A static numeric IP address works just as well.
|
|
101
|
+
|
|
102
|
+
A quick way to verify that the dashcam is online is by using `curl`.
|
|
103
|
+
|
|
104
|
+
```sh
|
|
105
|
+
$ curl http://dashcam.example.net/blackvue_vod.cgi
|
|
106
|
+
v:1.00
|
|
107
|
+
n:/Record/20181026_135003_PF.mp4,s:1000000
|
|
108
|
+
n:/Record/20181026_140658_PF.mp4,s:1000000
|
|
109
|
+
n:/Record/20181026_140953_PF.mp4,s:1000000
|
|
110
|
+
...
|
|
111
|
+
$
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Another way is by browsing to: `http://dashcam.example.net/blackvue_vod.cgi`.
|
|
115
|
+
|
|
116
|
+
## Usage
|
|
117
|
+
|
|
118
|
+
### Installation
|
|
119
|
+
|
|
120
|
+
BlackVue Sync is a single script, and can be obtained in a number of ways:
|
|
121
|
+
|
|
122
|
+
* **[uv](https://docs.astral.sh/uv/)**: Run with `uvx blackvuesync <args>`, or install with `uv tool install blackvuesync` and run with `blackvuesync <args>`.
|
|
123
|
+
* **[Pip](https://pypi.org/project/pip/):** Install with `pip install blackvuesync` and run with `blackvuesync <args>`.
|
|
124
|
+
* **Direct:** [Download from GitHub](https://raw.githubusercontent.com/acolomba/blackvuesync/refs/heads/main/blackvuesync.py), save to the desired location, and either run it with `python3 blackvuesync.py <args>`, or mark it executable and run it with `blackvuesync.py <args>`.
|
|
125
|
+
|
|
126
|
+
The interactive instructions assume a uv or Pip installation.
|
|
127
|
+
|
|
128
|
+
### Manual Usage
|
|
129
|
+
|
|
130
|
+
The dashcam address is the only required parameter. The `--dry-run` option makes it so that the script communicates what it would do without actually doing anything. Example:
|
|
131
|
+
|
|
132
|
+
```sh
|
|
133
|
+
blackvuesync dashcam.example.net --dry-run
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
It's also possible to specify a destination directory other than the current directory with `--destination`:
|
|
137
|
+
|
|
138
|
+
```sh
|
|
139
|
+
blackvuesync dashcam.example.net --destination /data/dashcam --dry-run
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
A retention period can be indicated with the `--keep` option. Recordings prior to the retention period will be removed from the destination directory. Accepted units are `d` for days and `w` for weeks. If no unit is indicated, days are assumed.
|
|
143
|
+
|
|
144
|
+
```sh
|
|
145
|
+
blackvuesync dashcam.example.net --destination /data/dashcam --keep 2w --dry-run
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
A typical invocation would be:
|
|
149
|
+
|
|
150
|
+
```sh
|
|
151
|
+
blackvuesync dashcam.example.net --destination /data/dashcam --keep 2w
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Other options:
|
|
155
|
+
|
|
156
|
+
* `--grouping`: Groups downloaded recordings in directories according to different schemes. Grouping speeds up loading recordings in the BlackVue Viewer app. The supported groupings are:
|
|
157
|
+
* `daily`: By day, e.g. 2018-10-26;
|
|
158
|
+
* `weekly`: By week, with the directory indicating the date of that week's monday, e.g. 2018-10-22;
|
|
159
|
+
* `monthly`: By month, e.g. 2018-10;
|
|
160
|
+
* `yearly`: By year, e.g. 2018;
|
|
161
|
+
* `none`: No grouping, the default.
|
|
162
|
+
* `--priority`: Downloads recordings with different priorities: `date` downloads oldest to newest; `rdate` downloads newest to oldest; `type` downloads manual, event (all types), normal and (non-event) parking recordings in that order. Defaults to `date`.
|
|
163
|
+
* `--max-used-disk`: Downloads stop once the specified used disk percentage threshold is reached. Defaults to `90` (i.e. 90%.)
|
|
164
|
+
* `--timeout`: Sets a timeout for establishing a connection to the dashcam, in seconds. Defaults to `10.0` seconds.
|
|
165
|
+
* `--quiet`: Quiets down output messages, except for unexpected errors. Takes precedence over `--verbose`.
|
|
166
|
+
* `--verbose`: Increases verbosity. Can be specified multiple times to indicate additional verbosity.
|
|
167
|
+
|
|
168
|
+
### Unattended Usage
|
|
169
|
+
|
|
170
|
+
#### Plain cron
|
|
171
|
+
|
|
172
|
+
The script can run periodically by setting up a [cron](https://en.wikipedia.org/wiki/Cron) job on UNIX systems.
|
|
173
|
+
|
|
174
|
+
Simple example with crontab for a hypothetical `media` user:
|
|
175
|
+
|
|
176
|
+
```crontab
|
|
177
|
+
*/15 * * * * /home/media/bin/blackvuesync.py dashcam.example.net --keep 2w --destination /data/dashcam --cron
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
The `--cron` option changes the logging level with the assumption that the output may be emailed. When this option is enabled, the script only produces logs when it downloads recordings and when it encounters unexpected errors. One would typically see an email only after driving or when something goes wrong.
|
|
181
|
+
|
|
182
|
+
Note that if the dashcam is unreachable for whatever reason, in `--cron` mode no output is generated, since this is an expected condition whenever the dashcam is away from the local network.
|
|
183
|
+
|
|
184
|
+
If cron jobs overlap, the script recognizes that another instance is currently running via a lock file on the destination directory. For the lock to work correctly, the destination directory must be on a local filesystem relative to the script.
|
|
185
|
+
|
|
186
|
+
#### NAS
|
|
187
|
+
|
|
188
|
+
Many NAS systems allow running commands periodically at set intervals.
|
|
189
|
+
|
|
190
|
+
##### openmediavault
|
|
191
|
+
|
|
192
|
+
The [openmediavault](http://www.openmediavault.org/) NAS solution allows running [scheduled jobs](https://openmediavault.readthedocs.io/en/latest/administration/general/cron.html) with support for mail notifications.
|
|
193
|
+
|
|
194
|
+
Example:
|
|
195
|
+
|
|
196
|
+

|
|
197
|
+
|
|
198
|
+
#### Docker
|
|
199
|
+
|
|
200
|
+
##### Overview
|
|
201
|
+
|
|
202
|
+
The [acolomba/blackvuesync](https://hub.docker.com/r/acolomba/blackvuesync/) docker image sets up a cron job internal to the container that runs the synchronization operation every 15 minutes.
|
|
203
|
+
|
|
204
|
+
##### Quick Start
|
|
205
|
+
|
|
206
|
+
It's a good idea to do a single, interactive dry run first with verbose logging:
|
|
207
|
+
|
|
208
|
+
```sh
|
|
209
|
+
docker run -it --rm \
|
|
210
|
+
-e ADDRESS=dashcam.example.net \
|
|
211
|
+
-v $PWD:/recordings \
|
|
212
|
+
-e DRY_RUN=1 \
|
|
213
|
+
-e VERBOSE=1 \
|
|
214
|
+
-e RUN_ONCE=1 \
|
|
215
|
+
--name blackvuesync \
|
|
216
|
+
acolomba/blackvuesync
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Once that works, a typical invocation would be similar to:
|
|
220
|
+
|
|
221
|
+
```sh
|
|
222
|
+
docker run -d --restart unless-stopped \
|
|
223
|
+
-v /data/dashcam:/recordings \
|
|
224
|
+
-e ADDRESS=dashcam.example.net \
|
|
225
|
+
-e PUID=$(id -u) \
|
|
226
|
+
-e PGID=$(id -g) \
|
|
227
|
+
-e TZ="America/New_York" \
|
|
228
|
+
-e KEEP=2w \
|
|
229
|
+
--name blackvuesync \
|
|
230
|
+
acolomba/blackvuesync
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
##### Docker Compose
|
|
234
|
+
|
|
235
|
+
[Docker Compose](https://docs.docker.com/compose/) may offer an easier, more repeatable and extensible option for running a BlackVueSync Docker container.
|
|
236
|
+
|
|
237
|
+
After downloading the Docker [Compose file](https://raw.githubusercontent.com/acolomba/blackvuesync/main/docker-compose.yml) and editing its values as desired, BlackVueSync can be started with:
|
|
238
|
+
|
|
239
|
+
```sh
|
|
240
|
+
docker-compose up -d
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
OR, depending on the Docker version:
|
|
244
|
+
|
|
245
|
+
```sh
|
|
246
|
+
docker compose up -d
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
##### Reference
|
|
250
|
+
|
|
251
|
+
These options are required for the docker image to operate correctly:
|
|
252
|
+
|
|
253
|
+
* The `/recordings` volume mapped to the desired destination of the downloaded recordings.
|
|
254
|
+
* The `ADDRESS` parameter set to the dashcam address.
|
|
255
|
+
* The `PUID` and `PGID` parameters set to the desired destination directory's user id and group id.
|
|
256
|
+
* The `TZ` parameter set to the same [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) as the dashcam. Note that BlackVue dashcams do not respect Daylight Savings Time, so their clock needs to be adjusted periodically.
|
|
257
|
+
|
|
258
|
+
Other parameters:
|
|
259
|
+
|
|
260
|
+
* `GROUPING`: Groups downloaded recordings in directories, `daily`, `weekly`, `monthly`, `yearly` and `none` are supported. (Default: `none`.)
|
|
261
|
+
* `KEEP`: Sets the retention period of downloaded recordings. Recordings prior to the retention period will be removed from the destination. Accepted units are `d` for days and `w` for weeks. If no unit is indicated, days are assumed. (Default: empty, meaning recordings are kept forever.)
|
|
262
|
+
* `PRIORITY`: Sets the priority to download recordings. Pick `date` to download from oldest to newest; pick `rdate` to download from newset to oldest; pick `type` to download manual, event (all types), normal and (non-event) parking recordings in that order. Defaults to `date`.
|
|
263
|
+
* `MAX_USED_DISK`: If set to a percentage value, stops downloading if the amount of used disk space exceeds the indicated percentage value. (Default: `90`, i.e. 90%.)
|
|
264
|
+
* `TIMEOUT`: If set to a float value, sets the timeout in seconds for connecting to the dashcam. (Default: `10.0` seconds.)
|
|
265
|
+
* `VERBOSE`: If set to a number greater than zero, increases logging verbosity. (Default: `0`.)
|
|
266
|
+
* `QUIET`: If set to any value, quiets down logs: only unexpected errors will be logged. (Default: empty.)
|
|
267
|
+
* `CRON`: Set by default, makes it so downloads of normal recordings and unexpected error conditions are logged. Can be set to `""` to disable.
|
|
268
|
+
* `DRY_RUN`: If set to any value, makes it so that the script communicates what it would do without actually doing anything. (Default: empty.)
|
|
269
|
+
* `RUN_ONCE`: If set to any value, the docker image runs the sync operation once and exits without setting up the cron job. (Default: empty. Not supported in Docker Compose.)
|
|
270
|
+
|
|
271
|
+
## License
|
|
272
|
+
|
|
273
|
+
This project is licensed under the MIT License - see the [COPYING](COPYING) file for details
|
|
274
|
+
|
|
275
|
+
Copyright 2018-2026 [Alessandro Colomba](https://github.com/acolomba)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
blackvuesync.py,sha256=zuqaM0efaDVDQsdcvLVonq0sPCDW0qN_D19glVJWp88,31505
|
|
2
|
+
blackvuesync-2.0.dist-info/licenses/COPYING,sha256=62rRn5EEnnhpc40RvkeFskxpUXBJa-hhexMdoPhXnl0,1088
|
|
3
|
+
blackvuesync-2.0.dist-info/METADATA,sha256=Y3AJt-nD0aB91HNC60VBoPfV_tTWFGYrg_MbqW3MImQ,14724
|
|
4
|
+
blackvuesync-2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
5
|
+
blackvuesync-2.0.dist-info/entry_points.txt,sha256=N20WR55b3HhumtX8LJDqn8KyoqZib7L6OjcGLBc5IZQ,51
|
|
6
|
+
blackvuesync-2.0.dist-info/top_level.txt,sha256=dCQPIgB4Zz9KRErjB7KU5x_CCq3j62_XKuKXM1VWapA,13
|
|
7
|
+
blackvuesync-2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
Copyright 2018 Alessandro Colomba (https://github.com/acolomba)
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
|
4
|
+
documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
|
|
5
|
+
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
|
|
6
|
+
persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
7
|
+
|
|
8
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
|
|
9
|
+
Software.
|
|
10
|
+
|
|
11
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
|
|
12
|
+
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
13
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
|
14
|
+
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
blackvuesync
|
blackvuesync.py
ADDED
|
@@ -0,0 +1,912 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Synchronizes recordings from a BlackVue dashcam with a local directory over a LAN.
|
|
4
|
+
https://github.com/acolomba/blackvuesync
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
# Copyright 2018-2026 Alessandro Colomba (https://github.com/acolomba)
|
|
10
|
+
#
|
|
11
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
|
|
12
|
+
# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
|
|
13
|
+
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
|
|
14
|
+
# persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
15
|
+
#
|
|
16
|
+
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
|
|
17
|
+
# Software.
|
|
18
|
+
#
|
|
19
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
|
|
20
|
+
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
21
|
+
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
|
22
|
+
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
23
|
+
|
|
24
|
+
__version__ = "2.0"
|
|
25
|
+
|
|
26
|
+
import argparse
|
|
27
|
+
import datetime
|
|
28
|
+
import errno
|
|
29
|
+
import fcntl
|
|
30
|
+
import glob
|
|
31
|
+
import http.client
|
|
32
|
+
import logging
|
|
33
|
+
import os
|
|
34
|
+
import re
|
|
35
|
+
import shutil
|
|
36
|
+
import socket
|
|
37
|
+
import stat
|
|
38
|
+
import sys
|
|
39
|
+
import time
|
|
40
|
+
import urllib
|
|
41
|
+
import urllib.parse
|
|
42
|
+
import urllib.request
|
|
43
|
+
from collections.abc import Callable
|
|
44
|
+
from dataclasses import dataclass
|
|
45
|
+
|
|
46
|
+
# logging
|
|
47
|
+
logging.basicConfig(format="%(asctime)s: %(levelname)s %(message)s")
|
|
48
|
+
|
|
49
|
+
# root logger
|
|
50
|
+
logger = logging.getLogger()
|
|
51
|
+
|
|
52
|
+
# cron logger (remains active in cron mode)
|
|
53
|
+
cron_logger = logging.getLogger("cron")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def set_logging_levels(verbosity: int, is_cron_mode: bool) -> None:
|
|
57
|
+
"""sets up the logging levels according to the desired verbosity and operation mode"""
|
|
58
|
+
if verbosity == -1:
|
|
59
|
+
logger.setLevel(logging.ERROR)
|
|
60
|
+
cron_logger.setLevel(logging.ERROR)
|
|
61
|
+
elif verbosity == 0:
|
|
62
|
+
logger.setLevel(logging.ERROR if is_cron_mode else logging.WARN)
|
|
63
|
+
cron_logger.setLevel(logging.INFO if is_cron_mode else logging.WARN)
|
|
64
|
+
elif verbosity == 1:
|
|
65
|
+
logger.setLevel(logging.INFO)
|
|
66
|
+
cron_logger.setLevel(logging.INFO)
|
|
67
|
+
else:
|
|
68
|
+
logger.setLevel(logging.DEBUG)
|
|
69
|
+
cron_logger.setLevel(logging.DEBUG)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def flush_logs() -> None:
|
|
73
|
+
"""flushes all logging handlers"""
|
|
74
|
+
for handler in logging.root.handlers:
|
|
75
|
+
handler.flush()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# max disk usage percent
|
|
79
|
+
max_disk_used_percent = None
|
|
80
|
+
|
|
81
|
+
# socket timeout
|
|
82
|
+
socket_timeout = None
|
|
83
|
+
|
|
84
|
+
# indicator that we're doing a dry run
|
|
85
|
+
dry_run = None
|
|
86
|
+
|
|
87
|
+
# session key reserved for test isolation
|
|
88
|
+
session_key: str | None = None
|
|
89
|
+
|
|
90
|
+
# keep and cutoff date; only recordings from this date on are downloaded and kept
|
|
91
|
+
keep_re = re.compile(r"""(?P<range>\d+)(?P<unit>[dw]?)""")
|
|
92
|
+
cutoff_date: datetime.date | None = None
|
|
93
|
+
|
|
94
|
+
# errno codes for unavailable dashcam
|
|
95
|
+
dashcam_unavailable_errno_codes = (
|
|
96
|
+
errno.EHOSTDOWN, # host is down
|
|
97
|
+
errno.EHOSTUNREACH, # host is unreachable
|
|
98
|
+
errno.ENETUNREACH, # network is unreachable
|
|
99
|
+
errno.ETIMEDOUT, # connection timed out
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# for unit testing
|
|
103
|
+
today = datetime.date.today()
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def calc_cutoff_date(keep: str) -> datetime.date:
|
|
107
|
+
"""given a retention period, calculates the date before which files should be deleted"""
|
|
108
|
+
|
|
109
|
+
if (keep_match := re.fullmatch(keep_re, keep)) is None:
|
|
110
|
+
raise RuntimeError("KEEP must be in the format <number>[dw]")
|
|
111
|
+
|
|
112
|
+
keep_range = int(keep_match.group("range"))
|
|
113
|
+
|
|
114
|
+
if keep_range < 1:
|
|
115
|
+
raise RuntimeError("KEEP must be greater than one.")
|
|
116
|
+
|
|
117
|
+
keep_unit = keep_match.group("unit") or "d"
|
|
118
|
+
|
|
119
|
+
if keep_unit == "d" or keep_unit is None:
|
|
120
|
+
keep_range_timedelta = datetime.timedelta(days=keep_range)
|
|
121
|
+
elif keep_unit == "w":
|
|
122
|
+
keep_range_timedelta = datetime.timedelta(weeks=keep_range)
|
|
123
|
+
else:
|
|
124
|
+
# this indicates a coding error
|
|
125
|
+
raise RuntimeError(f"unknown KEEP unit : {keep_unit}")
|
|
126
|
+
|
|
127
|
+
return today - keep_range_timedelta
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass(frozen=True)
|
|
131
|
+
class Recording:
|
|
132
|
+
"""represents a recording from the dashcam; the dashcam serves the list of video recording filenames (front and rear)"""
|
|
133
|
+
|
|
134
|
+
filename: str
|
|
135
|
+
base_filename: str
|
|
136
|
+
group_name: str | None
|
|
137
|
+
datetime: datetime.datetime
|
|
138
|
+
type: str
|
|
139
|
+
direction: str
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# dashcam recording filename regular expression
|
|
143
|
+
#
|
|
144
|
+
# references:
|
|
145
|
+
# - https://support.blackvue.com.au/hc/en-us/articles/13301776266895-Video-File-Naming
|
|
146
|
+
# N: Normal
|
|
147
|
+
# E: Event
|
|
148
|
+
# P: Parking motion detection
|
|
149
|
+
# M: Manual
|
|
150
|
+
# I: Parking impact
|
|
151
|
+
# O: Overspeed
|
|
152
|
+
# A: Hard acceleration
|
|
153
|
+
# T: Hard cornering
|
|
154
|
+
# B: Hard braking
|
|
155
|
+
# R: Geofence-enter (Fleet)
|
|
156
|
+
# X: Geofence-exit (Fleet)
|
|
157
|
+
# G: Geofence-pass (Fleet)
|
|
158
|
+
# D: Drowsiness (DMS)
|
|
159
|
+
# L: Distraction (DMS)
|
|
160
|
+
# Y: Seatbelt not detected (DMS)
|
|
161
|
+
# F: Driver undetected (DMS)
|
|
162
|
+
#
|
|
163
|
+
# F: Front camera
|
|
164
|
+
# R: Rear camera
|
|
165
|
+
# I: Interior camera
|
|
166
|
+
# O: Optional camera
|
|
167
|
+
#
|
|
168
|
+
# L or S: upload flag, Substream or Live
|
|
169
|
+
filename_re = re.compile(
|
|
170
|
+
r"""(?P<base_filename>(?P<year>\d\d\d\d)(?P<month>\d\d)(?P<day>\d\d)
|
|
171
|
+
_(?P<hour>\d\d)(?P<minute>\d\d)(?P<second>\d\d))
|
|
172
|
+
_(?P<type>[NEPMIOATBRXGDLYF])
|
|
173
|
+
(?P<direction>[FRIO])
|
|
174
|
+
(?P<upload>[LS]?)
|
|
175
|
+
\.(?P<extension>mp4)""",
|
|
176
|
+
re.VERBOSE,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def to_recording(filename: str, grouping: str) -> Recording | None:
|
|
181
|
+
"""extracts recording information from a filename"""
|
|
182
|
+
if (filename_match := re.fullmatch(filename_re, filename)) is None:
|
|
183
|
+
return None
|
|
184
|
+
|
|
185
|
+
year = int(filename_match.group("year"))
|
|
186
|
+
month = int(filename_match.group("month"))
|
|
187
|
+
day = int(filename_match.group("day"))
|
|
188
|
+
hour = int(filename_match.group("hour"))
|
|
189
|
+
minute = int(filename_match.group("minute"))
|
|
190
|
+
second = int(filename_match.group("second"))
|
|
191
|
+
recording_datetime = datetime.datetime(year, month, day, hour, minute, second)
|
|
192
|
+
|
|
193
|
+
recording_base_filename = filename_match.group("base_filename")
|
|
194
|
+
recording_group_name = get_group_name(recording_datetime, grouping)
|
|
195
|
+
recording_type = filename_match.group("type")
|
|
196
|
+
recording_direction = filename_match.group("direction")
|
|
197
|
+
|
|
198
|
+
return Recording(
|
|
199
|
+
filename,
|
|
200
|
+
recording_base_filename,
|
|
201
|
+
recording_group_name,
|
|
202
|
+
recording_datetime,
|
|
203
|
+
recording_type,
|
|
204
|
+
recording_direction,
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# pattern of a recording filename as returned in each line from from the dashcam index page
|
|
209
|
+
file_line_re = re.compile(r"n:/Record/(?P<filename>.*\.mp4),s:1000000\r\n")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def get_filenames(file_lines: list[str]) -> list[str]:
|
|
213
|
+
"""extracts the recording filenames from the lines returned by the dashcam index page"""
|
|
214
|
+
filenames = []
|
|
215
|
+
for file_line in file_lines:
|
|
216
|
+
# the first line is "v:1.00", which won't match, so we skip it
|
|
217
|
+
if file_line_match := re.fullmatch(file_line_re, file_line):
|
|
218
|
+
filenames.append(file_line_match.group("filename"))
|
|
219
|
+
|
|
220
|
+
return filenames
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def get_dashcam_filenames(base_url: str) -> list[str]:
|
|
224
|
+
"""gets the recording filenames from the dashcam"""
|
|
225
|
+
try:
|
|
226
|
+
url = urllib.parse.urljoin(base_url, "blackvue_vod.cgi")
|
|
227
|
+
request = urllib.request.Request(url)
|
|
228
|
+
if session_key:
|
|
229
|
+
request.add_header("X-Session-Key", session_key)
|
|
230
|
+
|
|
231
|
+
response = urllib.request.urlopen(request)
|
|
232
|
+
|
|
233
|
+
response_status_code = response.getcode()
|
|
234
|
+
if response_status_code != 200:
|
|
235
|
+
raise RuntimeError(
|
|
236
|
+
f"Error response from : {base_url} ; status code : {response_status_code}"
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
charset = response.info().get_param("charset", "UTF-8")
|
|
240
|
+
file_lines = [x.decode(charset) for x in response.readlines()]
|
|
241
|
+
|
|
242
|
+
return get_filenames(file_lines)
|
|
243
|
+
except urllib.error.URLError as e:
|
|
244
|
+
if isinstance(e.reason, OSError) and (
|
|
245
|
+
isinstance(e.reason, TimeoutError)
|
|
246
|
+
or e.reason.errno in dashcam_unavailable_errno_codes
|
|
247
|
+
):
|
|
248
|
+
raise UserWarning(f"Dashcam unavailable : {e}") from e
|
|
249
|
+
|
|
250
|
+
raise RuntimeError(
|
|
251
|
+
f"Cannot obtain list of recordings from dashcam at address : {base_url}; error : {e}"
|
|
252
|
+
) from e
|
|
253
|
+
except socket.timeout as e:
|
|
254
|
+
raise UserWarning(
|
|
255
|
+
f"Timeout communicating with dashcam at address : {base_url}; error : {e}"
|
|
256
|
+
) from e
|
|
257
|
+
except http.client.RemoteDisconnected as e:
|
|
258
|
+
raise UserWarning(
|
|
259
|
+
f"Dashcam disconnected without a response; address : {base_url}; error : {e}"
|
|
260
|
+
) from e
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def get_group_name(recording_datetime: datetime.datetime, grouping: str) -> str | None:
|
|
264
|
+
"""determines the group name for a given recording according to the indicated grouping"""
|
|
265
|
+
if grouping == "daily":
|
|
266
|
+
return recording_datetime.date().isoformat()
|
|
267
|
+
|
|
268
|
+
if grouping == "weekly":
|
|
269
|
+
recording_date = recording_datetime.date()
|
|
270
|
+
|
|
271
|
+
# day of the week (mon = 0, ..., sun = 6)
|
|
272
|
+
recording_weekday = recording_date.weekday()
|
|
273
|
+
recording_weekday_delta = datetime.timedelta(days=recording_weekday)
|
|
274
|
+
recording_mon_date = recording_date - recording_weekday_delta
|
|
275
|
+
return recording_mon_date.isoformat()
|
|
276
|
+
|
|
277
|
+
if grouping == "monthly":
|
|
278
|
+
return recording_datetime.date().strftime("%Y-%m")
|
|
279
|
+
|
|
280
|
+
if grouping == "yearly":
|
|
281
|
+
return recording_datetime.date().strftime("%Y")
|
|
282
|
+
|
|
283
|
+
return None
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
# download speed units for conversion to a natural representation
|
|
287
|
+
speed_units = [(1000000, "Mbps"), (1000, "Kbps"), (1, "bps")]
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def to_natural_speed(speed_bps: int) -> tuple[int, str]:
|
|
291
|
+
"""returns a natural representation of a given download speed in bps as an scalar+unit tuple (base 10)"""
|
|
292
|
+
for speed_unit_multiplier, speed_unit_name in speed_units:
|
|
293
|
+
if speed_bps > speed_unit_multiplier:
|
|
294
|
+
return int(speed_bps / speed_unit_multiplier), speed_unit_name
|
|
295
|
+
|
|
296
|
+
return 0, "bps"
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def format_natural_speed(speed_bps: int | None) -> str:
|
|
300
|
+
"""formats download speed in bps as a human-readable string like ' (123Mbps)', or empty string if None"""
|
|
301
|
+
if not speed_bps:
|
|
302
|
+
return ""
|
|
303
|
+
|
|
304
|
+
speed_value, speed_unit = to_natural_speed(speed_bps)
|
|
305
|
+
return f" ({speed_value}{speed_unit})"
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def get_filepath(destination: str, group_name: str | None, filename: str) -> str:
|
|
309
|
+
"""constructs a path for a recording file from the destination, group name and filename (or glob pattern)"""
|
|
310
|
+
if group_name:
|
|
311
|
+
return os.path.join(destination, group_name, filename)
|
|
312
|
+
return os.path.join(destination, filename)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def download_file(
|
|
316
|
+
base_url: str, filename: str, destination: str, group_name: str | None
|
|
317
|
+
) -> tuple[bool, int | None]:
|
|
318
|
+
"""downloads a file from the dashcam to the destination directory; returns whether data was transferred"""
|
|
319
|
+
# if we have a group name, we may not have ensured it exists yet
|
|
320
|
+
if group_name:
|
|
321
|
+
group_filepath = os.path.join(destination, group_name)
|
|
322
|
+
ensure_destination(group_filepath)
|
|
323
|
+
|
|
324
|
+
destination_filepath = get_filepath(destination, group_name, filename)
|
|
325
|
+
|
|
326
|
+
if os.path.exists(destination_filepath):
|
|
327
|
+
logger.debug("Ignoring already downloaded file : %s", filename)
|
|
328
|
+
return False, None
|
|
329
|
+
|
|
330
|
+
temp_filepath = os.path.join(destination, f".{filename}")
|
|
331
|
+
if os.path.exists(temp_filepath):
|
|
332
|
+
logger.debug("Found incomplete download : %s", temp_filepath)
|
|
333
|
+
|
|
334
|
+
if dry_run:
|
|
335
|
+
logger.debug("DRY RUN Would download file : %s", filename)
|
|
336
|
+
return True, None
|
|
337
|
+
|
|
338
|
+
try:
|
|
339
|
+
url = urllib.parse.urljoin(base_url, f"Record/{filename}")
|
|
340
|
+
|
|
341
|
+
start = time.perf_counter()
|
|
342
|
+
try:
|
|
343
|
+
# request
|
|
344
|
+
request = urllib.request.Request(url)
|
|
345
|
+
if session_key:
|
|
346
|
+
request.add_header("X-Session-Key", session_key)
|
|
347
|
+
|
|
348
|
+
# downloads file
|
|
349
|
+
with urllib.request.urlopen(request) as response:
|
|
350
|
+
headers = response.info()
|
|
351
|
+
size = headers.get("Content-Length")
|
|
352
|
+
|
|
353
|
+
# writes response to temp file
|
|
354
|
+
with open(temp_filepath, "wb") as f:
|
|
355
|
+
f.write(response.read())
|
|
356
|
+
finally:
|
|
357
|
+
end = time.perf_counter()
|
|
358
|
+
elapsed_s = end - start
|
|
359
|
+
|
|
360
|
+
os.rename(temp_filepath, destination_filepath)
|
|
361
|
+
|
|
362
|
+
speed_bps = int(10.0 * float(size) / elapsed_s) if size else None
|
|
363
|
+
speed_str = format_natural_speed(speed_bps)
|
|
364
|
+
logger.debug("Downloaded file : %s%s", filename, speed_str)
|
|
365
|
+
|
|
366
|
+
return True, speed_bps
|
|
367
|
+
except urllib.error.URLError as e:
|
|
368
|
+
# data corruption may lead to error status codes; logs a warning (cron) and returns normally
|
|
369
|
+
cron_logger.warning(
|
|
370
|
+
"Could not download file : %s; error : %s; ignoring.", filename, e
|
|
371
|
+
)
|
|
372
|
+
return False, None
|
|
373
|
+
except socket.timeout as e:
|
|
374
|
+
raise UserWarning(
|
|
375
|
+
f"Timeout communicating with dashcam at address : {base_url}; error : {e}"
|
|
376
|
+
) from e
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def download_recording(base_url: str, recording: Recording, destination: str) -> None:
|
|
380
|
+
"""downloads the set of recordings, including gps data, for the given filename from the dashcam to the destination
|
|
381
|
+
directory"""
|
|
382
|
+
# first checks that we have enough room left
|
|
383
|
+
disk_usage = shutil.disk_usage(destination)
|
|
384
|
+
disk_used_percent = disk_usage.used / disk_usage.total * 100.0
|
|
385
|
+
|
|
386
|
+
if max_disk_used_percent is not None and disk_used_percent > max_disk_used_percent:
|
|
387
|
+
raise RuntimeError(
|
|
388
|
+
f"Not enough disk space left. Max used disk space percentage allowed : {max_disk_used_percent}%"
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
# whether any file of a recording (video, thumbnail, gps, accel.) was downloaded
|
|
392
|
+
any_downloaded = False
|
|
393
|
+
|
|
394
|
+
# downloads the video recording
|
|
395
|
+
filename = recording.filename
|
|
396
|
+
downloaded, speed_bps = download_file(
|
|
397
|
+
base_url, filename, destination, recording.group_name
|
|
398
|
+
)
|
|
399
|
+
any_downloaded |= downloaded
|
|
400
|
+
|
|
401
|
+
# downloads the thumbnail file
|
|
402
|
+
thm_filename = (
|
|
403
|
+
f"{recording.base_filename}_{recording.type}{recording.direction}.thm"
|
|
404
|
+
)
|
|
405
|
+
downloaded, _ = download_file(
|
|
406
|
+
base_url, thm_filename, destination, recording.group_name
|
|
407
|
+
)
|
|
408
|
+
any_downloaded |= downloaded
|
|
409
|
+
|
|
410
|
+
# downloads the accelerometer data
|
|
411
|
+
tgf_filename = f"{recording.base_filename}_{recording.type}.3gf"
|
|
412
|
+
downloaded, _ = download_file(
|
|
413
|
+
base_url, tgf_filename, destination, recording.group_name
|
|
414
|
+
)
|
|
415
|
+
any_downloaded |= downloaded
|
|
416
|
+
|
|
417
|
+
# downloads the gps data
|
|
418
|
+
gps_filename = f"{recording.base_filename}_{recording.type}.gps"
|
|
419
|
+
downloaded, _ = download_file(
|
|
420
|
+
base_url, gps_filename, destination, recording.group_name
|
|
421
|
+
)
|
|
422
|
+
any_downloaded |= downloaded
|
|
423
|
+
|
|
424
|
+
# logs if any part of a recording was downloaded (or would have been)
|
|
425
|
+
if any_downloaded:
|
|
426
|
+
# recording logger, depends on type of recording
|
|
427
|
+
recording_logger = cron_logger if recording.type in ("N", "M") else logger
|
|
428
|
+
|
|
429
|
+
if not dry_run:
|
|
430
|
+
speed_str = format_natural_speed(speed_bps)
|
|
431
|
+
recording_logger.info(
|
|
432
|
+
"Downloaded recording : %s (%s%s)%s",
|
|
433
|
+
recording.base_filename,
|
|
434
|
+
recording.type,
|
|
435
|
+
recording.direction,
|
|
436
|
+
speed_str,
|
|
437
|
+
)
|
|
438
|
+
else:
|
|
439
|
+
recording_logger.info(
|
|
440
|
+
"DRY RUN Would download recording : %s (%s%s)",
|
|
441
|
+
recording.base_filename,
|
|
442
|
+
recording.type,
|
|
443
|
+
recording.direction,
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def sort_recordings(recordings: list[Recording], recording_priority: str) -> None:
|
|
448
|
+
"""sorts recordings in place according to the given priority"""
|
|
449
|
+
|
|
450
|
+
# preferred orderings (by type and direction)
|
|
451
|
+
recording_types = "MEIBOATRXGNP"
|
|
452
|
+
recording_directions = "FRIO"
|
|
453
|
+
|
|
454
|
+
# tomorrow, for reverse datetime sorting
|
|
455
|
+
tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
|
|
456
|
+
|
|
457
|
+
def datetime_sort_key(recording: Recording) -> tuple[datetime.datetime, int]:
|
|
458
|
+
"""sorts by datetime, then front/rear direction, then recording type"""
|
|
459
|
+
return recording.datetime, recording_directions.find(recording.direction)
|
|
460
|
+
|
|
461
|
+
def rev_datetime_sort_key(recording: Recording) -> tuple[datetime.timedelta, int]:
|
|
462
|
+
"""sorts by newest to oldest datetime, then front/rear/interior direction"""
|
|
463
|
+
return tomorrow - recording.datetime, recording_directions.find(
|
|
464
|
+
recording.direction
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
def manual_event_sort_key(
|
|
468
|
+
recording: Recording,
|
|
469
|
+
) -> tuple[int, datetime.datetime, int]:
|
|
470
|
+
"""sorts by recording type (manual and events first), then datetime, then front/rear/interior direction"""
|
|
471
|
+
return (
|
|
472
|
+
recording_types.find(recording.type),
|
|
473
|
+
recording.datetime,
|
|
474
|
+
recording_directions.find(recording.direction),
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
sort_key: Callable[[Recording], tuple[object, ...]]
|
|
478
|
+
if recording_priority == "date":
|
|
479
|
+
# least recent first
|
|
480
|
+
sort_key = datetime_sort_key
|
|
481
|
+
elif recording_priority == "rdate":
|
|
482
|
+
# most recent first
|
|
483
|
+
sort_key = rev_datetime_sort_key
|
|
484
|
+
elif recording_priority == "type":
|
|
485
|
+
# manual, event, normal, parking
|
|
486
|
+
sort_key = manual_event_sort_key
|
|
487
|
+
else:
|
|
488
|
+
# this indicates a coding error
|
|
489
|
+
raise RuntimeError(f"unknown recording priority : {recording_priority}")
|
|
490
|
+
|
|
491
|
+
recordings.sort(key=sort_key)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
# group name globs, keyed by grouping
|
|
495
|
+
group_name_globs = {
|
|
496
|
+
"none": None,
|
|
497
|
+
"daily": "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]",
|
|
498
|
+
"weekly": "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]",
|
|
499
|
+
"monthly": "[0-9][0-9][0-9][0-9]-[0-9][0-9]",
|
|
500
|
+
"yearly": "[0-9][0-9][0-9][0-9]",
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
# represents a recording downloaded to the destination; matches all files (video front/rear, gps, etc.)
|
|
505
|
+
@dataclass(frozen=True)
|
|
506
|
+
class DownloadedRecording:
|
|
507
|
+
base_filename: str
|
|
508
|
+
group_name: str | None
|
|
509
|
+
datetime: datetime.datetime
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
# downloaded recording filename regular expression
|
|
513
|
+
downloaded_filename_re = re.compile(
|
|
514
|
+
r"""^(?P<base_filename>(?P<year>\d\d\d\d)(?P<month>\d\d)(?P<day>\d\d)
|
|
515
|
+
_(?P<hour>\d\d)(?P<minute>\d\d)(?P<second>\d\d))_""",
|
|
516
|
+
re.VERBOSE,
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def to_downloaded_recording(filename: str, grouping: str) -> DownloadedRecording | None:
|
|
521
|
+
"""extracts destination recording information from a filename"""
|
|
522
|
+
if (filename_match := re.match(downloaded_filename_re, filename)) is None:
|
|
523
|
+
return None
|
|
524
|
+
|
|
525
|
+
year = int(filename_match.group("year"))
|
|
526
|
+
month = int(filename_match.group("month"))
|
|
527
|
+
day = int(filename_match.group("day"))
|
|
528
|
+
hour = int(filename_match.group("hour"))
|
|
529
|
+
minute = int(filename_match.group("minute"))
|
|
530
|
+
second = int(filename_match.group("second"))
|
|
531
|
+
recording_datetime = datetime.datetime(year, month, day, hour, minute, second)
|
|
532
|
+
|
|
533
|
+
recording_base_filename = filename_match.group("base_filename")
|
|
534
|
+
recording_group_name = get_group_name(recording_datetime, grouping)
|
|
535
|
+
|
|
536
|
+
return DownloadedRecording(
|
|
537
|
+
recording_base_filename, recording_group_name, recording_datetime
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
# downloaded recording filename glob pattern
|
|
542
|
+
downloaded_filename_glob = (
|
|
543
|
+
"[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9][0-9][0-9]_*.*"
|
|
544
|
+
)
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def get_downloaded_recordings(
|
|
548
|
+
destination: str, grouping: str
|
|
549
|
+
) -> set[DownloadedRecording]:
|
|
550
|
+
"""reads files from the destination directory and returns them as recording records"""
|
|
551
|
+
group_name_glob = group_name_globs[grouping]
|
|
552
|
+
|
|
553
|
+
downloaded_filepath_glob = get_filepath(
|
|
554
|
+
destination, group_name_glob, downloaded_filename_glob
|
|
555
|
+
)
|
|
556
|
+
|
|
557
|
+
downloaded_filepaths = glob.glob(downloaded_filepath_glob)
|
|
558
|
+
|
|
559
|
+
return {
|
|
560
|
+
r
|
|
561
|
+
for p in downloaded_filepaths
|
|
562
|
+
if (r := to_downloaded_recording(os.path.basename(p), grouping)) is not None
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def get_outdated_recordings(
|
|
567
|
+
destination: str, grouping: str
|
|
568
|
+
) -> list[DownloadedRecording]:
|
|
569
|
+
"""returns the recordings prior to the cutoff date"""
|
|
570
|
+
if cutoff_date is None:
|
|
571
|
+
return []
|
|
572
|
+
|
|
573
|
+
downloaded_recordings = get_downloaded_recordings(destination, grouping)
|
|
574
|
+
|
|
575
|
+
return [x for x in downloaded_recordings if x.datetime.date() < cutoff_date]
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def get_current_recordings(recordings: list[Recording]) -> list[Recording]:
|
|
579
|
+
"""returns the recordings that are after or on the cutoff date"""
|
|
580
|
+
return (
|
|
581
|
+
recordings
|
|
582
|
+
if cutoff_date is None
|
|
583
|
+
else [x for x in recordings if x.datetime.date() >= cutoff_date]
|
|
584
|
+
)
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def get_filtered_recordings(
|
|
588
|
+
recordings: list[Recording], recording_filter: tuple[str, ...] | None
|
|
589
|
+
) -> list[Recording]:
|
|
590
|
+
"""returns recordings filtered by recording_filter"""
|
|
591
|
+
return (
|
|
592
|
+
recordings
|
|
593
|
+
if recording_filter is None
|
|
594
|
+
else [x for x in recordings if f"{x.type}{x.direction}" in recording_filter]
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def ensure_destination(destination: str) -> None:
|
|
599
|
+
"""ensures the destination directory exists, creates if not, verifies it's writeable"""
|
|
600
|
+
# if no destination, creates it
|
|
601
|
+
if not os.path.exists(destination):
|
|
602
|
+
os.makedirs(destination)
|
|
603
|
+
return
|
|
604
|
+
|
|
605
|
+
# destination exists, tests if directory
|
|
606
|
+
if not os.path.isdir(destination):
|
|
607
|
+
raise RuntimeError(f"download destination is not a directory : {destination}")
|
|
608
|
+
|
|
609
|
+
# destination is a directory, tests if writable
|
|
610
|
+
if not os.access(destination, os.W_OK):
|
|
611
|
+
raise RuntimeError(
|
|
612
|
+
f"download destination directory not writable : {destination}"
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def prepare_destination(destination: str, grouping: str) -> None:
|
|
617
|
+
"""prepares the destination, ensuring it's valid and removing excess recordings"""
|
|
618
|
+
# optionally removes outdated recordings
|
|
619
|
+
if cutoff_date:
|
|
620
|
+
outdated_recordings = get_outdated_recordings(destination, grouping)
|
|
621
|
+
|
|
622
|
+
for outdated_recording in outdated_recordings:
|
|
623
|
+
if dry_run:
|
|
624
|
+
logger.info(
|
|
625
|
+
"DRY RUN Would remove outdated recording : %s",
|
|
626
|
+
outdated_recording.base_filename,
|
|
627
|
+
)
|
|
628
|
+
continue
|
|
629
|
+
|
|
630
|
+
logger.info(
|
|
631
|
+
"Removing outdated recording : %s", outdated_recording.base_filename
|
|
632
|
+
)
|
|
633
|
+
|
|
634
|
+
outdated_recording_glob = (
|
|
635
|
+
f"{outdated_recording.base_filename}_[NEPMIOATBRXGDLYF]*.*"
|
|
636
|
+
)
|
|
637
|
+
outdated_filepath_glob = get_filepath(
|
|
638
|
+
destination, outdated_recording.group_name, outdated_recording_glob
|
|
639
|
+
)
|
|
640
|
+
|
|
641
|
+
outdated_filepaths = glob.glob(outdated_filepath_glob)
|
|
642
|
+
|
|
643
|
+
for outdated_filepath in outdated_filepaths:
|
|
644
|
+
os.remove(outdated_filepath)
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def sync(
|
|
648
|
+
address: str,
|
|
649
|
+
destination: str,
|
|
650
|
+
grouping: str,
|
|
651
|
+
download_priority: str,
|
|
652
|
+
recording_filter: tuple[str, ...] | None,
|
|
653
|
+
) -> None:
|
|
654
|
+
"""synchronizes the recordings at the dashcam address with the destination directory"""
|
|
655
|
+
prepare_destination(destination, grouping)
|
|
656
|
+
|
|
657
|
+
base_url = f"http://{address}"
|
|
658
|
+
dashcam_filenames = get_dashcam_filenames(base_url)
|
|
659
|
+
dashcam_recordings = [
|
|
660
|
+
r for x in dashcam_filenames if (r := to_recording(x, grouping)) is not None
|
|
661
|
+
]
|
|
662
|
+
|
|
663
|
+
# figures out which recordings are current and should be downloaded
|
|
664
|
+
current_dashcam_recordings = get_current_recordings(dashcam_recordings)
|
|
665
|
+
|
|
666
|
+
# filters recordings according to recording_filter tuple
|
|
667
|
+
current_dashcam_recordings = get_filtered_recordings(
|
|
668
|
+
current_dashcam_recordings, recording_filter
|
|
669
|
+
)
|
|
670
|
+
|
|
671
|
+
# sorts the dashcam recordings so we download them according to some priority
|
|
672
|
+
sort_recordings(current_dashcam_recordings, download_priority)
|
|
673
|
+
|
|
674
|
+
for recording in current_dashcam_recordings:
|
|
675
|
+
download_recording(base_url, recording, destination)
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def is_empty_directory(dirpath: str) -> bool:
|
|
679
|
+
"""tests if a directory is empty, ignoring anything that's not a video recording"""
|
|
680
|
+
return all(not x.endswith(".mp4") for x in os.listdir(dirpath))
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
# temp filename regular expression
|
|
684
|
+
temp_filename_glob = ".[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9][0-9][0-9]_[NEPMIOATBRXGDLYF]*.*"
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def clean_destination(destination: str, grouping: str) -> None:
|
|
688
|
+
"""removes temporary artifacts from the destination directory"""
|
|
689
|
+
# removes temporary files from interrupted downloads
|
|
690
|
+
temp_filepath_glob = os.path.join(destination, temp_filename_glob)
|
|
691
|
+
temp_filepaths = glob.glob(temp_filepath_glob)
|
|
692
|
+
|
|
693
|
+
for temp_filepath in temp_filepaths:
|
|
694
|
+
if not dry_run:
|
|
695
|
+
logger.debug("Removing temporary file : %s", temp_filepath)
|
|
696
|
+
os.remove(temp_filepath)
|
|
697
|
+
else:
|
|
698
|
+
logger.debug("DRY RUN Would remove temporary file : %s", temp_filepath)
|
|
699
|
+
|
|
700
|
+
# removes empty grouping directories; ignores dotfiles such as .DS_Store
|
|
701
|
+
group_name_glob = group_name_globs[grouping]
|
|
702
|
+
if group_name_glob:
|
|
703
|
+
group_filepath_glob = os.path.join(destination, group_name_glob)
|
|
704
|
+
|
|
705
|
+
group_filepaths = glob.glob(group_filepath_glob)
|
|
706
|
+
|
|
707
|
+
for group_filepath in group_filepaths:
|
|
708
|
+
if is_empty_directory(group_filepath):
|
|
709
|
+
if not dry_run:
|
|
710
|
+
logger.debug("Removing grouping directory : %s", group_filepath)
|
|
711
|
+
shutil.rmtree(group_filepath)
|
|
712
|
+
else:
|
|
713
|
+
logger.debug(
|
|
714
|
+
"DRY RUN Would remove grouping directory : %s", group_filepath
|
|
715
|
+
)
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def lock(destination: str) -> int:
|
|
719
|
+
"""creates a lock to ensure only one instance is running on a given destination; adapted from:
|
|
720
|
+
https://stackoverflow.com/questions/220525/ensure-a-single-instance-of-an-application-in-linux
|
|
721
|
+
"""
|
|
722
|
+
# Establish lock file settings
|
|
723
|
+
lf_path = os.path.join(destination, ".blackvuesync.lock")
|
|
724
|
+
lf_flags = os.O_WRONLY | os.O_CREAT
|
|
725
|
+
lf_mode = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH # This is 0o222, i.e. 146
|
|
726
|
+
|
|
727
|
+
# Create lock file
|
|
728
|
+
# Regarding umask, see https://stackoverflow.com/a/15015748/832230
|
|
729
|
+
umask_original = os.umask(0)
|
|
730
|
+
|
|
731
|
+
try:
|
|
732
|
+
lf_fd = os.open(lf_path, lf_flags, lf_mode)
|
|
733
|
+
finally:
|
|
734
|
+
os.umask(umask_original)
|
|
735
|
+
|
|
736
|
+
try:
|
|
737
|
+
fcntl.lockf(lf_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
738
|
+
|
|
739
|
+
return lf_fd
|
|
740
|
+
except OSError as e:
|
|
741
|
+
raise UserWarning(
|
|
742
|
+
f"Another instance is already running for destination : {destination}"
|
|
743
|
+
) from e
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def unlock(lf_fd: int) -> None:
|
|
747
|
+
"""unlocks the lock file; does not remove because another process may lock it in the meantime"""
|
|
748
|
+
fcntl.lockf(lf_fd, fcntl.LOCK_UN)
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def parse_args() -> argparse.Namespace:
|
|
752
|
+
"""parses the command-line arguments"""
|
|
753
|
+
arg_parser = argparse.ArgumentParser(
|
|
754
|
+
description="Synchronizes BlackVue dashcam recordings with a local directory.",
|
|
755
|
+
epilog="Bug reports: https://github.com/acolomba/BlackVueSync",
|
|
756
|
+
)
|
|
757
|
+
arg_parser.add_argument(
|
|
758
|
+
"address", metavar="ADDRESS", help="dashcam IP address or name"
|
|
759
|
+
)
|
|
760
|
+
arg_parser.add_argument(
|
|
761
|
+
"-d",
|
|
762
|
+
"--destination",
|
|
763
|
+
metavar="DEST",
|
|
764
|
+
help="sets the destination directory to DEST; defaults to the current directory",
|
|
765
|
+
)
|
|
766
|
+
arg_parser.add_argument(
|
|
767
|
+
"-g",
|
|
768
|
+
"--grouping",
|
|
769
|
+
metavar="GROUPING",
|
|
770
|
+
default="none",
|
|
771
|
+
choices=["none", "daily", "weekly", "monthly", "yearly"],
|
|
772
|
+
help="groups recording by day, week, month or year under a directory named after the date; so respectively 2019-06-15, 2019-06-09 (Mon), 2019-07 or 2019; defaults to none, indicating no grouping",
|
|
773
|
+
)
|
|
774
|
+
arg_parser.add_argument(
|
|
775
|
+
"-k",
|
|
776
|
+
"--keep",
|
|
777
|
+
metavar="KEEP_RANGE",
|
|
778
|
+
help="""keeps recordings in the given range, removing the rest; defaults to days, but can suffix with d, w for days or weeks respectively""",
|
|
779
|
+
)
|
|
780
|
+
arg_parser.add_argument(
|
|
781
|
+
"-p",
|
|
782
|
+
"--priority",
|
|
783
|
+
metavar="DOWNLOAD_PRIORITY",
|
|
784
|
+
default="date",
|
|
785
|
+
choices=["date", "rdate", "type"],
|
|
786
|
+
help="sets the recording download priority; date: downloads in chronological order from oldest to newest; rdate: downloads in chronological order from newest to oldest; type: prioritizes manual, event, normal and then parkingrecordings; defaults to date",
|
|
787
|
+
)
|
|
788
|
+
arg_parser.add_argument(
|
|
789
|
+
"-f",
|
|
790
|
+
"--filter",
|
|
791
|
+
default=None,
|
|
792
|
+
help="downloads recordings filtered by event type and camera direction; e.g.: --filter PF PR downloads only Parking Front and Parking Rear recordings",
|
|
793
|
+
nargs="+",
|
|
794
|
+
)
|
|
795
|
+
arg_parser.add_argument(
|
|
796
|
+
"-u",
|
|
797
|
+
"--max-used-disk",
|
|
798
|
+
metavar="DISK_USAGE_PERCENT",
|
|
799
|
+
default=90,
|
|
800
|
+
type=int,
|
|
801
|
+
choices=range(5, 99),
|
|
802
|
+
help="stops downloading recordings if disk is over DISK_USAGE_PERCENT used; defaults to 90",
|
|
803
|
+
)
|
|
804
|
+
arg_parser.add_argument(
|
|
805
|
+
"-t",
|
|
806
|
+
"--timeout",
|
|
807
|
+
metavar="TIMEOUT",
|
|
808
|
+
default=10.0,
|
|
809
|
+
type=float,
|
|
810
|
+
help="sets the connection timeout in seconds (float); defaults to 10.0 seconds",
|
|
811
|
+
)
|
|
812
|
+
arg_parser.add_argument(
|
|
813
|
+
"-v", "--verbose", action="count", default=0, help="increases verbosity"
|
|
814
|
+
)
|
|
815
|
+
arg_parser.add_argument(
|
|
816
|
+
"-q",
|
|
817
|
+
"--quiet",
|
|
818
|
+
action="store_true",
|
|
819
|
+
help="quiets down output messages; overrides verbosity options",
|
|
820
|
+
)
|
|
821
|
+
arg_parser.add_argument(
|
|
822
|
+
"--cron",
|
|
823
|
+
action="store_true",
|
|
824
|
+
help="cron mode, only logs normal recordings at default verbosity",
|
|
825
|
+
)
|
|
826
|
+
arg_parser.add_argument(
|
|
827
|
+
"--dry-run", action="store_true", help="shows what the program would do"
|
|
828
|
+
)
|
|
829
|
+
arg_parser.add_argument(
|
|
830
|
+
"--session-key",
|
|
831
|
+
metavar="SESSION_KEY",
|
|
832
|
+
help="session key; reserved for test isolation",
|
|
833
|
+
)
|
|
834
|
+
arg_parser.add_argument(
|
|
835
|
+
"--version",
|
|
836
|
+
action="version",
|
|
837
|
+
default=__version__,
|
|
838
|
+
version=f"%(prog)s {__version__}",
|
|
839
|
+
help="shows the version and exits",
|
|
840
|
+
)
|
|
841
|
+
|
|
842
|
+
return arg_parser.parse_args()
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def main() -> int:
|
|
846
|
+
"""run forrest run"""
|
|
847
|
+
# dry-run is a global setting
|
|
848
|
+
global dry_run
|
|
849
|
+
global max_disk_used_percent
|
|
850
|
+
global cutoff_date
|
|
851
|
+
global socket_timeout
|
|
852
|
+
global session_key
|
|
853
|
+
|
|
854
|
+
args = parse_args()
|
|
855
|
+
|
|
856
|
+
dry_run = args.dry_run
|
|
857
|
+
session_key = args.session_key
|
|
858
|
+
if dry_run:
|
|
859
|
+
logger.info("DRY RUN No action will be taken.")
|
|
860
|
+
|
|
861
|
+
max_disk_used_percent = args.max_used_disk
|
|
862
|
+
|
|
863
|
+
set_logging_levels(-1 if args.quiet else args.verbose, args.cron)
|
|
864
|
+
|
|
865
|
+
# sets socket timeout
|
|
866
|
+
socket_timeout = args.timeout
|
|
867
|
+
if socket_timeout <= 0:
|
|
868
|
+
raise argparse.ArgumentTypeError("TIMEOUT must be greater than zero.")
|
|
869
|
+
socket.setdefaulttimeout(socket_timeout)
|
|
870
|
+
|
|
871
|
+
# lock file file descriptor
|
|
872
|
+
lf_fd = None
|
|
873
|
+
|
|
874
|
+
try:
|
|
875
|
+
if args.keep:
|
|
876
|
+
cutoff_date = calc_cutoff_date(args.keep)
|
|
877
|
+
logger.info("Recording cutoff date : %s", cutoff_date)
|
|
878
|
+
|
|
879
|
+
# prepares the local file destination
|
|
880
|
+
destination = args.destination or os.getcwd()
|
|
881
|
+
ensure_destination(destination)
|
|
882
|
+
|
|
883
|
+
# grouping
|
|
884
|
+
grouping = args.grouping
|
|
885
|
+
|
|
886
|
+
lf_fd = lock(destination)
|
|
887
|
+
|
|
888
|
+
try:
|
|
889
|
+
sync(args.address, destination, grouping, args.priority, args.filter)
|
|
890
|
+
finally:
|
|
891
|
+
# removes temporary files (if we synced successfully, these are temp files from lost recordings)
|
|
892
|
+
clean_destination(destination, grouping)
|
|
893
|
+
except UserWarning as e:
|
|
894
|
+
logger.warning(e.args[0])
|
|
895
|
+
return 0 if args.cron else 1
|
|
896
|
+
except RuntimeError as e:
|
|
897
|
+
logger.error(e.args[0])
|
|
898
|
+
return 2
|
|
899
|
+
except Exception as e:
|
|
900
|
+
logger.exception(e)
|
|
901
|
+
return 3
|
|
902
|
+
finally:
|
|
903
|
+
if lf_fd:
|
|
904
|
+
unlock(lf_fd)
|
|
905
|
+
|
|
906
|
+
flush_logs()
|
|
907
|
+
|
|
908
|
+
return 0
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
if __name__ == "__main__":
|
|
912
|
+
sys.exit(main())
|