py-pve-cloud-backup 3.5.18__tar.gz → 3.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {py_pve_cloud_backup-3.5.18/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.6.0}/PKG-INFO +3 -3
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/pyproject.toml +1 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/requirements.txt +2 -4
- py_pve_cloud_backup-3.6.0/src/pve_cloud_backup/_version.py +1 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/bdd.py +43 -79
- py_pve_cloud_backup-3.6.0/src/pve_cloud_backup/daemon/bdd_prune.py +141 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/brctl.py +14 -22
- py_pve_cloud_backup-3.6.0/src/pve_cloud_backup/daemon/db.py +182 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/funcs.py +12 -24
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/ext_zfs.py +1 -1
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/funcs.py +2 -4
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/main.py +1 -1
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/net.py +8 -2
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0/src/py_pve_cloud_backup.egg-info}/PKG-INFO +3 -3
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +2 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/entry_points.txt +1 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/requires.txt +2 -2
- py_pve_cloud_backup-3.5.18/src/pve_cloud_backup/_version.py +0 -1
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/LICENSE.md +0 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/README.md +0 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/setup.cfg +0 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/restore.py +0 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/git.py +0 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
- {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
{py_pve_cloud_backup-3.5.18/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.6.0}/PKG-INFO
RENAMED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: py-pve-cloud-backup
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.6.0
|
|
4
4
|
Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
|
|
5
5
|
License-Expression: GPL-3.0-or-later
|
|
6
6
|
License-File: LICENSE.md
|
|
7
|
-
Requires-Dist: py-pve-cloud<3.
|
|
7
|
+
Requires-Dist: py-pve-cloud<3.6.0,>=3.5.0
|
|
8
8
|
Requires-Dist: kubernetes==34.1.0
|
|
9
9
|
Requires-Dist: requests==2.32.5
|
|
10
10
|
Requires-Dist: zstandard==0.25.0
|
|
11
|
-
Requires-Dist:
|
|
11
|
+
Requires-Dist: aiosqlite==0.21.0
|
|
12
12
|
Requires-Dist: python-socketio[asyncio_client]==5.16.4
|
|
13
13
|
Requires-Dist: aiohttp==3.14.3
|
|
14
14
|
Dynamic: license-file
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
# pessimistic operator for local tdd installs (pip install -e .)
|
|
2
|
-
py-pve-cloud>=3.
|
|
2
|
+
py-pve-cloud>=3.5.0,<3.6.0
|
|
3
3
|
kubernetes==34.1.0
|
|
4
4
|
requests==2.32.5
|
|
5
5
|
zstandard==0.25.0
|
|
6
|
-
|
|
7
|
-
# or removed from controller for pickeling
|
|
8
|
-
tinydb==4.8.2
|
|
6
|
+
aiosqlite==0.21.0
|
|
9
7
|
python-socketio[asyncio_client]==5.16.4
|
|
10
8
|
aiohttp==3.14.3
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "3.6.0"
|
|
@@ -7,11 +7,13 @@ import struct
|
|
|
7
7
|
|
|
8
8
|
import zstandard as zstd
|
|
9
9
|
from pve_cloud.lib.backup_rpc import Command
|
|
10
|
-
from tinydb import Query, TinyDB
|
|
11
10
|
|
|
11
|
+
from pve_cloud_backup.daemon.db import (get_stack_secrets, get_volume_metas,
|
|
12
|
+
init_db, insert_stack_secret,
|
|
13
|
+
insert_volume_meta)
|
|
12
14
|
from pve_cloud_backup.daemon.funcs import (copy_backup_generic,
|
|
13
15
|
get_backup_base_dir,
|
|
14
|
-
|
|
16
|
+
init_backup_dir)
|
|
15
17
|
from pve_cloud_backup.fetcher.net import send_cchunk
|
|
16
18
|
|
|
17
19
|
log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
|
|
@@ -188,126 +190,87 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
188
190
|
finally:
|
|
189
191
|
lock.release()
|
|
190
192
|
|
|
191
|
-
case Command.
|
|
192
|
-
# read meta dict size
|
|
193
|
+
case Command.STACK_SECRETS:
|
|
193
194
|
dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
|
|
195
|
+
logger.debug(f"reading secret dict of size {dict_size}")
|
|
194
196
|
meta_dict = pickle.loads((await reader.readexactly(dict_size)))
|
|
195
197
|
|
|
196
|
-
|
|
198
|
+
await insert_stack_secret(meta_dict)
|
|
197
199
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
secret_db = TinyDB(db_path)
|
|
201
|
-
secret_db.insert(meta_dict)
|
|
200
|
+
writer.write(b"\x01")
|
|
201
|
+
await writer.drain()
|
|
202
202
|
|
|
203
203
|
case Command.VOLUME_META:
|
|
204
204
|
dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
|
|
205
205
|
meta_dict = pickle.loads((await reader.readexactly(dict_size)))
|
|
206
|
-
db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
|
|
207
206
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
207
|
+
await insert_volume_meta(meta_dict)
|
|
208
|
+
|
|
209
|
+
writer.write(b"\x01")
|
|
210
|
+
await writer.drain()
|
|
212
211
|
|
|
213
212
|
# funcs called by brctl for restores
|
|
214
213
|
case Command.LIST_BACKUPS:
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
lock = await get_lock(db_path)
|
|
218
|
-
async with lock:
|
|
219
|
-
# we call borg on all our backups and send a return string that is strictly for display via the cli tool
|
|
220
|
-
timestamp_archives = get_volume_metas()
|
|
214
|
+
volume_metas = await get_volume_metas()
|
|
221
215
|
|
|
222
216
|
# simply return all archives
|
|
223
|
-
|
|
224
|
-
writer.write(struct.pack("!I", len(
|
|
217
|
+
metas_pickled = pickle.dumps(volume_metas)
|
|
218
|
+
writer.write(struct.pack("!I", len(metas_pickled)))
|
|
225
219
|
await writer.drain()
|
|
226
220
|
|
|
227
|
-
writer.write(
|
|
221
|
+
writer.write(metas_pickled)
|
|
228
222
|
await writer.drain()
|
|
229
223
|
|
|
230
224
|
case Command.LIST_BACKUP_DETAILS:
|
|
231
225
|
timestamp = (await reader.readline()).decode().rstrip("\n")
|
|
232
226
|
|
|
233
|
-
|
|
227
|
+
volume_metas = await get_volume_metas(timestamp_filter=timestamp)
|
|
234
228
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
# this time we need the filter for displaying details of a certain backup
|
|
239
|
-
timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
|
|
240
|
-
|
|
241
|
-
# return the archive
|
|
242
|
-
archive_pickled = pickle.dumps(timestamp_archives[timestamp])
|
|
243
|
-
writer.write(struct.pack("!I", len(archive_pickled)))
|
|
229
|
+
# return the volume metas
|
|
230
|
+
metas_pickled = pickle.dumps(volume_metas[timestamp])
|
|
231
|
+
writer.write(struct.pack("!I", len(metas_pickled)))
|
|
244
232
|
await writer.drain()
|
|
245
233
|
|
|
246
|
-
writer.write(
|
|
234
|
+
writer.write(metas_pickled)
|
|
247
235
|
await writer.drain()
|
|
248
236
|
|
|
249
|
-
# return
|
|
250
|
-
|
|
237
|
+
# return the secrets
|
|
238
|
+
stack_secrets = await get_stack_secrets(timestamp)
|
|
251
239
|
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
stack = (await reader.readline()).decode().rstrip("\n")
|
|
257
|
-
|
|
258
|
-
Meta = Query()
|
|
259
|
-
ns_secrets = secret_db.get(
|
|
260
|
-
(Meta.timestamp == timestamp) & (Meta.stack == stack)
|
|
261
|
-
)
|
|
262
|
-
|
|
263
|
-
meta_pickled = pickle.dumps(ns_secrets)
|
|
264
|
-
writer.write(struct.pack("!I", len(meta_pickled)))
|
|
265
|
-
await writer.drain()
|
|
240
|
+
meta_pickled = pickle.dumps(stack_secrets)
|
|
241
|
+
writer.write(struct.pack("!I", len(meta_pickled)))
|
|
242
|
+
await writer.drain()
|
|
266
243
|
|
|
267
|
-
|
|
268
|
-
|
|
244
|
+
writer.write(meta_pickled)
|
|
245
|
+
await writer.drain()
|
|
269
246
|
|
|
270
247
|
case Command.INIT_RESTORE_PROCEDURE:
|
|
271
248
|
timestamp = (await reader.readline()).decode().rstrip("\n")
|
|
272
249
|
logger.info("init restore procedure for %s", timestamp)
|
|
273
250
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
lock = await get_lock(db_path)
|
|
277
|
-
async with lock:
|
|
278
|
-
# we call borg on all our backups and send a return string that is strictly for display via the cli tool
|
|
279
|
-
# this time we need the filter for displaying details of a certain backup
|
|
280
|
-
timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
|
|
251
|
+
volume_metas = await get_volume_metas(timestamp_filter=timestamp)
|
|
281
252
|
|
|
282
253
|
# return the archive
|
|
283
|
-
|
|
284
|
-
writer.write(struct.pack("!I", len(
|
|
254
|
+
metas_pickled = pickle.dumps(volume_metas[timestamp])
|
|
255
|
+
writer.write(struct.pack("!I", len(metas_pickled)))
|
|
285
256
|
await writer.drain()
|
|
286
257
|
|
|
287
|
-
writer.write(
|
|
258
|
+
writer.write(metas_pickled)
|
|
288
259
|
await writer.drain()
|
|
289
260
|
|
|
290
|
-
|
|
291
|
-
# return k8s secret requests
|
|
292
|
-
db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
|
|
261
|
+
stack_secrets = await get_stack_secrets(timestamp)
|
|
293
262
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
Meta = Query()
|
|
299
|
-
ns_secrets = secret_db.get(
|
|
300
|
-
(
|
|
301
|
-
Meta.timestamp == timestamp
|
|
302
|
-
) # timestamp is our unique id for the backup
|
|
263
|
+
if not stack_secrets:
|
|
264
|
+
raise RuntimeError(
|
|
265
|
+
f"No stack secrets found for timestamp {timestamp}"
|
|
303
266
|
)
|
|
304
267
|
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
268
|
+
meta_pickled = pickle.dumps(stack_secrets)
|
|
269
|
+
writer.write(struct.pack("!I", len(meta_pickled)))
|
|
270
|
+
await writer.drain()
|
|
308
271
|
|
|
309
|
-
|
|
310
|
-
|
|
272
|
+
writer.write(meta_pickled)
|
|
273
|
+
await writer.drain()
|
|
311
274
|
|
|
312
275
|
case Command.REQUEST_ARCHIVE:
|
|
313
276
|
# next the client requests the archives which we extract here and pipe via a stream
|
|
@@ -369,6 +332,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
369
332
|
|
|
370
333
|
|
|
371
334
|
async def run():
|
|
335
|
+
await init_db()
|
|
372
336
|
certs_dir = "/opt/bdd/certs"
|
|
373
337
|
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
|
374
338
|
ssl_context.load_cert_chain(
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import subprocess
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from pve_cloud_backup.daemon.db import (get_db_timestamps,
|
|
9
|
+
prune_db_by_timestamps)
|
|
10
|
+
from pve_cloud_backup.daemon.funcs import get_backup_base_dir
|
|
11
|
+
|
|
12
|
+
log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
|
|
13
|
+
log_level = getattr(logging, log_level_str, logging.INFO)
|
|
14
|
+
|
|
15
|
+
logging.basicConfig(level=log_level)
|
|
16
|
+
logger = logging.getLogger("bdd-prune")
|
|
17
|
+
|
|
18
|
+
os.environ["BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK"] = "yes"
|
|
19
|
+
os.environ["BORG_RELOCATED_REPO_ACCESS_IS_OK"] = "yes"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def prune_repo(repo_path: str, keep_daily: int, keep_weekly: int, keep_monthly: int):
|
|
23
|
+
cmd = [
|
|
24
|
+
"borg",
|
|
25
|
+
"prune",
|
|
26
|
+
"-v",
|
|
27
|
+
"--list",
|
|
28
|
+
f"--keep-daily={keep_daily}",
|
|
29
|
+
f"--keep-weekly={keep_weekly}",
|
|
30
|
+
f"--keep-monthly={keep_monthly}",
|
|
31
|
+
repo_path,
|
|
32
|
+
]
|
|
33
|
+
subprocess.run(cmd, check=True)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def check_repo(repo_path: str, verify_data: bool):
|
|
37
|
+
cmd = ["borg", "-v", "check"]
|
|
38
|
+
if verify_data:
|
|
39
|
+
cmd.append("--verify-data")
|
|
40
|
+
|
|
41
|
+
cmd.append(repo_path)
|
|
42
|
+
subprocess.run(cmd, check=True)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def borg_list_timestamps(archive_dir: str):
|
|
46
|
+
list_result = subprocess.run(
|
|
47
|
+
["borg", "list", archive_dir, "--json"], capture_output=True, check=True
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
timestamps = []
|
|
51
|
+
archives = json.loads(list_result.stdout)["archives"]
|
|
52
|
+
for archive in archives:
|
|
53
|
+
archive_name = archive["archive"]
|
|
54
|
+
timestamp = archive_name.split("_", 1)[1]
|
|
55
|
+
timestamps.append(timestamp)
|
|
56
|
+
|
|
57
|
+
return timestamps
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def collect_borg_timestamps(base_dir: str) -> set:
|
|
61
|
+
borg_timestamps = set()
|
|
62
|
+
|
|
63
|
+
k8s_base = Path(base_dir) / "borg-k8s"
|
|
64
|
+
if k8s_base.exists():
|
|
65
|
+
for ns_dir in k8s_base.iterdir():
|
|
66
|
+
if ns_dir.is_dir():
|
|
67
|
+
borg_timestamps.update(borg_list_timestamps(str(ns_dir)))
|
|
68
|
+
|
|
69
|
+
flat_dirs = ["borg-nextcloud", "borg-git", "borg-postgres"]
|
|
70
|
+
for dir_name in flat_dirs:
|
|
71
|
+
dir_path = Path(base_dir) / dir_name
|
|
72
|
+
if dir_path.exists():
|
|
73
|
+
borg_timestamps.update(borg_list_timestamps(str(dir_path)))
|
|
74
|
+
|
|
75
|
+
return borg_timestamps
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def prune_database(borg_timestamps: set):
|
|
79
|
+
db_timestamps = get_db_timestamps()
|
|
80
|
+
# delete all timestamp ids from the sqlite database that were not found in borg
|
|
81
|
+
timestamps_to_delete = db_timestamps - borg_timestamps
|
|
82
|
+
|
|
83
|
+
logger.info(f"pruning {len(timestamps_to_delete)} timestamps from database!")
|
|
84
|
+
prune_db_by_timestamps(timestamps_to_delete)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def run_maintenance(
|
|
88
|
+
keep_daily: int, keep_weekly: int, keep_monthly: int, verify_data: bool
|
|
89
|
+
):
|
|
90
|
+
base_dir = get_backup_base_dir()
|
|
91
|
+
|
|
92
|
+
logger.info("performing prune and check on borg archives...")
|
|
93
|
+
k8s_base = Path(base_dir) / "borg-k8s"
|
|
94
|
+
if k8s_base.exists():
|
|
95
|
+
for ns_dir in k8s_base.iterdir():
|
|
96
|
+
if ns_dir.is_dir():
|
|
97
|
+
prune_repo(str(ns_dir), keep_daily, keep_weekly, keep_monthly)
|
|
98
|
+
check_repo(str(ns_dir), verify_data)
|
|
99
|
+
|
|
100
|
+
flat_dirs = ["borg-nextcloud", "borg-git", "borg-postgres"]
|
|
101
|
+
for dir_name in flat_dirs:
|
|
102
|
+
dir_path = Path(base_dir) / dir_name
|
|
103
|
+
if dir_path.exists():
|
|
104
|
+
prune_repo(str(dir_path), keep_daily, keep_weekly, keep_monthly)
|
|
105
|
+
check_repo(str(dir_path), verify_data)
|
|
106
|
+
|
|
107
|
+
borg_timestamps = collect_borg_timestamps(base_dir)
|
|
108
|
+
|
|
109
|
+
prune_database(borg_timestamps)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def main():
|
|
113
|
+
parser = argparse.ArgumentParser(
|
|
114
|
+
description="Borg repository maintenance (prune + check)"
|
|
115
|
+
)
|
|
116
|
+
parser.add_argument(
|
|
117
|
+
"--keep-daily",
|
|
118
|
+
type=int,
|
|
119
|
+
default=int(os.environ.get("KEEP_DAILY", 7)),
|
|
120
|
+
)
|
|
121
|
+
parser.add_argument(
|
|
122
|
+
"--keep-weekly",
|
|
123
|
+
type=int,
|
|
124
|
+
default=int(os.environ.get("KEEP_WEEKLY", 4)),
|
|
125
|
+
)
|
|
126
|
+
parser.add_argument(
|
|
127
|
+
"--keep-monthly",
|
|
128
|
+
type=int,
|
|
129
|
+
default=int(os.environ.get("KEEP_MONTHLY", 6)),
|
|
130
|
+
)
|
|
131
|
+
parser.add_argument(
|
|
132
|
+
"--verify-data",
|
|
133
|
+
action="store_true",
|
|
134
|
+
default=os.environ.get("VERIFY_DATA", "").lower() in ("1", "true", "yes"),
|
|
135
|
+
help="Run borg check with --verify-data",
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
args = parser.parse_args()
|
|
139
|
+
run_maintenance(
|
|
140
|
+
args.keep_daily, args.keep_weekly, args.keep_monthly, args.verify_data
|
|
141
|
+
)
|
{py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/brctl.py
RENAMED
|
@@ -52,8 +52,8 @@ async def list_backup_details_remote(args):
|
|
|
52
52
|
# launch our rpc service to read discovery secrets and prepare
|
|
53
53
|
# acessing the backup server
|
|
54
54
|
logger.info(f"connecting to {pve_host} via {jump_host}")
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
volume_metas = None
|
|
56
|
+
stack_secrets = None
|
|
57
57
|
|
|
58
58
|
async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
|
|
59
59
|
if args.use_mc_gw:
|
|
@@ -79,8 +79,8 @@ async def list_backup_details_remote(args):
|
|
|
79
79
|
|
|
80
80
|
sio.disconnect()
|
|
81
81
|
|
|
82
|
-
|
|
83
|
-
|
|
82
|
+
volume_metas = result["volume_metas"]
|
|
83
|
+
stack_secrets = result["stack_secrets"]
|
|
84
84
|
else:
|
|
85
85
|
# we will connect directly to the backup server
|
|
86
86
|
# todo: here we can also pass the correct tls config
|
|
@@ -112,29 +112,21 @@ async def list_backup_details_remote(args):
|
|
|
112
112
|
|
|
113
113
|
# read the archives
|
|
114
114
|
dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
# first we group metas
|
|
118
|
-
k8s_stack = metas[0]["stack"]
|
|
119
|
-
|
|
120
|
-
print(f"k8s stack {k8s_stack}:")
|
|
121
|
-
|
|
122
|
-
# query the server for backup secrets
|
|
123
|
-
writer.write((k8s_stack + "\n").encode())
|
|
124
|
-
await writer.drain()
|
|
115
|
+
volume_metas = pickle.loads((await reader.readexactly(dict_size)))
|
|
125
116
|
|
|
126
117
|
# read the the meta information
|
|
127
118
|
dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
|
|
128
|
-
|
|
119
|
+
stack_secrets = pickle.loads((await reader.readexactly(dict_size)))
|
|
129
120
|
|
|
121
|
+
# load secrets
|
|
130
122
|
namespace_secret_dict = pickle.loads(
|
|
131
|
-
base64.b64decode(
|
|
123
|
+
base64.b64decode(stack_secrets["namespace_secret_dict_b64"])
|
|
132
124
|
)
|
|
133
125
|
|
|
134
126
|
namespace_k8s_metas = {}
|
|
135
127
|
|
|
136
128
|
# group metas by namespace
|
|
137
|
-
for meta in
|
|
129
|
+
for meta in volume_metas:
|
|
138
130
|
if meta["namespace"] not in namespace_k8s_metas:
|
|
139
131
|
namespace_k8s_metas[meta["namespace"]] = []
|
|
140
132
|
|
|
@@ -205,7 +197,7 @@ async def list_backups_remote(args):
|
|
|
205
197
|
# launch our rpc service to read discovery secrets and prepare
|
|
206
198
|
# acessing the backup server
|
|
207
199
|
logger.info(f"connecting to {pve_host} via {jump_host}")
|
|
208
|
-
|
|
200
|
+
volume_metas = None
|
|
209
201
|
|
|
210
202
|
async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
|
|
211
203
|
if args.use_mc_gw:
|
|
@@ -230,7 +222,7 @@ async def list_backups_remote(args):
|
|
|
230
222
|
|
|
231
223
|
sio.disconnect()
|
|
232
224
|
|
|
233
|
-
|
|
225
|
+
volume_metas = result["volume_metas"]
|
|
234
226
|
|
|
235
227
|
else:
|
|
236
228
|
tls_disc_raw = await pxrpc.get_cloud_secret(
|
|
@@ -255,15 +247,15 @@ async def list_backups_remote(args):
|
|
|
255
247
|
|
|
256
248
|
# read the response archives size and then the archives
|
|
257
249
|
dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
|
|
258
|
-
|
|
250
|
+
volume_metas = pickle.loads((await reader.readexactly(dict_size)))
|
|
259
251
|
|
|
260
252
|
if args.json:
|
|
261
|
-
print(json.dumps(sorted(
|
|
253
|
+
print(json.dumps(sorted(volume_metas)))
|
|
262
254
|
return
|
|
263
255
|
|
|
264
256
|
print("available backup timestamps (ids):")
|
|
265
257
|
|
|
266
|
-
for timestamp in sorted(
|
|
258
|
+
for timestamp in sorted(volume_metas):
|
|
267
259
|
print(f"- timestamp {timestamp}")
|
|
268
260
|
|
|
269
261
|
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import sqlite3
|
|
3
|
+
from typing import Optional, Set
|
|
4
|
+
|
|
5
|
+
import aiosqlite
|
|
6
|
+
|
|
7
|
+
DB_PATH = "/mnt/bdd-zfs-data/backup-metadata.db"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
async def init_db():
|
|
11
|
+
# todo: this should be refactored to a deduplicated approach
|
|
12
|
+
# as namespace secrets can get quite large and daily backups will
|
|
13
|
+
# result in a big db
|
|
14
|
+
async with aiosqlite.connect(DB_PATH) as db:
|
|
15
|
+
# create table for storing kubernetes volume related information
|
|
16
|
+
await db.execute(
|
|
17
|
+
"""
|
|
18
|
+
CREATE TABLE IF NOT EXISTS volume_meta (
|
|
19
|
+
image_name TEXT NOT NULL,
|
|
20
|
+
timestamp TEXT NOT NULL,
|
|
21
|
+
data_json TEXT NOT NULL,
|
|
22
|
+
PRIMARY KEY (image_name, timestamp)
|
|
23
|
+
)
|
|
24
|
+
"""
|
|
25
|
+
)
|
|
26
|
+
await db.execute(
|
|
27
|
+
"CREATE INDEX IF NOT EXISTS idx_volume_meta_timestamp ON volume_meta(timestamp)"
|
|
28
|
+
)
|
|
29
|
+
await db.execute(
|
|
30
|
+
"CREATE INDEX IF NOT EXISTS idx_volume_meta_image ON volume_meta(image_name)"
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
# table for storing secrets of multiple k8s namespaces per row (grouped by proxmox cloud stack)
|
|
34
|
+
await db.execute(
|
|
35
|
+
"""
|
|
36
|
+
CREATE TABLE IF NOT EXISTS stack_secrets (
|
|
37
|
+
timestamp TEXT PRIMARY KEY,
|
|
38
|
+
stack TEXT NOT NULL,
|
|
39
|
+
data_json TEXT NOT NULL
|
|
40
|
+
)
|
|
41
|
+
"""
|
|
42
|
+
)
|
|
43
|
+
await db.execute(
|
|
44
|
+
"CREATE INDEX IF NOT EXISTS idx_stack_secrets_timestamp ON stack_secrets(timestamp)"
|
|
45
|
+
)
|
|
46
|
+
await db.execute(
|
|
47
|
+
"CREATE INDEX IF NOT EXISTS idx_stack_secrets_stack ON stack_secrets(stack)"
|
|
48
|
+
)
|
|
49
|
+
await db.commit()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def insert_volume_meta(meta_dict: dict) -> int:
|
|
53
|
+
image_name = meta_dict["image_name"]
|
|
54
|
+
timestamp = meta_dict["timestamp"]
|
|
55
|
+
|
|
56
|
+
async with aiosqlite.connect(DB_PATH) as db:
|
|
57
|
+
cursor = await db.execute(
|
|
58
|
+
"INSERT OR REPLACE INTO volume_meta (image_name, timestamp, data_json) VALUES (?, ?, ?)",
|
|
59
|
+
(image_name, timestamp, json.dumps(meta_dict)),
|
|
60
|
+
)
|
|
61
|
+
await db.commit()
|
|
62
|
+
return cursor.lastrowid
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def get_volume_meta(image_name: str, timestamp: str) -> Optional[dict]:
|
|
66
|
+
"""
|
|
67
|
+
Get metadata of a single kubernetes pv + pvc.
|
|
68
|
+
"""
|
|
69
|
+
async with aiosqlite.connect(DB_PATH) as db:
|
|
70
|
+
cursor = await db.execute(
|
|
71
|
+
"SELECT data_json FROM volume_meta WHERE image_name = ? AND timestamp = ?",
|
|
72
|
+
(image_name, timestamp),
|
|
73
|
+
)
|
|
74
|
+
row = await cursor.fetchone()
|
|
75
|
+
if row:
|
|
76
|
+
return json.loads(row[0])
|
|
77
|
+
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
async def get_volume_metas(timestamp_filter: Optional[str] = None) -> dict:
|
|
82
|
+
"""
|
|
83
|
+
Query the database for mutliple k8s pv + pvc volume metadata entries. Returns them in a map, grouped by their
|
|
84
|
+
backup timestamp id.
|
|
85
|
+
"""
|
|
86
|
+
async with aiosqlite.connect(DB_PATH) as db:
|
|
87
|
+
if timestamp_filter:
|
|
88
|
+
cursor = await db.execute(
|
|
89
|
+
"SELECT timestamp, data_json FROM volume_meta WHERE timestamp = ?",
|
|
90
|
+
(timestamp_filter,),
|
|
91
|
+
)
|
|
92
|
+
else:
|
|
93
|
+
cursor = await db.execute("SELECT timestamp, data_json FROM volume_meta")
|
|
94
|
+
|
|
95
|
+
rows = await cursor.fetchall()
|
|
96
|
+
result = {}
|
|
97
|
+
for row in rows:
|
|
98
|
+
timestamp = row[0]
|
|
99
|
+
if timestamp not in result:
|
|
100
|
+
result[timestamp] = []
|
|
101
|
+
result[timestamp].append(json.loads(row[1]))
|
|
102
|
+
return result
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
async def insert_stack_secret(data: dict) -> int:
|
|
106
|
+
timestamp = data["timestamp"]
|
|
107
|
+
stack = data["stack"]
|
|
108
|
+
|
|
109
|
+
async with aiosqlite.connect(DB_PATH) as db:
|
|
110
|
+
cursor = await db.execute(
|
|
111
|
+
"INSERT INTO stack_secrets (timestamp, stack, data_json) VALUES (?, ?, ?)",
|
|
112
|
+
(timestamp, stack, json.dumps(data)),
|
|
113
|
+
)
|
|
114
|
+
await db.commit()
|
|
115
|
+
return cursor.lastrowid
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
async def get_stack_secrets(timestamp: str) -> Optional[dict]:
|
|
119
|
+
async with aiosqlite.connect(DB_PATH) as db:
|
|
120
|
+
cursor = await db.execute(
|
|
121
|
+
"SELECT data_json FROM stack_secrets WHERE timestamp = ?", (timestamp,)
|
|
122
|
+
)
|
|
123
|
+
row = await cursor.fetchone()
|
|
124
|
+
if row:
|
|
125
|
+
return json.loads(row[0])
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def get_db_timestamps() -> Set[str]:
|
|
130
|
+
timestamps = set()
|
|
131
|
+
|
|
132
|
+
with sqlite3.connect(DB_PATH) as conn:
|
|
133
|
+
cursor = conn.cursor()
|
|
134
|
+
cursor.execute("SELECT DISTINCT timestamp FROM volume_meta")
|
|
135
|
+
for row in cursor.fetchall():
|
|
136
|
+
timestamps.add(row[0])
|
|
137
|
+
|
|
138
|
+
cursor.execute("SELECT DISTINCT timestamp FROM stack_secrets")
|
|
139
|
+
for row in cursor.fetchall():
|
|
140
|
+
timestamps.add(row[0])
|
|
141
|
+
|
|
142
|
+
return timestamps
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def prune_db_by_timestamps(timestamps_to_delete: Set[str]):
|
|
146
|
+
with sqlite3.connect(DB_PATH) as conn:
|
|
147
|
+
# create temp table with timestamp ids that will be deleted
|
|
148
|
+
conn.execute(
|
|
149
|
+
"""
|
|
150
|
+
CREATE TEMP TABLE IF NOT EXISTS timestamps_to_delete (
|
|
151
|
+
timestamp TEXT PRIMARY KEY
|
|
152
|
+
)
|
|
153
|
+
"""
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
conn.executemany(
|
|
157
|
+
"INSERT OR IGNORE INTO timestamps_to_delete (timestamp) VALUES (?)",
|
|
158
|
+
((ts,) for ts in timestamps_to_delete),
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# prune the two tables
|
|
162
|
+
conn.execute(
|
|
163
|
+
"""
|
|
164
|
+
DELETE FROM volume_meta
|
|
165
|
+
WHERE EXISTS (
|
|
166
|
+
SELECT 1
|
|
167
|
+
FROM timestamps_to_delete d
|
|
168
|
+
WHERE d.timestamp = volume_meta.timestamp
|
|
169
|
+
)
|
|
170
|
+
"""
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
conn.execute(
|
|
174
|
+
"""
|
|
175
|
+
DELETE FROM stack_secrets
|
|
176
|
+
WHERE EXISTS (
|
|
177
|
+
SELECT 1
|
|
178
|
+
FROM timestamps_to_delete d
|
|
179
|
+
WHERE d.timestamp = stack_secrets.timestamp
|
|
180
|
+
)
|
|
181
|
+
"""
|
|
182
|
+
)
|
{py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/funcs.py
RENAMED
|
@@ -7,7 +7,8 @@ from pathlib import Path
|
|
|
7
7
|
|
|
8
8
|
from kubernetes import client
|
|
9
9
|
from kubernetes.client.rest import ApiException
|
|
10
|
-
|
|
10
|
+
|
|
11
|
+
from pve_cloud_backup.daemon.db import get_volume_metas as db_get_volume_metas
|
|
11
12
|
|
|
12
13
|
logger = logging.getLogger("bdd")
|
|
13
14
|
|
|
@@ -51,12 +52,9 @@ def copy_backup_generic():
|
|
|
51
52
|
shutil.copy2(full_source_path, full_dest_path)
|
|
52
53
|
|
|
53
54
|
|
|
54
|
-
def get_volume_metas(timestamp_filter=None):
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
archives = []
|
|
55
|
+
async def get_volume_metas(timestamp_filter=None):
|
|
56
|
+
db_result = await db_get_volume_metas(timestamp_filter)
|
|
58
57
|
|
|
59
|
-
# iterate all k8s namespaced borg repos
|
|
60
58
|
k8s_base_path = f"{get_backup_base_dir()}/borg-k8s"
|
|
61
59
|
namespace_repos = [
|
|
62
60
|
name
|
|
@@ -64,6 +62,7 @@ def get_volume_metas(timestamp_filter=None):
|
|
|
64
62
|
if os.path.isdir(os.path.join(k8s_base_path, name))
|
|
65
63
|
]
|
|
66
64
|
|
|
65
|
+
archives = []
|
|
67
66
|
for repo in namespace_repos:
|
|
68
67
|
list_result = subprocess.run(
|
|
69
68
|
["borg", "list", f"{get_backup_base_dir()}/borg-k8s/{repo}", "--json"],
|
|
@@ -77,29 +76,18 @@ def get_volume_metas(timestamp_filter=None):
|
|
|
77
76
|
|
|
78
77
|
archives.extend(json.loads(list_result.stdout)["archives"])
|
|
79
78
|
|
|
80
|
-
|
|
79
|
+
result = {}
|
|
81
80
|
for archive in archives:
|
|
82
81
|
image = archive["archive"].split("_", 1)[0]
|
|
83
82
|
timestamp = archive["archive"].split("_", 1)[1]
|
|
84
83
|
|
|
85
84
|
if timestamp_filter is not None and timestamp_filter != timestamp:
|
|
86
|
-
continue # skip filtered
|
|
87
|
-
|
|
88
|
-
if timestamp not in timestamp_archives:
|
|
89
|
-
timestamp_archives[timestamp] = []
|
|
90
|
-
|
|
91
|
-
Meta = Query()
|
|
92
|
-
image_meta = volume_meta_db.get(
|
|
93
|
-
(Meta.image_name == image) & (Meta.timestamp == timestamp)
|
|
94
|
-
)
|
|
95
|
-
|
|
96
|
-
if image_meta is None:
|
|
97
|
-
logger.error(
|
|
98
|
-
f"None meta found {timestamp}, image_name {image}, archive {archive}"
|
|
99
|
-
)
|
|
100
|
-
del timestamp_archives[timestamp]
|
|
101
85
|
continue
|
|
102
86
|
|
|
103
|
-
|
|
87
|
+
if timestamp not in result:
|
|
88
|
+
result[timestamp] = []
|
|
89
|
+
|
|
90
|
+
if timestamp in db_result:
|
|
91
|
+
result[timestamp].extend(db_result[timestamp])
|
|
104
92
|
|
|
105
|
-
return
|
|
93
|
+
return result
|
{py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/funcs.py
RENAMED
|
@@ -333,9 +333,7 @@ async def post_volume_meta(namespace_volume_meta, timestamp, k8s_stack, backup_a
|
|
|
333
333
|
await net.volume_meta(backup_addr, body)
|
|
334
334
|
|
|
335
335
|
|
|
336
|
-
async def
|
|
337
|
-
namespace_secrets, timestamp, k8s_stack, backup_addr
|
|
338
|
-
):
|
|
336
|
+
async def post_k8s_stack_secrets(namespace_secrets, timestamp, k8s_stack, backup_addr):
|
|
339
337
|
|
|
340
338
|
namespace_secret_dict_b64 = base64.b64encode(
|
|
341
339
|
pickle.dumps(namespace_secrets)
|
|
@@ -347,7 +345,7 @@ async def post_k8s_namespace_secrets(
|
|
|
347
345
|
}
|
|
348
346
|
logger.debug(f"posting {body}")
|
|
349
347
|
|
|
350
|
-
await net.
|
|
348
|
+
await net.stack_secrets(backup_addr, body)
|
|
351
349
|
|
|
352
350
|
|
|
353
351
|
def cleanup(namespace_volume_meta, timestamp, unique_pools):
|
{py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/net.py
RENAMED
|
@@ -231,6 +231,8 @@ async def archive(backup_addr, request_dict, chunk_generator, compress=True):
|
|
|
231
231
|
|
|
232
232
|
|
|
233
233
|
async def meta(backup_addr, cmd, meta_dict):
|
|
234
|
+
logger.info("sending meta request: %s", cmd)
|
|
235
|
+
|
|
234
236
|
if backup_addr.startswith("https://"):
|
|
235
237
|
sio = await get_sio_mc_client(backup_addr)
|
|
236
238
|
|
|
@@ -255,6 +257,10 @@ async def meta(backup_addr, cmd, meta_dict):
|
|
|
255
257
|
writer.write(meta_pickled)
|
|
256
258
|
await writer.drain()
|
|
257
259
|
|
|
260
|
+
ack = await reader.readexactly(1)
|
|
261
|
+
if ack != b"\x01":
|
|
262
|
+
raise RuntimeError("Expected x01 ack byte!")
|
|
263
|
+
|
|
258
264
|
writer.close()
|
|
259
265
|
|
|
260
266
|
|
|
@@ -262,5 +268,5 @@ async def volume_meta(backup_addr, meta_dict):
|
|
|
262
268
|
await meta(backup_addr, Command.VOLUME_META, meta_dict)
|
|
263
269
|
|
|
264
270
|
|
|
265
|
-
async def
|
|
266
|
-
await meta(backup_addr, Command.
|
|
271
|
+
async def stack_secrets(backup_addr, meta_dict):
|
|
272
|
+
await meta(backup_addr, Command.STACK_SECRETS, meta_dict)
|
{py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0/src/py_pve_cloud_backup.egg-info}/PKG-INFO
RENAMED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: py-pve-cloud-backup
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.6.0
|
|
4
4
|
Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
|
|
5
5
|
License-Expression: GPL-3.0-or-later
|
|
6
6
|
License-File: LICENSE.md
|
|
7
|
-
Requires-Dist: py-pve-cloud<3.
|
|
7
|
+
Requires-Dist: py-pve-cloud<3.6.0,>=3.5.0
|
|
8
8
|
Requires-Dist: kubernetes==34.1.0
|
|
9
9
|
Requires-Dist: requests==2.32.5
|
|
10
10
|
Requires-Dist: zstandard==0.25.0
|
|
11
|
-
Requires-Dist:
|
|
11
|
+
Requires-Dist: aiosqlite==0.21.0
|
|
12
12
|
Requires-Dist: python-socketio[asyncio_client]==5.16.4
|
|
13
13
|
Requires-Dist: aiohttp==3.14.3
|
|
14
14
|
Dynamic: license-file
|
|
@@ -4,7 +4,9 @@ pyproject.toml
|
|
|
4
4
|
requirements.txt
|
|
5
5
|
src/pve_cloud_backup/_version.py
|
|
6
6
|
src/pve_cloud_backup/daemon/bdd.py
|
|
7
|
+
src/pve_cloud_backup/daemon/bdd_prune.py
|
|
7
8
|
src/pve_cloud_backup/daemon/brctl.py
|
|
9
|
+
src/pve_cloud_backup/daemon/db.py
|
|
8
10
|
src/pve_cloud_backup/daemon/funcs.py
|
|
9
11
|
src/pve_cloud_backup/daemon/restore.py
|
|
10
12
|
src/pve_cloud_backup/fetcher/ext_zfs.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "3.5.18"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/restore.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/git.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/nextcloud.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/patroni.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|