py-pve-cloud-backup 3.5.18__tar.gz → 3.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {py_pve_cloud_backup-3.5.18/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.6.0}/PKG-INFO +3 -3
  2. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/pyproject.toml +1 -0
  3. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/requirements.txt +2 -4
  4. py_pve_cloud_backup-3.6.0/src/pve_cloud_backup/_version.py +1 -0
  5. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/bdd.py +43 -79
  6. py_pve_cloud_backup-3.6.0/src/pve_cloud_backup/daemon/bdd_prune.py +141 -0
  7. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/brctl.py +14 -22
  8. py_pve_cloud_backup-3.6.0/src/pve_cloud_backup/daemon/db.py +182 -0
  9. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/funcs.py +12 -24
  10. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/ext_zfs.py +1 -1
  11. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/funcs.py +2 -4
  12. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/main.py +1 -1
  13. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/net.py +8 -2
  14. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0/src/py_pve_cloud_backup.egg-info}/PKG-INFO +3 -3
  15. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +2 -0
  16. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/entry_points.txt +1 -0
  17. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/requires.txt +2 -2
  18. py_pve_cloud_backup-3.5.18/src/pve_cloud_backup/_version.py +0 -1
  19. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/LICENSE.md +0 -0
  20. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/README.md +0 -0
  21. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/setup.cfg +0 -0
  22. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/daemon/restore.py +0 -0
  23. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/git.py +0 -0
  24. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
  25. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
  26. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
  27. {py_pve_cloud_backup-3.5.18 → py_pve_cloud_backup-3.6.0}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
@@ -1,14 +1,14 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.18
3
+ Version: 3.6.0
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
7
- Requires-Dist: py-pve-cloud<3.5.0,>=3.4.6
7
+ Requires-Dist: py-pve-cloud<3.6.0,>=3.5.0
8
8
  Requires-Dist: kubernetes==34.1.0
9
9
  Requires-Dist: requests==2.32.5
10
10
  Requires-Dist: zstandard==0.25.0
11
- Requires-Dist: tinydb==4.8.2
11
+ Requires-Dist: aiosqlite==0.21.0
12
12
  Requires-Dist: python-socketio[asyncio_client]==5.16.4
13
13
  Requires-Dist: aiohttp==3.14.3
14
14
  Dynamic: license-file
@@ -19,3 +19,4 @@ brctl = "pve_cloud_backup.daemon.brctl:main"
19
19
  fetcher = "pve_cloud_backup.fetcher.main:main"
20
20
  ext-zfs = "pve_cloud_backup.fetcher.ext_zfs:main"
21
21
  pxc-restore = "pve_cloud_backup.daemon.restore:main"
22
+ bdd-prune = "pve_cloud_backup.daemon.bdd_prune:main"
@@ -1,10 +1,8 @@
1
1
  # pessimistic operator for local tdd installs (pip install -e .)
2
- py-pve-cloud>=3.4.6,<3.5.0
2
+ py-pve-cloud>=3.5.0,<3.6.0
3
3
  kubernetes==34.1.0
4
4
  requests==2.32.5
5
5
  zstandard==0.25.0
6
- # todo: this should probably be moved to top level py-pve-cloud
7
- # or removed from controller for pickeling
8
- tinydb==4.8.2
6
+ aiosqlite==0.21.0
9
7
  python-socketio[asyncio_client]==5.16.4
10
8
  aiohttp==3.14.3
@@ -0,0 +1 @@
1
+ __version__ = "3.6.0"
@@ -7,11 +7,13 @@ import struct
7
7
 
8
8
  import zstandard as zstd
9
9
  from pve_cloud.lib.backup_rpc import Command
10
- from tinydb import Query, TinyDB
11
10
 
11
+ from pve_cloud_backup.daemon.db import (get_stack_secrets, get_volume_metas,
12
+ init_db, insert_stack_secret,
13
+ insert_volume_meta)
12
14
  from pve_cloud_backup.daemon.funcs import (copy_backup_generic,
13
15
  get_backup_base_dir,
14
- get_volume_metas, init_backup_dir)
16
+ init_backup_dir)
15
17
  from pve_cloud_backup.fetcher.net import send_cchunk
16
18
 
17
19
  log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
@@ -188,126 +190,87 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
188
190
  finally:
189
191
  lock.release()
190
192
 
191
- case Command.NAMESPACE_SECRETS:
192
- # read meta dict size
193
+ case Command.STACK_SECRETS:
193
194
  dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
195
+ logger.debug(f"reading secret dict of size {dict_size}")
194
196
  meta_dict = pickle.loads((await reader.readexactly(dict_size)))
195
197
 
196
- db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
198
+ await insert_stack_secret(meta_dict)
197
199
 
198
- lock = await get_lock(db_path)
199
- async with lock:
200
- secret_db = TinyDB(db_path)
201
- secret_db.insert(meta_dict)
200
+ writer.write(b"\x01")
201
+ await writer.drain()
202
202
 
203
203
  case Command.VOLUME_META:
204
204
  dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
205
205
  meta_dict = pickle.loads((await reader.readexactly(dict_size)))
206
- db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
207
206
 
208
- lock = await get_lock(db_path)
209
- async with lock:
210
- secret_db = TinyDB(db_path)
211
- secret_db.insert(meta_dict)
207
+ await insert_volume_meta(meta_dict)
208
+
209
+ writer.write(b"\x01")
210
+ await writer.drain()
212
211
 
213
212
  # funcs called by brctl for restores
214
213
  case Command.LIST_BACKUPS:
215
- db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
216
-
217
- lock = await get_lock(db_path)
218
- async with lock:
219
- # we call borg on all our backups and send a return string that is strictly for display via the cli tool
220
- timestamp_archives = get_volume_metas()
214
+ volume_metas = await get_volume_metas()
221
215
 
222
216
  # simply return all archives
223
- archives_pickled = pickle.dumps(timestamp_archives)
224
- writer.write(struct.pack("!I", len(archives_pickled)))
217
+ metas_pickled = pickle.dumps(volume_metas)
218
+ writer.write(struct.pack("!I", len(metas_pickled)))
225
219
  await writer.drain()
226
220
 
227
- writer.write(archives_pickled)
221
+ writer.write(metas_pickled)
228
222
  await writer.drain()
229
223
 
230
224
  case Command.LIST_BACKUP_DETAILS:
231
225
  timestamp = (await reader.readline()).decode().rstrip("\n")
232
226
 
233
- db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
227
+ volume_metas = await get_volume_metas(timestamp_filter=timestamp)
234
228
 
235
- lock = await get_lock(db_path)
236
- async with lock:
237
- # we call borg on all our backups and send a return string that is strictly for display via the cli tool
238
- # this time we need the filter for displaying details of a certain backup
239
- timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
240
-
241
- # return the archive
242
- archive_pickled = pickle.dumps(timestamp_archives[timestamp])
243
- writer.write(struct.pack("!I", len(archive_pickled)))
229
+ # return the volume metas
230
+ metas_pickled = pickle.dumps(volume_metas[timestamp])
231
+ writer.write(struct.pack("!I", len(metas_pickled)))
244
232
  await writer.drain()
245
233
 
246
- writer.write(archive_pickled)
234
+ writer.write(metas_pickled)
247
235
  await writer.drain()
248
236
 
249
- # return k8s secret requests
250
- db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
237
+ # return the secrets
238
+ stack_secrets = await get_stack_secrets(timestamp)
251
239
 
252
- lock = await get_lock(db_path)
253
- async with lock:
254
- secret_db = TinyDB(db_path)
255
-
256
- stack = (await reader.readline()).decode().rstrip("\n")
257
-
258
- Meta = Query()
259
- ns_secrets = secret_db.get(
260
- (Meta.timestamp == timestamp) & (Meta.stack == stack)
261
- )
262
-
263
- meta_pickled = pickle.dumps(ns_secrets)
264
- writer.write(struct.pack("!I", len(meta_pickled)))
265
- await writer.drain()
240
+ meta_pickled = pickle.dumps(stack_secrets)
241
+ writer.write(struct.pack("!I", len(meta_pickled)))
242
+ await writer.drain()
266
243
 
267
- writer.write(meta_pickled)
268
- await writer.drain()
244
+ writer.write(meta_pickled)
245
+ await writer.drain()
269
246
 
270
247
  case Command.INIT_RESTORE_PROCEDURE:
271
248
  timestamp = (await reader.readline()).decode().rstrip("\n")
272
249
  logger.info("init restore procedure for %s", timestamp)
273
250
 
274
- db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
275
-
276
- lock = await get_lock(db_path)
277
- async with lock:
278
- # we call borg on all our backups and send a return string that is strictly for display via the cli tool
279
- # this time we need the filter for displaying details of a certain backup
280
- timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
251
+ volume_metas = await get_volume_metas(timestamp_filter=timestamp)
281
252
 
282
253
  # return the archive
283
- archive_pickled = pickle.dumps(timestamp_archives[timestamp])
284
- writer.write(struct.pack("!I", len(archive_pickled)))
254
+ metas_pickled = pickle.dumps(volume_metas[timestamp])
255
+ writer.write(struct.pack("!I", len(metas_pickled)))
285
256
  await writer.drain()
286
257
 
287
- writer.write(archive_pickled)
258
+ writer.write(metas_pickled)
288
259
  await writer.drain()
289
260
 
290
- # client then queries secrets of the backup to restore
291
- # return k8s secret requests
292
- db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
261
+ stack_secrets = await get_stack_secrets(timestamp)
293
262
 
294
- lock = await get_lock(db_path)
295
- async with lock:
296
- secret_db = TinyDB(db_path)
297
-
298
- Meta = Query()
299
- ns_secrets = secret_db.get(
300
- (
301
- Meta.timestamp == timestamp
302
- ) # timestamp is our unique id for the backup
263
+ if not stack_secrets:
264
+ raise RuntimeError(
265
+ f"No stack secrets found for timestamp {timestamp}"
303
266
  )
304
267
 
305
- meta_pickled = pickle.dumps(ns_secrets)
306
- writer.write(struct.pack("!I", len(meta_pickled)))
307
- await writer.drain()
268
+ meta_pickled = pickle.dumps(stack_secrets)
269
+ writer.write(struct.pack("!I", len(meta_pickled)))
270
+ await writer.drain()
308
271
 
309
- writer.write(meta_pickled)
310
- await writer.drain()
272
+ writer.write(meta_pickled)
273
+ await writer.drain()
311
274
 
312
275
  case Command.REQUEST_ARCHIVE:
313
276
  # next the client requests the archives which we extract here and pipe via a stream
@@ -369,6 +332,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
369
332
 
370
333
 
371
334
  async def run():
335
+ await init_db()
372
336
  certs_dir = "/opt/bdd/certs"
373
337
  ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
374
338
  ssl_context.load_cert_chain(
@@ -0,0 +1,141 @@
1
+ import argparse
2
+ import json
3
+ import logging
4
+ import os
5
+ import subprocess
6
+ from pathlib import Path
7
+
8
+ from pve_cloud_backup.daemon.db import (get_db_timestamps,
9
+ prune_db_by_timestamps)
10
+ from pve_cloud_backup.daemon.funcs import get_backup_base_dir
11
+
12
+ log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
13
+ log_level = getattr(logging, log_level_str, logging.INFO)
14
+
15
+ logging.basicConfig(level=log_level)
16
+ logger = logging.getLogger("bdd-prune")
17
+
18
+ os.environ["BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK"] = "yes"
19
+ os.environ["BORG_RELOCATED_REPO_ACCESS_IS_OK"] = "yes"
20
+
21
+
22
+ def prune_repo(repo_path: str, keep_daily: int, keep_weekly: int, keep_monthly: int):
23
+ cmd = [
24
+ "borg",
25
+ "prune",
26
+ "-v",
27
+ "--list",
28
+ f"--keep-daily={keep_daily}",
29
+ f"--keep-weekly={keep_weekly}",
30
+ f"--keep-monthly={keep_monthly}",
31
+ repo_path,
32
+ ]
33
+ subprocess.run(cmd, check=True)
34
+
35
+
36
+ def check_repo(repo_path: str, verify_data: bool):
37
+ cmd = ["borg", "-v", "check"]
38
+ if verify_data:
39
+ cmd.append("--verify-data")
40
+
41
+ cmd.append(repo_path)
42
+ subprocess.run(cmd, check=True)
43
+
44
+
45
+ def borg_list_timestamps(archive_dir: str):
46
+ list_result = subprocess.run(
47
+ ["borg", "list", archive_dir, "--json"], capture_output=True, check=True
48
+ )
49
+
50
+ timestamps = []
51
+ archives = json.loads(list_result.stdout)["archives"]
52
+ for archive in archives:
53
+ archive_name = archive["archive"]
54
+ timestamp = archive_name.split("_", 1)[1]
55
+ timestamps.append(timestamp)
56
+
57
+ return timestamps
58
+
59
+
60
+ def collect_borg_timestamps(base_dir: str) -> set:
61
+ borg_timestamps = set()
62
+
63
+ k8s_base = Path(base_dir) / "borg-k8s"
64
+ if k8s_base.exists():
65
+ for ns_dir in k8s_base.iterdir():
66
+ if ns_dir.is_dir():
67
+ borg_timestamps.update(borg_list_timestamps(str(ns_dir)))
68
+
69
+ flat_dirs = ["borg-nextcloud", "borg-git", "borg-postgres"]
70
+ for dir_name in flat_dirs:
71
+ dir_path = Path(base_dir) / dir_name
72
+ if dir_path.exists():
73
+ borg_timestamps.update(borg_list_timestamps(str(dir_path)))
74
+
75
+ return borg_timestamps
76
+
77
+
78
+ def prune_database(borg_timestamps: set):
79
+ db_timestamps = get_db_timestamps()
80
+ # delete all timestamp ids from the sqlite database that were not found in borg
81
+ timestamps_to_delete = db_timestamps - borg_timestamps
82
+
83
+ logger.info(f"pruning {len(timestamps_to_delete)} timestamps from database!")
84
+ prune_db_by_timestamps(timestamps_to_delete)
85
+
86
+
87
+ def run_maintenance(
88
+ keep_daily: int, keep_weekly: int, keep_monthly: int, verify_data: bool
89
+ ):
90
+ base_dir = get_backup_base_dir()
91
+
92
+ logger.info("performing prune and check on borg archives...")
93
+ k8s_base = Path(base_dir) / "borg-k8s"
94
+ if k8s_base.exists():
95
+ for ns_dir in k8s_base.iterdir():
96
+ if ns_dir.is_dir():
97
+ prune_repo(str(ns_dir), keep_daily, keep_weekly, keep_monthly)
98
+ check_repo(str(ns_dir), verify_data)
99
+
100
+ flat_dirs = ["borg-nextcloud", "borg-git", "borg-postgres"]
101
+ for dir_name in flat_dirs:
102
+ dir_path = Path(base_dir) / dir_name
103
+ if dir_path.exists():
104
+ prune_repo(str(dir_path), keep_daily, keep_weekly, keep_monthly)
105
+ check_repo(str(dir_path), verify_data)
106
+
107
+ borg_timestamps = collect_borg_timestamps(base_dir)
108
+
109
+ prune_database(borg_timestamps)
110
+
111
+
112
+ def main():
113
+ parser = argparse.ArgumentParser(
114
+ description="Borg repository maintenance (prune + check)"
115
+ )
116
+ parser.add_argument(
117
+ "--keep-daily",
118
+ type=int,
119
+ default=int(os.environ.get("KEEP_DAILY", 7)),
120
+ )
121
+ parser.add_argument(
122
+ "--keep-weekly",
123
+ type=int,
124
+ default=int(os.environ.get("KEEP_WEEKLY", 4)),
125
+ )
126
+ parser.add_argument(
127
+ "--keep-monthly",
128
+ type=int,
129
+ default=int(os.environ.get("KEEP_MONTHLY", 6)),
130
+ )
131
+ parser.add_argument(
132
+ "--verify-data",
133
+ action="store_true",
134
+ default=os.environ.get("VERIFY_DATA", "").lower() in ("1", "true", "yes"),
135
+ help="Run borg check with --verify-data",
136
+ )
137
+
138
+ args = parser.parse_args()
139
+ run_maintenance(
140
+ args.keep_daily, args.keep_weekly, args.keep_monthly, args.verify_data
141
+ )
@@ -52,8 +52,8 @@ async def list_backup_details_remote(args):
52
52
  # launch our rpc service to read discovery secrets and prepare
53
53
  # acessing the backup server
54
54
  logger.info(f"connecting to {pve_host} via {jump_host}")
55
- metas = None
56
- stack_meta = None
55
+ volume_metas = None
56
+ stack_secrets = None
57
57
 
58
58
  async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
59
59
  if args.use_mc_gw:
@@ -79,8 +79,8 @@ async def list_backup_details_remote(args):
79
79
 
80
80
  sio.disconnect()
81
81
 
82
- metas = result["metas"]
83
- stack_meta = result["stack_meta"]
82
+ volume_metas = result["volume_metas"]
83
+ stack_secrets = result["stack_secrets"]
84
84
  else:
85
85
  # we will connect directly to the backup server
86
86
  # todo: here we can also pass the correct tls config
@@ -112,29 +112,21 @@ async def list_backup_details_remote(args):
112
112
 
113
113
  # read the archives
114
114
  dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
115
- metas = pickle.loads((await reader.readexactly(dict_size)))
116
-
117
- # first we group metas
118
- k8s_stack = metas[0]["stack"]
119
-
120
- print(f"k8s stack {k8s_stack}:")
121
-
122
- # query the server for backup secrets
123
- writer.write((k8s_stack + "\n").encode())
124
- await writer.drain()
115
+ volume_metas = pickle.loads((await reader.readexactly(dict_size)))
125
116
 
126
117
  # read the the meta information
127
118
  dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
128
- stack_meta = pickle.loads((await reader.readexactly(dict_size)))
119
+ stack_secrets = pickle.loads((await reader.readexactly(dict_size)))
129
120
 
121
+ # load secrets
130
122
  namespace_secret_dict = pickle.loads(
131
- base64.b64decode(stack_meta["namespace_secret_dict_b64"])
123
+ base64.b64decode(stack_secrets["namespace_secret_dict_b64"])
132
124
  )
133
125
 
134
126
  namespace_k8s_metas = {}
135
127
 
136
128
  # group metas by namespace
137
- for meta in metas:
129
+ for meta in volume_metas:
138
130
  if meta["namespace"] not in namespace_k8s_metas:
139
131
  namespace_k8s_metas[meta["namespace"]] = []
140
132
 
@@ -205,7 +197,7 @@ async def list_backups_remote(args):
205
197
  # launch our rpc service to read discovery secrets and prepare
206
198
  # acessing the backup server
207
199
  logger.info(f"connecting to {pve_host} via {jump_host}")
208
- archives = None
200
+ volume_metas = None
209
201
 
210
202
  async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
211
203
  if args.use_mc_gw:
@@ -230,7 +222,7 @@ async def list_backups_remote(args):
230
222
 
231
223
  sio.disconnect()
232
224
 
233
- archives = result["archives"]
225
+ volume_metas = result["volume_metas"]
234
226
 
235
227
  else:
236
228
  tls_disc_raw = await pxrpc.get_cloud_secret(
@@ -255,15 +247,15 @@ async def list_backups_remote(args):
255
247
 
256
248
  # read the response archives size and then the archives
257
249
  dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
258
- archives = pickle.loads((await reader.readexactly(dict_size)))
250
+ volume_metas = pickle.loads((await reader.readexactly(dict_size)))
259
251
 
260
252
  if args.json:
261
- print(json.dumps(sorted(archives)))
253
+ print(json.dumps(sorted(volume_metas)))
262
254
  return
263
255
 
264
256
  print("available backup timestamps (ids):")
265
257
 
266
- for timestamp in sorted(archives):
258
+ for timestamp in sorted(volume_metas):
267
259
  print(f"- timestamp {timestamp}")
268
260
 
269
261
 
@@ -0,0 +1,182 @@
1
+ import json
2
+ import sqlite3
3
+ from typing import Optional, Set
4
+
5
+ import aiosqlite
6
+
7
+ DB_PATH = "/mnt/bdd-zfs-data/backup-metadata.db"
8
+
9
+
10
+ async def init_db():
11
+ # todo: this should be refactored to a deduplicated approach
12
+ # as namespace secrets can get quite large and daily backups will
13
+ # result in a big db
14
+ async with aiosqlite.connect(DB_PATH) as db:
15
+ # create table for storing kubernetes volume related information
16
+ await db.execute(
17
+ """
18
+ CREATE TABLE IF NOT EXISTS volume_meta (
19
+ image_name TEXT NOT NULL,
20
+ timestamp TEXT NOT NULL,
21
+ data_json TEXT NOT NULL,
22
+ PRIMARY KEY (image_name, timestamp)
23
+ )
24
+ """
25
+ )
26
+ await db.execute(
27
+ "CREATE INDEX IF NOT EXISTS idx_volume_meta_timestamp ON volume_meta(timestamp)"
28
+ )
29
+ await db.execute(
30
+ "CREATE INDEX IF NOT EXISTS idx_volume_meta_image ON volume_meta(image_name)"
31
+ )
32
+
33
+ # table for storing secrets of multiple k8s namespaces per row (grouped by proxmox cloud stack)
34
+ await db.execute(
35
+ """
36
+ CREATE TABLE IF NOT EXISTS stack_secrets (
37
+ timestamp TEXT PRIMARY KEY,
38
+ stack TEXT NOT NULL,
39
+ data_json TEXT NOT NULL
40
+ )
41
+ """
42
+ )
43
+ await db.execute(
44
+ "CREATE INDEX IF NOT EXISTS idx_stack_secrets_timestamp ON stack_secrets(timestamp)"
45
+ )
46
+ await db.execute(
47
+ "CREATE INDEX IF NOT EXISTS idx_stack_secrets_stack ON stack_secrets(stack)"
48
+ )
49
+ await db.commit()
50
+
51
+
52
+ async def insert_volume_meta(meta_dict: dict) -> int:
53
+ image_name = meta_dict["image_name"]
54
+ timestamp = meta_dict["timestamp"]
55
+
56
+ async with aiosqlite.connect(DB_PATH) as db:
57
+ cursor = await db.execute(
58
+ "INSERT OR REPLACE INTO volume_meta (image_name, timestamp, data_json) VALUES (?, ?, ?)",
59
+ (image_name, timestamp, json.dumps(meta_dict)),
60
+ )
61
+ await db.commit()
62
+ return cursor.lastrowid
63
+
64
+
65
+ async def get_volume_meta(image_name: str, timestamp: str) -> Optional[dict]:
66
+ """
67
+ Get metadata of a single kubernetes pv + pvc.
68
+ """
69
+ async with aiosqlite.connect(DB_PATH) as db:
70
+ cursor = await db.execute(
71
+ "SELECT data_json FROM volume_meta WHERE image_name = ? AND timestamp = ?",
72
+ (image_name, timestamp),
73
+ )
74
+ row = await cursor.fetchone()
75
+ if row:
76
+ return json.loads(row[0])
77
+
78
+ return None
79
+
80
+
81
+ async def get_volume_metas(timestamp_filter: Optional[str] = None) -> dict:
82
+ """
83
+ Query the database for mutliple k8s pv + pvc volume metadata entries. Returns them in a map, grouped by their
84
+ backup timestamp id.
85
+ """
86
+ async with aiosqlite.connect(DB_PATH) as db:
87
+ if timestamp_filter:
88
+ cursor = await db.execute(
89
+ "SELECT timestamp, data_json FROM volume_meta WHERE timestamp = ?",
90
+ (timestamp_filter,),
91
+ )
92
+ else:
93
+ cursor = await db.execute("SELECT timestamp, data_json FROM volume_meta")
94
+
95
+ rows = await cursor.fetchall()
96
+ result = {}
97
+ for row in rows:
98
+ timestamp = row[0]
99
+ if timestamp not in result:
100
+ result[timestamp] = []
101
+ result[timestamp].append(json.loads(row[1]))
102
+ return result
103
+
104
+
105
+ async def insert_stack_secret(data: dict) -> int:
106
+ timestamp = data["timestamp"]
107
+ stack = data["stack"]
108
+
109
+ async with aiosqlite.connect(DB_PATH) as db:
110
+ cursor = await db.execute(
111
+ "INSERT INTO stack_secrets (timestamp, stack, data_json) VALUES (?, ?, ?)",
112
+ (timestamp, stack, json.dumps(data)),
113
+ )
114
+ await db.commit()
115
+ return cursor.lastrowid
116
+
117
+
118
+ async def get_stack_secrets(timestamp: str) -> Optional[dict]:
119
+ async with aiosqlite.connect(DB_PATH) as db:
120
+ cursor = await db.execute(
121
+ "SELECT data_json FROM stack_secrets WHERE timestamp = ?", (timestamp,)
122
+ )
123
+ row = await cursor.fetchone()
124
+ if row:
125
+ return json.loads(row[0])
126
+ return None
127
+
128
+
129
+ def get_db_timestamps() -> Set[str]:
130
+ timestamps = set()
131
+
132
+ with sqlite3.connect(DB_PATH) as conn:
133
+ cursor = conn.cursor()
134
+ cursor.execute("SELECT DISTINCT timestamp FROM volume_meta")
135
+ for row in cursor.fetchall():
136
+ timestamps.add(row[0])
137
+
138
+ cursor.execute("SELECT DISTINCT timestamp FROM stack_secrets")
139
+ for row in cursor.fetchall():
140
+ timestamps.add(row[0])
141
+
142
+ return timestamps
143
+
144
+
145
+ def prune_db_by_timestamps(timestamps_to_delete: Set[str]):
146
+ with sqlite3.connect(DB_PATH) as conn:
147
+ # create temp table with timestamp ids that will be deleted
148
+ conn.execute(
149
+ """
150
+ CREATE TEMP TABLE IF NOT EXISTS timestamps_to_delete (
151
+ timestamp TEXT PRIMARY KEY
152
+ )
153
+ """
154
+ )
155
+
156
+ conn.executemany(
157
+ "INSERT OR IGNORE INTO timestamps_to_delete (timestamp) VALUES (?)",
158
+ ((ts,) for ts in timestamps_to_delete),
159
+ )
160
+
161
+ # prune the two tables
162
+ conn.execute(
163
+ """
164
+ DELETE FROM volume_meta
165
+ WHERE EXISTS (
166
+ SELECT 1
167
+ FROM timestamps_to_delete d
168
+ WHERE d.timestamp = volume_meta.timestamp
169
+ )
170
+ """
171
+ )
172
+
173
+ conn.execute(
174
+ """
175
+ DELETE FROM stack_secrets
176
+ WHERE EXISTS (
177
+ SELECT 1
178
+ FROM timestamps_to_delete d
179
+ WHERE d.timestamp = stack_secrets.timestamp
180
+ )
181
+ """
182
+ )
@@ -7,7 +7,8 @@ from pathlib import Path
7
7
 
8
8
  from kubernetes import client
9
9
  from kubernetes.client.rest import ApiException
10
- from tinydb import Query, TinyDB
10
+
11
+ from pve_cloud_backup.daemon.db import get_volume_metas as db_get_volume_metas
11
12
 
12
13
  logger = logging.getLogger("bdd")
13
14
 
@@ -51,12 +52,9 @@ def copy_backup_generic():
51
52
  shutil.copy2(full_source_path, full_dest_path)
52
53
 
53
54
 
54
- def get_volume_metas(timestamp_filter=None):
55
- volume_meta_db = TinyDB(f"{get_backup_base_dir()}/volume-meta-db.json")
56
-
57
- archives = []
55
+ async def get_volume_metas(timestamp_filter=None):
56
+ db_result = await db_get_volume_metas(timestamp_filter)
58
57
 
59
- # iterate all k8s namespaced borg repos
60
58
  k8s_base_path = f"{get_backup_base_dir()}/borg-k8s"
61
59
  namespace_repos = [
62
60
  name
@@ -64,6 +62,7 @@ def get_volume_metas(timestamp_filter=None):
64
62
  if os.path.isdir(os.path.join(k8s_base_path, name))
65
63
  ]
66
64
 
65
+ archives = []
67
66
  for repo in namespace_repos:
68
67
  list_result = subprocess.run(
69
68
  ["borg", "list", f"{get_backup_base_dir()}/borg-k8s/{repo}", "--json"],
@@ -77,29 +76,18 @@ def get_volume_metas(timestamp_filter=None):
77
76
 
78
77
  archives.extend(json.loads(list_result.stdout)["archives"])
79
78
 
80
- timestamp_archives = {}
79
+ result = {}
81
80
  for archive in archives:
82
81
  image = archive["archive"].split("_", 1)[0]
83
82
  timestamp = archive["archive"].split("_", 1)[1]
84
83
 
85
84
  if timestamp_filter is not None and timestamp_filter != timestamp:
86
- continue # skip filtered
87
-
88
- if timestamp not in timestamp_archives:
89
- timestamp_archives[timestamp] = []
90
-
91
- Meta = Query()
92
- image_meta = volume_meta_db.get(
93
- (Meta.image_name == image) & (Meta.timestamp == timestamp)
94
- )
95
-
96
- if image_meta is None:
97
- logger.error(
98
- f"None meta found {timestamp}, image_name {image}, archive {archive}"
99
- )
100
- del timestamp_archives[timestamp]
101
85
  continue
102
86
 
103
- timestamp_archives[timestamp].append(image_meta)
87
+ if timestamp not in result:
88
+ result[timestamp] = []
89
+
90
+ if timestamp in db_result:
91
+ result[timestamp].extend(db_result[timestamp])
104
92
 
105
- return timestamp_archives
93
+ return result
@@ -54,7 +54,7 @@ async def run():
54
54
  backup_config["k8s_stack"],
55
55
  backup_addr,
56
56
  )
57
- await funcs.post_k8s_namespace_secrets(
57
+ await funcs.post_k8s_stack_secrets(
58
58
  namespace_secrets_zfs,
59
59
  timestamp,
60
60
  backup_config["k8s_stack"],
@@ -333,9 +333,7 @@ async def post_volume_meta(namespace_volume_meta, timestamp, k8s_stack, backup_a
333
333
  await net.volume_meta(backup_addr, body)
334
334
 
335
335
 
336
- async def post_k8s_namespace_secrets(
337
- namespace_secrets, timestamp, k8s_stack, backup_addr
338
- ):
336
+ async def post_k8s_stack_secrets(namespace_secrets, timestamp, k8s_stack, backup_addr):
339
337
 
340
338
  namespace_secret_dict_b64 = base64.b64encode(
341
339
  pickle.dumps(namespace_secrets)
@@ -347,7 +345,7 @@ async def post_k8s_namespace_secrets(
347
345
  }
348
346
  logger.debug(f"posting {body}")
349
347
 
350
- await net.namespace_secrets(backup_addr, body)
348
+ await net.stack_secrets(backup_addr, body)
351
349
 
352
350
 
353
351
  def cleanup(namespace_volume_meta, timestamp, unique_pools):
@@ -101,7 +101,7 @@ async def run():
101
101
  backup_config["k8s_stack"],
102
102
  backup_addr,
103
103
  )
104
- await funcs.post_k8s_namespace_secrets(
104
+ await funcs.post_k8s_stack_secrets(
105
105
  namespace_secrets | namespace_secrets_zfs,
106
106
  timestamp,
107
107
  backup_config["k8s_stack"],
@@ -231,6 +231,8 @@ async def archive(backup_addr, request_dict, chunk_generator, compress=True):
231
231
 
232
232
 
233
233
  async def meta(backup_addr, cmd, meta_dict):
234
+ logger.info("sending meta request: %s", cmd)
235
+
234
236
  if backup_addr.startswith("https://"):
235
237
  sio = await get_sio_mc_client(backup_addr)
236
238
 
@@ -255,6 +257,10 @@ async def meta(backup_addr, cmd, meta_dict):
255
257
  writer.write(meta_pickled)
256
258
  await writer.drain()
257
259
 
260
+ ack = await reader.readexactly(1)
261
+ if ack != b"\x01":
262
+ raise RuntimeError("Expected x01 ack byte!")
263
+
258
264
  writer.close()
259
265
 
260
266
 
@@ -262,5 +268,5 @@ async def volume_meta(backup_addr, meta_dict):
262
268
  await meta(backup_addr, Command.VOLUME_META, meta_dict)
263
269
 
264
270
 
265
- async def namespace_secrets(backup_addr, meta_dict):
266
- await meta(backup_addr, Command.NAMESPACE_SECRETS, meta_dict)
271
+ async def stack_secrets(backup_addr, meta_dict):
272
+ await meta(backup_addr, Command.STACK_SECRETS, meta_dict)
@@ -1,14 +1,14 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.18
3
+ Version: 3.6.0
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
7
- Requires-Dist: py-pve-cloud<3.5.0,>=3.4.6
7
+ Requires-Dist: py-pve-cloud<3.6.0,>=3.5.0
8
8
  Requires-Dist: kubernetes==34.1.0
9
9
  Requires-Dist: requests==2.32.5
10
10
  Requires-Dist: zstandard==0.25.0
11
- Requires-Dist: tinydb==4.8.2
11
+ Requires-Dist: aiosqlite==0.21.0
12
12
  Requires-Dist: python-socketio[asyncio_client]==5.16.4
13
13
  Requires-Dist: aiohttp==3.14.3
14
14
  Dynamic: license-file
@@ -4,7 +4,9 @@ pyproject.toml
4
4
  requirements.txt
5
5
  src/pve_cloud_backup/_version.py
6
6
  src/pve_cloud_backup/daemon/bdd.py
7
+ src/pve_cloud_backup/daemon/bdd_prune.py
7
8
  src/pve_cloud_backup/daemon/brctl.py
9
+ src/pve_cloud_backup/daemon/db.py
8
10
  src/pve_cloud_backup/daemon/funcs.py
9
11
  src/pve_cloud_backup/daemon/restore.py
10
12
  src/pve_cloud_backup/fetcher/ext_zfs.py
@@ -1,4 +1,5 @@
1
1
  [console_scripts]
2
+ bdd-prune = pve_cloud_backup.daemon.bdd_prune:main
2
3
  brctl = pve_cloud_backup.daemon.brctl:main
3
4
  daemon = pve_cloud_backup.daemon.bdd:main
4
5
  ext-zfs = pve_cloud_backup.fetcher.ext_zfs:main
@@ -1,7 +1,7 @@
1
- py-pve-cloud<3.5.0,>=3.4.6
1
+ py-pve-cloud<3.6.0,>=3.5.0
2
2
  kubernetes==34.1.0
3
3
  requests==2.32.5
4
4
  zstandard==0.25.0
5
- tinydb==4.8.2
5
+ aiosqlite==0.21.0
6
6
  python-socketio[asyncio_client]==5.16.4
7
7
  aiohttp==3.14.3
@@ -1 +0,0 @@
1
- __version__ = "3.5.18"