py-pve-cloud-backup 3.5.9__tar.gz → 3.5.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {py_pve_cloud_backup-3.5.9/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.11}/PKG-INFO +2 -2
  2. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/requirements.txt +1 -1
  3. py_pve_cloud_backup-3.5.11/src/pve_cloud_backup/_version.py +1 -0
  4. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/bdd.py +52 -50
  5. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/brctl.py +11 -15
  6. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/restore.py +265 -139
  7. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11/src/py_pve_cloud_backup.egg-info}/PKG-INFO +2 -2
  8. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/requires.txt +1 -1
  9. py_pve_cloud_backup-3.5.9/src/pve_cloud_backup/_version.py +0 -1
  10. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/LICENSE.md +0 -0
  11. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/README.md +0 -0
  12. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/pyproject.toml +0 -0
  13. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/setup.cfg +0 -0
  14. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/funcs.py +0 -0
  15. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/ext_zfs.py +0 -0
  16. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/funcs.py +0 -0
  17. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/git.py +0 -0
  18. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/main.py +0 -0
  19. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/net.py +0 -0
  20. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
  21. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
  22. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +0 -0
  23. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
  24. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/entry_points.txt +0 -0
  25. {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
@@ -1,10 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.9
3
+ Version: 3.5.11
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
7
- Requires-Dist: py-pve-cloud<3.5.0,>=3.4.1
7
+ Requires-Dist: py-pve-cloud<3.5.0,>=3.4.3
8
8
  Requires-Dist: kubernetes==34.1.0
9
9
  Requires-Dist: requests==2.32.5
10
10
  Requires-Dist: zstandard==0.25.0
@@ -1,5 +1,5 @@
1
1
  # pessimistic operator for local tdd installs (pip install -e .)
2
- py-pve-cloud>=3.4.1,<3.5.0
2
+ py-pve-cloud>=3.4.3,<3.5.0
3
3
  kubernetes==34.1.0
4
4
  requests==2.32.5
5
5
  zstandard==0.25.0
@@ -0,0 +1 @@
1
+ __version__ = "3.5.11"
@@ -67,6 +67,10 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
67
67
 
68
68
  # send ping pong while waiting on lock
69
69
  # maybe this is also needed in other rpc call types
70
+ # todo: currently this is the most likely point where a disconnect might happen
71
+ # this connection here is by far the longest running. It would however be nice that
72
+ # the entire app has retries for connects and can handle connection outages well.
73
+ # implement something like toxiproxy in e2e testing and make resilient.
70
74
  lock = await get_lock(backup_dir)
71
75
  # store ref to kill in case of failure => this will stop the archive from being committet
72
76
  borg_proc = None
@@ -106,12 +110,18 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
106
110
  )
107
111
 
108
112
  # read compressed chunks
113
+ dbg_chunk_count = 0
114
+
109
115
  while True:
110
116
  # client first always sends chunk size
111
117
  chunk_size = struct.unpack("!I", (await reader.readexactly(4)))[
112
118
  0
113
119
  ]
114
- logger.debug(f"received chunk size {chunk_size}")
120
+
121
+ # log chunk size on every 100th chunk in dbg log level
122
+ dbg_chunk_count += 1
123
+ if dbg_chunk_count % 100 == 0:
124
+ logger.debug(f"received chunk size {chunk_size}")
115
125
 
116
126
  if chunk_size == 0:
117
127
  logger.debug("received chunk size 0, finished")
@@ -149,6 +159,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
149
159
  )
150
160
 
151
161
  if borg_proc:
162
+ logger.info("terminating borg process")
152
163
  borg_proc.terminate()
153
164
 
154
165
  try:
@@ -160,7 +171,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
160
171
  borg_proc.kill()
161
172
  await borg_proc.wait()
162
173
 
163
- # cleanup repo
174
+ logger.debug(f"cleaning up borg repo {borg_archive}")
164
175
  await asyncio.create_subprocess_exec(
165
176
  "borg",
166
177
  "delete",
@@ -237,13 +248,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
237
248
  async with lock:
238
249
  secret_db = TinyDB(db_path)
239
250
 
240
- while True:
241
- # listen for secret requests
242
251
  stack = (await reader.readline()).decode().rstrip("\n")
243
252
 
244
- if stack == "##BRCTL-DONE":
245
- break # done signal
246
-
247
253
  Meta = Query()
248
254
  ns_secrets = secret_db.get(
249
255
  (Meta.timestamp == timestamp) & (Meta.stack == stack)
@@ -256,7 +262,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
256
262
  writer.write(meta_pickled)
257
263
  await writer.drain()
258
264
 
259
- case Command.RESTORE_PROCEDURE:
265
+ case Command.INIT_RESTORE_PROCEDURE:
260
266
  timestamp = (await reader.readline()).decode().rstrip("\n")
261
267
  logger.info(timestamp)
262
268
 
@@ -298,53 +304,49 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
298
304
  writer.write(meta_pickled)
299
305
  await writer.drain()
300
306
 
301
- logger.info(
302
- "send initial config / secrets - waiting for archive requests"
303
- )
307
+ logger.info(
308
+ "send initial config / secrets - waiting for archive requests"
309
+ )
310
+
311
+ case Command.REQUEST_ARCHIVE:
304
312
  # next the client requests the archives which we extract here and pipe via a stream
305
- while True:
306
- # open the extract process and send the stream the output
307
- request_archive = (await reader.readline()).decode().rstrip("\n")
308
- logger.info(request_archive)
309
-
310
- if request_archive == "##BRCTL-DONE":
311
- break # done signal
312
-
313
- request_artifact = (await reader.readline()).decode().rstrip("\n")
314
- logger.info(request_artifact)
315
-
316
- backup_dir = f"{get_backup_base_dir()}/{request_archive}"
317
- lock = await get_lock(backup_dir)
318
- async with lock:
319
- logger.info(
320
- f"running borg extract on {backup_dir}::{request_artifact}"
321
- )
322
- proc = await asyncio.create_subprocess_exec(
323
- "borg",
324
- "extract",
325
- "--sparse",
326
- "--stdout",
327
- f"{backup_dir}::{request_artifact}",
328
- stdout=asyncio.subprocess.PIPE,
329
- )
313
+ # open the extract process and send the stream the output
314
+ request_archive = (await reader.readline()).decode().rstrip("\n")
315
+ logger.info(request_archive)
316
+
317
+ request_artifact = (await reader.readline()).decode().rstrip("\n")
318
+ logger.info(request_artifact)
330
319
 
331
- compressor = zstd.ZstdCompressor(
332
- level=1, threads=6
333
- ).compressobj()
334
- while True:
335
- chunk = await proc.stdout.read(4 * 1024 * 1024 * 10) # 4MB
336
- if not chunk:
337
- break
320
+ backup_dir = f"{get_backup_base_dir()}/{request_archive}"
321
+ lock = await get_lock(backup_dir)
322
+ async with lock:
323
+ logger.info(
324
+ f"running borg extract on {backup_dir}::{request_artifact}"
325
+ )
326
+ proc = await asyncio.create_subprocess_exec(
327
+ "borg",
328
+ "extract",
329
+ "--sparse",
330
+ "--stdout",
331
+ f"{backup_dir}::{request_artifact}",
332
+ stdout=asyncio.subprocess.PIPE,
333
+ )
338
334
 
339
- # compress and send the chunk
340
- await send_cchunk(writer, compressor.compress(chunk))
335
+ compressor = zstd.ZstdCompressor(level=1, threads=6).compressobj()
336
+ while True:
337
+ chunk = await proc.stdout.read(4 * 1024 * 1024 * 10) # 4MB
338
+ if not chunk:
339
+ break
341
340
 
342
- # send the rest in the compressor
343
- await send_cchunk(writer, compressor.flush())
341
+ # compress and send the chunk
342
+ await send_cchunk(writer, compressor.compress(chunk))
344
343
 
345
- logger.info("sending eof")
346
- writer.write(struct.pack("!I", 0))
347
- await writer.drain()
344
+ # send the rest in the compressor
345
+ await send_cchunk(writer, compressor.flush())
346
+
347
+ logger.info("sending eof")
348
+ writer.write(struct.pack("!I", 0))
349
+ await writer.drain()
348
350
 
349
351
  except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError) as e:
350
352
  logger.warning("Client disconnected: %s", e, exc_info=True)
@@ -19,7 +19,7 @@ from kubernetes.client import (ApiException, V1ConfigMapVolumeSource,
19
19
  from kubernetes.config.kube_config import KubeConfigLoader
20
20
  from pve_cloud.cli.pvclu import (get_cloud_domain, get_ssh_master_kubeconfig,
21
21
  get_ssh_remote_master_kubeconfig)
22
- from pve_cloud.cli.pxrpc import launch_pxrpc
22
+ from pve_cloud.cli.pxrpc import get_simple_pxrpc, launch_pxrpc
23
23
  from pve_cloud.lib.backup_rpc import Command
24
24
  from pve_cloud.lib.inventory import (get_cloud_domain, get_cluster_vars,
25
25
  get_online_pve_host,
@@ -55,10 +55,10 @@ async def list_backup_details_remote(args):
55
55
  metas = None
56
56
  stack_meta = None
57
57
 
58
- with launch_pxrpc(jump_host, pve_host) as (pxrpc, pve_host_conn):
58
+ async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
59
59
  if args.use_mc_gw:
60
60
  # we will create a socket connection to the multi cloud gateway for that we need to fetch secrets
61
- ext_mc_raw = pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
61
+ ext_mc_raw = await pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
62
62
  if not ext_mc_raw:
63
63
  raise RuntimeError(
64
64
  f"No multi cloud services could be discovered for {pve_cluster} - {cloud_domain}!"
@@ -85,7 +85,7 @@ async def list_backup_details_remote(args):
85
85
  # we will connect directly to the backup server
86
86
  # todo: here we can also pass the correct tls config
87
87
 
88
- tls_disc_raw = pxrpc.get_cloud_secret(
88
+ tls_disc_raw = await pxrpc.get_cloud_secret(
89
89
  cloud_domain, f"{bdd_stack_name}-bdd-tls-discovery"
90
90
  )
91
91
  if not tls_disc_raw:
@@ -128,11 +128,6 @@ async def list_backup_details_remote(args):
128
128
  dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
129
129
  stack_meta = pickle.loads((await reader.readexactly(dict_size)))
130
130
 
131
- # send a terminator
132
- # todo: probably not needed anymore
133
- writer.write("##BRCTL-DONE\n".encode())
134
- await writer.drain()
135
-
136
131
  namespace_secret_dict = pickle.loads(
137
132
  base64.b64decode(stack_meta["namespace_secret_dict_b64"])
138
133
  )
@@ -213,9 +208,9 @@ async def list_backups_remote(args):
213
208
  logger.info(f"connecting to {pve_host} via {jump_host}")
214
209
  archives = None
215
210
 
216
- with launch_pxrpc(jump_host, pve_host) as (pxrpc, pve_host_conn):
211
+ async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
217
212
  if args.use_mc_gw:
218
- ext_mc_raw = pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
213
+ ext_mc_raw = await pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
219
214
  if not ext_mc_raw:
220
215
  raise RuntimeError(
221
216
  f"No multi cloud services could be discovered for {pve_cluster} - {cloud_domain}!"
@@ -239,7 +234,7 @@ async def list_backups_remote(args):
239
234
  archives = result["archives"]
240
235
 
241
236
  else:
242
- tls_disc_raw = pxrpc.get_cloud_secret(
237
+ tls_disc_raw = await pxrpc.get_cloud_secret(
243
238
  cloud_domain, f"{bdd_stack_name}-bdd-tls-discovery"
244
239
  )
245
240
  if not tls_disc_raw:
@@ -299,6 +294,7 @@ async def launch_restore_job(args):
299
294
 
300
295
  kubeconfig_dict = None
301
296
 
297
+ # todo: maybe this can be simplyfied relying more heavily on pxrpc.get_simple_pxrpc concepts
302
298
  if raw_pxc_inv["plugin"] == "pxc.cloud.kubespray_inv":
303
299
  kubespray_inv = raw_pxc_inv
304
300
 
@@ -368,9 +364,9 @@ async def launch_restore_job(args):
368
364
  # acessing the backup server
369
365
  logger.info(f"connecting to {pve_host} via {jump_host}")
370
366
 
371
- with launch_pxrpc(jump_host, pve_host) as (pxrpc, pve_host_conn):
367
+ async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
372
368
  if args.use_mc_gw:
373
- ext_mc_raw = pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
369
+ ext_mc_raw = await pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
374
370
  if not ext_mc_raw:
375
371
  raise RuntimeError(
376
372
  f"No multi cloud services could be discovered for {pve_cluster} - {cloud_domain}!"
@@ -384,7 +380,7 @@ async def launch_restore_job(args):
384
380
  serializable_args["bdd_stack_name"] = bdd_stack_name
385
381
 
386
382
  else:
387
- tls_disc_raw = pxrpc.get_cloud_secret(
383
+ tls_disc_raw = await pxrpc.get_cloud_secret(
388
384
  cloud_domain, f"{bdd_stack_name}-bdd-tls-discovery"
389
385
  )
390
386
  if not tls_disc_raw:
@@ -9,6 +9,7 @@ import ssl
9
9
  import struct
10
10
  import subprocess
11
11
  import uuid
12
+ from contextlib import asynccontextmanager
12
13
  from pprint import pformat
13
14
 
14
15
  import asyncssh
@@ -27,6 +28,8 @@ log_level = getattr(logging, log_level_str, logging.INFO)
27
28
  logging.basicConfig(level=log_level)
28
29
  logger = logging.getLogger("pxc-restore")
29
30
 
31
+ SIO_MAX_RETRIES = 3
32
+
30
33
 
31
34
  # these functions are necessary to convert python k8s naming to camel case
32
35
  def to_camel_case(snake_str):
@@ -48,67 +51,86 @@ def convert_keys_to_camel_case(obj):
48
51
  return obj
49
52
 
50
53
 
54
+ @asynccontextmanager
55
+ async def get_direct_conn(restore_args):
56
+ ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
57
+ ssl_ctx.check_hostname = False
58
+ ssl_ctx.verify_mode = ssl.CERT_NONE
59
+
60
+ reader, writer = await asyncio.open_connection(
61
+ restore_args["bdd_host"], 8085, ssl=ssl_ctx
62
+ )
63
+ yield reader, writer
64
+
65
+ writer.close()
66
+
67
+
68
+ @asynccontextmanager
69
+ async def get_sio_conn(restore_args):
70
+ log_debug = os.getenv("LOG_LEVEL") == "DEBUG"
71
+ sio = socketio.AsyncClient(logger=log_debug, engineio_logger=log_debug)
72
+
73
+ await sio.connect(
74
+ f"https://{restore_args['mc_gw_host']}",
75
+ auth={
76
+ "token": restore_args["mc_ext_token"],
77
+ "bdd_stack_name": restore_args["bdd_stack_name"],
78
+ },
79
+ transports=["websocket"],
80
+ )
81
+
82
+ yield sio
83
+
84
+ await sio.disconnect()
85
+
86
+
51
87
  async def init_procedure_bdd(restore_args):
52
88
  if restore_args["use_mc_gw"]:
53
- log_debug = os.getenv("LOG_LEVEL") == "DEBUG"
54
- sio = socketio.AsyncClient(logger=log_debug, engineio_logger=log_debug)
55
-
56
- await sio.connect(
57
- f"https://{restore_args['mc_gw_host']}",
58
- auth={
59
- "token": restore_args["mc_ext_token"],
60
- "bdd_stack_name": restore_args["bdd_stack_name"],
61
- },
62
- transports=["websocket"],
63
- )
64
- result_pickled = await sio.call(
65
- "init_restore", restore_args["timestamp"], timeout=30
66
- )
67
- logger.info("init restore result")
68
- result = pickle.loads(result_pickled)
69
- logger.info(result)
70
89
 
71
- return sio, result["metas_grouped_by_ns"], result["namespace_secret_dict"]
90
+ async with get_sio_conn(restore_args) as sio:
91
+ result_pickled = await sio.call(
92
+ "init_restore", restore_args["timestamp"], timeout=30
93
+ )
94
+ logger.info("init restore result")
95
+ result = pickle.loads(result_pickled)
96
+ logger.info(result)
97
+
98
+ return result["metas_grouped_by_ns"], result["namespace_secret_dict"]
72
99
 
73
100
  else:
74
- # connect to the backup server and start the restore procedure
75
- # we simply trust the host here as the ca is only available from
76
- # postgres secrets
77
- ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
78
- ssl_ctx.check_hostname = False
79
- ssl_ctx.verify_mode = ssl.CERT_NONE
80
-
81
- reader, writer = await asyncio.open_connection(
82
- restore_args["bdd_host"], 8085, ssl=ssl_ctx
83
- )
84
- writer.write(struct.pack("B", Command.RESTORE_PROCEDURE.value))
85
- await writer.drain()
86
101
 
87
- # first we send the timestamp and receive our meta information for the restore
88
- writer.write((restore_args["timestamp"] + "\n").encode())
89
- await writer.drain()
102
+ async with get_direct_conn(restore_args) as (reader, writer):
103
+ # connect to the backup server and start the restore procedure
104
+ # we simply trust the host here as the ca is only available from
105
+ # postgres secrets
106
+ writer.write(struct.pack("B", Command.INIT_RESTORE_PROCEDURE.value))
107
+ await writer.drain()
90
108
 
91
- # read the archives
92
- dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
93
- metas = pickle.loads((await reader.readexactly(dict_size)))
109
+ # first we send the timestamp and receive our meta information for the restore
110
+ writer.write((restore_args["timestamp"] + "\n").encode())
111
+ await writer.drain()
94
112
 
95
- metas_grouped_by_ns = {}
113
+ # read the archives
114
+ dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
115
+ metas = pickle.loads((await reader.readexactly(dict_size)))
96
116
 
97
- for meta in metas:
98
- if meta["namespace"] not in metas_grouped_by_ns:
99
- metas_grouped_by_ns[meta["namespace"]] = []
117
+ metas_grouped_by_ns = {}
100
118
 
101
- metas_grouped_by_ns[meta["namespace"]].append(meta)
119
+ for meta in metas:
120
+ if meta["namespace"] not in metas_grouped_by_ns:
121
+ metas_grouped_by_ns[meta["namespace"]] = []
102
122
 
103
- # read the the meta information
104
- dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
105
- stack_meta = pickle.loads((await reader.readexactly(dict_size)))
123
+ metas_grouped_by_ns[meta["namespace"]].append(meta)
106
124
 
107
- namespace_secret_dict = pickle.loads(
108
- base64.b64decode(stack_meta["namespace_secret_dict_b64"])
109
- )
125
+ # read the the meta information
126
+ dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
127
+ stack_meta = pickle.loads((await reader.readexactly(dict_size)))
128
+
129
+ namespace_secret_dict = pickle.loads(
130
+ base64.b64decode(stack_meta["namespace_secret_dict_b64"])
131
+ )
110
132
 
111
- return (reader, writer), metas_grouped_by_ns, namespace_secret_dict
133
+ return metas_grouped_by_ns, namespace_secret_dict
112
134
 
113
135
 
114
136
  def clean_pvc_dict(pvc_dict):
@@ -175,9 +197,7 @@ async def procedure():
175
197
  restore_args = json.loads(base64.b64decode(os.getenv("PXC_RESTORE_ARGS")))
176
198
  logger.info(restore_args)
177
199
 
178
- connection, metas_grouped_by_ns, namespace_secret_dict = await init_procedure_bdd(
179
- restore_args
180
- )
200
+ metas_grouped_by_ns, namespace_secret_dict = await init_procedure_bdd(restore_args)
181
201
 
182
202
  # now we start the restore procedure
183
203
  config.load_incluster_config()
@@ -466,14 +486,15 @@ async def procedure():
466
486
  # first we create a zvol of identical size
467
487
  # todo: remove -rest stub and replace with new id generation
468
488
  # todo: write converter function to create zvol in exact size like k8s did zfs list -t volume -o name,volsize
469
- cmd = f"sudo zfs create -s -V {parse_quantity(pvc_dict['spec']['resources']['requests']['storage'])} tank-pv/pvc-{restore_pvc_uuid}"
470
- logger.info("Executing create: %s", cmd)
489
+ zvol_create_cmd = f"sudo zfs create -s -V {parse_quantity(pvc_dict['spec']['resources']['requests']['storage'])} tank-pv/pvc-{restore_pvc_uuid}"
490
+ logger.info("Executing create: %s", zvol_create_cmd)
471
491
 
472
- await ssh.run(cmd, check=True)
492
+ await ssh.run(zvol_create_cmd, check=True)
473
493
 
474
494
  # then we pipe the compressed stream into dd zstd decompression pipeline
495
+ import_cmd = f"zstd --decompress --stdout | sudo dd of=/dev/zvol/tank-pv/pvc-{restore_pvc_uuid} bs=4M status=none"
475
496
  proc = await ssh.create_process(
476
- f"zstd --decompress --stdout | sudo dd of=/dev/zvol/tank-pv/pvc-{restore_pvc_uuid} bs=4M status=none",
497
+ import_cmd,
477
498
  encoding=None,
478
499
  )
479
500
 
@@ -485,45 +506,105 @@ async def procedure():
485
506
  )
486
507
 
487
508
  if restore_args["use_mc_gw"]:
488
- await connection.call(
489
- "init_request",
490
- {"archive": request_archive, "artifact": request_artifact},
491
- timeout=30,
492
- )
493
- else:
494
- reader, writer = connection
509
+ for attempt in range(1, SIO_MAX_RETRIES + 1):
510
+ try:
511
+ async with get_sio_conn(restore_args) as sio:
512
+ await sio.call(
513
+ "init_request",
514
+ {
515
+ "archive": request_archive,
516
+ "artifact": request_artifact,
517
+ },
518
+ timeout=30,
519
+ )
495
520
 
496
- # bdd server does readline()
497
- writer.write(request_archive.encode())
498
- await writer.drain()
521
+ logger.info(
522
+ "send request artifact / archive - requesting chunks"
523
+ )
499
524
 
500
- writer.write(request_artifact.encode())
501
- await writer.drain()
525
+ while True:
526
+ chunk = await sio.call(
527
+ "request_chunk", timeout=30
528
+ )
529
+ if not chunk:
530
+ break
502
531
 
503
- logger.info("send request artifact / archive - requesting chunks")
532
+ proc.stdin.write(chunk)
533
+ await proc.stdin.drain()
504
534
 
505
- # read compressed chunks
506
- while True:
507
- chunk = None
508
- if restore_args["use_mc_gw"]:
509
- chunk = await connection.call("request_chunk", timeout=30)
510
- if not chunk:
535
+ # success
511
536
  break
512
- else:
513
- reader, writer = connection
514
537
 
515
- # client first always sends chunk size
516
- chunk_size = struct.unpack(
517
- "!I", (await reader.readexactly(4))
518
- )[0]
519
- if chunk_size == 0:
520
- logger.debug("received eof")
521
- break # client sends 0 chunk size at the end to signal that its finished uploading
538
+ except socketio.exceptions.TimeoutError:
539
+ logger.warn(f"Error on attempt {attempt}")
540
+ if attempt == SIO_MAX_RETRIES:
541
+ raise
522
542
 
523
- chunk = await reader.readexactly(chunk_size)
543
+ # gracefully terminate input pipeline
544
+ logger.debug("terminating dd zvol import proc")
545
+ proc.terminate()
546
+
547
+ try:
548
+ await asyncio.wait_for(proc.wait(), timeout=30)
549
+ except asyncio.TimeoutError:
550
+ logger.warning(
551
+ "terminate timed out, force killing import subprocess!"
552
+ )
553
+ proc.kill()
554
+ await proc.wait()
555
+
556
+ # destroy the zfs volume
557
+ logger.debug(
558
+ f"cleaning up zfs vol tank-pv/pvc-{restore_pvc_uuid}"
559
+ )
560
+ await ssh.run(
561
+ f"sudo zfs destroy tank-pv/pvc-{restore_pvc_uuid}"
562
+ )
524
563
 
525
- proc.stdin.write(chunk)
526
- await proc.stdin.drain()
564
+ logger.info("Retrying...")
565
+ await asyncio.sleep(10)
566
+
567
+ # rerun init commands
568
+ # todo: generic retry could make this much learner combining with direct connect
569
+ logger.debug("relaunching zvol create cmd")
570
+ await ssh.run(zvol_create_cmd, check=True)
571
+ proc = await ssh.create_process(
572
+ import_cmd,
573
+ encoding=None,
574
+ )
575
+
576
+ else:
577
+ async with get_direct_conn(restore_args) as (reader, writer):
578
+
579
+ writer.write(
580
+ struct.pack("B", Command.REQUEST_ARCHIVE.value)
581
+ )
582
+ await writer.drain()
583
+
584
+ # bdd server does readline()
585
+ writer.write(request_archive.encode())
586
+ await writer.drain()
587
+
588
+ writer.write(request_artifact.encode())
589
+ await writer.drain()
590
+
591
+ logger.info(
592
+ "send request artifact / archive - requesting chunks"
593
+ )
594
+
595
+ while True:
596
+ # client first always sends chunk size
597
+ chunk_size = struct.unpack(
598
+ "!I", (await reader.readexactly(4))
599
+ )[0]
600
+ if chunk_size == 0:
601
+ logger.debug("received eof")
602
+ break # client sends 0 chunk size at the end to signal that its finished uploading
603
+
604
+ chunk = await reader.readexactly(chunk_size)
605
+
606
+ proc.stdin.write(chunk)
607
+ await proc.stdin.drain()
527
608
 
528
609
  logger.info("done reading closing proc")
529
610
  proc.stdin.close()
@@ -659,22 +740,6 @@ async def procedure():
659
740
  f"requesting borg archive stream from bdd {request_archive} - {request_artifact} into rbd import {pool}/{new_csi_image_name}"
660
741
  )
661
742
 
662
- if restore_args["use_mc_gw"]:
663
- await connection.call(
664
- "init_request",
665
- {"archive": request_archive, "artifact": request_artifact},
666
- timeout=30,
667
- )
668
- else:
669
- reader, writer = connection
670
-
671
- # bdd server does readline()
672
- writer.write(request_archive.encode())
673
- await writer.drain()
674
-
675
- writer.write(request_artifact.encode())
676
- await writer.drain()
677
-
678
743
  # pipe the resulting stream into rbd import
679
744
  rbd_import_proc = await asyncio.create_subprocess_exec(
680
745
  "rbd",
@@ -684,32 +749,108 @@ async def procedure():
684
749
  stdin=asyncio.subprocess.PIPE,
685
750
  )
686
751
 
687
- # read compressed chunks
688
- decompressor = zstd.ZstdDecompressor().decompressobj()
689
- while True:
690
- chunk = None
691
- if restore_args["use_mc_gw"]:
692
- chunk = await connection.call("request_chunk", timeout=30)
693
- if not chunk:
752
+ if restore_args["use_mc_gw"]:
753
+ for attempt in range(1, SIO_MAX_RETRIES + 1):
754
+ try:
755
+ async with get_sio_conn(restore_args) as sio:
756
+ await sio.call(
757
+ "init_request",
758
+ {
759
+ "archive": request_archive,
760
+ "artifact": request_artifact,
761
+ },
762
+ timeout=30,
763
+ )
764
+ # read compressed chunks
765
+ decompressor = zstd.ZstdDecompressor().decompressobj()
766
+ while True:
767
+ chunk = await sio.call("request_chunk", timeout=30)
768
+ if not chunk:
769
+ break
770
+
771
+ # decompress and write
772
+ decompressed_chunk = decompressor.decompress(chunk)
773
+ if decompressed_chunk:
774
+ rbd_import_proc.stdin.write(decompressed_chunk)
775
+ await rbd_import_proc.stdin.drain()
776
+
777
+ # success
694
778
  break
695
- else:
696
- reader, writer = connection
697
779
 
698
- # client first always sends chunk size
699
- chunk_size = struct.unpack("!I", (await reader.readexactly(4)))[
700
- 0
701
- ]
702
- if chunk_size == 0:
703
- logger.debug("received eof")
704
- break # client sends 0 chunk size at the end to signal that its finished uploading
780
+ except socketio.exceptions.TimeoutError:
781
+ logger.warn(f"Error on attempt {attempt}")
782
+ if attempt == SIO_MAX_RETRIES:
783
+ raise
784
+
785
+ # gracefully terminate input pipeline
786
+ logger.debug("terminating rbd import proc")
787
+ rbd_import_proc.terminate()
788
+
789
+ try:
790
+ await asyncio.wait_for(
791
+ rbd_import_proc.wait(), timeout=30
792
+ )
793
+ except asyncio.TimeoutError:
794
+ logger.warning(
795
+ "terminate timed out, force killing import subprocess!"
796
+ )
797
+ rbd_import_proc.kill()
798
+ await rbd_import_proc.wait()
705
799
 
706
- chunk = await reader.readexactly(chunk_size)
800
+ # destroy the rbd image
801
+ logger.debug(
802
+ f"cleanup rbd image {pool}/{new_csi_image_name}"
803
+ )
804
+ await asyncio.create_subprocess_exec(
805
+ "rbd",
806
+ "rm",
807
+ f"{pool}/{new_csi_image_name}",
808
+ )
707
809
 
708
- # decompress and write
709
- decompressed_chunk = decompressor.decompress(chunk)
710
- if decompressed_chunk:
711
- rbd_import_proc.stdin.write(decompressed_chunk)
712
- await rbd_import_proc.stdin.drain()
810
+ logger.info("Retrying...")
811
+ await asyncio.sleep(10)
812
+
813
+ # rerun init commands
814
+ # todo: generic retry could make this much learner combining with direct connect
815
+ rbd_import_proc = await asyncio.create_subprocess_exec(
816
+ "rbd",
817
+ "import",
818
+ "-",
819
+ f"{pool}/{new_csi_image_name}",
820
+ stdin=asyncio.subprocess.PIPE,
821
+ )
822
+
823
+ else:
824
+ async with get_direct_conn(restore_args) as (reader, writer):
825
+
826
+ writer.write(struct.pack("B", Command.REQUEST_ARCHIVE.value))
827
+ await writer.drain()
828
+
829
+ # bdd server does readline()
830
+ writer.write(request_archive.encode())
831
+ await writer.drain()
832
+
833
+ writer.write(request_artifact.encode())
834
+ await writer.drain()
835
+
836
+ # read compressed chunks
837
+ decompressor = zstd.ZstdDecompressor().decompressobj()
838
+ while True:
839
+ # client first always sends chunk size
840
+ chunk_size = struct.unpack(
841
+ "!I", (await reader.readexactly(4))
842
+ )[0]
843
+ if chunk_size == 0:
844
+ logger.debug("received eof")
845
+ break # client sends 0 chunk size at the end to signal that its finished uploading
846
+
847
+ chunk = await reader.readexactly(chunk_size)
848
+
849
+ # decompress and write
850
+ decompressed_chunk = decompressor.decompress(chunk)
851
+ if decompressed_chunk:
852
+ rbd_import_proc.stdin.write(decompressed_chunk)
853
+ await rbd_import_proc.stdin.drain()
713
854
 
714
855
  # the decompressor does not always return a decompressed chunk but might retain
715
856
  # and return empty. at the end we need to call flush to get everything out
@@ -835,21 +976,6 @@ async def procedure():
835
976
  )
836
977
  )
837
978
 
838
- if restore_args["use_mc_gw"]:
839
- await connection.call("request_done", timeout=30)
840
-
841
- await connection.disconnect()
842
-
843
- else:
844
- reader, writer = connection
845
-
846
- # send the done signal to bdd server
847
- writer.write("##BRCTL-DONE\n".encode())
848
- await writer.drain()
849
-
850
- # close the writer here
851
- writer.close()
852
-
853
979
  # scale back up again
854
980
  if restore_args["auto_scale"]:
855
981
  # auto downscale deployments and statefulsets of namespace
@@ -1,10 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.9
3
+ Version: 3.5.11
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
7
- Requires-Dist: py-pve-cloud<3.5.0,>=3.4.1
7
+ Requires-Dist: py-pve-cloud<3.5.0,>=3.4.3
8
8
  Requires-Dist: kubernetes==34.1.0
9
9
  Requires-Dist: requests==2.32.5
10
10
  Requires-Dist: zstandard==0.25.0
@@ -1,4 +1,4 @@
1
- py-pve-cloud<3.5.0,>=3.4.1
1
+ py-pve-cloud<3.5.0,>=3.4.3
2
2
  kubernetes==34.1.0
3
3
  requests==2.32.5
4
4
  zstandard==0.25.0
@@ -1 +0,0 @@
1
- __version__ = "3.5.9"