py-pve-cloud-backup 3.5.14__tar.gz → 3.5.16__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {py_pve_cloud_backup-3.5.14/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.16}/PKG-INFO +1 -1
  2. py_pve_cloud_backup-3.5.16/src/pve_cloud_backup/_version.py +1 -0
  3. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/bdd.py +11 -32
  4. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/brctl.py +2 -13
  5. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/restore.py +12 -35
  6. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/funcs.py +2 -2
  7. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/net.py +56 -79
  8. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16/src/py_pve_cloud_backup.egg-info}/PKG-INFO +1 -1
  9. py_pve_cloud_backup-3.5.14/src/pve_cloud_backup/_version.py +0 -1
  10. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/LICENSE.md +0 -0
  11. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/README.md +0 -0
  12. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/pyproject.toml +0 -0
  13. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/requirements.txt +0 -0
  14. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/setup.cfg +0 -0
  15. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/funcs.py +0 -0
  16. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/ext_zfs.py +0 -0
  17. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/git.py +0 -0
  18. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/main.py +0 -0
  19. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
  20. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
  21. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +0 -0
  22. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
  23. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/entry_points.txt +0 -0
  24. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/requires.txt +0 -0
  25. {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.14
3
+ Version: 3.5.16
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -0,0 +1 @@
1
+ __version__ = "3.5.16"
@@ -72,6 +72,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
72
72
  # the entire app has retries for connects and can handle connection outages well.
73
73
  # implement something like toxiproxy in e2e testing and make resilient.
74
74
  lock = await get_lock(backup_dir)
75
+
75
76
  # store ref to kill in case of failure => this will stop the archive from being committet
76
77
  borg_proc = None
77
78
  borg_archive = f"{backup_dir}::{archive_name}_{timestamp}"
@@ -142,7 +143,6 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
142
143
 
143
144
  # the decompressor does not always return a decompressed chunk but might retain
144
145
  # and return empty. at the end we need to call flush to get everything out
145
- logger.debug("flushing, draining and closing")
146
146
  borg_proc.stdin.write(decompressor.flush())
147
147
  await borg_proc.stdin.drain()
148
148
 
@@ -159,13 +159,12 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
159
159
  BrokenPipeError,
160
160
  ) as e:
161
161
  logger.warning(
162
- "Client error on transmission: %s, gracefully terminating borg...",
162
+ "Client error on transmission: %s",
163
163
  e,
164
164
  exc_info=True,
165
165
  )
166
166
 
167
167
  if borg_proc:
168
- logger.info("terminating borg process")
169
168
  borg_proc.terminate()
170
169
 
171
170
  try:
@@ -183,6 +182,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
183
182
  "delete",
184
183
  borg_archive,
185
184
  )
185
+
186
186
  # reraise exception for main close handler
187
187
  raise
188
188
  finally:
@@ -226,7 +226,6 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
226
226
 
227
227
  writer.write(archives_pickled)
228
228
  await writer.drain()
229
- logger.debug("send archives")
230
229
 
231
230
  case Command.LIST_BACKUP_DETAILS:
232
231
  timestamp = (await reader.readline()).decode().rstrip("\n")
@@ -270,7 +269,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
270
269
 
271
270
  case Command.INIT_RESTORE_PROCEDURE:
272
271
  timestamp = (await reader.readline()).decode().rstrip("\n")
273
- logger.info(timestamp)
272
+ logger.info("init restore procedure for %s", timestamp)
274
273
 
275
274
  db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
276
275
 
@@ -310,18 +309,16 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
310
309
  writer.write(meta_pickled)
311
310
  await writer.drain()
312
311
 
313
- logger.info(
314
- "send initial config / secrets - waiting for archive requests"
315
- )
316
-
317
312
  case Command.REQUEST_ARCHIVE:
318
313
  # next the client requests the archives which we extract here and pipe via a stream
319
314
  # open the extract process and send the stream the output
320
315
  request_archive = (await reader.readline()).decode().rstrip("\n")
321
- logger.info(request_archive)
322
-
323
316
  request_artifact = (await reader.readline()).decode().rstrip("\n")
324
- logger.info(request_artifact)
317
+ logger.info(
318
+ "request archive: %s, request artifact: %s",
319
+ request_archive,
320
+ request_artifact,
321
+ )
325
322
 
326
323
  backup_dir = f"{get_backup_base_dir()}/{request_archive}"
327
324
  lock = await get_lock(backup_dir)
@@ -360,7 +357,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
360
357
  # send the rest in the compressor
361
358
  await send_cchunk(writer, reader, compressor.flush())
362
359
 
363
- logger.info("sending eof")
360
+ logger.info("finished extracting, sending eof")
364
361
  writer.write(struct.pack("!I", 0))
365
362
  await writer.drain()
366
363
 
@@ -368,7 +365,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
368
365
  logger.warning("Client disconnected: %s", e, exc_info=True)
369
366
  finally:
370
367
  writer.close()
371
- # dont await on server side
368
+ # dont await to avoid race conditions
372
369
 
373
370
 
374
371
  async def run():
@@ -386,25 +383,7 @@ async def run():
386
383
 
387
384
 
388
385
  def main():
389
- # # wait for drive to be available
390
- # while True:
391
- # try:
392
- # get_backup_base_dir()
393
- # logger.info("Backup drive is available!")
394
- # break
395
- # except FileNotFoundError as e:
396
- # logger.debug(e)
397
- # logger.info("Backup drive not found, startup delayed.")
398
- # time.sleep(5)
399
-
400
386
  if ENV == "PRODUCTION":
401
387
  copy_backup_generic()
402
388
 
403
- # backup_store_env_vars = ["PXC_BACKUP_BASE_DIR", "PXC_REMOVABLE_DATASTORES"]
404
- # num_defined = len([var for var in backup_store_env_vars if os.getenv(var)])
405
- # if num_defined != 1:
406
- # raise Exception(
407
- # f"Number of defined backup store vars is {num_defined} but should only be exactly 1 defined!"
408
- # )
409
-
410
389
  asyncio.run(run())
@@ -19,7 +19,7 @@ from kubernetes.client import (ApiException, V1ConfigMapVolumeSource,
19
19
  from kubernetes.config.kube_config import KubeConfigLoader
20
20
  from pve_cloud.cli.pvclu import (get_cloud_domain, get_ssh_master_kubeconfig,
21
21
  get_ssh_remote_master_kubeconfig)
22
- from pve_cloud.cli.pxrpc import get_simple_pxrpc, launch_pxrpc
22
+ from pve_cloud.cli.pxrpc import get_simple_pxrpc
23
23
  from pve_cloud.lib.backup_rpc import Command
24
24
  from pve_cloud.lib.inventory import (get_cloud_domain, get_cluster_vars,
25
25
  get_online_pve_host,
@@ -84,7 +84,6 @@ async def list_backup_details_remote(args):
84
84
  else:
85
85
  # we will connect directly to the backup server
86
86
  # todo: here we can also pass the correct tls config
87
-
88
87
  tls_disc_raw = await pxrpc.get_cloud_secret(
89
88
  cloud_domain, f"{bdd_stack_name}-bdd-tls-discovery"
90
89
  )
@@ -418,15 +417,7 @@ async def launch_restore_job(args):
418
417
  env_vars = [
419
418
  V1EnvVar(
420
419
  name="PXC_RESTORE_ARGS",
421
- value=base64.b64encode(
422
- json.dumps(
423
- serializable_args
424
- # | {
425
- # "cloud_domain": cloud_domain,
426
- # "stack_name": kubespray_inv["stack_name"],
427
- # }
428
- ).encode()
429
- ).decode(),
420
+ value=base64.b64encode(json.dumps(serializable_args).encode()).decode(),
430
421
  ),
431
422
  V1EnvVar(name="LOG_LEVEL", value=args.log_level),
432
423
  ]
@@ -539,14 +530,12 @@ def get_parser():
539
530
  "--inventory",
540
531
  type=str,
541
532
  help="PVE cloud kubespray inventory yaml file or pxc external hosts k0s conform inventory file, in this cluster the restore job will be launched.",
542
- # required=True,
543
533
  )
544
534
  base_parser.add_argument(
545
535
  "--use-mc-gw",
546
536
  action="store_true",
547
537
  help="Configures the backup job with the clouds external gateway instead of the internal bdd servers ip.",
548
538
  )
549
- # todo: implement bdd-host-address
550
539
 
551
540
  subparsers = parser.add_subparsers(dest="command", required=True)
552
541
 
@@ -13,14 +13,12 @@ from contextlib import asynccontextmanager
13
13
  from pprint import pformat
14
14
 
15
15
  import asyncssh
16
- import paramiko
17
16
  import socketio
18
17
  import zstandard as zstd
19
18
  from kubernetes import client, config
20
19
  from kubernetes.client.rest import ApiException
21
20
  from kubernetes.utils.quantity import parse_quantity
22
21
  from pve_cloud.lib.backup_rpc import Command
23
- from tinydb import Query, TinyDB
24
22
 
25
23
  log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
26
24
  log_level = getattr(logging, log_level_str, logging.INFO)
@@ -51,6 +49,7 @@ def convert_keys_to_camel_case(obj):
51
49
  return obj
52
50
 
53
51
 
52
+ # connection helpers to either gateway or server directory
54
53
  @asynccontextmanager
55
54
  async def get_direct_conn(restore_args):
56
55
  ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
@@ -67,8 +66,7 @@ async def get_direct_conn(restore_args):
67
66
 
68
67
  @asynccontextmanager
69
68
  async def get_sio_conn(restore_args):
70
- log_debug = os.getenv("LOG_LEVEL") == "DEBUG"
71
- sio = socketio.AsyncClient() # (logger=log_debug, engineio_logger=log_debug)
69
+ sio = socketio.AsyncClient()
72
70
 
73
71
  await sio.connect(
74
72
  f"https://{restore_args['mc_gw_host']}",
@@ -86,7 +84,6 @@ async def get_sio_conn(restore_args):
86
84
 
87
85
  async def init_procedure_bdd(restore_args):
88
86
  if restore_args["use_mc_gw"]:
89
-
90
87
  async with get_sio_conn(restore_args) as sio:
91
88
  result_pickled = await sio.call(
92
89
  "init_restore", restore_args["timestamp"], timeout=30
@@ -98,7 +95,6 @@ async def init_procedure_bdd(restore_args):
98
95
  return result["metas_grouped_by_ns"], result["namespace_secret_dict"]
99
96
 
100
97
  else:
101
-
102
98
  async with get_direct_conn(restore_args) as (reader, writer):
103
99
  # connect to the backup server and start the restore procedure
104
100
  # we simply trust the host here as the ca is only available from
@@ -483,9 +479,8 @@ async def procedure():
483
479
  known_hosts=None,
484
480
  ) as ssh:
485
481
  restore_pvc_uuid = str(uuid.uuid4())
482
+
486
483
  # first we create a zvol of identical size
487
- # todo: remove -rest stub and replace with new id generation
488
- # todo: write converter function to create zvol in exact size like k8s did zfs list -t volume -o name,volsize
489
484
  zvol_create_cmd = f"sudo zfs create -s -V {parse_quantity(pvc_dict['spec']['resources']['requests']['storage'])} tank-pv/pvc-{restore_pvc_uuid}"
490
485
  logger.info("Executing create: %s", zvol_create_cmd)
491
486
 
@@ -506,6 +501,7 @@ async def procedure():
506
501
  )
507
502
 
508
503
  if restore_args["use_mc_gw"]:
504
+ # retry mechanism for more instable remote restore procedure via the gateway
509
505
  for attempt in range(1, SIO_MAX_RETRIES + 1):
510
506
  try:
511
507
  async with get_sio_conn(restore_args) as sio:
@@ -518,10 +514,6 @@ async def procedure():
518
514
  timeout=30,
519
515
  )
520
516
 
521
- logger.info(
522
- "send request artifact / archive - requesting chunks"
523
- )
524
-
525
517
  while True:
526
518
  chunk = await sio.call(
527
519
  "request_chunk", timeout=30
@@ -540,8 +532,9 @@ async def procedure():
540
532
  if attempt == SIO_MAX_RETRIES:
541
533
  raise
542
534
 
535
+ logger.info("Retrying...")
536
+
543
537
  # gracefully terminate input pipeline
544
- logger.debug("terminating dd zvol import proc")
545
538
  proc.terminate()
546
539
 
547
540
  try:
@@ -553,27 +546,21 @@ async def procedure():
553
546
  proc.kill()
554
547
  await proc.wait()
555
548
 
556
- logger.info("Retrying...")
549
+ # wait before retrying
557
550
  await asyncio.sleep(10)
558
551
 
559
- # destroy the zfs volume
560
- logger.debug(
561
- f"cleaning up zfs vol tank-pv/pvc-{restore_pvc_uuid}"
562
- )
563
552
  cleanup_proc = await ssh.run(
564
553
  f"sudo zfs destroy tank-pv/pvc-{restore_pvc_uuid}"
565
554
  )
566
555
 
567
556
  await cleanup_proc.wait()
568
-
569
557
  logger.debug(
570
558
  f"cleanup retcode {cleanup_proc.returncode}"
571
559
  )
572
560
 
573
561
  # rerun init commands
574
- # todo: generic retry could make this much learner combining with direct connect
562
+ # todo: generic retry could make this much cleaner combining with direct connect
575
563
  logger.debug("relaunching zvol create cmd")
576
- await ssh.run(zvol_create_cmd, check=True)
577
564
  proc = await ssh.create_process(
578
565
  import_cmd,
579
566
  encoding=None,
@@ -581,7 +568,6 @@ async def procedure():
581
568
 
582
569
  else:
583
570
  async with get_direct_conn(restore_args) as (reader, writer):
584
-
585
571
  writer.write(
586
572
  struct.pack("B", Command.REQUEST_ARCHIVE.value)
587
573
  )
@@ -594,10 +580,6 @@ async def procedure():
594
580
  writer.write(request_artifact.encode())
595
581
  await writer.drain()
596
582
 
597
- logger.info(
598
- "send request artifact / archive - requesting chunks"
599
- )
600
-
601
583
  while True:
602
584
  # client first always sends chunk size
603
585
  chunk_size = struct.unpack(
@@ -616,7 +598,7 @@ async def procedure():
616
598
  proc.stdin.write(chunk)
617
599
  await proc.stdin.drain()
618
600
 
619
- logger.info("done reading closing proc")
601
+ # close the process regularly
620
602
  proc.stdin.close()
621
603
  await proc.wait()
622
604
 
@@ -739,7 +721,6 @@ async def procedure():
739
721
  )
740
722
 
741
723
  elif target_provisioner == "rbd.csi.ceph.com":
742
-
743
724
  new_csi_image_name = f"csi-vol-{uuid.uuid4()}"
744
725
 
745
726
  # send to the bdd server what we want to request
@@ -760,6 +741,7 @@ async def procedure():
760
741
  )
761
742
 
762
743
  if restore_args["use_mc_gw"]:
744
+ # retry mechanism for more instable remote restore procedures via multi cloud gateway
763
745
  for attempt in range(1, SIO_MAX_RETRIES + 1):
764
746
  try:
765
747
  async with get_sio_conn(restore_args) as sio:
@@ -792,8 +774,9 @@ async def procedure():
792
774
  if attempt == SIO_MAX_RETRIES:
793
775
  raise
794
776
 
777
+ logger.info("retrying...")
778
+
795
779
  # gracefully terminate input pipeline
796
- logger.debug("terminating rbd import proc")
797
780
  rbd_import_proc.terminate()
798
781
 
799
782
  try:
@@ -808,12 +791,8 @@ async def procedure():
808
791
  await rbd_import_proc.wait()
809
792
 
810
793
  # destroy the rbd image
811
- logger.info("Retrying...")
812
794
  await asyncio.sleep(10)
813
795
 
814
- logger.debug(
815
- f"cleanup rbd image {pool}/{new_csi_image_name}"
816
- )
817
796
  cleanup_proc = await asyncio.create_subprocess_exec(
818
797
  "rbd",
819
798
  "rm",
@@ -821,7 +800,6 @@ async def procedure():
821
800
  )
822
801
 
823
802
  await cleanup_proc.wait()
824
-
825
803
  logger.debug(f"cleanup retcode {cleanup_proc.returncode}")
826
804
 
827
805
  # rerun init commands
@@ -836,7 +814,6 @@ async def procedure():
836
814
 
837
815
  else:
838
816
  async with get_direct_conn(restore_args) as (reader, writer):
839
-
840
817
  writer.write(struct.pack("B", Command.REQUEST_ARCHIVE.value))
841
818
  await writer.drain()
842
819
 
@@ -260,7 +260,7 @@ async def send_export(send_command, semaphore):
260
260
 
261
261
  await proc.wait()
262
262
 
263
- await net.archive_async(backup_addr, request_dict, async_chunk_generator)
263
+ await net.archive(backup_addr, request_dict, async_chunk_generator)
264
264
 
265
265
 
266
266
  async def send_backups(namespace_volume_meta, timestamp, backup_addr):
@@ -512,7 +512,7 @@ async def zfs_snap_and_send(
512
512
 
513
513
  logger.info("dd exit code %s", proc.exit_status)
514
514
 
515
- await net.archive_async(
515
+ await net.archive(
516
516
  backup_addr, request_dict, chunk_generator, compress=False
517
517
  )
518
518
 
@@ -1,4 +1,5 @@
1
1
  import asyncio
2
+ import inspect
2
3
  import logging
3
4
  import os
4
5
  import pickle
@@ -28,7 +29,6 @@ async def archive_init(reader, writer, request_dict):
28
29
  # intialize archive command
29
30
  writer.write(struct.pack("B", Command.ARCHIVE.value))
30
31
  await writer.drain()
31
- logger.debug("send command")
32
32
 
33
33
  # send the archive request dict
34
34
  req_dict_pickled = pickle.dumps(request_dict)
@@ -36,7 +36,6 @@ async def archive_init(reader, writer, request_dict):
36
36
  await writer.drain()
37
37
  writer.write(req_dict_pickled)
38
38
  await writer.drain()
39
- logger.debug("send dict")
40
39
 
41
40
  # wait for go signal, server needs to aquire write lock
42
41
  # we dont
@@ -51,13 +50,12 @@ async def archive_init(reader, writer, request_dict):
51
50
  logger.error("recieved incorrect go signal")
52
51
  raise Exception("Incorrect go signal!")
53
52
  else:
53
+ logger.debug("received go")
54
54
  break
55
55
 
56
- logger.debug("received go")
57
-
58
56
 
57
+ # generic send and ack function
59
58
  async def send_cchunk(writer, reader, compressed_chunk):
60
- # compress the chunk
61
59
  if compressed_chunk: # only send if something actually got compressed
62
60
  # send size + chunk
63
61
  writer.write(struct.pack("!I", len(compressed_chunk)))
@@ -76,9 +74,7 @@ async def get_sio_mc_client(backup_addr):
76
74
  "Tried to initialize multicloud proxy without providing MC_EXT_TOKEN env var!"
77
75
  )
78
76
 
79
- log_debug = os.getenv("LOG_LEVEL") == "DEBUG"
80
- sio = socketio.AsyncClient() # (logger=log_debug, engineio_logger=log_debug)
81
-
77
+ sio = socketio.AsyncClient()
82
78
  await sio.connect(
83
79
  backup_addr,
84
80
  auth={
@@ -87,31 +83,39 @@ async def get_sio_mc_client(backup_addr):
87
83
  },
88
84
  transports=["websocket"],
89
85
  )
86
+
90
87
  logger.debug(f"Connected sio client to {backup_addr}")
88
+
91
89
  return sio
92
90
 
93
91
 
92
+ # sio framework takes care of acking messages
94
93
  async def sio_send_cchunk(sio, compressed_chunk):
95
94
  if compressed_chunk:
96
95
  await sio.call("backup_chunk", compressed_chunk)
97
96
 
98
97
 
99
98
  async def wait_archive_init(sio, request_dict):
99
+ # init archive request, this doesn't necessarily immediatly
100
+ # lock and ready the server for writing
100
101
  initial = await sio.call(
101
102
  "archive_init",
102
103
  request_dict,
103
104
  timeout=30,
104
105
  )
106
+ logger.debug("received init response: %s", initial)
107
+
105
108
  if initial["status"] == "ERR":
106
109
  raise RuntimeError(initial["error"])
107
110
 
108
111
  if initial["status"] == "ACQUIRED":
109
- return
112
+ return # server acquired lock for backup repo
110
113
 
111
- logger.info("waiting for lock")
112
- # status WAIT
114
+ # ==> status WAIT
115
+ logger.info("waiting for lock...")
113
116
  while True:
114
117
  wait_call = await sio.call("wait_archive", timeout=30)
118
+ logger.debug("wait archive response: %s", wait_call)
115
119
 
116
120
  if wait_call["status"] == "ERR":
117
121
  raise RuntimeError(initial["error"])
@@ -119,24 +123,21 @@ async def wait_archive_init(sio, request_dict):
119
123
  if wait_call["status"] == "ACQUIRED":
120
124
  return
121
125
 
122
- # continue wait
123
- logger.info("continueing waiting")
124
-
125
126
 
126
127
  # compress parameter exists for chunk generators that already do the compression
127
128
  # the receiving side ALWAYS expects a compressed stream
128
- async def archive_async(backup_addr, request_dict, chunk_generator, compress=True):
129
- logger.info(request_dict)
129
+ async def archive(backup_addr, request_dict, chunk_generator, compress=True):
130
+ logger.info("sending archive request: %s", request_dict)
131
+ logger.debug("generator async: %s", inspect.isasyncgenfunction(chunk_generator))
130
132
 
133
+ # we assume that we are sending to a multicloud gateway if the addr starts with https://
134
+ # direct connects via tcp are if the backup_addr is a hostname / ip address
131
135
  if backup_addr.startswith("https://"):
132
136
  logger.info(f"sending archive to mc gateway {backup_addr}")
133
137
 
134
- # retry is currently only implemented for async generators
135
- # which are the primary generator type for large image backups
136
138
  for attempt in range(1, SIO_MAX_RETRIES + 1):
137
139
  sio = None
138
140
  try:
139
-
140
141
  # connection to mc gw
141
142
  sio = await get_sio_mc_client(backup_addr)
142
143
 
@@ -148,18 +149,27 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
148
149
  threads=6,
149
150
  ).compressobj()
150
151
 
151
- async for chunk in chunk_generator():
152
- await sio_send_cchunk(sio, compressor.compress(chunk))
152
+ if inspect.isasyncgenfunction(chunk_generator):
153
+ async for chunk in chunk_generator():
154
+ await sio_send_cchunk(sio, compressor.compress(chunk))
155
+ else:
156
+ for chunk in chunk_generator():
157
+ await sio_send_cchunk(sio, compressor.compress(chunk))
153
158
 
154
159
  await sio_send_cchunk(sio, compressor.flush())
155
160
 
156
161
  else:
157
- async for chunk in chunk_generator():
158
- await sio.call("backup_chunk", chunk)
159
-
162
+ if inspect.isasyncgenfunction(chunk_generator):
163
+ async for chunk in chunk_generator():
164
+ await sio.call("backup_chunk", chunk)
165
+ else:
166
+ for chunk in chunk_generator():
167
+ await sio.call("backup_chunk", chunk)
168
+
169
+ # signal the server that we are done
160
170
  await sio.call("backup_eof")
161
171
 
162
- break # finished successfully
172
+ break # finished successfully, break try loop
163
173
 
164
174
  except socketio.exceptions.TimeoutError:
165
175
  logger.warn(f"Error on attempt {attempt}")
@@ -170,10 +180,12 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
170
180
  await asyncio.sleep(10)
171
181
 
172
182
  finally:
183
+ # always close the sio connection
173
184
  if sio:
174
185
  await sio.disconnect()
175
186
 
176
187
  else:
188
+ # direct connection to backup server
177
189
  reader, writer = await asyncio.open_connection(
178
190
  backup_addr, 8085, ssl=get_strict_client_ssl_ctx()
179
191
  )
@@ -184,65 +196,30 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
184
196
  # compressor = zlib.compressobj(level=1)
185
197
  if compress:
186
198
  compressor = zstd.ZstdCompressor(level=1, threads=6).compressobj()
187
- async for chunk in chunk_generator():
188
- await send_cchunk(writer, reader, compressor.compress(chunk))
199
+ if inspect.isasyncgenfunction(chunk_generator):
200
+ async for chunk in chunk_generator():
201
+ await send_cchunk(writer, reader, compressor.compress(chunk))
202
+ else:
203
+ for chunk in chunk_generator():
204
+ await send_cchunk(writer, reader, compressor.compress(chunk))
205
+
189
206
  # send rest in compressor, compress doesnt always return a byte array, see bdd.py doc
190
207
  # send size first again
191
208
  await send_cchunk(writer, reader, compressor.flush())
192
209
 
193
210
  else:
194
- async for chunk in chunk_generator():
195
- writer.write(struct.pack("!I", len(chunk)))
196
- await writer.drain()
197
- writer.write(chunk)
198
- await writer.drain()
199
-
200
- # send eof to server, signal that we are done
201
- logger.debug("sending eof")
202
- writer.write(struct.pack("!I", 0))
203
- await writer.drain()
204
-
205
- # close the writer here, stdout needs to be closed by caller
206
- writer.close()
207
-
208
-
209
- async def archive(backup_addr, request_dict, chunk_generator):
210
- logger.info(request_dict)
211
- if backup_addr.startswith("https://"):
212
- sio = await get_sio_mc_client(backup_addr)
213
-
214
- await wait_archive_init(sio, request_dict)
215
-
216
- compressor = zstd.ZstdCompressor(
217
- level=1,
218
- threads=6,
219
- ).compressobj()
220
-
221
- for chunk in chunk_generator():
222
- await sio_send_cchunk(sio, compressor.compress(chunk))
223
-
224
- await sio_send_cchunk(sio, compressor.flush())
225
-
226
- await sio.call("backup_eof")
227
-
228
- await sio.disconnect()
229
-
230
- else:
231
- reader, writer = await asyncio.open_connection(
232
- backup_addr, 8085, ssl=get_strict_client_ssl_ctx()
233
- )
234
-
235
- await archive_init(reader, writer, request_dict)
236
-
237
- # initialize the synchronous generator and start reading chunks, compress and send
238
- # compressor = zlib.compressobj(level=1)
239
- compressor = zstd.ZstdCompressor(level=1, threads=6).compressobj()
240
- for chunk in chunk_generator():
241
- await send_cchunk(writer, reader, compressor.compress(chunk))
242
-
243
- # send rest in compressor, compress doesnt always return a byte array, see bdd.py doc
244
- # send size first again
245
- await send_cchunk(writer, reader, compressor.flush())
211
+ if inspect.isasyncgenfunction(chunk_generator):
212
+ async for chunk in chunk_generator():
213
+ writer.write(struct.pack("!I", len(chunk)))
214
+ await writer.drain()
215
+ writer.write(chunk)
216
+ await writer.drain()
217
+ else:
218
+ for chunk in chunk_generator():
219
+ writer.write(struct.pack("!I", len(chunk)))
220
+ await writer.drain()
221
+ writer.write(chunk)
222
+ await writer.drain()
246
223
 
247
224
  # send eof to server, signal that we are done
248
225
  logger.debug("sending eof")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.14
3
+ Version: 3.5.16
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -1 +0,0 @@
1
- __version__ = "3.5.14"