py-pve-cloud-backup 3.5.14__tar.gz → 3.5.16__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {py_pve_cloud_backup-3.5.14/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.16}/PKG-INFO +1 -1
- py_pve_cloud_backup-3.5.16/src/pve_cloud_backup/_version.py +1 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/bdd.py +11 -32
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/brctl.py +2 -13
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/restore.py +12 -35
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/funcs.py +2 -2
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/net.py +56 -79
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16/src/py_pve_cloud_backup.egg-info}/PKG-INFO +1 -1
- py_pve_cloud_backup-3.5.14/src/pve_cloud_backup/_version.py +0 -1
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/LICENSE.md +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/README.md +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/pyproject.toml +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/requirements.txt +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/setup.cfg +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/funcs.py +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/ext_zfs.py +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/git.py +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/main.py +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/entry_points.txt +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/requires.txt +0 -0
- {py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "3.5.16"
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/bdd.py
RENAMED
|
@@ -72,6 +72,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
72
72
|
# the entire app has retries for connects and can handle connection outages well.
|
|
73
73
|
# implement something like toxiproxy in e2e testing and make resilient.
|
|
74
74
|
lock = await get_lock(backup_dir)
|
|
75
|
+
|
|
75
76
|
# store ref to kill in case of failure => this will stop the archive from being committet
|
|
76
77
|
borg_proc = None
|
|
77
78
|
borg_archive = f"{backup_dir}::{archive_name}_{timestamp}"
|
|
@@ -142,7 +143,6 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
142
143
|
|
|
143
144
|
# the decompressor does not always return a decompressed chunk but might retain
|
|
144
145
|
# and return empty. at the end we need to call flush to get everything out
|
|
145
|
-
logger.debug("flushing, draining and closing")
|
|
146
146
|
borg_proc.stdin.write(decompressor.flush())
|
|
147
147
|
await borg_proc.stdin.drain()
|
|
148
148
|
|
|
@@ -159,13 +159,12 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
159
159
|
BrokenPipeError,
|
|
160
160
|
) as e:
|
|
161
161
|
logger.warning(
|
|
162
|
-
"Client error on transmission: %s
|
|
162
|
+
"Client error on transmission: %s",
|
|
163
163
|
e,
|
|
164
164
|
exc_info=True,
|
|
165
165
|
)
|
|
166
166
|
|
|
167
167
|
if borg_proc:
|
|
168
|
-
logger.info("terminating borg process")
|
|
169
168
|
borg_proc.terminate()
|
|
170
169
|
|
|
171
170
|
try:
|
|
@@ -183,6 +182,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
183
182
|
"delete",
|
|
184
183
|
borg_archive,
|
|
185
184
|
)
|
|
185
|
+
|
|
186
186
|
# reraise exception for main close handler
|
|
187
187
|
raise
|
|
188
188
|
finally:
|
|
@@ -226,7 +226,6 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
226
226
|
|
|
227
227
|
writer.write(archives_pickled)
|
|
228
228
|
await writer.drain()
|
|
229
|
-
logger.debug("send archives")
|
|
230
229
|
|
|
231
230
|
case Command.LIST_BACKUP_DETAILS:
|
|
232
231
|
timestamp = (await reader.readline()).decode().rstrip("\n")
|
|
@@ -270,7 +269,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
270
269
|
|
|
271
270
|
case Command.INIT_RESTORE_PROCEDURE:
|
|
272
271
|
timestamp = (await reader.readline()).decode().rstrip("\n")
|
|
273
|
-
logger.info(timestamp)
|
|
272
|
+
logger.info("init restore procedure for %s", timestamp)
|
|
274
273
|
|
|
275
274
|
db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
|
|
276
275
|
|
|
@@ -310,18 +309,16 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
310
309
|
writer.write(meta_pickled)
|
|
311
310
|
await writer.drain()
|
|
312
311
|
|
|
313
|
-
logger.info(
|
|
314
|
-
"send initial config / secrets - waiting for archive requests"
|
|
315
|
-
)
|
|
316
|
-
|
|
317
312
|
case Command.REQUEST_ARCHIVE:
|
|
318
313
|
# next the client requests the archives which we extract here and pipe via a stream
|
|
319
314
|
# open the extract process and send the stream the output
|
|
320
315
|
request_archive = (await reader.readline()).decode().rstrip("\n")
|
|
321
|
-
logger.info(request_archive)
|
|
322
|
-
|
|
323
316
|
request_artifact = (await reader.readline()).decode().rstrip("\n")
|
|
324
|
-
logger.info(
|
|
317
|
+
logger.info(
|
|
318
|
+
"request archive: %s, request artifact: %s",
|
|
319
|
+
request_archive,
|
|
320
|
+
request_artifact,
|
|
321
|
+
)
|
|
325
322
|
|
|
326
323
|
backup_dir = f"{get_backup_base_dir()}/{request_archive}"
|
|
327
324
|
lock = await get_lock(backup_dir)
|
|
@@ -360,7 +357,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
360
357
|
# send the rest in the compressor
|
|
361
358
|
await send_cchunk(writer, reader, compressor.flush())
|
|
362
359
|
|
|
363
|
-
logger.info("sending eof")
|
|
360
|
+
logger.info("finished extracting, sending eof")
|
|
364
361
|
writer.write(struct.pack("!I", 0))
|
|
365
362
|
await writer.drain()
|
|
366
363
|
|
|
@@ -368,7 +365,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
368
365
|
logger.warning("Client disconnected: %s", e, exc_info=True)
|
|
369
366
|
finally:
|
|
370
367
|
writer.close()
|
|
371
|
-
# dont await
|
|
368
|
+
# dont await to avoid race conditions
|
|
372
369
|
|
|
373
370
|
|
|
374
371
|
async def run():
|
|
@@ -386,25 +383,7 @@ async def run():
|
|
|
386
383
|
|
|
387
384
|
|
|
388
385
|
def main():
|
|
389
|
-
# # wait for drive to be available
|
|
390
|
-
# while True:
|
|
391
|
-
# try:
|
|
392
|
-
# get_backup_base_dir()
|
|
393
|
-
# logger.info("Backup drive is available!")
|
|
394
|
-
# break
|
|
395
|
-
# except FileNotFoundError as e:
|
|
396
|
-
# logger.debug(e)
|
|
397
|
-
# logger.info("Backup drive not found, startup delayed.")
|
|
398
|
-
# time.sleep(5)
|
|
399
|
-
|
|
400
386
|
if ENV == "PRODUCTION":
|
|
401
387
|
copy_backup_generic()
|
|
402
388
|
|
|
403
|
-
# backup_store_env_vars = ["PXC_BACKUP_BASE_DIR", "PXC_REMOVABLE_DATASTORES"]
|
|
404
|
-
# num_defined = len([var for var in backup_store_env_vars if os.getenv(var)])
|
|
405
|
-
# if num_defined != 1:
|
|
406
|
-
# raise Exception(
|
|
407
|
-
# f"Number of defined backup store vars is {num_defined} but should only be exactly 1 defined!"
|
|
408
|
-
# )
|
|
409
|
-
|
|
410
389
|
asyncio.run(run())
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/brctl.py
RENAMED
|
@@ -19,7 +19,7 @@ from kubernetes.client import (ApiException, V1ConfigMapVolumeSource,
|
|
|
19
19
|
from kubernetes.config.kube_config import KubeConfigLoader
|
|
20
20
|
from pve_cloud.cli.pvclu import (get_cloud_domain, get_ssh_master_kubeconfig,
|
|
21
21
|
get_ssh_remote_master_kubeconfig)
|
|
22
|
-
from pve_cloud.cli.pxrpc import get_simple_pxrpc
|
|
22
|
+
from pve_cloud.cli.pxrpc import get_simple_pxrpc
|
|
23
23
|
from pve_cloud.lib.backup_rpc import Command
|
|
24
24
|
from pve_cloud.lib.inventory import (get_cloud_domain, get_cluster_vars,
|
|
25
25
|
get_online_pve_host,
|
|
@@ -84,7 +84,6 @@ async def list_backup_details_remote(args):
|
|
|
84
84
|
else:
|
|
85
85
|
# we will connect directly to the backup server
|
|
86
86
|
# todo: here we can also pass the correct tls config
|
|
87
|
-
|
|
88
87
|
tls_disc_raw = await pxrpc.get_cloud_secret(
|
|
89
88
|
cloud_domain, f"{bdd_stack_name}-bdd-tls-discovery"
|
|
90
89
|
)
|
|
@@ -418,15 +417,7 @@ async def launch_restore_job(args):
|
|
|
418
417
|
env_vars = [
|
|
419
418
|
V1EnvVar(
|
|
420
419
|
name="PXC_RESTORE_ARGS",
|
|
421
|
-
value=base64.b64encode(
|
|
422
|
-
json.dumps(
|
|
423
|
-
serializable_args
|
|
424
|
-
# | {
|
|
425
|
-
# "cloud_domain": cloud_domain,
|
|
426
|
-
# "stack_name": kubespray_inv["stack_name"],
|
|
427
|
-
# }
|
|
428
|
-
).encode()
|
|
429
|
-
).decode(),
|
|
420
|
+
value=base64.b64encode(json.dumps(serializable_args).encode()).decode(),
|
|
430
421
|
),
|
|
431
422
|
V1EnvVar(name="LOG_LEVEL", value=args.log_level),
|
|
432
423
|
]
|
|
@@ -539,14 +530,12 @@ def get_parser():
|
|
|
539
530
|
"--inventory",
|
|
540
531
|
type=str,
|
|
541
532
|
help="PVE cloud kubespray inventory yaml file or pxc external hosts k0s conform inventory file, in this cluster the restore job will be launched.",
|
|
542
|
-
# required=True,
|
|
543
533
|
)
|
|
544
534
|
base_parser.add_argument(
|
|
545
535
|
"--use-mc-gw",
|
|
546
536
|
action="store_true",
|
|
547
537
|
help="Configures the backup job with the clouds external gateway instead of the internal bdd servers ip.",
|
|
548
538
|
)
|
|
549
|
-
# todo: implement bdd-host-address
|
|
550
539
|
|
|
551
540
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
552
541
|
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/restore.py
RENAMED
|
@@ -13,14 +13,12 @@ from contextlib import asynccontextmanager
|
|
|
13
13
|
from pprint import pformat
|
|
14
14
|
|
|
15
15
|
import asyncssh
|
|
16
|
-
import paramiko
|
|
17
16
|
import socketio
|
|
18
17
|
import zstandard as zstd
|
|
19
18
|
from kubernetes import client, config
|
|
20
19
|
from kubernetes.client.rest import ApiException
|
|
21
20
|
from kubernetes.utils.quantity import parse_quantity
|
|
22
21
|
from pve_cloud.lib.backup_rpc import Command
|
|
23
|
-
from tinydb import Query, TinyDB
|
|
24
22
|
|
|
25
23
|
log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
|
|
26
24
|
log_level = getattr(logging, log_level_str, logging.INFO)
|
|
@@ -51,6 +49,7 @@ def convert_keys_to_camel_case(obj):
|
|
|
51
49
|
return obj
|
|
52
50
|
|
|
53
51
|
|
|
52
|
+
# connection helpers to either gateway or server directory
|
|
54
53
|
@asynccontextmanager
|
|
55
54
|
async def get_direct_conn(restore_args):
|
|
56
55
|
ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
|
@@ -67,8 +66,7 @@ async def get_direct_conn(restore_args):
|
|
|
67
66
|
|
|
68
67
|
@asynccontextmanager
|
|
69
68
|
async def get_sio_conn(restore_args):
|
|
70
|
-
|
|
71
|
-
sio = socketio.AsyncClient() # (logger=log_debug, engineio_logger=log_debug)
|
|
69
|
+
sio = socketio.AsyncClient()
|
|
72
70
|
|
|
73
71
|
await sio.connect(
|
|
74
72
|
f"https://{restore_args['mc_gw_host']}",
|
|
@@ -86,7 +84,6 @@ async def get_sio_conn(restore_args):
|
|
|
86
84
|
|
|
87
85
|
async def init_procedure_bdd(restore_args):
|
|
88
86
|
if restore_args["use_mc_gw"]:
|
|
89
|
-
|
|
90
87
|
async with get_sio_conn(restore_args) as sio:
|
|
91
88
|
result_pickled = await sio.call(
|
|
92
89
|
"init_restore", restore_args["timestamp"], timeout=30
|
|
@@ -98,7 +95,6 @@ async def init_procedure_bdd(restore_args):
|
|
|
98
95
|
return result["metas_grouped_by_ns"], result["namespace_secret_dict"]
|
|
99
96
|
|
|
100
97
|
else:
|
|
101
|
-
|
|
102
98
|
async with get_direct_conn(restore_args) as (reader, writer):
|
|
103
99
|
# connect to the backup server and start the restore procedure
|
|
104
100
|
# we simply trust the host here as the ca is only available from
|
|
@@ -483,9 +479,8 @@ async def procedure():
|
|
|
483
479
|
known_hosts=None,
|
|
484
480
|
) as ssh:
|
|
485
481
|
restore_pvc_uuid = str(uuid.uuid4())
|
|
482
|
+
|
|
486
483
|
# first we create a zvol of identical size
|
|
487
|
-
# todo: remove -rest stub and replace with new id generation
|
|
488
|
-
# todo: write converter function to create zvol in exact size like k8s did zfs list -t volume -o name,volsize
|
|
489
484
|
zvol_create_cmd = f"sudo zfs create -s -V {parse_quantity(pvc_dict['spec']['resources']['requests']['storage'])} tank-pv/pvc-{restore_pvc_uuid}"
|
|
490
485
|
logger.info("Executing create: %s", zvol_create_cmd)
|
|
491
486
|
|
|
@@ -506,6 +501,7 @@ async def procedure():
|
|
|
506
501
|
)
|
|
507
502
|
|
|
508
503
|
if restore_args["use_mc_gw"]:
|
|
504
|
+
# retry mechanism for more instable remote restore procedure via the gateway
|
|
509
505
|
for attempt in range(1, SIO_MAX_RETRIES + 1):
|
|
510
506
|
try:
|
|
511
507
|
async with get_sio_conn(restore_args) as sio:
|
|
@@ -518,10 +514,6 @@ async def procedure():
|
|
|
518
514
|
timeout=30,
|
|
519
515
|
)
|
|
520
516
|
|
|
521
|
-
logger.info(
|
|
522
|
-
"send request artifact / archive - requesting chunks"
|
|
523
|
-
)
|
|
524
|
-
|
|
525
517
|
while True:
|
|
526
518
|
chunk = await sio.call(
|
|
527
519
|
"request_chunk", timeout=30
|
|
@@ -540,8 +532,9 @@ async def procedure():
|
|
|
540
532
|
if attempt == SIO_MAX_RETRIES:
|
|
541
533
|
raise
|
|
542
534
|
|
|
535
|
+
logger.info("Retrying...")
|
|
536
|
+
|
|
543
537
|
# gracefully terminate input pipeline
|
|
544
|
-
logger.debug("terminating dd zvol import proc")
|
|
545
538
|
proc.terminate()
|
|
546
539
|
|
|
547
540
|
try:
|
|
@@ -553,27 +546,21 @@ async def procedure():
|
|
|
553
546
|
proc.kill()
|
|
554
547
|
await proc.wait()
|
|
555
548
|
|
|
556
|
-
|
|
549
|
+
# wait before retrying
|
|
557
550
|
await asyncio.sleep(10)
|
|
558
551
|
|
|
559
|
-
# destroy the zfs volume
|
|
560
|
-
logger.debug(
|
|
561
|
-
f"cleaning up zfs vol tank-pv/pvc-{restore_pvc_uuid}"
|
|
562
|
-
)
|
|
563
552
|
cleanup_proc = await ssh.run(
|
|
564
553
|
f"sudo zfs destroy tank-pv/pvc-{restore_pvc_uuid}"
|
|
565
554
|
)
|
|
566
555
|
|
|
567
556
|
await cleanup_proc.wait()
|
|
568
|
-
|
|
569
557
|
logger.debug(
|
|
570
558
|
f"cleanup retcode {cleanup_proc.returncode}"
|
|
571
559
|
)
|
|
572
560
|
|
|
573
561
|
# rerun init commands
|
|
574
|
-
# todo: generic retry could make this much
|
|
562
|
+
# todo: generic retry could make this much cleaner combining with direct connect
|
|
575
563
|
logger.debug("relaunching zvol create cmd")
|
|
576
|
-
await ssh.run(zvol_create_cmd, check=True)
|
|
577
564
|
proc = await ssh.create_process(
|
|
578
565
|
import_cmd,
|
|
579
566
|
encoding=None,
|
|
@@ -581,7 +568,6 @@ async def procedure():
|
|
|
581
568
|
|
|
582
569
|
else:
|
|
583
570
|
async with get_direct_conn(restore_args) as (reader, writer):
|
|
584
|
-
|
|
585
571
|
writer.write(
|
|
586
572
|
struct.pack("B", Command.REQUEST_ARCHIVE.value)
|
|
587
573
|
)
|
|
@@ -594,10 +580,6 @@ async def procedure():
|
|
|
594
580
|
writer.write(request_artifact.encode())
|
|
595
581
|
await writer.drain()
|
|
596
582
|
|
|
597
|
-
logger.info(
|
|
598
|
-
"send request artifact / archive - requesting chunks"
|
|
599
|
-
)
|
|
600
|
-
|
|
601
583
|
while True:
|
|
602
584
|
# client first always sends chunk size
|
|
603
585
|
chunk_size = struct.unpack(
|
|
@@ -616,7 +598,7 @@ async def procedure():
|
|
|
616
598
|
proc.stdin.write(chunk)
|
|
617
599
|
await proc.stdin.drain()
|
|
618
600
|
|
|
619
|
-
|
|
601
|
+
# close the process regularly
|
|
620
602
|
proc.stdin.close()
|
|
621
603
|
await proc.wait()
|
|
622
604
|
|
|
@@ -739,7 +721,6 @@ async def procedure():
|
|
|
739
721
|
)
|
|
740
722
|
|
|
741
723
|
elif target_provisioner == "rbd.csi.ceph.com":
|
|
742
|
-
|
|
743
724
|
new_csi_image_name = f"csi-vol-{uuid.uuid4()}"
|
|
744
725
|
|
|
745
726
|
# send to the bdd server what we want to request
|
|
@@ -760,6 +741,7 @@ async def procedure():
|
|
|
760
741
|
)
|
|
761
742
|
|
|
762
743
|
if restore_args["use_mc_gw"]:
|
|
744
|
+
# retry mechanism for more instable remote restore procedures via multi cloud gateway
|
|
763
745
|
for attempt in range(1, SIO_MAX_RETRIES + 1):
|
|
764
746
|
try:
|
|
765
747
|
async with get_sio_conn(restore_args) as sio:
|
|
@@ -792,8 +774,9 @@ async def procedure():
|
|
|
792
774
|
if attempt == SIO_MAX_RETRIES:
|
|
793
775
|
raise
|
|
794
776
|
|
|
777
|
+
logger.info("retrying...")
|
|
778
|
+
|
|
795
779
|
# gracefully terminate input pipeline
|
|
796
|
-
logger.debug("terminating rbd import proc")
|
|
797
780
|
rbd_import_proc.terminate()
|
|
798
781
|
|
|
799
782
|
try:
|
|
@@ -808,12 +791,8 @@ async def procedure():
|
|
|
808
791
|
await rbd_import_proc.wait()
|
|
809
792
|
|
|
810
793
|
# destroy the rbd image
|
|
811
|
-
logger.info("Retrying...")
|
|
812
794
|
await asyncio.sleep(10)
|
|
813
795
|
|
|
814
|
-
logger.debug(
|
|
815
|
-
f"cleanup rbd image {pool}/{new_csi_image_name}"
|
|
816
|
-
)
|
|
817
796
|
cleanup_proc = await asyncio.create_subprocess_exec(
|
|
818
797
|
"rbd",
|
|
819
798
|
"rm",
|
|
@@ -821,7 +800,6 @@ async def procedure():
|
|
|
821
800
|
)
|
|
822
801
|
|
|
823
802
|
await cleanup_proc.wait()
|
|
824
|
-
|
|
825
803
|
logger.debug(f"cleanup retcode {cleanup_proc.returncode}")
|
|
826
804
|
|
|
827
805
|
# rerun init commands
|
|
@@ -836,7 +814,6 @@ async def procedure():
|
|
|
836
814
|
|
|
837
815
|
else:
|
|
838
816
|
async with get_direct_conn(restore_args) as (reader, writer):
|
|
839
|
-
|
|
840
817
|
writer.write(struct.pack("B", Command.REQUEST_ARCHIVE.value))
|
|
841
818
|
await writer.drain()
|
|
842
819
|
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/funcs.py
RENAMED
|
@@ -260,7 +260,7 @@ async def send_export(send_command, semaphore):
|
|
|
260
260
|
|
|
261
261
|
await proc.wait()
|
|
262
262
|
|
|
263
|
-
await net.
|
|
263
|
+
await net.archive(backup_addr, request_dict, async_chunk_generator)
|
|
264
264
|
|
|
265
265
|
|
|
266
266
|
async def send_backups(namespace_volume_meta, timestamp, backup_addr):
|
|
@@ -512,7 +512,7 @@ async def zfs_snap_and_send(
|
|
|
512
512
|
|
|
513
513
|
logger.info("dd exit code %s", proc.exit_status)
|
|
514
514
|
|
|
515
|
-
await net.
|
|
515
|
+
await net.archive(
|
|
516
516
|
backup_addr, request_dict, chunk_generator, compress=False
|
|
517
517
|
)
|
|
518
518
|
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/net.py
RENAMED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
import inspect
|
|
2
3
|
import logging
|
|
3
4
|
import os
|
|
4
5
|
import pickle
|
|
@@ -28,7 +29,6 @@ async def archive_init(reader, writer, request_dict):
|
|
|
28
29
|
# intialize archive command
|
|
29
30
|
writer.write(struct.pack("B", Command.ARCHIVE.value))
|
|
30
31
|
await writer.drain()
|
|
31
|
-
logger.debug("send command")
|
|
32
32
|
|
|
33
33
|
# send the archive request dict
|
|
34
34
|
req_dict_pickled = pickle.dumps(request_dict)
|
|
@@ -36,7 +36,6 @@ async def archive_init(reader, writer, request_dict):
|
|
|
36
36
|
await writer.drain()
|
|
37
37
|
writer.write(req_dict_pickled)
|
|
38
38
|
await writer.drain()
|
|
39
|
-
logger.debug("send dict")
|
|
40
39
|
|
|
41
40
|
# wait for go signal, server needs to aquire write lock
|
|
42
41
|
# we dont
|
|
@@ -51,13 +50,12 @@ async def archive_init(reader, writer, request_dict):
|
|
|
51
50
|
logger.error("recieved incorrect go signal")
|
|
52
51
|
raise Exception("Incorrect go signal!")
|
|
53
52
|
else:
|
|
53
|
+
logger.debug("received go")
|
|
54
54
|
break
|
|
55
55
|
|
|
56
|
-
logger.debug("received go")
|
|
57
|
-
|
|
58
56
|
|
|
57
|
+
# generic send and ack function
|
|
59
58
|
async def send_cchunk(writer, reader, compressed_chunk):
|
|
60
|
-
# compress the chunk
|
|
61
59
|
if compressed_chunk: # only send if something actually got compressed
|
|
62
60
|
# send size + chunk
|
|
63
61
|
writer.write(struct.pack("!I", len(compressed_chunk)))
|
|
@@ -76,9 +74,7 @@ async def get_sio_mc_client(backup_addr):
|
|
|
76
74
|
"Tried to initialize multicloud proxy without providing MC_EXT_TOKEN env var!"
|
|
77
75
|
)
|
|
78
76
|
|
|
79
|
-
|
|
80
|
-
sio = socketio.AsyncClient() # (logger=log_debug, engineio_logger=log_debug)
|
|
81
|
-
|
|
77
|
+
sio = socketio.AsyncClient()
|
|
82
78
|
await sio.connect(
|
|
83
79
|
backup_addr,
|
|
84
80
|
auth={
|
|
@@ -87,31 +83,39 @@ async def get_sio_mc_client(backup_addr):
|
|
|
87
83
|
},
|
|
88
84
|
transports=["websocket"],
|
|
89
85
|
)
|
|
86
|
+
|
|
90
87
|
logger.debug(f"Connected sio client to {backup_addr}")
|
|
88
|
+
|
|
91
89
|
return sio
|
|
92
90
|
|
|
93
91
|
|
|
92
|
+
# sio framework takes care of acking messages
|
|
94
93
|
async def sio_send_cchunk(sio, compressed_chunk):
|
|
95
94
|
if compressed_chunk:
|
|
96
95
|
await sio.call("backup_chunk", compressed_chunk)
|
|
97
96
|
|
|
98
97
|
|
|
99
98
|
async def wait_archive_init(sio, request_dict):
|
|
99
|
+
# init archive request, this doesn't necessarily immediatly
|
|
100
|
+
# lock and ready the server for writing
|
|
100
101
|
initial = await sio.call(
|
|
101
102
|
"archive_init",
|
|
102
103
|
request_dict,
|
|
103
104
|
timeout=30,
|
|
104
105
|
)
|
|
106
|
+
logger.debug("received init response: %s", initial)
|
|
107
|
+
|
|
105
108
|
if initial["status"] == "ERR":
|
|
106
109
|
raise RuntimeError(initial["error"])
|
|
107
110
|
|
|
108
111
|
if initial["status"] == "ACQUIRED":
|
|
109
|
-
return
|
|
112
|
+
return # server acquired lock for backup repo
|
|
110
113
|
|
|
111
|
-
|
|
112
|
-
|
|
114
|
+
# ==> status WAIT
|
|
115
|
+
logger.info("waiting for lock...")
|
|
113
116
|
while True:
|
|
114
117
|
wait_call = await sio.call("wait_archive", timeout=30)
|
|
118
|
+
logger.debug("wait archive response: %s", wait_call)
|
|
115
119
|
|
|
116
120
|
if wait_call["status"] == "ERR":
|
|
117
121
|
raise RuntimeError(initial["error"])
|
|
@@ -119,24 +123,21 @@ async def wait_archive_init(sio, request_dict):
|
|
|
119
123
|
if wait_call["status"] == "ACQUIRED":
|
|
120
124
|
return
|
|
121
125
|
|
|
122
|
-
# continue wait
|
|
123
|
-
logger.info("continueing waiting")
|
|
124
|
-
|
|
125
126
|
|
|
126
127
|
# compress parameter exists for chunk generators that already do the compression
|
|
127
128
|
# the receiving side ALWAYS expects a compressed stream
|
|
128
|
-
async def
|
|
129
|
-
logger.info(request_dict)
|
|
129
|
+
async def archive(backup_addr, request_dict, chunk_generator, compress=True):
|
|
130
|
+
logger.info("sending archive request: %s", request_dict)
|
|
131
|
+
logger.debug("generator async: %s", inspect.isasyncgenfunction(chunk_generator))
|
|
130
132
|
|
|
133
|
+
# we assume that we are sending to a multicloud gateway if the addr starts with https://
|
|
134
|
+
# direct connects via tcp are if the backup_addr is a hostname / ip address
|
|
131
135
|
if backup_addr.startswith("https://"):
|
|
132
136
|
logger.info(f"sending archive to mc gateway {backup_addr}")
|
|
133
137
|
|
|
134
|
-
# retry is currently only implemented for async generators
|
|
135
|
-
# which are the primary generator type for large image backups
|
|
136
138
|
for attempt in range(1, SIO_MAX_RETRIES + 1):
|
|
137
139
|
sio = None
|
|
138
140
|
try:
|
|
139
|
-
|
|
140
141
|
# connection to mc gw
|
|
141
142
|
sio = await get_sio_mc_client(backup_addr)
|
|
142
143
|
|
|
@@ -148,18 +149,27 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
|
|
|
148
149
|
threads=6,
|
|
149
150
|
).compressobj()
|
|
150
151
|
|
|
151
|
-
|
|
152
|
-
|
|
152
|
+
if inspect.isasyncgenfunction(chunk_generator):
|
|
153
|
+
async for chunk in chunk_generator():
|
|
154
|
+
await sio_send_cchunk(sio, compressor.compress(chunk))
|
|
155
|
+
else:
|
|
156
|
+
for chunk in chunk_generator():
|
|
157
|
+
await sio_send_cchunk(sio, compressor.compress(chunk))
|
|
153
158
|
|
|
154
159
|
await sio_send_cchunk(sio, compressor.flush())
|
|
155
160
|
|
|
156
161
|
else:
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
162
|
+
if inspect.isasyncgenfunction(chunk_generator):
|
|
163
|
+
async for chunk in chunk_generator():
|
|
164
|
+
await sio.call("backup_chunk", chunk)
|
|
165
|
+
else:
|
|
166
|
+
for chunk in chunk_generator():
|
|
167
|
+
await sio.call("backup_chunk", chunk)
|
|
168
|
+
|
|
169
|
+
# signal the server that we are done
|
|
160
170
|
await sio.call("backup_eof")
|
|
161
171
|
|
|
162
|
-
break # finished successfully
|
|
172
|
+
break # finished successfully, break try loop
|
|
163
173
|
|
|
164
174
|
except socketio.exceptions.TimeoutError:
|
|
165
175
|
logger.warn(f"Error on attempt {attempt}")
|
|
@@ -170,10 +180,12 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
|
|
|
170
180
|
await asyncio.sleep(10)
|
|
171
181
|
|
|
172
182
|
finally:
|
|
183
|
+
# always close the sio connection
|
|
173
184
|
if sio:
|
|
174
185
|
await sio.disconnect()
|
|
175
186
|
|
|
176
187
|
else:
|
|
188
|
+
# direct connection to backup server
|
|
177
189
|
reader, writer = await asyncio.open_connection(
|
|
178
190
|
backup_addr, 8085, ssl=get_strict_client_ssl_ctx()
|
|
179
191
|
)
|
|
@@ -184,65 +196,30 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
|
|
|
184
196
|
# compressor = zlib.compressobj(level=1)
|
|
185
197
|
if compress:
|
|
186
198
|
compressor = zstd.ZstdCompressor(level=1, threads=6).compressobj()
|
|
187
|
-
|
|
188
|
-
|
|
199
|
+
if inspect.isasyncgenfunction(chunk_generator):
|
|
200
|
+
async for chunk in chunk_generator():
|
|
201
|
+
await send_cchunk(writer, reader, compressor.compress(chunk))
|
|
202
|
+
else:
|
|
203
|
+
for chunk in chunk_generator():
|
|
204
|
+
await send_cchunk(writer, reader, compressor.compress(chunk))
|
|
205
|
+
|
|
189
206
|
# send rest in compressor, compress doesnt always return a byte array, see bdd.py doc
|
|
190
207
|
# send size first again
|
|
191
208
|
await send_cchunk(writer, reader, compressor.flush())
|
|
192
209
|
|
|
193
210
|
else:
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
writer.close()
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
async def archive(backup_addr, request_dict, chunk_generator):
|
|
210
|
-
logger.info(request_dict)
|
|
211
|
-
if backup_addr.startswith("https://"):
|
|
212
|
-
sio = await get_sio_mc_client(backup_addr)
|
|
213
|
-
|
|
214
|
-
await wait_archive_init(sio, request_dict)
|
|
215
|
-
|
|
216
|
-
compressor = zstd.ZstdCompressor(
|
|
217
|
-
level=1,
|
|
218
|
-
threads=6,
|
|
219
|
-
).compressobj()
|
|
220
|
-
|
|
221
|
-
for chunk in chunk_generator():
|
|
222
|
-
await sio_send_cchunk(sio, compressor.compress(chunk))
|
|
223
|
-
|
|
224
|
-
await sio_send_cchunk(sio, compressor.flush())
|
|
225
|
-
|
|
226
|
-
await sio.call("backup_eof")
|
|
227
|
-
|
|
228
|
-
await sio.disconnect()
|
|
229
|
-
|
|
230
|
-
else:
|
|
231
|
-
reader, writer = await asyncio.open_connection(
|
|
232
|
-
backup_addr, 8085, ssl=get_strict_client_ssl_ctx()
|
|
233
|
-
)
|
|
234
|
-
|
|
235
|
-
await archive_init(reader, writer, request_dict)
|
|
236
|
-
|
|
237
|
-
# initialize the synchronous generator and start reading chunks, compress and send
|
|
238
|
-
# compressor = zlib.compressobj(level=1)
|
|
239
|
-
compressor = zstd.ZstdCompressor(level=1, threads=6).compressobj()
|
|
240
|
-
for chunk in chunk_generator():
|
|
241
|
-
await send_cchunk(writer, reader, compressor.compress(chunk))
|
|
242
|
-
|
|
243
|
-
# send rest in compressor, compress doesnt always return a byte array, see bdd.py doc
|
|
244
|
-
# send size first again
|
|
245
|
-
await send_cchunk(writer, reader, compressor.flush())
|
|
211
|
+
if inspect.isasyncgenfunction(chunk_generator):
|
|
212
|
+
async for chunk in chunk_generator():
|
|
213
|
+
writer.write(struct.pack("!I", len(chunk)))
|
|
214
|
+
await writer.drain()
|
|
215
|
+
writer.write(chunk)
|
|
216
|
+
await writer.drain()
|
|
217
|
+
else:
|
|
218
|
+
for chunk in chunk_generator():
|
|
219
|
+
writer.write(struct.pack("!I", len(chunk)))
|
|
220
|
+
await writer.drain()
|
|
221
|
+
writer.write(chunk)
|
|
222
|
+
await writer.drain()
|
|
246
223
|
|
|
247
224
|
# send eof to server, signal that we are done
|
|
248
225
|
logger.debug("sending eof")
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "3.5.14"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/daemon/funcs.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/ext_zfs.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/git.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/main.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/nextcloud.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.14 → py_pve_cloud_backup-3.5.16}/src/pve_cloud_backup/fetcher/patroni.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|