py-pve-cloud-backup 3.5.9__tar.gz → 3.5.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {py_pve_cloud_backup-3.5.9/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.11}/PKG-INFO +2 -2
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/requirements.txt +1 -1
- py_pve_cloud_backup-3.5.11/src/pve_cloud_backup/_version.py +1 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/bdd.py +52 -50
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/brctl.py +11 -15
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/restore.py +265 -139
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11/src/py_pve_cloud_backup.egg-info}/PKG-INFO +2 -2
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/requires.txt +1 -1
- py_pve_cloud_backup-3.5.9/src/pve_cloud_backup/_version.py +0 -1
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/LICENSE.md +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/README.md +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/pyproject.toml +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/setup.cfg +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/funcs.py +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/ext_zfs.py +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/funcs.py +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/git.py +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/main.py +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/net.py +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/entry_points.txt +0 -0
- {py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
{py_pve_cloud_backup-3.5.9/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.11}/PKG-INFO
RENAMED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: py-pve-cloud-backup
|
|
3
|
-
Version: 3.5.
|
|
3
|
+
Version: 3.5.11
|
|
4
4
|
Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
|
|
5
5
|
License-Expression: GPL-3.0-or-later
|
|
6
6
|
License-File: LICENSE.md
|
|
7
|
-
Requires-Dist: py-pve-cloud<3.5.0,>=3.4.
|
|
7
|
+
Requires-Dist: py-pve-cloud<3.5.0,>=3.4.3
|
|
8
8
|
Requires-Dist: kubernetes==34.1.0
|
|
9
9
|
Requires-Dist: requests==2.32.5
|
|
10
10
|
Requires-Dist: zstandard==0.25.0
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "3.5.11"
|
|
@@ -67,6 +67,10 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
67
67
|
|
|
68
68
|
# send ping pong while waiting on lock
|
|
69
69
|
# maybe this is also needed in other rpc call types
|
|
70
|
+
# todo: currently this is the most likely point where a disconnect might happen
|
|
71
|
+
# this connection here is by far the longest running. It would however be nice that
|
|
72
|
+
# the entire app has retries for connects and can handle connection outages well.
|
|
73
|
+
# implement something like toxiproxy in e2e testing and make resilient.
|
|
70
74
|
lock = await get_lock(backup_dir)
|
|
71
75
|
# store ref to kill in case of failure => this will stop the archive from being committet
|
|
72
76
|
borg_proc = None
|
|
@@ -106,12 +110,18 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
106
110
|
)
|
|
107
111
|
|
|
108
112
|
# read compressed chunks
|
|
113
|
+
dbg_chunk_count = 0
|
|
114
|
+
|
|
109
115
|
while True:
|
|
110
116
|
# client first always sends chunk size
|
|
111
117
|
chunk_size = struct.unpack("!I", (await reader.readexactly(4)))[
|
|
112
118
|
0
|
|
113
119
|
]
|
|
114
|
-
|
|
120
|
+
|
|
121
|
+
# log chunk size on every 100th chunk in dbg log level
|
|
122
|
+
dbg_chunk_count += 1
|
|
123
|
+
if dbg_chunk_count % 100 == 0:
|
|
124
|
+
logger.debug(f"received chunk size {chunk_size}")
|
|
115
125
|
|
|
116
126
|
if chunk_size == 0:
|
|
117
127
|
logger.debug("received chunk size 0, finished")
|
|
@@ -149,6 +159,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
149
159
|
)
|
|
150
160
|
|
|
151
161
|
if borg_proc:
|
|
162
|
+
logger.info("terminating borg process")
|
|
152
163
|
borg_proc.terminate()
|
|
153
164
|
|
|
154
165
|
try:
|
|
@@ -160,7 +171,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
160
171
|
borg_proc.kill()
|
|
161
172
|
await borg_proc.wait()
|
|
162
173
|
|
|
163
|
-
|
|
174
|
+
logger.debug(f"cleaning up borg repo {borg_archive}")
|
|
164
175
|
await asyncio.create_subprocess_exec(
|
|
165
176
|
"borg",
|
|
166
177
|
"delete",
|
|
@@ -237,13 +248,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
237
248
|
async with lock:
|
|
238
249
|
secret_db = TinyDB(db_path)
|
|
239
250
|
|
|
240
|
-
while True:
|
|
241
|
-
# listen for secret requests
|
|
242
251
|
stack = (await reader.readline()).decode().rstrip("\n")
|
|
243
252
|
|
|
244
|
-
if stack == "##BRCTL-DONE":
|
|
245
|
-
break # done signal
|
|
246
|
-
|
|
247
253
|
Meta = Query()
|
|
248
254
|
ns_secrets = secret_db.get(
|
|
249
255
|
(Meta.timestamp == timestamp) & (Meta.stack == stack)
|
|
@@ -256,7 +262,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
256
262
|
writer.write(meta_pickled)
|
|
257
263
|
await writer.drain()
|
|
258
264
|
|
|
259
|
-
case Command.
|
|
265
|
+
case Command.INIT_RESTORE_PROCEDURE:
|
|
260
266
|
timestamp = (await reader.readline()).decode().rstrip("\n")
|
|
261
267
|
logger.info(timestamp)
|
|
262
268
|
|
|
@@ -298,53 +304,49 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
298
304
|
writer.write(meta_pickled)
|
|
299
305
|
await writer.drain()
|
|
300
306
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
307
|
+
logger.info(
|
|
308
|
+
"send initial config / secrets - waiting for archive requests"
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
case Command.REQUEST_ARCHIVE:
|
|
304
312
|
# next the client requests the archives which we extract here and pipe via a stream
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
break # done signal
|
|
312
|
-
|
|
313
|
-
request_artifact = (await reader.readline()).decode().rstrip("\n")
|
|
314
|
-
logger.info(request_artifact)
|
|
315
|
-
|
|
316
|
-
backup_dir = f"{get_backup_base_dir()}/{request_archive}"
|
|
317
|
-
lock = await get_lock(backup_dir)
|
|
318
|
-
async with lock:
|
|
319
|
-
logger.info(
|
|
320
|
-
f"running borg extract on {backup_dir}::{request_artifact}"
|
|
321
|
-
)
|
|
322
|
-
proc = await asyncio.create_subprocess_exec(
|
|
323
|
-
"borg",
|
|
324
|
-
"extract",
|
|
325
|
-
"--sparse",
|
|
326
|
-
"--stdout",
|
|
327
|
-
f"{backup_dir}::{request_artifact}",
|
|
328
|
-
stdout=asyncio.subprocess.PIPE,
|
|
329
|
-
)
|
|
313
|
+
# open the extract process and send the stream the output
|
|
314
|
+
request_archive = (await reader.readline()).decode().rstrip("\n")
|
|
315
|
+
logger.info(request_archive)
|
|
316
|
+
|
|
317
|
+
request_artifact = (await reader.readline()).decode().rstrip("\n")
|
|
318
|
+
logger.info(request_artifact)
|
|
330
319
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
320
|
+
backup_dir = f"{get_backup_base_dir()}/{request_archive}"
|
|
321
|
+
lock = await get_lock(backup_dir)
|
|
322
|
+
async with lock:
|
|
323
|
+
logger.info(
|
|
324
|
+
f"running borg extract on {backup_dir}::{request_artifact}"
|
|
325
|
+
)
|
|
326
|
+
proc = await asyncio.create_subprocess_exec(
|
|
327
|
+
"borg",
|
|
328
|
+
"extract",
|
|
329
|
+
"--sparse",
|
|
330
|
+
"--stdout",
|
|
331
|
+
f"{backup_dir}::{request_artifact}",
|
|
332
|
+
stdout=asyncio.subprocess.PIPE,
|
|
333
|
+
)
|
|
338
334
|
|
|
339
|
-
|
|
340
|
-
|
|
335
|
+
compressor = zstd.ZstdCompressor(level=1, threads=6).compressobj()
|
|
336
|
+
while True:
|
|
337
|
+
chunk = await proc.stdout.read(4 * 1024 * 1024 * 10) # 4MB
|
|
338
|
+
if not chunk:
|
|
339
|
+
break
|
|
341
340
|
|
|
342
|
-
#
|
|
343
|
-
await send_cchunk(writer, compressor.
|
|
341
|
+
# compress and send the chunk
|
|
342
|
+
await send_cchunk(writer, compressor.compress(chunk))
|
|
344
343
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
344
|
+
# send the rest in the compressor
|
|
345
|
+
await send_cchunk(writer, compressor.flush())
|
|
346
|
+
|
|
347
|
+
logger.info("sending eof")
|
|
348
|
+
writer.write(struct.pack("!I", 0))
|
|
349
|
+
await writer.drain()
|
|
348
350
|
|
|
349
351
|
except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError) as e:
|
|
350
352
|
logger.warning("Client disconnected: %s", e, exc_info=True)
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/brctl.py
RENAMED
|
@@ -19,7 +19,7 @@ from kubernetes.client import (ApiException, V1ConfigMapVolumeSource,
|
|
|
19
19
|
from kubernetes.config.kube_config import KubeConfigLoader
|
|
20
20
|
from pve_cloud.cli.pvclu import (get_cloud_domain, get_ssh_master_kubeconfig,
|
|
21
21
|
get_ssh_remote_master_kubeconfig)
|
|
22
|
-
from pve_cloud.cli.pxrpc import launch_pxrpc
|
|
22
|
+
from pve_cloud.cli.pxrpc import get_simple_pxrpc, launch_pxrpc
|
|
23
23
|
from pve_cloud.lib.backup_rpc import Command
|
|
24
24
|
from pve_cloud.lib.inventory import (get_cloud_domain, get_cluster_vars,
|
|
25
25
|
get_online_pve_host,
|
|
@@ -55,10 +55,10 @@ async def list_backup_details_remote(args):
|
|
|
55
55
|
metas = None
|
|
56
56
|
stack_meta = None
|
|
57
57
|
|
|
58
|
-
with
|
|
58
|
+
async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
|
|
59
59
|
if args.use_mc_gw:
|
|
60
60
|
# we will create a socket connection to the multi cloud gateway for that we need to fetch secrets
|
|
61
|
-
ext_mc_raw = pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
|
|
61
|
+
ext_mc_raw = await pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
|
|
62
62
|
if not ext_mc_raw:
|
|
63
63
|
raise RuntimeError(
|
|
64
64
|
f"No multi cloud services could be discovered for {pve_cluster} - {cloud_domain}!"
|
|
@@ -85,7 +85,7 @@ async def list_backup_details_remote(args):
|
|
|
85
85
|
# we will connect directly to the backup server
|
|
86
86
|
# todo: here we can also pass the correct tls config
|
|
87
87
|
|
|
88
|
-
tls_disc_raw = pxrpc.get_cloud_secret(
|
|
88
|
+
tls_disc_raw = await pxrpc.get_cloud_secret(
|
|
89
89
|
cloud_domain, f"{bdd_stack_name}-bdd-tls-discovery"
|
|
90
90
|
)
|
|
91
91
|
if not tls_disc_raw:
|
|
@@ -128,11 +128,6 @@ async def list_backup_details_remote(args):
|
|
|
128
128
|
dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
|
|
129
129
|
stack_meta = pickle.loads((await reader.readexactly(dict_size)))
|
|
130
130
|
|
|
131
|
-
# send a terminator
|
|
132
|
-
# todo: probably not needed anymore
|
|
133
|
-
writer.write("##BRCTL-DONE\n".encode())
|
|
134
|
-
await writer.drain()
|
|
135
|
-
|
|
136
131
|
namespace_secret_dict = pickle.loads(
|
|
137
132
|
base64.b64decode(stack_meta["namespace_secret_dict_b64"])
|
|
138
133
|
)
|
|
@@ -213,9 +208,9 @@ async def list_backups_remote(args):
|
|
|
213
208
|
logger.info(f"connecting to {pve_host} via {jump_host}")
|
|
214
209
|
archives = None
|
|
215
210
|
|
|
216
|
-
with
|
|
211
|
+
async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
|
|
217
212
|
if args.use_mc_gw:
|
|
218
|
-
ext_mc_raw = pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
|
|
213
|
+
ext_mc_raw = await pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
|
|
219
214
|
if not ext_mc_raw:
|
|
220
215
|
raise RuntimeError(
|
|
221
216
|
f"No multi cloud services could be discovered for {pve_cluster} - {cloud_domain}!"
|
|
@@ -239,7 +234,7 @@ async def list_backups_remote(args):
|
|
|
239
234
|
archives = result["archives"]
|
|
240
235
|
|
|
241
236
|
else:
|
|
242
|
-
tls_disc_raw = pxrpc.get_cloud_secret(
|
|
237
|
+
tls_disc_raw = await pxrpc.get_cloud_secret(
|
|
243
238
|
cloud_domain, f"{bdd_stack_name}-bdd-tls-discovery"
|
|
244
239
|
)
|
|
245
240
|
if not tls_disc_raw:
|
|
@@ -299,6 +294,7 @@ async def launch_restore_job(args):
|
|
|
299
294
|
|
|
300
295
|
kubeconfig_dict = None
|
|
301
296
|
|
|
297
|
+
# todo: maybe this can be simplyfied relying more heavily on pxrpc.get_simple_pxrpc concepts
|
|
302
298
|
if raw_pxc_inv["plugin"] == "pxc.cloud.kubespray_inv":
|
|
303
299
|
kubespray_inv = raw_pxc_inv
|
|
304
300
|
|
|
@@ -368,9 +364,9 @@ async def launch_restore_job(args):
|
|
|
368
364
|
# acessing the backup server
|
|
369
365
|
logger.info(f"connecting to {pve_host} via {jump_host}")
|
|
370
366
|
|
|
371
|
-
with
|
|
367
|
+
async with get_simple_pxrpc(pve_host, jump_host) as pxrpc:
|
|
372
368
|
if args.use_mc_gw:
|
|
373
|
-
ext_mc_raw = pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
|
|
369
|
+
ext_mc_raw = await pxrpc.get_cloud_secret(cloud_domain, "external-mc-token")
|
|
374
370
|
if not ext_mc_raw:
|
|
375
371
|
raise RuntimeError(
|
|
376
372
|
f"No multi cloud services could be discovered for {pve_cluster} - {cloud_domain}!"
|
|
@@ -384,7 +380,7 @@ async def launch_restore_job(args):
|
|
|
384
380
|
serializable_args["bdd_stack_name"] = bdd_stack_name
|
|
385
381
|
|
|
386
382
|
else:
|
|
387
|
-
tls_disc_raw = pxrpc.get_cloud_secret(
|
|
383
|
+
tls_disc_raw = await pxrpc.get_cloud_secret(
|
|
388
384
|
cloud_domain, f"{bdd_stack_name}-bdd-tls-discovery"
|
|
389
385
|
)
|
|
390
386
|
if not tls_disc_raw:
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/restore.py
RENAMED
|
@@ -9,6 +9,7 @@ import ssl
|
|
|
9
9
|
import struct
|
|
10
10
|
import subprocess
|
|
11
11
|
import uuid
|
|
12
|
+
from contextlib import asynccontextmanager
|
|
12
13
|
from pprint import pformat
|
|
13
14
|
|
|
14
15
|
import asyncssh
|
|
@@ -27,6 +28,8 @@ log_level = getattr(logging, log_level_str, logging.INFO)
|
|
|
27
28
|
logging.basicConfig(level=log_level)
|
|
28
29
|
logger = logging.getLogger("pxc-restore")
|
|
29
30
|
|
|
31
|
+
SIO_MAX_RETRIES = 3
|
|
32
|
+
|
|
30
33
|
|
|
31
34
|
# these functions are necessary to convert python k8s naming to camel case
|
|
32
35
|
def to_camel_case(snake_str):
|
|
@@ -48,67 +51,86 @@ def convert_keys_to_camel_case(obj):
|
|
|
48
51
|
return obj
|
|
49
52
|
|
|
50
53
|
|
|
54
|
+
@asynccontextmanager
|
|
55
|
+
async def get_direct_conn(restore_args):
|
|
56
|
+
ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
|
57
|
+
ssl_ctx.check_hostname = False
|
|
58
|
+
ssl_ctx.verify_mode = ssl.CERT_NONE
|
|
59
|
+
|
|
60
|
+
reader, writer = await asyncio.open_connection(
|
|
61
|
+
restore_args["bdd_host"], 8085, ssl=ssl_ctx
|
|
62
|
+
)
|
|
63
|
+
yield reader, writer
|
|
64
|
+
|
|
65
|
+
writer.close()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@asynccontextmanager
|
|
69
|
+
async def get_sio_conn(restore_args):
|
|
70
|
+
log_debug = os.getenv("LOG_LEVEL") == "DEBUG"
|
|
71
|
+
sio = socketio.AsyncClient(logger=log_debug, engineio_logger=log_debug)
|
|
72
|
+
|
|
73
|
+
await sio.connect(
|
|
74
|
+
f"https://{restore_args['mc_gw_host']}",
|
|
75
|
+
auth={
|
|
76
|
+
"token": restore_args["mc_ext_token"],
|
|
77
|
+
"bdd_stack_name": restore_args["bdd_stack_name"],
|
|
78
|
+
},
|
|
79
|
+
transports=["websocket"],
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
yield sio
|
|
83
|
+
|
|
84
|
+
await sio.disconnect()
|
|
85
|
+
|
|
86
|
+
|
|
51
87
|
async def init_procedure_bdd(restore_args):
|
|
52
88
|
if restore_args["use_mc_gw"]:
|
|
53
|
-
log_debug = os.getenv("LOG_LEVEL") == "DEBUG"
|
|
54
|
-
sio = socketio.AsyncClient(logger=log_debug, engineio_logger=log_debug)
|
|
55
|
-
|
|
56
|
-
await sio.connect(
|
|
57
|
-
f"https://{restore_args['mc_gw_host']}",
|
|
58
|
-
auth={
|
|
59
|
-
"token": restore_args["mc_ext_token"],
|
|
60
|
-
"bdd_stack_name": restore_args["bdd_stack_name"],
|
|
61
|
-
},
|
|
62
|
-
transports=["websocket"],
|
|
63
|
-
)
|
|
64
|
-
result_pickled = await sio.call(
|
|
65
|
-
"init_restore", restore_args["timestamp"], timeout=30
|
|
66
|
-
)
|
|
67
|
-
logger.info("init restore result")
|
|
68
|
-
result = pickle.loads(result_pickled)
|
|
69
|
-
logger.info(result)
|
|
70
89
|
|
|
71
|
-
|
|
90
|
+
async with get_sio_conn(restore_args) as sio:
|
|
91
|
+
result_pickled = await sio.call(
|
|
92
|
+
"init_restore", restore_args["timestamp"], timeout=30
|
|
93
|
+
)
|
|
94
|
+
logger.info("init restore result")
|
|
95
|
+
result = pickle.loads(result_pickled)
|
|
96
|
+
logger.info(result)
|
|
97
|
+
|
|
98
|
+
return result["metas_grouped_by_ns"], result["namespace_secret_dict"]
|
|
72
99
|
|
|
73
100
|
else:
|
|
74
|
-
# connect to the backup server and start the restore procedure
|
|
75
|
-
# we simply trust the host here as the ca is only available from
|
|
76
|
-
# postgres secrets
|
|
77
|
-
ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
|
78
|
-
ssl_ctx.check_hostname = False
|
|
79
|
-
ssl_ctx.verify_mode = ssl.CERT_NONE
|
|
80
|
-
|
|
81
|
-
reader, writer = await asyncio.open_connection(
|
|
82
|
-
restore_args["bdd_host"], 8085, ssl=ssl_ctx
|
|
83
|
-
)
|
|
84
|
-
writer.write(struct.pack("B", Command.RESTORE_PROCEDURE.value))
|
|
85
|
-
await writer.drain()
|
|
86
101
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
102
|
+
async with get_direct_conn(restore_args) as (reader, writer):
|
|
103
|
+
# connect to the backup server and start the restore procedure
|
|
104
|
+
# we simply trust the host here as the ca is only available from
|
|
105
|
+
# postgres secrets
|
|
106
|
+
writer.write(struct.pack("B", Command.INIT_RESTORE_PROCEDURE.value))
|
|
107
|
+
await writer.drain()
|
|
90
108
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
109
|
+
# first we send the timestamp and receive our meta information for the restore
|
|
110
|
+
writer.write((restore_args["timestamp"] + "\n").encode())
|
|
111
|
+
await writer.drain()
|
|
94
112
|
|
|
95
|
-
|
|
113
|
+
# read the archives
|
|
114
|
+
dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
|
|
115
|
+
metas = pickle.loads((await reader.readexactly(dict_size)))
|
|
96
116
|
|
|
97
|
-
|
|
98
|
-
if meta["namespace"] not in metas_grouped_by_ns:
|
|
99
|
-
metas_grouped_by_ns[meta["namespace"]] = []
|
|
117
|
+
metas_grouped_by_ns = {}
|
|
100
118
|
|
|
101
|
-
|
|
119
|
+
for meta in metas:
|
|
120
|
+
if meta["namespace"] not in metas_grouped_by_ns:
|
|
121
|
+
metas_grouped_by_ns[meta["namespace"]] = []
|
|
102
122
|
|
|
103
|
-
|
|
104
|
-
dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
|
|
105
|
-
stack_meta = pickle.loads((await reader.readexactly(dict_size)))
|
|
123
|
+
metas_grouped_by_ns[meta["namespace"]].append(meta)
|
|
106
124
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
125
|
+
# read the the meta information
|
|
126
|
+
dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
|
|
127
|
+
stack_meta = pickle.loads((await reader.readexactly(dict_size)))
|
|
128
|
+
|
|
129
|
+
namespace_secret_dict = pickle.loads(
|
|
130
|
+
base64.b64decode(stack_meta["namespace_secret_dict_b64"])
|
|
131
|
+
)
|
|
110
132
|
|
|
111
|
-
|
|
133
|
+
return metas_grouped_by_ns, namespace_secret_dict
|
|
112
134
|
|
|
113
135
|
|
|
114
136
|
def clean_pvc_dict(pvc_dict):
|
|
@@ -175,9 +197,7 @@ async def procedure():
|
|
|
175
197
|
restore_args = json.loads(base64.b64decode(os.getenv("PXC_RESTORE_ARGS")))
|
|
176
198
|
logger.info(restore_args)
|
|
177
199
|
|
|
178
|
-
|
|
179
|
-
restore_args
|
|
180
|
-
)
|
|
200
|
+
metas_grouped_by_ns, namespace_secret_dict = await init_procedure_bdd(restore_args)
|
|
181
201
|
|
|
182
202
|
# now we start the restore procedure
|
|
183
203
|
config.load_incluster_config()
|
|
@@ -466,14 +486,15 @@ async def procedure():
|
|
|
466
486
|
# first we create a zvol of identical size
|
|
467
487
|
# todo: remove -rest stub and replace with new id generation
|
|
468
488
|
# todo: write converter function to create zvol in exact size like k8s did zfs list -t volume -o name,volsize
|
|
469
|
-
|
|
470
|
-
logger.info("Executing create: %s",
|
|
489
|
+
zvol_create_cmd = f"sudo zfs create -s -V {parse_quantity(pvc_dict['spec']['resources']['requests']['storage'])} tank-pv/pvc-{restore_pvc_uuid}"
|
|
490
|
+
logger.info("Executing create: %s", zvol_create_cmd)
|
|
471
491
|
|
|
472
|
-
await ssh.run(
|
|
492
|
+
await ssh.run(zvol_create_cmd, check=True)
|
|
473
493
|
|
|
474
494
|
# then we pipe the compressed stream into dd zstd decompression pipeline
|
|
495
|
+
import_cmd = f"zstd --decompress --stdout | sudo dd of=/dev/zvol/tank-pv/pvc-{restore_pvc_uuid} bs=4M status=none"
|
|
475
496
|
proc = await ssh.create_process(
|
|
476
|
-
|
|
497
|
+
import_cmd,
|
|
477
498
|
encoding=None,
|
|
478
499
|
)
|
|
479
500
|
|
|
@@ -485,45 +506,105 @@ async def procedure():
|
|
|
485
506
|
)
|
|
486
507
|
|
|
487
508
|
if restore_args["use_mc_gw"]:
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
509
|
+
for attempt in range(1, SIO_MAX_RETRIES + 1):
|
|
510
|
+
try:
|
|
511
|
+
async with get_sio_conn(restore_args) as sio:
|
|
512
|
+
await sio.call(
|
|
513
|
+
"init_request",
|
|
514
|
+
{
|
|
515
|
+
"archive": request_archive,
|
|
516
|
+
"artifact": request_artifact,
|
|
517
|
+
},
|
|
518
|
+
timeout=30,
|
|
519
|
+
)
|
|
495
520
|
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
521
|
+
logger.info(
|
|
522
|
+
"send request artifact / archive - requesting chunks"
|
|
523
|
+
)
|
|
499
524
|
|
|
500
|
-
|
|
501
|
-
|
|
525
|
+
while True:
|
|
526
|
+
chunk = await sio.call(
|
|
527
|
+
"request_chunk", timeout=30
|
|
528
|
+
)
|
|
529
|
+
if not chunk:
|
|
530
|
+
break
|
|
502
531
|
|
|
503
|
-
|
|
532
|
+
proc.stdin.write(chunk)
|
|
533
|
+
await proc.stdin.drain()
|
|
504
534
|
|
|
505
|
-
|
|
506
|
-
while True:
|
|
507
|
-
chunk = None
|
|
508
|
-
if restore_args["use_mc_gw"]:
|
|
509
|
-
chunk = await connection.call("request_chunk", timeout=30)
|
|
510
|
-
if not chunk:
|
|
535
|
+
# success
|
|
511
536
|
break
|
|
512
|
-
else:
|
|
513
|
-
reader, writer = connection
|
|
514
537
|
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
if chunk_size == 0:
|
|
520
|
-
logger.debug("received eof")
|
|
521
|
-
break # client sends 0 chunk size at the end to signal that its finished uploading
|
|
538
|
+
except socketio.exceptions.TimeoutError:
|
|
539
|
+
logger.warn(f"Error on attempt {attempt}")
|
|
540
|
+
if attempt == SIO_MAX_RETRIES:
|
|
541
|
+
raise
|
|
522
542
|
|
|
523
|
-
|
|
543
|
+
# gracefully terminate input pipeline
|
|
544
|
+
logger.debug("terminating dd zvol import proc")
|
|
545
|
+
proc.terminate()
|
|
546
|
+
|
|
547
|
+
try:
|
|
548
|
+
await asyncio.wait_for(proc.wait(), timeout=30)
|
|
549
|
+
except asyncio.TimeoutError:
|
|
550
|
+
logger.warning(
|
|
551
|
+
"terminate timed out, force killing import subprocess!"
|
|
552
|
+
)
|
|
553
|
+
proc.kill()
|
|
554
|
+
await proc.wait()
|
|
555
|
+
|
|
556
|
+
# destroy the zfs volume
|
|
557
|
+
logger.debug(
|
|
558
|
+
f"cleaning up zfs vol tank-pv/pvc-{restore_pvc_uuid}"
|
|
559
|
+
)
|
|
560
|
+
await ssh.run(
|
|
561
|
+
f"sudo zfs destroy tank-pv/pvc-{restore_pvc_uuid}"
|
|
562
|
+
)
|
|
524
563
|
|
|
525
|
-
|
|
526
|
-
|
|
564
|
+
logger.info("Retrying...")
|
|
565
|
+
await asyncio.sleep(10)
|
|
566
|
+
|
|
567
|
+
# rerun init commands
|
|
568
|
+
# todo: generic retry could make this much learner combining with direct connect
|
|
569
|
+
logger.debug("relaunching zvol create cmd")
|
|
570
|
+
await ssh.run(zvol_create_cmd, check=True)
|
|
571
|
+
proc = await ssh.create_process(
|
|
572
|
+
import_cmd,
|
|
573
|
+
encoding=None,
|
|
574
|
+
)
|
|
575
|
+
|
|
576
|
+
else:
|
|
577
|
+
async with get_direct_conn(restore_args) as (reader, writer):
|
|
578
|
+
|
|
579
|
+
writer.write(
|
|
580
|
+
struct.pack("B", Command.REQUEST_ARCHIVE.value)
|
|
581
|
+
)
|
|
582
|
+
await writer.drain()
|
|
583
|
+
|
|
584
|
+
# bdd server does readline()
|
|
585
|
+
writer.write(request_archive.encode())
|
|
586
|
+
await writer.drain()
|
|
587
|
+
|
|
588
|
+
writer.write(request_artifact.encode())
|
|
589
|
+
await writer.drain()
|
|
590
|
+
|
|
591
|
+
logger.info(
|
|
592
|
+
"send request artifact / archive - requesting chunks"
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
while True:
|
|
596
|
+
# client first always sends chunk size
|
|
597
|
+
chunk_size = struct.unpack(
|
|
598
|
+
"!I", (await reader.readexactly(4))
|
|
599
|
+
)[0]
|
|
600
|
+
if chunk_size == 0:
|
|
601
|
+
logger.debug("received eof")
|
|
602
|
+
break # client sends 0 chunk size at the end to signal that its finished uploading
|
|
603
|
+
|
|
604
|
+
chunk = await reader.readexactly(chunk_size)
|
|
605
|
+
|
|
606
|
+
proc.stdin.write(chunk)
|
|
607
|
+
await proc.stdin.drain()
|
|
527
608
|
|
|
528
609
|
logger.info("done reading closing proc")
|
|
529
610
|
proc.stdin.close()
|
|
@@ -659,22 +740,6 @@ async def procedure():
|
|
|
659
740
|
f"requesting borg archive stream from bdd {request_archive} - {request_artifact} into rbd import {pool}/{new_csi_image_name}"
|
|
660
741
|
)
|
|
661
742
|
|
|
662
|
-
if restore_args["use_mc_gw"]:
|
|
663
|
-
await connection.call(
|
|
664
|
-
"init_request",
|
|
665
|
-
{"archive": request_archive, "artifact": request_artifact},
|
|
666
|
-
timeout=30,
|
|
667
|
-
)
|
|
668
|
-
else:
|
|
669
|
-
reader, writer = connection
|
|
670
|
-
|
|
671
|
-
# bdd server does readline()
|
|
672
|
-
writer.write(request_archive.encode())
|
|
673
|
-
await writer.drain()
|
|
674
|
-
|
|
675
|
-
writer.write(request_artifact.encode())
|
|
676
|
-
await writer.drain()
|
|
677
|
-
|
|
678
743
|
# pipe the resulting stream into rbd import
|
|
679
744
|
rbd_import_proc = await asyncio.create_subprocess_exec(
|
|
680
745
|
"rbd",
|
|
@@ -684,32 +749,108 @@ async def procedure():
|
|
|
684
749
|
stdin=asyncio.subprocess.PIPE,
|
|
685
750
|
)
|
|
686
751
|
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
752
|
+
if restore_args["use_mc_gw"]:
|
|
753
|
+
for attempt in range(1, SIO_MAX_RETRIES + 1):
|
|
754
|
+
try:
|
|
755
|
+
async with get_sio_conn(restore_args) as sio:
|
|
756
|
+
await sio.call(
|
|
757
|
+
"init_request",
|
|
758
|
+
{
|
|
759
|
+
"archive": request_archive,
|
|
760
|
+
"artifact": request_artifact,
|
|
761
|
+
},
|
|
762
|
+
timeout=30,
|
|
763
|
+
)
|
|
764
|
+
# read compressed chunks
|
|
765
|
+
decompressor = zstd.ZstdDecompressor().decompressobj()
|
|
766
|
+
while True:
|
|
767
|
+
chunk = await sio.call("request_chunk", timeout=30)
|
|
768
|
+
if not chunk:
|
|
769
|
+
break
|
|
770
|
+
|
|
771
|
+
# decompress and write
|
|
772
|
+
decompressed_chunk = decompressor.decompress(chunk)
|
|
773
|
+
if decompressed_chunk:
|
|
774
|
+
rbd_import_proc.stdin.write(decompressed_chunk)
|
|
775
|
+
await rbd_import_proc.stdin.drain()
|
|
776
|
+
|
|
777
|
+
# success
|
|
694
778
|
break
|
|
695
|
-
else:
|
|
696
|
-
reader, writer = connection
|
|
697
779
|
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
780
|
+
except socketio.exceptions.TimeoutError:
|
|
781
|
+
logger.warn(f"Error on attempt {attempt}")
|
|
782
|
+
if attempt == SIO_MAX_RETRIES:
|
|
783
|
+
raise
|
|
784
|
+
|
|
785
|
+
# gracefully terminate input pipeline
|
|
786
|
+
logger.debug("terminating rbd import proc")
|
|
787
|
+
rbd_import_proc.terminate()
|
|
788
|
+
|
|
789
|
+
try:
|
|
790
|
+
await asyncio.wait_for(
|
|
791
|
+
rbd_import_proc.wait(), timeout=30
|
|
792
|
+
)
|
|
793
|
+
except asyncio.TimeoutError:
|
|
794
|
+
logger.warning(
|
|
795
|
+
"terminate timed out, force killing import subprocess!"
|
|
796
|
+
)
|
|
797
|
+
rbd_import_proc.kill()
|
|
798
|
+
await rbd_import_proc.wait()
|
|
705
799
|
|
|
706
|
-
|
|
800
|
+
# destroy the rbd image
|
|
801
|
+
logger.debug(
|
|
802
|
+
f"cleanup rbd image {pool}/{new_csi_image_name}"
|
|
803
|
+
)
|
|
804
|
+
await asyncio.create_subprocess_exec(
|
|
805
|
+
"rbd",
|
|
806
|
+
"rm",
|
|
807
|
+
f"{pool}/{new_csi_image_name}",
|
|
808
|
+
)
|
|
707
809
|
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
810
|
+
logger.info("Retrying...")
|
|
811
|
+
await asyncio.sleep(10)
|
|
812
|
+
|
|
813
|
+
# rerun init commands
|
|
814
|
+
# todo: generic retry could make this much learner combining with direct connect
|
|
815
|
+
rbd_import_proc = await asyncio.create_subprocess_exec(
|
|
816
|
+
"rbd",
|
|
817
|
+
"import",
|
|
818
|
+
"-",
|
|
819
|
+
f"{pool}/{new_csi_image_name}",
|
|
820
|
+
stdin=asyncio.subprocess.PIPE,
|
|
821
|
+
)
|
|
822
|
+
|
|
823
|
+
else:
|
|
824
|
+
async with get_direct_conn(restore_args) as (reader, writer):
|
|
825
|
+
|
|
826
|
+
writer.write(struct.pack("B", Command.REQUEST_ARCHIVE.value))
|
|
827
|
+
await writer.drain()
|
|
828
|
+
|
|
829
|
+
# bdd server does readline()
|
|
830
|
+
writer.write(request_archive.encode())
|
|
831
|
+
await writer.drain()
|
|
832
|
+
|
|
833
|
+
writer.write(request_artifact.encode())
|
|
834
|
+
await writer.drain()
|
|
835
|
+
|
|
836
|
+
# read compressed chunks
|
|
837
|
+
decompressor = zstd.ZstdDecompressor().decompressobj()
|
|
838
|
+
while True:
|
|
839
|
+
# client first always sends chunk size
|
|
840
|
+
chunk_size = struct.unpack(
|
|
841
|
+
"!I", (await reader.readexactly(4))
|
|
842
|
+
)[0]
|
|
843
|
+
if chunk_size == 0:
|
|
844
|
+
logger.debug("received eof")
|
|
845
|
+
break # client sends 0 chunk size at the end to signal that its finished uploading
|
|
846
|
+
|
|
847
|
+
chunk = await reader.readexactly(chunk_size)
|
|
848
|
+
|
|
849
|
+
# decompress and write
|
|
850
|
+
decompressed_chunk = decompressor.decompress(chunk)
|
|
851
|
+
if decompressed_chunk:
|
|
852
|
+
rbd_import_proc.stdin.write(decompressed_chunk)
|
|
853
|
+
await rbd_import_proc.stdin.drain()
|
|
713
854
|
|
|
714
855
|
# the decompressor does not always return a decompressed chunk but might retain
|
|
715
856
|
# and return empty. at the end we need to call flush to get everything out
|
|
@@ -835,21 +976,6 @@ async def procedure():
|
|
|
835
976
|
)
|
|
836
977
|
)
|
|
837
978
|
|
|
838
|
-
if restore_args["use_mc_gw"]:
|
|
839
|
-
await connection.call("request_done", timeout=30)
|
|
840
|
-
|
|
841
|
-
await connection.disconnect()
|
|
842
|
-
|
|
843
|
-
else:
|
|
844
|
-
reader, writer = connection
|
|
845
|
-
|
|
846
|
-
# send the done signal to bdd server
|
|
847
|
-
writer.write("##BRCTL-DONE\n".encode())
|
|
848
|
-
await writer.drain()
|
|
849
|
-
|
|
850
|
-
# close the writer here
|
|
851
|
-
writer.close()
|
|
852
|
-
|
|
853
979
|
# scale back up again
|
|
854
980
|
if restore_args["auto_scale"]:
|
|
855
981
|
# auto downscale deployments and statefulsets of namespace
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11/src/py_pve_cloud_backup.egg-info}/PKG-INFO
RENAMED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: py-pve-cloud-backup
|
|
3
|
-
Version: 3.5.
|
|
3
|
+
Version: 3.5.11
|
|
4
4
|
Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
|
|
5
5
|
License-Expression: GPL-3.0-or-later
|
|
6
6
|
License-File: LICENSE.md
|
|
7
|
-
Requires-Dist: py-pve-cloud<3.5.0,>=3.4.
|
|
7
|
+
Requires-Dist: py-pve-cloud<3.5.0,>=3.4.3
|
|
8
8
|
Requires-Dist: kubernetes==34.1.0
|
|
9
9
|
Requires-Dist: requests==2.32.5
|
|
10
10
|
Requires-Dist: zstandard==0.25.0
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "3.5.9"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/daemon/funcs.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/ext_zfs.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/funcs.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/git.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/main.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/net.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/nextcloud.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.9 → py_pve_cloud_backup-3.5.11}/src/pve_cloud_backup/fetcher/patroni.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|