py-pve-cloud-backup 3.5.7__tar.gz → 3.5.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {py_pve_cloud_backup-3.5.7/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.9}/PKG-INFO +1 -1
  2. py_pve_cloud_backup-3.5.9/src/pve_cloud_backup/_version.py +1 -0
  3. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/daemon/bdd.py +49 -15
  4. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/daemon/restore.py +2 -1
  5. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/fetcher/net.py +39 -17
  6. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9/src/py_pve_cloud_backup.egg-info}/PKG-INFO +1 -1
  7. py_pve_cloud_backup-3.5.7/src/pve_cloud_backup/_version.py +0 -1
  8. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/LICENSE.md +0 -0
  9. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/README.md +0 -0
  10. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/pyproject.toml +0 -0
  11. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/requirements.txt +0 -0
  12. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/setup.cfg +0 -0
  13. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/daemon/brctl.py +0 -0
  14. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/daemon/funcs.py +0 -0
  15. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/fetcher/ext_zfs.py +0 -0
  16. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/fetcher/funcs.py +0 -0
  17. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/fetcher/git.py +0 -0
  18. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/fetcher/main.py +0 -0
  19. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
  20. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
  21. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +0 -0
  22. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
  23. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/py_pve_cloud_backup.egg-info/entry_points.txt +0 -0
  24. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/py_pve_cloud_backup.egg-info/requires.txt +0 -0
  25. {py_pve_cloud_backup-3.5.7 → py_pve_cloud_backup-3.5.9}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.7
3
+ Version: 3.5.9
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -0,0 +1 @@
1
+ __version__ = "3.5.9"
@@ -68,19 +68,22 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
68
68
  # send ping pong while waiting on lock
69
69
  # maybe this is also needed in other rpc call types
70
70
  lock = await get_lock(backup_dir)
71
-
72
- while True:
73
- try:
74
- await asyncio.wait_for(lock.acquire(), timeout=5)
75
- logger.info(f"accuired lock {backup_dir}")
76
- break
77
- except asyncio.TimeoutError:
78
- writer.write(b"\x02") # 0x02 byte means continue waiting
79
- await writer.drain()
80
- logger.debug("send keepalive waiting for lock, continueing...")
81
-
71
+ # store ref to kill in case of failure => this will stop the archive from being committet
72
+ borg_proc = None
73
+ borg_archive = f"{backup_dir}::{archive_name}_{timestamp}"
82
74
  try:
83
- borg_archive = f"{backup_dir}::{archive_name}_{timestamp}"
75
+ # lock wait mechanism
76
+ while True:
77
+ try:
78
+ await asyncio.wait_for(lock.acquire(), timeout=5)
79
+ logger.info(f"accuired lock {backup_dir}")
80
+ break
81
+ except asyncio.TimeoutError:
82
+ writer.write(b"\x02") # 0x02 byte means continue waiting
83
+ await writer.drain()
84
+ logger.debug(
85
+ "send keepalive waiting for lock, continueing..."
86
+ )
84
87
 
85
88
  # send continue signal, meaning we have the lock and export can start.
86
89
  writer.write(b"\x01") # signal = 0x01 means "continue"
@@ -132,8 +135,39 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
132
135
  exit_code = await borg_proc.wait()
133
136
 
134
137
  if exit_code != 0:
135
- raise Exception(f"Borg failed with code {exit_code}")
138
+ raise RuntimeError(f"Borg failed with code {exit_code}")
139
+
140
+ except (
141
+ asyncio.IncompleteReadError,
142
+ ConnectionResetError,
143
+ BrokenPipeError,
144
+ ) as e:
145
+ logger.warning(
146
+ "Client error on transmission: %s, gracefully terminating borg...",
147
+ e,
148
+ exc_info=True,
149
+ )
150
+
151
+ if borg_proc:
152
+ borg_proc.terminate()
153
+
154
+ try:
155
+ await asyncio.wait_for(borg_proc.wait(), timeout=30)
156
+ except asyncio.TimeoutError:
157
+ logger.warning(
158
+ "terminate timed out, force killing borg subprocess!"
159
+ )
160
+ borg_proc.kill()
161
+ await borg_proc.wait()
136
162
 
163
+ # cleanup repo
164
+ await asyncio.create_subprocess_exec(
165
+ "borg",
166
+ "delete",
167
+ borg_archive,
168
+ )
169
+ # reraise exception for main close handler
170
+ raise
137
171
  finally:
138
172
  lock.release()
139
173
 
@@ -312,8 +346,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
312
346
  writer.write(struct.pack("!I", 0))
313
347
  await writer.drain()
314
348
 
315
- except asyncio.IncompleteReadError as e:
316
- logger.error("Client disconnected", e)
349
+ except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError) as e:
350
+ logger.warning("Client disconnected: %s", e, exc_info=True)
317
351
  finally:
318
352
  writer.close()
319
353
  # dont await on server side
@@ -50,7 +50,8 @@ def convert_keys_to_camel_case(obj):
50
50
 
51
51
  async def init_procedure_bdd(restore_args):
52
52
  if restore_args["use_mc_gw"]:
53
- sio = socketio.AsyncClient(logger=True, engineio_logger=True)
53
+ log_debug = os.getenv("LOG_LEVEL") == "DEBUG"
54
+ sio = socketio.AsyncClient(logger=log_debug, engineio_logger=log_debug)
54
55
 
55
56
  await sio.connect(
56
57
  f"https://{restore_args['mc_gw_host']}",
@@ -11,6 +11,8 @@ from pve_cloud.lib.backup_rpc import Command
11
11
 
12
12
  logger = logging.getLogger("fetcher")
13
13
 
14
+ SIO_MAX_RETRIES = 3
15
+
14
16
 
15
17
  def get_strict_client_ssl_ctx():
16
18
  ca_cert_path = os.getenv("BDD_CA_CERT_PATH")
@@ -70,7 +72,8 @@ async def get_sio_mc_client(backup_addr):
70
72
  "Tried to initialize multicloud proxy without providing MC_EXT_TOKEN env var!"
71
73
  )
72
74
 
73
- sio = socketio.AsyncClient(logger=True, engineio_logger=True)
75
+ log_debug = os.getenv("LOG_LEVEL") == "DEBUG"
76
+ sio = socketio.AsyncClient(logger=log_debug, engineio_logger=log_debug)
74
77
 
75
78
  await sio.connect(
76
79
  backup_addr,
@@ -123,29 +126,48 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
123
126
 
124
127
  if backup_addr.startswith("https://"):
125
128
  logger.info(f"sending archive to mc gateway {backup_addr}")
126
- # connection to mc gw
127
- sio = await get_sio_mc_client(backup_addr)
128
129
 
129
- await wait_archive_init(sio, request_dict)
130
+ # retry is currently only implemented for async generators
131
+ # which are the primary generator type for large image backups
132
+ for attempt in range(1, SIO_MAX_RETRIES + 1):
133
+ sio = None
134
+ try:
130
135
 
131
- if compress:
132
- compressor = zstd.ZstdCompressor(
133
- level=1,
134
- threads=6,
135
- ).compressobj()
136
+ # connection to mc gw
137
+ sio = await get_sio_mc_client(backup_addr)
136
138
 
137
- async for chunk in chunk_generator():
138
- await sio_send_cchunk(sio, compressor.compress(chunk))
139
+ await wait_archive_init(sio, request_dict)
139
140
 
140
- await sio_send_cchunk(sio, compressor.flush())
141
+ if compress:
142
+ compressor = zstd.ZstdCompressor(
143
+ level=1,
144
+ threads=6,
145
+ ).compressobj()
141
146
 
142
- else:
143
- async for chunk in chunk_generator():
144
- await sio.call("backup_chunk", chunk)
147
+ async for chunk in chunk_generator():
148
+ await sio_send_cchunk(sio, compressor.compress(chunk))
145
149
 
146
- await sio.call("backup_eof")
150
+ await sio_send_cchunk(sio, compressor.flush())
147
151
 
148
- await sio.disconnect()
152
+ else:
153
+ async for chunk in chunk_generator():
154
+ await sio.call("backup_chunk", chunk)
155
+
156
+ await sio.call("backup_eof")
157
+
158
+ break # finished successfully
159
+
160
+ except socketio.exceptions.TimeoutError:
161
+ logger.warn(f"Error on attempt {attempt}")
162
+ if attempt == SIO_MAX_RETRIES:
163
+ raise
164
+
165
+ logger.info("Retrying...")
166
+ await asyncio.sleep(10)
167
+
168
+ finally:
169
+ if sio:
170
+ await sio.disconnect()
149
171
 
150
172
  else:
151
173
  reader, writer = await asyncio.open_connection(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.7
3
+ Version: 3.5.9
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -1 +0,0 @@
1
- __version__ = "3.5.7"