py-pve-cloud-backup 3.5.2__tar.gz → 3.5.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {py_pve_cloud_backup-3.5.2/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.4}/PKG-INFO +1 -1
  2. py_pve_cloud_backup-3.5.4/src/pve_cloud_backup/_version.py +1 -0
  3. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/daemon/bdd.py +21 -2
  4. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/fetcher/net.py +25 -22
  5. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4/src/py_pve_cloud_backup.egg-info}/PKG-INFO +1 -1
  6. py_pve_cloud_backup-3.5.2/src/pve_cloud_backup/_version.py +0 -1
  7. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/LICENSE.md +0 -0
  8. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/README.md +0 -0
  9. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/pyproject.toml +0 -0
  10. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/requirements.txt +0 -0
  11. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/setup.cfg +0 -0
  12. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/daemon/brctl.py +0 -0
  13. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/daemon/funcs.py +0 -0
  14. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/daemon/restore.py +0 -0
  15. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/fetcher/ext_zfs.py +0 -0
  16. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/fetcher/funcs.py +0 -0
  17. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/fetcher/git.py +0 -0
  18. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/fetcher/main.py +0 -0
  19. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
  20. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
  21. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +0 -0
  22. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
  23. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/py_pve_cloud_backup.egg-info/entry_points.txt +0 -0
  24. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/py_pve_cloud_backup.egg-info/requires.txt +0 -0
  25. {py_pve_cloud_backup-3.5.2 → py_pve_cloud_backup-3.5.4}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.2
3
+ Version: 3.5.4
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -0,0 +1 @@
1
+ __version__ = "3.5.4"
@@ -63,8 +63,21 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
63
63
  else:
64
64
  backup_dir = init_backup_dir(borg_archive_type)
65
65
 
66
- # lock locally, we have one borg archive per archive type
67
- async with get_lock(backup_dir):
66
+ # send ping pong while waiting on lock
67
+ # maybe this is also needed in other rpc call types
68
+ lock = get_lock(backup_dir)
69
+
70
+ while True:
71
+ try:
72
+ await asyncio.wait_for(lock.acquire(), timeout=5)
73
+ logger.info(f"accuired lock {backup_dir}")
74
+ break
75
+ except asyncio.TimeoutError:
76
+ writer.write(b"\x02")
77
+ await writer.drain()
78
+ logger.debug("send keepalive waiting for lock, continueing...")
79
+
80
+ try:
68
81
  borg_archive = f"{backup_dir}::{archive_name}_{timestamp}"
69
82
  logger.info(f"accuired lock {backup_dir}")
70
83
 
@@ -94,7 +107,9 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
94
107
  chunk_size = struct.unpack("!I", (await reader.readexactly(4)))[
95
108
  0
96
109
  ]
110
+ logger.debug(f"received chunk size {chunk_size}")
97
111
  if chunk_size == 0:
112
+ logger.debug("received chunk size 0, finished")
98
113
  break # client sends 0 chunk size at the end to signal that its finished uploading
99
114
  chunk = await reader.readexactly(chunk_size)
100
115
 
@@ -106,6 +121,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
106
121
 
107
122
  # the decompressor does not always return a decompressed chunk but might retain
108
123
  # and return empty. at the end we need to call flush to get everything out
124
+ logger.debug("flushing, draining and closing")
109
125
  borg_proc.stdin.write(decompressor.flush())
110
126
  await borg_proc.stdin.drain()
111
127
 
@@ -116,6 +132,9 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
116
132
  if exit_code != 0:
117
133
  raise Exception(f"Borg failed with code {exit_code}")
118
134
 
135
+ finally:
136
+ lock.release()
137
+
119
138
  case Command.NAMESPACE_SECRETS:
120
139
  # read meta dict size
121
140
  dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
@@ -39,10 +39,18 @@ async def archive_init(reader, writer, request_dict):
39
39
  # wait for go signal, server needs to aquire write lock
40
40
  # we dont
41
41
  logger.debug("waiting for go from bdd")
42
- signal = await reader.readexactly(1)
43
- if signal != b"\x01":
44
- logger.error("recieved incorrect go signal")
45
- raise Exception("Incorrect go signal!")
42
+ while True:
43
+ signal = await reader.readexactly(1)
44
+ if signal == b"\x02":
45
+ logger.debug("waiting for lock continues...")
46
+ continue
47
+
48
+ if signal != b"\x01":
49
+ logger.error("recieved incorrect go signal")
50
+ raise Exception("Incorrect go signal!")
51
+ else:
52
+ break
53
+
46
54
  logger.debug("received go")
47
55
 
48
56
 
@@ -72,10 +80,15 @@ async def get_sio_mc_client(backup_addr):
72
80
  },
73
81
  transports=["websocket"],
74
82
  )
75
-
83
+ logger.debug(f"Connected sio client to {backup_addr}")
76
84
  return sio
77
85
 
78
86
 
87
+ async def sio_send_cchunk(sio, compressed_chunk):
88
+ if compressed_chunk:
89
+ await sio.call("backup_chunk", compressed_chunk)
90
+
91
+
79
92
  # compress parameter exists for chunk generators that already do the compression
80
93
  # the receiving side ALWAYS expects a compressed stream
81
94
  async def archive_async(backup_addr, request_dict, chunk_generator, compress=True):
@@ -86,12 +99,14 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
86
99
  # connection to mc gw
87
100
  sio = await get_sio_mc_client(backup_addr)
88
101
 
102
+ logger.debug("sending archive init call")
89
103
  result = await sio.call(
90
104
  "archive_init",
91
105
  request_dict,
92
106
  timeout=30,
93
107
  )
94
108
 
109
+ logger.debug(f"init response {result['ok']}")
95
110
  if not result["ok"]:
96
111
  raise RuntimeError(result["error"])
97
112
 
@@ -102,15 +117,9 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
102
117
  ).compressobj()
103
118
 
104
119
  async for chunk in chunk_generator():
105
- await sio.call(
106
- "backup_chunk",
107
- compressor.compress(chunk),
108
- )
120
+ await sio_send_cchunk(sio, compressor.compress(chunk))
109
121
 
110
- await sio.call(
111
- "backup_chunk",
112
- compressor.flush(),
113
- )
122
+ await sio_send_cchunk(sio, compressor.flush())
114
123
 
115
124
  else:
116
125
  async for chunk in chunk_generator():
@@ -173,15 +182,9 @@ async def archive(backup_addr, request_dict, chunk_generator):
173
182
  ).compressobj()
174
183
 
175
184
  for chunk in chunk_generator():
176
- await sio.call(
177
- "backup_chunk",
178
- compressor.compress(chunk),
179
- )
180
-
181
- await sio.call(
182
- "backup_chunk",
183
- compressor.flush(),
184
- )
185
+ await sio_send_cchunk(sio, compressor.compress(chunk))
186
+
187
+ await sio_send_cchunk(sio, compressor.flush())
185
188
 
186
189
  await sio.call("backup_eof")
187
190
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.2
3
+ Version: 3.5.4
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -1 +0,0 @@
1
- __version__ = "3.5.2"