py-pve-cloud-backup 3.5.3__tar.gz → 3.5.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {py_pve_cloud_backup-3.5.3/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.5}/PKG-INFO +1 -1
  2. py_pve_cloud_backup-3.5.5/src/pve_cloud_backup/_version.py +1 -0
  3. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/daemon/bdd.py +40 -16
  4. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/fetcher/net.py +12 -4
  5. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5/src/py_pve_cloud_backup.egg-info}/PKG-INFO +1 -1
  6. py_pve_cloud_backup-3.5.3/src/pve_cloud_backup/_version.py +0 -1
  7. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/LICENSE.md +0 -0
  8. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/README.md +0 -0
  9. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/pyproject.toml +0 -0
  10. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/requirements.txt +0 -0
  11. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/setup.cfg +0 -0
  12. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/daemon/brctl.py +0 -0
  13. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/daemon/funcs.py +0 -0
  14. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/daemon/restore.py +0 -0
  15. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/fetcher/ext_zfs.py +0 -0
  16. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/fetcher/funcs.py +0 -0
  17. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/fetcher/git.py +0 -0
  18. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/fetcher/main.py +0 -0
  19. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
  20. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
  21. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +0 -0
  22. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
  23. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/py_pve_cloud_backup.egg-info/entry_points.txt +0 -0
  24. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/py_pve_cloud_backup.egg-info/requires.txt +0 -0
  25. {py_pve_cloud_backup-3.5.3 → py_pve_cloud_backup-3.5.5}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.3
3
+ Version: 3.5.5
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -0,0 +1 @@
1
+ __version__ = "3.5.5"
@@ -25,14 +25,15 @@ ENV = os.getenv("ENV", "TESTING")
25
25
  BACKUP_TYPES = ["k8s", "nextcloud", "git", "postgres"]
26
26
 
27
27
  lock_dict = {}
28
-
28
+ lock_dict_lock = asyncio.Lock()
29
29
 
30
30
  # to prevent from writing to the same borg archive parallel
31
- def get_lock(backup_dir):
32
- if backup_dir not in lock_dict:
33
- lock_dict[backup_dir] = asyncio.Lock()
31
+ async def get_lock(backup_dir):
32
+ async with lock_dict_lock:
33
+ if backup_dir not in lock_dict:
34
+ lock_dict[backup_dir] = asyncio.Lock()
34
35
 
35
- return lock_dict[backup_dir]
36
+ return lock_dict[backup_dir]
36
37
 
37
38
 
38
39
  async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
@@ -63,8 +64,21 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
63
64
  else:
64
65
  backup_dir = init_backup_dir(borg_archive_type)
65
66
 
66
- # lock locally, we have one borg archive per archive type
67
- async with get_lock(backup_dir):
67
+ # send ping pong while waiting on lock
68
+ # maybe this is also needed in other rpc call types
69
+ lock = await get_lock(backup_dir)
70
+
71
+ while True:
72
+ try:
73
+ await asyncio.wait_for(lock.acquire(), timeout=5)
74
+ logger.info(f"accuired lock {backup_dir}")
75
+ break
76
+ except asyncio.TimeoutError:
77
+ writer.write(b"\x02")
78
+ await writer.drain()
79
+ logger.debug("send keepalive waiting for lock, continueing...")
80
+
81
+ try:
68
82
  borg_archive = f"{backup_dir}::{archive_name}_{timestamp}"
69
83
  logger.info(f"accuired lock {backup_dir}")
70
84
 
@@ -119,6 +133,9 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
119
133
  if exit_code != 0:
120
134
  raise Exception(f"Borg failed with code {exit_code}")
121
135
 
136
+ finally:
137
+ lock.release()
138
+
122
139
  case Command.NAMESPACE_SECRETS:
123
140
  # read meta dict size
124
141
  dict_size = struct.unpack("!I", (await reader.readexactly(4)))[0]
@@ -126,7 +143,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
126
143
 
127
144
  db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
128
145
 
129
- async with get_lock(db_path):
146
+ lock = await get_lock(db_path)
147
+ async with lock:
130
148
  secret_db = TinyDB(db_path)
131
149
  secret_db.insert(meta_dict)
132
150
 
@@ -135,7 +153,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
135
153
  meta_dict = pickle.loads((await reader.readexactly(dict_size)))
136
154
  db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
137
155
 
138
- async with get_lock(db_path):
156
+ lock = await get_lock(db_path)
157
+ async with lock:
139
158
  secret_db = TinyDB(db_path)
140
159
  secret_db.insert(meta_dict)
141
160
 
@@ -143,7 +162,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
143
162
  case Command.LIST_BACKUPS:
144
163
  db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
145
164
 
146
- async with get_lock(db_path):
165
+ lock = await get_lock(db_path)
166
+ async with lock:
147
167
  # we call borg on all our backups and send a return string that is strictly for display via the cli tool
148
168
  timestamp_archives = get_volume_metas()
149
169
 
@@ -161,7 +181,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
161
181
 
162
182
  db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
163
183
 
164
- async with get_lock(db_path):
184
+ lock = await get_lock(db_path)
185
+ async with lock:
165
186
  # we call borg on all our backups and send a return string that is strictly for display via the cli tool
166
187
  # this time we need the filter for displaying details of a certain backup
167
188
  timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
@@ -177,7 +198,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
177
198
  # return k8s secret requests
178
199
  db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
179
200
 
180
- async with get_lock(db_path):
201
+ lock = await get_lock(db_path)
202
+ async with lock:
181
203
  secret_db = TinyDB(db_path)
182
204
 
183
205
  while True:
@@ -205,7 +227,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
205
227
 
206
228
  db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
207
229
 
208
- async with get_lock(db_path):
230
+ lock = await get_lock(db_path)
231
+ async with lock:
209
232
  # we call borg on all our backups and send a return string that is strictly for display via the cli tool
210
233
  # this time we need the filter for displaying details of a certain backup
211
234
  timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
@@ -222,7 +245,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
222
245
  # return k8s secret requests
223
246
  db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
224
247
 
225
- async with get_lock(db_path):
248
+ lock = await get_lock(db_path)
249
+ async with lock:
226
250
  secret_db = TinyDB(db_path)
227
251
 
228
252
  Meta = Query()
@@ -255,8 +279,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
255
279
  logger.info(request_artifact)
256
280
 
257
281
  backup_dir = f"{get_backup_base_dir()}/{request_archive}"
258
-
259
- async with get_lock(backup_dir):
282
+ lock = await get_lock(backup_dir):
283
+ async with lock:
260
284
  logger.info(
261
285
  f"running borg extract on {backup_dir}::{request_artifact}"
262
286
  )
@@ -39,10 +39,18 @@ async def archive_init(reader, writer, request_dict):
39
39
  # wait for go signal, server needs to aquire write lock
40
40
  # we dont
41
41
  logger.debug("waiting for go from bdd")
42
- signal = await reader.readexactly(1)
43
- if signal != b"\x01":
44
- logger.error("recieved incorrect go signal")
45
- raise Exception("Incorrect go signal!")
42
+ while True:
43
+ signal = await reader.readexactly(1)
44
+ if signal == b"\x02":
45
+ logger.debug("waiting for lock continues...")
46
+ continue
47
+
48
+ if signal != b"\x01":
49
+ logger.error("recieved incorrect go signal")
50
+ raise Exception("Incorrect go signal!")
51
+ else:
52
+ break
53
+
46
54
  logger.debug("received go")
47
55
 
48
56
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.3
3
+ Version: 3.5.5
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -1 +0,0 @@
1
- __version__ = "3.5.3"