py-pve-cloud-backup 3.5.4__tar.gz → 3.5.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {py_pve_cloud_backup-3.5.4/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.6}/PKG-INFO +1 -1
  2. py_pve_cloud_backup-3.5.6/src/pve_cloud_backup/_version.py +1 -0
  3. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/bdd.py +23 -15
  4. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/net.py +29 -18
  5. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6/src/py_pve_cloud_backup.egg-info}/PKG-INFO +1 -1
  6. py_pve_cloud_backup-3.5.4/src/pve_cloud_backup/_version.py +0 -1
  7. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/LICENSE.md +0 -0
  8. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/README.md +0 -0
  9. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/pyproject.toml +0 -0
  10. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/requirements.txt +0 -0
  11. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/setup.cfg +0 -0
  12. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/brctl.py +0 -0
  13. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/funcs.py +0 -0
  14. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/restore.py +0 -0
  15. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/ext_zfs.py +0 -0
  16. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/funcs.py +0 -0
  17. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/git.py +0 -0
  18. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/main.py +0 -0
  19. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
  20. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
  21. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +0 -0
  22. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
  23. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/entry_points.txt +0 -0
  24. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/requires.txt +0 -0
  25. {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.4
3
+ Version: 3.5.6
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -0,0 +1 @@
1
+ __version__ = "3.5.6"
@@ -25,14 +25,16 @@ ENV = os.getenv("ENV", "TESTING")
25
25
  BACKUP_TYPES = ["k8s", "nextcloud", "git", "postgres"]
26
26
 
27
27
  lock_dict = {}
28
+ lock_dict_lock = asyncio.Lock()
28
29
 
29
30
 
30
31
  # to prevent from writing to the same borg archive parallel
31
- def get_lock(backup_dir):
32
- if backup_dir not in lock_dict:
33
- lock_dict[backup_dir] = asyncio.Lock()
32
+ async def get_lock(backup_dir):
33
+ async with lock_dict_lock:
34
+ if backup_dir not in lock_dict:
35
+ lock_dict[backup_dir] = asyncio.Lock()
34
36
 
35
- return lock_dict[backup_dir]
37
+ return lock_dict[backup_dir]
36
38
 
37
39
 
38
40
  async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
@@ -65,7 +67,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
65
67
 
66
68
  # send ping pong while waiting on lock
67
69
  # maybe this is also needed in other rpc call types
68
- lock = get_lock(backup_dir)
70
+ lock = await get_lock(backup_dir)
69
71
 
70
72
  while True:
71
73
  try:
@@ -79,7 +81,6 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
79
81
 
80
82
  try:
81
83
  borg_archive = f"{backup_dir}::{archive_name}_{timestamp}"
82
- logger.info(f"accuired lock {backup_dir}")
83
84
 
84
85
  # send continue signal, meaning we have the lock and export can start.
85
86
  writer.write(b"\x01") # signal = 0x01 means "continue"
@@ -142,7 +143,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
142
143
 
143
144
  db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
144
145
 
145
- async with get_lock(db_path):
146
+ lock = await get_lock(db_path)
147
+ async with lock:
146
148
  secret_db = TinyDB(db_path)
147
149
  secret_db.insert(meta_dict)
148
150
 
@@ -151,7 +153,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
151
153
  meta_dict = pickle.loads((await reader.readexactly(dict_size)))
152
154
  db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
153
155
 
154
- async with get_lock(db_path):
156
+ lock = await get_lock(db_path)
157
+ async with lock:
155
158
  secret_db = TinyDB(db_path)
156
159
  secret_db.insert(meta_dict)
157
160
 
@@ -159,7 +162,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
159
162
  case Command.LIST_BACKUPS:
160
163
  db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
161
164
 
162
- async with get_lock(db_path):
165
+ lock = await get_lock(db_path)
166
+ async with lock:
163
167
  # we call borg on all our backups and send a return string that is strictly for display via the cli tool
164
168
  timestamp_archives = get_volume_metas()
165
169
 
@@ -177,7 +181,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
177
181
 
178
182
  db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
179
183
 
180
- async with get_lock(db_path):
184
+ lock = await get_lock(db_path)
185
+ async with lock:
181
186
  # we call borg on all our backups and send a return string that is strictly for display via the cli tool
182
187
  # this time we need the filter for displaying details of a certain backup
183
188
  timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
@@ -193,7 +198,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
193
198
  # return k8s secret requests
194
199
  db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
195
200
 
196
- async with get_lock(db_path):
201
+ lock = await get_lock(db_path)
202
+ async with lock:
197
203
  secret_db = TinyDB(db_path)
198
204
 
199
205
  while True:
@@ -221,7 +227,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
221
227
 
222
228
  db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
223
229
 
224
- async with get_lock(db_path):
230
+ lock = await get_lock(db_path)
231
+ async with lock:
225
232
  # we call borg on all our backups and send a return string that is strictly for display via the cli tool
226
233
  # this time we need the filter for displaying details of a certain backup
227
234
  timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
@@ -238,7 +245,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
238
245
  # return k8s secret requests
239
246
  db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
240
247
 
241
- async with get_lock(db_path):
248
+ lock = await get_lock(db_path)
249
+ async with lock:
242
250
  secret_db = TinyDB(db_path)
243
251
 
244
252
  Meta = Query()
@@ -271,8 +279,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
271
279
  logger.info(request_artifact)
272
280
 
273
281
  backup_dir = f"{get_backup_base_dir()}/{request_archive}"
274
-
275
- async with get_lock(backup_dir):
282
+ lock = await get_lock(backup_dir)
283
+ async with lock:
276
284
  logger.info(
277
285
  f"running borg extract on {backup_dir}::{request_artifact}"
278
286
  )
@@ -89,6 +89,33 @@ async def sio_send_cchunk(sio, compressed_chunk):
89
89
  await sio.call("backup_chunk", compressed_chunk)
90
90
 
91
91
 
92
+ async def wait_archive_init(sio, request_dict):
93
+ initial = await sio.call(
94
+ "archive_init",
95
+ request_dict,
96
+ timeout=30,
97
+ )
98
+ if initial["status"] == "ERR":
99
+ raise RuntimeError(initial["error"])
100
+
101
+ if initial["status"] == "ACQUIRED":
102
+ return
103
+
104
+ logger.info("waiting for lock")
105
+ # status WAIT
106
+ while True:
107
+ wait_call = await sio.call("wait_archive", timeout=30)
108
+
109
+ if wait_call["status"] == "ERR":
110
+ raise RuntimeError(initial["error"])
111
+
112
+ if wait_call["status"] == "ACQUIRED":
113
+ return
114
+
115
+ # continue wait
116
+ logger.info("continueing waiting")
117
+
118
+
92
119
  # compress parameter exists for chunk generators that already do the compression
93
120
  # the receiving side ALWAYS expects a compressed stream
94
121
  async def archive_async(backup_addr, request_dict, chunk_generator, compress=True):
@@ -99,16 +126,7 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
99
126
  # connection to mc gw
100
127
  sio = await get_sio_mc_client(backup_addr)
101
128
 
102
- logger.debug("sending archive init call")
103
- result = await sio.call(
104
- "archive_init",
105
- request_dict,
106
- timeout=30,
107
- )
108
-
109
- logger.debug(f"init response {result['ok']}")
110
- if not result["ok"]:
111
- raise RuntimeError(result["error"])
129
+ await wait_archive_init(sio, request_dict)
112
130
 
113
131
  if compress:
114
132
  compressor = zstd.ZstdCompressor(
@@ -167,14 +185,7 @@ async def archive(backup_addr, request_dict, chunk_generator):
167
185
  if backup_addr.startswith("https://"):
168
186
  sio = await get_sio_mc_client(backup_addr)
169
187
 
170
- result = await sio.call(
171
- "archive_init",
172
- request_dict,
173
- timeout=30,
174
- )
175
-
176
- if not result["ok"]:
177
- raise RuntimeError(result["error"])
188
+ await wait_archive_init(sio, request_dict)
178
189
 
179
190
  compressor = zstd.ZstdCompressor(
180
191
  level=1,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: py-pve-cloud-backup
3
- Version: 3.5.4
3
+ Version: 3.5.6
4
4
  Author-email: Tobias Huebner <tobias.huebner@vmzberlin.com>
5
5
  License-Expression: GPL-3.0-or-later
6
6
  License-File: LICENSE.md
@@ -1 +0,0 @@
1
- __version__ = "3.5.4"