py-pve-cloud-backup 3.5.4__tar.gz → 3.5.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {py_pve_cloud_backup-3.5.4/src/py_pve_cloud_backup.egg-info → py_pve_cloud_backup-3.5.6}/PKG-INFO +1 -1
- py_pve_cloud_backup-3.5.6/src/pve_cloud_backup/_version.py +1 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/bdd.py +23 -15
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/net.py +29 -18
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6/src/py_pve_cloud_backup.egg-info}/PKG-INFO +1 -1
- py_pve_cloud_backup-3.5.4/src/pve_cloud_backup/_version.py +0 -1
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/LICENSE.md +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/README.md +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/pyproject.toml +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/requirements.txt +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/setup.cfg +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/brctl.py +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/funcs.py +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/restore.py +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/ext_zfs.py +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/funcs.py +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/git.py +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/main.py +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/nextcloud.py +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/patroni.py +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/SOURCES.txt +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/dependency_links.txt +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/entry_points.txt +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/requires.txt +0 -0
- {py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "3.5.6"
|
|
@@ -25,14 +25,16 @@ ENV = os.getenv("ENV", "TESTING")
|
|
|
25
25
|
BACKUP_TYPES = ["k8s", "nextcloud", "git", "postgres"]
|
|
26
26
|
|
|
27
27
|
lock_dict = {}
|
|
28
|
+
lock_dict_lock = asyncio.Lock()
|
|
28
29
|
|
|
29
30
|
|
|
30
31
|
# to prevent from writing to the same borg archive parallel
|
|
31
|
-
def get_lock(backup_dir):
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
async def get_lock(backup_dir):
|
|
33
|
+
async with lock_dict_lock:
|
|
34
|
+
if backup_dir not in lock_dict:
|
|
35
|
+
lock_dict[backup_dir] = asyncio.Lock()
|
|
34
36
|
|
|
35
|
-
|
|
37
|
+
return lock_dict[backup_dir]
|
|
36
38
|
|
|
37
39
|
|
|
38
40
|
async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
|
|
@@ -65,7 +67,7 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
65
67
|
|
|
66
68
|
# send ping pong while waiting on lock
|
|
67
69
|
# maybe this is also needed in other rpc call types
|
|
68
|
-
lock = get_lock(backup_dir)
|
|
70
|
+
lock = await get_lock(backup_dir)
|
|
69
71
|
|
|
70
72
|
while True:
|
|
71
73
|
try:
|
|
@@ -79,7 +81,6 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
79
81
|
|
|
80
82
|
try:
|
|
81
83
|
borg_archive = f"{backup_dir}::{archive_name}_{timestamp}"
|
|
82
|
-
logger.info(f"accuired lock {backup_dir}")
|
|
83
84
|
|
|
84
85
|
# send continue signal, meaning we have the lock and export can start.
|
|
85
86
|
writer.write(b"\x01") # signal = 0x01 means "continue"
|
|
@@ -142,7 +143,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
142
143
|
|
|
143
144
|
db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
|
|
144
145
|
|
|
145
|
-
|
|
146
|
+
lock = await get_lock(db_path)
|
|
147
|
+
async with lock:
|
|
146
148
|
secret_db = TinyDB(db_path)
|
|
147
149
|
secret_db.insert(meta_dict)
|
|
148
150
|
|
|
@@ -151,7 +153,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
151
153
|
meta_dict = pickle.loads((await reader.readexactly(dict_size)))
|
|
152
154
|
db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
|
|
153
155
|
|
|
154
|
-
|
|
156
|
+
lock = await get_lock(db_path)
|
|
157
|
+
async with lock:
|
|
155
158
|
secret_db = TinyDB(db_path)
|
|
156
159
|
secret_db.insert(meta_dict)
|
|
157
160
|
|
|
@@ -159,7 +162,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
159
162
|
case Command.LIST_BACKUPS:
|
|
160
163
|
db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
|
|
161
164
|
|
|
162
|
-
|
|
165
|
+
lock = await get_lock(db_path)
|
|
166
|
+
async with lock:
|
|
163
167
|
# we call borg on all our backups and send a return string that is strictly for display via the cli tool
|
|
164
168
|
timestamp_archives = get_volume_metas()
|
|
165
169
|
|
|
@@ -177,7 +181,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
177
181
|
|
|
178
182
|
db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
|
|
179
183
|
|
|
180
|
-
|
|
184
|
+
lock = await get_lock(db_path)
|
|
185
|
+
async with lock:
|
|
181
186
|
# we call borg on all our backups and send a return string that is strictly for display via the cli tool
|
|
182
187
|
# this time we need the filter for displaying details of a certain backup
|
|
183
188
|
timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
|
|
@@ -193,7 +198,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
193
198
|
# return k8s secret requests
|
|
194
199
|
db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
|
|
195
200
|
|
|
196
|
-
|
|
201
|
+
lock = await get_lock(db_path)
|
|
202
|
+
async with lock:
|
|
197
203
|
secret_db = TinyDB(db_path)
|
|
198
204
|
|
|
199
205
|
while True:
|
|
@@ -221,7 +227,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
221
227
|
|
|
222
228
|
db_path = f"{get_backup_base_dir()}/volume-meta-db.json"
|
|
223
229
|
|
|
224
|
-
|
|
230
|
+
lock = await get_lock(db_path)
|
|
231
|
+
async with lock:
|
|
225
232
|
# we call borg on all our backups and send a return string that is strictly for display via the cli tool
|
|
226
233
|
# this time we need the filter for displaying details of a certain backup
|
|
227
234
|
timestamp_archives = get_volume_metas(timestamp_filter=timestamp)
|
|
@@ -238,7 +245,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
238
245
|
# return k8s secret requests
|
|
239
246
|
db_path = f"{get_backup_base_dir()}/ns-secret-db.json"
|
|
240
247
|
|
|
241
|
-
|
|
248
|
+
lock = await get_lock(db_path)
|
|
249
|
+
async with lock:
|
|
242
250
|
secret_db = TinyDB(db_path)
|
|
243
251
|
|
|
244
252
|
Meta = Query()
|
|
@@ -271,8 +279,8 @@ async def handle_client(reader: asyncio.StreamReader, writer: asyncio.StreamWrit
|
|
|
271
279
|
logger.info(request_artifact)
|
|
272
280
|
|
|
273
281
|
backup_dir = f"{get_backup_base_dir()}/{request_archive}"
|
|
274
|
-
|
|
275
|
-
async with
|
|
282
|
+
lock = await get_lock(backup_dir)
|
|
283
|
+
async with lock:
|
|
276
284
|
logger.info(
|
|
277
285
|
f"running borg extract on {backup_dir}::{request_artifact}"
|
|
278
286
|
)
|
|
@@ -89,6 +89,33 @@ async def sio_send_cchunk(sio, compressed_chunk):
|
|
|
89
89
|
await sio.call("backup_chunk", compressed_chunk)
|
|
90
90
|
|
|
91
91
|
|
|
92
|
+
async def wait_archive_init(sio, request_dict):
|
|
93
|
+
initial = await sio.call(
|
|
94
|
+
"archive_init",
|
|
95
|
+
request_dict,
|
|
96
|
+
timeout=30,
|
|
97
|
+
)
|
|
98
|
+
if initial["status"] == "ERR":
|
|
99
|
+
raise RuntimeError(initial["error"])
|
|
100
|
+
|
|
101
|
+
if initial["status"] == "ACQUIRED":
|
|
102
|
+
return
|
|
103
|
+
|
|
104
|
+
logger.info("waiting for lock")
|
|
105
|
+
# status WAIT
|
|
106
|
+
while True:
|
|
107
|
+
wait_call = await sio.call("wait_archive", timeout=30)
|
|
108
|
+
|
|
109
|
+
if wait_call["status"] == "ERR":
|
|
110
|
+
raise RuntimeError(initial["error"])
|
|
111
|
+
|
|
112
|
+
if wait_call["status"] == "ACQUIRED":
|
|
113
|
+
return
|
|
114
|
+
|
|
115
|
+
# continue wait
|
|
116
|
+
logger.info("continueing waiting")
|
|
117
|
+
|
|
118
|
+
|
|
92
119
|
# compress parameter exists for chunk generators that already do the compression
|
|
93
120
|
# the receiving side ALWAYS expects a compressed stream
|
|
94
121
|
async def archive_async(backup_addr, request_dict, chunk_generator, compress=True):
|
|
@@ -99,16 +126,7 @@ async def archive_async(backup_addr, request_dict, chunk_generator, compress=Tru
|
|
|
99
126
|
# connection to mc gw
|
|
100
127
|
sio = await get_sio_mc_client(backup_addr)
|
|
101
128
|
|
|
102
|
-
|
|
103
|
-
result = await sio.call(
|
|
104
|
-
"archive_init",
|
|
105
|
-
request_dict,
|
|
106
|
-
timeout=30,
|
|
107
|
-
)
|
|
108
|
-
|
|
109
|
-
logger.debug(f"init response {result['ok']}")
|
|
110
|
-
if not result["ok"]:
|
|
111
|
-
raise RuntimeError(result["error"])
|
|
129
|
+
await wait_archive_init(sio, request_dict)
|
|
112
130
|
|
|
113
131
|
if compress:
|
|
114
132
|
compressor = zstd.ZstdCompressor(
|
|
@@ -167,14 +185,7 @@ async def archive(backup_addr, request_dict, chunk_generator):
|
|
|
167
185
|
if backup_addr.startswith("https://"):
|
|
168
186
|
sio = await get_sio_mc_client(backup_addr)
|
|
169
187
|
|
|
170
|
-
|
|
171
|
-
"archive_init",
|
|
172
|
-
request_dict,
|
|
173
|
-
timeout=30,
|
|
174
|
-
)
|
|
175
|
-
|
|
176
|
-
if not result["ok"]:
|
|
177
|
-
raise RuntimeError(result["error"])
|
|
188
|
+
await wait_archive_init(sio, request_dict)
|
|
178
189
|
|
|
179
190
|
compressor = zstd.ZstdCompressor(
|
|
180
191
|
level=1,
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "3.5.4"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/brctl.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/funcs.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/daemon/restore.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/ext_zfs.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/funcs.py
RENAMED
|
File without changes
|
|
File without changes
|
{py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/main.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/nextcloud.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/pve_cloud_backup/fetcher/patroni.py
RENAMED
|
File without changes
|
{py_pve_cloud_backup-3.5.4 → py_pve_cloud_backup-3.5.6}/src/py_pve_cloud_backup.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|