hlsync 1.17.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of hlsync might be problematic. Click here for more details.
- hlsync/__init__.py +5 -0
- hlsync/__main__.py +4 -0
- hlsync/_version.py +2 -0
- hlsync/cli.py +2909 -0
- hlsync/comparison.py +278 -0
- hlsync/config.py +576 -0
- hlsync/gitignore.py +59 -0
- hlsync/pattern_operands.py +33 -0
- hlsync/progress.py +57 -0
- hlsync/rules.py +276 -0
- hlsync/selection.py +159 -0
- hlsync/snapshot.py +193 -0
- hlsync/storage.py +29 -0
- hlsync/transfer.py +465 -0
- hlsync/transport.py +761 -0
- hlsync-1.17.2.dist-info/METADATA +687 -0
- hlsync-1.17.2.dist-info/RECORD +21 -0
- hlsync-1.17.2.dist-info/WHEEL +5 -0
- hlsync-1.17.2.dist-info/entry_points.txt +2 -0
- hlsync-1.17.2.dist-info/licenses/LICENSE +106 -0
- hlsync-1.17.2.dist-info/top_level.txt +1 -0
hlsync/transport.py
ADDED
|
@@ -0,0 +1,761 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ftplib
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import ssl
|
|
7
|
+
from calendar import timegm
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from datetime import UTC, datetime
|
|
11
|
+
from pathlib import PurePosixPath
|
|
12
|
+
from typing import BinaryIO, Protocol
|
|
13
|
+
from uuid import uuid4
|
|
14
|
+
|
|
15
|
+
from hlsync.config import ProfileConfiguration
|
|
16
|
+
from hlsync.rules import RuleSet
|
|
17
|
+
from hlsync.selection import FileSelection
|
|
18
|
+
from hlsync.snapshot import EntryKind, SnapshotError, TreeEntry, TreeSnapshot
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class TransportError(RuntimeError):
|
|
22
|
+
"""Raised when a remote transport cannot be used safely."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class PathOperationError(TransportError):
|
|
26
|
+
"""Raised when one remote path fails but the FTPS session remains usable."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
_HLSYNC_ARTIFACT_NAME = re.compile(
|
|
30
|
+
r"^\.(?P<destination>.+)\.hlsync-(?P<kind>upload|backup)-"
|
|
31
|
+
r"(?P<token>[0-9a-f]{32})$"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class _HLSyncArtifact:
|
|
37
|
+
path: str
|
|
38
|
+
destination: str
|
|
39
|
+
kind: str
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class _ArtifactRecoveryPlan:
|
|
44
|
+
projected: TreeSnapshot
|
|
45
|
+
uploads_to_remove: tuple[str, ...]
|
|
46
|
+
backups_to_remove: tuple[str, ...]
|
|
47
|
+
backups_to_restore: tuple[tuple[str, str], ...]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _hlsync_artifact(path: str) -> _HLSyncArtifact | None:
|
|
51
|
+
remote = PurePosixPath(path)
|
|
52
|
+
match = _HLSYNC_ARTIFACT_NAME.fullmatch(remote.name)
|
|
53
|
+
if match is None:
|
|
54
|
+
return None
|
|
55
|
+
destination = remote.with_name(match.group("destination")).as_posix()
|
|
56
|
+
return _HLSyncArtifact(path, destination, match.group("kind"))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _parse_modify_timestamp(
|
|
60
|
+
value: str,
|
|
61
|
+
path: str,
|
|
62
|
+
*,
|
|
63
|
+
source: str,
|
|
64
|
+
) -> tuple[int, int]:
|
|
65
|
+
base, separator, fraction = value.partition(".")
|
|
66
|
+
if len(base) != 14 or not base.isdigit() or (
|
|
67
|
+
separator and (not fraction or not fraction.isdigit())
|
|
68
|
+
):
|
|
69
|
+
raise TransportError(
|
|
70
|
+
f"invalid {source} timestamp {value!r} for '{path}'"
|
|
71
|
+
)
|
|
72
|
+
try:
|
|
73
|
+
timestamp = datetime.strptime(base, "%Y%m%d%H%M%S")
|
|
74
|
+
except ValueError as error:
|
|
75
|
+
raise TransportError(
|
|
76
|
+
f"invalid {source} timestamp {value!r} for '{path}'"
|
|
77
|
+
) from error
|
|
78
|
+
fraction_digits = min(len(fraction), 9)
|
|
79
|
+
fractional_ns = int(fraction[:9].ljust(9, "0")) if fraction else 0
|
|
80
|
+
precision_ns = 10 ** (9 - fraction_digits) if fraction else 1_000_000_000
|
|
81
|
+
return timegm(timestamp.timetuple()) * 1_000_000_000 + fractional_ns, precision_ns
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _parse_mlsd_modify(value: str, path: str) -> tuple[int, int]:
|
|
85
|
+
return _parse_modify_timestamp(value, path, source="MLSD modify")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _parse_mdtm_response(response: str, path: str) -> tuple[int, int]:
|
|
89
|
+
code, separator, value = response.partition(" ")
|
|
90
|
+
if code != "213" or not separator or not value.strip():
|
|
91
|
+
raise TransportError(
|
|
92
|
+
f"invalid MDTM response for '{path}': {response}"
|
|
93
|
+
)
|
|
94
|
+
return _parse_modify_timestamp(value.strip(), path, source="MDTM")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _relative_remote_path(value: str) -> str:
|
|
98
|
+
path = PurePosixPath(value)
|
|
99
|
+
if path.is_absolute() or not path.parts or ".." in path.parts:
|
|
100
|
+
raise TransportError(f"remote operation path must be relative: {value!r}")
|
|
101
|
+
return path.as_posix()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class RemoteTransport(Protocol):
|
|
105
|
+
def connect(self) -> None: ...
|
|
106
|
+
|
|
107
|
+
def snapshot(
|
|
108
|
+
self,
|
|
109
|
+
rules: RuleSet,
|
|
110
|
+
selector: FileSelection | None = None,
|
|
111
|
+
*,
|
|
112
|
+
include_excluded: bool = False,
|
|
113
|
+
traverse_excluded: bool = False,
|
|
114
|
+
artifact_recovery: Callable[[str], None] | None = None,
|
|
115
|
+
artifact_preview: Callable[[str], None] | None = None,
|
|
116
|
+
directory_progress: Callable[[PurePosixPath], None] | None = None,
|
|
117
|
+
directory_counts: Callable[[int, int], None] | None = None,
|
|
118
|
+
) -> TreeSnapshot: ...
|
|
119
|
+
|
|
120
|
+
def list_directory(
|
|
121
|
+
self,
|
|
122
|
+
relative_directory: PurePosixPath,
|
|
123
|
+
rules: RuleSet,
|
|
124
|
+
) -> TreeSnapshot: ...
|
|
125
|
+
|
|
126
|
+
def make_directory(self, relative_path: str) -> None: ...
|
|
127
|
+
|
|
128
|
+
def upload_file(
|
|
129
|
+
self,
|
|
130
|
+
local_path: BinaryIO,
|
|
131
|
+
relative_path: str,
|
|
132
|
+
*,
|
|
133
|
+
size: int,
|
|
134
|
+
modified_ns: int,
|
|
135
|
+
replace: bool,
|
|
136
|
+
byte_progress: Callable[[int], None] | None = None,
|
|
137
|
+
) -> None: ...
|
|
138
|
+
|
|
139
|
+
def download_file(self, relative_path: str, destination: BinaryIO) -> None: ...
|
|
140
|
+
|
|
141
|
+
def delete_path(self, relative_path: str, *, is_directory: bool) -> None: ...
|
|
142
|
+
|
|
143
|
+
def close(self) -> None: ...
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@dataclass
|
|
147
|
+
class ExplicitFTPSTransport:
|
|
148
|
+
configuration: ProfileConfiguration
|
|
149
|
+
timeout: float = 30.0
|
|
150
|
+
ssl_context: ssl.SSLContext | None = None
|
|
151
|
+
|
|
152
|
+
def __post_init__(self) -> None:
|
|
153
|
+
self._client: ftplib.FTP_TLS | None = None
|
|
154
|
+
self._use_mlsd = True
|
|
155
|
+
self._timestamp_command = "MFMT"
|
|
156
|
+
|
|
157
|
+
def connect(self) -> None:
|
|
158
|
+
username = os.environ.get(self.configuration.username_env)
|
|
159
|
+
password = os.environ.get(self.configuration.password_env)
|
|
160
|
+
missing = [
|
|
161
|
+
name
|
|
162
|
+
for name, value in (
|
|
163
|
+
(self.configuration.username_env, username),
|
|
164
|
+
(self.configuration.password_env, password),
|
|
165
|
+
)
|
|
166
|
+
if value is None
|
|
167
|
+
]
|
|
168
|
+
if missing:
|
|
169
|
+
raise TransportError(
|
|
170
|
+
f"missing credential environment variable(s): {', '.join(missing)}"
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
client = ftplib.FTP_TLS(
|
|
174
|
+
context=self.ssl_context or ssl.create_default_context(),
|
|
175
|
+
timeout=self.timeout,
|
|
176
|
+
)
|
|
177
|
+
try:
|
|
178
|
+
client.connect(self.configuration.host, self.configuration.port)
|
|
179
|
+
client.auth()
|
|
180
|
+
client.login(username, password)
|
|
181
|
+
client.prot_p()
|
|
182
|
+
client.cwd(self.configuration.remote_root)
|
|
183
|
+
features = client.sendcmd("FEAT")
|
|
184
|
+
advertised = {
|
|
185
|
+
line.strip().upper().split(" ", 1)[0]
|
|
186
|
+
for line in features.splitlines()
|
|
187
|
+
}
|
|
188
|
+
self._timestamp_command = (
|
|
189
|
+
"MDTM"
|
|
190
|
+
if "MFMT" not in advertised
|
|
191
|
+
and "vsftpd" in client.getwelcome().lower()
|
|
192
|
+
else "MFMT"
|
|
193
|
+
)
|
|
194
|
+
self._use_mlsd = any(
|
|
195
|
+
line.strip().upper().split(" ", 1)[0] in {"MLST", "MLSD"}
|
|
196
|
+
for line in features.splitlines()
|
|
197
|
+
)
|
|
198
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
199
|
+
try:
|
|
200
|
+
client.close()
|
|
201
|
+
finally:
|
|
202
|
+
raise TransportError(
|
|
203
|
+
f"could not open explicit FTPS profile at "
|
|
204
|
+
f"{self.configuration.host}:{self.configuration.port}"
|
|
205
|
+
f"{self.configuration.remote_root}: {error}"
|
|
206
|
+
) from error
|
|
207
|
+
self._client = client
|
|
208
|
+
|
|
209
|
+
def close(self) -> None:
|
|
210
|
+
if self._client is None:
|
|
211
|
+
return
|
|
212
|
+
client, self._client = self._client, None
|
|
213
|
+
try:
|
|
214
|
+
client.quit()
|
|
215
|
+
except (OSError, ftplib.Error):
|
|
216
|
+
client.close()
|
|
217
|
+
|
|
218
|
+
def snapshot(
|
|
219
|
+
self,
|
|
220
|
+
rules: RuleSet,
|
|
221
|
+
selector: FileSelection | None = None,
|
|
222
|
+
*,
|
|
223
|
+
include_excluded: bool = False,
|
|
224
|
+
traverse_excluded: bool = False,
|
|
225
|
+
artifact_recovery: Callable[[str], None] | None = None,
|
|
226
|
+
artifact_preview: Callable[[str], None] | None = None,
|
|
227
|
+
directory_progress: Callable[[PurePosixPath], None] | None = None,
|
|
228
|
+
directory_counts: Callable[[int, int], None] | None = None,
|
|
229
|
+
) -> TreeSnapshot:
|
|
230
|
+
if self._client is None:
|
|
231
|
+
raise TransportError("FTPS transport is not connected")
|
|
232
|
+
entries: list[TreeEntry] = []
|
|
233
|
+
completed_directories = 0
|
|
234
|
+
discovered_directories = 1
|
|
235
|
+
if directory_counts is not None:
|
|
236
|
+
directory_counts(0, 1)
|
|
237
|
+
|
|
238
|
+
def walk(relative_directory: PurePosixPath) -> None:
|
|
239
|
+
nonlocal completed_directories, discovered_directories
|
|
240
|
+
if directory_progress is not None:
|
|
241
|
+
directory_progress(relative_directory)
|
|
242
|
+
listing = self.list_directory(relative_directory, rules)
|
|
243
|
+
if artifact_recovery is not None:
|
|
244
|
+
recoveries = self._recover_artifacts(listing, selector, rules)
|
|
245
|
+
for recovery in recoveries:
|
|
246
|
+
artifact_recovery(recovery)
|
|
247
|
+
if recoveries:
|
|
248
|
+
listing = self.list_directory(relative_directory, rules)
|
|
249
|
+
elif artifact_preview is not None:
|
|
250
|
+
recovery = self._plan_artifact_recovery(listing, selector, rules)
|
|
251
|
+
for path in recovery.uploads_to_remove:
|
|
252
|
+
artifact_preview(f"Would remove abandoned upload '{path}'.")
|
|
253
|
+
for path in recovery.backups_to_remove:
|
|
254
|
+
artifact_preview(f"Would remove old backup '{path}'.")
|
|
255
|
+
for _, destination in recovery.backups_to_restore:
|
|
256
|
+
artifact_preview(
|
|
257
|
+
f"Would restore interrupted replacement '{destination}'."
|
|
258
|
+
)
|
|
259
|
+
listing = recovery.projected
|
|
260
|
+
descendants: list[PurePosixPath] = []
|
|
261
|
+
for entry in listing.entries:
|
|
262
|
+
relative_path = entry.path
|
|
263
|
+
relative = PurePosixPath(relative_path)
|
|
264
|
+
kind = entry.kind
|
|
265
|
+
excluded = entry.excluded
|
|
266
|
+
visible = not excluded or include_excluded
|
|
267
|
+
selected = visible and (
|
|
268
|
+
selector is None or selector.matches(relative_path)
|
|
269
|
+
)
|
|
270
|
+
if selected:
|
|
271
|
+
entries.append(entry)
|
|
272
|
+
selection_may_descend = (
|
|
273
|
+
selector is None
|
|
274
|
+
or selector.may_match_descendant(relative_path)
|
|
275
|
+
)
|
|
276
|
+
local_exclusion_may_descend = (
|
|
277
|
+
traverse_excluded
|
|
278
|
+
or not excluded
|
|
279
|
+
or rules.may_include_descendant(relative_path, target="local")
|
|
280
|
+
)
|
|
281
|
+
if (
|
|
282
|
+
kind == "directory"
|
|
283
|
+
and selection_may_descend
|
|
284
|
+
and local_exclusion_may_descend
|
|
285
|
+
and not entry.remote_excluded
|
|
286
|
+
):
|
|
287
|
+
descendants.append(relative)
|
|
288
|
+
discovered_directories += len(descendants)
|
|
289
|
+
completed_directories += 1
|
|
290
|
+
if directory_counts is not None:
|
|
291
|
+
directory_counts(completed_directories, discovered_directories)
|
|
292
|
+
for descendant in descendants:
|
|
293
|
+
walk(descendant)
|
|
294
|
+
|
|
295
|
+
walk(PurePosixPath())
|
|
296
|
+
return TreeSnapshot(tuple(entries))
|
|
297
|
+
|
|
298
|
+
def list_directory(
|
|
299
|
+
self,
|
|
300
|
+
relative_directory: PurePosixPath,
|
|
301
|
+
rules: RuleSet,
|
|
302
|
+
) -> TreeSnapshot:
|
|
303
|
+
"""Normalize the negotiated listing format into shared tree metadata."""
|
|
304
|
+
client = self._connected_client()
|
|
305
|
+
remote_directory = relative_directory.as_posix()
|
|
306
|
+
if remote_directory == ".":
|
|
307
|
+
remote_directory = ""
|
|
308
|
+
try:
|
|
309
|
+
children = sorted(
|
|
310
|
+
client.mlsd(
|
|
311
|
+
remote_directory,
|
|
312
|
+
facts=["type", "size", "modify"],
|
|
313
|
+
) if self._use_mlsd else self._list_facts(relative_directory),
|
|
314
|
+
key=lambda item: item[0],
|
|
315
|
+
)
|
|
316
|
+
except ftplib.error_perm as error:
|
|
317
|
+
display_path = remote_directory or "."
|
|
318
|
+
raise PathOperationError(
|
|
319
|
+
f"could not list remote directory '{display_path}': {error}"
|
|
320
|
+
) from error
|
|
321
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
322
|
+
display_path = remote_directory or "."
|
|
323
|
+
raise TransportError(
|
|
324
|
+
f"could not list remote directory '{display_path}': {error}"
|
|
325
|
+
) from error
|
|
326
|
+
|
|
327
|
+
entries: list[TreeEntry] = []
|
|
328
|
+
for name, facts in children:
|
|
329
|
+
entry_type = facts.get("type", "").lower()
|
|
330
|
+
if entry_type in {"cdir", "pdir"}:
|
|
331
|
+
continue
|
|
332
|
+
if not name or "/" in name or name in {".", ".."}:
|
|
333
|
+
raise TransportError(f"invalid name in remote listing: {name!r}")
|
|
334
|
+
relative_path = (relative_directory / name).as_posix()
|
|
335
|
+
if entry_type == "dir":
|
|
336
|
+
kind: EntryKind = "directory"
|
|
337
|
+
elif entry_type == "file":
|
|
338
|
+
kind = "file"
|
|
339
|
+
elif entry_type.startswith("os.unix=slink"):
|
|
340
|
+
kind = "symlink"
|
|
341
|
+
else:
|
|
342
|
+
raise TransportError(
|
|
343
|
+
f"unsupported remote entry type {entry_type!r} "
|
|
344
|
+
f"for '{relative_path}'"
|
|
345
|
+
)
|
|
346
|
+
excluded = rules.excludes(
|
|
347
|
+
relative_path,
|
|
348
|
+
target="local",
|
|
349
|
+
is_directory=kind == "directory",
|
|
350
|
+
)
|
|
351
|
+
remote_excluded = rules.excludes(
|
|
352
|
+
relative_path,
|
|
353
|
+
target="remote",
|
|
354
|
+
is_directory=kind == "directory",
|
|
355
|
+
)
|
|
356
|
+
if kind != "file":
|
|
357
|
+
entries.append(
|
|
358
|
+
TreeEntry(
|
|
359
|
+
relative_path,
|
|
360
|
+
kind,
|
|
361
|
+
excluded=excluded,
|
|
362
|
+
remote_excluded=remote_excluded,
|
|
363
|
+
)
|
|
364
|
+
)
|
|
365
|
+
continue
|
|
366
|
+
size_value = facts.get("size")
|
|
367
|
+
modified_value = facts.get("modify")
|
|
368
|
+
if size_value is None or modified_value is None:
|
|
369
|
+
raise TransportError(
|
|
370
|
+
f"remote file metadata is incomplete for '{relative_path}'"
|
|
371
|
+
)
|
|
372
|
+
try:
|
|
373
|
+
size = int(size_value)
|
|
374
|
+
except ValueError as error:
|
|
375
|
+
raise TransportError(
|
|
376
|
+
f"invalid MLSD size {size_value!r} for '{relative_path}'"
|
|
377
|
+
) from error
|
|
378
|
+
modified_ns, precision_ns = _parse_mlsd_modify(
|
|
379
|
+
modified_value,
|
|
380
|
+
relative_path,
|
|
381
|
+
)
|
|
382
|
+
try:
|
|
383
|
+
entries.append(
|
|
384
|
+
TreeEntry(
|
|
385
|
+
relative_path,
|
|
386
|
+
kind,
|
|
387
|
+
size=size,
|
|
388
|
+
modified_ns=modified_ns,
|
|
389
|
+
timestamp_precision_ns=precision_ns,
|
|
390
|
+
excluded=excluded,
|
|
391
|
+
remote_excluded=remote_excluded,
|
|
392
|
+
)
|
|
393
|
+
)
|
|
394
|
+
except SnapshotError as error:
|
|
395
|
+
raise TransportError(
|
|
396
|
+
f"invalid remote file metadata for '{relative_path}': {error}"
|
|
397
|
+
) from error
|
|
398
|
+
return TreeSnapshot(tuple(entries))
|
|
399
|
+
|
|
400
|
+
def _list_facts(
|
|
401
|
+
self, directory: PurePosixPath,
|
|
402
|
+
) -> list[tuple[str, dict[str, str]]]:
|
|
403
|
+
client = self._connected_client()
|
|
404
|
+
lines: list[str] = []
|
|
405
|
+
# Keep options separate from paths; servers disagree on LIST -a PATH.
|
|
406
|
+
previous = client.pwd()
|
|
407
|
+
client.cwd(f"./{directory.as_posix()}")
|
|
408
|
+
try:
|
|
409
|
+
client.retrlines("LIST -a", lines.append)
|
|
410
|
+
finally:
|
|
411
|
+
try:
|
|
412
|
+
client.cwd(previous)
|
|
413
|
+
except (OSError, ftplib.Error) as error:
|
|
414
|
+
client.close()
|
|
415
|
+
self._client = None
|
|
416
|
+
raise TransportError(
|
|
417
|
+
"could not restore remote working directory"
|
|
418
|
+
) from error
|
|
419
|
+
children: list[tuple[str, dict[str, str]]] = []
|
|
420
|
+
client.voidcmd("TYPE I")
|
|
421
|
+
for line in lines:
|
|
422
|
+
if re.fullmatch(r"total \d+", line):
|
|
423
|
+
continue
|
|
424
|
+
fields = line.split(maxsplit=8)
|
|
425
|
+
if len(fields) != 9 or not re.fullmatch(
|
|
426
|
+
r"[-dl][rwxStTs-]{9}[+@.]?", fields[0]
|
|
427
|
+
):
|
|
428
|
+
raise TransportError(f"unsupported LIST entry: {line!r}")
|
|
429
|
+
name = fields[8]
|
|
430
|
+
kind = fields[0][0]
|
|
431
|
+
if kind == "l":
|
|
432
|
+
name = name.split(" -> ", 1)[0]
|
|
433
|
+
if name in {".", ".."}:
|
|
434
|
+
continue
|
|
435
|
+
if not name or "/" in name or "\r" in name or "\n" in name:
|
|
436
|
+
raise TransportError(f"invalid name in remote listing: {name!r}")
|
|
437
|
+
facts = {"type": {"d": "dir", "-": "file", "l": "OS.unix=slink"}[kind]}
|
|
438
|
+
if kind == "-":
|
|
439
|
+
remote_path = f"./{(directory / name).as_posix()}"
|
|
440
|
+
size = client.size(remote_path)
|
|
441
|
+
if size is None:
|
|
442
|
+
raise TransportError(f"missing remote size for '{name}'")
|
|
443
|
+
response = client.sendcmd(f"MDTM {remote_path}")
|
|
444
|
+
_parse_mdtm_response(response, remote_path)
|
|
445
|
+
facts.update(size=str(size), modify=response.partition(" ")[2].strip())
|
|
446
|
+
children.append((name, facts))
|
|
447
|
+
return children
|
|
448
|
+
|
|
449
|
+
def _connected_client(self) -> ftplib.FTP_TLS:
|
|
450
|
+
if self._client is None:
|
|
451
|
+
raise TransportError("FTPS transport is not connected")
|
|
452
|
+
return self._client
|
|
453
|
+
|
|
454
|
+
def make_directory(self, relative_path: str) -> None:
|
|
455
|
+
path = _relative_remote_path(relative_path)
|
|
456
|
+
try:
|
|
457
|
+
self._connected_client().mkd(path)
|
|
458
|
+
except ftplib.error_perm as error:
|
|
459
|
+
raise PathOperationError(
|
|
460
|
+
f"could not create remote directory '{path}': {error}"
|
|
461
|
+
) from error
|
|
462
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
463
|
+
raise TransportError(
|
|
464
|
+
f"could not create remote directory '{path}': {error}"
|
|
465
|
+
) from error
|
|
466
|
+
|
|
467
|
+
def _plan_artifact_recovery(
|
|
468
|
+
self,
|
|
469
|
+
listing: TreeSnapshot,
|
|
470
|
+
selector: FileSelection | None,
|
|
471
|
+
rules: RuleSet,
|
|
472
|
+
) -> _ArtifactRecoveryPlan:
|
|
473
|
+
entries = {entry.path: entry for entry in listing.entries}
|
|
474
|
+
artifacts = tuple(
|
|
475
|
+
artifact
|
|
476
|
+
for path in sorted(entries)
|
|
477
|
+
for artifact in (_hlsync_artifact(path),)
|
|
478
|
+
if artifact is not None
|
|
479
|
+
and (selector is None or selector.matches(artifact.destination))
|
|
480
|
+
)
|
|
481
|
+
for artifact in artifacts:
|
|
482
|
+
if entries[artifact.path].kind != "file":
|
|
483
|
+
raise TransportError(
|
|
484
|
+
f"reserved HLSync artifact is not a file: '{artifact.path}'"
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
uploads = tuple(
|
|
488
|
+
artifact.path for artifact in artifacts if artifact.kind == "upload"
|
|
489
|
+
)
|
|
490
|
+
projected = dict(entries)
|
|
491
|
+
for path in uploads:
|
|
492
|
+
projected.pop(path)
|
|
493
|
+
|
|
494
|
+
backups_by_destination: dict[str, list[_HLSyncArtifact]] = {}
|
|
495
|
+
for artifact in artifacts:
|
|
496
|
+
if artifact.kind == "backup":
|
|
497
|
+
backups_by_destination.setdefault(
|
|
498
|
+
artifact.destination,
|
|
499
|
+
[],
|
|
500
|
+
).append(artifact)
|
|
501
|
+
obsolete_backups: list[str] = []
|
|
502
|
+
restorations: list[tuple[str, str]] = []
|
|
503
|
+
for destination, backups in sorted(backups_by_destination.items()):
|
|
504
|
+
if destination not in entries and len(backups) > 1:
|
|
505
|
+
names = ", ".join(backup.path for backup in backups)
|
|
506
|
+
raise TransportError(
|
|
507
|
+
f"multiple HLSync backups require manual recovery for "
|
|
508
|
+
f"'{destination}': {names}"
|
|
509
|
+
)
|
|
510
|
+
if destination in entries:
|
|
511
|
+
for backup in backups:
|
|
512
|
+
projected.pop(backup.path)
|
|
513
|
+
obsolete_backups.append(backup.path)
|
|
514
|
+
continue
|
|
515
|
+
backup = backups[0]
|
|
516
|
+
projected.pop(backup.path)
|
|
517
|
+
source = entries[backup.path]
|
|
518
|
+
projected[destination] = TreeEntry(
|
|
519
|
+
destination,
|
|
520
|
+
"file",
|
|
521
|
+
size=source.size,
|
|
522
|
+
modified_ns=source.modified_ns,
|
|
523
|
+
timestamp_precision_ns=source.timestamp_precision_ns,
|
|
524
|
+
excluded=rules.excludes(destination, target="local"),
|
|
525
|
+
remote_excluded=rules.excludes(destination, target="remote"),
|
|
526
|
+
)
|
|
527
|
+
restorations.append((backup.path, destination))
|
|
528
|
+
return _ArtifactRecoveryPlan(
|
|
529
|
+
TreeSnapshot(tuple(projected[path] for path in sorted(projected))),
|
|
530
|
+
uploads,
|
|
531
|
+
tuple(obsolete_backups),
|
|
532
|
+
tuple(restorations),
|
|
533
|
+
)
|
|
534
|
+
|
|
535
|
+
def _recover_artifacts(
|
|
536
|
+
self,
|
|
537
|
+
listing: TreeSnapshot,
|
|
538
|
+
selector: FileSelection | None,
|
|
539
|
+
rules: RuleSet,
|
|
540
|
+
) -> tuple[str, ...]:
|
|
541
|
+
recovery = self._plan_artifact_recovery(listing, selector, rules)
|
|
542
|
+
messages: list[str] = []
|
|
543
|
+
for path in recovery.uploads_to_remove:
|
|
544
|
+
self.delete_path(path, is_directory=False)
|
|
545
|
+
messages.append(f"Removed abandoned upload '{path}'.")
|
|
546
|
+
for path in recovery.backups_to_remove:
|
|
547
|
+
self.delete_path(path, is_directory=False)
|
|
548
|
+
messages.append(f"Removed old backup '{path}'.")
|
|
549
|
+
client = self._connected_client()
|
|
550
|
+
for backup, destination in recovery.backups_to_restore:
|
|
551
|
+
try:
|
|
552
|
+
client.rename(backup, destination)
|
|
553
|
+
except ftplib.error_perm as error:
|
|
554
|
+
raise PathOperationError(
|
|
555
|
+
f"could not restore HLSync backup '{backup}' to "
|
|
556
|
+
f"'{destination}': {error}"
|
|
557
|
+
) from error
|
|
558
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
559
|
+
raise TransportError(
|
|
560
|
+
f"could not restore HLSync backup '{backup}' to "
|
|
561
|
+
f"'{destination}': {error}"
|
|
562
|
+
) from error
|
|
563
|
+
messages.append(
|
|
564
|
+
f"Restored interrupted replacement '{destination}'."
|
|
565
|
+
)
|
|
566
|
+
return tuple(messages)
|
|
567
|
+
|
|
568
|
+
def upload_file(
|
|
569
|
+
self,
|
|
570
|
+
local_path: BinaryIO,
|
|
571
|
+
relative_path: str,
|
|
572
|
+
*,
|
|
573
|
+
size: int,
|
|
574
|
+
modified_ns: int,
|
|
575
|
+
replace: bool,
|
|
576
|
+
byte_progress: Callable[[int], None] | None = None,
|
|
577
|
+
) -> None:
|
|
578
|
+
path = _relative_remote_path(relative_path)
|
|
579
|
+
remote = PurePosixPath(path)
|
|
580
|
+
token = uuid4().hex
|
|
581
|
+
temporary = str(remote.with_name(f".{remote.name}.hlsync-upload-{token}"))
|
|
582
|
+
backup = str(remote.with_name(f".{remote.name}.hlsync-backup-{token}"))
|
|
583
|
+
client = self._connected_client()
|
|
584
|
+
timestamp = datetime.fromtimestamp(
|
|
585
|
+
modified_ns // 1_000_000_000,
|
|
586
|
+
tz=UTC,
|
|
587
|
+
).strftime("%Y%m%d%H%M%S")
|
|
588
|
+
|
|
589
|
+
def discard(candidate: str) -> None:
|
|
590
|
+
try:
|
|
591
|
+
client.delete(candidate)
|
|
592
|
+
except (OSError, ftplib.Error):
|
|
593
|
+
pass
|
|
594
|
+
|
|
595
|
+
stage = "upload staging file (STOR)"
|
|
596
|
+
try:
|
|
597
|
+
callback = (
|
|
598
|
+
(lambda block: byte_progress(len(block)))
|
|
599
|
+
if byte_progress is not None else None
|
|
600
|
+
)
|
|
601
|
+
client.storbinary(f"STOR {temporary}", local_path, callback=callback)
|
|
602
|
+
stage = "verify upload size (SIZE)"
|
|
603
|
+
uploaded_size = client.size(temporary)
|
|
604
|
+
if uploaded_size != size:
|
|
605
|
+
raise TransportError(
|
|
606
|
+
f"remote upload size mismatch for '{path}': "
|
|
607
|
+
f"expected {size}, got {uploaded_size}"
|
|
608
|
+
)
|
|
609
|
+
stage = f"set timestamp ({self._timestamp_command})"
|
|
610
|
+
response = client.sendcmd(
|
|
611
|
+
f"{self._timestamp_command} {timestamp} {temporary}"
|
|
612
|
+
)
|
|
613
|
+
if len(response) < 3 or not response[:3].isdigit() or response[0] != "2":
|
|
614
|
+
raise TransportError(
|
|
615
|
+
f"remote server did not accept modification time for "
|
|
616
|
+
f"'{path}': {response}"
|
|
617
|
+
)
|
|
618
|
+
stage = "verify timestamp (MDTM)"
|
|
619
|
+
verification = client.sendcmd(f"MDTM {temporary}")
|
|
620
|
+
remote_modified_ns, remote_precision_ns = _parse_mdtm_response(
|
|
621
|
+
verification,
|
|
622
|
+
path,
|
|
623
|
+
)
|
|
624
|
+
expected_modified_ns = (
|
|
625
|
+
modified_ns // 1_000_000_000 * 1_000_000_000
|
|
626
|
+
)
|
|
627
|
+
comparison_precision_ns = max(
|
|
628
|
+
remote_precision_ns,
|
|
629
|
+
1_000_000_000,
|
|
630
|
+
)
|
|
631
|
+
if (
|
|
632
|
+
remote_modified_ns // comparison_precision_ns
|
|
633
|
+
!= expected_modified_ns // comparison_precision_ns
|
|
634
|
+
):
|
|
635
|
+
raise TransportError(
|
|
636
|
+
f"remote modification time verification failed for "
|
|
637
|
+
f"'{path}': expected {timestamp}, got "
|
|
638
|
+
f"{verification.partition(' ')[2]}"
|
|
639
|
+
)
|
|
640
|
+
current_source = os.fstat(local_path.fileno())
|
|
641
|
+
if (
|
|
642
|
+
current_source.st_size != size
|
|
643
|
+
or current_source.st_mtime_ns != modified_ns
|
|
644
|
+
):
|
|
645
|
+
raise TransportError(
|
|
646
|
+
f"local source changed while staging remote file '{path}'"
|
|
647
|
+
)
|
|
648
|
+
except TransportError:
|
|
649
|
+
discard(temporary)
|
|
650
|
+
raise
|
|
651
|
+
except ftplib.error_perm as error:
|
|
652
|
+
discard(temporary)
|
|
653
|
+
hint = (
|
|
654
|
+
" Check mdtm_write and file permissions on the FTP server."
|
|
655
|
+
if stage == "set timestamp (MDTM)"
|
|
656
|
+
else ""
|
|
657
|
+
)
|
|
658
|
+
raise PathOperationError(
|
|
659
|
+
f"could not {stage} for '{path}': {error}.{hint}"
|
|
660
|
+
) from error
|
|
661
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
662
|
+
discard(temporary)
|
|
663
|
+
raise TransportError(
|
|
664
|
+
f"could not {stage} for '{path}': {error}"
|
|
665
|
+
) from error
|
|
666
|
+
|
|
667
|
+
if not replace:
|
|
668
|
+
try:
|
|
669
|
+
client.rename(temporary, path)
|
|
670
|
+
except ftplib.error_perm as error:
|
|
671
|
+
discard(temporary)
|
|
672
|
+
raise PathOperationError(
|
|
673
|
+
f"could not install remote file '{path}': {error}"
|
|
674
|
+
) from error
|
|
675
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
676
|
+
discard(temporary)
|
|
677
|
+
raise TransportError(
|
|
678
|
+
f"could not install remote file '{path}': {error}"
|
|
679
|
+
) from error
|
|
680
|
+
return
|
|
681
|
+
|
|
682
|
+
try:
|
|
683
|
+
client.rename(path, backup)
|
|
684
|
+
except ftplib.error_perm as error:
|
|
685
|
+
discard(temporary)
|
|
686
|
+
raise PathOperationError(
|
|
687
|
+
f"could not stage replacement of remote file '{path}': {error}"
|
|
688
|
+
) from error
|
|
689
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
690
|
+
discard(temporary)
|
|
691
|
+
raise TransportError(
|
|
692
|
+
f"could not stage replacement of remote file '{path}': {error}"
|
|
693
|
+
) from error
|
|
694
|
+
try:
|
|
695
|
+
client.rename(temporary, path)
|
|
696
|
+
except ftplib.error_perm as error:
|
|
697
|
+
discard(temporary)
|
|
698
|
+
try:
|
|
699
|
+
client.rename(backup, path)
|
|
700
|
+
except (OSError, ftplib.Error, ssl.SSLError) as restore_error:
|
|
701
|
+
raise TransportError(
|
|
702
|
+
f"could not install remote file '{path}' and could not "
|
|
703
|
+
f"restore its backup '{backup}': {restore_error}"
|
|
704
|
+
) from error
|
|
705
|
+
raise PathOperationError(
|
|
706
|
+
f"could not install remote file '{path}'; its prior version "
|
|
707
|
+
f"was restored: {error}"
|
|
708
|
+
) from error
|
|
709
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
710
|
+
discard(temporary)
|
|
711
|
+
try:
|
|
712
|
+
client.rename(backup, path)
|
|
713
|
+
except (OSError, ftplib.Error, ssl.SSLError) as restore_error:
|
|
714
|
+
raise TransportError(
|
|
715
|
+
f"could not install remote file '{path}' and could not "
|
|
716
|
+
f"restore its backup '{backup}': {restore_error}"
|
|
717
|
+
) from error
|
|
718
|
+
raise TransportError(
|
|
719
|
+
f"could not install remote file '{path}'; its prior version "
|
|
720
|
+
f"was restored: {error}"
|
|
721
|
+
) from error
|
|
722
|
+
try:
|
|
723
|
+
client.delete(backup)
|
|
724
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
725
|
+
raise TransportError(
|
|
726
|
+
f"installed remote file '{path}' but could not remove backup "
|
|
727
|
+
f"'{backup}': {error}"
|
|
728
|
+
) from error
|
|
729
|
+
|
|
730
|
+
def download_file(self, relative_path: str, destination: BinaryIO) -> None:
|
|
731
|
+
path = _relative_remote_path(relative_path)
|
|
732
|
+
try:
|
|
733
|
+
self._connected_client().retrbinary(f"RETR {path}", destination.write)
|
|
734
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
735
|
+
raise TransportError(
|
|
736
|
+
f"could not download remote file '{path}': {error}"
|
|
737
|
+
) from error
|
|
738
|
+
|
|
739
|
+
def delete_path(self, relative_path: str, *, is_directory: bool) -> None:
|
|
740
|
+
path = _relative_remote_path(relative_path)
|
|
741
|
+
try:
|
|
742
|
+
client = self._connected_client()
|
|
743
|
+
if is_directory:
|
|
744
|
+
client.rmd(path)
|
|
745
|
+
else:
|
|
746
|
+
client.delete(path)
|
|
747
|
+
except ftplib.error_perm as error:
|
|
748
|
+
raise PathOperationError(
|
|
749
|
+
f"could not delete remote path '{path}': {error}"
|
|
750
|
+
) from error
|
|
751
|
+
except (OSError, ftplib.Error, ssl.SSLError) as error:
|
|
752
|
+
raise TransportError(
|
|
753
|
+
f"could not delete remote path '{path}': {error}"
|
|
754
|
+
) from error
|
|
755
|
+
|
|
756
|
+
def __enter__(self) -> ExplicitFTPSTransport:
|
|
757
|
+
self.connect()
|
|
758
|
+
return self
|
|
759
|
+
|
|
760
|
+
def __exit__(self, *_: object) -> None:
|
|
761
|
+
self.close()
|