rclone-api 1.5.66__py3-none-any.whl → 1.5.68__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rclone_api/fs/filesystem.py +18 -3
- rclone_api/s3/multipart/upload_parts_resumable.py +24 -5
- {rclone_api-1.5.66.dist-info → rclone_api-1.5.68.dist-info}/METADATA +1 -1
- {rclone_api-1.5.66.dist-info → rclone_api-1.5.68.dist-info}/RECORD +8 -8
- {rclone_api-1.5.66.dist-info → rclone_api-1.5.68.dist-info}/WHEEL +0 -0
- {rclone_api-1.5.66.dist-info → rclone_api-1.5.68.dist-info}/entry_points.txt +0 -0
- {rclone_api-1.5.66.dist-info → rclone_api-1.5.68.dist-info}/licenses/LICENSE +0 -0
- {rclone_api-1.5.66.dist-info → rclone_api-1.5.68.dist-info}/top_level.txt +0 -0
rclone_api/fs/filesystem.py
CHANGED
@@ -230,9 +230,24 @@ class RemoteFS(FS):
|
|
230
230
|
|
231
231
|
|
232
232
|
class FSPath:
|
233
|
-
|
234
|
-
|
235
|
-
|
233
|
+
|
234
|
+
@staticmethod
|
235
|
+
def from_path(path: Path | str) -> "FSPath":
|
236
|
+
return RealFS.from_path(path)
|
237
|
+
|
238
|
+
def __init__(self, fs: FS | Path, path: str | None = None) -> None:
|
239
|
+
self.fs: FS
|
240
|
+
self.path: str
|
241
|
+
if isinstance(fs, Path):
|
242
|
+
real_path = RealFS.from_path(fs)
|
243
|
+
self.fs = real_path.fs
|
244
|
+
self.path = real_path.path
|
245
|
+
else:
|
246
|
+
assert (
|
247
|
+
path is not None
|
248
|
+
), "path must be non None, when not auto converting from Path"
|
249
|
+
self.fs = fs
|
250
|
+
self.path = path
|
236
251
|
self.fs_holder: FS | None = None
|
237
252
|
|
238
253
|
def set_owner(self) -> None:
|
@@ -20,7 +20,7 @@ from rclone_api.types import (
|
|
20
20
|
_LOCK = threading.Lock()
|
21
21
|
|
22
22
|
|
23
|
-
def
|
23
|
+
def _log(msg: str) -> None:
|
24
24
|
print(msg)
|
25
25
|
if os.getenv("LOG_UPLOAD_S3_RESUMABLE") == "1":
|
26
26
|
log_path = Path("log") / "s3_resumable_upload.log"
|
@@ -32,6 +32,17 @@ def _maybe_log(msg: str) -> None:
|
|
32
32
|
f.write("\n")
|
33
33
|
|
34
34
|
|
35
|
+
def _log_completed_item(msg: str) -> None:
|
36
|
+
if os.getenv("LOG_UPLOAD_S3_RESUMABLE") == "1":
|
37
|
+
log_path = Path("log") / "s3_resumable_upload_completed.log"
|
38
|
+
with _LOCK:
|
39
|
+
log_path.parent.mkdir(parents=True, exist_ok=True)
|
40
|
+
# log_path.write_text(msg, append=True)
|
41
|
+
with open(log_path, mode="a", encoding="utf-8") as f:
|
42
|
+
f.write(msg)
|
43
|
+
f.write("\n")
|
44
|
+
|
45
|
+
|
35
46
|
@dataclass
|
36
47
|
class UploadPart:
|
37
48
|
chunk: Path
|
@@ -74,7 +85,7 @@ def upload_task(self: RcloneImpl, upload_part: UploadPart) -> UploadPart:
|
|
74
85
|
msg += f"# Chunk size: {upload_part.chunk.stat().st_size} bytes\n"
|
75
86
|
msg += f"# Range: {upload_part.chunk.name}\n"
|
76
87
|
msg += "##############################################################\n"
|
77
|
-
|
88
|
+
_log(msg)
|
78
89
|
self.copy_to(upload_part.chunk.as_posix(), upload_part.dst_part)
|
79
90
|
return upload_part
|
80
91
|
except Exception as e:
|
@@ -346,7 +357,7 @@ def upload_parts_resumable(
|
|
346
357
|
|
347
358
|
if len(exceptions) > 0:
|
348
359
|
msg = f"Failed to copy parts: {exceptions}"
|
349
|
-
|
360
|
+
_log(msg)
|
350
361
|
return Exception(msg, exceptions)
|
351
362
|
|
352
363
|
finished_parts: list[int] = info_json.fetch_all_finished_part_numbers()
|
@@ -355,7 +366,15 @@ def upload_parts_resumable(
|
|
355
366
|
diff_set = set(all_part_numbers).symmetric_difference(set(finished_parts))
|
356
367
|
all_part_numbers_done = len(diff_set) == 0
|
357
368
|
# print(f"all_part_numbers_done: {all_part_numbers_done}")
|
358
|
-
msg = f"all_part_numbers_done: {all_part_numbers_done}"
|
359
|
-
|
369
|
+
# msg = f"all_part_numbers_done: {all_part_numbers_done}"
|
370
|
+
full_path = os.path.join(dst_dir, src_name)
|
371
|
+
if all_part_numbers_done:
|
372
|
+
msg = f"Upload completed: {full_path} ({len(finished_parts)}/{len(all_part_numbers)})"
|
373
|
+
_log(msg)
|
374
|
+
info_json.save()
|
375
|
+
else:
|
376
|
+
msg = f"Upload failed for {full_path} ({len(finished_parts)}/{len(all_part_numbers)})"
|
377
|
+
_log(msg)
|
378
|
+
_log_completed_item(msg)
|
360
379
|
|
361
380
|
return None
|
@@ -40,7 +40,7 @@ rclone_api/detail/copy_file_parts_resumable.py,sha256=RoUWV2eBWEvuuTfsvrz5BhtvX3
|
|
40
40
|
rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
|
41
41
|
rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
|
42
42
|
rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
|
43
|
-
rclone_api/fs/filesystem.py,sha256=
|
43
|
+
rclone_api/fs/filesystem.py,sha256=F7hq-pV-TSGZKmkJczST5aSXwxIq8wlWTjRaj1wbupY,11685
|
44
44
|
rclone_api/fs/walk.py,sha256=Yqn_SS25gTL0ANWSQxnAdN-ELBO1PMZfYYnyxoGafWA,875
|
45
45
|
rclone_api/fs/walk_threaded.py,sha256=zlbooQ7WbteUK1cCjm1Yj66VpDzS8Qm3DcHBSz1Gbjk,1591
|
46
46
|
rclone_api/fs/walk_threaded_walker.py,sha256=xi0QkmmzI35NcaA08FLjGe7ox9tLZcgXtOZCT8nMuWU,718
|
@@ -55,12 +55,12 @@ rclone_api/s3/multipart/info_json.py,sha256=-e8UCwrqjAP64U8PmH-o2ciJ6TN48DwHktJf
|
|
55
55
|
rclone_api/s3/multipart/merge_state.py,sha256=ziTB9CYV-OWaky5C1fOT9hifSY2zgUrk5HmX1Xeu2UA,4978
|
56
56
|
rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
|
57
57
|
rclone_api/s3/multipart/upload_parts_inline.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
|
58
|
-
rclone_api/s3/multipart/upload_parts_resumable.py,sha256=
|
58
|
+
rclone_api/s3/multipart/upload_parts_resumable.py,sha256=tdOkEfoK9zENaTfgVyTWB5GNqn-IYl-zJn-rJLcoVTo,12388
|
59
59
|
rclone_api/s3/multipart/upload_parts_server_side_merge.py,sha256=Fp2pdrs5dONQI9LkfNolgAGj1-Z2V1SsRd0r0sreuXI,18040
|
60
60
|
rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
|
61
|
-
rclone_api-1.5.
|
62
|
-
rclone_api-1.5.
|
63
|
-
rclone_api-1.5.
|
64
|
-
rclone_api-1.5.
|
65
|
-
rclone_api-1.5.
|
66
|
-
rclone_api-1.5.
|
61
|
+
rclone_api-1.5.68.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
|
62
|
+
rclone_api-1.5.68.dist-info/METADATA,sha256=I_SjMm5849WPDwolAnXqXFH3rDAKPW4nUrBMnIw_0EI,37305
|
63
|
+
rclone_api-1.5.68.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
|
64
|
+
rclone_api-1.5.68.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
|
65
|
+
rclone_api-1.5.68.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
|
66
|
+
rclone_api-1.5.68.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|