rclone-api 1.2.13__py2.py3-none-any.whl → 1.2.154__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rclone_api/__init__.py +45 -38
- rclone_api/logging.py +39 -0
- rclone_api/mount.py +9 -111
- rclone_api/mount_read_chunker.py +123 -0
- rclone_api/profile/mount_copy_bytes.py +1 -1
- rclone_api/rclone.py +2 -1
- rclone_api/s3/chunk_file.py +146 -145
- rclone_api/s3/types.py +1 -1
- rclone_api/s3/upload_file_multipart.py +1 -1
- {rclone_api-1.2.13.dist-info → rclone_api-1.2.154.dist-info}/METADATA +1 -1
- {rclone_api-1.2.13.dist-info → rclone_api-1.2.154.dist-info}/RECORD +15 -13
- {rclone_api-1.2.13.dist-info → rclone_api-1.2.154.dist-info}/LICENSE +0 -0
- {rclone_api-1.2.13.dist-info → rclone_api-1.2.154.dist-info}/WHEEL +0 -0
- {rclone_api-1.2.13.dist-info → rclone_api-1.2.154.dist-info}/entry_points.txt +0 -0
- {rclone_api-1.2.13.dist-info → rclone_api-1.2.154.dist-info}/top_level.txt +0 -0
rclone_api/__init__.py
CHANGED
|
@@ -1,38 +1,45 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
from .
|
|
5
|
-
from .
|
|
6
|
-
from .
|
|
7
|
-
from .
|
|
8
|
-
from .
|
|
9
|
-
from .
|
|
10
|
-
from .
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
from .
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
"
|
|
25
|
-
"
|
|
26
|
-
"
|
|
27
|
-
"
|
|
28
|
-
"
|
|
29
|
-
"
|
|
30
|
-
"
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"
|
|
38
|
-
|
|
1
|
+
# Import logging module to activate default configuration
|
|
2
|
+
import rclone_api.logging # noqa: F401
|
|
3
|
+
|
|
4
|
+
from .completed_process import CompletedProcess
|
|
5
|
+
from .config import Config, Parsed, Section
|
|
6
|
+
from .diff import DiffItem, DiffOption, DiffType
|
|
7
|
+
from .dir import Dir
|
|
8
|
+
from .dir_listing import DirListing
|
|
9
|
+
from .file import File
|
|
10
|
+
from .filelist import FileList
|
|
11
|
+
|
|
12
|
+
# Import the configure_logging function to make it available at package level
|
|
13
|
+
from .logging import configure_logging
|
|
14
|
+
from .process import Process
|
|
15
|
+
from .rclone import Rclone, rclone_verbose
|
|
16
|
+
from .remote import Remote
|
|
17
|
+
from .rpath import RPath
|
|
18
|
+
from .s3.types import MultiUploadResult
|
|
19
|
+
from .types import ListingOption, Order, SizeResult, SizeSuffix
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"Rclone",
|
|
23
|
+
"File",
|
|
24
|
+
"Config",
|
|
25
|
+
"Remote",
|
|
26
|
+
"Dir",
|
|
27
|
+
"RPath",
|
|
28
|
+
"DirListing",
|
|
29
|
+
"FileList",
|
|
30
|
+
"Process",
|
|
31
|
+
"DiffItem",
|
|
32
|
+
"DiffType",
|
|
33
|
+
"rclone_verbose",
|
|
34
|
+
"CompletedProcess",
|
|
35
|
+
"DiffOption",
|
|
36
|
+
"ListingOption",
|
|
37
|
+
"Order",
|
|
38
|
+
"ListingOption",
|
|
39
|
+
"SizeResult",
|
|
40
|
+
"Parsed",
|
|
41
|
+
"Section",
|
|
42
|
+
"MultiUploadResult",
|
|
43
|
+
"SizeSuffix",
|
|
44
|
+
"configure_logging",
|
|
45
|
+
]
|
rclone_api/logging.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def setup_default_logging():
|
|
6
|
+
"""Set up default logging configuration if none exists."""
|
|
7
|
+
if not logging.root.handlers:
|
|
8
|
+
logging.basicConfig(
|
|
9
|
+
level=logging.INFO,
|
|
10
|
+
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
|
11
|
+
handlers=[
|
|
12
|
+
logging.StreamHandler(sys.stdout),
|
|
13
|
+
# Uncomment to add file logging
|
|
14
|
+
# logging.FileHandler('rclone_api.log')
|
|
15
|
+
],
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def configure_logging(level=logging.INFO, log_file=None):
|
|
20
|
+
"""Configure logging for the rclone_api package.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
level: The logging level (default: logging.INFO)
|
|
24
|
+
log_file: Optional path to a log file
|
|
25
|
+
"""
|
|
26
|
+
handlers = [logging.StreamHandler(sys.stdout)]
|
|
27
|
+
if log_file:
|
|
28
|
+
handlers.append(logging.FileHandler(log_file))
|
|
29
|
+
|
|
30
|
+
logging.basicConfig(
|
|
31
|
+
level=level,
|
|
32
|
+
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
|
33
|
+
handlers=handlers,
|
|
34
|
+
force=True, # Override any existing configuration
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# Call setup_default_logging when this module is imported
|
|
39
|
+
setup_default_logging()
|
rclone_api/mount.py
CHANGED
|
@@ -4,17 +4,14 @@ import platform
|
|
|
4
4
|
import shutil
|
|
5
5
|
import subprocess
|
|
6
6
|
import time
|
|
7
|
-
import traceback
|
|
8
7
|
import warnings
|
|
9
8
|
import weakref
|
|
10
|
-
from concurrent.futures import
|
|
9
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
11
10
|
from dataclasses import dataclass
|
|
12
11
|
from pathlib import Path
|
|
13
|
-
from threading import Lock, Semaphore
|
|
14
12
|
from typing import Any
|
|
15
13
|
|
|
16
14
|
from rclone_api.process import Process
|
|
17
|
-
from rclone_api.types import FilePart
|
|
18
15
|
|
|
19
16
|
_SYSTEM = platform.system() # "Linux", "Darwin", "Windows", etc.
|
|
20
17
|
|
|
@@ -37,6 +34,14 @@ def _cleanup_mounts() -> None:
|
|
|
37
34
|
executor.submit(mount.close)
|
|
38
35
|
|
|
39
36
|
|
|
37
|
+
def _cache_dir_delete_on_exit(cache_dir: Path) -> None:
|
|
38
|
+
if cache_dir.exists():
|
|
39
|
+
try:
|
|
40
|
+
shutil.rmtree(cache_dir)
|
|
41
|
+
except Exception as e:
|
|
42
|
+
warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
|
|
43
|
+
|
|
44
|
+
|
|
40
45
|
atexit.register(_cleanup_mounts)
|
|
41
46
|
|
|
42
47
|
|
|
@@ -278,110 +283,3 @@ def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
|
|
|
278
283
|
else:
|
|
279
284
|
if verbose:
|
|
280
285
|
print(f"{mount_path} successfully cleaned up.")
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
def _cache_dir_delete_on_exit(cache_dir: Path) -> None:
|
|
284
|
-
if cache_dir.exists():
|
|
285
|
-
try:
|
|
286
|
-
shutil.rmtree(cache_dir)
|
|
287
|
-
except Exception as e:
|
|
288
|
-
warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
def _read_from_mount_task(
|
|
292
|
-
offset: int, size: int, path: Path, verbose: bool
|
|
293
|
-
) -> bytes | Exception:
|
|
294
|
-
if verbose or True:
|
|
295
|
-
print(f"Fetching chunk: offset={offset}, size={size}, path={path}")
|
|
296
|
-
try:
|
|
297
|
-
with path.open("rb") as f:
|
|
298
|
-
f.seek(offset)
|
|
299
|
-
payload = f.read(size)
|
|
300
|
-
assert len(payload) == size, f"Invalid read size: {len(payload)}"
|
|
301
|
-
return payload
|
|
302
|
-
|
|
303
|
-
except KeyboardInterrupt as e:
|
|
304
|
-
import _thread
|
|
305
|
-
|
|
306
|
-
_thread.interrupt_main()
|
|
307
|
-
return Exception(e)
|
|
308
|
-
except Exception as e:
|
|
309
|
-
stack_trace = traceback.format_exc()
|
|
310
|
-
warnings.warn(
|
|
311
|
-
f"Error fetching file chunk at offset {offset} + {size}: {e}\n{stack_trace}"
|
|
312
|
-
)
|
|
313
|
-
return e
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
class MultiMountFileChunker:
|
|
317
|
-
def __init__(
|
|
318
|
-
self,
|
|
319
|
-
filename: str,
|
|
320
|
-
filesize: int,
|
|
321
|
-
mounts: list[Mount],
|
|
322
|
-
executor: ThreadPoolExecutor,
|
|
323
|
-
verbose: bool | None,
|
|
324
|
-
) -> None:
|
|
325
|
-
from rclone_api.util import get_verbose
|
|
326
|
-
|
|
327
|
-
self.filename = filename
|
|
328
|
-
self.filesize = filesize
|
|
329
|
-
self.executor = executor
|
|
330
|
-
self.mounts_processing: list[Mount] = []
|
|
331
|
-
self.mounts_availabe: list[Mount] = mounts
|
|
332
|
-
self.semaphore = Semaphore(len(mounts))
|
|
333
|
-
self.lock = Lock()
|
|
334
|
-
self.verbose = get_verbose(verbose)
|
|
335
|
-
|
|
336
|
-
def shutdown(self) -> None:
|
|
337
|
-
self.executor.shutdown(wait=True, cancel_futures=True)
|
|
338
|
-
with ThreadPoolExecutor() as executor:
|
|
339
|
-
for mount in self.mounts_processing:
|
|
340
|
-
executor.submit(lambda: mount.close())
|
|
341
|
-
|
|
342
|
-
def _acquire_mount(self) -> Mount:
|
|
343
|
-
self.semaphore.acquire()
|
|
344
|
-
with self.lock:
|
|
345
|
-
mount = self.mounts_availabe.pop()
|
|
346
|
-
self.mounts_processing.append(mount)
|
|
347
|
-
return mount
|
|
348
|
-
|
|
349
|
-
def _release_mount(self, mount: Mount) -> None:
|
|
350
|
-
with self.lock:
|
|
351
|
-
self.mounts_processing.remove(mount)
|
|
352
|
-
self.mounts_availabe.append(mount)
|
|
353
|
-
self.semaphore.release()
|
|
354
|
-
|
|
355
|
-
def fetch(self, offset: int, size: int, extra: Any) -> Future[FilePart]:
|
|
356
|
-
if self.verbose:
|
|
357
|
-
print(f"Fetching data range: offset={offset}, size={size}")
|
|
358
|
-
|
|
359
|
-
assert size > 0, f"Invalid size: {size}"
|
|
360
|
-
assert offset >= 0, f"Invalid offset: {offset}"
|
|
361
|
-
assert (
|
|
362
|
-
offset + size <= self.filesize
|
|
363
|
-
), f"Invalid offset + size: {offset} + {size} ({offset+size}) <= {self.filesize}"
|
|
364
|
-
|
|
365
|
-
try:
|
|
366
|
-
mount = self._acquire_mount()
|
|
367
|
-
path = mount.mount_path / self.filename
|
|
368
|
-
|
|
369
|
-
def task_fetch_file_range(
|
|
370
|
-
size=size, path=path, mount=mount, verbose=self.verbose
|
|
371
|
-
) -> FilePart:
|
|
372
|
-
bytes_or_err = _read_from_mount_task(
|
|
373
|
-
offset=offset, size=size, path=path, verbose=verbose
|
|
374
|
-
)
|
|
375
|
-
self._release_mount(mount)
|
|
376
|
-
|
|
377
|
-
if isinstance(bytes_or_err, Exception):
|
|
378
|
-
return FilePart(payload=bytes_or_err, extra=extra)
|
|
379
|
-
out = FilePart(payload=bytes_or_err, extra=extra)
|
|
380
|
-
return out
|
|
381
|
-
|
|
382
|
-
fut = self.executor.submit(task_fetch_file_range)
|
|
383
|
-
return fut
|
|
384
|
-
except Exception as e:
|
|
385
|
-
warnings.warn(f"Error fetching file chunk: {e}")
|
|
386
|
-
fp = FilePart(payload=e, extra=extra)
|
|
387
|
-
return self.executor.submit(lambda: fp)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import traceback
|
|
3
|
+
from concurrent.futures import Future, ThreadPoolExecutor
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from threading import Lock, Semaphore
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from rclone_api.mount import Mount
|
|
9
|
+
from rclone_api.types import FilePart
|
|
10
|
+
|
|
11
|
+
# Create a logger for this module
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _read_from_mount_task(
|
|
16
|
+
offset: int, size: int, path: Path, verbose: bool
|
|
17
|
+
) -> bytes | Exception:
|
|
18
|
+
if verbose:
|
|
19
|
+
logger.debug(f"Fetching chunk: offset={offset}, size={size}, path={path}")
|
|
20
|
+
try:
|
|
21
|
+
with path.open("rb") as f:
|
|
22
|
+
f.seek(offset)
|
|
23
|
+
payload = f.read(size)
|
|
24
|
+
assert len(payload) == size, f"Invalid read size: {len(payload)}"
|
|
25
|
+
return payload
|
|
26
|
+
|
|
27
|
+
except KeyboardInterrupt as e:
|
|
28
|
+
import _thread
|
|
29
|
+
|
|
30
|
+
logger.error("KeyboardInterrupt received during chunk read")
|
|
31
|
+
_thread.interrupt_main()
|
|
32
|
+
return Exception(e)
|
|
33
|
+
except Exception as e:
|
|
34
|
+
stack_trace = traceback.format_exc()
|
|
35
|
+
logger.error(
|
|
36
|
+
f"Error fetching file chunk at offset {offset} + {size}: {e}\n{stack_trace}"
|
|
37
|
+
)
|
|
38
|
+
return e
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class MultiMountFileChunker:
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
filename: str,
|
|
45
|
+
filesize: int,
|
|
46
|
+
mounts: list[Mount],
|
|
47
|
+
executor: ThreadPoolExecutor,
|
|
48
|
+
verbose: bool | None,
|
|
49
|
+
) -> None:
|
|
50
|
+
from rclone_api.util import get_verbose
|
|
51
|
+
|
|
52
|
+
self.filename = filename
|
|
53
|
+
self.filesize = filesize
|
|
54
|
+
self.executor = executor
|
|
55
|
+
self.mounts_processing: list[Mount] = []
|
|
56
|
+
self.mounts_availabe: list[Mount] = mounts
|
|
57
|
+
self.semaphore = Semaphore(len(mounts))
|
|
58
|
+
self.lock = Lock()
|
|
59
|
+
self.verbose = get_verbose(verbose)
|
|
60
|
+
logger.info(
|
|
61
|
+
f"Initialized MultiMountFileChunker for {filename} ({filesize} bytes)"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def shutdown(self) -> None:
|
|
65
|
+
logger.info("Shutting down MultiMountFileChunker")
|
|
66
|
+
self.executor.shutdown(wait=True, cancel_futures=True)
|
|
67
|
+
with ThreadPoolExecutor() as executor:
|
|
68
|
+
for mount in self.mounts_processing:
|
|
69
|
+
executor.submit(lambda: mount.close())
|
|
70
|
+
logger.debug("MultiMountFileChunker shutdown complete")
|
|
71
|
+
|
|
72
|
+
def _acquire_mount(self) -> Mount:
|
|
73
|
+
logger.debug("Acquiring mount")
|
|
74
|
+
self.semaphore.acquire()
|
|
75
|
+
with self.lock:
|
|
76
|
+
mount = self.mounts_availabe.pop()
|
|
77
|
+
self.mounts_processing.append(mount)
|
|
78
|
+
logger.debug(f"Mount acquired: {mount}")
|
|
79
|
+
return mount
|
|
80
|
+
|
|
81
|
+
def _release_mount(self, mount: Mount) -> None:
|
|
82
|
+
logger.debug(f"Releasing mount: {mount}")
|
|
83
|
+
with self.lock:
|
|
84
|
+
self.mounts_processing.remove(mount)
|
|
85
|
+
self.mounts_availabe.append(mount)
|
|
86
|
+
self.semaphore.release()
|
|
87
|
+
logger.debug("Mount released")
|
|
88
|
+
|
|
89
|
+
def fetch(self, offset: int, size: int, extra: Any) -> Future[FilePart]:
|
|
90
|
+
if self.verbose:
|
|
91
|
+
logger.debug(f"Fetching data range: offset={offset}, size={size}")
|
|
92
|
+
|
|
93
|
+
assert size > 0, f"Invalid size: {size}"
|
|
94
|
+
assert offset >= 0, f"Invalid offset: {offset}"
|
|
95
|
+
assert (
|
|
96
|
+
offset + size <= self.filesize
|
|
97
|
+
), f"Invalid offset + size: {offset} + {size} ({offset+size}) <= {self.filesize}"
|
|
98
|
+
|
|
99
|
+
try:
|
|
100
|
+
mount = self._acquire_mount()
|
|
101
|
+
path = mount.mount_path / self.filename
|
|
102
|
+
|
|
103
|
+
def task_fetch_file_range(
|
|
104
|
+
size=size, path=path, mount=mount, verbose=self.verbose
|
|
105
|
+
) -> FilePart:
|
|
106
|
+
bytes_or_err = _read_from_mount_task(
|
|
107
|
+
offset=offset, size=size, path=path, verbose=verbose
|
|
108
|
+
)
|
|
109
|
+
self._release_mount(mount)
|
|
110
|
+
|
|
111
|
+
if isinstance(bytes_or_err, Exception):
|
|
112
|
+
logger.warning(f"Fetch task returned exception: {bytes_or_err}")
|
|
113
|
+
return FilePart(payload=bytes_or_err, extra=extra)
|
|
114
|
+
logger.debug(f"Successfully fetched {size} bytes from offset {offset}")
|
|
115
|
+
out = FilePart(payload=bytes_or_err, extra=extra)
|
|
116
|
+
return out
|
|
117
|
+
|
|
118
|
+
fut = self.executor.submit(task_fetch_file_range)
|
|
119
|
+
return fut
|
|
120
|
+
except Exception as e:
|
|
121
|
+
logger.error(f"Error setting up file chunk fetch: {e}", exc_info=True)
|
|
122
|
+
fp = FilePart(payload=e, extra=extra)
|
|
123
|
+
return self.executor.submit(lambda: fp)
|
|
@@ -14,7 +14,7 @@ import psutil
|
|
|
14
14
|
from dotenv import load_dotenv
|
|
15
15
|
|
|
16
16
|
from rclone_api import Config, Rclone, SizeSuffix
|
|
17
|
-
from rclone_api.
|
|
17
|
+
from rclone_api.mount_read_chunker import MultiMountFileChunker
|
|
18
18
|
from rclone_api.types import FilePart
|
|
19
19
|
|
|
20
20
|
os.environ["RCLONE_API_VERBOSE"] = "1"
|
rclone_api/rclone.py
CHANGED
|
@@ -26,7 +26,8 @@ from rclone_api.dir_listing import DirListing
|
|
|
26
26
|
from rclone_api.exec import RcloneExec
|
|
27
27
|
from rclone_api.file import File
|
|
28
28
|
from rclone_api.group_files import group_files
|
|
29
|
-
from rclone_api.mount import Mount,
|
|
29
|
+
from rclone_api.mount import Mount, clean_mount, prepare_mount
|
|
30
|
+
from rclone_api.mount_read_chunker import MultiMountFileChunker
|
|
30
31
|
from rclone_api.process import Process
|
|
31
32
|
from rclone_api.remote import Remote
|
|
32
33
|
from rclone_api.rpath import RPath
|
rclone_api/s3/chunk_file.py
CHANGED
|
@@ -1,145 +1,146 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
from concurrent.futures import Future
|
|
4
|
-
from dataclasses import dataclass
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from queue import Queue
|
|
7
|
-
from threading import Event
|
|
8
|
-
from typing import Any, Callable
|
|
9
|
-
|
|
10
|
-
from rclone_api.
|
|
11
|
-
from rclone_api.s3.chunk_types import UploadState
|
|
12
|
-
from rclone_api.types import EndOfStream
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
#
|
|
101
|
-
#
|
|
102
|
-
#
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
1
|
+
import logging
|
|
2
|
+
import time
|
|
3
|
+
from concurrent.futures import Future
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from queue import Queue
|
|
7
|
+
from threading import Event
|
|
8
|
+
from typing import Any, Callable
|
|
9
|
+
|
|
10
|
+
from rclone_api.mount_read_chunker import FilePart
|
|
11
|
+
from rclone_api.s3.chunk_types import UploadState
|
|
12
|
+
from rclone_api.types import EndOfStream
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__) # noqa
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _get_file_size(file_path: Path, timeout: int = 60) -> int:
|
|
18
|
+
sleep_time = timeout / 60 if timeout > 0 else 1
|
|
19
|
+
start = time.time()
|
|
20
|
+
while True:
|
|
21
|
+
expired = time.time() - start > timeout
|
|
22
|
+
try:
|
|
23
|
+
time.sleep(sleep_time)
|
|
24
|
+
if file_path.exists():
|
|
25
|
+
return file_path.stat().st_size
|
|
26
|
+
except FileNotFoundError as e:
|
|
27
|
+
if expired:
|
|
28
|
+
print(f"File not found: {file_path}, exception is {e}")
|
|
29
|
+
raise
|
|
30
|
+
if expired:
|
|
31
|
+
raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class S3FileInfo:
|
|
36
|
+
upload_id: str
|
|
37
|
+
part_number: int
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def file_chunker(
|
|
41
|
+
upload_state: UploadState,
|
|
42
|
+
fetcher: Callable[[int, int, Any], Future[FilePart]],
|
|
43
|
+
max_chunks: int | None,
|
|
44
|
+
cancel_signal: Event,
|
|
45
|
+
queue_upload: Queue[FilePart | EndOfStream],
|
|
46
|
+
) -> None:
|
|
47
|
+
count = 0
|
|
48
|
+
|
|
49
|
+
def should_stop() -> bool:
|
|
50
|
+
nonlocal count
|
|
51
|
+
|
|
52
|
+
if max_chunks is None:
|
|
53
|
+
return False
|
|
54
|
+
if count >= max_chunks:
|
|
55
|
+
logger.info(
|
|
56
|
+
f"Stopping file chunker after {count} chunks because it exceeded max_chunks {max_chunks}"
|
|
57
|
+
)
|
|
58
|
+
return True
|
|
59
|
+
count += 1
|
|
60
|
+
return False
|
|
61
|
+
|
|
62
|
+
upload_info = upload_state.upload_info
|
|
63
|
+
file_path = upload_info.src_file_path
|
|
64
|
+
chunk_size = upload_info.chunk_size
|
|
65
|
+
# src = Path(file_path)
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
part_number = 1
|
|
69
|
+
done_part_numbers: set[int] = {
|
|
70
|
+
p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
|
|
71
|
+
}
|
|
72
|
+
num_parts = upload_info.total_chunks()
|
|
73
|
+
|
|
74
|
+
def next_part_number() -> int | None:
|
|
75
|
+
nonlocal part_number
|
|
76
|
+
while part_number in done_part_numbers:
|
|
77
|
+
part_number += 1
|
|
78
|
+
if part_number > num_parts:
|
|
79
|
+
return None
|
|
80
|
+
return part_number
|
|
81
|
+
|
|
82
|
+
if cancel_signal.is_set():
|
|
83
|
+
logger.info(
|
|
84
|
+
f"Cancel signal is set for file chunker while processing {file_path}, returning"
|
|
85
|
+
)
|
|
86
|
+
return
|
|
87
|
+
|
|
88
|
+
while not should_stop():
|
|
89
|
+
logger.debug("Processing next chunk")
|
|
90
|
+
curr_part_number = next_part_number()
|
|
91
|
+
if curr_part_number is None:
|
|
92
|
+
logger.info(f"File {file_path} has completed chunking all parts")
|
|
93
|
+
break
|
|
94
|
+
assert curr_part_number is not None
|
|
95
|
+
offset = (curr_part_number - 1) * chunk_size
|
|
96
|
+
file_size = upload_info.file_size
|
|
97
|
+
|
|
98
|
+
assert offset < file_size, f"Offset {offset} is greater than file size"
|
|
99
|
+
|
|
100
|
+
# Open the file, seek, read the chunk, and close immediately.
|
|
101
|
+
# with open(file_path, "rb") as f:
|
|
102
|
+
# f.seek(offset)
|
|
103
|
+
# data = f.read(chunk_size)
|
|
104
|
+
|
|
105
|
+
# data = chunk_fetcher(offset, chunk_size).result()
|
|
106
|
+
|
|
107
|
+
assert curr_part_number is not None
|
|
108
|
+
cpn: int = curr_part_number
|
|
109
|
+
|
|
110
|
+
def on_complete(fut: Future[FilePart]) -> None:
|
|
111
|
+
logger.debug("Chunk read complete")
|
|
112
|
+
fp: FilePart = fut.result()
|
|
113
|
+
if fp.is_error():
|
|
114
|
+
logger.warning(
|
|
115
|
+
f"Error reading file: {fp}, skipping part {part_number}"
|
|
116
|
+
)
|
|
117
|
+
return
|
|
118
|
+
|
|
119
|
+
if fp.n_bytes() == 0:
|
|
120
|
+
logger.warning(f"Empty data for part {part_number} of {file_path}")
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f"Empty data for part {part_number} of {file_path}"
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
if isinstance(fp.payload, Exception):
|
|
126
|
+
logger.warning(f"Error reading file because of error: {fp.payload}")
|
|
127
|
+
return
|
|
128
|
+
|
|
129
|
+
done_part_numbers.add(part_number)
|
|
130
|
+
queue_upload.put(fp)
|
|
131
|
+
|
|
132
|
+
offset = (curr_part_number - 1) * chunk_size
|
|
133
|
+
logger.info(
|
|
134
|
+
f"Reading chunk {curr_part_number} of {num_parts} for {file_path}"
|
|
135
|
+
)
|
|
136
|
+
fut = fetcher(offset, file_size, S3FileInfo(upload_info.upload_id, cpn))
|
|
137
|
+
fut.add_done_callback(on_complete)
|
|
138
|
+
# wait until the queue_upload queue can accept the next chunk
|
|
139
|
+
while queue_upload.full():
|
|
140
|
+
time.sleep(0.1)
|
|
141
|
+
except Exception as e:
|
|
142
|
+
|
|
143
|
+
logger.error(f"Error reading file: {e}", exc_info=True)
|
|
144
|
+
finally:
|
|
145
|
+
logger.info(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
|
|
146
|
+
queue_upload.put(EndOfStream())
|
rclone_api/s3/types.py
CHANGED
|
@@ -10,7 +10,7 @@ from typing import Any, Callable
|
|
|
10
10
|
|
|
11
11
|
from botocore.client import BaseClient
|
|
12
12
|
|
|
13
|
-
from rclone_api.
|
|
13
|
+
from rclone_api.mount_read_chunker import FilePart
|
|
14
14
|
from rclone_api.s3.chunk_file import S3FileInfo, file_chunker
|
|
15
15
|
from rclone_api.s3.chunk_types import (
|
|
16
16
|
FinishedPiece,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
rclone_api/__init__.py,sha256=
|
|
1
|
+
rclone_api/__init__.py,sha256=dPa4vtqtaO47JZa-YUjOgVa2gCSHLdGxM-_xX_4C7Yw,1150
|
|
2
2
|
rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
|
|
3
3
|
rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
|
|
4
4
|
rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
|
|
@@ -11,9 +11,11 @@ rclone_api/exec.py,sha256=Pd7pUBd8ib5MzqvMybG2DQISPRbDRu20VjVRL2mLAVY,1076
|
|
|
11
11
|
rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
|
|
12
12
|
rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
|
|
13
13
|
rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
|
|
14
|
-
rclone_api/
|
|
14
|
+
rclone_api/logging.py,sha256=fJ4Hr4baAEv93oOOiyzNfoQ8eD0MuErT3NHMjBC3W_w,1184
|
|
15
|
+
rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
|
|
16
|
+
rclone_api/mount_read_chunker.py,sha256=IH2YcB-N22oiJLkp7KlpG3A4VZkkHOqTYvDLyor2e7Q,4505
|
|
15
17
|
rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
|
|
16
|
-
rclone_api/rclone.py,sha256=
|
|
18
|
+
rclone_api/rclone.py,sha256=WSCbIo4NtD871nvR1ZG9v7ihbLi7z1Ku4Pg4mgrNxZw,49364
|
|
17
19
|
rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
|
|
18
20
|
rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
|
|
19
21
|
rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
|
|
@@ -25,17 +27,17 @@ rclone_api/cmd/copy_large_s3.py,sha256=fYHyHq2YZT_dfMbS7SCpEeLCaWD-BU-jcpKP9eKf1
|
|
|
25
27
|
rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
|
|
26
28
|
rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
|
|
27
29
|
rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
|
|
28
|
-
rclone_api/profile/mount_copy_bytes.py,sha256=
|
|
30
|
+
rclone_api/profile/mount_copy_bytes.py,sha256=okzcfpmLcQvh5IUcIwZs9jLPSxFMv2igt2-kHoEmlfE,8571
|
|
29
31
|
rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
|
|
30
32
|
rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
|
|
31
|
-
rclone_api/s3/chunk_file.py,sha256=
|
|
33
|
+
rclone_api/s3/chunk_file.py,sha256=xtg9g4BvaFsipyfj6p5iRitR53jXjBqX0tmtO7Vf3Us,5068
|
|
32
34
|
rclone_api/s3/chunk_types.py,sha256=I0YCWFgxCvmt8cp4tMabiiwiD2yKTcbA6ZL2D3xnn5w,8781
|
|
33
35
|
rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
|
|
34
|
-
rclone_api/s3/types.py,sha256=
|
|
35
|
-
rclone_api/s3/upload_file_multipart.py,sha256=
|
|
36
|
-
rclone_api-1.2.
|
|
37
|
-
rclone_api-1.2.
|
|
38
|
-
rclone_api-1.2.
|
|
39
|
-
rclone_api-1.2.
|
|
40
|
-
rclone_api-1.2.
|
|
41
|
-
rclone_api-1.2.
|
|
36
|
+
rclone_api/s3/types.py,sha256=Elmh__gvZJyJyElYwMmvYZIBIunDJiTRAbEg21GmsRU,1604
|
|
37
|
+
rclone_api/s3/upload_file_multipart.py,sha256=inoMOQDZZYqTitJz3f0BBHo3F9ZYm8VhL4UTzPmcdm0,11385
|
|
38
|
+
rclone_api-1.2.154.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
|
|
39
|
+
rclone_api-1.2.154.dist-info/METADATA,sha256=RTR14SPQNbWbnFKlWuMGj-y3CdZ4tU3x70odXfdXJVQ,4538
|
|
40
|
+
rclone_api-1.2.154.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
|
|
41
|
+
rclone_api-1.2.154.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
|
|
42
|
+
rclone_api-1.2.154.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
|
|
43
|
+
rclone_api-1.2.154.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|