rclone-api 1.2.13__tar.gz → 1.2.154__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {rclone_api-1.2.13 → rclone_api-1.2.154}/PKG-INFO +1 -1
  2. {rclone_api-1.2.13 → rclone_api-1.2.154}/pyproject.toml +1 -1
  3. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/__init__.py +45 -38
  4. rclone_api-1.2.154/src/rclone_api/logging.py +39 -0
  5. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/mount.py +9 -111
  6. rclone_api-1.2.154/src/rclone_api/mount_read_chunker.py +123 -0
  7. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/profile/mount_copy_bytes.py +1 -1
  8. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/rclone.py +2 -1
  9. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/s3/chunk_file.py +146 -145
  10. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/s3/types.py +1 -1
  11. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/s3/upload_file_multipart.py +1 -1
  12. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api.egg-info/PKG-INFO +1 -1
  13. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api.egg-info/SOURCES.txt +2 -0
  14. {rclone_api-1.2.13 → rclone_api-1.2.154}/.aiderignore +0 -0
  15. {rclone_api-1.2.13 → rclone_api-1.2.154}/.github/workflows/lint.yml +0 -0
  16. {rclone_api-1.2.13 → rclone_api-1.2.154}/.github/workflows/push_macos.yml +0 -0
  17. {rclone_api-1.2.13 → rclone_api-1.2.154}/.github/workflows/push_ubuntu.yml +0 -0
  18. {rclone_api-1.2.13 → rclone_api-1.2.154}/.github/workflows/push_win.yml +0 -0
  19. {rclone_api-1.2.13 → rclone_api-1.2.154}/.gitignore +0 -0
  20. {rclone_api-1.2.13 → rclone_api-1.2.154}/.pylintrc +0 -0
  21. {rclone_api-1.2.13 → rclone_api-1.2.154}/.vscode/launch.json +0 -0
  22. {rclone_api-1.2.13 → rclone_api-1.2.154}/.vscode/settings.json +0 -0
  23. {rclone_api-1.2.13 → rclone_api-1.2.154}/.vscode/tasks.json +0 -0
  24. {rclone_api-1.2.13 → rclone_api-1.2.154}/LICENSE +0 -0
  25. {rclone_api-1.2.13 → rclone_api-1.2.154}/MANIFEST.in +0 -0
  26. {rclone_api-1.2.13 → rclone_api-1.2.154}/README.md +0 -0
  27. {rclone_api-1.2.13 → rclone_api-1.2.154}/clean +0 -0
  28. {rclone_api-1.2.13 → rclone_api-1.2.154}/install +0 -0
  29. {rclone_api-1.2.13 → rclone_api-1.2.154}/lint +0 -0
  30. {rclone_api-1.2.13 → rclone_api-1.2.154}/requirements.testing.txt +0 -0
  31. {rclone_api-1.2.13 → rclone_api-1.2.154}/setup.cfg +0 -0
  32. {rclone_api-1.2.13 → rclone_api-1.2.154}/setup.py +0 -0
  33. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/assets/example.txt +0 -0
  34. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/cli.py +0 -0
  35. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/cmd/copy_large_s3.py +0 -0
  36. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/cmd/list_files.py +0 -0
  37. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/completed_process.py +0 -0
  38. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/config.py +0 -0
  39. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/convert.py +0 -0
  40. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/deprecated.py +0 -0
  41. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/diff.py +0 -0
  42. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/dir.py +0 -0
  43. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/dir_listing.py +0 -0
  44. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/exec.py +0 -0
  45. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/experimental/flags.py +0 -0
  46. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/experimental/flags_base.py +0 -0
  47. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/file.py +0 -0
  48. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/filelist.py +0 -0
  49. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/group_files.py +0 -0
  50. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/process.py +0 -0
  51. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/remote.py +0 -0
  52. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/rpath.py +0 -0
  53. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/s3/api.py +0 -0
  54. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/s3/basic_ops.py +0 -0
  55. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/s3/chunk_types.py +0 -0
  56. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/s3/create.py +0 -0
  57. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/scan_missing_folders.py +0 -0
  58. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/types.py +0 -0
  59. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/util.py +0 -0
  60. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api/walk.py +0 -0
  61. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api.egg-info/dependency_links.txt +0 -0
  62. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api.egg-info/entry_points.txt +0 -0
  63. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api.egg-info/requires.txt +0 -0
  64. {rclone_api-1.2.13 → rclone_api-1.2.154}/src/rclone_api.egg-info/top_level.txt +0 -0
  65. {rclone_api-1.2.13 → rclone_api-1.2.154}/test +0 -0
  66. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/archive/test_paramiko.py.disabled +0 -0
  67. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_cmd_list_files.py +0 -0
  68. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_copy.py +0 -0
  69. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_copy_bytes.py +0 -0
  70. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_copy_file_resumable_s3.py +0 -0
  71. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_copy_files.py +0 -0
  72. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_diff.py +0 -0
  73. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_group_files.py +0 -0
  74. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_is_synced.py +0 -0
  75. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_ls.py +0 -0
  76. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_mount.py +0 -0
  77. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_mount_s3.py +0 -0
  78. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_obscure.py +0 -0
  79. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_rclone_config.py +0 -0
  80. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_remote_control.py +0 -0
  81. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_remotes.py +0 -0
  82. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_s3.py +0 -0
  83. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_scan_missing_folders.py +0 -0
  84. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_size_files.py +0 -0
  85. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_size_suffix.py +0 -0
  86. {rclone_api-1.2.13 → rclone_api-1.2.154}/tests/test_walk.py +0 -0
  87. {rclone_api-1.2.13 → rclone_api-1.2.154}/tox.ini +0 -0
  88. {rclone_api-1.2.13 → rclone_api-1.2.154}/upload_package.sh +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: rclone_api
3
- Version: 1.2.13
3
+ Version: 1.2.154
4
4
  Summary: rclone api in python
5
5
  Home-page: https://github.com/zackees/rclone-api
6
6
  License: BSD 3-Clause License
@@ -23,7 +23,7 @@ dependencies = [
23
23
  ]
24
24
 
25
25
  # Change this with the version number bump.
26
- version = "1.2.13"
26
+ version = "1.2.154"
27
27
 
28
28
  [tool.setuptools]
29
29
  package-dir = {"" = "src"}
@@ -1,38 +1,45 @@
1
- from .completed_process import CompletedProcess
2
- from .config import Config, Parsed, Section
3
- from .diff import DiffItem, DiffOption, DiffType
4
- from .dir import Dir
5
- from .dir_listing import DirListing
6
- from .file import File
7
- from .filelist import FileList
8
- from .process import Process
9
- from .rclone import Rclone, rclone_verbose
10
- from .remote import Remote
11
- from .rpath import RPath
12
- from .s3.types import MultiUploadResult
13
- from .types import ListingOption, Order, SizeResult, SizeSuffix
14
-
15
- __all__ = [
16
- "Rclone",
17
- "File",
18
- "Config",
19
- "Remote",
20
- "Dir",
21
- "RPath",
22
- "DirListing",
23
- "FileList",
24
- "Process",
25
- "DiffItem",
26
- "DiffType",
27
- "rclone_verbose",
28
- "CompletedProcess",
29
- "DiffOption",
30
- "ListingOption",
31
- "Order",
32
- "ListingOption",
33
- "SizeResult",
34
- "Parsed",
35
- "Section",
36
- "MultiUploadResult",
37
- "SizeSuffix",
38
- ]
1
+ # Import logging module to activate default configuration
2
+ import rclone_api.logging # noqa: F401
3
+
4
+ from .completed_process import CompletedProcess
5
+ from .config import Config, Parsed, Section
6
+ from .diff import DiffItem, DiffOption, DiffType
7
+ from .dir import Dir
8
+ from .dir_listing import DirListing
9
+ from .file import File
10
+ from .filelist import FileList
11
+
12
+ # Import the configure_logging function to make it available at package level
13
+ from .logging import configure_logging
14
+ from .process import Process
15
+ from .rclone import Rclone, rclone_verbose
16
+ from .remote import Remote
17
+ from .rpath import RPath
18
+ from .s3.types import MultiUploadResult
19
+ from .types import ListingOption, Order, SizeResult, SizeSuffix
20
+
21
+ __all__ = [
22
+ "Rclone",
23
+ "File",
24
+ "Config",
25
+ "Remote",
26
+ "Dir",
27
+ "RPath",
28
+ "DirListing",
29
+ "FileList",
30
+ "Process",
31
+ "DiffItem",
32
+ "DiffType",
33
+ "rclone_verbose",
34
+ "CompletedProcess",
35
+ "DiffOption",
36
+ "ListingOption",
37
+ "Order",
38
+ "ListingOption",
39
+ "SizeResult",
40
+ "Parsed",
41
+ "Section",
42
+ "MultiUploadResult",
43
+ "SizeSuffix",
44
+ "configure_logging",
45
+ ]
@@ -0,0 +1,39 @@
1
+ import logging
2
+ import sys
3
+
4
+
5
+ def setup_default_logging():
6
+ """Set up default logging configuration if none exists."""
7
+ if not logging.root.handlers:
8
+ logging.basicConfig(
9
+ level=logging.INFO,
10
+ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
11
+ handlers=[
12
+ logging.StreamHandler(sys.stdout),
13
+ # Uncomment to add file logging
14
+ # logging.FileHandler('rclone_api.log')
15
+ ],
16
+ )
17
+
18
+
19
+ def configure_logging(level=logging.INFO, log_file=None):
20
+ """Configure logging for the rclone_api package.
21
+
22
+ Args:
23
+ level: The logging level (default: logging.INFO)
24
+ log_file: Optional path to a log file
25
+ """
26
+ handlers = [logging.StreamHandler(sys.stdout)]
27
+ if log_file:
28
+ handlers.append(logging.FileHandler(log_file))
29
+
30
+ logging.basicConfig(
31
+ level=level,
32
+ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
33
+ handlers=handlers,
34
+ force=True, # Override any existing configuration
35
+ )
36
+
37
+
38
+ # Call setup_default_logging when this module is imported
39
+ setup_default_logging()
@@ -4,17 +4,14 @@ import platform
4
4
  import shutil
5
5
  import subprocess
6
6
  import time
7
- import traceback
8
7
  import warnings
9
8
  import weakref
10
- from concurrent.futures import Future, ThreadPoolExecutor
9
+ from concurrent.futures import ThreadPoolExecutor
11
10
  from dataclasses import dataclass
12
11
  from pathlib import Path
13
- from threading import Lock, Semaphore
14
12
  from typing import Any
15
13
 
16
14
  from rclone_api.process import Process
17
- from rclone_api.types import FilePart
18
15
 
19
16
  _SYSTEM = platform.system() # "Linux", "Darwin", "Windows", etc.
20
17
 
@@ -37,6 +34,14 @@ def _cleanup_mounts() -> None:
37
34
  executor.submit(mount.close)
38
35
 
39
36
 
37
+ def _cache_dir_delete_on_exit(cache_dir: Path) -> None:
38
+ if cache_dir.exists():
39
+ try:
40
+ shutil.rmtree(cache_dir)
41
+ except Exception as e:
42
+ warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
43
+
44
+
40
45
  atexit.register(_cleanup_mounts)
41
46
 
42
47
 
@@ -278,110 +283,3 @@ def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
278
283
  else:
279
284
  if verbose:
280
285
  print(f"{mount_path} successfully cleaned up.")
281
-
282
-
283
- def _cache_dir_delete_on_exit(cache_dir: Path) -> None:
284
- if cache_dir.exists():
285
- try:
286
- shutil.rmtree(cache_dir)
287
- except Exception as e:
288
- warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
289
-
290
-
291
- def _read_from_mount_task(
292
- offset: int, size: int, path: Path, verbose: bool
293
- ) -> bytes | Exception:
294
- if verbose or True:
295
- print(f"Fetching chunk: offset={offset}, size={size}, path={path}")
296
- try:
297
- with path.open("rb") as f:
298
- f.seek(offset)
299
- payload = f.read(size)
300
- assert len(payload) == size, f"Invalid read size: {len(payload)}"
301
- return payload
302
-
303
- except KeyboardInterrupt as e:
304
- import _thread
305
-
306
- _thread.interrupt_main()
307
- return Exception(e)
308
- except Exception as e:
309
- stack_trace = traceback.format_exc()
310
- warnings.warn(
311
- f"Error fetching file chunk at offset {offset} + {size}: {e}\n{stack_trace}"
312
- )
313
- return e
314
-
315
-
316
- class MultiMountFileChunker:
317
- def __init__(
318
- self,
319
- filename: str,
320
- filesize: int,
321
- mounts: list[Mount],
322
- executor: ThreadPoolExecutor,
323
- verbose: bool | None,
324
- ) -> None:
325
- from rclone_api.util import get_verbose
326
-
327
- self.filename = filename
328
- self.filesize = filesize
329
- self.executor = executor
330
- self.mounts_processing: list[Mount] = []
331
- self.mounts_availabe: list[Mount] = mounts
332
- self.semaphore = Semaphore(len(mounts))
333
- self.lock = Lock()
334
- self.verbose = get_verbose(verbose)
335
-
336
- def shutdown(self) -> None:
337
- self.executor.shutdown(wait=True, cancel_futures=True)
338
- with ThreadPoolExecutor() as executor:
339
- for mount in self.mounts_processing:
340
- executor.submit(lambda: mount.close())
341
-
342
- def _acquire_mount(self) -> Mount:
343
- self.semaphore.acquire()
344
- with self.lock:
345
- mount = self.mounts_availabe.pop()
346
- self.mounts_processing.append(mount)
347
- return mount
348
-
349
- def _release_mount(self, mount: Mount) -> None:
350
- with self.lock:
351
- self.mounts_processing.remove(mount)
352
- self.mounts_availabe.append(mount)
353
- self.semaphore.release()
354
-
355
- def fetch(self, offset: int, size: int, extra: Any) -> Future[FilePart]:
356
- if self.verbose:
357
- print(f"Fetching data range: offset={offset}, size={size}")
358
-
359
- assert size > 0, f"Invalid size: {size}"
360
- assert offset >= 0, f"Invalid offset: {offset}"
361
- assert (
362
- offset + size <= self.filesize
363
- ), f"Invalid offset + size: {offset} + {size} ({offset+size}) <= {self.filesize}"
364
-
365
- try:
366
- mount = self._acquire_mount()
367
- path = mount.mount_path / self.filename
368
-
369
- def task_fetch_file_range(
370
- size=size, path=path, mount=mount, verbose=self.verbose
371
- ) -> FilePart:
372
- bytes_or_err = _read_from_mount_task(
373
- offset=offset, size=size, path=path, verbose=verbose
374
- )
375
- self._release_mount(mount)
376
-
377
- if isinstance(bytes_or_err, Exception):
378
- return FilePart(payload=bytes_or_err, extra=extra)
379
- out = FilePart(payload=bytes_or_err, extra=extra)
380
- return out
381
-
382
- fut = self.executor.submit(task_fetch_file_range)
383
- return fut
384
- except Exception as e:
385
- warnings.warn(f"Error fetching file chunk: {e}")
386
- fp = FilePart(payload=e, extra=extra)
387
- return self.executor.submit(lambda: fp)
@@ -0,0 +1,123 @@
1
+ import logging
2
+ import traceback
3
+ from concurrent.futures import Future, ThreadPoolExecutor
4
+ from pathlib import Path
5
+ from threading import Lock, Semaphore
6
+ from typing import Any
7
+
8
+ from rclone_api.mount import Mount
9
+ from rclone_api.types import FilePart
10
+
11
+ # Create a logger for this module
12
+ logger = logging.getLogger(__name__)
13
+
14
+
15
+ def _read_from_mount_task(
16
+ offset: int, size: int, path: Path, verbose: bool
17
+ ) -> bytes | Exception:
18
+ if verbose:
19
+ logger.debug(f"Fetching chunk: offset={offset}, size={size}, path={path}")
20
+ try:
21
+ with path.open("rb") as f:
22
+ f.seek(offset)
23
+ payload = f.read(size)
24
+ assert len(payload) == size, f"Invalid read size: {len(payload)}"
25
+ return payload
26
+
27
+ except KeyboardInterrupt as e:
28
+ import _thread
29
+
30
+ logger.error("KeyboardInterrupt received during chunk read")
31
+ _thread.interrupt_main()
32
+ return Exception(e)
33
+ except Exception as e:
34
+ stack_trace = traceback.format_exc()
35
+ logger.error(
36
+ f"Error fetching file chunk at offset {offset} + {size}: {e}\n{stack_trace}"
37
+ )
38
+ return e
39
+
40
+
41
+ class MultiMountFileChunker:
42
+ def __init__(
43
+ self,
44
+ filename: str,
45
+ filesize: int,
46
+ mounts: list[Mount],
47
+ executor: ThreadPoolExecutor,
48
+ verbose: bool | None,
49
+ ) -> None:
50
+ from rclone_api.util import get_verbose
51
+
52
+ self.filename = filename
53
+ self.filesize = filesize
54
+ self.executor = executor
55
+ self.mounts_processing: list[Mount] = []
56
+ self.mounts_availabe: list[Mount] = mounts
57
+ self.semaphore = Semaphore(len(mounts))
58
+ self.lock = Lock()
59
+ self.verbose = get_verbose(verbose)
60
+ logger.info(
61
+ f"Initialized MultiMountFileChunker for {filename} ({filesize} bytes)"
62
+ )
63
+
64
+ def shutdown(self) -> None:
65
+ logger.info("Shutting down MultiMountFileChunker")
66
+ self.executor.shutdown(wait=True, cancel_futures=True)
67
+ with ThreadPoolExecutor() as executor:
68
+ for mount in self.mounts_processing:
69
+ executor.submit(lambda: mount.close())
70
+ logger.debug("MultiMountFileChunker shutdown complete")
71
+
72
+ def _acquire_mount(self) -> Mount:
73
+ logger.debug("Acquiring mount")
74
+ self.semaphore.acquire()
75
+ with self.lock:
76
+ mount = self.mounts_availabe.pop()
77
+ self.mounts_processing.append(mount)
78
+ logger.debug(f"Mount acquired: {mount}")
79
+ return mount
80
+
81
+ def _release_mount(self, mount: Mount) -> None:
82
+ logger.debug(f"Releasing mount: {mount}")
83
+ with self.lock:
84
+ self.mounts_processing.remove(mount)
85
+ self.mounts_availabe.append(mount)
86
+ self.semaphore.release()
87
+ logger.debug("Mount released")
88
+
89
+ def fetch(self, offset: int, size: int, extra: Any) -> Future[FilePart]:
90
+ if self.verbose:
91
+ logger.debug(f"Fetching data range: offset={offset}, size={size}")
92
+
93
+ assert size > 0, f"Invalid size: {size}"
94
+ assert offset >= 0, f"Invalid offset: {offset}"
95
+ assert (
96
+ offset + size <= self.filesize
97
+ ), f"Invalid offset + size: {offset} + {size} ({offset+size}) <= {self.filesize}"
98
+
99
+ try:
100
+ mount = self._acquire_mount()
101
+ path = mount.mount_path / self.filename
102
+
103
+ def task_fetch_file_range(
104
+ size=size, path=path, mount=mount, verbose=self.verbose
105
+ ) -> FilePart:
106
+ bytes_or_err = _read_from_mount_task(
107
+ offset=offset, size=size, path=path, verbose=verbose
108
+ )
109
+ self._release_mount(mount)
110
+
111
+ if isinstance(bytes_or_err, Exception):
112
+ logger.warning(f"Fetch task returned exception: {bytes_or_err}")
113
+ return FilePart(payload=bytes_or_err, extra=extra)
114
+ logger.debug(f"Successfully fetched {size} bytes from offset {offset}")
115
+ out = FilePart(payload=bytes_or_err, extra=extra)
116
+ return out
117
+
118
+ fut = self.executor.submit(task_fetch_file_range)
119
+ return fut
120
+ except Exception as e:
121
+ logger.error(f"Error setting up file chunk fetch: {e}", exc_info=True)
122
+ fp = FilePart(payload=e, extra=extra)
123
+ return self.executor.submit(lambda: fp)
@@ -14,7 +14,7 @@ import psutil
14
14
  from dotenv import load_dotenv
15
15
 
16
16
  from rclone_api import Config, Rclone, SizeSuffix
17
- from rclone_api.mount import MultiMountFileChunker
17
+ from rclone_api.mount_read_chunker import MultiMountFileChunker
18
18
  from rclone_api.types import FilePart
19
19
 
20
20
  os.environ["RCLONE_API_VERBOSE"] = "1"
@@ -26,7 +26,8 @@ from rclone_api.dir_listing import DirListing
26
26
  from rclone_api.exec import RcloneExec
27
27
  from rclone_api.file import File
28
28
  from rclone_api.group_files import group_files
29
- from rclone_api.mount import Mount, MultiMountFileChunker, clean_mount, prepare_mount
29
+ from rclone_api.mount import Mount, clean_mount, prepare_mount
30
+ from rclone_api.mount_read_chunker import MultiMountFileChunker
30
31
  from rclone_api.process import Process
31
32
  from rclone_api.remote import Remote
32
33
  from rclone_api.rpath import RPath
@@ -1,145 +1,146 @@
1
- import time
2
- import warnings
3
- from concurrent.futures import Future
4
- from dataclasses import dataclass
5
- from pathlib import Path
6
- from queue import Queue
7
- from threading import Event
8
- from typing import Any, Callable
9
-
10
- from rclone_api.mount import FilePart
11
- from rclone_api.s3.chunk_types import UploadState
12
- from rclone_api.types import EndOfStream
13
- from rclone_api.util import locked_print
14
-
15
-
16
- def _get_file_size(file_path: Path, timeout: int = 60) -> int:
17
- sleep_time = timeout / 60 if timeout > 0 else 1
18
- start = time.time()
19
- while True:
20
- expired = time.time() - start > timeout
21
- try:
22
- time.sleep(sleep_time)
23
- if file_path.exists():
24
- return file_path.stat().st_size
25
- except FileNotFoundError as e:
26
- if expired:
27
- print(f"File not found: {file_path}, exception is {e}")
28
- raise
29
- if expired:
30
- raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
31
-
32
-
33
- @dataclass
34
- class S3FileInfo:
35
- upload_id: str
36
- part_number: int
37
-
38
-
39
- def file_chunker(
40
- upload_state: UploadState,
41
- fetcher: Callable[[int, int, Any], Future[FilePart]],
42
- max_chunks: int | None,
43
- cancel_signal: Event,
44
- queue_upload: Queue[FilePart | EndOfStream],
45
- ) -> None:
46
- count = 0
47
-
48
- def should_stop() -> bool:
49
- nonlocal count
50
-
51
- if max_chunks is None:
52
- return False
53
- if count >= max_chunks:
54
- print(
55
- f"Stopping file chunker after {count} chunks because it exceeded max_chunks {max_chunks}"
56
- )
57
- return True
58
- count += 1
59
- return False
60
-
61
- upload_info = upload_state.upload_info
62
- file_path = upload_info.src_file_path
63
- chunk_size = upload_info.chunk_size
64
- # src = Path(file_path)
65
-
66
- try:
67
- part_number = 1
68
- done_part_numbers: set[int] = {
69
- p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
70
- }
71
- num_parts = upload_info.total_chunks()
72
-
73
- def next_part_number() -> int | None:
74
- nonlocal part_number
75
- while part_number in done_part_numbers:
76
- part_number += 1
77
- if part_number > num_parts:
78
- return None
79
- return part_number
80
-
81
- if cancel_signal.is_set():
82
- print(
83
- f"Cancel signal is set for file chunker while processing {file_path}, returning"
84
- )
85
- return
86
-
87
- while not should_stop():
88
- print("@@@@@@@@@@")
89
- curr_part_number = next_part_number()
90
- if curr_part_number is None:
91
- locked_print(f"File {file_path} has completed chunking all parts")
92
- break
93
- assert curr_part_number is not None
94
- offset = (curr_part_number - 1) * chunk_size
95
- file_size = upload_info.file_size
96
-
97
- assert offset < file_size, f"Offset {offset} is greater than file size"
98
-
99
- # Open the file, seek, read the chunk, and close immediately.
100
- # with open(file_path, "rb") as f:
101
- # f.seek(offset)
102
- # data = f.read(chunk_size)
103
-
104
- # data = chunk_fetcher(offset, chunk_size).result()
105
-
106
- assert curr_part_number is not None
107
- cpn: int = curr_part_number
108
-
109
- def on_complete(fut: Future[FilePart]) -> None:
110
- print("ON COMPLETE")
111
- fp: FilePart = fut.result()
112
- if fp.is_error():
113
- warnings.warn(
114
- f"Error reading file: {fp}, skipping part {part_number}"
115
- )
116
- return
117
-
118
- if fp.n_bytes() == 0:
119
- warnings.warn(f"Empty data for part {part_number} of {file_path}")
120
- raise ValueError(
121
- f"Empty data for part {part_number} of {file_path}"
122
- )
123
-
124
- if isinstance(fp.payload, Exception):
125
- warnings.warn(f"Error reading file because of error: {fp.payload}")
126
- return
127
-
128
- done_part_numbers.add(part_number)
129
- queue_upload.put(fp)
130
-
131
- offset = (curr_part_number - 1) * chunk_size
132
- print(f"Reading chunk {curr_part_number} of {num_parts} for {file_path}")
133
- fut = fetcher(offset, file_size, S3FileInfo(upload_info.upload_id, cpn))
134
- fut.add_done_callback(on_complete)
135
- # wait until the queue_upload queue can accept the next chunk
136
- while queue_upload.full():
137
- time.sleep(0.1)
138
- except Exception as e:
139
-
140
- warnings.warn(f"Error reading file: {e}")
141
- finally:
142
- print("#############################################################")
143
- print(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
144
- print("#############################################################")
145
- queue_upload.put(EndOfStream())
1
+ import logging
2
+ import time
3
+ from concurrent.futures import Future
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+ from queue import Queue
7
+ from threading import Event
8
+ from typing import Any, Callable
9
+
10
+ from rclone_api.mount_read_chunker import FilePart
11
+ from rclone_api.s3.chunk_types import UploadState
12
+ from rclone_api.types import EndOfStream
13
+
14
+ logger = logging.getLogger(__name__) # noqa
15
+
16
+
17
+ def _get_file_size(file_path: Path, timeout: int = 60) -> int:
18
+ sleep_time = timeout / 60 if timeout > 0 else 1
19
+ start = time.time()
20
+ while True:
21
+ expired = time.time() - start > timeout
22
+ try:
23
+ time.sleep(sleep_time)
24
+ if file_path.exists():
25
+ return file_path.stat().st_size
26
+ except FileNotFoundError as e:
27
+ if expired:
28
+ print(f"File not found: {file_path}, exception is {e}")
29
+ raise
30
+ if expired:
31
+ raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
32
+
33
+
34
+ @dataclass
35
+ class S3FileInfo:
36
+ upload_id: str
37
+ part_number: int
38
+
39
+
40
+ def file_chunker(
41
+ upload_state: UploadState,
42
+ fetcher: Callable[[int, int, Any], Future[FilePart]],
43
+ max_chunks: int | None,
44
+ cancel_signal: Event,
45
+ queue_upload: Queue[FilePart | EndOfStream],
46
+ ) -> None:
47
+ count = 0
48
+
49
+ def should_stop() -> bool:
50
+ nonlocal count
51
+
52
+ if max_chunks is None:
53
+ return False
54
+ if count >= max_chunks:
55
+ logger.info(
56
+ f"Stopping file chunker after {count} chunks because it exceeded max_chunks {max_chunks}"
57
+ )
58
+ return True
59
+ count += 1
60
+ return False
61
+
62
+ upload_info = upload_state.upload_info
63
+ file_path = upload_info.src_file_path
64
+ chunk_size = upload_info.chunk_size
65
+ # src = Path(file_path)
66
+
67
+ try:
68
+ part_number = 1
69
+ done_part_numbers: set[int] = {
70
+ p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
71
+ }
72
+ num_parts = upload_info.total_chunks()
73
+
74
+ def next_part_number() -> int | None:
75
+ nonlocal part_number
76
+ while part_number in done_part_numbers:
77
+ part_number += 1
78
+ if part_number > num_parts:
79
+ return None
80
+ return part_number
81
+
82
+ if cancel_signal.is_set():
83
+ logger.info(
84
+ f"Cancel signal is set for file chunker while processing {file_path}, returning"
85
+ )
86
+ return
87
+
88
+ while not should_stop():
89
+ logger.debug("Processing next chunk")
90
+ curr_part_number = next_part_number()
91
+ if curr_part_number is None:
92
+ logger.info(f"File {file_path} has completed chunking all parts")
93
+ break
94
+ assert curr_part_number is not None
95
+ offset = (curr_part_number - 1) * chunk_size
96
+ file_size = upload_info.file_size
97
+
98
+ assert offset < file_size, f"Offset {offset} is greater than file size"
99
+
100
+ # Open the file, seek, read the chunk, and close immediately.
101
+ # with open(file_path, "rb") as f:
102
+ # f.seek(offset)
103
+ # data = f.read(chunk_size)
104
+
105
+ # data = chunk_fetcher(offset, chunk_size).result()
106
+
107
+ assert curr_part_number is not None
108
+ cpn: int = curr_part_number
109
+
110
+ def on_complete(fut: Future[FilePart]) -> None:
111
+ logger.debug("Chunk read complete")
112
+ fp: FilePart = fut.result()
113
+ if fp.is_error():
114
+ logger.warning(
115
+ f"Error reading file: {fp}, skipping part {part_number}"
116
+ )
117
+ return
118
+
119
+ if fp.n_bytes() == 0:
120
+ logger.warning(f"Empty data for part {part_number} of {file_path}")
121
+ raise ValueError(
122
+ f"Empty data for part {part_number} of {file_path}"
123
+ )
124
+
125
+ if isinstance(fp.payload, Exception):
126
+ logger.warning(f"Error reading file because of error: {fp.payload}")
127
+ return
128
+
129
+ done_part_numbers.add(part_number)
130
+ queue_upload.put(fp)
131
+
132
+ offset = (curr_part_number - 1) * chunk_size
133
+ logger.info(
134
+ f"Reading chunk {curr_part_number} of {num_parts} for {file_path}"
135
+ )
136
+ fut = fetcher(offset, file_size, S3FileInfo(upload_info.upload_id, cpn))
137
+ fut.add_done_callback(on_complete)
138
+ # wait until the queue_upload queue can accept the next chunk
139
+ while queue_upload.full():
140
+ time.sleep(0.1)
141
+ except Exception as e:
142
+
143
+ logger.error(f"Error reading file: {e}", exc_info=True)
144
+ finally:
145
+ logger.info(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
146
+ queue_upload.put(EndOfStream())
@@ -4,7 +4,7 @@ from enum import Enum
4
4
  from pathlib import Path
5
5
  from typing import Any, Callable
6
6
 
7
- from rclone_api.mount import FilePart
7
+ from rclone_api.mount_read_chunker import FilePart
8
8
 
9
9
 
10
10
  class S3Provider(Enum):
@@ -10,7 +10,7 @@ from typing import Any, Callable
10
10
 
11
11
  from botocore.client import BaseClient
12
12
 
13
- from rclone_api.mount import FilePart
13
+ from rclone_api.mount_read_chunker import FilePart
14
14
  from rclone_api.s3.chunk_file import S3FileInfo, file_chunker
15
15
  from rclone_api.s3.chunk_types import (
16
16
  FinishedPiece,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: rclone_api
3
- Version: 1.2.13
3
+ Version: 1.2.154
4
4
  Summary: rclone api in python
5
5
  Home-page: https://github.com/zackees/rclone-api
6
6
  License: BSD 3-Clause License
@@ -33,7 +33,9 @@ src/rclone_api/exec.py
33
33
  src/rclone_api/file.py
34
34
  src/rclone_api/filelist.py
35
35
  src/rclone_api/group_files.py
36
+ src/rclone_api/logging.py
36
37
  src/rclone_api/mount.py
38
+ src/rclone_api/mount_read_chunker.py
37
39
  src/rclone_api/process.py
38
40
  src/rclone_api/rclone.py
39
41
  src/rclone_api/remote.py
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes