rclone-api 1.2.12__tar.gz → 1.2.14__tar.gz

{rclone_api-1.2.12 → rclone_api-1.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.2.12
+Version: 1.2.14
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.2.12 → rclone_api-1.2.14}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.2.12"
+version = "1.2.14"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.2.12 → rclone_api-1.2.14}/src/rclone_api/mount.py

@@ -4,17 +4,14 @@ import platform
 import shutil
 import subprocess
 import time
-import traceback
 import warnings
 import weakref
-from concurrent.futures import Future, ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
-from threading import Lock, Semaphore
 from typing import Any
 
 from rclone_api.process import Process
-from rclone_api.types import FilePart
 
 _SYSTEM = platform.system()  # "Linux", "Darwin", "Windows", etc.
 
@@ -37,6 +34,14 @@ def _cleanup_mounts() -> None:
             executor.submit(mount.close)
 
 
+def _cache_dir_delete_on_exit(cache_dir: Path) -> None:
+    if cache_dir.exists():
+        try:
+            shutil.rmtree(cache_dir)
+        except Exception as e:
+            warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
+
+
 atexit.register(_cleanup_mounts)
 
 
@@ -278,110 +283,3 @@ def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
     else:
         if verbose:
             print(f"{mount_path} successfully cleaned up.")
-
-
-def _cache_dir_delete_on_exit(cache_dir: Path) -> None:
-    if cache_dir.exists():
-        try:
-            shutil.rmtree(cache_dir)
-        except Exception as e:
-            warnings.warn(f"Error removing cache directory {cache_dir}: {e}")
-
-
-def _read_from_mount_task(
-    offset: int, size: int, path: Path, verbose: bool
-) -> bytes | Exception:
-    if verbose or True:
-        print(f"Fetching chunk: offset={offset}, size={size}, path={path}")
-    try:
-        with path.open("rb") as f:
-            f.seek(offset)
-            payload = f.read(size)
-            assert len(payload) == size, f"Invalid read size: {len(payload)}"
-            return payload
-
-    except KeyboardInterrupt as e:
-        import _thread
-
-        _thread.interrupt_main()
-        return Exception(e)
-    except Exception as e:
-        stack_trace = traceback.format_exc()
-        warnings.warn(
-            f"Error fetching file chunk at offset {offset} + {size}: {e}\n{stack_trace}"
-        )
-        return e
-
-
-class MultiMountFileChunker:
-    def __init__(
-        self,
-        filename: str,
-        filesize: int,
-        mounts: list[Mount],
-        executor: ThreadPoolExecutor,
-        verbose: bool | None,
-    ) -> None:
-        from rclone_api.util import get_verbose
-
-        self.filename = filename
-        self.filesize = filesize
-        self.executor = executor
-        self.mounts_processing: list[Mount] = []
-        self.mounts_availabe: list[Mount] = mounts
-        self.semaphore = Semaphore(len(mounts))
-        self.lock = Lock()
-        self.verbose = get_verbose(verbose)
-
-    def shutdown(self) -> None:
-        self.executor.shutdown(wait=True, cancel_futures=True)
-        with ThreadPoolExecutor() as executor:
-            for mount in self.mounts_processing:
-                executor.submit(lambda: mount.close())
-
-    def _acquire_mount(self) -> Mount:
-        self.semaphore.acquire()
-        with self.lock:
-            mount = self.mounts_availabe.pop()
-            self.mounts_processing.append(mount)
-        return mount
-
-    def _release_mount(self, mount: Mount) -> None:
-        with self.lock:
-            self.mounts_processing.remove(mount)
-            self.mounts_availabe.append(mount)
-            self.semaphore.release()
-
-    def fetch(self, offset: int, size: int, extra: Any) -> Future[FilePart]:
-        if self.verbose:
-            print(f"Fetching data range: offset={offset}, size={size}")
-
-        assert size > 0, f"Invalid size: {size}"
-        assert offset >= 0, f"Invalid offset: {offset}"
-        assert (
-            offset + size <= self.filesize
-        ), f"Invalid offset + size: {offset} + {size} ({offset+size}) <= {self.filesize}"
-
-        try:
-            mount = self._acquire_mount()
-            path = mount.mount_path / self.filename
-
-            def task_fetch_file_range(
-                size=size, path=path, mount=mount, verbose=self.verbose
-            ) -> FilePart:
-                bytes_or_err = _read_from_mount_task(
-                    offset=offset, size=size, path=path, verbose=verbose
-                )
-                self._release_mount(mount)
-
-                if isinstance(bytes_or_err, Exception):
-                    return FilePart(payload=bytes_or_err, extra=extra)
-                out = FilePart(payload=bytes_or_err, extra=extra)
-                return out
-
-            fut = self.executor.submit(task_fetch_file_range)
-            return fut
-        except Exception as e:
-            warnings.warn(f"Error fetching file chunk: {e}")
-            fp = FilePart(payload=e, extra=extra)
-            return self.executor.submit(lambda: fp)

rclone_api-1.2.14/src/rclone_api/mount_read_chunker.py

@@ -0,0 +1,123 @@
+import logging
+import traceback
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from threading import Lock, Semaphore
+from typing import Any
+
+from rclone_api.mount import Mount
+from rclone_api.types import FilePart
+
+# Create a logger for this module
+logger = logging.getLogger(__name__)
+
+
+def _read_from_mount_task(
+    offset: int, size: int, path: Path, verbose: bool
+) -> bytes | Exception:
+    if verbose:
+        logger.debug(f"Fetching chunk: offset={offset}, size={size}, path={path}")
+    try:
+        with path.open("rb") as f:
+            f.seek(offset)
+            payload = f.read(size)
+            assert len(payload) == size, f"Invalid read size: {len(payload)}"
+            return payload
+
+    except KeyboardInterrupt as e:
+        import _thread
+
+        logger.error("KeyboardInterrupt received during chunk read")
+        _thread.interrupt_main()
+        return Exception(e)
+    except Exception as e:
+        stack_trace = traceback.format_exc()
+        logger.error(
+            f"Error fetching file chunk at offset {offset} + {size}: {e}\n{stack_trace}"
+        )
+        return e
+
+
+class MultiMountFileChunker:
+    def __init__(
+        self,
+        filename: str,
+        filesize: int,
+        mounts: list[Mount],
+        executor: ThreadPoolExecutor,
+        verbose: bool | None,
+    ) -> None:
+        from rclone_api.util import get_verbose
+
+        self.filename = filename
+        self.filesize = filesize
+        self.executor = executor
+        self.mounts_processing: list[Mount] = []
+        self.mounts_availabe: list[Mount] = mounts
+        self.semaphore = Semaphore(len(mounts))
+        self.lock = Lock()
+        self.verbose = get_verbose(verbose)
+        logger.info(
+            f"Initialized MultiMountFileChunker for {filename} ({filesize} bytes)"
+        )
+
+    def shutdown(self) -> None:
+        logger.info("Shutting down MultiMountFileChunker")
+        self.executor.shutdown(wait=True, cancel_futures=True)
+        with ThreadPoolExecutor() as executor:
+            for mount in self.mounts_processing:
+                executor.submit(lambda: mount.close())
+        logger.debug("MultiMountFileChunker shutdown complete")
+
+    def _acquire_mount(self) -> Mount:
+        logger.debug("Acquiring mount")
+        self.semaphore.acquire()
+        with self.lock:
+            mount = self.mounts_availabe.pop()
+            self.mounts_processing.append(mount)
+        logger.debug(f"Mount acquired: {mount}")
+        return mount
+
+    def _release_mount(self, mount: Mount) -> None:
+        logger.debug(f"Releasing mount: {mount}")
+        with self.lock:
+            self.mounts_processing.remove(mount)
+            self.mounts_availabe.append(mount)
+            self.semaphore.release()
+        logger.debug("Mount released")
+
+    def fetch(self, offset: int, size: int, extra: Any) -> Future[FilePart]:
+        if self.verbose:
+            logger.debug(f"Fetching data range: offset={offset}, size={size}")
+
+        assert size > 0, f"Invalid size: {size}"
+        assert offset >= 0, f"Invalid offset: {offset}"
+        assert (
+            offset + size <= self.filesize
+        ), f"Invalid offset + size: {offset} + {size} ({offset+size}) <= {self.filesize}"
+
+        try:
+            mount = self._acquire_mount()
+            path = mount.mount_path / self.filename
+
+            def task_fetch_file_range(
+                size=size, path=path, mount=mount, verbose=self.verbose
+            ) -> FilePart:
+                bytes_or_err = _read_from_mount_task(
+                    offset=offset, size=size, path=path, verbose=verbose
+                )
+                self._release_mount(mount)
+
+                if isinstance(bytes_or_err, Exception):
+                    logger.warning(f"Fetch task returned exception: {bytes_or_err}")
+                    return FilePart(payload=bytes_or_err, extra=extra)
+                logger.debug(f"Successfully fetched {size} bytes from offset {offset}")
+                out = FilePart(payload=bytes_or_err, extra=extra)
+                return out
+
+            fut = self.executor.submit(task_fetch_file_range)
+            return fut
+        except Exception as e:
+            logger.error(f"Error setting up file chunk fetch: {e}", exc_info=True)
+            fp = FilePart(payload=e, extra=extra)
+            return self.executor.submit(lambda: fp)

{rclone_api-1.2.12 → rclone_api-1.2.14}/src/rclone_api/profile/mount_copy_bytes.py

@@ -14,7 +14,7 @@ import psutil
 from dotenv import load_dotenv
 
 from rclone_api import Config, Rclone, SizeSuffix
-from rclone_api.mount import MultiMountFileChunker
+from rclone_api.mount_read_chunker import MultiMountFileChunker
 from rclone_api.types import FilePart
 
 os.environ["RCLONE_API_VERBOSE"] = "1"

{rclone_api-1.2.12 → rclone_api-1.2.14}/src/rclone_api/rclone.py

@@ -26,7 +26,8 @@ from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File
 from rclone_api.group_files import group_files
-from rclone_api.mount import Mount, MultiMountFileChunker, clean_mount, prepare_mount
+from rclone_api.mount import Mount, clean_mount, prepare_mount
+from rclone_api.mount_read_chunker import MultiMountFileChunker
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath

{rclone_api-1.2.12 → rclone_api-1.2.14}/src/rclone_api/s3/chunk_file.py

@@ -7,7 +7,7 @@ from queue import Queue
 from threading import Event
 from typing import Any, Callable
 
-from rclone_api.mount import FilePart
+from rclone_api.mount_read_chunker import FilePart
 from rclone_api.s3.chunk_types import UploadState
 from rclone_api.types import EndOfStream
 from rclone_api.util import locked_print
@@ -85,6 +85,7 @@ def file_chunker(
             return
 
         while not should_stop():
+            print("@@@@@@@@@@")
             curr_part_number = next_part_number()
             if curr_part_number is None:
                 locked_print(f"File {file_path} has completed chunking all parts")
@@ -106,6 +107,7 @@ def file_chunker(
             cpn: int = curr_part_number
 
             def on_complete(fut: Future[FilePart]) -> None:
+                print("ON COMPLETE")
                 fp: FilePart = fut.result()
                 if fp.is_error():
                     warnings.warn(
@@ -127,6 +129,7 @@ def file_chunker(
                 queue_upload.put(fp)
 
             offset = (curr_part_number - 1) * chunk_size
+            print(f"Reading chunk {curr_part_number} of {num_parts} for {file_path}")
             fut = fetcher(offset, file_size, S3FileInfo(upload_info.upload_id, cpn))
             fut.add_done_callback(on_complete)
             # wait until the queue_upload queue can accept the next chunk

{rclone_api-1.2.12 → rclone_api-1.2.14}/src/rclone_api/s3/types.py

@@ -4,7 +4,7 @@ from enum import Enum
 from pathlib import Path
 from typing import Any, Callable
 
-from rclone_api.mount import FilePart
+from rclone_api.mount_read_chunker import FilePart
 
 
 class S3Provider(Enum):

{rclone_api-1.2.12 → rclone_api-1.2.14}/src/rclone_api/s3/upload_file_multipart.py

@@ -10,7 +10,7 @@ from typing import Any, Callable
 
 from botocore.client import BaseClient
 
-from rclone_api.mount import FilePart
+from rclone_api.mount_read_chunker import FilePart
 from rclone_api.s3.chunk_file import S3FileInfo, file_chunker
 from rclone_api.s3.chunk_types import (
     FinishedPiece,

{rclone_api-1.2.12 → rclone_api-1.2.14}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.2.12
+Version: 1.2.14
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.2.12 → rclone_api-1.2.14}/src/rclone_api.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ src/rclone_api/file.py
 src/rclone_api/filelist.py
 src/rclone_api/group_files.py
 src/rclone_api/mount.py
+src/rclone_api/mount_read_chunker.py
 src/rclone_api/process.py
 src/rclone_api/rclone.py
 src/rclone_api/remote.py