rclone-api 1.3.0__tar.gz → 1.3.2__tar.gz

{rclone_api-1.3.0 → rclone_api-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.0
+Version: 1.3.2
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.0 → rclone_api-1.3.2}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.3.0"
+version = "1.3.2"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.3.0 → rclone_api-1.3.2}/src/rclone_api/__init__.py

@@ -1,6 +1,6 @@
 # Import logging module to activate default configuration
 
-from rclone_api import logging
+from rclone_api import log
 
 from .completed_process import CompletedProcess
 from .config import Config, Parsed, Section
@@ -11,7 +11,7 @@ from .file import File
 from .filelist import FileList
 
 # Import the configure_logging function to make it available at package level
-from .logging import configure_logging
+from .log import configure_logging, setup_default_logging
 from .process import Process
 from .rclone import Rclone, rclone_verbose
 from .remote import Remote
@@ -43,5 +43,7 @@ __all__ = [
     "MultiUploadResult",
     "SizeSuffix",
     "configure_logging",
-    "logging",
+    "log",
 ]
+
+setup_default_logging()

rclone_api-1.3.0/src/rclone_api/logging.py → rclone_api-1.3.2/src/rclone_api/log.py

@@ -1,9 +1,14 @@
 import logging
 import sys
 
+_INITIALISED = False
+
 
 def setup_default_logging():
     """Set up default logging configuration if none exists."""
+    global _INITIALISED
+    if _INITIALISED:
+        return
     if not logging.root.handlers:
         logging.basicConfig(
             level=logging.INFO,

{rclone_api-1.3.0 → rclone_api-1.3.2}/src/rclone_api/profile/mount_copy_bytes.py

@@ -164,9 +164,17 @@ def _run_profile(
     diff = (time.time() - start) / num
     net_io_end = psutil.net_io_counters()
     # self.assertEqual(len(bytes_or_err), size)
-    assert (
-        bytes_count == SizeSuffix(size * num).as_int()
-    ), f"Length: {SizeSuffix(bytes_count)} != {SizeSuffix(size* num)}"
+    # assert (
+    #     bytes_count == SizeSuffix(size * num).as_int()
+    # ), f"Length: {SizeSuffix(bytes_count)} != {SizeSuffix(size* num)}"
+
+    if bytes_count != SizeSuffix(size * num).as_int():
+        print("\n######################## ERROR ########################")
+        print(f"Error: Length: {SizeSuffix(bytes_count)} != {SizeSuffix(size* num)}")
+        print(f"  Bytes count: {bytes_count}")
+        print(f"  Size: {SizeSuffix(size * num)}")
+        print(f"  num: {num}")
+        print("########################################################\n")
 
     # print io stats
     bytes_sent = (net_io_end.bytes_sent - net_io_start.bytes_sent) // num
@@ -203,11 +211,11 @@ def test_profile_copy_bytes(
         sizes = [size.as_int()]
     else:
         sizes = [
-            # 1024 * 1024 * 1,
+            1024 * 1024 * 1,
             # 1024 * 1024 * 2,
             # 1024 * 1024 * 4,
             # 1024 * 1024 * 8,
-            1024 * 1024 * 16,
+            # 1024 * 1024 * 16,
             # 1024 * 1024 * 32,
             1024 * 1024 * 64,
             1024 * 1024 * 128,

{rclone_api-1.3.0 → rclone_api-1.3.2}/src/rclone_api/rclone.py

@@ -682,7 +682,7 @@ class Rclone:
         save_state_json: Path,
         chunk_size: SizeSuffix | None = None,
         read_threads: int = 8,
-        write_threads: int = 16,
+        write_threads: int = 8,
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
@@ -829,74 +829,6 @@ class Rclone:
         finally:
             chunk_fetcher.shutdown()
 
-    def _copy_bytes(
-        self,
-        src: str,
-        offset: int,
-        length: int,
-        transfers: int = 1,  # Note, increasing transfers can result in devestating drop in performance.
-        # If outfile is supplied then bytes are written to this file and success returns bytes(0)
-        outfile: Path | None = None,
-        mount_log: Path | None = None,
-        direct_io: bool = True,
-    ) -> bytes | Exception:
-        """Copy bytes from a file to another file."""
-        from rclone_api.util import random_str
-
-        tmp_mnts = Path("tmp_mnts") / random_str(12)
-        src_parent_path = Path(src).parent.as_posix()
-        src_file = Path(src).name
-        other_args: list[str] = [
-            "--no-modtime",
-            # "--vfs-read-wait", "1s"
-        ]
-        vfs_read_chunk_size = SizeSuffix(length // transfers)
-        vfs_read_chunk_size_limit = SizeSuffix(length)
-        vfs_read_chunk_streams = transfers
-        vfs_disk_space_total_size = SizeSuffix(length)
-        # --max-read-ahead SizeSuffix
-        max_read_ahead = SizeSuffix(vfs_read_chunk_size.as_int())
-
-        # other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
-        other_args += ["--vfs-read-chunk-size", str(vfs_read_chunk_size)]
-        other_args += ["--vfs-read-chunk-size-limit", str(vfs_read_chunk_size_limit)]
-        other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
-        other_args += ["--vfs-disk-space-total-size", str(vfs_disk_space_total_size)]
-        other_args += ["--max-read-ahead", str(max_read_ahead)]
-        other_args += ["--read-only"]
-        if direct_io:
-            other_args += ["--direct-io"]
-
-        with TemporaryDirectory() as tmpdir:
-            cache_dir = Path(tmpdir) / "cache"
-            other_args += ["--cache-dir", str(cache_dir.absolute())]
-            try:
-                # use scoped mount to do the read, then write the bytes to the destination
-                with self.scoped_mount(
-                    src_parent_path,
-                    tmp_mnts,
-                    use_links=True,
-                    verbose=mount_log is not None,
-                    vfs_cache_mode="minimal",
-                    other_args=other_args,
-                    log=mount_log,
-                    cache_dir=cache_dir,
-                    cache_dir_delete_on_exit=True,
-                ):
-                    src_file_mnt = tmp_mnts / src_file
-                    with open(src_file_mnt, "rb") as f:
-                        f.seek(offset)
-                        data = f.read(length)
-                        if outfile is None:
-                            return data
-                        with open(outfile, "wb") as out:
-                            out.write(data)
-                            del data
-                        return bytes(0)
-
-            except Exception as e:
-                return e
-
     def get_multi_mount_file_chunker(
         self,
         src: str,
@@ -928,16 +860,19 @@ class Rclone:
         if direct_io:
             other_args += ["--direct-io"]
 
+        base_mount_dir = Path("tmp_mnts")
+        base_cache_dir = Path("cache")
+
         filename = Path(src).name
         with ThreadPoolExecutor(max_workers=threads) as executor:
             futures: list[Future] = []
             try:
                 for i in range(threads):
-                    tmp_mnts = Path("tmp_mnts") / random_str(12)
+                    tmp_mnts = base_mount_dir / random_str(12)
                     verbose = mount_log is not None
 
                     src_parent_path = Path(src).parent.as_posix()
-                    cache_dir = Path("cache") / random_str(12)
+                    cache_dir = base_cache_dir / random_str(12)
 
                     def task(
                         src_parent_path=src_parent_path,
@@ -992,7 +927,7 @@ class Rclone:
         )
         return filechunker
 
-    def copy_bytes(
+    def copy_bytes_multimount(
         self,
         src: str,
         offset: int,
@@ -1004,7 +939,7 @@ class Rclone:
         mount_log: Path | None = None,
         direct_io: bool = True,
     ) -> bytes | Exception:
-        """Copy bytes from a file to another file."""
+        """Copy a slice of bytes from the src file to dst. Parallelism is achieved through multiple mounted files."""
         from rclone_api.types import FilePart
 
         # determine number of threads from chunk size
@@ -1022,12 +957,13 @@ class Rclone:
             payload = fp.payload
             if isinstance(payload, Exception):
                 return payload
-            if outfile is None:
-                out = payload.read_bytes()
-                payload.unlink()
-                return out
-            shutil.move(payload, outfile)
-            return bytes(0)
+            try:
+                if outfile is None:
+                    return payload.read_bytes()
+                shutil.move(payload, outfile)
+                return bytes(0)
+            finally:
+                fp.close()
 
         except Exception as e:
             warnings.warn(f"Error copying bytes: {e}")

rclone_api-1.3.2/src/rclone_api/s3/chunk_task.py

@@ -0,0 +1,196 @@
+import logging
+import time
+from concurrent.futures import Future
+from pathlib import Path
+from queue import Queue
+from threading import Event, Lock
+from typing import Any, Callable
+
+from rclone_api.mount_read_chunker import FilePart
+from rclone_api.s3.chunk_types import S3FileInfo, UploadState
+from rclone_api.types import EndOfStream
+
+logger = logging.getLogger(__name__)  # noqa
+
+
+# def _get_file_size(file_path: Path, timeout: int = 60) -> int:
+#     sleep_time = timeout / 60 if timeout > 0 else 1
+#     start = time.time()
+#     while True:
+#         expired = time.time() - start > timeout
+#         try:
+#             time.sleep(sleep_time)
+#             if file_path.exists():
+#                 return file_path.stat().st_size
+#         except FileNotFoundError as e:
+#             if expired:
+#                 print(f"File not found: {file_path}, exception is {e}")
+#                 raise
+#         if expired:
+#             raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
+
+
+class _ShouldStopChecker:
+    def __init__(self, max_chunks: int | None) -> None:
+        self.count = 0
+        self.max_chunks = max_chunks
+
+    def should_stop(self) -> bool:
+        if self.max_chunks is None:
+            return False
+        if self.count >= self.max_chunks:
+            logger.info(
+                f"Stopping file chunker after {self.count} chunks because it exceeded max_chunks {self.max_chunks}"
+            )
+            return True
+        # self.count += 1
+        return False
+
+    def increment(self):
+        self.count += 1
+
+
+class _PartNumberTracker:
+    def __init__(
+        self, start_part_value: int, last_part_value: int, done_parts: set[int]
+    ) -> None:
+        # self._num_parts = (last_part_value - start_part_value) + 1
+        self._start_part_value = start_part_value
+        self._last_part_value = last_part_value
+        self._done_part_numbers: set[int] = done_parts
+        self._curr_part_number = start_part_value
+        self._finished = False
+        self._lock = Lock()
+
+    def next_part_number(self) -> int | None:
+        with self._lock:
+            while self._curr_part_number in self._done_part_numbers:
+                self._curr_part_number += 1
+            if self._curr_part_number > self._last_part_value:
+                self._finished = True
+                return None
+            curr_part_number = self._curr_part_number
+            self._curr_part_number += (
+                1  # prevent a second thread from getting the same part number
+            )
+            return curr_part_number
+
+    def is_finished(self) -> bool:
+        with self._lock:
+            return self._finished
+
+    def add_finished_part_number(self, part_number: int) -> None:
+        with self._lock:
+            self._done_part_numbers.add(part_number)
+
+
+class _OnCompleteHandler:
+    def __init__(
+        self,
+        part_number_tracker: _PartNumberTracker,
+        file_path: Path,
+        queue_upload: Queue[FilePart | EndOfStream],
+    ) -> None:
+        self.part_number_tracker = part_number_tracker
+        self.file_path = file_path
+        self.queue_upload = queue_upload
+
+    def on_complete(self, fut: Future[FilePart]) -> None:
+        logger.debug("Chunk read complete")
+        fp: FilePart = fut.result()
+        extra: S3FileInfo = fp.extra
+        assert isinstance(extra, S3FileInfo)
+        part_number = extra.part_number
+        if fp.is_error():
+            logger.warning(f"Error reading file: {fp}, skipping part {part_number}")
+            return
+
+        if fp.n_bytes() == 0:
+            logger.warning(f"Empty data for part {part_number} of {self.file_path}")
+            raise ValueError(f"Empty data for part {part_number} of {self.file_path}")
+
+        if isinstance(fp.payload, Exception):
+            logger.warning(f"Error reading file because of error: {fp.payload}")
+            return
+
+        # done_part_numbers.add(part_number)
+        # queue_upload.put(fp)
+        self.part_number_tracker.add_finished_part_number(part_number)
+        self.queue_upload.put(fp)
+
+
+def file_chunker(
+    upload_state: UploadState,
+    fetcher: Callable[[int, int, Any], Future[FilePart]],
+    max_chunks: int | None,
+    cancel_signal: Event,
+    queue_upload: Queue[FilePart | EndOfStream],
+) -> None:
+    final_part_number = upload_state.upload_info.total_chunks() + 1
+    should_stop_checker = _ShouldStopChecker(max_chunks)
+
+    upload_info = upload_state.upload_info
+    file_path = upload_info.src_file_path
+    chunk_size = upload_info.chunk_size
+
+    done_part_numbers: set[int] = {
+        p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
+    }
+
+    part_tracker = _PartNumberTracker(
+        start_part_value=1,
+        last_part_value=final_part_number,
+        done_parts=done_part_numbers,
+    )
+
+    callback = _OnCompleteHandler(part_tracker, file_path, queue_upload)
+
+    try:
+        num_parts = upload_info.total_chunks()
+
+        if cancel_signal.is_set():
+            logger.info(
+                f"Cancel signal is set for file chunker while processing {file_path}, returning"
+            )
+            return
+
+        while not should_stop_checker.should_stop():
+            should_stop_checker.increment()
+            logger.debug("Processing next chunk")
+            curr_part_number = part_tracker.next_part_number()
+            if curr_part_number is None:
+                logger.info(f"File {file_path} has completed chunking all parts")
+                break
+
+            assert curr_part_number is not None
+            offset = (curr_part_number - 1) * chunk_size
+            file_size = upload_info.file_size
+
+            assert offset < file_size, f"Offset {offset} is greater than file size"
+            fetch_size = max(0, min(chunk_size, file_size - offset))
+            if fetch_size == 0:
+                logger.error(
+                    f"Empty data for part {curr_part_number} of {file_path}, is this the last chunk?"
+                )
+                # assert final_part_number == curr_part_number, f"Final part number is {final_part_number} but current part number is {curr_part_number}"
+                if final_part_number != curr_part_number:
+                    raise ValueError(
+                        f"This should have been the last part, but it is not: {final_part_number} != {curr_part_number}"
+                    )
+
+            assert curr_part_number is not None
+            logger.info(
+                f"Reading chunk {curr_part_number} of {num_parts} for {file_path}"
+            )
+            fut = fetcher(
+                offset, fetch_size, S3FileInfo(upload_info.upload_id, curr_part_number)
+            )
+            fut.add_done_callback(callback.on_complete)
+            # wait until the queue_upload queue can accept the next chunk
+            while queue_upload.full():
+                time.sleep(0.1)
+    except Exception as e:
+        logger.error(f"Error reading file: {e}", exc_info=True)
+    finally:
+        logger.info(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
+        queue_upload.put(EndOfStream())

{rclone_api-1.3.0 → rclone_api-1.3.2}/src/rclone_api/s3/chunk_types.py

@@ -14,6 +14,12 @@ from rclone_api.util import locked_print
 _SAVE_STATE_LOCK = Lock()
 
 
+@dataclass
+class S3FileInfo:
+    upload_id: str
+    part_number: int
+
+
 @dataclass
 class UploadInfo:
     s3_client: BaseClient

{rclone_api-1.3.0 → rclone_api-1.3.2}/src/rclone_api/s3/upload_file_multipart.py

@@ -11,7 +11,7 @@ from typing import Any, Callable
 from botocore.client import BaseClient
 
 from rclone_api.mount_read_chunker import FilePart
-from rclone_api.s3.chunk_file import S3FileInfo, file_chunker
+from rclone_api.s3.chunk_task import S3FileInfo, file_chunker
 from rclone_api.s3.chunk_types import (
     FinishedPiece,
     UploadInfo,

{rclone_api-1.3.0 → rclone_api-1.3.2}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.0
+Version: 1.3.2
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.0 → rclone_api-1.3.2}/src/rclone_api.egg-info/SOURCES.txt

@@ -33,7 +33,7 @@ src/rclone_api/exec.py
 src/rclone_api/file.py
 src/rclone_api/filelist.py
 src/rclone_api/group_files.py
-src/rclone_api/logging.py
+src/rclone_api/log.py
 src/rclone_api/mount.py
 src/rclone_api/mount_read_chunker.py
 src/rclone_api/process.py
@@ -58,7 +58,7 @@ src/rclone_api/experimental/flags_base.py
 src/rclone_api/profile/mount_copy_bytes.py
 src/rclone_api/s3/api.py
 src/rclone_api/s3/basic_ops.py
-src/rclone_api/s3/chunk_file.py
+src/rclone_api/s3/chunk_task.py
 src/rclone_api/s3/chunk_types.py
 src/rclone_api/s3/create.py
 src/rclone_api/s3/types.py

{rclone_api-1.3.0 → rclone_api-1.3.2}/tests/test_copy_bytes.py

@@ -72,7 +72,7 @@ class RcloneCopyBytesTester(unittest.TestCase):
 
     def test_copy_bytes(self) -> None:
         rclone = Rclone(_generate_rclone_config())
-        bytes_or_err: bytes | Exception = rclone.copy_bytes(
+        bytes_or_err: bytes | Exception = rclone.copy_bytes_multimount(
             src="dst:rclone-api-unit-test/zachs_video/breaking_ai_mind.mp4",
             offset=0,
             length=1024 * 1024,
@@ -92,7 +92,7 @@ class RcloneCopyBytesTester(unittest.TestCase):
             tmp = Path(tmpdir) / "tmp.mp4"
             log = Path(tmpdir) / "log.txt"
             rclone = Rclone(_generate_rclone_config())
-            bytes_or_err: bytes | Exception = rclone.copy_bytes(
+            bytes_or_err: bytes | Exception = rclone.copy_bytes_multimount(
                 src="dst:rclone-api-unit-test/zachs_video/breaking_ai_mind.mp4",
                 offset=0,
                 length=1024 * 1024,

rclone_api-1.3.0/src/rclone_api/s3/chunk_file.py

@@ -1,146 +0,0 @@
-import logging
-import time
-from concurrent.futures import Future
-from dataclasses import dataclass
-from pathlib import Path
-from queue import Queue
-from threading import Event
-from typing import Any, Callable
-
-from rclone_api.mount_read_chunker import FilePart
-from rclone_api.s3.chunk_types import UploadState
-from rclone_api.types import EndOfStream
-
-logger = logging.getLogger(__name__)  # noqa
-
-
-def _get_file_size(file_path: Path, timeout: int = 60) -> int:
-    sleep_time = timeout / 60 if timeout > 0 else 1
-    start = time.time()
-    while True:
-        expired = time.time() - start > timeout
-        try:
-            time.sleep(sleep_time)
-            if file_path.exists():
-                return file_path.stat().st_size
-        except FileNotFoundError as e:
-            if expired:
-                print(f"File not found: {file_path}, exception is {e}")
-                raise
-        if expired:
-            raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
-
-
-@dataclass
-class S3FileInfo:
-    upload_id: str
-    part_number: int
-
-
-def file_chunker(
-    upload_state: UploadState,
-    fetcher: Callable[[int, int, Any], Future[FilePart]],
-    max_chunks: int | None,
-    cancel_signal: Event,
-    queue_upload: Queue[FilePart | EndOfStream],
-) -> None:
-    count = 0
-
-    def should_stop() -> bool:
-        nonlocal count
-
-        if max_chunks is None:
-            return False
-        if count >= max_chunks:
-            logger.info(
-                f"Stopping file chunker after {count} chunks because it exceeded max_chunks {max_chunks}"
-            )
-            return True
-        count += 1
-        return False
-
-    upload_info = upload_state.upload_info
-    file_path = upload_info.src_file_path
-    chunk_size = upload_info.chunk_size
-    # src = Path(file_path)
-
-    try:
-        part_number = 1
-        done_part_numbers: set[int] = {
-            p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
-        }
-        num_parts = upload_info.total_chunks()
-
-        def next_part_number() -> int | None:
-            nonlocal part_number
-            while part_number in done_part_numbers:
-                part_number += 1
-            if part_number > num_parts:
-                return None
-            return part_number
-
-        if cancel_signal.is_set():
-            logger.info(
-                f"Cancel signal is set for file chunker while processing {file_path}, returning"
-            )
-            return
-
-        while not should_stop():
-            logger.debug("Processing next chunk")
-            curr_part_number = next_part_number()
-            if curr_part_number is None:
-                logger.info(f"File {file_path} has completed chunking all parts")
-                break
-            assert curr_part_number is not None
-            offset = (curr_part_number - 1) * chunk_size
-            file_size = upload_info.file_size
-
-            assert offset < file_size, f"Offset {offset} is greater than file size"
-
-            # Open the file, seek, read the chunk, and close immediately.
-            # with open(file_path, "rb") as f:
-            #     f.seek(offset)
-            #     data = f.read(chunk_size)
-
-            # data = chunk_fetcher(offset, chunk_size).result()
-
-            assert curr_part_number is not None
-            cpn: int = curr_part_number
-
-            def on_complete(fut: Future[FilePart]) -> None:
-                logger.debug("Chunk read complete")
-                fp: FilePart = fut.result()
-                if fp.is_error():
-                    logger.warning(
-                        f"Error reading file: {fp}, skipping part {part_number}"
-                    )
-                    return
-
-                if fp.n_bytes() == 0:
-                    logger.warning(f"Empty data for part {part_number} of {file_path}")
-                    raise ValueError(
-                        f"Empty data for part {part_number} of {file_path}"
-                    )
-
-                if isinstance(fp.payload, Exception):
-                    logger.warning(f"Error reading file because of error: {fp.payload}")
-                    return
-
-                done_part_numbers.add(part_number)
-                queue_upload.put(fp)
-
-            offset = (curr_part_number - 1) * chunk_size
-            logger.info(
-                f"Reading chunk {curr_part_number} of {num_parts} for {file_path}"
-            )
-            fut = fetcher(offset, file_size, S3FileInfo(upload_info.upload_id, cpn))
-            fut.add_done_callback(on_complete)
-            # wait until the queue_upload queue can accept the next chunk
-            while queue_upload.full():
-                time.sleep(0.1)
-    except Exception as e:
-
-        logger.error(f"Error reading file: {e}", exc_info=True)
-    finally:
-        logger.info(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
-        queue_upload.put(EndOfStream())