rclone-api 1.2.154__py2.py3-none-any.whl → 1.3.1__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -1,5 +1,6 @@
 # Import logging module to activate default configuration
-import rclone_api.logging  # noqa: F401
+
+from rclone_api import log
 
 from .completed_process import CompletedProcess
 from .config import Config, Parsed, Section
@@ -10,7 +11,7 @@ from .file import File
 from .filelist import FileList
 
 # Import the configure_logging function to make it available at package level
-from .logging import configure_logging
+from .log import configure_logging, setup_default_logging
 from .process import Process
 from .rclone import Rclone, rclone_verbose
 from .remote import Remote
@@ -42,4 +43,7 @@ __all__ = [
     "MultiUploadResult",
     "SizeSuffix",
     "configure_logging",
+    "log",
 ]
+
+setup_default_logging()

rclone_api/{logging.py → log.py}

@@ -1,9 +1,14 @@
 import logging
 import sys
 
+_INITIALISED = False
+
 
 def setup_default_logging():
     """Set up default logging configuration if none exists."""
+    global _INITIALISED
+    if _INITIALISED:
+        return
     if not logging.root.handlers:
         logging.basicConfig(
             level=logging.INFO,

rclone_api/mount_read_chunker.py

@@ -94,7 +94,7 @@ class MultiMountFileChunker:
         assert offset >= 0, f"Invalid offset: {offset}"
         assert (
             offset + size <= self.filesize
-        ), f"Invalid offset + size: {offset} + {size} ({offset+size}) <= {self.filesize}"
+        ), f"Invalid offset + size: {offset} + {size} ({offset+size}) > {self.filesize}"
 
         try:
             mount = self._acquire_mount()

rclone_api/s3/chunk_task.py

@@ -0,0 +1,215 @@
+import logging
+import time
+from concurrent.futures import Future
+from pathlib import Path
+from queue import Queue
+from threading import Event, Lock
+from typing import Any, Callable
+
+from rclone_api.mount_read_chunker import FilePart
+from rclone_api.s3.chunk_types import S3FileInfo, UploadState
+from rclone_api.types import EndOfStream
+
+logger = logging.getLogger(__name__)  # noqa
+
+
+# def _get_file_size(file_path: Path, timeout: int = 60) -> int:
+#     sleep_time = timeout / 60 if timeout > 0 else 1
+#     start = time.time()
+#     while True:
+#         expired = time.time() - start > timeout
+#         try:
+#             time.sleep(sleep_time)
+#             if file_path.exists():
+#                 return file_path.stat().st_size
+#         except FileNotFoundError as e:
+#             if expired:
+#                 print(f"File not found: {file_path}, exception is {e}")
+#                 raise
+#         if expired:
+#             raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
+
+
+class _ShouldStopChecker:
+    def __init__(self, max_chunks: int | None) -> None:
+        self.count = 0
+        self.max_chunks = max_chunks
+
+    def should_stop(self) -> bool:
+        if self.max_chunks is None:
+            return False
+        if self.count >= self.max_chunks:
+            logger.info(
+                f"Stopping file chunker after {self.count} chunks because it exceeded max_chunks {self.max_chunks}"
+            )
+            return True
+        # self.count += 1
+        return False
+
+    def increment(self):
+        self.count += 1
+
+
+class _PartNumberTracker:
+    def __init__(
+        self, start_part_value: int, last_part_value: int, done_parts: set[int]
+    ) -> None:
+        # self._num_parts = (last_part_value - start_part_value) + 1
+        self._start_part_value = start_part_value
+        self._last_part_value = last_part_value
+        self._done_part_numbers: set[int] = done_parts
+        self._curr_part_number = start_part_value
+        self._finished = False
+        self._lock = Lock()
+
+    def next_part_number(self) -> int | None:
+        with self._lock:
+            while self._curr_part_number in self._done_part_numbers:
+                self._curr_part_number += 1
+            if self._curr_part_number > self._last_part_value:
+                self._finished = True
+                return None
+            curr_part_number = self._curr_part_number
+            self._curr_part_number += (
+                1  # prevent a second thread from getting the same part number
+            )
+            return curr_part_number
+
+    def is_finished(self) -> bool:
+        with self._lock:
+            return self._finished
+
+    def add_finished_part_number(self, part_number: int) -> None:
+        with self._lock:
+            self._done_part_numbers.add(part_number)
+
+
+class _OnCompleteHandler:
+    def __init__(
+        self,
+        part_number_tracker: _PartNumberTracker,
+        file_path: Path,
+        queue_upload: Queue[FilePart | EndOfStream],
+    ) -> None:
+        self.part_number_tracker = part_number_tracker
+        self.file_path = file_path
+        self.queue_upload = queue_upload
+
+    def on_complete(self, fut: Future[FilePart]) -> None:
+        logger.debug("Chunk read complete")
+        fp: FilePart = fut.result()
+        extra: S3FileInfo = fp.extra
+        assert isinstance(extra, S3FileInfo)
+        part_number = extra.part_number
+        if fp.is_error():
+            logger.warning(f"Error reading file: {fp}, skipping part {part_number}")
+            return
+
+        if fp.n_bytes() == 0:
+            logger.warning(f"Empty data for part {part_number} of {self.file_path}")
+            raise ValueError(f"Empty data for part {part_number} of {self.file_path}")
+
+        if isinstance(fp.payload, Exception):
+            logger.warning(f"Error reading file because of error: {fp.payload}")
+            return
+
+        # done_part_numbers.add(part_number)
+        # queue_upload.put(fp)
+        self.part_number_tracker.add_finished_part_number(part_number)
+        self.queue_upload.put(fp)
+
+
+def file_chunker(
+    upload_state: UploadState,
+    fetcher: Callable[[int, int, Any], Future[FilePart]],
+    max_chunks: int | None,
+    cancel_signal: Event,
+    queue_upload: Queue[FilePart | EndOfStream],
+) -> None:
+    final_part_number = upload_state.upload_info.total_chunks() + 1
+    should_stop_checker = _ShouldStopChecker(max_chunks)
+
+    upload_info = upload_state.upload_info
+    file_path = upload_info.src_file_path
+    chunk_size = upload_info.chunk_size
+    # src = Path(file_path)
+
+    # for p in upload_state.parts:
+    #     if not isinstance(p, EndOfStream):
+    #         part_tracker.add_done_part_number(p.part_number)
+
+    done_part_numbers: set[int] = {
+        p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
+    }
+
+    part_tracker = _PartNumberTracker(
+        start_part_value=1,
+        last_part_value=final_part_number,
+        done_parts=done_part_numbers,
+    )
+
+    callback = _OnCompleteHandler(part_tracker, file_path, queue_upload)
+
+    try:
+        num_parts = upload_info.total_chunks()
+
+        if cancel_signal.is_set():
+            logger.info(
+                f"Cancel signal is set for file chunker while processing {file_path}, returning"
+            )
+            return
+
+        while not should_stop_checker.should_stop():
+            should_stop_checker.increment()
+            logger.debug("Processing next chunk")
+            curr_part_number = part_tracker.next_part_number()
+            if curr_part_number is None:
+                logger.info(f"File {file_path} has completed chunking all parts")
+                break
+
+            assert curr_part_number is not None
+            offset = (curr_part_number - 1) * chunk_size
+            file_size = upload_info.file_size
+
+            assert offset < file_size, f"Offset {offset} is greater than file size"
+            fetch_size = max(0, min(chunk_size, file_size - offset))
+
+            # assert fetch_size > 0, f"Invalid fetch size: {fetch_size}"
+
+            if fetch_size == 0:
+                logger.error(
+                    f"Empty data for part {curr_part_number} of {file_path}, is this the last chunk?"
+                )
+                # assert final_part_number == curr_part_number, f"Final part number is {final_part_number} but current part number is {curr_part_number}"
+                if final_part_number != curr_part_number:
+                    raise ValueError(
+                        f"This should have been the last part, but it is not: {final_part_number} != {curr_part_number}"
+                    )
+
+            # Open the file, seek, read the chunk, and close immediately.
+            # with open(file_path, "rb") as f:
+            #     f.seek(offset)
+            #     data = f.read(chunk_size)
+
+            # data = chunk_fetcher(offset, chunk_size).result()
+
+            assert curr_part_number is not None
+            # cpn: int = curr_part_number
+
+            # offset = (curr_part_number - 1) * chunk_size
+            logger.info(
+                f"Reading chunk {curr_part_number} of {num_parts} for {file_path}"
+            )
+            fut = fetcher(
+                offset, fetch_size, S3FileInfo(upload_info.upload_id, curr_part_number)
+            )
+            fut.add_done_callback(callback.on_complete)
+            # wait until the queue_upload queue can accept the next chunk
+            while queue_upload.full():
+                time.sleep(0.1)
+    except Exception as e:
+
+        logger.error(f"Error reading file: {e}", exc_info=True)
+    finally:
+        logger.info(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
+        queue_upload.put(EndOfStream())

rclone_api/s3/chunk_types.py

@@ -14,6 +14,12 @@ from rclone_api.util import locked_print
 _SAVE_STATE_LOCK = Lock()
 
 
+@dataclass
+class S3FileInfo:
+    upload_id: str
+    part_number: int
+
+
 @dataclass
 class UploadInfo:
     s3_client: BaseClient

rclone_api/s3/upload_file_multipart.py

@@ -11,7 +11,7 @@ from typing import Any, Callable
 from botocore.client import BaseClient
 
 from rclone_api.mount_read_chunker import FilePart
-from rclone_api.s3.chunk_file import S3FileInfo, file_chunker
+from rclone_api.s3.chunk_task import S3FileInfo, file_chunker
 from rclone_api.s3.chunk_types import (
     FinishedPiece,
     UploadInfo,

{rclone_api-1.2.154.dist-info → rclone_api-1.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.2.154
+Version: 1.3.1
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.2.154.dist-info → rclone_api-1.3.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=dPa4vtqtaO47JZa-YUjOgVa2gCSHLdGxM-_xX_4C7Yw,1150
+rclone_api/__init__.py,sha256=96hecpVEvaLklfaL5K_uBV-gaftt9kklC0XMlutZMf8,1197
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -11,9 +11,9 @@ rclone_api/exec.py,sha256=Pd7pUBd8ib5MzqvMybG2DQISPRbDRu20VjVRL2mLAVY,1076
 rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/logging.py,sha256=fJ4Hr4baAEv93oOOiyzNfoQ8eD0MuErT3NHMjBC3W_w,1184
+rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
-rclone_api/mount_read_chunker.py,sha256=IH2YcB-N22oiJLkp7KlpG3A4VZkkHOqTYvDLyor2e7Q,4505
+rclone_api/mount_read_chunker.py,sha256=Pz4s6AGigB_-vTAEdLK9Fv116j87yN4Qg0Uz1COwPtQ,4504
 rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
 rclone_api/rclone.py,sha256=WSCbIo4NtD871nvR1ZG9v7ihbLi7z1Ku4Pg4mgrNxZw,49364
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
@@ -30,14 +30,14 @@ rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ
 rclone_api/profile/mount_copy_bytes.py,sha256=okzcfpmLcQvh5IUcIwZs9jLPSxFMv2igt2-kHoEmlfE,8571
 rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_file.py,sha256=xtg9g4BvaFsipyfj6p5iRitR53jXjBqX0tmtO7Vf3Us,5068
-rclone_api/s3/chunk_types.py,sha256=I0YCWFgxCvmt8cp4tMabiiwiD2yKTcbA6ZL2D3xnn5w,8781
+rclone_api/s3/chunk_task.py,sha256=qJLoacUIucGh7h4Puo7wfqSYYoz7dxG-5cJ3TopgzwM,7735
+rclone_api/s3/chunk_types.py,sha256=oSWv8No9V3BeM7IcGnowyR2a7YrszdAXzEJlxaeZcp0,8852
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=Elmh__gvZJyJyElYwMmvYZIBIunDJiTRAbEg21GmsRU,1604
-rclone_api/s3/upload_file_multipart.py,sha256=inoMOQDZZYqTitJz3f0BBHo3F9ZYm8VhL4UTzPmcdm0,11385
-rclone_api-1.2.154.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.2.154.dist-info/METADATA,sha256=RTR14SPQNbWbnFKlWuMGj-y3CdZ4tU3x70odXfdXJVQ,4538
-rclone_api-1.2.154.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.2.154.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.2.154.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.2.154.dist-info/RECORD,,
+rclone_api/s3/upload_file_multipart.py,sha256=UVMTSeP98fSQdOYcCdi9tV5ZjOxRDuhZbiBbVaf-rCM,11385
+rclone_api-1.3.1.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.3.1.dist-info/METADATA,sha256=ZjOE584_hiG9__K0BqmA3OYmTb076iT18dznebi5D98,4536
+rclone_api-1.3.1.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.3.1.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.3.1.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.3.1.dist-info/RECORD,,

rclone_api/s3/chunk_file.py

@@ -1,146 +0,0 @@
-import logging
-import time
-from concurrent.futures import Future
-from dataclasses import dataclass
-from pathlib import Path
-from queue import Queue
-from threading import Event
-from typing import Any, Callable
-
-from rclone_api.mount_read_chunker import FilePart
-from rclone_api.s3.chunk_types import UploadState
-from rclone_api.types import EndOfStream
-
-logger = logging.getLogger(__name__)  # noqa
-
-
-def _get_file_size(file_path: Path, timeout: int = 60) -> int:
-    sleep_time = timeout / 60 if timeout > 0 else 1
-    start = time.time()
-    while True:
-        expired = time.time() - start > timeout
-        try:
-            time.sleep(sleep_time)
-            if file_path.exists():
-                return file_path.stat().st_size
-        except FileNotFoundError as e:
-            if expired:
-                print(f"File not found: {file_path}, exception is {e}")
-                raise
-        if expired:
-            raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
-
-
-@dataclass
-class S3FileInfo:
-    upload_id: str
-    part_number: int
-
-
-def file_chunker(
-    upload_state: UploadState,
-    fetcher: Callable[[int, int, Any], Future[FilePart]],
-    max_chunks: int | None,
-    cancel_signal: Event,
-    queue_upload: Queue[FilePart | EndOfStream],
-) -> None:
-    count = 0
-
-    def should_stop() -> bool:
-        nonlocal count
-
-        if max_chunks is None:
-            return False
-        if count >= max_chunks:
-            logger.info(
-                f"Stopping file chunker after {count} chunks because it exceeded max_chunks {max_chunks}"
-            )
-            return True
-        count += 1
-        return False
-
-    upload_info = upload_state.upload_info
-    file_path = upload_info.src_file_path
-    chunk_size = upload_info.chunk_size
-    # src = Path(file_path)
-
-    try:
-        part_number = 1
-        done_part_numbers: set[int] = {
-            p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
-        }
-        num_parts = upload_info.total_chunks()
-
-        def next_part_number() -> int | None:
-            nonlocal part_number
-            while part_number in done_part_numbers:
-                part_number += 1
-            if part_number > num_parts:
-                return None
-            return part_number
-
-        if cancel_signal.is_set():
-            logger.info(
-                f"Cancel signal is set for file chunker while processing {file_path}, returning"
-            )
-            return
-
-        while not should_stop():
-            logger.debug("Processing next chunk")
-            curr_part_number = next_part_number()
-            if curr_part_number is None:
-                logger.info(f"File {file_path} has completed chunking all parts")
-                break
-            assert curr_part_number is not None
-            offset = (curr_part_number - 1) * chunk_size
-            file_size = upload_info.file_size
-
-            assert offset < file_size, f"Offset {offset} is greater than file size"
-
-            # Open the file, seek, read the chunk, and close immediately.
-            # with open(file_path, "rb") as f:
-            #     f.seek(offset)
-            #     data = f.read(chunk_size)
-
-            # data = chunk_fetcher(offset, chunk_size).result()
-
-            assert curr_part_number is not None
-            cpn: int = curr_part_number
-
-            def on_complete(fut: Future[FilePart]) -> None:
-                logger.debug("Chunk read complete")
-                fp: FilePart = fut.result()
-                if fp.is_error():
-                    logger.warning(
-                        f"Error reading file: {fp}, skipping part {part_number}"
-                    )
-                    return
-
-                if fp.n_bytes() == 0:
-                    logger.warning(f"Empty data for part {part_number} of {file_path}")
-                    raise ValueError(
-                        f"Empty data for part {part_number} of {file_path}"
-                    )
-
-                if isinstance(fp.payload, Exception):
-                    logger.warning(f"Error reading file because of error: {fp.payload}")
-                    return
-
-                done_part_numbers.add(part_number)
-                queue_upload.put(fp)
-
-            offset = (curr_part_number - 1) * chunk_size
-            logger.info(
-                f"Reading chunk {curr_part_number} of {num_parts} for {file_path}"
-            )
-            fut = fetcher(offset, file_size, S3FileInfo(upload_info.upload_id, cpn))
-            fut.add_done_callback(on_complete)
-            # wait until the queue_upload queue can accept the next chunk
-            while queue_upload.full():
-                time.sleep(0.1)
-    except Exception as e:
-
-        logger.error(f"Error reading file: {e}", exc_info=True)
-    finally:
-        logger.info(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
-        queue_upload.put(EndOfStream())