rclone-api 1.2.14__py2.py3-none-any.whl → 1.2.154__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -1,38 +1,45 @@
-from .completed_process import CompletedProcess
-from .config import Config, Parsed, Section
-from .diff import DiffItem, DiffOption, DiffType
-from .dir import Dir
-from .dir_listing import DirListing
-from .file import File
-from .filelist import FileList
-from .process import Process
-from .rclone import Rclone, rclone_verbose
-from .remote import Remote
-from .rpath import RPath
-from .s3.types import MultiUploadResult
-from .types import ListingOption, Order, SizeResult, SizeSuffix
-
-__all__ = [
-    "Rclone",
-    "File",
-    "Config",
-    "Remote",
-    "Dir",
-    "RPath",
-    "DirListing",
-    "FileList",
-    "Process",
-    "DiffItem",
-    "DiffType",
-    "rclone_verbose",
-    "CompletedProcess",
-    "DiffOption",
-    "ListingOption",
-    "Order",
-    "ListingOption",
-    "SizeResult",
-    "Parsed",
-    "Section",
-    "MultiUploadResult",
-    "SizeSuffix",
-]
+# Import logging module to activate default configuration
+import rclone_api.logging  # noqa: F401
+
+from .completed_process import CompletedProcess
+from .config import Config, Parsed, Section
+from .diff import DiffItem, DiffOption, DiffType
+from .dir import Dir
+from .dir_listing import DirListing
+from .file import File
+from .filelist import FileList
+
+# Import the configure_logging function to make it available at package level
+from .logging import configure_logging
+from .process import Process
+from .rclone import Rclone, rclone_verbose
+from .remote import Remote
+from .rpath import RPath
+from .s3.types import MultiUploadResult
+from .types import ListingOption, Order, SizeResult, SizeSuffix
+
+__all__ = [
+    "Rclone",
+    "File",
+    "Config",
+    "Remote",
+    "Dir",
+    "RPath",
+    "DirListing",
+    "FileList",
+    "Process",
+    "DiffItem",
+    "DiffType",
+    "rclone_verbose",
+    "CompletedProcess",
+    "DiffOption",
+    "ListingOption",
+    "Order",
+    "ListingOption",
+    "SizeResult",
+    "Parsed",
+    "Section",
+    "MultiUploadResult",
+    "SizeSuffix",
+    "configure_logging",
+]

rclone_api/logging.py

@@ -0,0 +1,39 @@
+import logging
+import sys
+
+
+def setup_default_logging():
+    """Set up default logging configuration if none exists."""
+    if not logging.root.handlers:
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            handlers=[
+                logging.StreamHandler(sys.stdout),
+                # Uncomment to add file logging
+                # logging.FileHandler('rclone_api.log')
+            ],
+        )
+
+
+def configure_logging(level=logging.INFO, log_file=None):
+    """Configure logging for the rclone_api package.
+
+    Args:
+        level: The logging level (default: logging.INFO)
+        log_file: Optional path to a log file
+    """
+    handlers = [logging.StreamHandler(sys.stdout)]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file))
+
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        handlers=handlers,
+        force=True,  # Override any existing configuration
+    )
+
+
+# Call setup_default_logging when this module is imported
+setup_default_logging()

rclone_api/s3/chunk_file.py

@@ -1,145 +1,146 @@
-import time
-import warnings
-from concurrent.futures import Future
-from dataclasses import dataclass
-from pathlib import Path
-from queue import Queue
-from threading import Event
-from typing import Any, Callable
-
-from rclone_api.mount_read_chunker import FilePart
-from rclone_api.s3.chunk_types import UploadState
-from rclone_api.types import EndOfStream
-from rclone_api.util import locked_print
-
-
-def _get_file_size(file_path: Path, timeout: int = 60) -> int:
-    sleep_time = timeout / 60 if timeout > 0 else 1
-    start = time.time()
-    while True:
-        expired = time.time() - start > timeout
-        try:
-            time.sleep(sleep_time)
-            if file_path.exists():
-                return file_path.stat().st_size
-        except FileNotFoundError as e:
-            if expired:
-                print(f"File not found: {file_path}, exception is {e}")
-                raise
-        if expired:
-            raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
-
-
-@dataclass
-class S3FileInfo:
-    upload_id: str
-    part_number: int
-
-
-def file_chunker(
-    upload_state: UploadState,
-    fetcher: Callable[[int, int, Any], Future[FilePart]],
-    max_chunks: int | None,
-    cancel_signal: Event,
-    queue_upload: Queue[FilePart | EndOfStream],
-) -> None:
-    count = 0
-
-    def should_stop() -> bool:
-        nonlocal count
-
-        if max_chunks is None:
-            return False
-        if count >= max_chunks:
-            print(
-                f"Stopping file chunker after {count} chunks because it exceeded max_chunks {max_chunks}"
-            )
-            return True
-        count += 1
-        return False
-
-    upload_info = upload_state.upload_info
-    file_path = upload_info.src_file_path
-    chunk_size = upload_info.chunk_size
-    # src = Path(file_path)
-
-    try:
-        part_number = 1
-        done_part_numbers: set[int] = {
-            p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
-        }
-        num_parts = upload_info.total_chunks()
-
-        def next_part_number() -> int | None:
-            nonlocal part_number
-            while part_number in done_part_numbers:
-                part_number += 1
-            if part_number > num_parts:
-                return None
-            return part_number
-
-        if cancel_signal.is_set():
-            print(
-                f"Cancel signal is set for file chunker while processing {file_path}, returning"
-            )
-            return
-
-        while not should_stop():
-            print("@@@@@@@@@@")
-            curr_part_number = next_part_number()
-            if curr_part_number is None:
-                locked_print(f"File {file_path} has completed chunking all parts")
-                break
-            assert curr_part_number is not None
-            offset = (curr_part_number - 1) * chunk_size
-            file_size = upload_info.file_size
-
-            assert offset < file_size, f"Offset {offset} is greater than file size"
-
-            # Open the file, seek, read the chunk, and close immediately.
-            # with open(file_path, "rb") as f:
-            #     f.seek(offset)
-            #     data = f.read(chunk_size)
-
-            # data = chunk_fetcher(offset, chunk_size).result()
-
-            assert curr_part_number is not None
-            cpn: int = curr_part_number
-
-            def on_complete(fut: Future[FilePart]) -> None:
-                print("ON COMPLETE")
-                fp: FilePart = fut.result()
-                if fp.is_error():
-                    warnings.warn(
-                        f"Error reading file: {fp}, skipping part {part_number}"
-                    )
-                    return
-
-                if fp.n_bytes() == 0:
-                    warnings.warn(f"Empty data for part {part_number} of {file_path}")
-                    raise ValueError(
-                        f"Empty data for part {part_number} of {file_path}"
-                    )
-
-                if isinstance(fp.payload, Exception):
-                    warnings.warn(f"Error reading file because of error: {fp.payload}")
-                    return
-
-                done_part_numbers.add(part_number)
-                queue_upload.put(fp)
-
-            offset = (curr_part_number - 1) * chunk_size
-            print(f"Reading chunk {curr_part_number} of {num_parts} for {file_path}")
-            fut = fetcher(offset, file_size, S3FileInfo(upload_info.upload_id, cpn))
-            fut.add_done_callback(on_complete)
-            # wait until the queue_upload queue can accept the next chunk
-            while queue_upload.full():
-                time.sleep(0.1)
-    except Exception as e:
-
-        warnings.warn(f"Error reading file: {e}")
-    finally:
-        print("#############################################################")
-        print(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
-        print("#############################################################")
-        queue_upload.put(EndOfStream())
+import logging
+import time
+from concurrent.futures import Future
+from dataclasses import dataclass
+from pathlib import Path
+from queue import Queue
+from threading import Event
+from typing import Any, Callable
+
+from rclone_api.mount_read_chunker import FilePart
+from rclone_api.s3.chunk_types import UploadState
+from rclone_api.types import EndOfStream
+
+logger = logging.getLogger(__name__)  # noqa
+
+
+def _get_file_size(file_path: Path, timeout: int = 60) -> int:
+    sleep_time = timeout / 60 if timeout > 0 else 1
+    start = time.time()
+    while True:
+        expired = time.time() - start > timeout
+        try:
+            time.sleep(sleep_time)
+            if file_path.exists():
+                return file_path.stat().st_size
+        except FileNotFoundError as e:
+            if expired:
+                print(f"File not found: {file_path}, exception is {e}")
+                raise
+        if expired:
+            raise TimeoutError(f"File {file_path} not found after {timeout} seconds")
+
+
+@dataclass
+class S3FileInfo:
+    upload_id: str
+    part_number: int
+
+
+def file_chunker(
+    upload_state: UploadState,
+    fetcher: Callable[[int, int, Any], Future[FilePart]],
+    max_chunks: int | None,
+    cancel_signal: Event,
+    queue_upload: Queue[FilePart | EndOfStream],
+) -> None:
+    count = 0
+
+    def should_stop() -> bool:
+        nonlocal count
+
+        if max_chunks is None:
+            return False
+        if count >= max_chunks:
+            logger.info(
+                f"Stopping file chunker after {count} chunks because it exceeded max_chunks {max_chunks}"
+            )
+            return True
+        count += 1
+        return False
+
+    upload_info = upload_state.upload_info
+    file_path = upload_info.src_file_path
+    chunk_size = upload_info.chunk_size
+    # src = Path(file_path)
+
+    try:
+        part_number = 1
+        done_part_numbers: set[int] = {
+            p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
+        }
+        num_parts = upload_info.total_chunks()
+
+        def next_part_number() -> int | None:
+            nonlocal part_number
+            while part_number in done_part_numbers:
+                part_number += 1
+            if part_number > num_parts:
+                return None
+            return part_number
+
+        if cancel_signal.is_set():
+            logger.info(
+                f"Cancel signal is set for file chunker while processing {file_path}, returning"
+            )
+            return
+
+        while not should_stop():
+            logger.debug("Processing next chunk")
+            curr_part_number = next_part_number()
+            if curr_part_number is None:
+                logger.info(f"File {file_path} has completed chunking all parts")
+                break
+            assert curr_part_number is not None
+            offset = (curr_part_number - 1) * chunk_size
+            file_size = upload_info.file_size
+
+            assert offset < file_size, f"Offset {offset} is greater than file size"
+
+            # Open the file, seek, read the chunk, and close immediately.
+            # with open(file_path, "rb") as f:
+            #     f.seek(offset)
+            #     data = f.read(chunk_size)
+
+            # data = chunk_fetcher(offset, chunk_size).result()
+
+            assert curr_part_number is not None
+            cpn: int = curr_part_number
+
+            def on_complete(fut: Future[FilePart]) -> None:
+                logger.debug("Chunk read complete")
+                fp: FilePart = fut.result()
+                if fp.is_error():
+                    logger.warning(
+                        f"Error reading file: {fp}, skipping part {part_number}"
+                    )
+                    return
+
+                if fp.n_bytes() == 0:
+                    logger.warning(f"Empty data for part {part_number} of {file_path}")
+                    raise ValueError(
+                        f"Empty data for part {part_number} of {file_path}"
+                    )
+
+                if isinstance(fp.payload, Exception):
+                    logger.warning(f"Error reading file because of error: {fp.payload}")
+                    return
+
+                done_part_numbers.add(part_number)
+                queue_upload.put(fp)
+
+            offset = (curr_part_number - 1) * chunk_size
+            logger.info(
+                f"Reading chunk {curr_part_number} of {num_parts} for {file_path}"
+            )
+            fut = fetcher(offset, file_size, S3FileInfo(upload_info.upload_id, cpn))
+            fut.add_done_callback(on_complete)
+            # wait until the queue_upload queue can accept the next chunk
+            while queue_upload.full():
+                time.sleep(0.1)
+    except Exception as e:
+
+        logger.error(f"Error reading file: {e}", exc_info=True)
+    finally:
+        logger.info(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
+        queue_upload.put(EndOfStream())

{rclone_api-1.2.14.dist-info → rclone_api-1.2.154.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.2.14
+Version: 1.2.154
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.2.14.dist-info → rclone_api-1.2.154.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=6TVA3JUVxr76wzy29XRTX_xTdQ5JBGwuheNvQc8EgsU,863
+rclone_api/__init__.py,sha256=dPa4vtqtaO47JZa-YUjOgVa2gCSHLdGxM-_xX_4C7Yw,1150
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -11,6 +11,7 @@ rclone_api/exec.py,sha256=Pd7pUBd8ib5MzqvMybG2DQISPRbDRu20VjVRL2mLAVY,1076
 rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
+rclone_api/logging.py,sha256=fJ4Hr4baAEv93oOOiyzNfoQ8eD0MuErT3NHMjBC3W_w,1184
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
 rclone_api/mount_read_chunker.py,sha256=IH2YcB-N22oiJLkp7KlpG3A4VZkkHOqTYvDLyor2e7Q,4505
 rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
@@ -29,14 +30,14 @@ rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ
 rclone_api/profile/mount_copy_bytes.py,sha256=okzcfpmLcQvh5IUcIwZs9jLPSxFMv2igt2-kHoEmlfE,8571
 rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
-rclone_api/s3/chunk_file.py,sha256=NqwEJp79VIiuOuCpvYq8w84zRr4_WTVqpb_5dJhyR8U,4973
+rclone_api/s3/chunk_file.py,sha256=xtg9g4BvaFsipyfj6p5iRitR53jXjBqX0tmtO7Vf3Us,5068
 rclone_api/s3/chunk_types.py,sha256=I0YCWFgxCvmt8cp4tMabiiwiD2yKTcbA6ZL2D3xnn5w,8781
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=Elmh__gvZJyJyElYwMmvYZIBIunDJiTRAbEg21GmsRU,1604
 rclone_api/s3/upload_file_multipart.py,sha256=inoMOQDZZYqTitJz3f0BBHo3F9ZYm8VhL4UTzPmcdm0,11385
-rclone_api-1.2.14.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.2.14.dist-info/METADATA,sha256=wQaHcpw0KiZ7cpJkzz3oM0FkWR3Dyv4lkqlYoXsJEdI,4537
-rclone_api-1.2.14.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.2.14.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.2.14.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.2.14.dist-info/RECORD,,
+rclone_api-1.2.154.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.2.154.dist-info/METADATA,sha256=RTR14SPQNbWbnFKlWuMGj-y3CdZ4tU3x70odXfdXJVQ,4538
+rclone_api-1.2.154.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.2.154.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.2.154.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.2.154.dist-info/RECORD,,