rclone-api 1.1.29__py2.py3-none-any.whl → 1.1.31__py2.py3-none-any.whl

rclone_api/exec.py

@@ -23,7 +23,9 @@ class RcloneExec:
             cmd, self.rclone_config, self.rclone_exe, check=check, capture=capture
         )
 
-    def launch_process(self, cmd: list[str], capture: bool | None) -> Process:
+    def launch_process(
+        self, cmd: list[str], capture: bool | None, log: Path | None
+    ) -> Process:
         """Launch rclone process."""
 
         args: ProcessArgs = ProcessArgs(
@@ -32,6 +34,7 @@ class RcloneExec:
             rclone_exe=self.rclone_exe,
             cmd_list=cmd,
             capture_stdout=capture,
+            log=log,
         )
         process = Process(args)
         return process

rclone_api/mount.py

@@ -105,8 +105,8 @@ def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
     mount_path = mount.mount_path if isinstance(mount, Mount) else mount
     try:
         mount_exists = mount_path.exists()
-    except OSError as e:
-        warnings.warn(f"Error checking {mount_path}: {e}")
+    except OSError:
+        # warnings.warn(f"Error checking {mount_path}: {e}")
         mount_exists = True
 
     # Give the system a moment (if unmount is in progress, etc.)
@@ -139,8 +139,9 @@ def clean_mount(mount: Mount | Path, verbose: bool = False, wait=True) -> None:
             mount_path.rmdir()
             if verbose:
                 print(f"Successfully removed mount directory {mount_path}")
-        except Exception as e:
-            warnings.warn(f"Failed to remove mount {mount_path}: {e}")
+        except Exception:
+            # warnings.warn(f"Failed to remove mount {mount_path}: {e}")
+            pass
     else:
         warnings.warn(f"Unsupported platform: {_SYSTEM}")
 

rclone_api/process.py

@@ -1,5 +1,4 @@
 import atexit
-import os
 import subprocess
 import threading
 import time
@@ -13,12 +12,6 @@ from rclone_api.config import Config
 from rclone_api.util import get_verbose
 
 
-def _get_verbose(verbose: bool | None) -> bool:
-    if verbose is not None:
-        return verbose
-    return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
-
-
 @dataclass
 class ProcessArgs:
     cmd: list[str]
@@ -27,12 +20,14 @@ class ProcessArgs:
     cmd_list: list[str]
     verbose: bool | None = None
     capture_stdout: bool | None = None
+    log: Path | None = None
 
 
 class Process:
     def __init__(self, args: ProcessArgs) -> None:
         assert args.rclone_exe.exists()
         self.args = args
+        self.log = args.log
         self.tempdir: TemporaryDirectory | None = None
         verbose = get_verbose(args.verbose)
         if isinstance(args.rclone_conf, Config):
@@ -52,6 +47,9 @@ class Process:
             + ["--config", str(rclone_conf.resolve())]
             + args.cmd
         )
+        if self.args.log:
+            self.args.log.parent.mkdir(parents=True, exist_ok=True)
+            self.cmd += ["--log-file", str(self.args.log)]
         if verbose:
             cmd_str = subprocess.list2cmdline(self.cmd)
             print(f"Running: {cmd_str}")

rclone_api/profiling/mount_copy_bytes.py

@@ -0,0 +1,145 @@
+"""
+Unit test file.
+"""
+
+import os
+import time
+import unittest
+
+import psutil
+from dotenv import load_dotenv
+
+from rclone_api import Config, Rclone, SizeSuffix
+
+load_dotenv()
+
+BUCKET_NAME = os.getenv("BUCKET_NAME")  # Default if not in .env
+
+
+def _generate_rclone_config() -> Config:
+
+    # assert that .env exists for this test
+    assert os.path.exists(
+        ".env"
+    ), "this test requires that the secret .env file exists with the credentials"
+
+    # BUCKET_NAME = os.getenv("BUCKET_NAME", "TorrentBooks")  # Default if not in .env
+
+    # Load additional environment variables
+    BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
+    BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
+    SRC_SFTP_HOST = os.getenv("SRC_SFTP_HOST")
+    SRC_SFTP_USER = os.getenv("SRC_SFTP_USER")
+    SRC_SFTP_PORT = os.getenv("SRC_SFTP_PORT")
+    SRC_SFTP_PASS = os.getenv("SRC_SFTP_PASS")
+    # BUCKET_URL = os.getenv("BUCKET_URL")
+    BUCKET_URL = "sfo3.digitaloceanspaces.com"
+
+    config_text = f"""
+[dst]
+type = s3
+provider = DigitalOcean
+access_key_id = {BUCKET_KEY_PUBLIC}
+secret_access_key = {BUCKET_KEY_SECRET}
+endpoint = {BUCKET_URL}
+bucket = {BUCKET_NAME}
+
+[src]
+type = sftp
+host = {SRC_SFTP_HOST}
+user = {SRC_SFTP_USER}
+port = {SRC_SFTP_PORT}
+pass = {SRC_SFTP_PASS}
+
+"""
+    # _CONFIG_PATH.write_text(config_text, encoding="utf-8")
+    # print(f"Config file written to: {_CONFIG_PATH}")
+    return Config(config_text)
+
+
+class RcloneProfileCopyBytes(unittest.TestCase):
+    """Test rclone functionality."""
+
+    def setUp(self) -> None:
+        """Check if all required environment variables are set before running tests."""
+        required_vars = [
+            "BUCKET_NAME",
+            "BUCKET_KEY_SECRET",
+            "BUCKET_KEY_PUBLIC",
+            "BUCKET_URL",
+        ]
+        missing = [var for var in required_vars if not os.getenv(var)]
+        if missing:
+            self.skipTest(
+                f"Missing required environment variables: {', '.join(missing)}"
+            )
+        os.environ["RCLONE_API_VERBOSE"] = "1"
+
+    def test_profile_copy_bytes(self) -> None:
+        rclone = Rclone(_generate_rclone_config())
+        sizes = [
+            1024 * 1024 * 1,
+            1024 * 1024 * 2,
+            1024 * 1024 * 4,
+            1024 * 1024 * 8,
+            1024 * 1024 * 16,
+            1024 * 1024 * 32,
+            1024 * 1024 * 64,
+        ]
+        # transfer_list = [1, 2, 4, 8, 16]
+        transfer_list = [1, 2, 4]
+
+        # src_file = "dst:rclone-api-unit-test/zachs_video/internaly_ai_alignment.mp4"
+        # sftp mount
+        src_file = "src:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst"
+
+        for size in sizes:
+            for transfers in transfer_list:
+                print("\n\n")
+                print("#" * 80)
+                print(
+                    f"# Started test download of {SizeSuffix(size)} with {transfers} transfers"
+                )
+                print("#" * 80)
+                net_io_start = psutil.net_io_counters()
+                start = time.time()
+                bytes_or_err: bytes | Exception = rclone.copy_bytes(
+                    src=src_file,
+                    offset=0,
+                    length=size,
+                    direct_io=True,
+                    transfers=transfers,
+                )
+                diff = time.time() - start
+                net_io_end = psutil.net_io_counters()
+                if isinstance(bytes_or_err, Exception):
+                    print(bytes_or_err)
+                    self.fail(f"Error: {bytes_or_err}")
+                assert isinstance(bytes_or_err, bytes)
+                self.assertEqual(len(bytes_or_err), size)
+
+                # print io stats
+                bytes_sent = net_io_end.bytes_sent - net_io_start.bytes_sent
+                bytes_recv = net_io_end.bytes_recv - net_io_start.bytes_recv
+                packets_sent = net_io_end.packets_sent - net_io_start.packets_sent
+                efficiency = size / (bytes_recv)
+                efficiency_100 = efficiency * 100
+                efficiency_str = f"{efficiency_100:.2f}"
+
+                bytes_send_suffix = SizeSuffix(bytes_sent)
+                bytes_recv_suffix = SizeSuffix(bytes_recv)
+                range_size = SizeSuffix(size)
+
+                print(f"\nFinished downloading {range_size} with {transfers} transfers")
+                print("Net IO stats:")
+                print(f"Bytes sent: {bytes_send_suffix}")
+                print(f"Bytes received: {bytes_recv_suffix}")
+                print(f"Packets sent: {packets_sent}")
+                print(f"Efficiency: {efficiency_str}%")
+                print(f"Time: {diff:.1f} seconds")
+
+        print("done")
+
+
+if __name__ == "__main__":
+    unittest.main()

rclone_api/rclone.py

@@ -80,8 +80,10 @@ class Rclone:
     ) -> subprocess.CompletedProcess:
         return self._exec.execute(cmd, check=check, capture=capture)
 
-    def _launch_process(self, cmd: list[str], capture: bool | None = None) -> Process:
-        return self._exec.launch_process(cmd, capture=capture)
+    def _launch_process(
+        self, cmd: list[str], capture: bool | None = None, log: Path | None = None
+    ) -> Process:
+        return self._exec.launch_process(cmd, capture=capture, log=log)
 
     def webgui(self, other_args: list[str] | None = None) -> Process:
         """Launch the Rclone web GUI."""
@@ -684,6 +686,7 @@ class Rclone:
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
         mount_path: Path | None = None,
+        mount_log: Path | None = None,
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
         from rclone_api.s3.api import S3Client
@@ -724,6 +727,7 @@ class Rclone:
             use_links=True,
             vfs_cache_mode="minimal",
             verbose=False,
+            log=mount_log,
             other_args=other_args,
         ):
             path_info: S3PathInfo = split_s3_path(dst)
@@ -816,7 +820,11 @@ class Rclone:
         src: str,
         offset: int,
         length: int,
-        transfers: int = 16,
+        transfers: int = 1,  # Note, increasing transfers can result in devestating drop in performance.
+        # If outfile is supplied then bytes are written to this file and success returns bytes(0)
+        outfile: Path | None = None,
+        mount_log: Path | None = None,
+        direct_io: bool = True,
     ) -> bytes | Exception:
         """Copy bytes from a file to another file."""
         from rclone_api.util import random_str
@@ -825,8 +833,7 @@ class Rclone:
         src_parent_path = Path(src).parent.as_posix()
         src_file = Path(src).name
         other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
-        unit_chunk_size = length
-        vfs_read_chunk_size = unit_chunk_size
+        vfs_read_chunk_size = length // transfers
         vfs_read_chunk_size_limit = length
         vfs_read_chunk_streams = transfers
         vfs_disk_space_total_size = length
@@ -835,7 +842,8 @@ class Rclone:
         other_args += ["--vfs-read-chunk-streams", str(vfs_read_chunk_streams)]
         other_args += ["--vfs-disk-space-total-size", str(vfs_disk_space_total_size)]
         other_args += ["--read-only"]
-        other_args += ["--direct-io"]
+        if direct_io:
+            other_args += ["--direct-io"]
 
         try:
             # use scoped mount to do the read, then write the bytes to the destination
@@ -843,15 +851,22 @@ class Rclone:
                 src_parent_path,
                 tmp_mnt,
                 use_links=True,
-                verbose=False,
+                verbose=mount_log is not None,
                 vfs_cache_mode="minimal",
                 other_args=other_args,
+                log=mount_log,
             ):
                 src_file_mnt = tmp_mnt / src_file
                 with open(src_file_mnt, "rb") as f:
                     f.seek(offset)
                     data = f.read(length)
-                    return data
+                    if outfile is None:
+                        return data
+                    with open(outfile, "wb") as out:
+                        out.write(data)
+                        del data
+                    return bytes(0)
+
         except Exception as e:
             return e
 
@@ -887,6 +902,7 @@ class Rclone:
         use_links: bool | None = None,
         vfs_cache_mode: str | None = None,
         verbose: bool | None = None,
+        log: Path | None = None,
         other_args: list[str] | None = None,
     ) -> Mount:
         """Mount a remote or directory to a local path.
@@ -921,7 +937,7 @@ class Rclone:
             cmd_list.append("-vvvv")
         if other_args:
             cmd_list += other_args
-        proc = self._launch_process(cmd_list)
+        proc = self._launch_process(cmd_list, log=log)
         mount_read_only = "--read-only" in cmd_list  # hint to allow fast teardown
         mount: Mount = Mount(mount_path=outdir, process=proc, read_only=mount_read_only)
         return mount
@@ -935,6 +951,7 @@ class Rclone:
         use_links: bool | None = None,
         vfs_cache_mode: str | None = None,
         verbose: bool | None = None,
+        log: Path | None = None,
         other_args: list[str] | None = None,
     ) -> Generator[Mount, None, None]:
         """Like mount, but can be used in a context manager."""
@@ -947,6 +964,7 @@ class Rclone:
             use_links=use_links,
             vfs_cache_mode=vfs_cache_mode,
             verbose=verbose,
+            log=log,
             other_args=other_args,
         )
         try:

{rclone_api-1.1.29.dist-info → rclone_api-1.1.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.1.29
+Version: 1.1.31
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License
@@ -12,6 +12,7 @@ License-File: LICENSE
 Requires-Dist: pyright>=1.1.393
 Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: certifi>=2025.1.31
+Requires-Dist: psutil
 Requires-Dist: boto3<=1.35.99,>=1.20.1
 Dynamic: home-page
 

{rclone_api-1.1.29.dist-info → rclone_api-1.1.31.dist-info}/RECORD

@@ -7,13 +7,13 @@ rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
 rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
 rclone_api/dir.py,sha256=i4h7LX5hB_WmVixxDRWL_l1nifvscrdWct_8Wx7wHZc,3540
 rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,1882
-rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
+rclone_api/exec.py,sha256=Pd7pUBd8ib5MzqvMybG2DQISPRbDRu20VjVRL2mLAVY,1076
 rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/mount.py,sha256=g9YkKCGJbWvGF7gzSKtWa8pEgphuW0-e0SySxtOirH4,6093
-rclone_api/process.py,sha256=Hgn8MGEPkBt8C6C4oIuh-n1t1GkFF2miPlIE1lh_Zbc,5045
-rclone_api/rclone.py,sha256=bdAhJAW5KuxmTCZsSp5LfVUgD5WWfaKtuWyZTM-AGtI,41881
+rclone_api/mount.py,sha256=ryAjkX4_kFeFZWLiBVpcGy2VilpvVhFbWeWfEX4jMKs,6104
+rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
+rclone_api/rclone.py,sha256=meZkXmIM1M_L0Xu2hf0fcm-tG1cF5t6AHSIM6TOXUQM,42624
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
@@ -25,6 +25,7 @@ rclone_api/cmd/copy_large_s3.py,sha256=-rfedi-ZzPUdCSP8ai9LRL0y1xVkvN-viQQlk8HVU
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
+rclone_api/profiling/mount_copy_bytes.py,sha256=SgorSAG9lOlqMxzLt5N2UX_fepANpFWKM5NaQiBrsFE,4745
 rclone_api/s3/api.py,sha256=qxtRDUpHYqJ7StJRtP8U_PbF_BvYRg705568SyvF-R0,3770
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_file.py,sha256=YELR-EzR7RHpzCDGpYdzlwu21NZW5wttIDvLoONI4aU,3477
@@ -32,9 +33,9 @@ rclone_api/s3/chunk_types.py,sha256=LbXayXY1KgVU1LkdbASD_BQ7TpVpwVnzMjtz--8LBaE,
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=yBnJ38Tjk6RlydJ-sqZ7DSfyFloy8KDYJ0mv3vlOzLE,1388
 rclone_api/s3/upload_file_multipart.py,sha256=1jQAdk35Fa9Tcq36bS65262cs7AcNG2DAFQ-NdYlWSw,9961
-rclone_api-1.1.29.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.1.29.dist-info/METADATA,sha256=BmYo65cIJ7V2miFqEfW5Kk1Y2-tpcMz1cbTUQNTZlNo,4514
-rclone_api-1.1.29.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.1.29.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
-rclone_api-1.1.29.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.1.29.dist-info/RECORD,,
+rclone_api-1.1.31.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.1.31.dist-info/METADATA,sha256=5gEXYe7terl_H8vFbUZkWyD2wy2c05W-6EKuNgQya6Y,4537
+rclone_api-1.1.31.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.1.31.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.1.31.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.1.31.dist-info/RECORD,,