rclone-api 1.4.24__py2.py3-none-any.whl → 1.4.27__py2.py3-none-any.whl

rclone_api/__init__.py

@@ -366,33 +366,6 @@ class Rclone:
         """Read text from a file."""
         return self.impl.read_text(src=src)
 
-    def copy_file_resumable_s3(
-        self,
-        src: str,
-        dst: str,
-        save_state_json: Path,
-        chunk_size: SizeSuffix | None = None,
-        read_threads: int = 8,
-        write_threads: int = 8,
-        retries: int = 3,
-        verbose: bool | None = None,
-        max_chunks_before_suspension: int | None = None,
-        backend_log: Path | None = None,
-    ) -> MultiUploadResult:
-        """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
-        return self.impl.copy_file_resumable_s3(
-            src=src,
-            dst=dst,
-            save_state_json=save_state_json,
-            chunk_size=chunk_size,
-            read_threads=read_threads,
-            write_threads=write_threads,
-            retries=retries,
-            verbose=verbose,
-            max_chunks_before_suspension=max_chunks_before_suspension,
-            backend_log=backend_log,
-        )
-
     def copy_bytes(
         self,
         src: str,
@@ -423,7 +396,7 @@ class Rclone:
         """Copy a remote to another remote."""
         return self.impl.copy_remote(src=src, dst=dst, args=args)
 
-    def copy_file_parts(
+    def copy_file_s3_resumable(
         self,
         src: str,  # src:/Bucket/path/myfile.large.zst
         dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/part.{part_number:05d}.start-end
@@ -432,7 +405,7 @@ class Rclone:
         merge_threads: int = 4,  # Number of threads to use for merging the parts
     ) -> Exception | None:
         """Copy a file in parts."""
-        return self.impl.copy_file_parts(
+        return self.impl.copy_file_s3_resumable(
             src=src,
             dst_dir=dst_dir,
             part_infos=part_infos,

rclone_api/cmd/copy_large_s3.py

@@ -87,7 +87,7 @@ def main() -> int:
     #     save_state_json=args.save_state_json,
     #     verbose=args.verbose,
     # )
-    err: Exception | None = rclone.copy_file_parts(
+    err: Exception | None = rclone.copy_file_s3_resumable(
         src=args.src,
         dst_dir=args.dst,
     )

rclone_api/cmd/copy_large_s3_finish.py

@@ -31,7 +31,7 @@ def list_files(rclone: Rclone, path: str):
 def _parse_args() -> Args:
     parser = argparse.ArgumentParser(description="List files in a remote path.")
     parser.add_argument("src", help="Directory that holds the info.json file")
-    parser.add_argument("-v", "--verbose", help="Verbose output", action="store_true")
+    parser.add_argument("--no-verbose", help="Verbose output", action="store_true")
     parser.add_argument(
         "--config", help="Path to rclone config file", type=Path, required=False
     )
@@ -45,7 +45,7 @@ def _parse_args() -> Args:
     out = Args(
         config_path=config,
         src=args.src,
-        verbose=args.verbose,
+        verbose=not args.no_verbose,
     )
     return out
 
@@ -65,7 +65,7 @@ def main() -> int:
     rclone = Rclone(rclone_conf=args.config_path)
     info_path = _get_info_path(src=args.src)
     s3_server_side_multi_part_merge(
-        rclone=rclone.impl, info_path=info_path, max_workers=5
+        rclone=rclone.impl, info_path=info_path, max_workers=5, verbose=args.verbose
     )
     return 0
 

rclone_api/rclone_impl.py

@@ -6,7 +6,6 @@ import os
 import random
 import subprocess
 import time
-import traceback
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
 from datetime import datetime
@@ -34,10 +33,7 @@ from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
 from rclone_api.s3.create import S3Credentials
 from rclone_api.s3.types import (
-    MultiUploadResult,
-    S3MutliPartUploadConfig,
     S3Provider,
-    S3UploadTarget,
 )
 from rclone_api.types import (
     ListingOption,
@@ -787,7 +783,7 @@ class RcloneImpl:
         except subprocess.CalledProcessError:
             return False
 
-    def copy_file_parts(
+    def copy_file_s3_resumable(
         self,
         src: str,  # src:/Bucket/path/myfile.large.zst
         dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/
@@ -938,105 +934,6 @@ class RcloneImpl:
         )
         return s3_creds
 
-    def copy_file_resumable_s3(
-        self,
-        src: str,
-        dst: str,
-        save_state_json: Path,
-        chunk_size: SizeSuffix | None = None,
-        read_threads: int = 8,
-        write_threads: int = 8,
-        retries: int = 3,
-        verbose: bool | None = None,
-        max_chunks_before_suspension: int | None = None,
-        backend_log: Path | None = None,
-    ) -> MultiUploadResult:
-        """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
-        from rclone_api.http_server import HttpFetcher, HttpServer
-        from rclone_api.s3.api import S3Client
-        from rclone_api.util import S3PathInfo, split_s3_path
-
-        src_path = Path(src)
-        name = src_path.name
-        src_parent_path = Path(src).parent.as_posix()
-
-        size_result: SizeResult | Exception = self.size_files(src_parent_path, [name])
-        if isinstance(size_result, Exception):
-            raise size_result
-        target_size = SizeSuffix(size_result.total_size)
-
-        chunk_size = chunk_size or SizeSuffix("64M")
-        MAX_CHUNKS = 10000
-        min_chunk_size = SizeSuffix(size_result.total_size // (MAX_CHUNKS - 1))
-        if min_chunk_size > chunk_size:
-            warnings.warn(
-                f"Chunk size {chunk_size} is too small for file size {size_result.total_size}, setting to {min_chunk_size}"
-            )
-            chunk_size = SizeSuffix(min_chunk_size)
-
-        if target_size < SizeSuffix("5M"):
-            # fallback to normal copy
-            completed_proc = self.copy_to(src, dst, check=True)
-            if completed_proc.ok:
-                return MultiUploadResult.UPLOADED_FRESH
-
-        if size_result.total_size <= 0:
-            raise ValueError(
-                f"File {src} has size {size_result.total_size}, is this a directory?"
-            )
-
-        path_info: S3PathInfo = split_s3_path(dst)
-        # remote = path_info.remote
-        bucket_name = path_info.bucket
-        s3_key = path_info.key
-        s3_creds: S3Credentials = self.get_s3_credentials(dst, verbose=verbose)
-
-        port = random.randint(10000, 20000)
-        http_server: HttpServer = self.serve_http(
-            src=src_path.parent.as_posix(),
-            addr=f"localhost:{port}",
-            serve_http_log=backend_log,
-        )
-        chunk_fetcher: HttpFetcher = http_server.get_fetcher(
-            path=src_path.name,
-            n_threads=read_threads,
-        )
-
-        client = S3Client(s3_creds)
-        upload_config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
-            chunk_size=chunk_size.as_int(),
-            chunk_fetcher=chunk_fetcher.bytes_fetcher,
-            max_write_threads=write_threads,
-            retries=retries,
-            resume_path_json=save_state_json,
-            max_chunks_before_suspension=max_chunks_before_suspension,
-        )
-
-        print(f"Uploading {name} to {s3_key} in bucket {bucket_name}")
-        print(f"Source: {src_path}")
-        print(f"bucket_name: {bucket_name}")
-        print(f"upload_config: {upload_config}")
-
-        upload_target = S3UploadTarget(
-            src_file=src_path,
-            src_file_size=size_result.total_size,
-            bucket_name=bucket_name,
-            s3_key=s3_key,
-        )
-
-        try:
-            out: MultiUploadResult = client.upload_file_multipart(
-                upload_target=upload_target,
-                upload_config=upload_config,
-            )
-            return out
-        except Exception as e:
-            print(f"Error uploading file: {e}")
-            traceback.print_exc()
-            raise
-        finally:
-            chunk_fetcher.shutdown()
-
     def copy_bytes(
         self,
         src: str,

rclone_api-1.4.27.dist-info/METADATA

@@ -0,0 +1,555 @@
+Metadata-Version: 2.2
+Name: rclone_api
+Version: 1.4.27
+Summary: rclone api in python
+Home-page: https://github.com/zackees/rclone-api
+License: BSD 3-Clause License
+Keywords: template-python-cmd
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyright>=1.1.393
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: certifi>=2025.1.31
+Requires-Dist: psutil
+Requires-Dist: boto3<=1.35.99,>=1.20.1
+Requires-Dist: sqlmodel>=0.0.23
+Requires-Dist: psycopg2-binary>=2.9.10
+Requires-Dist: httpx>=0.28.1
+Dynamic: home-page
+
+# rclone-api
+
+
+![perpetualmaniac_faster_400fd528-df15-4a04-8ad3-3cca786d7bca (2)](https://github.com/user-attachments/assets/65138e38-b115-447c-849a-4adbd27e4b67)
+
+
+<!--
+[![Linting](https://github.com/zackees/rclone-api/actions/workflows/lint.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/lint.yml)
+[![MacOS_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_macos.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_macos.yml)
+[![Ubuntu_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
+[![Win_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
+-->
+
+
+Got a lot of data to transfer quickly? This package is for you.
+
+This library was built out of necessity to transfer large amounts of AI training data. Aggressive default means this api will transfer faster than rclone does in stock settings.
+
+You must have rclone in your path to use this library. You'd want this anyway because rclone is still under heavy developement.
+
+# Install
+
+`pip install rclone-api`
+
+# Quick
+
+In addition to providing easy python use for rclone, this package provides additional features:
+
+  * Resumable multi-part uploads when s3 is the destination.
+  * Diffing src/dst repos as a stream of `list[str]`.
+  * Efficient copying of byte ranges of a file.
+  * Aggressive default settings for copying / syncing operations for extreme performance.
+  * Some platform specific fixes.
+
+
+## Example
+
+```python
+
+from rclone_api import Rclone, DirListing, Config
+
+RCLONE_CONFIG = Config("""
+[dst]
+type = s3
+account = *********
+key = ************
+""")
+
+
+def test_ls_glob_png(self) -> None:
+    rclone = Rclone(RCLONE_CONFIG)
+    path = f"dst:{BUCKET_NAME}/my_data"
+    listing: DirListing = rclone.ls(path, glob="*.png")
+    self.assertGreater(len(listing.files), 0)
+    for file in listing.files:
+        self.assertIsInstance(file, File)
+        # test that it ends with .png
+        self.assertTrue(file.name.endswith(".png"))
+    # there should be no directories with this glob
+    self.assertEqual(len(listing.dirs), 0)
+```
+
+## API
+
+```python
+
+# from rclone_api import Rclone
+# Rclone is the main api entry point.
+class Rclone:
+    def __init__(
+        self, rclone_conf: Path | Config, rclone_exe: Path | None = None
+    ) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        self.impl: RcloneImpl = RcloneImpl(rclone_conf, rclone_exe)
+
+    def webgui(self, other_args: list[str] | None = None) -> Process:
+        """Launch the Rclone web GUI."""
+        return self.impl.webgui(other_args=other_args)
+
+    def launch_server(
+        self,
+        addr: str,
+        user: str | None = None,
+        password: str | None = None,
+        other_args: list[str] | None = None,
+    ) -> Process:
+        """Launch the Rclone server so it can receive commands"""
+        return self.impl.launch_server(
+            addr=addr, user=user, password=password, other_args=other_args
+        )
+
+    def remote_control(
+        self,
+        addr: str,
+        user: str | None = None,
+        password: str | None = None,
+        capture: bool | None = None,
+        other_args: list[str] | None = None,
+    ) -> CompletedProcess:
+        return self.impl.remote_control(
+            addr=addr,
+            user=user,
+            password=password,
+            capture=capture,
+            other_args=other_args,
+        )
+
+    def obscure(self, password: str) -> str:
+        """Obscure a password for use in rclone config files."""
+        return self.impl.obscure(password=password)
+
+    def ls_stream(
+        self,
+        path: str,
+        max_depth: int = -1,
+        fast_list: bool = False,
+    ) -> FilesStream:
+        """
+        List files in the given path
+
+        Args:
+            src: Remote path to list
+            max_depth: Maximum recursion depth (-1 for unlimited)
+            fast_list: Use fast list (only use when getting THE entire data repository from the root/bucket, or it's small)
+        """
+        return self.impl.ls_stream(path=path, max_depth=max_depth, fast_list=fast_list)
+
+    def save_to_db(
+        self,
+        src: str,
+        db_url: str,
+        max_depth: int = -1,
+        fast_list: bool = False,
+    ) -> None:
+        """
+        Save files to a database (sqlite, mysql, postgres)
+
+        Args:
+            src: Remote path to list, this will be used to populate an entire table, so always use the root-most path.
+            db_url: Database URL, like sqlite:///data.db or mysql://user:pass@localhost/db or postgres://user:pass@localhost/db
+            max_depth: Maximum depth to traverse (-1 for unlimited)
+            fast_list: Use fast list (only use when getting THE entire data repository from the root/bucket)
+
+        """
+        return self.impl.save_to_db(
+            src=src, db_url=db_url, max_depth=max_depth, fast_list=fast_list
+        )
+
+    def ls(
+        self,
+        path: Dir | Remote | str | None = None,
+        max_depth: int | None = None,
+        glob: str | None = None,
+        order: Order = Order.NORMAL,
+        listing_option: ListingOption = ListingOption.ALL,
+    ) -> DirListing:
+        return self.impl.ls(
+            path=path,
+            max_depth=max_depth,
+            glob=glob,
+            order=order,
+            listing_option=listing_option,
+        )
+
+    def listremotes(self) -> list[Remote]:
+        return self.impl.listremotes()
+
+    def diff(
+        self,
+        src: str,
+        dst: str,
+        min_size: (
+            str | None
+        ) = None,  # e. g. "1MB" - see rclone documentation: https://rclone.org/commands/rclone_check/
+        max_size: (
+            str | None
+        ) = None,  # e. g. "1GB" - see rclone documentation: https://rclone.org/commands/rclone_check/
+        diff_option: DiffOption = DiffOption.COMBINED,
+        fast_list: bool = True,
+        size_only: bool | None = None,
+        checkers: int | None = None,
+        other_args: list[str] | None = None,
+    ) -> Generator[DiffItem, None, None]:
+        """Be extra careful with the src and dst values. If you are off by one
+        parent directory, you will get a huge amount of false diffs."""
+        return self.impl.diff(
+            src=src,
+            dst=dst,
+            min_size=min_size,
+            max_size=max_size,
+            diff_option=diff_option,
+            fast_list=fast_list,
+            size_only=size_only,
+            checkers=checkers,
+            other_args=other_args,
+        )
+
+    def walk(
+        self,
+        path: Dir | Remote | str,
+        max_depth: int = -1,
+        breadth_first: bool = True,
+        order: Order = Order.NORMAL,
+    ) -> Generator[DirListing, None, None]:
+        """Walk through the given path recursively.
+
+        Args:
+            path: Remote path or Remote object to walk through
+            max_depth: Maximum depth to traverse (-1 for unlimited)
+
+        Yields:
+            DirListing: Directory listing for each directory encountered
+        """
+        return self.impl.walk(
+            path=path, max_depth=max_depth, breadth_first=breadth_first, order=order
+        )
+
+    def scan_missing_folders(
+        self,
+        src: Dir | Remote | str,
+        dst: Dir | Remote | str,
+        max_depth: int = -1,
+        order: Order = Order.NORMAL,
+    ) -> Generator[Dir, None, None]:
+        """Walk through the given path recursively.
+
+        WORK IN PROGRESS!!
+
+        Args:
+            src: Source directory or Remote to walk through
+            dst: Destination directory or Remote to walk through
+            max_depth: Maximum depth to traverse (-1 for unlimited)
+
+        Yields:
+            DirListing: Directory listing for each directory encountered
+        """
+        return self.impl.scan_missing_folders(
+            src=src, dst=dst, max_depth=max_depth, order=order
+        )
+
+    def cleanup(
+        self, path: str, other_args: list[str] | None = None
+    ) -> CompletedProcess:
+        """Cleanup any resources used by the Rclone instance."""
+        return self.impl.cleanup(path=path, other_args=other_args)
+
+    def copy_to(
+        self,
+        src: File | str,
+        dst: File | str,
+        check: bool | None = None,
+        verbose: bool | None = None,
+        other_args: list[str] | None = None,
+    ) -> CompletedProcess:
+        """Copy one file from source to destination.
+
+        Warning - slow.
+
+        """
+        return self.impl.copy_to(
+            src=src, dst=dst, check=check, verbose=verbose, other_args=other_args
+        )
+
+    def copy_files(
+        self,
+        src: str,
+        dst: str,
+        files: list[str] | Path,
+        check: bool | None = None,
+        max_backlog: int | None = None,
+        verbose: bool | None = None,
+        checkers: int | None = None,
+        transfers: int | None = None,
+        low_level_retries: int | None = None,
+        retries: int | None = None,
+        retries_sleep: str | None = None,
+        metadata: bool | None = None,
+        timeout: str | None = None,
+        max_partition_workers: int | None = None,
+        multi_thread_streams: int | None = None,
+        other_args: list[str] | None = None,
+    ) -> list[CompletedProcess]:
+        """Copy multiple files from source to destination.
+
+        Args:
+            payload: Dictionary of source and destination file paths
+        """
+        return self.impl.copy_files(
+            src=src,
+            dst=dst,
+            files=files,
+            check=check,
+            max_backlog=max_backlog,
+            verbose=verbose,
+            checkers=checkers,
+            transfers=transfers,
+            low_level_retries=low_level_retries,
+            retries=retries,
+            retries_sleep=retries_sleep,
+            metadata=metadata,
+            timeout=timeout,
+            max_partition_workers=max_partition_workers,
+            multi_thread_streams=multi_thread_streams,
+            other_args=other_args,
+        )
+
+    def copy(
+        self,
+        src: Dir | str,
+        dst: Dir | str,
+        check: bool | None = None,
+        transfers: int | None = None,
+        checkers: int | None = None,
+        multi_thread_streams: int | None = None,
+        low_level_retries: int | None = None,
+        retries: int | None = None,
+        other_args: list[str] | None = None,
+    ) -> CompletedProcess:
+        """Copy files from source to destination.
+
+        Args:
+            src: Source directory
+            dst: Destination directory
+        """
+        return self.impl.copy(
+            src=src,
+            dst=dst,
+            check=check,
+            transfers=transfers,
+            checkers=checkers,
+            multi_thread_streams=multi_thread_streams,
+            low_level_retries=low_level_retries,
+            retries=retries,
+            other_args=other_args,
+        )
+
+    def purge(self, path: Dir | str) -> CompletedProcess:
+        """Purge a directory"""
+        return self.impl.purge(path=path)
+
+    def delete_files(
+        self,
+        files: str | File | list[str] | list[File],
+        check: bool | None = None,
+        rmdirs=False,
+        verbose: bool | None = None,
+        max_partition_workers: int | None = None,
+        other_args: list[str] | None = None,
+    ) -> CompletedProcess:
+        """Delete a directory"""
+        return self.impl.delete_files(
+            files=files,
+            check=check,
+            rmdirs=rmdirs,
+            verbose=verbose,
+            max_partition_workers=max_partition_workers,
+            other_args=other_args,
+        )
+
+    def exists(self, path: Dir | Remote | str | File) -> bool:
+        """Check if a file or directory exists."""
+        return self.impl.exists(path=path)
+
+    def is_synced(self, src: str | Dir, dst: str | Dir) -> bool:
+        """Check if two directories are in sync."""
+        return self.impl.is_synced(src=src, dst=dst)
+
+    def modtime(self, src: str) -> str | Exception:
+        """Get the modification time of a file or directory."""
+        return self.impl.modtime(src=src)
+
+    def modtime_dt(self, src: str) -> datetime | Exception:
+        """Get the modification time of a file or directory."""
+        return self.impl.modtime_dt(src=src)
+
+    def write_text(
+        self,
+        text: str,
+        dst: str,
+    ) -> Exception | None:
+        """Write text to a file."""
+        return self.impl.write_text(text=text, dst=dst)
+
+    def write_bytes(
+        self,
+        data: bytes,
+        dst: str,
+    ) -> Exception | None:
+        """Write bytes to a file."""
+        return self.impl.write_bytes(data=data, dst=dst)
+
+    def read_bytes(self, src: str) -> bytes | Exception:
+        """Read bytes from a file."""
+        return self.impl.read_bytes(src=src)
+
+    def read_text(self, src: str) -> str | Exception:
+        """Read text from a file."""
+        return self.impl.read_text(src=src)
+
+    def copy_bytes(
+        self,
+        src: str,
+        offset: int | SizeSuffix,
+        length: int | SizeSuffix,
+        outfile: Path,
+        other_args: list[str] | None = None,
+    ) -> Exception | None:
+        """Copy a slice of bytes from the src file to dst."""
+        return self.impl.copy_bytes(
+            src=src,
+            offset=offset,
+            length=length,
+            outfile=outfile,
+            other_args=other_args,
+        )
+
+    def copy_dir(
+        self, src: str | Dir, dst: str | Dir, args: list[str] | None = None
+    ) -> CompletedProcess:
+        """Copy a directory from source to destination."""
+        # convert src to str, also dst
+        return self.impl.copy_dir(src=src, dst=dst, args=args)
+
+    def copy_remote(
+        self, src: Remote, dst: Remote, args: list[str] | None = None
+    ) -> CompletedProcess:
+        """Copy a remote to another remote."""
+        return self.impl.copy_remote(src=src, dst=dst, args=args)
+
+    def copy_file_s3_resumable(
+        self,
+        src: str,  # src:/Bucket/path/myfile.large.zst
+        dst_dir: str,  # dst:/Bucket/path/myfile.large.zst-parts/part.{part_number:05d}.start-end
+        part_infos: list[PartInfo] | None = None,
+        upload_threads: int = 8,  # Number of reader and writer threads to use
+        merge_threads: int = 4,  # Number of threads to use for merging the parts
+    ) -> Exception | None:
+        """Copy a file in parts."""
+        return self.impl.copy_file_s3_resumable(
+            src=src,
+            dst_dir=dst_dir,
+            part_infos=part_infos,
+            upload_threads=upload_threads,
+            merge_threads=merge_threads,
+        )
+
+    def mount(
+        self,
+        src: Remote | Dir | str,
+        outdir: Path,
+        allow_writes: bool | None = False,
+        use_links: bool | None = None,
+        vfs_cache_mode: str | None = None,
+        verbose: bool | None = None,
+        cache_dir: Path | None = None,
+        cache_dir_delete_on_exit: bool | None = None,
+        log: Path | None = None,
+        other_args: list[str] | None = None,
+    ) -> Mount:
+        """Mount a remote or directory to a local path.
+
+        Args:
+            src: Remote or directory to mount
+            outdir: Local path to mount to
+
+        Returns:
+            CompletedProcess from the mount command execution
+
+        Raises:
+            subprocess.CalledProcessError: If the mount operation fails
+        """
+        return self.impl.mount(
+            src=src,
+            outdir=outdir,
+            allow_writes=allow_writes,
+            use_links=use_links,
+            vfs_cache_mode=vfs_cache_mode,
+            verbose=verbose,
+            cache_dir=cache_dir,
+            cache_dir_delete_on_exit=cache_dir_delete_on_exit,
+            log=log,
+            other_args=other_args,
+        )
+
+    def serve_http(
+        self,
+        src: str,
+        addr: str = "localhost:8080",
+        other_args: list[str] | None = None,
+    ) -> HttpServer:
+        """Serve a remote or directory via HTTP. The returned HttpServer has a client which can be used to
+        fetch files or parts.
+
+        Args:
+            src: Remote or directory to serve
+            addr: Network address and port to serve on (default: localhost:8080)
+        """
+        return self.impl.serve_http(src=src, addr=addr, other_args=other_args)
+
+    def size_files(
+        self,
+        src: str,
+        files: list[str],
+        fast_list: bool = False,  # Recommend that this is False
+        other_args: list[str] | None = None,
+        check: bool | None = False,
+        verbose: bool | None = None,
+    ) -> SizeResult | Exception:
+        """Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
+        return self.impl.size_files(
+            src=src,
+            files=files,
+            fast_list=fast_list,
+            other_args=other_args,
+            check=check,
+            verbose=verbose,
+        )
+
+    def size_file(self, src: str) -> SizeSuffix | Exception:
+        """Get the size of a file."""
+        return self.impl.size_file(src=src)
+```
+
+
+To develop software, run `. ./activate`
+
+# Windows
+
+This environment requires you to use `git-bash`.
+
+# Linting
+
+Run `./lint`

{rclone_api-1.4.24.dist-info → rclone_api-1.4.27.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=wvLRTEYvwLfkBx_LuWI62cZQA-pEz0xo0oVh_9DCOuM,17876
+rclone_api/__init__.py,sha256=MyVBox2r4zRoMt3mWDTw7My24n9SAxGoCxkvgAo6Nps,16898
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -18,7 +18,7 @@ rclone_api/http_server.py,sha256=LhovQu2AI-Z7zQIWflWelCiCDLnWzisL32Rs5350kxE,885
 rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
 rclone_api/process.py,sha256=tGooS5NLdPuqHh7hCH8SfK44A6LGftPQCPQUNgSo0a0,5714
-rclone_api/rclone_impl.py,sha256=WBLkQpQq4lGPla1uJBzpp1yf4kS3ub7fxpbU6SdJyZY,49873
+rclone_api/rclone_impl.py,sha256=mFFpU4ngWuZfvtwoIaOl2iTVYrWNRte-TBtbZzVhWaU,46108
 rclone_api/remote.py,sha256=mTgMTQTwxUmbLjTpr-AGTId2ycXKI9mLX5L7PPpDIoc,520
 rclone_api/rpath.py,sha256=Y1JjQWcie39EgQrq-UtbfDz5yDLCwwfu27W7AQXllSE,2860
 rclone_api/scan_missing_folders.py,sha256=-8NCwpCaHeHrX-IepCoAEsX1rl8S-GOCxcIhTr_w3gA,4747
@@ -26,8 +26,8 @@ rclone_api/types.py,sha256=2ngxwpdNy88y0teeYJ5Vz5NiLK1rfaFx8Xf99i0J-Js,12155
 rclone_api/util.py,sha256=yY72YKpmpT_ZM7AleVtPpl0YZZYQPTwTdqKn9qPwm8Y,9290
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
-rclone_api/cmd/copy_large_s3.py,sha256=yhPwbtGz9MmlronB-biiYUfNVclOsxfX9GIhe3ai3g4,3463
-rclone_api/cmd/copy_large_s3_finish.py,sha256=DMGEKKslsmBp2pYZwMA24oGIApYheqT04nZ-iNP2vLI,2283
+rclone_api/cmd/copy_large_s3.py,sha256=O1cAfrgL-M1F-j93qH2w6F9zNoJeXG_4v_pUwL0sXxY,3470
+rclone_api/cmd/copy_large_s3_finish.py,sha256=SNCqkvu8YtxPKmBp37WVMP876YhxV0kJDoYuOSNPaPY,2309
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/cmd/save_to_db.py,sha256=ylvnhg_yzexM-m6Zr7XDiswvoDVSl56ELuFAdb9gqBY,1957
 rclone_api/db/__init__.py,sha256=OSRUdnSWUlDTOHmjdjVmxYTUNpTbtaJ5Ll9sl-PfZg0,40
@@ -51,9 +51,9 @@ rclone_api/s3/multipart/upload_parts_inline.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9
 rclone_api/s3/multipart/upload_parts_resumable.py,sha256=diJoUpVYow6No_dNgOZIYVsv43k4evb6zixqpzWJaUk,9771
 rclone_api/s3/multipart/upload_parts_server_side_merge.py,sha256=Fp2pdrs5dONQI9LkfNolgAGj1-Z2V1SsRd0r0sreuXI,18040
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.4.24.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.4.24.dist-info/METADATA,sha256=2vOkBf_iR7d3Au2fujA3PZaonjMIXHTzPfHb3ut1Ml0,4628
-rclone_api-1.4.24.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.4.24.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.4.24.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.4.24.dist-info/RECORD,,
+rclone_api-1.4.27.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.4.27.dist-info/METADATA,sha256=EfHFkopInrpyHiwq_Uai3U5VJIL0lDiR92qPctLEUHk,18480
+rclone_api-1.4.27.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.4.27.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.4.27.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.4.27.dist-info/RECORD,,

rclone_api-1.4.24.dist-info/METADATA

@@ -1,154 +0,0 @@
-Metadata-Version: 2.2
-Name: rclone_api
-Version: 1.4.24
-Summary: rclone api in python
-Home-page: https://github.com/zackees/rclone-api
-License: BSD 3-Clause License
-Keywords: template-python-cmd
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: pyright>=1.1.393
-Requires-Dist: python-dotenv>=1.0.0
-Requires-Dist: certifi>=2025.1.31
-Requires-Dist: psutil
-Requires-Dist: boto3<=1.35.99,>=1.20.1
-Requires-Dist: sqlmodel>=0.0.23
-Requires-Dist: psycopg2-binary>=2.9.10
-Requires-Dist: httpx>=0.28.1
-Dynamic: home-page
-
-# rclone-api
-
-[![Linting](https://github.com/zackees/rclone-api/actions/workflows/lint.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/lint.yml)
-[![MacOS_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_macos.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_macos.yml)
-[![Ubuntu_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
-[![Win_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
-
-Started off as just python bindings to rclone, but this project is now adding features to fill in the gaps of the rclone app. The big features is streaming file diffs, stream file listings.
-
-You will need to have rclone installed and on your path.
-
-One of the benefits of this api is that it does not use 'shell=True' so therefore ctrl-c will work well in gracefully shutting down
-
-# Install
-
-`pip install rclone-api`
-
-
-# Examples
-
-You can use env variables or use a `.env` file to store your secrets.
-
-
-# Rclone API Usage Examples
-
-This script demonstrates how to interact with DigitalOcean Spaces using `rclone_api`. 
-
-## Setup & Usage
-
-Ensure you have set the required environment variables:
-
-- `BUCKET_NAME`
-- `BUCKET_KEY_PUBLIC`
-- `BUCKET_KEY_SECRET`
-- `BUCKET_URL`
-
-Then, run the following Python script:
-
-```python
-import os
-from rclone_api import Config, DirListing, File, Rclone, Remote
-
-# Load environment variables
-BUCKET_NAME = os.getenv("BUCKET_NAME")
-BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
-BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
-BUCKET_URL = "sfo3.digitaloceanspaces.com"
-
-# Generate Rclone Configuration
-def generate_rclone_config() -> Config:
-    config_text = f"""
-    [dst]
-    type = s3
-    provider = DigitalOcean
-    access_key_id = {BUCKET_KEY_PUBLIC}
-    secret_access_key = {BUCKET_KEY_SECRET}
-    endpoint = {BUCKET_URL}
-    """
-    return Config(config_text)
-
-rclone = Rclone(generate_rclone_config())
-
-# List Available Remotes
-print("\n=== Available Remotes ===")
-remotes = rclone.listremotes()
-for remote in remotes:
-    print(remote)
-
-# List Contents of the Root Bucket
-print("\n=== Listing Root Bucket ===")
-listing = rclone.ls(f"dst:{BUCKET_NAME}", max_depth=-1)
-
-print("\nDirectories:")
-for dir in listing.dirs:
-    print(dir)
-
-print("\nFiles:")
-for file in listing.files:
-    print(file)
-
-# List a Specific Subdirectory
-print("\n=== Listing 'zachs_video' Subdirectory ===")
-path = f"dst:{BUCKET_NAME}/zachs_video"
-listing = rclone.ls(path)
-print(listing)
-
-# List PNG Files in a Subdirectory
-print("\n=== Listing PNG Files ===")
-listing = rclone.ls(path, glob="*.png")
-
-if listing.files:
-    for file in listing.files:
-        print(file)
-
-# Copy a File
-print("\n=== Copying a File ===")
-if listing.files:
-    file = listing.files[0]
-    new_path = f"dst:{BUCKET_NAME}/zachs_video/{file.name}_copy"
-    rclone.copyfile(file, new_path)
-    print(f"Copied {file.name} to {new_path}")
-
-# Copy Multiple Files
-print("\n=== Copying Multiple Files ===")
-if listing.files:
-    file_mapping = {file.name: file.name + "_copy" for file in listing.files[:2]}
-    rclone.copyfiles(file_mapping)
-    print(f"Copied files: {file_mapping}")
-
-# Delete a File
-print("\n=== Deleting a File ===")
-file_to_delete = f"dst:{BUCKET_NAME}/zachs_video/sample.png_copy"
-rclone.delete_files([file_to_delete])
-print(f"Deleted {file_to_delete}")
-
-# Walk Through a Directory
-print("\n=== Walking Through a Directory ===")
-for dirlisting in rclone.walk(f"dst:{BUCKET_NAME}", max_depth=1):
-    print(dirlisting)
-
-print("Done.")
-```
-
-
-To develop software, run `. ./activate`
-
-# Windows
-
-This environment requires you to use `git-bash`.
-
-# Linting
-
-Run `./lint`