rclone-api 1.5.50__py3-none-any.whl → 1.5.52__py3-none-any.whl

rclone_api/__init__.py

@@ -438,6 +438,37 @@ class Rclone:
         """
         return self.impl.cleanup(src=src, other_args=other_args)
 
+    def copy_file_s3(
+        self,
+        src: Path,
+        dst: str,
+        verbose: bool | None = None,
+    ) -> Exception | None:
+        """
+        Copy a file to S3.
+
+        Args:
+            src: Local file path
+            dst: Destination path in S3
+            verbose: Whether to show detailed progress
+
+        Returns:
+            None if successful, Exception if an error occurred
+        """
+        return self.impl.copy_file_s3(src=src, dst=dst, verbose=verbose)
+
+    def is_s3(self, dst: str) -> bool:
+        """
+        Check if a path is an S3 bucket.
+
+        Args:
+            dst: Path to check
+
+        Returns:
+            True if the path is an S3 bucket, False otherwise
+        """
+        return self.impl.is_s3(dst=dst)
+
     def copy_to(
         self,
         src: File | str,

rclone_api/fs/filesystem.py

@@ -1,17 +1,20 @@
 import abc
+import logging
 import shutil
 import warnings
 from pathlib import Path
 
 from rclone_api.config import Config
 
+logger = logging.getLogger(__name__)
+
 
 class FS(abc.ABC):
     def __init__(self) -> None:
         pass
 
     @abc.abstractmethod
-    def copy(self, src: Path | str, dest: Path | str) -> None:
+    def copy(self, src: Path | str, dst: Path | str) -> None:
         pass
 
     @abc.abstractmethod
@@ -67,8 +70,8 @@ class RealFS(FS):
     def cwd(self) -> "FSPath":
         return RealFS.from_path(Path.cwd())
 
-    def copy(self, src: Path | str, dest: Path | str) -> None:
-        shutil.copy(str(src), str(dest))
+    def copy(self, src: Path | str, dst: Path | str) -> None:
+        shutil.copy(str(src), str(dst))
 
     def read_bytes(self, path: Path | str) -> bytes:
         with open(path, "rb") as f:
@@ -132,10 +135,31 @@ class RemoteFS(FS):
     def _to_remote_path(self, path: str | Path) -> str:
         return Path(path).relative_to(self.src).as_posix()
 
-    def copy(self, src: Path | str, dest: Path | str) -> None:
-        src = self._to_str(src)
-        dest = self._to_remote_path(dest)
-        self.rclone.copy(src, dest)
+    def copy(self, src: Path | str, dst: Path | str) -> None:
+        from rclone_api.completed_process import CompletedProcess
+
+        src = src if isinstance(src, Path) else Path(src)
+        if not src.is_file():
+            raise FileNotFoundError(f"File not found: {src}")
+        dst = self._to_remote_path(dst)
+
+        is_s3 = self.rclone.is_s3(dst)
+        if is_s3:
+            filesize = src.stat().st_size
+            if filesize < 512 * 1024 * 1024:
+                logger.info(f"S3 OPTIMIZED: Copying {src} -> {dst}")
+                err = self.rclone.copy_file_s3(src, dst)
+                if isinstance(err, Exception):
+                    raise FileNotFoundError(
+                        f"File not found: {src}, specified by {err}"
+                    )
+                return
+        # Fallback.
+        logging.info(f"Copying {src} -> {dst}")
+        src_path = src.as_posix()
+        cp: CompletedProcess = self.rclone.copy(src_path, dst)
+        if cp.returncode != 0:
+            raise FileNotFoundError(f"File not found: {src}, specified by {cp.stderr}")
 
     def read_bytes(self, path: Path | str) -> bytes:
         path = self._to_str(path)
@@ -146,7 +170,7 @@ class RemoteFS(FS):
 
     def write_binary(self, path: Path | str, data: bytes) -> None:
         path = self._to_str(path)
-        self.rclone.write_bytes(data, path)
+        self.rclone.write_bytes(data, path)  # Already optimized for s3.
 
     def exists(self, path: Path | str) -> bool:
         from rclone_api.http_server import HttpServer

rclone_api/rclone_impl.py

@@ -2,6 +2,7 @@
 Unit test file.
 """
 
+import logging
 import os
 import random
 import subprocess
@@ -33,6 +34,7 @@ from rclone_api.mount import Mount
 from rclone_api.process import Process
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
+from rclone_api.s3.api import S3Client
 from rclone_api.s3.create import S3Credentials
 from rclone_api.s3.types import (
     S3Provider,
@@ -56,6 +58,8 @@ from rclone_api.util import (
 # Enable tracing memory usage always
 tracemalloc.start()
 
+logger = logging.getLogger(__name__)
+
 
 def rclone_verbose(verbose: bool | None) -> bool:
     if verbose is not None:
@@ -825,6 +829,52 @@ class RcloneImpl:
         except subprocess.CalledProcessError:
             return False
 
+    def _s3_client(self, src: str, verbose: bool | None = None) -> S3Client:
+        """Get an S3 client."""
+        verbose = get_verbose(verbose)
+        s3_creds = self.get_s3_credentials(remote=src, verbose=verbose)
+        s3_client = S3Client(s3_creds=s3_creds, verbose=verbose)
+        return s3_client
+
+    def copy_file_s3(
+        self,
+        src: Path,
+        dst: str,
+        verbose: bool | None = None,
+    ) -> Exception | None:
+        """Copy a file to S3."""
+        from rclone_api.s3.types import S3UploadTarget
+        from rclone_api.util import S3PathInfo
+
+        dst_is_s3 = self.is_s3(dst)
+        if not dst_is_s3:
+            return ValueError(f"Destination is not an S3 remote: {dst}")
+        s3_client = self._s3_client(dst, verbose=verbose)
+
+        path_info: S3PathInfo = S3PathInfo.from_str(dst)
+        target: S3UploadTarget = S3UploadTarget(
+            src_file=src,
+            src_file_size=src.stat().st_size,
+            bucket_name=path_info.bucket,
+            s3_key=path_info.key,
+        )
+        out = s3_client.upload_file(target=target)
+        return out
+
+    def is_s3(self, dst: str) -> bool:
+        """Check if a remote is an S3 remote."""
+        from rclone_api.util import S3PathInfo
+
+        path_info: S3PathInfo = S3PathInfo.from_str(dst)
+        remote = path_info.remote
+        parsed: Parsed = self.config.parse()
+        sections: dict[str, Section] = parsed.sections
+        if remote not in sections:
+            return False
+        section: Section = sections[remote]
+        t = section.type()
+        return t in ["s3", "b2"]
+
     def copy_file_s3_resumable(
         self,
         src: str,  # src:/Bucket/path/myfile.large.zst
@@ -864,15 +914,28 @@ class RcloneImpl:
     def write_bytes(
         self,
         dst: str,
-        data: bytes,
+        data: bytes | Path,
+        verbose: bool | None = None,
     ) -> Exception | None:
         """Write bytes to a file."""
-        with TemporaryDirectory() as tmpdir:
-            tmpfile = Path(tmpdir) / "file.bin"
-            tmpfile.write_bytes(data)
-            completed_proc = self.copy_to(str(tmpfile), dst, check=True)
-            if completed_proc.returncode != 0:
-                return Exception(f"Failed to write bytes to {dst}", completed_proc)
+
+        try:
+            if isinstance(data, Path):
+                data = data.read_bytes()
+
+            with TemporaryDirectory() as tmpdir:
+                tmpfile = Path(tmpdir) / "file.bin"
+                tmpfile.write_bytes(data)
+                dst_is_s3 = self.is_s3(dst)
+                if dst_is_s3:
+                    return self.copy_file_s3(tmpfile, dst, verbose=verbose)
+
+                completed_proc = self.copy_to(str(tmpfile), dst, check=True)
+                if completed_proc.returncode != 0:
+                    return Exception(f"Failed to write bytes to {dst}", completed_proc)
+        except Exception as e:
+            logging.error(f"Failed to write bytes to {dst}")
+            return e
         return None
 
     def read_bytes(self, src: str) -> bytes | Exception:
@@ -925,10 +988,10 @@ class RcloneImpl:
     def get_s3_credentials(
         self, remote: str, verbose: bool | None = None
     ) -> S3Credentials:
-        from rclone_api.util import S3PathInfo, split_s3_path
+        from rclone_api.util import S3PathInfo
 
         verbose = get_verbose(verbose)
-        path_info: S3PathInfo = split_s3_path(remote)
+        path_info: S3PathInfo = S3PathInfo.from_str(remote)
 
         # path_info: S3PathInfo = split_s3_path(remote)
         remote = path_info.remote

rclone_api/s3/api.py

@@ -27,6 +27,10 @@ class S3Client:
             s3_creds=s3_creds, s3_config=S3Config(verbose=verbose)
         )
 
+    @property
+    def bucket_name(self) -> str:
+        return self.credentials.bucket_name
+
     def list_bucket_contents(self, bucket_name: str) -> None:
         list_bucket_contents(self.client, bucket_name)
 

rclone_api/types.py

@@ -31,6 +31,12 @@ class S3PathInfo:
     bucket: str
     key: str
 
+    @staticmethod
+    def from_str(src: str) -> "S3PathInfo":
+        from rclone_api.util import split_s3_path
+
+        return split_s3_path(src)
+
 
 @dataclass
 class SizeResult:

{rclone_api-1.5.50.dist-info → rclone_api-1.5.52.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rclone_api
-Version: 1.5.50
+Version: 1.5.52
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.5.50.dist-info → rclone_api-1.5.52.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=SDvu6dsXpgJTnsqT6aH8P-46lWTirF99tgXToj-_R94,34097
+rclone_api/__init__.py,sha256=M3-jlyxhO7Md3FGMO8qBJ1Z7oLcbOjWshCHh2-CqKG4,34885
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=DEBubN0ziUfAFYe6cmDn7YoTqRJPdGronFyRHZvWUso,6070
@@ -20,11 +20,11 @@ rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=LZqEhuKZunbWVqmsOIqkkCotaxWJpdFRS1InXveoU5E,1428
 rclone_api/mount_util.py,sha256=jqhJEVTHV6c6lOOzUYb4FLMbqDMHdz7-QRcdH-IobFc,10154
 rclone_api/process.py,sha256=MeWiN-TrrN0HmtWexBPxGwf84z6f-_E5yaXE-YtLYpY,5879
-rclone_api/rclone_impl.py,sha256=b9vcRH9__A8cYfeXaGnaLJG7JzdlaGTzVAiUaHNSO6g,50150
+rclone_api/rclone_impl.py,sha256=s00SHQGcKskbauWiIZQBQKSR0NOJ6SSiXhMf0fwkl9U,52328
 rclone_api/remote.py,sha256=mTgMTQTwxUmbLjTpr-AGTId2ycXKI9mLX5L7PPpDIoc,520
 rclone_api/rpath.py,sha256=Y1JjQWcie39EgQrq-UtbfDz5yDLCwwfu27W7AQXllSE,2860
 rclone_api/scan_missing_folders.py,sha256=-8NCwpCaHeHrX-IepCoAEsX1rl8S-GOCxcIhTr_w3gA,4747
-rclone_api/types.py,sha256=2ngxwpdNy88y0teeYJ5Vz5NiLK1rfaFx8Xf99i0J-Js,12155
+rclone_api/types.py,sha256=59Rw7NdHw35X6iiWMemO61IzCbrjgfNHUlzY3Dq0wdM,12303
 rclone_api/util.py,sha256=Xa_VifjNT_h9IMkqbnAmgSkEqqwczkGH_TcUUJHx-no,9546
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
 rclone_api/cmd/analyze.py,sha256=RHbvk1G5ZUc3qLqlm1AZEyQzd_W_ZjcbCNDvW4YpTKQ,1252
@@ -40,8 +40,8 @@ rclone_api/detail/copy_file_parts_resumable.py,sha256=RoUWV2eBWEvuuTfsvrz5BhtvX3
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/fs/filesystem.py,sha256=xUkoSV2R9dByKgueZ3x2GjyCn9fM7tK1ydgK1Whiph0,8933
-rclone_api/s3/api.py,sha256=6E4xEOxtpP6niiAFEpgB1-ckWJclNyRsJ3D11Qm4RwU,4069
+rclone_api/fs/filesystem.py,sha256=KgaLqk6OCePcWWtPgh9NDf3dMJzYpHdhQspvzgQ0r1s,9919
+rclone_api/s3/api.py,sha256=HpheMOGHcCc0CyNdv0zvB0S92g312urjRdssRrkdpb8,4162
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=waEYe-iYQ1_BR3NCS4BrzVrK9UANvH1EcbXx2I6Z_NM,6839
 rclone_api/s3/create.py,sha256=_Q-faQ4Zl8XKTB28gireRxVXWP-YNxoAK4bligxDtiI,3998
@@ -55,9 +55,9 @@ rclone_api/s3/multipart/upload_parts_inline.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9
 rclone_api/s3/multipart/upload_parts_resumable.py,sha256=6-nlMclS8jyVvMvFbQDcZOX9MY1WbCcKA_s9bwuYxnk,9793
 rclone_api/s3/multipart/upload_parts_server_side_merge.py,sha256=Fp2pdrs5dONQI9LkfNolgAGj1-Z2V1SsRd0r0sreuXI,18040
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.5.50.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.5.50.dist-info/METADATA,sha256=SoxiYtc9A7AVTeWiTMqqsSakoS1LXaTKisjQ5hpwZ08,37305
-rclone_api-1.5.50.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rclone_api-1.5.50.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
-rclone_api-1.5.50.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.5.50.dist-info/RECORD,,
+rclone_api-1.5.52.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.5.52.dist-info/METADATA,sha256=pzUkl7zLuD22M8z-gOpbEkACDpT0tZ6r62N3hJ-LwMc,37305
+rclone_api-1.5.52.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rclone_api-1.5.52.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
+rclone_api-1.5.52.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.5.52.dist-info/RECORD,,