rclone-api 1.2.1__tar.gz → 1.2.3__tar.gz

{rclone_api-1.2.1 → rclone_api-1.2.3}/.gitignore

@@ -150,3 +150,4 @@ t.py
 mount
 t2.py
 chunk_store
+state.json

{rclone_api-1.2.1 → rclone_api-1.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.2.1
+Version: 1.2.3
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.2.1 → rclone_api-1.2.3}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.2.1"
+version = "1.2.3"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.2.1 → rclone_api-1.2.3}/src/rclone_api/cmd/copy_large_s3.py

@@ -45,7 +45,7 @@ def _parse_args() -> Args:
         "--read-threads",
         help="Number of concurrent read threads per chunk, only one chunk will be read at a time",
         type=int,
-        default=16,
+        default=8,
     )
     parser.add_argument(
         "--write-threads",

{rclone_api-1.2.1 → rclone_api-1.2.3}/src/rclone_api/mount.py

@@ -115,7 +115,7 @@ def prepare_mount(outdir: Path, verbose: bool) -> None:
 def wait_for_mount(
     path: Path,
     mount_process: Any,
-    timeout: int = 10,
+    timeout: int = 20,
     post_mount_delay: int = 5,
     poll_interval: float = 1.0,
     check_mount_flag: bool = False,
@@ -292,6 +292,7 @@ class MultiMountFileChunker:
     def __init__(
         self,
         filename: str,
+        filesize: int,
         chunk_size: SizeSuffix,
         mounts: list[Mount],
         executor: ThreadPoolExecutor,
@@ -300,6 +301,7 @@ class MultiMountFileChunker:
         from rclone_api.util import get_verbose
 
         self.filename = filename
+        self.filesize = filesize
         self.chunk_size = chunk_size
         self.executor = executor
         self.mounts_processing: list[Mount] = []
@@ -309,7 +311,7 @@ class MultiMountFileChunker:
         self.verbose = get_verbose(verbose)
 
     def close(self) -> None:
-        self.executor.shutdown(wait=False, cancel_futures=True)
+        self.executor.shutdown(wait=True, cancel_futures=True)
         with ThreadPoolExecutor() as executor:
             for mount in self.mounts_processing:
                 executor.submit(lambda: mount.close())
@@ -319,9 +321,11 @@ class MultiMountFileChunker:
             print(f"Fetching data range: offset={offset}, size={size}")
         try:
             try:
+                if offset + size > self.filesize:
+                    size = self.filesize - offset
                 assert (
-                    offset % self.chunk_size.as_int() == 0
-                ), f"Invalid offset: {offset}"
+                    offset + size <= self.filesize
+                ), f"Invalid offset + size: {offset + size}, it is beyond the end of the file."
                 assert size > 0, f"Invalid size: {size}"
                 assert offset >= 0, f"Invalid offset: {offset}"
             except AssertionError as e:
@@ -356,6 +360,12 @@ class MultiMountFileChunker:
                             f"Fetching chunk: offset={offset}, size={size}, path={path}"
                         )
                     try:
+                        # make sure we don't overflow the file size
+                        # assert (
+                        #     offset + size <= self.chunk_size.as_int()
+                        # ), f"Invalid offset + size: {offset + size}, it is beyond the end of the file."
+                        if offset + size > self.filesize:
+                            size = self.filesize - offset
                         with path.open("rb") as f:
                             f.seek(offset)
                             return f.read(size)
@@ -369,7 +379,7 @@ class MultiMountFileChunker:
                     except Exception as e:
                         stack_trace = traceback.format_exc()
                         warnings.warn(
-                            f"Error fetching file chunk at offset{offset} + {size}: {e}\n{stack_trace}"
+                            f"Error fetching file chunk at offset {offset} + {size}: {e}\n{stack_trace}"
                         )
                         return e
                     finally:

{rclone_api-1.2.1 → rclone_api-1.2.3}/src/rclone_api/rclone.py

@@ -353,13 +353,11 @@ class Rclone:
         check: bool | None = None,
         verbose: bool | None = None,
         other_args: list[str] | None = None,
-    ) -> None:
-        """Copy multiple files from source to destination.
+    ) -> CompletedProcess:
+        """Copy one file from source to destination.
 
         Warning - slow.
 
-        Args:
-            payload: Dictionary of source and destination file paths
         """
         check = get_check(check)
         verbose = get_verbose(verbose)
@@ -368,7 +366,8 @@ class Rclone:
         cmd_list: list[str] = ["copyto", src, dst]
         if other_args is not None:
             cmd_list += other_args
-        self._run(cmd_list, check=check)
+        cp = self._run(cmd_list, check=check)
+        return CompletedProcess.from_subprocess(cp)
 
     def copy_files(
         self,
@@ -680,18 +679,17 @@ class Rclone:
         dst: str,
         save_state_json: Path,
         chunk_size: SizeSuffix | None = None,
-        read_threads: int = 16,
+        read_threads: int = 8,
         write_threads: int = 16,
         retries: int = 3,
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
-        mount_path: Path | None = None,
         mount_log: Path | None = None,
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
         from rclone_api.s3.api import S3Client
         from rclone_api.s3.create import S3Credentials
-        from rclone_api.util import S3PathInfo, random_str, split_s3_path
+        from rclone_api.util import S3PathInfo, split_s3_path
 
         other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
         chunk_size = chunk_size or SizeSuffix("64M")
@@ -717,105 +715,117 @@ class Rclone:
         other_args += ["--direct-io"]
         # --vfs-cache-max-size
         other_args += ["--vfs-cache-max-size", vfs_disk_space_total_size.as_str()]
-        mount_path = mount_path or Path("tmp_mnts") / random_str(12)
+        mount_path = Path("tmp_mnts") / "RCLONE_API_DYNAMIC_MOUNT"
         src_path = Path(src)
         name = src_path.name
-        parent_path = str(src_path.parent.as_posix())
-        with self.scoped_mount(
-            parent_path,
-            mount_path,
-            use_links=True,
-            vfs_cache_mode="minimal",
-            verbose=False,
-            log=mount_log,
-            other_args=other_args,
-        ):
-            path_info: S3PathInfo = split_s3_path(dst)
-            remote = path_info.remote
-            bucket_name = path_info.bucket
-            s3_key = path_info.key
-            parsed: Parsed = self.config.parse()
-            sections: dict[str, Section] = parsed.sections
-            if remote not in sections:
-                raise ValueError(
-                    f"Remote {remote} not found in rclone config, remotes are: {sections.keys()}"
-                )
 
-            section: Section = sections[remote]
-            dst_type = section.type()
-            if dst_type != "s3" and dst_type != "b2":
-                raise ValueError(
-                    f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
-                )
-
-            def get_provider_str(section=section) -> str | None:
-                type: str = section.type()
-                provider: str | None = section.provider()
-                if provider is not None:
-                    return provider
-                if type == "b2":
-                    return S3Provider.BACKBLAZE.value
-                if type != "s3":
-                    raise ValueError(f"Remote {remote} is not an S3 remote")
-                return S3Provider.S3.value
-
-            provider: str
-            if provided_provider_str := get_provider_str():
-                if verbose:
-                    print(f"Using provided provider: {provided_provider_str}")
-                provider = provided_provider_str
-            else:
-                if verbose:
-                    print(f"Using default provider: {S3Provider.S3.value}")
-                provider = S3Provider.S3.value
-            provider_enum = S3Provider.from_str(provider)
-
-            s3_creds: S3Credentials = S3Credentials(
-                provider=provider_enum,
-                access_key_id=section.access_key_id(),
-                secret_access_key=section.secret_access_key(),
-                endpoint_url=section.endpoint(),
+        src_parent_path = Path(src).parent.as_posix()
+        size_result: SizeResult = self.size_files(src_parent_path, [name])
+
+        target_size = SizeSuffix(size_result.total_size)
+        if target_size < SizeSuffix("5M"):
+            # fallback to normal copy
+            completed_proc = self.copy_to(src, dst, check=True)
+            if completed_proc.ok:
+                return MultiUploadResult.UPLOADED_FRESH
+        if size_result.total_size <= 0:
+            raise ValueError(
+                f"File {src} has size {size_result.total_size}, is this a directory?"
             )
 
-            chunk_fetcher: MultiMountFileChunker = self.get_multi_mount_file_chunker(
-                src=src_path.as_posix(),
-                chunk_size=chunk_size,
-                threads=read_threads,
-                mount_log=mount_log,
-                direct_io=True,
+        path_info: S3PathInfo = split_s3_path(dst)
+        remote = path_info.remote
+        bucket_name = path_info.bucket
+        s3_key = path_info.key
+        parsed: Parsed = self.config.parse()
+        sections: dict[str, Section] = parsed.sections
+        if remote not in sections:
+            raise ValueError(
+                f"Remote {remote} not found in rclone config, remotes are: {sections.keys()}"
             )
 
-            client = S3Client(s3_creds)
-            upload_config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
-                chunk_size=chunk_size.as_int(),
-                chunk_fetcher=chunk_fetcher.fetch,
-                max_write_threads=write_threads,
-                retries=retries,
-                resume_path_json=save_state_json,
-                max_chunks_before_suspension=max_chunks_before_suspension,
+        section: Section = sections[remote]
+        dst_type = section.type()
+        if dst_type != "s3" and dst_type != "b2":
+            raise ValueError(
+                f"Remote {remote} is not an S3 remote, it is of type {dst_type}"
             )
 
-            src_file = mount_path / name
+        def get_provider_str(section=section) -> str | None:
+            type: str = section.type()
+            provider: str | None = section.provider()
+            if provider is not None:
+                return provider
+            if type == "b2":
+                return S3Provider.BACKBLAZE.value
+            if type != "s3":
+                raise ValueError(f"Remote {remote} is not an S3 remote")
+            return S3Provider.S3.value
+
+        provider: str
+        if provided_provider_str := get_provider_str():
+            if verbose:
+                print(f"Using provided provider: {provided_provider_str}")
+            provider = provided_provider_str
+        else:
+            if verbose:
+                print(f"Using default provider: {S3Provider.S3.value}")
+            provider = S3Provider.S3.value
+        provider_enum = S3Provider.from_str(provider)
+
+        s3_creds: S3Credentials = S3Credentials(
+            provider=provider_enum,
+            access_key_id=section.access_key_id(),
+            secret_access_key=section.secret_access_key(),
+            endpoint_url=section.endpoint(),
+        )
 
-            print(f"Uploading {name} to {s3_key} in bucket {bucket_name}")
-            print(f"Source: {src_path}")
-            print(f"bucket_name: {bucket_name}")
-            print(f"upload_config: {upload_config}")
+        chunk_fetcher: MultiMountFileChunker = self.get_multi_mount_file_chunker(
+            src=src_path.as_posix(),
+            chunk_size=chunk_size,
+            threads=read_threads,
+            mount_log=mount_log,
+            direct_io=True,
+        )
 
-            upload_target = S3UploadTarget(
-                bucket_name=bucket_name,
-                src_file=src_file,
-                s3_key=s3_key,
-            )
+        client = S3Client(s3_creds)
+        upload_config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
+            chunk_size=chunk_size.as_int(),
+            chunk_fetcher=chunk_fetcher.fetch,
+            max_write_threads=write_threads,
+            retries=retries,
+            resume_path_json=save_state_json,
+            max_chunks_before_suspension=max_chunks_before_suspension,
+        )
 
-            try:
-                out: MultiUploadResult = client.upload_file_multipart(
-                    upload_target=upload_target,
-                    upload_config=upload_config,
-                )
-                return out
-            finally:
-                chunk_fetcher.close()
+        src_file = mount_path / name
+
+        print(f"Uploading {name} to {s3_key} in bucket {bucket_name}")
+        print(f"Source: {src_path}")
+        print(f"bucket_name: {bucket_name}")
+        print(f"upload_config: {upload_config}")
+
+        # get the file size
+
+        upload_target = S3UploadTarget(
+            src_file=src_file,
+            src_file_size=size_result.total_size,
+            bucket_name=bucket_name,
+            s3_key=s3_key,
+        )
+
+        try:
+            out: MultiUploadResult = client.upload_file_multipart(
+                upload_target=upload_target,
+                upload_config=upload_config,
+            )
+            return out
+        except Exception as e:
+            print(f"Error uploading file: {e}")
+            traceback.print_exc()
+            raise
+        finally:
+            chunk_fetcher.close()
 
     def _copy_bytes(
         self,
@@ -963,9 +973,17 @@ class Rclone:
                 for mount in mounts:
                     mount.close()
                 raise
+
+        src_path: Path = Path(src)
+        src_parent_path = src_path.parent.as_posix()
+        name = src_path.name
+        size_result: SizeResult = self.size_files(src_parent_path, [name])
+        filesize = size_result.total_size
+
         executor = ThreadPoolExecutor(max_workers=threads)
         filechunker: MultiMountFileChunker = MultiMountFileChunker(
             filename=filename,
+            filesize=filesize,
             chunk_size=chunk_size,
             mounts=mounts,
             executor=executor,

{rclone_api-1.2.1 → rclone_api-1.2.3}/src/rclone_api/s3/api.py

@@ -58,7 +58,12 @@ class S3Client:
         bucket_name = upload_target.bucket_name
 
         try:
-            filesize = upload_target.src_file.stat().st_size
+
+            if upload_target.src_file_size is None:
+                filesize = upload_target.src_file.stat().st_size
+            else:
+                filesize = upload_target.src_file_size
+
             if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
                 warnings.warn(
                     f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
@@ -73,6 +78,7 @@ class S3Client:
                 chunk_fetcher=upload_config.chunk_fetcher,
                 bucket_name=bucket_name,
                 file_path=upload_target.src_file,
+                file_size=filesize,
                 object_name=upload_target.s3_key,
                 resumable_info_path=resume_path_json,
                 chunk_size=chunk_size,

{rclone_api-1.2.1 → rclone_api-1.2.3}/src/rclone_api/s3/chunk_types.py

@@ -74,6 +74,10 @@ class FileChunk:
         return b""
 
     def close(self):
+        import traceback
+
+        stacktrace = traceback.format_stack()
+        locked_print(f"Closing file chunk: {self.filepart}\n{stacktrace}")
         if self.filepart.exists():
             self.filepart.unlink()
 
@@ -172,8 +176,12 @@ class UploadState:
     lock: Lock = Lock()
     parts: list[FinishedPiece | None] = field(default_factory=list)
 
-    def update_source_file(self, src_file: Path) -> None:
-        new_file_size = os.path.getsize(src_file)
+    def update_source_file(self, src_file: Path, known_file_size: int | None) -> None:
+        new_file_size = (
+            known_file_size
+            if known_file_size is not None
+            else os.path.getsize(src_file)
+        )
         if new_file_size != self.upload_info.file_size:
             raise ValueError("File size changed, cannot resume")
         self.upload_info.src_file_path = src_file

{rclone_api-1.2.1 → rclone_api-1.2.3}/src/rclone_api/s3/types.py

@@ -37,6 +37,7 @@ class S3UploadTarget:
     """Target information for S3 upload."""
 
     src_file: Path
+    src_file_size: int | None
     bucket_name: str
     s3_key: str
 

{rclone_api-1.2.1 → rclone_api-1.2.3}/src/rclone_api/s3/upload_file_multipart.py

@@ -76,6 +76,7 @@ def prepare_upload_file_multipart(
     s3_client: BaseClient,
     bucket_name: str,
     file_path: Path,
+    file_size: int | None,
     object_name: str,
     chunk_size: int,
     retries: int,
@@ -89,7 +90,7 @@ def prepare_upload_file_multipart(
     mpu = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_name)
     upload_id = mpu["UploadId"]
 
-    file_size = os.path.getsize(file_path)
+    file_size = file_size if file_size is not None else os.path.getsize(file_path)
 
     upload_info: UploadInfo = UploadInfo(
         s3_client=s3_client,
@@ -121,6 +122,7 @@ def upload_file_multipart(
     chunk_fetcher: Callable[[int, int], Future[bytes | Exception]],
     bucket_name: str,
     file_path: Path,
+    file_size: int | None,
     object_name: str,
     resumable_info_path: Path | None,
     chunk_size: int = 16 * 1024 * 1024,  # Default chunk size is 16MB; can be overridden
@@ -130,12 +132,12 @@ def upload_file_multipart(
     abort_transfer_on_failure: bool = False,
 ) -> MultiUploadResult:
     """Upload a file to the bucket using multipart upload with customizable chunk size."""
-    file_size = os.path.getsize(str(file_path))
-    if chunk_size > file_size:
-        warnings.warn(
-            f"Chunk size {chunk_size} is greater than file size {file_size}, using file size"
-        )
-        chunk_size = file_size
+    file_size = file_size if file_size is not None else os.path.getsize(str(file_path))
+    # if chunk_size > file_size:
+    #     warnings.warn(
+    #         f"Chunk size {chunk_size} is greater than file size {file_size}, using file size"
+    #     )
+    #     chunk_size = file_size
 
     if chunk_size < _MIN_UPLOAD_CHUNK_SIZE:
         raise ValueError(
@@ -160,6 +162,7 @@ def upload_file_multipart(
             s3_client=s3_client,
             bucket_name=bucket_name,
             file_path=file_path,
+            file_size=file_size,
             object_name=object_name,
             chunk_size=chunk_size,
             retries=retries,
@@ -194,7 +197,7 @@ def upload_file_multipart(
             upload_state = loaded_state
 
     try:
-        upload_state.update_source_file(file_path)
+        upload_state.update_source_file(file_path, file_size)
     except ValueError as e:
         locked_print(f"Cannot resume upload: {e}, size changed, starting over")
         _abort_previous_upload(upload_state)

{rclone_api-1.2.1 → rclone_api-1.2.3}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.2.1
+Version: 1.2.3
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.2.1 → rclone_api-1.2.3}/tests/test_copy_file_resumable_s3.py

@@ -135,8 +135,10 @@ class RcloneCopyResumableFileToS3(unittest.TestCase):
         print(f"Config file written to: {_CONFIG_PATH}")
         rclone = Rclone(_CONFIG_PATH)
         save_state_json = Path("state.json")
+        if save_state_json.exists():
+            save_state_json.unlink()
         rclone.copy_file_resumable_s3(
-            src="src:aa_misc_data/aa_misc_data/world_lending_library_2024_11.tar.zst.torrent",
+            src="dst:rclone-api-unit-test/zachs_video/perpetualmaniac_an_authoritative_school_teacher_forcing_a_stude_c65528d3-aa6f-4777-a56b-a919856d44e1.png",
             dst="dst:rclone-api-unit-test/test_data/test.torrent.testwrite",
             chunk_size=SizeSuffix("16MB"),
             retries=0,

{rclone_api-1.2.1 → rclone_api-1.2.3}/tests/test_s3.py

@@ -81,6 +81,7 @@ class RcloneS3Tester(unittest.TestCase):
 
             upload_target: S3UploadTarget = S3UploadTarget(
                 src_file=Path(tmpfile),
+                src_file_size=filesize,
                 bucket_name=BUCKET_NAME,
                 s3_key=dst_path,
             )