rclone-api 1.3.5__py2.py3-none-any.whl → 1.3.7__py2.py3-none-any.whl

rclone_api/s3/upload_file_multipart.py

@@ -258,9 +258,6 @@ def upload_file_multipart(
             try:
                 while True:
                     file_chunk: FilePart | EndOfStream = queue_upload.get()
-                    if file_chunk is EndOfStream:
-                        break
-
                     if isinstance(file_chunk, EndOfStream):
                         break
 
@@ -291,6 +288,12 @@ def upload_file_multipart(
         if not upload_state.is_done():
             upload_state.save()
             return MultiUploadResult.SUSPENDED
+        ######################## COMPLETE UPLOAD #######################
+        # Final part now is to complete the upload
+        msg = "\n########################################"
+        msg += f"# Upload complete, sorting {len(upload_state.parts)} parts to complete upload"
+        msg += "########################################\n"
+        locked_print(msg)
         parts: list[FinishedPiece] = [
             p for p in upload_state.parts if not isinstance(p, EndOfStream)
         ]

rclone_api/util.py

@@ -1,162 +1,183 @@
-import os
-import shutil
-import subprocess
-import warnings
-from pathlib import Path
-from tempfile import TemporaryDirectory
-from threading import Lock
-from typing import Any
-
-from rclone_api.config import Config
-from rclone_api.dir import Dir
-from rclone_api.remote import Remote
-from rclone_api.rpath import RPath
-from rclone_api.types import S3PathInfo
-
-# from .rclone import Rclone
-
-_PRINT_LOCK = Lock()
-
-
-def locked_print(*args, **kwargs):
-    with _PRINT_LOCK:
-        print(*args, **kwargs)
-
-
-def to_path(item: Dir | Remote | str, rclone: Any) -> RPath:
-    from rclone_api.rclone import Rclone
-
-    assert isinstance(rclone, Rclone)
-    # if str then it will be remote:path
-    if isinstance(item, str):
-        # return RPath(item)
-        # remote_name: str = item.split(":")[0]
-        parts = item.split(":")
-        remote_name = parts[0]
-        path = ":".join(parts[1:])
-        remote = Remote(name=remote_name, rclone=rclone)
-        out = RPath(
-            remote=remote,
-            path=path,
-            name="",
-            size=0,
-            mime_type="",
-            mod_time="",
-            is_dir=True,
-        )
-        out.set_rclone(rclone)
-        return out
-    elif isinstance(item, Dir):
-        return item.path
-    elif isinstance(item, Remote):
-        out = RPath(
-            remote=item,
-            path=str(item),
-            name=str(item),
-            size=0,
-            mime_type="inode/directory",
-            mod_time="",
-            is_dir=True,
-        )
-        out.set_rclone(rclone)
-        return out
-    else:
-        raise ValueError(f"Invalid type for item: {type(item)}")
-
-
-def get_verbose(verbose: bool | None) -> bool:
-    if verbose is not None:
-        return verbose
-    # get it from the environment
-    return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
-
-
-def get_check(check: bool | None) -> bool:
-    if check is not None:
-        return check
-    # get it from the environment
-    return bool(int(os.getenv("RCLONE_API_CHECK", "1")))
-
-
-def get_rclone_exe(rclone_exe: Path | None) -> Path:
-    if rclone_exe is None:
-
-        rclone_which_path = shutil.which("rclone")
-        if rclone_which_path is None:
-            raise ValueError("rclone executable not found")
-        return Path(rclone_which_path)
-    return rclone_exe
-
-
-def rclone_execute(
-    cmd: list[str],
-    rclone_conf: Path | Config,
-    rclone_exe: Path,
-    check: bool,
-    capture: bool | None = None,
-    verbose: bool | None = None,
-) -> subprocess.CompletedProcess:
-    tempdir: TemporaryDirectory | None = None
-    verbose = get_verbose(verbose)
-    capture = capture if isinstance(capture, bool) else True
-    assert verbose is not None
-
-    try:
-        if isinstance(rclone_conf, Config):
-            tempdir = TemporaryDirectory()
-            tmpfile = Path(tempdir.name) / "rclone.conf"
-            tmpfile.write_text(rclone_conf.text, encoding="utf-8")
-            rclone_conf = tmpfile
-        cmd = (
-            [str(rclone_exe.resolve())] + ["--config", str(rclone_conf.resolve())] + cmd
-        )
-        if verbose:
-            cmd_str = subprocess.list2cmdline(cmd)
-            print(f"\nRunning: {cmd_str}")
-        cp = subprocess.run(
-            cmd, capture_output=capture, encoding="utf-8", check=False, shell=False
-        )
-        if cp.returncode != 0:
-            cmd_str = subprocess.list2cmdline(cmd)
-            warnings.warn(
-                f"Error running: {cmd_str}, returncode: {cp.returncode}\n{cp.stdout}\n{cp.stderr}"
-            )
-            if check:
-                raise subprocess.CalledProcessError(
-                    cp.returncode, cmd, cp.stdout, cp.stderr
-                )
-        return cp
-    finally:
-        if tempdir:
-            try:
-                tempdir.cleanup()
-            except Exception as e:
-                print(f"Error cleaning up tempdir: {e}")
-
-
-def split_s3_path(path: str) -> S3PathInfo:
-    if ":" not in path:
-        raise ValueError(f"Invalid S3 path: {path}")
-
-    prts = path.split(":", 1)
-    remote = prts[0]
-    path = prts[1]
-    parts: list[str] = []
-    for part in path.split("/"):
-        part = part.strip()
-        if part:
-            parts.append(part)
-    if len(parts) < 2:
-        raise ValueError(f"Invalid S3 path: {path}")
-    bucket = parts[0]
-    key = "/".join(parts[1:])
-    assert bucket
-    assert key
-    return S3PathInfo(remote=remote, bucket=bucket, key=key)
-
-
-def random_str(length: int) -> str:
-    import random
-    import string
-
-    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
+import os
+import shutil
+import subprocess
+import warnings
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from threading import Lock
+from typing import Any
+
+from rclone_api.config import Config
+from rclone_api.dir import Dir
+from rclone_api.remote import Remote
+from rclone_api.rpath import RPath
+from rclone_api.types import S3PathInfo
+
+# from .rclone import Rclone
+
+_PRINT_LOCK = Lock()
+
+
+def locked_print(*args, **kwargs):
+    with _PRINT_LOCK:
+        print(*args, **kwargs)
+
+
+def to_path(item: Dir | Remote | str, rclone: Any) -> RPath:
+    from rclone_api.rclone import Rclone
+
+    assert isinstance(rclone, Rclone)
+    # if str then it will be remote:path
+    if isinstance(item, str):
+        # return RPath(item)
+        # remote_name: str = item.split(":")[0]
+        parts = item.split(":")
+        remote_name = parts[0]
+        path = ":".join(parts[1:])
+        remote = Remote(name=remote_name, rclone=rclone)
+        out = RPath(
+            remote=remote,
+            path=path,
+            name="",
+            size=0,
+            mime_type="",
+            mod_time="",
+            is_dir=True,
+        )
+        out.set_rclone(rclone)
+        return out
+    elif isinstance(item, Dir):
+        return item.path
+    elif isinstance(item, Remote):
+        out = RPath(
+            remote=item,
+            path=str(item),
+            name=str(item),
+            size=0,
+            mime_type="inode/directory",
+            mod_time="",
+            is_dir=True,
+        )
+        out.set_rclone(rclone)
+        return out
+    else:
+        raise ValueError(f"Invalid type for item: {type(item)}")
+
+
+def get_verbose(verbose: bool | None) -> bool:
+    if verbose is not None:
+        return verbose
+    # get it from the environment
+    return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
+
+
+def get_check(check: bool | None) -> bool:
+    if check is not None:
+        return check
+    # get it from the environment
+    return bool(int(os.getenv("RCLONE_API_CHECK", "1")))
+
+
+def get_rclone_exe(rclone_exe: Path | None) -> Path:
+    if rclone_exe is None:
+
+        rclone_which_path = shutil.which("rclone")
+        if rclone_which_path is None:
+            raise ValueError("rclone executable not found")
+        return Path(rclone_which_path)
+    return rclone_exe
+
+
+def rclone_execute(
+    cmd: list[str],
+    rclone_conf: Path | Config,
+    rclone_exe: Path,
+    check: bool,
+    capture: bool | Path | None = None,
+    verbose: bool | None = None,
+) -> subprocess.CompletedProcess:
+    tempdir: TemporaryDirectory | None = None
+    verbose = get_verbose(verbose)
+
+    # Handle the Path case for capture
+    output_file = None
+    if isinstance(capture, Path):
+        output_file = capture
+        capture = False  # Don't capture to memory when redirecting to file
+    else:
+        capture = capture if isinstance(capture, bool) else True
+
+    assert verbose is not None
+
+    try:
+        if isinstance(rclone_conf, Config):
+            tempdir = TemporaryDirectory()
+            tmpfile = Path(tempdir.name) / "rclone.conf"
+            tmpfile.write_text(rclone_conf.text, encoding="utf-8")
+            rclone_conf = tmpfile
+        cmd = (
+            [str(rclone_exe.resolve())] + ["--config", str(rclone_conf.resolve())] + cmd
+        )
+        if verbose:
+            cmd_str = subprocess.list2cmdline(cmd)
+            print(f"\nRunning: {cmd_str}")
+
+        # If output_file is set, redirect output to that file
+        if output_file:
+            with open(output_file, "w", encoding="utf-8") as f:
+                cp = subprocess.run(
+                    cmd,
+                    stdout=f,
+                    stderr=subprocess.PIPE,
+                    encoding="utf-8",
+                    check=False,
+                    shell=False,
+                )
+        else:
+            cp = subprocess.run(
+                cmd, capture_output=capture, encoding="utf-8", check=False, shell=False
+            )
+        if cp.returncode != 0:
+            cmd_str = subprocess.list2cmdline(cmd)
+            warnings.warn(
+                f"Error running: {cmd_str}, returncode: {cp.returncode}\n{cp.stdout}\n{cp.stderr}"
+            )
+            if check:
+                raise subprocess.CalledProcessError(
+                    cp.returncode, cmd, cp.stdout, cp.stderr
+                )
+        return cp
+    finally:
+        if tempdir:
+            try:
+                tempdir.cleanup()
+            except Exception as e:
+                print(f"Error cleaning up tempdir: {e}")
+
+
+def split_s3_path(path: str) -> S3PathInfo:
+    if ":" not in path:
+        raise ValueError(f"Invalid S3 path: {path}")
+
+    prts = path.split(":", 1)
+    remote = prts[0]
+    path = prts[1]
+    parts: list[str] = []
+    for part in path.split("/"):
+        part = part.strip()
+        if part:
+            parts.append(part)
+    if len(parts) < 2:
+        raise ValueError(f"Invalid S3 path: {path}")
+    bucket = parts[0]
+    key = "/".join(parts[1:])
+    assert bucket
+    assert key
+    return S3PathInfo(remote=remote, bucket=bucket, key=key)
+
+
+def random_str(length: int) -> str:
+    import random
+    import string
+
+    return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))

{rclone_api-1.3.5.dist-info → rclone_api-1.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.5
+Version: 1.3.7
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.5.dist-info → rclone_api-1.3.7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=96hecpVEvaLklfaL5K_uBV-gaftt9kklC0XMlutZMf8,1197
+rclone_api/__init__.py,sha256=bJ6x-7ySj1kC7xjQJqEEA-0cr46RUh_tvIZsebGcyu4,1224
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -7,37 +7,37 @@ rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
 rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
 rclone_api/dir.py,sha256=i4h7LX5hB_WmVixxDRWL_l1nifvscrdWct_8Wx7wHZc,3540
 rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,1882
-rclone_api/exec.py,sha256=Pd7pUBd8ib5MzqvMybG2DQISPRbDRu20VjVRL2mLAVY,1076
-rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
+rclone_api/exec.py,sha256=Bq0gkyZ10mEY0FRyzNZgdN4FaWP9vpeCk1kjpg-gN_8,1083
+rclone_api/file.py,sha256=HdqzNpUKyOTGLb8XCgEMDAug5av5sEB3ieDow6MnPnk,2809
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
 rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=TE_VIBMW7J1UkF_6HRCt8oi_jGdMov4S51bm2OgxFAM,10045
-rclone_api/mount_read_chunker.py,sha256=Pz4s6AGigB_-vTAEdLK9Fv116j87yN4Qg0Uz1COwPtQ,4504
-rclone_api/process.py,sha256=rBj_S86jC6nqCYop-jq8r9eMSteKeObxUrJMgH8LZvI,5084
-rclone_api/rclone.py,sha256=DOLiBeRSnHE1Q0MTnhzqfm--PhN4PJebcj_U0HsGg2Q,46647
+rclone_api/mount_read_chunker.py,sha256=bi4N-VkyPdM4RWUYhIqJ71lQrXanck4PpF3pY8k2xnQ,4722
+rclone_api/process.py,sha256=xmI4T9i1tQfAy9lfwVU3w8sndkDkbM5KVUNWiKHCemY,5226
+rclone_api/rclone.py,sha256=gL0b41BBedj23EmNJtZHpFI0_NGHbz2WT0PIq4KLZKg,50215
 rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
 rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
 rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
 rclone_api/types.py,sha256=XiiWoGXAzNxTEfRcszw3BeRF7ZATXHIAPFg2-aJzUfo,9926
-rclone_api/util.py,sha256=_Z-GUMVXnHYOGdo2dy2ie2P5fGgyg8KdGjHKicx68Ko,4573
+rclone_api/util.py,sha256=F9Q3zbWRsgPF4NG6OWB63cZ7GVq82lsraP47gmmDohU,5416
 rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
-rclone_api/cmd/copy_large_s3.py,sha256=fYHyHq2YZT_dfMbS7SCpEeLCaWD-BU-jcpKP9eKf1jk,3388
+rclone_api/cmd/copy_large_s3.py,sha256=nOpAUAQN1mJnf4EIZCh4OVCW7Q4_EXJeLFVe6r_9rZA,3369
 rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/profile/mount_copy_bytes.py,sha256=2NqCxqa5SCwpZyguAoWR9yqjvObmxy5gqkQeYbnmbJE,9066
+rclone_api/profile/mount_copy_bytes.py,sha256=nZtqMukLhSzHq64Pn1I8pXwjoraqWjCKey3WLAeubx0,9069
 rclone_api/s3/api.py,sha256=PafsIEyWDpLWAXsZAjFm9CY14vJpsDr9lOsn0kGRLZ0,4009
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=plrQyjuHFdJfV55kKH4wqo6QGll9n4_7BO4B1A7j6kY,7121
 rclone_api/s3/chunk_types.py,sha256=oSWv8No9V3BeM7IcGnowyR2a7YrszdAXzEJlxaeZcp0,8852
 rclone_api/s3/create.py,sha256=wgfkapv_j904CfKuWyiBIWJVxfAx_ftemFSUV14aT68,3149
 rclone_api/s3/types.py,sha256=Elmh__gvZJyJyElYwMmvYZIBIunDJiTRAbEg21GmsRU,1604
-rclone_api/s3/upload_file_multipart.py,sha256=UVMTSeP98fSQdOYcCdi9tV5ZjOxRDuhZbiBbVaf-rCM,11385
-rclone_api-1.3.5.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.3.5.dist-info/METADATA,sha256=TsRyMEuwiDpyJiJPnL8CJM6bGwEUDq8gIAZRPQy8fdo,4536
-rclone_api-1.3.5.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
-rclone_api-1.3.5.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
-rclone_api-1.3.5.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.3.5.dist-info/RECORD,,
+rclone_api/s3/upload_file_multipart.py,sha256=d8ZWqO8n9wsqRF6JjmvAFmG1aCkFqdSB1L8yxe_5qiY,11669
+rclone_api-1.3.7.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.3.7.dist-info/METADATA,sha256=OfqlMlZKb0YGl8hN6ae_Y1Icx2RKqlYj4UsMmcrAlEY,4536
+rclone_api-1.3.7.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.3.7.dist-info/entry_points.txt,sha256=TV8kwP3FRzYwUEr0RLC7aJh0W03SAefIJNXTJ-FdMIQ,200
+rclone_api-1.3.7.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.3.7.dist-info/RECORD,,