rclone-api 1.0.88__py2.py3-none-any.whl → 1.0.90__py2.py3-none-any.whl

rclone_api/s3/create.py

@@ -0,0 +1,69 @@
+import warnings
+
+import boto3
+from botocore.client import BaseClient
+from botocore.config import Config
+
+from rclone_api.s3.types import S3Credentials, S3Provider
+
+
+# Create a Boto3 session and S3 client, this is back blaze specific.
+# Add a function if you want to use a different S3 provider.
+# If AWS support is added in a fork then please merge it back here.
+def _create_backblaze_s3_client(creds: S3Credentials) -> BaseClient:
+    """Create and return an S3 client."""
+    region_name = creds.region_name
+    access_key = creds.access_key_id
+    secret_key = creds.secret_access_key
+    endpoint_url = creds.endpoint_url
+    region_name = region_name or "https://s3.us-west-002.backblazeb2.com"
+
+    session = boto3.session.Session()  # type: ignore
+    return session.client(
+        service_name="s3",
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        endpoint_url=endpoint_url,
+        config=Config(
+            signature_version="s3v4",
+            # Note that BackBlase has a boko3 bug where it doesn't support the new
+            # checksum header, the following line was an attempt of fix it on the newest
+            # version of boto3, but it didn't work.
+            # s3={"payload_signing_enabled": False},  # Disable checksum header
+        ),
+    )
+
+
+def _create_unknown_s3_client(creds: S3Credentials) -> BaseClient:
+    """Create and return an S3 client."""
+    access_key = creds.access_key_id
+    secret_key = creds.secret_access_key
+    endpoint_url = creds.endpoint_url
+    if (endpoint_url is not None) and not (endpoint_url.startswith("http")):
+        warnings.warn(f"Endpoint URL is schema niaive: {endpoint_url}, assuming HTTPS")
+        endpoint_url = f"https://{endpoint_url}"
+
+    session = boto3.session.Session()  # type: ignore
+    return session.client(
+        service_name="s3",
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        endpoint_url=endpoint_url,
+        config=Config(
+            signature_version="s3v4",
+            region_name=creds.region_name,
+            # Note that BackBlase has a boko3 bug where it doesn't support the new
+            # checksum header, the following line was an attempt of fix it on the newest
+            # version of boto3, but it didn't work.
+            # s3={"payload_signing_enabled": False},  # Disable checksum header
+        ),
+    )
+
+
+def create_s3_client(credentials: S3Credentials) -> BaseClient:
+    """Create and return an S3 client."""
+    provider = credentials.provider
+    if provider == S3Provider.BACKBLAZE:
+        return _create_backblaze_s3_client(credentials)
+    else:
+        return _create_unknown_s3_client(credentials)

rclone_api/s3/types.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+
+
+class S3Provider(Enum):
+    S3 = "s3"  # generic S3
+    BACKBLAZE = "b2"
+    DIGITAL_OCEAN = "DigitalOcean"
+
+    @staticmethod
+    def from_str(value: str) -> "S3Provider":
+        """Convert string to S3Provider."""
+        if value == "b2":
+            return S3Provider.BACKBLAZE
+        if value == "DigitalOcean":
+            return S3Provider.DIGITAL_OCEAN
+        raise ValueError(f"Unknown S3Provider: {value}")
+
+
+@dataclass
+class S3Credentials:
+    """Credentials for accessing S3."""
+
+    provider: S3Provider
+    access_key_id: str
+    secret_access_key: str
+    session_token: str | None = None
+    region_name: str | None = None
+    endpoint_url: str | None = None
+
+
+@dataclass
+class S3UploadTarget:
+    """Target information for S3 upload."""
+
+    src_file: Path
+    bucket_name: str
+    s3_key: str
+
+
+@dataclass
+class S3MutliPartUploadConfig:
+    """Input for multi-part upload."""
+
+    chunk_size: int
+    retries: int
+    resume_path_json: Path
+    max_chunks_before_suspension: int | None = None
+    mount_path: Path | None = None  # If set this will be used to mount the src file, otherwise it's one is chosen automatically
+
+
+
+class MultiUploadResult(Enum):
+    UPLOADED_FRESH = 1
+    UPLOADED_RESUME = 2
+    SUSPENDED = 3
+    ALREADY_DONE = 4

rclone_api/types.py

@@ -19,9 +19,11 @@ class Order(Enum):
     RANDOM = "random"
 
 
-# class GroupingOption(Enum):
-#     BUCKET = "bucket"
-#     REMOTE = "remote"
+@dataclass
+class S3PathInfo:
+    remote: str
+    bucket: str
+    key: str
 
 
 @dataclass

rclone_api/util.py

@@ -11,6 +11,7 @@ from rclone_api.config import Config
 from rclone_api.dir import Dir
 from rclone_api.remote import Remote
 from rclone_api.rpath import RPath
+from rclone_api.types import S3PathInfo
 
 # from .rclone import Rclone
 
@@ -126,7 +127,7 @@ def rclone_execute(
                 print(f"Error cleaning up tempdir: {e}")
 
 
-def wait_for_mount(path: Path, mount_process: Any, timeout: int = 60) -> None:
+def wait_for_mount(path: Path, mount_process: Any, timeout: int = 10) -> None:
     from rclone_api.process import Process
 
     assert isinstance(mount_process, Process)
@@ -134,7 +135,34 @@ def wait_for_mount(path: Path, mount_process: Any, timeout: int = 60) -> None:
     while time.time() < expire_time:
         rtn = mount_process.poll()
         if rtn is not None:
-            raise subprocess.CalledProcessError(rtn, mount_process.cmd)
+            cmd_str = subprocess.list2cmdline(mount_process.cmd)
+            raise subprocess.CalledProcessError(rtn, cmd_str)
         if path.exists():
-            return
-    raise TimeoutError(f"Path {path} did not exist after {timeout} seconds")
+            # how many files?
+            dircontents = os.listdir(str(path))
+            if len(dircontents) > 0:
+                print(f"Mount point {path}, waiting 5 seconds for files to appear.")
+                time.sleep(5)
+                return
+        time.sleep(1)
+
+
+def split_s3_path(path: str) -> S3PathInfo:
+    if ":" not in path:
+        raise ValueError(f"Invalid S3 path: {path}")
+
+    prts = path.split(":", 1)
+    remote = prts[0]
+    path = prts[1]
+    parts: list[str] = []
+    for part in path.split("/"):
+        part = part.strip()
+        if part:
+            parts.append(part)
+    if len(parts) < 2:
+        raise ValueError(f"Invalid S3 path: {path}")
+    bucket = parts[0]
+    key = "/".join(parts[1:])
+    assert bucket
+    assert key
+    return S3PathInfo(remote=remote, bucket=bucket, key=key)

{rclone_api-1.0.88.dist-info → rclone_api-1.0.90.dist-info}/METADATA

@@ -1,9 +1,8 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.0.88
+Version: 1.0.90
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
-Maintainer: Zachary Vorhies
 License: BSD 3-Clause License
 Keywords: template-python-cmd
 Classifier: Programming Language :: Python :: 3
@@ -12,8 +11,8 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyright>=1.1.393
 Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: boto3<=1.35.99,>=1.20.1
 Dynamic: home-page
-Dynamic: maintainer
 
 # rclone-api
 

rclone_api-1.0.90.dist-info/RECORD

@@ -0,0 +1,35 @@
+rclone_api/__init__.py,sha256=m7hQkaIsn9grAlziu9nvDlOIQY6VflYFwC1c_g0qCJo,833
+rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
+rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
+rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
+rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
+rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
+rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
+rclone_api/dir.py,sha256=i4h7LX5hB_WmVixxDRWL_l1nifvscrdWct_8Wx7wHZc,3540
+rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,1882
+rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
+rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
+rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
+rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
+rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
+rclone_api/rclone.py,sha256=W4CIhDEPJqY5X9-HXKp-DTVLhU5H8YUOjGYAHDVy-yU,41207
+rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
+rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
+rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
+rclone_api/types.py,sha256=NC3e78aXCx-sEQ-FqEaC9KzaJDdJhJrKa4Nwum_-Db0,563
+rclone_api/util.py,sha256=4bJF1CC8J2NW6UbJU3oSxmTJgN04uTovABZJn7c3-zk,5025
+rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
+rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
+rclone_api/cmd/copy_large_s3.py,sha256=FPU0S1Y9pApVLmWcdMT_3llywjEtOtwobXTnMXE-FhY,2690
+rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
+rclone_api/s3/api.py,sha256=Ju9A6uNHON2by2jJPtH-awe-3d588QLh5E-vjdV4kOY,2729
+rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
+rclone_api/s3/chunk_uploader.py,sha256=Wcm4h_A6ORaJvT9P9qSeknbJYch22RKZ4FGo1iePlAw,17385
+rclone_api/s3/create.py,sha256=X4mgjekgAoyHNeXnznHNS63DFViAhI0p7-fXDxml0y4,2701
+rclone_api/s3/types.py,sha256=BKvNkAFjLJK7Zd9xQya8gdjqJPLqdCPZSXD8GeFyP1A,1346
+rclone_api-1.0.90.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.0.90.dist-info/METADATA,sha256=7Y-NCkQi9yBFGqd90T-j9oA2rvA0rEu_-znW5LW_dZY,4479
+rclone_api-1.0.90.dist-info/WHEEL,sha256=rF4EZyR2XVS6irmOHQIJx2SUqXLZKRMUrjsg8UwN-XQ,109
+rclone_api-1.0.90.dist-info/entry_points.txt,sha256=6eNqTRXKhVf8CpWNjXiOa_0Du9tHiW_HD2iQSXRsUg8,132
+rclone_api-1.0.90.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.0.90.dist-info/RECORD,,

{rclone_api-1.0.88.dist-info → rclone_api-1.0.90.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

{rclone_api-1.0.88.dist-info → rclone_api-1.0.90.dist-info}/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+rclone-api-copylarge-s3 = rclone_api.cmd.copy_large_s3:main
 rclone-api-listfiles = rclone_api.cmd.list_files:main

rclone_api-1.0.88.dist-info/RECORD

@@ -1,29 +0,0 @@
-rclone_api/__init__.py,sha256=UH1aIcovOSoiWwubbgBs_fvX3YcMhHnLTYJzLnCPKVs,722
-rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
-rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
-rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
-rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
-rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
-rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
-rclone_api/dir.py,sha256=i4h7LX5hB_WmVixxDRWL_l1nifvscrdWct_8Wx7wHZc,3540
-rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,1882
-rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
-rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
-rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
-rclone_api/group_files.py,sha256=O8Awod4_ILGB4RfatBmiG4vXy9eO0rBhbEaZPr6X_sY,7969
-rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
-rclone_api/rclone.py,sha256=lfdOwUiFyXkiLNISkfb4_FIWwlmTo5Vgqfy9sS9by0s,34059
-rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
-rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
-rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
-rclone_api/types.py,sha256=9-qY0Z1NFwiIrvV3e4Ty-dVfed1I_482jPohJ-2hs-o,567
-rclone_api/util.py,sha256=XMrA2m_di4h8JTM-qyx2iyrFZn-l-or_SJOa5tEsDsI,4200
-rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
-rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
-rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
-rclone_api-1.0.88.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.0.88.dist-info/METADATA,sha256=1kuxxPVsUsTpNgzSy6clP88Cmt7s8Uxi8yEUn45uu24,4489
-rclone_api-1.0.88.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
-rclone_api-1.0.88.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
-rclone_api-1.0.88.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.0.88.dist-info/RECORD,,