rclone-api 1.5.8__py3-none-any.whl

rclone_api/remote.py

@@ -0,0 +1,21 @@
+from typing import Any
+
+
+class Remote:
+    """Remote (root) directory."""
+
+    def __init__(self, name: str, rclone: Any) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        if ":" in name:
+            raise ValueError("Remote name cannot contain ':'")
+
+        assert isinstance(rclone, RcloneImpl)
+        self.name = name
+        self.rclone: RcloneImpl = rclone
+
+    def __str__(self) -> str:
+        return f"{self.name}:"
+
+    def __repr__(self) -> str:
+        return f"Remote({self.name!r})"

rclone_api/rpath.py

@@ -0,0 +1,102 @@
+import json
+from datetime import datetime
+from typing import Any
+
+from rclone_api.remote import Remote
+
+
+class RPath:
+    """Remote file dataclass."""
+
+    def __init__(
+        self,
+        remote: Remote,
+        path: str,
+        name: str,
+        size: int,
+        mime_type: str,
+        mod_time: str,
+        is_dir: bool,
+    ) -> None:
+        from rclone_api.rclone_impl import RcloneImpl
+
+        if path.endswith("/"):
+            path = path[:-1]
+        self.remote = remote
+        self.path = path
+        self.name = name
+        self.size = size
+        self.mime_type = mime_type
+        self.mod_time = mod_time
+        self.is_dir = is_dir
+        self.rclone: RcloneImpl | None = None
+
+    def mod_time_dt(self) -> datetime:
+        """Return the modification time as a datetime object."""
+        return datetime.fromisoformat(self.mod_time)
+
+    def set_rclone(self, rclone: Any) -> None:
+        """Set the rclone object."""
+        from rclone_api.rclone_impl import RcloneImpl
+
+        assert isinstance(rclone, RcloneImpl)
+        self.rclone = rclone
+
+    @staticmethod
+    def from_dict(
+        data: dict, remote: Remote, parent_path: str | None = None
+    ) -> "RPath":
+        """Create a File from a dictionary."""
+        path = data["Path"]
+        if parent_path is not None:
+            path = f"{parent_path}/{path}"
+        return RPath(
+            remote,
+            path,
+            data["Name"],
+            data["Size"],
+            data["MimeType"],
+            data["ModTime"],
+            data["IsDir"],
+            # data["IsBucket"],
+        )
+
+    @staticmethod
+    def from_array(
+        data: list[dict], remote: Remote, parent_path: str | None = None
+    ) -> list["RPath"]:
+        """Create a File from a dictionary."""
+        out: list[RPath] = []
+        for d in data:
+            file: RPath = RPath.from_dict(d, remote, parent_path)
+            out.append(file)
+        return out
+
+    @staticmethod
+    def from_json_str(
+        json_str: str, remote: Remote, parent_path: str | None = None
+    ) -> list["RPath"]:
+        """Create a File from a JSON string."""
+        json_obj = json.loads(json_str)
+        if isinstance(json_obj, dict):
+            return [RPath.from_dict(json_obj, remote, parent_path)]
+        return RPath.from_array(json_obj, remote, parent_path)
+
+    def to_json(self) -> dict:
+        return {
+            "Path": self.path,
+            "Name": self.name,
+            "Size": self.size,
+            "MimeType": self.mime_type,
+            "ModTime": self.mod_time,
+            "IsDir": self.is_dir,
+            # "IsBucket": self.is_bucket,
+        }
+
+    def __str__(self) -> str:
+        return f"{self.remote.name}:{self.path}"
+
+    def __repr__(self):
+        data = self.to_json()
+        data["Remote"] = self.remote.name
+        return json.dumps(data)

rclone_api/s3/api.py

@@ -0,0 +1,109 @@
+import json
+import warnings
+
+from botocore.client import BaseClient
+
+from rclone_api.s3.basic_ops import (
+    download_file,
+    head,
+    list_bucket_contents,
+    upload_file,
+)
+from rclone_api.s3.create import S3Config, create_s3_client
+from rclone_api.s3.multipart.upload_parts_inline import (
+    MultiUploadResult,
+    upload_file_multipart,
+)
+from rclone_api.s3.types import S3Credentials, S3MutliPartUploadConfig, S3UploadTarget
+
+_MIN_THRESHOLD_FOR_CHUNKING = 5 * 1024 * 1024
+
+
+class S3Client:
+    def __init__(self, s3_creds: S3Credentials, verbose: bool = False) -> None:
+        self.verbose = verbose
+        self.credentials: S3Credentials = s3_creds
+        self.client: BaseClient = create_s3_client(
+            s3_creds=s3_creds, s3_config=S3Config(verbose=verbose)
+        )
+
+    def list_bucket_contents(self, bucket_name: str) -> None:
+        list_bucket_contents(self.client, bucket_name)
+
+    def upload_file(self, target: S3UploadTarget) -> Exception | None:
+        bucket_name = target.bucket_name
+        file_path = target.src_file
+        object_name = target.s3_key
+        return upload_file(
+            s3_client=self.client,
+            bucket_name=bucket_name,
+            file_path=file_path,
+            object_name=object_name,
+        )
+
+    def download_file(self, bucket_name: str, object_name: str, file_path: str) -> None:
+        download_file(self.client, bucket_name, object_name, file_path)
+
+    def head(self, bucket_name: str, object_name: str) -> dict | None:
+        return head(self.client, bucket_name, object_name)
+
+    def upload_file_multipart(
+        self,
+        upload_target: S3UploadTarget,
+        upload_config: S3MutliPartUploadConfig,
+    ) -> MultiUploadResult:
+
+        chunk_size = upload_config.chunk_size
+        retries = upload_config.retries
+        resume_path_json = upload_config.resume_path_json
+        max_chunks_before_suspension = upload_config.max_chunks_before_suspension
+        bucket_name = upload_target.bucket_name
+
+        try:
+
+            if upload_target.src_file_size is None:
+                filesize = upload_target.src_file.stat().st_size
+            else:
+                filesize = upload_target.src_file_size
+
+            if filesize < _MIN_THRESHOLD_FOR_CHUNKING:
+                warnings.warn(
+                    f"File size {filesize} is less than the minimum threshold for chunking ({_MIN_THRESHOLD_FOR_CHUNKING}), switching to single threaded upload."
+                )
+                err = self.upload_file(upload_target)
+                if err:
+                    raise err
+                return MultiUploadResult.UPLOADED_FRESH
+
+            out = upload_file_multipart(
+                s3_client=self.client,
+                chunk_fetcher=upload_config.chunk_fetcher,
+                bucket_name=bucket_name,
+                file_path=upload_target.src_file,
+                file_size=filesize,
+                object_name=upload_target.s3_key,
+                resumable_info_path=resume_path_json,
+                chunk_size=chunk_size,
+                retries=retries,
+                max_chunks_before_suspension=max_chunks_before_suspension,
+            )
+            return out
+        except Exception as e:
+            key = upload_target.s3_key
+            access_key_id = self.credentials.access_key_id[:4] + "..."
+            secret = self.credentials.secret_access_key[:4] + "..."
+            endpoint_url = self.credentials.endpoint_url
+            provider = self.credentials.provider.value
+            region_name = self.credentials.region_name
+            info_json = {
+                "bucket": bucket_name,
+                "key": key,
+                "access_key_id": access_key_id[:4] + "...",
+                "secret": secret[:4] + "...",
+                "endpoint_url": endpoint_url,
+                "provider": provider,
+                "region": region_name,
+            }
+            info_json_str = json.dumps(info_json, indent=2)
+            warnings.warn(f"Error uploading file: {e}\nInfo:\n\n{info_json_str}")
+            raise

rclone_api/s3/basic_ops.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+
+from botocore.client import BaseClient
+
+
+def list_bucket_contents(s3_client: BaseClient, bucket_name: str) -> None:
+    """List contents of the specified bucket."""
+    try:
+        response = s3_client.list_objects_v2(Bucket=bucket_name)
+        if "Contents" in response:
+            for obj in response["Contents"]:
+                print(f"File: {obj['Key']} | Size: {obj['Size']} bytes")
+        else:
+            print(f"The bucket '{bucket_name}' is empty.")
+    except Exception as e:
+        print(f"Error listing bucket contents: {e}")
+
+
+def upload_file(
+    s3_client: BaseClient,
+    bucket_name: str,
+    file_path: Path,
+    object_name: str,
+) -> Exception | None:
+    """Upload a file to the bucket."""
+    try:
+        s3_client.upload_file(str(file_path), bucket_name, object_name)
+        print(f"Uploaded {file_path} to {bucket_name}/{object_name}")
+    except Exception as e:
+        print(f"Error uploading file: {e}")
+        return e
+    return None
+
+
+def download_file(
+    s3_client: BaseClient, bucket_name: str, object_name: str, file_path: str
+) -> None:
+    """Download a file from the bucket."""
+    try:
+        s3_client.download_file(bucket_name, object_name, file_path)
+        print(f"Downloaded {object_name} from {bucket_name} to {file_path}")
+    except Exception as e:
+        print(f"Error downloading file: {e}")
+
+
+def head(s3_client: BaseClient, bucket_name: str, object_name: str) -> dict | None:
+    """
+    Retrieve metadata for the specified object using a HEAD operation.
+
+    :param s3_client: The S3 client to use.
+    :param bucket_name: The name of the bucket containing the object.
+    :param object_name: The key of the object.
+    :return: A dictionary containing the object's metadata if successful, otherwise None.
+    """
+    try:
+        response = s3_client.head_object(Bucket=bucket_name, Key=object_name)
+        print(f"Metadata for {object_name} in {bucket_name}: {response}")
+        return response
+    except Exception as e:
+        print(f"Error retrieving metadata for {object_name}: {e}")
+        return None

rclone_api/s3/chunk_task.py

@@ -0,0 +1,187 @@
+import logging
+import time
+from concurrent.futures import Future
+from pathlib import Path
+from queue import Queue
+from threading import Event, Lock
+from typing import Any, Callable
+
+from rclone_api.file_part import FilePart
+from rclone_api.s3.multipart.file_info import S3FileInfo
+from rclone_api.s3.multipart.upload_state import UploadState
+from rclone_api.types import EndOfStream
+
+logger = logging.getLogger(__name__)  # noqa
+
+
+class _ShouldStopChecker:
+    def __init__(self, max_chunks: int | None) -> None:
+        self.count = 0
+        self.max_chunks = max_chunks
+
+    def should_stop(self) -> bool:
+        if self.max_chunks is None:
+            return False
+        if self.count >= self.max_chunks:
+            logger.info(
+                f"Stopping file chunker after {self.count} chunks because it exceeded max_chunks {self.max_chunks}"
+            )
+            return True
+        # self.count += 1
+        return False
+
+    def increment(self):
+        self.count += 1
+
+
+class _PartNumberTracker:
+    def __init__(
+        self, start_part_value: int, last_part_value: int, done_parts: set[int]
+    ) -> None:
+        # self._num_parts = (last_part_value - start_part_value) + 1
+        self._start_part_value = start_part_value
+        self._last_part_value = last_part_value
+        self._done_part_numbers: set[int] = done_parts
+        self._curr_part_number = start_part_value
+        self._finished = False
+        self._lock = Lock()
+
+    def next_part_number(self) -> int | None:
+        with self._lock:
+            while self._curr_part_number in self._done_part_numbers:
+                self._curr_part_number += 1
+            if self._curr_part_number > self._last_part_value:
+                self._finished = True
+                return None
+            curr_part_number = self._curr_part_number
+            self._curr_part_number += (
+                1  # prevent a second thread from getting the same part number
+            )
+            return curr_part_number
+
+    def is_finished(self) -> bool:
+        with self._lock:
+            return self._finished
+
+    def add_finished_part_number(self, part_number: int) -> None:
+        with self._lock:
+            self._done_part_numbers.add(part_number)
+
+
+class _OnCompleteHandler:
+    def __init__(
+        self,
+        part_number_tracker: _PartNumberTracker,
+        file_path: Path,
+        queue_upload: Queue[FilePart | EndOfStream],
+    ) -> None:
+        self.part_number_tracker = part_number_tracker
+        self.file_path = file_path
+        self.queue_upload = queue_upload
+
+    def on_complete(self, fut: Future[FilePart]) -> None:
+        logger.debug("Chunk read complete")
+        fp: FilePart = fut.result()
+        extra: S3FileInfo = fp.extra
+        assert isinstance(extra, S3FileInfo)
+        part_number = extra.part_number
+        if fp.is_error():
+            logger.warning(f"Error reading file: {fp}, skipping part {part_number}")
+            return
+
+        if fp.n_bytes() == 0:
+            logger.warning(f"Empty data for part {part_number} of {self.file_path}")
+            raise ValueError(f"Empty data for part {part_number} of {self.file_path}")
+
+        if isinstance(fp.payload, Exception):
+            logger.warning(f"Error reading file because of error: {fp.payload}")
+            return
+
+        # done_part_numbers.add(part_number)
+        # queue_upload.put(fp)
+        self.part_number_tracker.add_finished_part_number(
+            part_number
+        )  # in memory database, not persistant to resume.json
+        self.queue_upload.put(fp)
+
+
+def file_chunker(
+    upload_state: UploadState,
+    fetcher: Callable[[int, int, Any], Future[FilePart]],
+    max_chunks: int | None,
+    cancel_signal: Event,
+    queue_upload: Queue[FilePart | EndOfStream],
+) -> None:
+    final_part_number = upload_state.upload_info.total_chunks() + 1
+    should_stop_checker = _ShouldStopChecker(max_chunks)
+
+    upload_info = upload_state.upload_info
+    file_path = upload_info.src_file_path
+    chunk_size = upload_info.chunk_size
+
+    done_part_numbers: set[int] = {
+        p.part_number for p in upload_state.parts if not isinstance(p, EndOfStream)
+    }
+
+    part_tracker = _PartNumberTracker(
+        start_part_value=1,
+        last_part_value=final_part_number,
+        done_parts=done_part_numbers,
+    )
+
+    callback = _OnCompleteHandler(part_tracker, file_path, queue_upload)
+
+    try:
+        num_parts = upload_info.total_chunks()
+
+        if cancel_signal.is_set():
+            logger.info(
+                f"Cancel signal is set for file chunker while processing {file_path}, returning"
+            )
+            return
+
+        while not should_stop_checker.should_stop():
+            should_stop_checker.increment()
+            logger.debug("Processing next chunk")
+            curr_part_number = part_tracker.next_part_number()
+            if curr_part_number is None:
+                logger.info(f"File {file_path} has completed chunking all parts")
+                break
+
+            assert curr_part_number is not None
+            offset = (curr_part_number - 1) * chunk_size
+            file_size = upload_info.file_size
+
+            assert offset < file_size, f"Offset {offset} is greater than file size"
+            fetch_size = max(0, min(chunk_size, file_size - offset))
+            if fetch_size == 0:
+                logger.error(
+                    f"Empty data for part {curr_part_number} of {file_path}, is this the last chunk?"
+                )
+                # assert final_part_number == curr_part_number, f"Final part number is {final_part_number} but current part number is {curr_part_number}"
+                if final_part_number != curr_part_number:
+                    raise ValueError(
+                        f"This should have been the last part, but it is not: {final_part_number} != {curr_part_number}"
+                    )
+
+            assert curr_part_number is not None
+            logger.info(
+                f"Reading chunk {curr_part_number} of {num_parts} for {file_path}"
+            )
+            logger.debug(
+                f"Fetching part {curr_part_number} with offset {offset} and size {fetch_size}"
+            )
+            fut = fetcher(
+                offset, fetch_size, S3FileInfo(upload_info.upload_id, curr_part_number)
+            )
+            fut.add_done_callback(callback.on_complete)
+            # wait until the queue_upload queue can accept the next chunk
+            qsize = queue_upload.qsize()
+            print(f"queue_upload_size: {qsize}")
+            while queue_upload.full():
+                time.sleep(0.1)
+    except Exception as e:
+        logger.error(f"Error reading file: {e}", exc_info=True)
+    finally:
+        logger.info(f"Finishing FILE CHUNKER for {file_path} and adding EndOfStream")
+        queue_upload.put(EndOfStream())

rclone_api/s3/create.py

@@ -0,0 +1,107 @@
+import warnings
+from dataclasses import dataclass
+
+import boto3
+from botocore.client import BaseClient
+from botocore.config import Config
+
+from rclone_api.s3.types import S3Credentials, S3Provider
+
+_DEFAULT_BACKBLAZE_ENDPOINT = "https://s3.us-west-002.backblazeb2.com"
+_MAX_CONNECTIONS = 10
+_TIMEOUT_READ = 120
+_TIMEOUT_CONNECT = 60
+
+
+@dataclass
+class S3Config:
+    max_pool_connections: int | None = None
+    timeout_connection: int | None = None
+    timeout_read: int | None = None
+    verbose: bool | None = None
+
+    def resolve_defaults(self) -> None:
+        self.max_pool_connections = self.max_pool_connections or _MAX_CONNECTIONS
+        self.timeout_connection = self.timeout_connection or _TIMEOUT_CONNECT
+        self.timeout_read = self.timeout_read or _TIMEOUT_READ
+        self.verbose = self.verbose or False
+
+
+# Create a Boto3 session and S3 client, this is back blaze specific.
+# Add a function if you want to use a different S3 provider.
+# If AWS support is added in a fork then please merge it back here.
+def _create_backblaze_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
+    """Create and return an S3 client."""
+    region_name = s3_creds.region_name
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
+    endpoint_url = endpoint_url or _DEFAULT_BACKBLAZE_ENDPOINT
+    s3_config.resolve_defaults()
+    session = boto3.session.Session()  # type: ignore
+    return session.client(
+        service_name="s3",
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        endpoint_url=endpoint_url,
+        # verify=False,  # Disables SSL certificate verification
+        config=Config(
+            signature_version="s3v4",
+            region_name=region_name,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
+            # Note that BackBlase has a boko3 bug where it doesn't support the new
+            # checksum header, the following line was an attempt of fix it on the newest
+            # version of boto3, but it didn't work.
+            s3={"payload_signing_enabled": False},  # Disable checksum header
+        ),
+    )
+
+
+def _create_unknown_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config
+) -> BaseClient:
+    """Create and return an S3 client."""
+    access_key = s3_creds.access_key_id
+    secret_key = s3_creds.secret_access_key
+    endpoint_url = s3_creds.endpoint_url
+    if (endpoint_url is not None) and not (endpoint_url.startswith("http")):
+        if s3_config.verbose:
+            warnings.warn(
+                f"Endpoint URL is schema naive: {endpoint_url}, assuming HTTPS"
+            )
+        endpoint_url = f"https://{endpoint_url}"
+    s3_config.resolve_defaults()
+    session = boto3.session.Session()  # type: ignore
+    return session.client(
+        service_name="s3",
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        endpoint_url=endpoint_url,
+        config=Config(
+            signature_version="s3v4",
+            region_name=s3_creds.region_name,
+            max_pool_connections=s3_config.max_pool_connections,
+            read_timeout=s3_config.timeout_read,
+            connect_timeout=s3_config.timeout_connection,
+        ),
+    )
+
+
+def create_s3_client(
+    s3_creds: S3Credentials, s3_config: S3Config | None = None
+) -> BaseClient:
+    """Create and return an S3 client."""
+    s3_config = s3_config or S3Config()
+    provider = s3_creds.provider
+    if provider == S3Provider.BACKBLAZE:
+        if s3_config.verbose:
+            print("Creating BackBlaze S3 client")
+        return _create_backblaze_s3_client(s3_creds=s3_creds, s3_config=s3_config)
+    else:
+        if s3_config.verbose:
+            print("Creating generic/unknown S3 client")
+        return _create_unknown_s3_client(s3_creds=s3_creds, s3_config=s3_config)

rclone_api/s3/multipart/file_info.py

@@ -0,0 +1,7 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class S3FileInfo:
+    upload_id: str
+    part_number: int

rclone_api/s3/multipart/finished_piece.py

@@ -0,0 +1,69 @@
+import warnings
+from dataclasses import dataclass
+
+from rclone_api.types import EndOfStream
+
+
+@dataclass
+class FinishedPiece:
+    part_number: int
+    etag: str
+
+    def to_json(self) -> dict:
+        # return {"part_number": self.part_number, "etag": self.etag}
+        # amazon s3 style dict
+        tag = self.etag
+        if not tag.startswith('"'):
+            tag = f'"{tag}"'
+        out = {"PartNumber": self.part_number, "ETag": self.etag}
+        return out
+
+    def __post_init__(self):
+        assert isinstance(self.part_number, int)
+        assert isinstance(self.etag, str)
+
+    @staticmethod
+    def to_json_array(
+        parts: list["FinishedPiece | EndOfStream"] | list["FinishedPiece"],
+    ) -> list[dict]:
+        non_none: list[FinishedPiece] = []
+        for p in parts:
+            if not isinstance(p, EndOfStream):
+                non_none.append(p)
+        non_none.sort(key=lambda x: x.part_number)
+        # all_nones: list[None] = [None for p in parts if p is None]
+        # assert len(all_nones) <= 1, "Only one None should be present"
+        count_eos = 0
+        for p in parts:
+            if p is EndOfStream:
+                count_eos += 1
+        # assert count_eos <= 1, "Only one EndOfStream should be present"
+        if count_eos > 1:
+            warnings.warn(f"Only one EndOfStream should be present, found {count_eos}")
+        out = [p.to_json() for p in non_none]
+        return out
+
+    @staticmethod
+    def from_json(json: dict | None) -> "FinishedPiece | EndOfStream":
+        if json is None:
+            return EndOfStream()
+        part_number = json.get("PartNumber") or json.get("part_number")
+        etag = json.get("ETag") or json.get("etag")
+        assert isinstance(etag, str)
+        # handle the double quotes around the etag
+        etag = etag.replace('"', "")
+        assert isinstance(part_number, int)
+        assert isinstance(etag, str)
+        return FinishedPiece(part_number=part_number, etag=etag)
+
+    @staticmethod
+    def from_json_array(json: dict) -> list["FinishedPiece"]:
+        tmp = [FinishedPiece.from_json(j) for j in json]
+        out: list[FinishedPiece] = []
+        for t in tmp:
+            if isinstance(t, FinishedPiece):
+                out.append(t)
+        return out
+
+    def __hash__(self) -> int:
+        return hash(self.part_number)