rclone-api 1.3.19__tar.gz → 1.3.20__tar.gz

{rclone_api-1.3.19 → rclone_api-1.3.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.19
+Version: 1.3.20
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.19 → rclone_api-1.3.20}/pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
 ]
 
 # Change this with the version number bump.
-version = "1.3.19"
+version = "1.3.20"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{rclone_api-1.3.19 → rclone_api-1.3.20}/src/rclone_api/__init__.py

@@ -9,6 +9,7 @@ from .dir import Dir
 from .dir_listing import DirListing
 from .file import File, FileItem
 from .filelist import FileList
+from .http_server import HttpFetcher, HttpServer, Range
 
 # Import the configure_logging function to make it available at package level
 from .log import configure_logging, setup_default_logging
@@ -45,6 +46,9 @@ __all__ = [
     "SizeSuffix",
     "configure_logging",
     "log",
+    "HttpServer",
+    "Range",
+    "HttpFetcher",
 ]
 
 setup_default_logging()

rclone_api-1.3.20/src/rclone_api/http_server.py

@@ -0,0 +1,233 @@
+"""
+Unit test file for testing rclone mount functionality.
+"""
+
+import tempfile
+import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from rclone_api.process import Process
+from rclone_api.types import FilePart, SizeSuffix, get_chunk_tmpdir
+
+_TIMEOUT = 10 * 60  # 10 minutes
+
+
+@dataclass
+class Range:
+    start: int
+    end: int
+
+    def to_header(self) -> dict[str, str]:
+        val = f"bytes={self.start}-{self.end-1}"
+        return {
+            "Range": val,
+        }
+
+
+_range = range
+
+
+class HttpServer:
+    """HTTP server configuration."""
+
+    def __init__(self, url: str, subpath: str, process: Process) -> None:
+        self.url = url
+        self.subpath = subpath
+        self.process: Process | None = process
+
+    def _get_file_url(self, path: str | Path) -> str:
+        # if self.subpath == "":
+        path = Path(path).as_posix()
+        return f"{self.url}/{path}"
+        # return f"{self.url}/{self.subpath}/{path}"
+
+    def get_fetcher(self, path: str, n_threads: int = 16) -> "HttpFetcher":
+        return HttpFetcher(self, path, n_threads=n_threads)
+
+    def get(self, path: str) -> bytes | Exception:
+        """Get bytes from the server."""
+        with tempfile.TemporaryFile() as file:
+            self.download(path, Path(file.name), None)
+            file.seek(0)
+            return file.read()
+
+    def size(self, path: str) -> int | Exception:
+        """Get size of the file from the server."""
+        try:
+            assert self.process is not None
+            # response = httpx.head(f"{self.url}/{path}")
+            url = self._get_file_url(path)
+            response = httpx.head(url)
+            response.raise_for_status()
+            size = int(response.headers["Content-Length"])
+            return size
+        except Exception as e:
+            warnings.warn(f"Failed to get size of {self.url}/{path}: {e}")
+            return e
+
+    def download(
+        self, path: str, dst: Path, range: Range | None = None
+    ) -> Path | Exception:
+        """Get bytes from the server."""
+        if not dst.parent.exists():
+            dst.parent.mkdir(parents=True, exist_ok=True)
+        headers: dict[str, str] = {}
+        if range:
+            headers.update(range.to_header())
+        url = self._get_file_url(path)
+        try:
+            with httpx.stream(
+                "GET", url, headers=headers, timeout=_TIMEOUT
+            ) as response:
+                response.raise_for_status()
+                with open(dst, "wb") as file:
+                    for chunk in response.iter_bytes(chunk_size=8192):
+                        if chunk:
+                            file.write(chunk)
+                        else:
+                            assert response.is_closed
+                # print(f"Downloaded bytes {start}-{end} to {dst}")
+                if range:
+                    print(f"Downloaded bytes {range.start}-{range.end} to {dst}")
+                else:
+                    size = dst.stat().st_size
+                    print(f"Downloaded {size} bytes to {dst}")
+                return dst
+        except Exception as e:
+            warnings.warn(f"Failed to download {url} to {dst}: {e}")
+            return e
+
+    def download_multi_threaded(
+        self,
+        src_path: str,
+        dst_path: Path,
+        chunk_size: int = 32 * 1024 * 1024,
+        n_threads: int = 16,
+        range: Range | None = None,
+    ) -> Path | Exception:
+        """Copy file from src to dst."""
+
+        finished: list[Path] = []
+        errors: list[Exception] = []
+
+        if range is None:
+            sz = self.size(src_path)
+            if isinstance(sz, Exception):
+                return sz
+            range = Range(0, sz)
+
+        with ThreadPoolExecutor(max_workers=n_threads) as executor:
+            try:
+                futures: list[Future[Path | Exception]] = []
+                for start in _range(range.start, range.end, chunk_size):
+                    end = min(start + chunk_size, range.end)
+                    r = Range(start=start, end=end)
+
+                    def task(r: Range = r) -> Path | Exception:
+                        dst = dst_path.with_suffix(f".{r.start}")
+                        out = self.download(src_path, dst, r)
+                        if isinstance(out, Exception):
+                            warnings.warn(f"Failed to download chunked: {out}")
+                        return out
+
+                    fut = executor.submit(task, r)
+                    futures.append(fut)
+                for fut in futures:
+                    result = fut.result()
+                    if isinstance(result, Exception):
+                        errors.append(result)
+                    else:
+                        finished.append(result)
+                if errors:
+                    for finished_file in finished:
+                        try:
+                            finished_file.unlink()
+                        except Exception as e:
+                            warnings.warn(f"Failed to delete file {finished_file}: {e}")
+                    return Exception(f"Failed to download chunked: {errors}")
+
+                if not dst_path.parent.exists():
+                    dst_path.parent.mkdir(parents=True, exist_ok=True)
+
+                count = 0
+                with open(dst_path, "wb") as file:
+                    for f in finished:
+                        print(f"Appending {f} to {dst_path}")
+                        with open(f, "rb") as part:
+                            # chunk = part.read(8192 * 4)
+                            while chunk := part.read(8192 * 4):
+                                if not chunk:
+                                    break
+                                count += len(chunk)
+                                file.write(chunk)
+                        print(f"Removing {f}")
+                        f.unlink()
+                # print(f"Downloaded {count} bytes to {dst_path}")
+                return dst_path
+            except Exception as e:
+                warnings.warn(f"Failed to copy chunked: {e}")
+                for f in finished:
+                    try:
+                        if f.exists():
+                            f.unlink()
+                    except Exception as ee:
+                        warnings.warn(f"Failed to delete file {f}: {ee}")
+                return e
+
+    def __enter__(self) -> "HttpServer":
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.shutdown()
+
+    def shutdown(self) -> None:
+        """Shutdown the server."""
+        if self.process:
+            self.process.terminate()
+            if self.process.stdout:
+                self.process.stdout.close()
+            if self.process.stderr:
+                self.process.stderr.close()
+
+
+class HttpFetcher:
+    def __init__(self, server: "HttpServer", path: str, n_threads: int) -> None:
+        self.server = server
+        self.path = path
+        self.executor = ThreadPoolExecutor(max_workers=n_threads)
+        from threading import Semaphore
+
+        self.semaphore = Semaphore(n_threads)
+
+    def fetch(
+        self, offset: int | SizeSuffix, size: int | SizeSuffix, extra: Any
+    ) -> Future[FilePart]:
+        if isinstance(offset, SizeSuffix):
+            offset = offset.as_int()
+        if isinstance(size, SizeSuffix):
+            size = size.as_int()
+
+        def task() -> FilePart:
+            from rclone_api.util import random_str
+
+            try:
+                range = Range(offset, offset + size)
+                dst = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
+                out = self.server.download(self.path, dst, range)
+                if isinstance(out, Exception):
+                    raise out
+                return FilePart(payload=dst, extra=extra)
+            finally:
+                self.semaphore.release()
+
+        self.semaphore.acquire()
+        fut = self.executor.submit(task)
+        return fut
+
+    def shutdown(self) -> None:
+        self.executor.shutdown(wait=True)

{rclone_api-1.3.19 → rclone_api-1.3.20}/src/rclone_api/profile/mount_copy_bytes.py

@@ -142,7 +142,7 @@ def _run_profile(
         filepart_or_err = future.result()
         if isinstance(filepart_or_err, Exception):
             assert False, f"Error: {filepart_or_err}"
-        filepart_or_err.close()
+        filepart_or_err.dispose()
     futures.clear()
 
     start = time.time()

{rclone_api-1.3.19 → rclone_api-1.3.20}/src/rclone_api/rclone.py

@@ -11,10 +11,11 @@ import traceback
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
 from contextlib import contextmanager
+from dataclasses import dataclass
 from fnmatch import fnmatch
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Generator
+from typing import Any, Callable, Generator
 
 from rclone_api import Dir
 from rclone_api.completed_process import CompletedProcess
@@ -26,6 +27,7 @@ from rclone_api.dir_listing import DirListing
 from rclone_api.exec import RcloneExec
 from rclone_api.file import File, FileItem
 from rclone_api.group_files import group_files
+from rclone_api.http_server import HttpServer
 from rclone_api.mount import Mount, clean_mount, prepare_mount
 from rclone_api.mount_read_chunker import MultiMountFileChunker
 from rclone_api.process import Process
@@ -38,6 +40,7 @@ from rclone_api.s3.types import (
     S3UploadTarget,
 )
 from rclone_api.types import (
+    FilePart,
     ListingOption,
     ModTimeStrategy,
     Order,
@@ -242,7 +245,7 @@ class Rclone:
 
     def ls(
         self,
-        path: Dir | Remote | str,
+        path: Dir | Remote | str | None = None,
         max_depth: int | None = None,
         glob: str | None = None,
         order: Order = Order.NORMAL,
@@ -258,6 +261,15 @@ class Rclone:
             List of File objects found at the path
         """
 
+        if path is None:
+            # list remotes instead
+            list_remotes: list[Remote] = self.listremotes()
+            dirs: list[Dir] = [Dir(remote) for remote in list_remotes]
+            for d in dirs:
+                d.path.path = ""
+            rpaths = [d.path for d in dirs]
+            return DirListing(rpaths)
+
         if isinstance(path, str):
             path = Dir(
                 to_path(path, self)
@@ -786,14 +798,33 @@ class Rclone:
         verbose: bool | None = None,
         max_chunks_before_suspension: int | None = None,
         mount_log: Path | None = None,
+        use_http_fetcher: bool = True,  # else use mount fetcher
     ) -> MultiUploadResult:
         """For massive files that rclone can't handle in one go, this function will copy the file in chunks to an S3 store"""
+        from rclone_api.http_server import HttpFetcher, HttpServer
         from rclone_api.s3.api import S3Client
         from rclone_api.s3.create import S3Credentials
         from rclone_api.util import S3PathInfo, split_s3_path
 
-        other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
+        src_path = Path(src)
+        name = src_path.name
+        src_parent_path = Path(src).parent.as_posix()
+
+        size_result: SizeResult = self.size_files(src_parent_path, [name])
+        target_size = SizeSuffix(size_result.total_size)
+
         chunk_size = chunk_size or SizeSuffix("64M")
+        MAX_CHUNKS = 10000
+        min_chunk_size = size_result.total_size // (MAX_CHUNKS - 1)
+        if min_chunk_size > chunk_size:
+            warnings.warn(
+                f"Chunk size {chunk_size} is too small for file size {size_result.total_size}, setting to {min_chunk_size}"
+            )
+            chunk_size = SizeSuffix(min_chunk_size)
+
+        other_args: list[str] = ["--no-modtime", "--vfs-read-wait", "1s"]
+
+        # BEGIN MOUNT SPECIFIC CONFIG
         unit_chunk_size = chunk_size / read_threads
         tmp_mount_dir = self._get_tmp_mount_dir()
         vfs_read_chunk_size = unit_chunk_size
@@ -818,18 +849,16 @@ class Rclone:
         # --vfs-cache-max-size
         other_args += ["--vfs-cache-max-size", vfs_disk_space_total_size.as_str()]
         mount_path = tmp_mount_dir / "RCLONE_API_DYNAMIC_MOUNT"
-        src_path = Path(src)
-        name = src_path.name
+        ## END MOUNT SPECIFIC CONFIG
 
-        src_parent_path = Path(src).parent.as_posix()
-        size_result: SizeResult = self.size_files(src_parent_path, [name])
+        # size_result: SizeResult = self.size_files(os.path.dirname(src), [name])
 
-        target_size = SizeSuffix(size_result.total_size)
         if target_size < SizeSuffix("5M"):
             # fallback to normal copy
             completed_proc = self.copy_to(src, dst, check=True)
             if completed_proc.ok:
                 return MultiUploadResult.UPLOADED_FRESH
+
         if size_result.total_size <= 0:
             raise ValueError(
                 f"File {src} has size {size_result.total_size}, is this a directory?"
@@ -882,18 +911,46 @@ class Rclone:
             endpoint_url=section.endpoint(),
         )
 
-        chunk_fetcher: MultiMountFileChunker = self.get_multi_mount_file_chunker(
-            src=src_path.as_posix(),
-            chunk_size=chunk_size,
-            threads=read_threads,
-            mount_log=mount_log,
-            direct_io=True,
-        )
+        @dataclass
+        class Fetcher:
+            fetch: Callable[[int, int, Any], Future[FilePart]]
+            shutdown: Callable[[], None]
+
+        def get_fetcher() -> Fetcher:
+            if use_http_fetcher:
+                import random
 
+                port = random.randint(10000, 20000)
+                http_server: HttpServer = self.serve_http(
+                    src=src_path.parent.as_posix(), addr=f"localhost:{port}"
+                )
+                chunk_fetcher: HttpFetcher = http_server.get_fetcher(
+                    path=src_path.name,
+                    n_threads=read_threads,
+                )
+                # return chunk_fetcher.fetch
+                return Fetcher(fetch=chunk_fetcher.fetch, shutdown=http_server.shutdown)
+            else:
+                # Use the mount fetcher, which relies on FUSE which has problems in Docker/Windows/MacOS
+                mount_fetcher: MultiMountFileChunker = (
+                    self.get_multi_mount_file_chunker(
+                        src=src_path.as_posix(),
+                        chunk_size=chunk_size,
+                        threads=read_threads,
+                        mount_log=mount_log,
+                        direct_io=True,
+                    )
+                )
+                # return chunk_fetcher.fetch
+                return Fetcher(
+                    fetch=mount_fetcher.fetch, shutdown=mount_fetcher.shutdown
+                )
+
+        fetcher = get_fetcher()
         client = S3Client(s3_creds)
         upload_config: S3MutliPartUploadConfig = S3MutliPartUploadConfig(
             chunk_size=chunk_size.as_int(),
-            chunk_fetcher=chunk_fetcher.fetch,
+            chunk_fetcher=fetcher.fetch,
             max_write_threads=write_threads,
             retries=retries,
             resume_path_json=save_state_json,
@@ -927,7 +984,8 @@ class Rclone:
             traceback.print_exc()
             raise
         finally:
-            chunk_fetcher.shutdown()
+            fetcher.shutdown()
+            fetcher.shutdown()
 
     def get_multi_mount_file_chunker(
         self,
@@ -1095,7 +1153,7 @@ class Rclone:
                 shutil.move(payload, outfile)
                 return bytes(0)
             finally:
-                fp.close()
+                fp.dispose()
 
         except Exception as e:
             warnings.warn(f"Error copying bytes: {e}")
@@ -1334,6 +1392,31 @@ class Rclone:
             raise ValueError("NFS serve process failed to start")
         return proc
 
+    def serve_http(
+        self,
+        src: str,
+        addr: str = "localhost:8080",
+        other_args: list[str] | None = None,
+    ) -> HttpServer:
+        """Serve a remote or directory via HTTP.
+
+        Args:
+            src: Remote or directory to serve
+            addr: Network address and port to serve on (default: localhost:8080)
+        """
+        _, subpath = src.split(":", 1)  # might not work on local paths.
+        cmd_list: list[str] = ["serve", "http", "--addr", addr, src]
+        if other_args:
+            cmd_list += other_args
+        proc = self._launch_process(cmd_list)
+        time.sleep(2)
+        if proc.poll() is not None:
+            raise ValueError("HTTP serve process failed to start")
+        out: HttpServer = HttpServer(
+            url=f"http://{addr}", subpath=subpath, process=proc
+        )
+        return out
+
     def size_files(
         self,
         src: str,

{rclone_api-1.3.19 → rclone_api-1.3.20}/src/rclone_api/remote.py

@@ -16,3 +16,6 @@ class Remote:
 
     def __str__(self) -> str:
         return f"{self.name}:"
+
+    def __repr__(self) -> str:
+        return f"Remote({self.name!r})"

{rclone_api-1.3.19 → rclone_api-1.3.20}/src/rclone_api/s3/upload_file_multipart.py

@@ -25,31 +25,39 @@ _MIN_UPLOAD_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
 
 
 def upload_task(
-    info: UploadInfo, chunk: bytes, part_number: int, retries: int
+    info: UploadInfo, chunk: FilePart, part_number: int, retries: int
 ) -> FinishedPiece:
-    assert len(chunk) > 0
+    file_or_err: Path | Exception = chunk.get_file()
+    if isinstance(file_or_err, Exception):
+        raise file_or_err
+    file: Path = file_or_err
+    size = os.path.getsize(file)
     retries = retries + 1  # Add one for the initial attempt
     for retry in range(retries):
         try:
             if retry > 0:
                 locked_print(f"Retrying part {part_number} for {info.src_file_path}")
             locked_print(
-                f"Uploading part {part_number} for {info.src_file_path} of size {len(chunk)}"
-            )
-            part = info.s3_client.upload_part(
-                Bucket=info.bucket_name,
-                Key=info.object_name,
-                PartNumber=part_number,
-                UploadId=info.upload_id,
-                Body=chunk,
-            )
-            out: FinishedPiece = FinishedPiece(
-                etag=part["ETag"], part_number=part_number
+                f"Uploading part {part_number} for {info.src_file_path} of size {size}"
             )
+
+            with open(file, "rb") as f:
+                part = info.s3_client.upload_part(
+                    Bucket=info.bucket_name,
+                    Key=info.object_name,
+                    PartNumber=part_number,
+                    UploadId=info.upload_id,
+                    Body=f,
+                )
+                out: FinishedPiece = FinishedPiece(
+                    etag=part["ETag"], part_number=part_number
+                )
+            chunk.dispose()
             return out
         except Exception as e:
             if retry == retries - 1:
                 locked_print(f"Error uploading part {part_number}: {e}")
+                chunk.dispose()
                 raise e
             else:
                 locked_print(f"Error uploading part {part_number}: {e}, retrying")
@@ -72,7 +80,7 @@ def handle_upload(
 
         part: FinishedPiece = upload_task(
             info=upload_info,
-            chunk=fp.load(),
+            chunk=fp,
             part_number=part_number,
             retries=upload_info.retries,
         )
@@ -83,7 +91,7 @@ def handle_upload(
         warnings.warn(msg)
         return e
     finally:
-        fp.close()
+        fp.dispose()
 
 
 def prepare_upload_file_multipart(

{rclone_api-1.3.19 → rclone_api-1.3.20}/src/rclone_api/types.py

@@ -289,7 +289,7 @@ atexit.register(_on_exit_cleanup)
 
 
 class FilePart:
-    def __init__(self, payload: bytes | Exception, extra: Any) -> None:
+    def __init__(self, payload: Path | bytes | Exception, extra: Any) -> None:
         from rclone_api.util import random_str
 
         self.extra = extra
@@ -298,12 +298,18 @@ class FilePart:
         if isinstance(payload, Exception):
             self.payload = payload
             return
-        self.payload = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
-        with _TMP_DIR_ACCESS_LOCK:
-            if not self.payload.parent.exists():
-                self.payload.parent.mkdir(parents=True, exist_ok=True)
-            self.payload.write_bytes(payload)
-        _add_for_cleanup(self.payload)
+        if isinstance(payload, bytes):
+            self.payload = get_chunk_tmpdir() / f"{random_str(12)}.chunk"
+            with _TMP_DIR_ACCESS_LOCK:
+                if not self.payload.parent.exists():
+                    self.payload.parent.mkdir(parents=True, exist_ok=True)
+                self.payload.write_bytes(payload)
+            _add_for_cleanup(self.payload)
+        if isinstance(payload, Path):
+            self.payload = payload
+
+    def get_file(self) -> Path | Exception:
+        return self.payload
 
     @property
     def size(self) -> int:
@@ -337,7 +343,7 @@ class FilePart:
     def is_error(self) -> bool:
         return isinstance(self.payload, Exception)
 
-    def close(self) -> None:
+    def dispose(self) -> None:
         with self._lock:
             if isinstance(self.payload, Exception):
                 warnings.warn(
@@ -352,4 +358,4 @@ class FilePart:
                     warnings.warn(f"Cannot close file part because of error: {e}")
 
     def __del__(self):
-        self.close()
+        self.dispose()

{rclone_api-1.3.19 → rclone_api-1.3.20}/src/rclone_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rclone_api
-Version: 1.3.19
+Version: 1.3.20
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.3.19 → rclone_api-1.3.20}/src/rclone_api.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ src/rclone_api/file.py
 src/rclone_api/file_item.py
 src/rclone_api/filelist.py
 src/rclone_api/group_files.py
+src/rclone_api/http_server.py
 src/rclone_api/log.py
 src/rclone_api/mount.py
 src/rclone_api/mount_read_chunker.py
@@ -89,6 +90,7 @@ tests/test_remote_control.py
 tests/test_remotes.py
 tests/test_s3.py
 tests/test_scan_missing_folders.py
+tests/test_serve_http.py
 tests/test_size_files.py
 tests/test_size_suffix.py
 tests/test_walk.py

{rclone_api-1.3.19 → rclone_api-1.3.20}/tests/test_copy_file_resumable_s3.py

@@ -117,7 +117,7 @@ class RcloneCopyResumableFileToS3(unittest.TestCase):
             )
         os.environ["RCLONE_API_VERBOSE"] = "1"
 
-    @unittest.skip("Skip for now - long running test")
+    # @unittest.skip("Skip for now - long running test")
     def test_upload_chunks(self) -> None:
         """Test basic Webdav serve functionality."""
         # config = _generate_rclone_config(PORT)

rclone_api-1.3.20/tests/test_serve_http.py

@@ -0,0 +1,255 @@
+"""
+Unit test file for testing rclone mount functionality.
+"""
+
+import atexit
+import os
+import shutil
+import subprocess
+import time
+import unittest
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from rclone_api import Config, Rclone
+from rclone_api.http_server import HttpServer, Range
+
+load_dotenv()
+
+_CLEANUP: list[Path] = []
+
+
+def _cleanup() -> None:
+    for p in _CLEANUP:
+        if p.exists():
+            # p.unlink()
+            if p.is_dir():
+
+                shutil.rmtree(p, ignore_errors=True)
+            else:
+                p.unlink()
+
+
+atexit.register(_cleanup)
+
+
+def _generate_rclone_config() -> Config:
+    # Load environment variables
+    BUCKET_KEY_SECRET = os.getenv("BUCKET_KEY_SECRET")
+    BUCKET_KEY_PUBLIC = os.getenv("BUCKET_KEY_PUBLIC")
+    BUCKET_URL = "sfo3.digitaloceanspaces.com"
+
+    config_text = f"""
+[dst]
+type = s3
+provider = DigitalOcean
+access_key_id = {BUCKET_KEY_PUBLIC}
+secret_access_key = {BUCKET_KEY_SECRET}
+endpoint = {BUCKET_URL}
+"""
+    return Config(config_text)
+
+
+class RcloneServeHttpTester(unittest.TestCase):
+    """Test rclone mount functionality."""
+
+    def setUp(self) -> None:
+        """Check if all required environment variables are set before running tests."""
+        required_vars = [
+            "BUCKET_NAME",
+            "BUCKET_KEY_SECRET",
+            "BUCKET_KEY_PUBLIC",
+            "BUCKET_URL",
+        ]
+        missing = [var for var in required_vars if not os.getenv(var)]
+        if missing:
+            self.skipTest(
+                f"Missing required environment variables: {', '.join(missing)}"
+            )
+
+        self.bucket_name = os.getenv("BUCKET_NAME")
+        self.mount_point = Path("test_tmp_serve_http")
+        _CLEANUP.append(self.mount_point)
+        # Create mount point directory if it doesn't exist
+        # self.mount_point.mkdir(exist_ok=True)
+        # make parents
+        parent = self.mount_point.parent
+        if not parent.exists():
+            parent.mkdir(parents=True)
+
+        os.environ["RCLONE_API_VERBOSE"] = "1"
+        self.rclone = Rclone(_generate_rclone_config())
+
+    # @unittest.skip("Skip for now")
+    def test_server_http(self) -> None:
+        """Test mounting a remote bucket."""
+        remote_path = f"dst:{self.bucket_name}"
+        http_server: HttpServer | None = None
+        try:
+            with self.rclone.serve_http(
+                remote_path, addr="localhost:8081"
+            ) as http_server:
+                resource_url = "zachs_video/internaly_ai_alignment.mp4"
+                expected_size = 73936110
+
+                actual_size = http_server.size(resource_url)
+                print(f"Actual size: {actual_size}")
+
+                self.assertEqual(actual_size, expected_size)
+                dst1 = self.mount_point / Path(
+                    "zachs_video/internaly_ai_alignment.mp4.1"
+                )
+                dst2 = self.mount_point / Path(
+                    "zachs_video/internaly_ai_alignment.mp4.2"
+                )
+
+                _CLEANUP.extend([Path("zachs_video"), dst1, dst2])
+
+                # out1 = http_server.copy_chunked(resource_url, dst1).result()
+                start = time.time()
+                out1 = http_server.download(resource_url, dst1)
+                print(f"(1) Time taken: {time.time() - start}")
+                start = time.time()
+                out2 = http_server.download_multi_threaded(resource_url, dst2)
+                print(f"(2) Time taken: {time.time() - start}")
+
+                assert not isinstance(out1, Exception)
+                assert not isinstance(out2, Exception)
+
+                s1 = dst1.stat().st_size
+                s2 = dst2.stat().st_size
+
+                print(f"Size of {dst1}: {dst1.stat().st_size}")
+                print(f"Size of {dst2}: {dst2.stat().st_size}")
+
+                if s1 != s2:
+                    # find the first index where there is a difference
+                    with open(dst1, "rb") as f1, open(dst2, "rb") as f2:
+                        bad_index = 0
+                        while (chunk1 := f1.read(1)) and (chunk2 := f2.read(1)):
+                            if chunk1 != chunk2:
+                                break
+                            bad_index += 1
+                        print("bad index: ", bad_index)
+
+                self.assertIsInstance(out1, Path)
+                self.assertIsInstance(out2, Path)
+                assert isinstance(out2, Path)
+
+                # print(f"Bytes written: {out1.stat().st_size}")
+                print(f"Bytes written: {out2.stat().st_size}")
+
+                def hash_bytes(fp: Path) -> str:
+                    import hashlib
+
+                    sha256 = hashlib.sha256()
+                    with open(fp, "rb") as f:
+                        while chunk := f.read(4096):
+                            sha256.update(chunk)
+                    return sha256.hexdigest()
+
+                hash1 = hash_bytes(dst1)
+                hash2 = hash_bytes(dst2)
+
+                print(dst1.absolute())
+                print(dst2.absolute())
+
+                self.assertEqual(hash1, hash2)
+                print("Done")
+
+        except subprocess.CalledProcessError as e:
+            self.fail(f"Mount operation failed: {str(e)}")
+        finally:
+            # Cleanup will happen in tearDown
+            pass
+
+    # @unittest.skip("Skip for now")
+    def test_small_range(self) -> None:
+        """Test mounting a remote bucket."""
+        remote_path = f"dst:{self.bucket_name}"
+        http_server: HttpServer | None = None
+        try:
+            with self.rclone.serve_http(
+                src=remote_path, addr="localhost:8082"
+            ) as http_server:
+                resource_url = "zachs_video/internaly_ai_alignment.mp4"
+                expected_size = 73936110
+
+                actual_size = http_server.size(resource_url)
+                print(f"Actual size: {actual_size}")
+
+                self.assertEqual(actual_size, expected_size)
+                dst1 = self.mount_point / Path(
+                    "zachs_video/internaly_ai_alignment.mp4-2.1"
+                )
+                dst2 = self.mount_point / Path(
+                    "zachs_video/internaly_ai_alignment.mp4-2.2"
+                )
+
+                _CLEANUP.extend([Path("zachs_video"), dst1, dst2])
+                range: Range = Range(0, 1000)
+
+                # out1 = http_server.copy_chunked(resource_url, dst1).result()
+                start = time.time()
+                out1 = http_server.download(resource_url, dst1, range=range)
+                print(f"(1) Time taken: {time.time() - start}")
+                start = time.time()
+                out2 = http_server.download_multi_threaded(
+                    resource_url, dst2, range=range
+                )
+                print(f"(2) Time taken: {time.time() - start}")
+
+                assert not isinstance(out1, Exception)
+                assert not isinstance(out2, Exception)
+
+                s1 = dst1.stat().st_size
+                s2 = dst2.stat().st_size
+
+                print(f"Size of {dst1}: {dst1.stat().st_size}")
+                print(f"Size of {dst2}: {dst2.stat().st_size}")
+
+                if s1 != s2:
+                    # find the first index where there is a difference
+                    with open(dst1, "rb") as f1, open(dst2, "rb") as f2:
+                        bad_index = 0
+                        while (chunk1 := f1.read(1)) and (chunk2 := f2.read(1)):
+                            if chunk1 != chunk2:
+                                break
+                            bad_index += 1
+                        print("bad index: ", bad_index)
+
+                self.assertIsInstance(out1, Path)
+                self.assertIsInstance(out2, Path)
+                assert isinstance(out2, Path)
+
+                # print(f"Bytes written: {out1.stat().st_size}")
+                print(f"Bytes written: {out2.stat().st_size}")
+
+                def hash_bytes(fp: Path) -> str:
+                    import hashlib
+
+                    sha256 = hashlib.sha256()
+                    with open(fp, "rb") as f:
+                        while chunk := f.read(4096):
+                            sha256.update(chunk)
+                    return sha256.hexdigest()
+
+                hash1 = hash_bytes(dst1)
+                hash2 = hash_bytes(dst2)
+
+                print(dst1.absolute())
+                print(dst2.absolute())
+
+                self.assertEqual(hash1, hash2)
+                print("Done")
+
+        except subprocess.CalledProcessError as e:
+            self.fail(f"Mount operation failed: {str(e)}")
+        finally:
+            # Cleanup will happen in tearDown
+            pass
+
+
+if __name__ == "__main__":
+    unittest.main()