rclone-api 1.5.12__py3-none-any.whl → 1.5.14__py3-none-any.whl

rclone_api/__init__.py

@@ -841,7 +841,7 @@ class Rclone:
     def serve_http(
         self,
         src: str,
-        addr: str = "localhost:8080",
+        addr: str | None = None,
         other_args: list[str] | None = None,
     ) -> HttpServer:
         """
@@ -862,7 +862,9 @@ class Rclone:
         Returns:
             HttpServer object with methods for accessing the served content
         """
-        return self.impl.serve_http(src=src, addr=addr, other_args=other_args)
+        return self.impl.serve_http(
+            src=src, cache_mode="minimal", addr=addr, other_args=other_args
+        )
 
     def size_files(
         self,

rclone_api/filesystem.py

@@ -0,0 +1,170 @@
+import abc
+import shutil
+from pathlib import Path
+
+
+class FileSystem(abc.ABC):
+    def __init__(self) -> None:
+        pass
+
+    @abc.abstractmethod
+    def copy(self, src: Path | str, dest: Path | str) -> None:
+        pass
+
+    @abc.abstractmethod
+    def read_binary(self, path: Path | str) -> bytes:
+        pass
+
+    @abc.abstractmethod
+    def exists(self, path: Path | str) -> bool:
+        pass
+
+    @abc.abstractmethod
+    def write_binary(self, path: Path | str, data: bytes) -> None:
+        pass
+
+    @abc.abstractmethod
+    def mkdir(self, path: str, parents=True, exist_ok=True) -> None:
+        pass
+
+    @abc.abstractmethod
+    def get_path(self, path: str) -> "FSPath":
+        pass
+
+    def read_text(self, path: Path | str) -> str:
+        utf = self.read_binary(path)
+        return utf.decode("utf-8")
+
+    def write_text(self, path: Path | str, data: str, encoding: str | None) -> None:
+        encoding = encoding or "utf-8"
+        utf = data.encode(encoding)
+        self.write_binary(path, utf)
+
+
+class RealFileSystem(FileSystem):
+
+    @staticmethod
+    def get_real_path(path: Path | str) -> "FSPath":
+        path_str = Path(path).as_posix()
+        return FSPath(RealFileSystem(), path_str)
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def copy(self, src: Path | str, dest: Path | str) -> None:
+        shutil.copy(str(src), str(dest))
+
+    def read_binary(self, path: Path | str) -> bytes:
+        with open(path, "rb") as f:
+            return f.read()
+
+    def write_binary(self, path: Path | str, data: bytes) -> None:
+        with open(path, "wb") as f:
+            f.write(data)
+
+    def exists(self, path: Path | str) -> bool:
+        return Path(path).exists()
+
+    def mkdir(self, path: str, parents=True, exist_ok=True) -> None:
+        Path(path).mkdir(parents=parents, exist_ok=exist_ok)
+
+    def get_path(self, path: str) -> "FSPath":
+        return FSPath(self, path)
+
+
+class RemoteFileSystem(FileSystem):
+    def __init__(self, rclone_conf: Path, src: str) -> None:
+        from rclone_api import HttpServer, Rclone
+
+        super().__init__()
+        self.rclone_conf = rclone_conf
+        self.rclone: Rclone = Rclone(rclone_conf)
+        self.server: HttpServer = self.rclone.serve_http(src=src)
+        self.shutdown = False
+
+    def _to_str(self, path: Path | str) -> str:
+        if isinstance(path, Path):
+            return path.as_posix()
+        return path
+
+    def copy(self, src: Path | str, dest: Path | str) -> None:
+        src = self._to_str(src)
+        dest = self._to_str(dest)
+        self.rclone.copy(src, dest)
+
+    def read_binary(self, path: Path | str) -> bytes:
+        path = self._to_str(path)
+        err = self.rclone.read_bytes(path)
+        if isinstance(err, Exception):
+            raise FileNotFoundError(f"File not found: {path}")
+        return err
+
+    def write_binary(self, path: Path | str, data: bytes) -> None:
+        path = self._to_str(path)
+        self.rclone.write_bytes(data, path)
+
+    def exists(self, path: Path | str) -> bool:
+        path = self._to_str(path)
+        return self.server.exists(path)
+
+    def mkdir(self, path: str, parents=True, exist_ok=True) -> None:
+        raise NotImplementedError("RemoteFileSystem does not support mkdir")
+
+    def get_path(self, path: str) -> "FSPath":
+        return FSPath(self, path)
+
+    def dispose(self) -> None:
+        if self.shutdown:
+            return
+        self.shutdown = True
+        self.server.shutdown()
+
+    def __del__(self) -> None:
+        self.dispose()
+
+
+class FSPath:
+    def __init__(self, fs: FileSystem, path: str) -> None:
+        self.path = path
+        self.fs = fs
+
+    def read_text(self) -> str:
+        return self.fs.read_text(self.path)
+
+    def read_binary(self) -> bytes:
+        return self.fs.read_binary(self.path)
+
+    def exists(self) -> bool:
+        return self.fs.exists(self.path)
+
+    def __str__(self) -> str:
+        return self.path
+
+    def __repr__(self) -> str:
+        return f"FSPath({self.path})"
+
+    def mkdir(self, parents=True, exist_ok=True) -> None:
+        self.fs.mkdir(self.path, parents=parents, exist_ok=exist_ok)
+
+    def write_text(self, data: str, encoding: str | None = None) -> None:
+        self.fs.write_text(self.path, data, encoding=encoding)
+
+    def rmtree(self, ignore_errors=False) -> None:
+        assert self.exists(), f"Path does not exist: {self.path}"
+        # check fs is RealFileSystem
+        assert isinstance(self.fs, RealFileSystem)
+        shutil.rmtree(self.path, ignore_errors=ignore_errors)
+
+    @property
+    def name(self) -> str:
+        return Path(self.path).name
+
+    @property
+    def parent(self) -> "FSPath":
+        parent_path = Path(self.path).parent
+        parent_str = parent_path.as_posix()
+        return FSPath(self.fs, parent_str)
+
+    def __truediv__(self, other: str) -> "FSPath":
+        new_path = Path(self.path) / other
+        return FSPath(self.fs, new_path.as_posix())

rclone_api/http_server.py

@@ -17,6 +17,7 @@ from rclone_api.process import Process
 from rclone_api.types import Range, SizeSuffix, get_chunk_tmpdir
 
 _TIMEOUT = 10 * 60  # 10 minutes
+_PUT_WARNED = False
 
 
 _range = range
@@ -74,16 +75,34 @@ class HttpServer:
 
     def put(self, path: str, data: bytes) -> Exception | None:
         """Put bytes to the server."""
+        global _PUT_WARNED
+        if not _PUT_WARNED:
+            _PUT_WARNED = True
+            warnings.warn("PUT method not implemented on the rclone binary as of 1.69")
         try:
             assert self.process is not None
             url = self._get_file_url(path)
-            response = httpx.put(url, content=data, timeout=_TIMEOUT)
+            headers = {"Content-Type": "application/octet-stream"}
+            response = httpx.post(url, content=data, timeout=_TIMEOUT, headers=headers)
+            print("Allowed methods:", response.headers.get("Allow"))
             response.raise_for_status()
             return None
         except Exception as e:
             warnings.warn(f"Failed to put {path} to {self.url}: {e}")
             return e
 
+    def delete(self, path: str) -> Exception | None:
+        """Remove file from the server."""
+        try:
+            assert self.process is not None
+            url = self._get_file_url(path)
+            response = httpx.delete(url)
+            response.raise_for_status()
+            return None
+        except Exception as e:
+            warnings.warn(f"Failed to remove {path} from {self.url}: {e}")
+            return e
+
     def download(
         self, path: str, dst: Path, range: Range | None = None
     ) -> Path | Exception:
@@ -223,11 +242,8 @@ class HttpServer:
     def shutdown(self) -> None:
         """Shutdown the server."""
         if self.process:
-            self.process.terminate()
-            if self.process.stdout:
-                self.process.stdout.close()
-            if self.process.stderr:
-                self.process.stderr.close()
+            self.process.dispose()
+            self.process = None
 
 
 class HttpFetcher:

rclone_api/process.py

@@ -30,6 +30,7 @@ class Process:
         ), f"rclone executable not found: {args.rclone_exe}"
         self.args = args
         self.log = args.log
+        self.cleaned_up = False
         self.tempfile: Path | None = None
 
         verbose = get_verbose(args.verbose)
@@ -75,11 +76,17 @@ class Process:
     def __enter__(self) -> "Process":
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+    def dispose(self) -> None:
+        if self.cleaned_up:
+            return
+        self.cleaned_up = True
         self.terminate()
         self.wait()
         self.cleanup()
 
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self.dispose()
+
     def cleanup(self) -> None:
         if self.tempfile:
             clear_temp_config_file(self.tempfile)

rclone_api/rclone_impl.py

@@ -6,6 +6,7 @@ import os
 import random
 import subprocess
 import time
+import tracemalloc
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
 from datetime import datetime
@@ -51,6 +52,9 @@ from rclone_api.util import (
     to_path,
 )
 
+# Enable tracing memory usage always
+tracemalloc.start()
+
 
 def rclone_verbose(verbose: bool | None) -> bool:
     if verbose is not None:
@@ -1164,6 +1168,7 @@ class RcloneImpl:
     def serve_http(
         self,
         src: str,
+        cache_mode: str | None,
         addr: str | None = None,
         serve_http_log: Path | None = None,
         other_args: list[str] | None = None,
@@ -1186,9 +1191,13 @@ class RcloneImpl:
             "0",
             "--vfs-read-chunk-size-limit",
             "512M",
-            "--vfs-cache-mode",
-            "off",
         ]
+
+        if cache_mode:
+            cmd_list += [
+                "--vfs-cache-mode",
+                cache_mode,
+            ]
         if serve_http_log:
             cmd_list += ["--log-file", str(serve_http_log)]
             cmd_list += ["-vvvv"]

rclone_api/s3/multipart/upload_parts_resumable.py

@@ -232,7 +232,7 @@ def upload_parts_resumable(
 
     atexit.register(lambda: shutil.rmtree(tmp_dir, ignore_errors=True))
 
-    with self.serve_http(src_dir) as http_server:
+    with self.serve_http(src_dir, cache_mode="minimal") as http_server:
         tmpdir: Path = Path(tmp_dir)
         write_semaphore = threading.Semaphore(threads)
         with ThreadPoolExecutor(max_workers=threads) as upload_executor:

{rclone_api-1.5.12.dist-info → rclone_api-1.5.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rclone_api
-Version: 1.5.12
+Version: 1.5.14
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.5.12.dist-info → rclone_api-1.5.14.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rclone_api/__init__.py,sha256=uhoHM66jDRlk0Hq4PTrk7QZE6VcJqiu5Jeo9nj34edk,32380
+rclone_api/__init__.py,sha256=tzjjY0b-VfJ8BFYs15S3OApVwd7S1JEYbv0gUz-13aE,32421
 rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
 rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
 rclone_api/config.py,sha256=f6jEAxVorGFr31oHfcsu5AJTtOJj2wR5tTSsbGGZuIw,2558
@@ -13,14 +13,15 @@ rclone_api/file_item.py,sha256=cH-AQYsxedhNPp4c8NHY1ad4Z7St4yf_VGbmiGD59no,1770
 rclone_api/file_part.py,sha256=i6ByS5_sae8Eba-4imBVTxd-xKC8ExWy7NR8QGr0ors,6155
 rclone_api/file_stream.py,sha256=_W3qnwCuigqA0hzXl2q5pAxSZDRaUSwet4BkT0lpnzs,1431
 rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
+rclone_api/filesystem.py,sha256=NlDlKWjX_WtaqjkwinMpqhaaU5QPkIrnTNKi7_k3tG0,4960
 rclone_api/group_files.py,sha256=H92xPW9lQnbNw5KbtZCl00bD6iRh9yRbCuxku4j_3dg,8036
-rclone_api/http_server.py,sha256=Rc7wHn4XeyJCwNRe6pr88vHtbJxFGdYNcgRVYNuUG3E,9784
+rclone_api/http_server.py,sha256=p1_S9VAViVvGif6NA_rxrDgMqnOmPTxJaHQ7B43FK70,10431
 rclone_api/install.py,sha256=Xb1BRn8rQcSpSd4dzmvIOELP2zM2DytUeIZ6jzv738A,2893
 rclone_api/log.py,sha256=VZHM7pNSXip2ZLBKMP7M1u-rp_F7zoafFDuR8CPUoKI,1271
 rclone_api/mount.py,sha256=LZqEhuKZunbWVqmsOIqkkCotaxWJpdFRS1InXveoU5E,1428
 rclone_api/mount_util.py,sha256=jqhJEVTHV6c6lOOzUYb4FLMbqDMHdz7-QRcdH-IobFc,10154
-rclone_api/process.py,sha256=tGooS5NLdPuqHh7hCH8SfK44A6LGftPQCPQUNgSo0a0,5714
-rclone_api/rclone_impl.py,sha256=kr0gvOSuiwxU15rvATYZmSzpWYDiAzjZ2WNe1wqI6NM,46345
+rclone_api/process.py,sha256=MeWiN-TrrN0HmtWexBPxGwf84z6f-_E5yaXE-YtLYpY,5879
+rclone_api/rclone_impl.py,sha256=STHQ_hT0chyp2yj3Cp-Vd0xXc1Pdpo3p9J3bTQEm0XE,46531
 rclone_api/remote.py,sha256=mTgMTQTwxUmbLjTpr-AGTId2ycXKI9mLX5L7PPpDIoc,520
 rclone_api/rpath.py,sha256=Y1JjQWcie39EgQrq-UtbfDz5yDLCwwfu27W7AQXllSE,2860
 rclone_api/scan_missing_folders.py,sha256=-8NCwpCaHeHrX-IepCoAEsX1rl8S-GOCxcIhTr_w3gA,4747
@@ -50,12 +51,12 @@ rclone_api/s3/multipart/info_json.py,sha256=-e8UCwrqjAP64U8PmH-o2ciJ6TN48DwHktJf
 rclone_api/s3/multipart/merge_state.py,sha256=ziTB9CYV-OWaky5C1fOT9hifSY2zgUrk5HmX1Xeu2UA,4978
 rclone_api/s3/multipart/upload_info.py,sha256=d6_OfzFR_vtDzCEegFfzCfWi2kUBUV4aXZzqAEVp1c4,1874
 rclone_api/s3/multipart/upload_parts_inline.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9akiew3MFjGmufLo2w,12503
-rclone_api/s3/multipart/upload_parts_resumable.py,sha256=diJoUpVYow6No_dNgOZIYVsv43k4evb6zixqpzWJaUk,9771
+rclone_api/s3/multipart/upload_parts_resumable.py,sha256=6-nlMclS8jyVvMvFbQDcZOX9MY1WbCcKA_s9bwuYxnk,9793
 rclone_api/s3/multipart/upload_parts_server_side_merge.py,sha256=Fp2pdrs5dONQI9LkfNolgAGj1-Z2V1SsRd0r0sreuXI,18040
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.5.12.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.5.12.dist-info/METADATA,sha256=7MG4gBbQ1cKx9EHM4hyR4LhpOuZVyE55YKfjQYNEox4,32657
-rclone_api-1.5.12.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-rclone_api-1.5.12.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
-rclone_api-1.5.12.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.5.12.dist-info/RECORD,,
+rclone_api-1.5.14.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.5.14.dist-info/METADATA,sha256=NEyfLjNiEDEL1_rdIVPkqPPr1fksSmZFpxgDFmkCFJQ,32657
+rclone_api-1.5.14.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+rclone_api-1.5.14.dist-info/entry_points.txt,sha256=fJteOlYVwgX3UbNuL9jJ0zUTuX2O79JFAeNgK7Sw7EQ,255
+rclone_api-1.5.14.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.5.14.dist-info/RECORD,,