rclone-api 1.5.55__py3-none-any.whl → 1.5.56__py3-none-any.whl

rclone_api/fs/filesystem.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from typing import Generator
 
 from rclone_api.config import Config
+from rclone_api.fs.walk_threaded_walker import FSWalker
 
 logger = logging.getLogger(__name__)
 
@@ -277,9 +278,19 @@ class FSPath:
     def walk(
         self,
     ) -> "Generator[tuple[FSPath, list[str], list[str]], None, None]":
-        from rclone_api.fs.walk import os_walk
+        from rclone_api.fs.walk import fs_walk
 
-        return os_walk(self)
+        return fs_walk(self)
+
+    def walk_begin(self, max_backlog: int = 8) -> FSWalker:
+        """
+        Threaded walker to hide latency.
+
+        with FSPath.walk_begin() as walker:
+            for root, dirnames, filenames in walker:
+                pass
+        """
+        return FSWalker(self, max_backlog=max_backlog)
 
     def relative_to(self, other: "FSPath") -> "FSPath":
         p = Path(self.path).relative_to(other.path)

rclone_api/fs/walk.py

@@ -4,7 +4,7 @@ from typing import Generator
 from rclone_api.fs.filesystem import FSPath, logger
 
 
-def os_walk(
+def fs_walk(
     self: FSPath,
 ) -> Generator[tuple[FSPath, list[str], list[str]], None, None]:
     root_path = self

rclone_api/fs/walk_threaded.py

@@ -0,0 +1,56 @@
+from queue import Queue
+from threading import Event, Thread
+from typing import Generator, List, Optional, Tuple
+
+from rclone_api.fs.filesystem import FSPath
+from rclone_api.fs.walk_threaded_walker import FSWalker
+
+### API
+
+
+def os_walk_threaded_begin(self: FSPath, max_backlog: int = 8) -> FSWalker:
+    return FSWalker(self, max_backlog)
+
+
+### Implementation
+
+
+class _FSWalkThread:
+    def __init__(self, fspath: FSPath, max_backlog: int = 8):
+        self.fspath = fspath
+        self.result_queue: Queue[Optional[Tuple[FSPath, List[str], List[str]]]] = Queue(
+            maxsize=max_backlog
+        )
+        self.thread = Thread(target=self.worker)
+        self.stop_event = Event()
+        self.start()
+
+    def worker(self):
+        from rclone_api.fs.walk import fs_walk
+
+        for root, dirnames, filenames in fs_walk(self.fspath):
+            if self.stop_event.is_set():
+                break
+            self.result_queue.put((root, dirnames, filenames))
+        self.result_queue.put(None)  # Sentinel value to indicate completion
+
+    def start(self):
+        self.thread.start()
+
+    def join(self):
+        self.thread.join()
+
+    def get_results(self) -> Generator[Tuple[FSPath, List[str], List[str]], None, None]:
+        while True:
+            result = self.result_queue.get()
+            if result is None:  # Check for sentinel value
+                break
+            yield result
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop_event.set()
+        self.join()

rclone_api/fs/walk_threaded_walker.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class FSWalker:
+    """Threaded"""
+
+    fspath: Any
+    max_backlog: int
+
+    def __enter__(self):
+        from rclone_api.fs.filesystem import FSPath
+        from rclone_api.fs.walk_threaded import _FSWalkThread
+
+        assert isinstance(
+            self.fspath, FSPath
+        ), f"Expected FSPath, got {type(self.fspath)}"
+        self.walker = _FSWalkThread(self.fspath, self.max_backlog)
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.walker.stop_event.set()
+        self.walker.join()
+
+    def __iter__(self):
+        return self.walk()
+
+    def walk(self):
+        return self.walker.get_results()

{rclone_api-1.5.55.dist-info → rclone_api-1.5.56.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rclone_api
-Version: 1.5.55
+Version: 1.5.56
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.5.55.dist-info → rclone_api-1.5.56.dist-info}/RECORD

@@ -40,8 +40,10 @@ rclone_api/detail/copy_file_parts_resumable.py,sha256=RoUWV2eBWEvuuTfsvrz5BhtvX3
 rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
-rclone_api/fs/filesystem.py,sha256=NjAe5T79YOUpE8cwKmFVQTjwdu-2iIJTXotV4iEwjlo,10617
-rclone_api/fs/walk.py,sha256=80MdbWoUMx0mfgpiHtvMKs7Plj656KyJ9ZSJBBqwlxQ,875
+rclone_api/fs/filesystem.py,sha256=ddOEfyPdQExs7EK_Et5rRVwqpIkthrpOp3sQZcSI2Ls,10984
+rclone_api/fs/walk.py,sha256=Yqn_SS25gTL0ANWSQxnAdN-ELBO1PMZfYYnyxoGafWA,875
+rclone_api/fs/walk_threaded.py,sha256=ad9MByUsGq3M0Tnv3yfFAK_FFSVJ23fKWQIrU4aLNUs,1579
+rclone_api/fs/walk_threaded_walker.py,sha256=gAJfn-oNH3cLNBCyG0gSSsnQsLz0xtOf6twNeH3kbxw,720
 rclone_api/s3/api.py,sha256=HpheMOGHcCc0CyNdv0zvB0S92g312urjRdssRrkdpb8,4162
 rclone_api/s3/basic_ops.py,sha256=hK3366xhVEzEcjz9Gk_8lFx6MRceAk72cax6mUrr6ko,2104
 rclone_api/s3/chunk_task.py,sha256=waEYe-iYQ1_BR3NCS4BrzVrK9UANvH1EcbXx2I6Z_NM,6839
@@ -56,9 +58,9 @@ rclone_api/s3/multipart/upload_parts_inline.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9
 rclone_api/s3/multipart/upload_parts_resumable.py,sha256=6-nlMclS8jyVvMvFbQDcZOX9MY1WbCcKA_s9bwuYxnk,9793
 rclone_api/s3/multipart/upload_parts_server_side_merge.py,sha256=Fp2pdrs5dONQI9LkfNolgAGj1-Z2V1SsRd0r0sreuXI,18040
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.5.55.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.5.55.dist-info/METADATA,sha256=1NBhvcPV0AsDF84Rkdj3d1W2rU35DR9HYm2rq7KcGnU,37305
-rclone_api-1.5.55.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rclone_api-1.5.55.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
-rclone_api-1.5.55.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.5.55.dist-info/RECORD,,
+rclone_api-1.5.56.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.5.56.dist-info/METADATA,sha256=6T5a5qEwy0amCkHxvF2omjBfMXD-OPtukKnyICtgFCo,37305
+rclone_api-1.5.56.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rclone_api-1.5.56.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
+rclone_api-1.5.56.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.5.56.dist-info/RECORD,,