rclone-api 1.5.70__py3-none-any.whl → 1.5.71__py3-none-any.whl

rclone_api/fs/walk.py

@@ -1,27 +1,62 @@
-from pathlib import Path
+import os
+from collections import OrderedDict
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from typing import Generator
 
 from rclone_api.fs.filesystem import FSPath, logger
 
+_FS_WALK_THREAD_MAX_BACKLOG = int(os.getenv("FS_WALK_THREAD_MAX_BACKLOG", "16"))
+# module‐wide executor
+_executor = ThreadPoolExecutor(max_workers=_FS_WALK_THREAD_MAX_BACKLOG)
 
-def fs_walk(
+
+def _list_dir(path: FSPath):
+    try:
+        filenames, dirnames = path.ls()
+    except Exception as e:
+        logger.warning(f"Unable to list directory {path}: {e}")
+        return None
+    return path, dirnames, filenames
+
+
+def fs_walk_parallel(
     self: FSPath,
 ) -> Generator[tuple[FSPath, list[str], list[str]], None, None]:
-    root_path = self
-    stack: list[str] = [self.path]
-
-    while stack:
-        curr_path = stack.pop()
-        curr_suffix_path = Path(curr_path).relative_to(Path(root_path.path)).as_posix()
-        current_dir = root_path / curr_suffix_path
-        try:
-            filenames, dirnames = current_dir.ls()
-        except Exception as e:
-            logger.warning(f"Unable to list directory {current_dir}: {e}")
-            continue
-
-        yield current_dir, dirnames, filenames
-
-        # Add subdirectories to stack for further traversal
-        for dirname in reversed(dirnames):
-            stack.append((current_dir / dirname).path)
+    """
+    Parallel version of fs_walk: walks `self` and lists
+    up to 16 directories at once using the global executor,
+    but yields results in the same order tasks were submitted.
+    """
+    root = self
+
+    # use an OrderedDict to remember submission order
+    futures: OrderedDict = OrderedDict()
+    # submit the root first
+    futures[_executor.submit(_list_dir, root)] = root
+
+    while futures:
+        # wait until at least one of them finishes
+        done, _ = wait(futures.keys(), return_when=FIRST_COMPLETED)
+
+        # iterate through our futures *in submission order*
+        for fut in list(futures.keys()):
+            if fut not in done:
+                continue
+
+            _ = futures.pop(fut)  # remove it in order
+            result = fut.result()
+            if result is None:
+                continue  # error already logged
+
+            current_dir, dirnames, filenames = result
+            yield current_dir, dirnames, filenames
+
+            # now schedule its children (they go at the end of our OrderedDict)
+            for dirname in dirnames:
+                sub = current_dir / dirname
+                futures[_executor.submit(_list_dir, sub)] = sub
+
+
+def fs_walk(self: FSPath) -> Generator[tuple[FSPath, list[str], list[str]], None, None]:
+    """Sequential API, now backed by the global-thread-pool parallel implementation."""
+    yield from fs_walk_parallel(self)

{rclone_api-1.5.70.dist-info → rclone_api-1.5.71.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rclone_api
-Version: 1.5.70
+Version: 1.5.71
 Summary: rclone api in python
 Home-page: https://github.com/zackees/rclone-api
 License: BSD 3-Clause License

{rclone_api-1.5.70.dist-info → rclone_api-1.5.71.dist-info}/RECORD

@@ -41,7 +41,7 @@ rclone_api/detail/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,336
 rclone_api/experimental/flags.py,sha256=qCVD--fSTmzlk9hloRLr0q9elzAOFzPsvVpKM3aB1Mk,2739
 rclone_api/experimental/flags_base.py,sha256=ajU_czkTcAxXYU-SlmiCfHY7aCQGHvpCLqJ-Z8uZLk0,2102
 rclone_api/fs/filesystem.py,sha256=2Ewa5g_842XU-UT4JDkPkq199pjknjoqpTiKtbRRRGk,11964
-rclone_api/fs/walk.py,sha256=Yqn_SS25gTL0ANWSQxnAdN-ELBO1PMZfYYnyxoGafWA,875
+rclone_api/fs/walk.py,sha256=gqxfJqzFSQlLwZ18ttO25SbCCs7Vcz_CtlDgjiqpNZc,2183
 rclone_api/fs/walk_threaded.py,sha256=zlbooQ7WbteUK1cCjm1Yj66VpDzS8Qm3DcHBSz1Gbjk,1591
 rclone_api/fs/walk_threaded_walker.py,sha256=xi0QkmmzI35NcaA08FLjGe7ox9tLZcgXtOZCT8nMuWU,718
 rclone_api/s3/api.py,sha256=HpheMOGHcCc0CyNdv0zvB0S92g312urjRdssRrkdpb8,4162
@@ -58,9 +58,9 @@ rclone_api/s3/multipart/upload_parts_inline.py,sha256=V7syKjFyVIe4U9Ahl5XgqVTzt9
 rclone_api/s3/multipart/upload_parts_resumable.py,sha256=tdOkEfoK9zENaTfgVyTWB5GNqn-IYl-zJn-rJLcoVTo,12388
 rclone_api/s3/multipart/upload_parts_server_side_merge.py,sha256=Fp2pdrs5dONQI9LkfNolgAGj1-Z2V1SsRd0r0sreuXI,18040
 rclone_api/s3/multipart/upload_state.py,sha256=f-Aq2NqtAaMUMhYitlICSNIxCKurWAl2gDEUVizLIqw,6019
-rclone_api-1.5.70.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
-rclone_api-1.5.70.dist-info/METADATA,sha256=5DqhnAOnUh_TpQ-XqlSrmZv8imYRt0vqJeKsRqwt020,37305
-rclone_api-1.5.70.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
-rclone_api-1.5.70.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
-rclone_api-1.5.70.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
-rclone_api-1.5.70.dist-info/RECORD,,
+rclone_api-1.5.71.dist-info/licenses/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
+rclone_api-1.5.71.dist-info/METADATA,sha256=qnLOllDyan73gwf8Nb5W5B9vhYnRUP1fRhRrA-5c56Q,37305
+rclone_api-1.5.71.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
+rclone_api-1.5.71.dist-info/entry_points.txt,sha256=ognh2e11HTjn73_KL5MWI67pBKS2jekBi-QTiRXySXA,316
+rclone_api-1.5.71.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
+rclone_api-1.5.71.dist-info/RECORD,,