rclone-api 1.0.13__py2.py3-none-any.whl → 1.0.14__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
rclone_api/__init__.py CHANGED
@@ -1,9 +1,9 @@
1
+ from .config import Config
1
2
  from .dir import Dir
2
3
  from .dir_listing import DirListing
3
4
  from .file import File
4
5
  from .rclone import Rclone
5
6
  from .remote import Remote
6
7
  from .rpath import RPath
7
- from .types import Config
8
8
 
9
9
  __all__ = ["Rclone", "File", "Config", "Remote", "Dir", "RPath", "DirListing"]
rclone_api/config.py CHANGED
@@ -1,19 +1,8 @@
1
- import subprocess
2
1
  from dataclasses import dataclass
3
- from pathlib import Path
4
-
5
- from rclone_api.types import Config
6
2
 
7
3
 
8
4
  @dataclass
9
- class RcloneExec:
10
- """Rclone execution dataclass."""
11
-
12
- rclone_config: Path | Config
13
- rclone_exe: Path
14
-
15
- def execute(self, cmd: list[str]) -> subprocess.CompletedProcess:
16
- """Execute rclone command."""
17
- from rclone_api.util import rclone_execute
5
+ class Config:
6
+ """Rclone configuration dataclass."""
18
7
 
19
- return rclone_execute(cmd, self.rclone_config, self.rclone_exe)
8
+ text: str
rclone_api/convert.py ADDED
@@ -0,0 +1,31 @@
1
+ from rclone_api.dir import Dir
2
+ from rclone_api.file import File
3
+ from rclone_api.remote import Remote
4
+
5
+
6
+ def convert_to_filestr_list(files: str | File | list[str] | list[File]) -> list[str]:
7
+ out: list[str] = []
8
+ if isinstance(files, str):
9
+ out.append(files)
10
+ elif isinstance(files, File):
11
+ out.append(str(files.path))
12
+ elif isinstance(files, list):
13
+ for f in files:
14
+ if isinstance(f, File):
15
+ f = str(f.path)
16
+ out.append(f)
17
+ else:
18
+ raise ValueError(f"Invalid type for file: {type(files)}")
19
+ return out
20
+
21
+
22
+ def convert_to_str(file_or_dir: str | File | Dir | Remote) -> str:
23
+ if isinstance(file_or_dir, str):
24
+ return file_or_dir
25
+ if isinstance(file_or_dir, File):
26
+ return str(file_or_dir.path)
27
+ if isinstance(file_or_dir, Dir):
28
+ return str(file_or_dir.path)
29
+ if isinstance(file_or_dir, Remote):
30
+ return str(file_or_dir)
31
+ raise ValueError(f"Invalid type for file_or_dir: {type(file_or_dir)}")
rclone_api/dir.py CHANGED
@@ -1,3 +1,4 @@
1
+ import json
1
2
  from typing import Generator
2
3
 
3
4
  from rclone_api.dir_listing import DirListing
@@ -12,6 +13,10 @@ class Dir:
12
13
  def remote(self) -> Remote:
13
14
  return self.path.remote
14
15
 
16
+ @property
17
+ def name(self) -> str:
18
+ return self.path.name
19
+
15
20
  def __init__(self, path: RPath | Remote) -> None:
16
21
  """Initialize Dir with either an RPath or Remote.
17
22
 
@@ -33,6 +38,8 @@ class Dir:
33
38
  self.path.set_rclone(path.rclone)
34
39
  else:
35
40
  self.path = path
41
+ # self.path.set_rclone(self.path.remote.rclone)
42
+ assert self.path.rclone is not None
36
43
 
37
44
  def ls(self, max_depth: int = 0) -> DirListing:
38
45
  """List files and directories in the given path."""
@@ -47,5 +54,14 @@ class Dir:
47
54
  assert self.path.rclone is not None
48
55
  return walk(self, max_depth=max_depth)
49
56
 
57
+ def to_json(self) -> dict:
58
+ """Convert the Dir to a JSON serializable dictionary."""
59
+ return self.path.to_json()
60
+
50
61
  def __str__(self) -> str:
51
62
  return str(self.path)
63
+
64
+ def __repr__(self) -> str:
65
+ data = self.path.to_json()
66
+ data_str = json.dumps(data)
67
+ return data_str
rclone_api/dir_listing.py CHANGED
@@ -1,5 +1,5 @@
1
- # from rclone_api.dir import Dir
2
- # from rclone_api.file import File
1
+ import json
2
+
3
3
  from rclone_api.rpath import RPath
4
4
 
5
5
 
@@ -12,3 +12,29 @@ class DirListing:
12
12
 
13
13
  self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
14
14
  self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
15
+
16
+ def __str__(self) -> str:
17
+ n_files = len(self.files)
18
+ n_dirs = len(self.dirs)
19
+ msg = f"Files: {n_files}\n"
20
+ if n_files > 0:
21
+ for f in self.files:
22
+ msg += f" {f}\n"
23
+ msg += f"Dirs: {n_dirs}\n"
24
+ if n_dirs > 0:
25
+ for d in self.dirs:
26
+ msg += f" {d}\n"
27
+ return msg
28
+
29
+ def __repr__(self) -> str:
30
+ dirs: list = []
31
+ files: list = []
32
+ for d in self.dirs:
33
+ dirs.append(d.path.to_json())
34
+ for f in self.files:
35
+ files.append(f.path.to_json())
36
+ json_obj = {
37
+ "dirs": dirs,
38
+ "files": files,
39
+ }
40
+ return json.dumps(json_obj, indent=2)
@@ -2,12 +2,7 @@ import subprocess
2
2
  from dataclasses import dataclass
3
3
  from pathlib import Path
4
4
 
5
-
6
- @dataclass
7
- class Config:
8
- """Rclone configuration dataclass."""
9
-
10
- text: str
5
+ from rclone_api.config import Config
11
6
 
12
7
 
13
8
  @dataclass
rclone_api/file.py CHANGED
@@ -1,3 +1,5 @@
1
+ import json
2
+
1
3
  from rclone_api.rpath import RPath
2
4
 
3
5
 
@@ -10,6 +12,10 @@ class File:
10
12
  ) -> None:
11
13
  self.path = path
12
14
 
15
+ @property
16
+ def name(self) -> str:
17
+ return self.path.name
18
+
13
19
  def read_text(self) -> str:
14
20
  """Read the file contents as bytes.
15
21
 
@@ -28,5 +34,14 @@ class File:
28
34
  result = self.path.rclone._run(["cat", self.path.path])
29
35
  return result.stdout
30
36
 
37
+ def to_json(self) -> dict:
38
+ """Convert the File to a JSON serializable dictionary."""
39
+ return self.path.to_json()
40
+
31
41
  def __str__(self) -> str:
32
42
  return str(self.path)
43
+
44
+ def __repr__(self) -> str:
45
+ data = self.path.to_json()
46
+ data_str = json.dumps(data)
47
+ return data_str
rclone_api/rclone.py CHANGED
@@ -1,97 +1,214 @@
1
- """
2
- Unit test file.
3
- """
4
-
5
- import subprocess
6
- from pathlib import Path
7
- from typing import Generator
8
-
9
- from rclone_api import Dir
10
- from rclone_api.dir_listing import DirListing
11
- from rclone_api.remote import Remote
12
- from rclone_api.rpath import RPath
13
- from rclone_api.types import Config, RcloneExec
14
- from rclone_api.util import get_rclone_exe
15
- from rclone_api.walk import walk
16
-
17
-
18
- class Rclone:
19
- def __init__(
20
- self, rclone_conf: Path | Config, rclone_exe: Path | None = None
21
- ) -> None:
22
- if isinstance(rclone_conf, Path):
23
- if not rclone_conf.exists():
24
- raise ValueError(f"Rclone config file not found: {rclone_conf}")
25
- self._exec = RcloneExec(rclone_conf, get_rclone_exe(rclone_exe))
26
-
27
- def _run(self, cmd: list[str]) -> subprocess.CompletedProcess:
28
- return self._exec.execute(cmd)
29
-
30
- def ls(self, path: Dir | Remote | str, max_depth: int | None = None) -> DirListing:
31
- """List files in the given path.
32
-
33
- Args:
34
- path: Remote path or Remote object to list
35
- max_depth: Maximum recursion depth (0 means no recursion)
36
-
37
- Returns:
38
- List of File objects found at the path
39
- """
40
- cmd = ["lsjson"]
41
- if max_depth is not None:
42
- cmd.append("--recursive")
43
- if max_depth > -1:
44
- cmd.append("--max-depth")
45
- cmd.append(str(max_depth))
46
- cmd.append(str(path))
47
- remote = path.remote if isinstance(path, Dir) else path
48
- assert isinstance(remote, Remote)
49
-
50
- cp = self._run(cmd)
51
- text = cp.stdout
52
- paths: list[RPath] = RPath.from_json_str(text, remote)
53
- for o in paths:
54
- o.set_rclone(self)
55
- return DirListing(paths)
56
-
57
- def listremotes(self) -> list[Remote]:
58
- cmd = ["listremotes"]
59
- cp = self._run(cmd)
60
- text: str = cp.stdout
61
- tmp = text.splitlines()
62
- tmp = [t.strip() for t in tmp]
63
- # strip out ":" from the end
64
- tmp = [t.replace(":", "") for t in tmp]
65
- out = [Remote(name=t, rclone=self) for t in tmp]
66
- return out
67
-
68
- def walk(
69
- self, path: Dir | Remote, max_depth: int = -1
70
- ) -> Generator[DirListing, None, None]:
71
- """Walk through the given path recursively.
72
-
73
- Args:
74
- path: Remote path or Remote object to walk through
75
- max_depth: Maximum depth to traverse (-1 for unlimited)
76
-
77
- Yields:
78
- DirListing: Directory listing for each directory encountered
79
- """
80
- if isinstance(path, Dir):
81
- # Create a Remote object for the path
82
- remote = path.remote
83
- rpath = RPath(
84
- remote=remote,
85
- path=path.path.path,
86
- name=path.path.name,
87
- size=0,
88
- mime_type="inode/directory",
89
- mod_time="",
90
- is_dir=True,
91
- )
92
- rpath.set_rclone(self)
93
- dir_obj = Dir(rpath)
94
- else:
95
- dir_obj = Dir(path)
96
-
97
- yield from walk(dir_obj, max_depth=max_depth)
1
+ """
2
+ Unit test file.
3
+ """
4
+
5
+ import subprocess
6
+ from concurrent.futures import ThreadPoolExecutor
7
+ from fnmatch import fnmatch
8
+ from pathlib import Path
9
+ from typing import Generator
10
+
11
+ from rclone_api import Dir
12
+ from rclone_api.config import Config
13
+ from rclone_api.convert import convert_to_filestr_list, convert_to_str
14
+ from rclone_api.dir_listing import DirListing
15
+ from rclone_api.exec import RcloneExec
16
+ from rclone_api.file import File
17
+ from rclone_api.remote import Remote
18
+ from rclone_api.rpath import RPath
19
+ from rclone_api.util import get_rclone_exe, to_path
20
+ from rclone_api.walk import walk
21
+
22
+
23
+ class Rclone:
24
+ def __init__(
25
+ self, rclone_conf: Path | Config, rclone_exe: Path | None = None
26
+ ) -> None:
27
+ if isinstance(rclone_conf, Path):
28
+ if not rclone_conf.exists():
29
+ raise ValueError(f"Rclone config file not found: {rclone_conf}")
30
+ self._exec = RcloneExec(rclone_conf, get_rclone_exe(rclone_exe))
31
+
32
+ def _run(self, cmd: list[str]) -> subprocess.CompletedProcess:
33
+ return self._exec.execute(cmd)
34
+
35
+ def ls(
36
+ self,
37
+ path: Dir | Remote | str,
38
+ max_depth: int | None = None,
39
+ glob: str | None = None,
40
+ ) -> DirListing:
41
+ """List files in the given path.
42
+
43
+ Args:
44
+ path: Remote path or Remote object to list
45
+ max_depth: Maximum recursion depth (0 means no recursion)
46
+
47
+ Returns:
48
+ List of File objects found at the path
49
+ """
50
+
51
+ if isinstance(path, str):
52
+ path = Dir(
53
+ to_path(path, self)
54
+ ) # assume it's a directory if ls is being called.
55
+
56
+ cmd = ["lsjson"]
57
+ if max_depth is not None:
58
+ cmd.append("--recursive")
59
+ if max_depth > -1:
60
+ cmd.append("--max-depth")
61
+ cmd.append(str(max_depth))
62
+ cmd.append(str(path))
63
+ remote = path.remote if isinstance(path, Dir) else path
64
+ assert isinstance(remote, Remote)
65
+
66
+ cp = self._run(cmd)
67
+ text = cp.stdout
68
+ parent_path: str | None = None
69
+ if isinstance(path, Dir):
70
+ parent_path = path.path.path
71
+ paths: list[RPath] = RPath.from_json_str(text, remote, parent_path=parent_path)
72
+ # print(parent_path)
73
+ for o in paths:
74
+ o.set_rclone(self)
75
+
76
+ # do we have a glob pattern?
77
+ if glob is not None:
78
+ paths = [p for p in paths if fnmatch(p.path, glob)]
79
+ return DirListing(paths)
80
+
81
+ def listremotes(self) -> list[Remote]:
82
+ cmd = ["listremotes"]
83
+ cp = self._run(cmd)
84
+ text: str = cp.stdout
85
+ tmp = text.splitlines()
86
+ tmp = [t.strip() for t in tmp]
87
+ # strip out ":" from the end
88
+ tmp = [t.replace(":", "") for t in tmp]
89
+ out = [Remote(name=t, rclone=self) for t in tmp]
90
+ return out
91
+
92
+ def walk(
93
+ self, path: Dir | Remote | str, max_depth: int = -1
94
+ ) -> Generator[DirListing, None, None]:
95
+ """Walk through the given path recursively.
96
+
97
+ Args:
98
+ path: Remote path or Remote object to walk through
99
+ max_depth: Maximum depth to traverse (-1 for unlimited)
100
+
101
+ Yields:
102
+ DirListing: Directory listing for each directory encountered
103
+ """
104
+ if isinstance(path, Dir):
105
+ # Create a Remote object for the path
106
+ remote = path.remote
107
+ rpath = RPath(
108
+ remote=remote,
109
+ path=path.path.path,
110
+ name=path.path.name,
111
+ size=0,
112
+ mime_type="inode/directory",
113
+ mod_time="",
114
+ is_dir=True,
115
+ )
116
+ rpath.set_rclone(self)
117
+ dir_obj = Dir(rpath)
118
+ elif isinstance(path, str):
119
+ dir_obj = Dir(to_path(path, self))
120
+ elif isinstance(path, Remote):
121
+ dir_obj = Dir(path)
122
+ else:
123
+ assert f"Invalid type for path: {type(path)}"
124
+
125
+ yield from walk(dir_obj, max_depth=max_depth)
126
+
127
+ def copyfile(self, src: File | str, dst: File | str) -> None:
128
+ """Copy a single file from source to destination.
129
+
130
+ Args:
131
+ src: Source file path (including remote if applicable)
132
+ dst: Destination file path (including remote if applicable)
133
+
134
+ Raises:
135
+ subprocess.CalledProcessError: If the copy operation fails
136
+ """
137
+ src = src if isinstance(src, str) else str(src.path)
138
+ dst = dst if isinstance(dst, str) else str(dst.path)
139
+ cmd_list: list[str] = ["copyto", src, dst]
140
+ self._run(cmd_list)
141
+
142
+ def copyfiles(self, filelist: dict[File, File] | dict[str, str]) -> None:
143
+ """Copy multiple files from source to destination.
144
+
145
+ Warning - slow.
146
+
147
+ Args:
148
+ payload: Dictionary of source and destination file paths
149
+ """
150
+ str_dict: dict[str, str] = {}
151
+ for src, dst in filelist.items():
152
+ src = src if isinstance(src, str) else str(src.path)
153
+ dst = dst if isinstance(dst, str) else str(dst.path)
154
+ str_dict[src] = dst
155
+
156
+ with ThreadPoolExecutor(max_workers=64) as executor:
157
+ for src, dst in str_dict.items(): # warning - slow
158
+ cmd_list: list[str] = ["copyto", src, dst]
159
+ # self._run(cmd_list)
160
+ executor.submit(self._run, cmd_list)
161
+
162
+ def copy(self, src: Dir, dst: Dir) -> None:
163
+ """Copy files from source to destination.
164
+
165
+ Args:
166
+ src: Source directory
167
+ dst: Destination directory
168
+ """
169
+ src_dir = src.path.path
170
+ dst_dir = dst.path.path
171
+ cmd_list: list[str] = ["copy", src_dir, dst_dir]
172
+ self._run(cmd_list)
173
+
174
+ def purge(self, path: Dir | str) -> None:
175
+ """Purge a directory"""
176
+ # path should always be a string
177
+ path = path if isinstance(path, str) else str(path.path)
178
+ cmd_list: list[str] = ["purge", str(path)]
179
+ self._run(cmd_list)
180
+
181
+ def deletefiles(self, files: str | File | list[str] | list[File]) -> None:
182
+ """Delete a directory"""
183
+ payload: list[str] = convert_to_filestr_list(files)
184
+ cmd_list: list[str] = ["delete"] + payload
185
+ self._run(cmd_list)
186
+
187
+ def exists(self, path: Dir | Remote | str | File) -> bool:
188
+ """Check if a file or directory exists."""
189
+ arg: str = convert_to_str(path)
190
+ assert isinstance(arg, str)
191
+ try:
192
+ self.ls(arg)
193
+ return True
194
+ except subprocess.CalledProcessError:
195
+ return False
196
+
197
+ def is_synced(self, src: str | Dir, dst: str | Dir) -> bool:
198
+ """Check if two directories are in sync."""
199
+ src = convert_to_str(src)
200
+ dst = convert_to_str(dst)
201
+ cmd_list: list[str] = ["check", str(src), str(dst)]
202
+ try:
203
+ self._run(cmd_list)
204
+ return True
205
+ except subprocess.CalledProcessError:
206
+ return False
207
+
208
+ def copy_dir(self, src: str | Dir, dst: str | Dir) -> None:
209
+ """Copy a directory from source to destination."""
210
+ # convert src to str, also dst
211
+ src = convert_to_str(src)
212
+ dst = convert_to_str(dst)
213
+ cmd_list: list[str] = ["copy", src, dst]
214
+ self._run(cmd_list)
rclone_api/rpath.py CHANGED
@@ -19,9 +19,6 @@ class RPath:
19
19
  ) -> None:
20
20
  from rclone_api.rclone import Rclone
21
21
 
22
- if "dst:" in path:
23
- raise ValueError(f"Invalid path: {path}")
24
-
25
22
  self.remote = remote
26
23
  self.path = path
27
24
  self.name = name
@@ -39,11 +36,16 @@ class RPath:
39
36
  self.rclone = rclone
40
37
 
41
38
  @staticmethod
42
- def from_dict(data: dict, remote: Remote) -> "RPath":
39
+ def from_dict(
40
+ data: dict, remote: Remote, parent_path: str | None = None
41
+ ) -> "RPath":
43
42
  """Create a File from a dictionary."""
43
+ path = data["Path"]
44
+ if parent_path is not None:
45
+ path = f"{parent_path}/{path}"
44
46
  return RPath(
45
47
  remote,
46
- data["Path"],
48
+ path,
47
49
  data["Name"],
48
50
  data["Size"],
49
51
  data["MimeType"],
@@ -53,21 +55,25 @@ class RPath:
53
55
  )
54
56
 
55
57
  @staticmethod
56
- def from_array(data: list[dict], remote: Remote) -> list["RPath"]:
58
+ def from_array(
59
+ data: list[dict], remote: Remote, parent_path: str | None = None
60
+ ) -> list["RPath"]:
57
61
  """Create a File from a dictionary."""
58
62
  out: list[RPath] = []
59
63
  for d in data:
60
- file: RPath = RPath.from_dict(d, remote)
64
+ file: RPath = RPath.from_dict(d, remote, parent_path)
61
65
  out.append(file)
62
66
  return out
63
67
 
64
68
  @staticmethod
65
- def from_json_str(json_str: str, remote: Remote) -> list["RPath"]:
69
+ def from_json_str(
70
+ json_str: str, remote: Remote, parent_path: str | None = None
71
+ ) -> list["RPath"]:
66
72
  """Create a File from a JSON string."""
67
73
  json_obj = json.loads(json_str)
68
74
  if isinstance(json_obj, dict):
69
- return [RPath.from_dict(json_obj, remote)]
70
- return RPath.from_array(json_obj, remote)
75
+ return [RPath.from_dict(json_obj, remote, parent_path)]
76
+ return RPath.from_array(json_obj, remote, parent_path)
71
77
 
72
78
  def to_json(self) -> dict:
73
79
  return {
@@ -82,3 +88,8 @@ class RPath:
82
88
 
83
89
  def __str__(self) -> str:
84
90
  return f"{self.remote.name}:{self.path}"
91
+
92
+ def __repr__(self):
93
+ data = self.to_json()
94
+ data["Remote"] = self.remote.name
95
+ return json.dumps(data)
rclone_api/util.py CHANGED
@@ -5,10 +5,10 @@ from pathlib import Path
5
5
  from tempfile import TemporaryDirectory
6
6
  from typing import Any
7
7
 
8
+ from rclone_api.config import Config
8
9
  from rclone_api.dir import Dir
9
10
  from rclone_api.remote import Remote
10
11
  from rclone_api.rpath import RPath
11
- from rclone_api.types import Config
12
12
 
13
13
  # from .rclone import Rclone
14
14
 
@@ -25,7 +25,7 @@ def to_path(item: Dir | Remote | str, rclone: Any) -> RPath:
25
25
  remote_name = parts[0]
26
26
  path = ":".join(parts[1:])
27
27
  remote = Remote(name=remote_name, rclone=rclone)
28
- return RPath(
28
+ out = RPath(
29
29
  remote=remote,
30
30
  path=path,
31
31
  name="",
@@ -34,10 +34,12 @@ def to_path(item: Dir | Remote | str, rclone: Any) -> RPath:
34
34
  mod_time="",
35
35
  is_dir=True,
36
36
  )
37
+ out.set_rclone(rclone)
38
+ return out
37
39
  elif isinstance(item, Dir):
38
40
  return item.path
39
41
  elif isinstance(item, Remote):
40
- return RPath(
42
+ out = RPath(
41
43
  remote=item,
42
44
  path=str(item),
43
45
  name=str(item),
@@ -46,6 +48,8 @@ def to_path(item: Dir | Remote | str, rclone: Any) -> RPath:
46
48
  mod_time="",
47
49
  is_dir=True,
48
50
  )
51
+ out.set_rclone(rclone)
52
+ return out
49
53
  else:
50
54
  raise ValueError(f"Invalid type for item: {type(item)}")
51
55
 
@@ -90,8 +94,16 @@ def rclone_execute(
90
94
  cmd_str = subprocess.list2cmdline(cmd)
91
95
  print(f"Running: {cmd_str}")
92
96
  cp = subprocess.run(
93
- cmd, capture_output=True, encoding="utf-8", check=True, shell=False
97
+ cmd, capture_output=True, encoding="utf-8", check=False, shell=False
94
98
  )
99
+ if cp.returncode != 0:
100
+ cmd_str = subprocess.list2cmdline(cmd)
101
+ print(
102
+ f"Error running: {cmd_str}, returncode: {cp.returncode}\n{cp.stdout}\n{cp.stderr}"
103
+ )
104
+ raise subprocess.CalledProcessError(
105
+ cp.returncode, cmd, cp.stdout, cp.stderr
106
+ )
95
107
  return cp
96
108
  finally:
97
109
  if tempdir:
rclone_api/walk.py CHANGED
@@ -1,87 +1,70 @@
1
- from concurrent.futures import ThreadPoolExecutor
2
- from queue import Empty, Queue
1
+ from queue import Queue
2
+ from threading import Thread
3
3
  from typing import Generator
4
4
 
5
5
  from rclone_api import Dir
6
6
  from rclone_api.dir_listing import DirListing
7
7
  from rclone_api.remote import Remote
8
8
 
9
+ _MAX_OUT_QUEUE_SIZE = 50
9
10
 
10
- def walk(
11
- dir: Dir | Remote, max_depth: int = -1, max_workers: int = 4
12
- ) -> Generator[DirListing, None, None]:
11
+
12
+ def _walk_runner(
13
+ queue: Queue[Dir], max_depth: int, out_queue: Queue[DirListing | None]
14
+ ) -> None:
15
+ try:
16
+ while not queue.empty():
17
+ current_dir = queue.get()
18
+ dirlisting = current_dir.ls()
19
+ out_queue.put(dirlisting)
20
+ dirs = dirlisting.dirs
21
+
22
+ if max_depth != 0 and len(dirs) > 0:
23
+ for child in dirs:
24
+ queue.put(child)
25
+ if max_depth < 0:
26
+ continue
27
+ if max_depth > 0:
28
+ max_depth -= 1
29
+ out_queue.put(None)
30
+ except KeyboardInterrupt:
31
+ import _thread
32
+
33
+ out_queue.put(None)
34
+
35
+ _thread.interrupt_main()
36
+
37
+
38
+ def walk(dir: Dir | Remote, max_depth: int = -1) -> Generator[DirListing, None, None]:
13
39
  """Walk through the given directory recursively.
14
40
 
15
41
  Args:
16
42
  dir: Directory or Remote to walk through
17
43
  max_depth: Maximum depth to traverse (-1 for unlimited)
18
- max_workers: Maximum number of concurrent workers
19
44
 
20
45
  Yields:
21
46
  DirListing: Directory listing for each directory encountered
22
47
  """
23
- # Convert Remote to Dir if needed
24
- if isinstance(dir, Remote):
25
- dir = Dir(dir)
26
- pending: Queue[tuple[Dir | None, int]] = Queue()
27
- results: Queue[DirListing | Exception] = Queue()
48
+ try:
49
+ # Convert Remote to Dir if needed
50
+ if isinstance(dir, Remote):
51
+ dir = Dir(dir)
28
52
 
29
- def worker():
30
- while True:
31
- try:
32
- # Add timeout to allow checking for sentinel value
33
- try:
34
- current_dir, depth = pending.get(timeout=0.1)
35
- except Empty:
36
- continue
53
+ in_queue: Queue[Dir] = Queue()
54
+ out_queue: Queue[DirListing] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
55
+ in_queue.put(dir)
37
56
 
38
- # Check for sentinel value
39
- if current_dir is None:
40
- pending.task_done()
41
- break
57
+ # Start worker thread
58
+ worker = Thread(
59
+ target=_walk_runner, args=(in_queue, max_depth, out_queue), daemon=True
60
+ )
61
+ worker.start()
42
62
 
43
- listing = current_dir.ls()
44
- results.put(listing)
45
-
46
- if max_depth == -1 or depth < max_depth:
47
- for d in listing.dirs:
48
- pending.put((d, depth + 1))
49
-
50
- pending.task_done()
51
- except Exception as e:
52
- results.put(e)
53
- pending.task_done()
63
+ while dirlisting := out_queue.get():
64
+ if dirlisting is None:
54
65
  break
55
- return None
56
-
57
- # Start workers
58
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
59
- workers = [executor.submit(worker) for _ in range(max_workers)]
60
-
61
- # Start walking
62
- pending.put((dir, 0))
63
-
64
- # Process results while workers are running
65
- completed = 0
66
- while completed < max_workers:
67
- try:
68
- result = results.get(timeout=0.1)
69
- if isinstance(result, Exception):
70
- # Propagate exception
71
- raise result
72
- yield result
73
- except Empty:
74
- # Check if any workers have completed
75
- completed = sum(1 for w in workers if w.done())
76
- continue
77
-
78
- # Signal workers to stop
79
- for _ in range(max_workers):
80
- pending.put((None, 0))
66
+ yield dirlisting
81
67
 
82
- # Drain any remaining results
83
- while not results.empty():
84
- result = results.get()
85
- if isinstance(result, Exception):
86
- raise result
87
- yield result
68
+ worker.join()
69
+ except KeyboardInterrupt:
70
+ pass
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: rclone_api
3
- Version: 1.0.13
3
+ Version: 1.0.14
4
4
  Summary: rclone api in python
5
5
  Home-page: https://github.com/zackees/rclone-api
6
6
  Maintainer: Zachary Vorhies
@@ -0,0 +1,19 @@
1
+ rclone_api/__init__.py,sha256=gca3IDhDdKBtyoS1ekGQ-xTa6zrShZ6Gb4XS8ChdlYo,266
2
+ rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
3
+ rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
4
+ rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
5
+ rclone_api/dir.py,sha256=xUV4i9_E7QHrqHld2IPBlon1CGaAp8qQYMKfJTyVvoQ,2088
6
+ rclone_api/dir_listing.py,sha256=8t5Jx9ZVOJPqGKTJbWaES6bjgogUT2bnpbPVWwK1Fcs,1124
7
+ rclone_api/exec.py,sha256=PgiNF5eQpsY8Pb62ZPwLli1WA9iMEJeMZwX43VEYKII,472
8
+ rclone_api/file.py,sha256=bb9FWQy54wKRydy0M12MQwNxycBwRSn_10ubYGAw0ag,1234
9
+ rclone_api/rclone.py,sha256=m2V3BQX0buK3mwImBiTSNTUh_JcdQJ27OeJWxqvgRZs,7455
10
+ rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
11
+ rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
12
+ rclone_api/util.py,sha256=uiTfOlKCFB5FdUkYR2-Hriz0fVyrWeuZtSgHgc9RrpA,3328
13
+ rclone_api/walk.py,sha256=J78-bY2AhNpt2ICsI5LqmXRE7oC6wVDoKoicIoU6XMg,1953
14
+ rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
15
+ rclone_api-1.0.14.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
16
+ rclone_api-1.0.14.dist-info/METADATA,sha256=pATg2JeY8MQ2I7kywlJj6x4KPdL0QKKcbxjxbvq1OCs,1375
17
+ rclone_api-1.0.14.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
18
+ rclone_api-1.0.14.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
19
+ rclone_api-1.0.14.dist-info/RECORD,,
@@ -1,18 +0,0 @@
1
- rclone_api/__init__.py,sha256=YH7KQaPwUiJWJiRf0NRKD7XHhMXsxWdXDjt9WLSwdjA,265
2
- rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
3
- rclone_api/config.py,sha256=UujjDNcpwgipgdyFPMhC3zikvlsrurUvvZiwXm5NlPg,471
4
- rclone_api/dir.py,sha256=tu0-Yy1s3O1MVY7Nkb6NOh6t9dXJ9DQvqOZVlWIexrI,1629
5
- rclone_api/dir_listing.py,sha256=NWleKHCCRW7_eh9JfRwE6r3QbjmiHD5ZEGQcd2vm4uY,458
6
- rclone_api/file.py,sha256=nJwJT7v2KwY_8g7UB3Y7O6pFfHn3nri9dJ3GS-cN3EE,874
7
- rclone_api/rclone.py,sha256=upMqUxYKC3LCjHlhI66_IdOD1t5PKo7V64sSdw0vAlk,3074
8
- rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
9
- rclone_api/rpath.py,sha256=QHD5zDjDIkivs3kCRYtTL8mP-DspXIvBVYWHSf_Y6Rg,2263
10
- rclone_api/types.py,sha256=Yp15HrjwZonSQhE323R0WP7fA4NWqKjAfM7z3OwHpWI,518
11
- rclone_api/util.py,sha256=xYUj0B9W5HDauTHh7Z8cnEQsTEuXvRcVNLtk3ZanlGM,2894
12
- rclone_api/walk.py,sha256=3GKnu9P5aPNiftfoi52U7wo1wixZxwxdqqS0_IlMGCE,2816
13
- rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
14
- rclone_api-1.0.13.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
15
- rclone_api-1.0.13.dist-info/METADATA,sha256=yEL4GNzAc-CwJt-rJVmDAl08L13RqytSSs1ZQR4KMA0,1375
16
- rclone_api-1.0.13.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
17
- rclone_api-1.0.13.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
18
- rclone_api-1.0.13.dist-info/RECORD,,