rclone-api 1.0.12__py2.py3-none-any.whl → 1.0.14__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
rclone_api/__init__.py CHANGED
@@ -1,9 +1,9 @@
1
+ from .config import Config
1
2
  from .dir import Dir
2
3
  from .dir_listing import DirListing
3
4
  from .file import File
4
5
  from .rclone import Rclone
5
6
  from .remote import Remote
6
7
  from .rpath import RPath
7
- from .types import Config
8
8
 
9
9
  __all__ = ["Rclone", "File", "Config", "Remote", "Dir", "RPath", "DirListing"]
rclone_api/config.py CHANGED
@@ -1,19 +1,8 @@
1
- import subprocess
2
1
  from dataclasses import dataclass
3
- from pathlib import Path
4
-
5
- from rclone_api.types import Config
6
2
 
7
3
 
8
4
  @dataclass
9
- class RcloneExec:
10
- """Rclone execution dataclass."""
11
-
12
- rclone_config: Path | Config
13
- rclone_exe: Path
14
-
15
- def execute(self, cmd: list[str]) -> subprocess.CompletedProcess:
16
- """Execute rclone command."""
17
- from rclone_api.util import rclone_execute
5
+ class Config:
6
+ """Rclone configuration dataclass."""
18
7
 
19
- return rclone_execute(cmd, self.rclone_config, self.rclone_exe)
8
+ text: str
rclone_api/convert.py ADDED
@@ -0,0 +1,31 @@
1
+ from rclone_api.dir import Dir
2
+ from rclone_api.file import File
3
+ from rclone_api.remote import Remote
4
+
5
+
6
+ def convert_to_filestr_list(files: str | File | list[str] | list[File]) -> list[str]:
7
+ out: list[str] = []
8
+ if isinstance(files, str):
9
+ out.append(files)
10
+ elif isinstance(files, File):
11
+ out.append(str(files.path))
12
+ elif isinstance(files, list):
13
+ for f in files:
14
+ if isinstance(f, File):
15
+ f = str(f.path)
16
+ out.append(f)
17
+ else:
18
+ raise ValueError(f"Invalid type for file: {type(files)}")
19
+ return out
20
+
21
+
22
+ def convert_to_str(file_or_dir: str | File | Dir | Remote) -> str:
23
+ if isinstance(file_or_dir, str):
24
+ return file_or_dir
25
+ if isinstance(file_or_dir, File):
26
+ return str(file_or_dir.path)
27
+ if isinstance(file_or_dir, Dir):
28
+ return str(file_or_dir.path)
29
+ if isinstance(file_or_dir, Remote):
30
+ return str(file_or_dir)
31
+ raise ValueError(f"Invalid type for file_or_dir: {type(file_or_dir)}")
rclone_api/dir.py CHANGED
@@ -1,3 +1,4 @@
1
+ import json
1
2
  from typing import Generator
2
3
 
3
4
  from rclone_api.dir_listing import DirListing
@@ -8,6 +9,14 @@ from rclone_api.rpath import RPath
8
9
  class Dir:
9
10
  """Remote file dataclass."""
10
11
 
12
+ @property
13
+ def remote(self) -> Remote:
14
+ return self.path.remote
15
+
16
+ @property
17
+ def name(self) -> str:
18
+ return self.path.name
19
+
11
20
  def __init__(self, path: RPath | Remote) -> None:
12
21
  """Initialize Dir with either an RPath or Remote.
13
22
 
@@ -17,6 +26,7 @@ class Dir:
17
26
  if isinstance(path, Remote):
18
27
  # Need to create an RPath for the Remote's root
19
28
  self.path = RPath(
29
+ remote=path,
20
30
  path=str(path),
21
31
  name=str(path),
22
32
  size=0,
@@ -28,11 +38,14 @@ class Dir:
28
38
  self.path.set_rclone(path.rclone)
29
39
  else:
30
40
  self.path = path
41
+ # self.path.set_rclone(self.path.remote.rclone)
42
+ assert self.path.rclone is not None
31
43
 
32
44
  def ls(self, max_depth: int = 0) -> DirListing:
33
45
  """List files and directories in the given path."""
34
46
  assert self.path.rclone is not None
35
- return self.path.rclone.ls(self.path.path, max_depth=max_depth)
47
+ dir = Dir(self.path)
48
+ return self.path.rclone.ls(dir, max_depth=max_depth)
36
49
 
37
50
  def walk(self, max_depth: int = -1) -> Generator[DirListing, None, None]:
38
51
  """List files and directories in the given path."""
@@ -41,5 +54,14 @@ class Dir:
41
54
  assert self.path.rclone is not None
42
55
  return walk(self, max_depth=max_depth)
43
56
 
57
+ def to_json(self) -> dict:
58
+ """Convert the Dir to a JSON serializable dictionary."""
59
+ return self.path.to_json()
60
+
44
61
  def __str__(self) -> str:
45
62
  return str(self.path)
63
+
64
+ def __repr__(self) -> str:
65
+ data = self.path.to_json()
66
+ data_str = json.dumps(data)
67
+ return data_str
rclone_api/dir_listing.py CHANGED
@@ -1,5 +1,5 @@
1
- # from rclone_api.dir import Dir
2
- # from rclone_api.file import File
1
+ import json
2
+
3
3
  from rclone_api.rpath import RPath
4
4
 
5
5
 
@@ -12,3 +12,29 @@ class DirListing:
12
12
 
13
13
  self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
14
14
  self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
15
+
16
+ def __str__(self) -> str:
17
+ n_files = len(self.files)
18
+ n_dirs = len(self.dirs)
19
+ msg = f"Files: {n_files}\n"
20
+ if n_files > 0:
21
+ for f in self.files:
22
+ msg += f" {f}\n"
23
+ msg += f"Dirs: {n_dirs}\n"
24
+ if n_dirs > 0:
25
+ for d in self.dirs:
26
+ msg += f" {d}\n"
27
+ return msg
28
+
29
+ def __repr__(self) -> str:
30
+ dirs: list = []
31
+ files: list = []
32
+ for d in self.dirs:
33
+ dirs.append(d.path.to_json())
34
+ for f in self.files:
35
+ files.append(f.path.to_json())
36
+ json_obj = {
37
+ "dirs": dirs,
38
+ "files": files,
39
+ }
40
+ return json.dumps(json_obj, indent=2)
@@ -2,12 +2,7 @@ import subprocess
2
2
  from dataclasses import dataclass
3
3
  from pathlib import Path
4
4
 
5
-
6
- @dataclass
7
- class Config:
8
- """Rclone configuration dataclass."""
9
-
10
- text: str
5
+ from rclone_api.config import Config
11
6
 
12
7
 
13
8
  @dataclass
rclone_api/file.py CHANGED
@@ -12,6 +12,10 @@ class File:
12
12
  ) -> None:
13
13
  self.path = path
14
14
 
15
+ @property
16
+ def name(self) -> str:
17
+ return self.path.name
18
+
15
19
  def read_text(self) -> str:
16
20
  """Read the file contents as bytes.
17
21
 
@@ -30,6 +34,14 @@ class File:
30
34
  result = self.path.rclone._run(["cat", self.path.path])
31
35
  return result.stdout
32
36
 
37
+ def to_json(self) -> dict:
38
+ """Convert the File to a JSON serializable dictionary."""
39
+ return self.path.to_json()
40
+
33
41
  def __str__(self) -> str:
34
- out = self.path.to_json()
35
- return json.dumps(out)
42
+ return str(self.path)
43
+
44
+ def __repr__(self) -> str:
45
+ data = self.path.to_json()
46
+ data_str = json.dumps(data)
47
+ return data_str
rclone_api/rclone.py CHANGED
@@ -3,15 +3,20 @@ Unit test file.
3
3
  """
4
4
 
5
5
  import subprocess
6
+ from concurrent.futures import ThreadPoolExecutor
7
+ from fnmatch import fnmatch
6
8
  from pathlib import Path
7
9
  from typing import Generator
8
10
 
9
11
  from rclone_api import Dir
12
+ from rclone_api.config import Config
13
+ from rclone_api.convert import convert_to_filestr_list, convert_to_str
10
14
  from rclone_api.dir_listing import DirListing
15
+ from rclone_api.exec import RcloneExec
16
+ from rclone_api.file import File
11
17
  from rclone_api.remote import Remote
12
18
  from rclone_api.rpath import RPath
13
- from rclone_api.types import Config, RcloneExec
14
- from rclone_api.util import get_rclone_exe
19
+ from rclone_api.util import get_rclone_exe, to_path
15
20
  from rclone_api.walk import walk
16
21
 
17
22
 
@@ -27,7 +32,12 @@ class Rclone:
27
32
  def _run(self, cmd: list[str]) -> subprocess.CompletedProcess:
28
33
  return self._exec.execute(cmd)
29
34
 
30
- def ls(self, path: str | Remote, max_depth: int = 0) -> DirListing:
35
+ def ls(
36
+ self,
37
+ path: Dir | Remote | str,
38
+ max_depth: int | None = None,
39
+ glob: str | None = None,
40
+ ) -> DirListing:
31
41
  """List files in the given path.
32
42
 
33
43
  Args:
@@ -37,16 +47,35 @@ class Rclone:
37
47
  Returns:
38
48
  List of File objects found at the path
39
49
  """
50
+
51
+ if isinstance(path, str):
52
+ path = Dir(
53
+ to_path(path, self)
54
+ ) # assume it's a directory if ls is being called.
55
+
40
56
  cmd = ["lsjson"]
41
- if max_depth > 0:
42
- cmd.extend(["--recursive", "--max-depth", str(max_depth)])
57
+ if max_depth is not None:
58
+ cmd.append("--recursive")
59
+ if max_depth > -1:
60
+ cmd.append("--max-depth")
61
+ cmd.append(str(max_depth))
43
62
  cmd.append(str(path))
63
+ remote = path.remote if isinstance(path, Dir) else path
64
+ assert isinstance(remote, Remote)
44
65
 
45
66
  cp = self._run(cmd)
46
67
  text = cp.stdout
47
- paths: list[RPath] = RPath.from_json_str(text)
68
+ parent_path: str | None = None
69
+ if isinstance(path, Dir):
70
+ parent_path = path.path.path
71
+ paths: list[RPath] = RPath.from_json_str(text, remote, parent_path=parent_path)
72
+ # print(parent_path)
48
73
  for o in paths:
49
74
  o.set_rclone(self)
75
+
76
+ # do we have a glob pattern?
77
+ if glob is not None:
78
+ paths = [p for p in paths if fnmatch(p.path, glob)]
50
79
  return DirListing(paths)
51
80
 
52
81
  def listremotes(self) -> list[Remote]:
@@ -61,7 +90,7 @@ class Rclone:
61
90
  return out
62
91
 
63
92
  def walk(
64
- self, path: str | Remote, max_depth: int = -1
93
+ self, path: Dir | Remote | str, max_depth: int = -1
65
94
  ) -> Generator[DirListing, None, None]:
66
95
  """Walk through the given path recursively.
67
96
 
@@ -72,11 +101,13 @@ class Rclone:
72
101
  Yields:
73
102
  DirListing: Directory listing for each directory encountered
74
103
  """
75
- if isinstance(path, str):
104
+ if isinstance(path, Dir):
76
105
  # Create a Remote object for the path
106
+ remote = path.remote
77
107
  rpath = RPath(
78
- path=path,
79
- name=path,
108
+ remote=remote,
109
+ path=path.path.path,
110
+ name=path.path.name,
80
111
  size=0,
81
112
  mime_type="inode/directory",
82
113
  mod_time="",
@@ -84,7 +115,100 @@ class Rclone:
84
115
  )
85
116
  rpath.set_rclone(self)
86
117
  dir_obj = Dir(rpath)
87
- else:
118
+ elif isinstance(path, str):
119
+ dir_obj = Dir(to_path(path, self))
120
+ elif isinstance(path, Remote):
88
121
  dir_obj = Dir(path)
122
+ else:
123
+ assert f"Invalid type for path: {type(path)}"
89
124
 
90
125
  yield from walk(dir_obj, max_depth=max_depth)
126
+
127
+ def copyfile(self, src: File | str, dst: File | str) -> None:
128
+ """Copy a single file from source to destination.
129
+
130
+ Args:
131
+ src: Source file path (including remote if applicable)
132
+ dst: Destination file path (including remote if applicable)
133
+
134
+ Raises:
135
+ subprocess.CalledProcessError: If the copy operation fails
136
+ """
137
+ src = src if isinstance(src, str) else str(src.path)
138
+ dst = dst if isinstance(dst, str) else str(dst.path)
139
+ cmd_list: list[str] = ["copyto", src, dst]
140
+ self._run(cmd_list)
141
+
142
+ def copyfiles(self, filelist: dict[File, File] | dict[str, str]) -> None:
143
+ """Copy multiple files from source to destination.
144
+
145
+ Warning - slow.
146
+
147
+ Args:
148
+ payload: Dictionary of source and destination file paths
149
+ """
150
+ str_dict: dict[str, str] = {}
151
+ for src, dst in filelist.items():
152
+ src = src if isinstance(src, str) else str(src.path)
153
+ dst = dst if isinstance(dst, str) else str(dst.path)
154
+ str_dict[src] = dst
155
+
156
+ with ThreadPoolExecutor(max_workers=64) as executor:
157
+ for src, dst in str_dict.items(): # warning - slow
158
+ cmd_list: list[str] = ["copyto", src, dst]
159
+ # self._run(cmd_list)
160
+ executor.submit(self._run, cmd_list)
161
+
162
+ def copy(self, src: Dir, dst: Dir) -> None:
163
+ """Copy files from source to destination.
164
+
165
+ Args:
166
+ src: Source directory
167
+ dst: Destination directory
168
+ """
169
+ src_dir = src.path.path
170
+ dst_dir = dst.path.path
171
+ cmd_list: list[str] = ["copy", src_dir, dst_dir]
172
+ self._run(cmd_list)
173
+
174
+ def purge(self, path: Dir | str) -> None:
175
+ """Purge a directory"""
176
+ # path should always be a string
177
+ path = path if isinstance(path, str) else str(path.path)
178
+ cmd_list: list[str] = ["purge", str(path)]
179
+ self._run(cmd_list)
180
+
181
+ def deletefiles(self, files: str | File | list[str] | list[File]) -> None:
182
+ """Delete a directory"""
183
+ payload: list[str] = convert_to_filestr_list(files)
184
+ cmd_list: list[str] = ["delete"] + payload
185
+ self._run(cmd_list)
186
+
187
+ def exists(self, path: Dir | Remote | str | File) -> bool:
188
+ """Check if a file or directory exists."""
189
+ arg: str = convert_to_str(path)
190
+ assert isinstance(arg, str)
191
+ try:
192
+ self.ls(arg)
193
+ return True
194
+ except subprocess.CalledProcessError:
195
+ return False
196
+
197
+ def is_synced(self, src: str | Dir, dst: str | Dir) -> bool:
198
+ """Check if two directories are in sync."""
199
+ src = convert_to_str(src)
200
+ dst = convert_to_str(dst)
201
+ cmd_list: list[str] = ["check", str(src), str(dst)]
202
+ try:
203
+ self._run(cmd_list)
204
+ return True
205
+ except subprocess.CalledProcessError:
206
+ return False
207
+
208
+ def copy_dir(self, src: str | Dir, dst: str | Dir) -> None:
209
+ """Copy a directory from source to destination."""
210
+ # convert src to str, also dst
211
+ src = convert_to_str(src)
212
+ dst = convert_to_str(dst)
213
+ cmd_list: list[str] = ["copy", src, dst]
214
+ self._run(cmd_list)
rclone_api/rpath.py CHANGED
@@ -1,12 +1,15 @@
1
1
  import json
2
2
  from typing import Any
3
3
 
4
+ from rclone_api.remote import Remote
5
+
4
6
 
5
7
  class RPath:
6
8
  """Remote file dataclass."""
7
9
 
8
10
  def __init__(
9
11
  self,
12
+ remote: Remote,
10
13
  path: str,
11
14
  name: str,
12
15
  size: int,
@@ -16,6 +19,7 @@ class RPath:
16
19
  ) -> None:
17
20
  from rclone_api.rclone import Rclone
18
21
 
22
+ self.remote = remote
19
23
  self.path = path
20
24
  self.name = name
21
25
  self.size = size
@@ -32,10 +36,16 @@ class RPath:
32
36
  self.rclone = rclone
33
37
 
34
38
  @staticmethod
35
- def from_dict(data: dict) -> "RPath":
39
+ def from_dict(
40
+ data: dict, remote: Remote, parent_path: str | None = None
41
+ ) -> "RPath":
36
42
  """Create a File from a dictionary."""
43
+ path = data["Path"]
44
+ if parent_path is not None:
45
+ path = f"{parent_path}/{path}"
37
46
  return RPath(
38
- data["Path"],
47
+ remote,
48
+ path,
39
49
  data["Name"],
40
50
  data["Size"],
41
51
  data["MimeType"],
@@ -45,21 +55,25 @@ class RPath:
45
55
  )
46
56
 
47
57
  @staticmethod
48
- def from_array(data: list[dict]) -> list["RPath"]:
58
+ def from_array(
59
+ data: list[dict], remote: Remote, parent_path: str | None = None
60
+ ) -> list["RPath"]:
49
61
  """Create a File from a dictionary."""
50
62
  out: list[RPath] = []
51
63
  for d in data:
52
- file: RPath = RPath.from_dict(d)
64
+ file: RPath = RPath.from_dict(d, remote, parent_path)
53
65
  out.append(file)
54
66
  return out
55
67
 
56
68
  @staticmethod
57
- def from_json_str(json_str: str) -> list["RPath"]:
69
+ def from_json_str(
70
+ json_str: str, remote: Remote, parent_path: str | None = None
71
+ ) -> list["RPath"]:
58
72
  """Create a File from a JSON string."""
59
73
  json_obj = json.loads(json_str)
60
74
  if isinstance(json_obj, dict):
61
- return [RPath.from_dict(json_obj)]
62
- return RPath.from_array(json_obj)
75
+ return [RPath.from_dict(json_obj, remote, parent_path)]
76
+ return RPath.from_array(json_obj, remote, parent_path)
63
77
 
64
78
  def to_json(self) -> dict:
65
79
  return {
@@ -73,5 +87,9 @@ class RPath:
73
87
  }
74
88
 
75
89
  def __str__(self) -> str:
76
- out = self.to_json()
77
- return json.dumps(out)
90
+ return f"{self.remote.name}:{self.path}"
91
+
92
+ def __repr__(self):
93
+ data = self.to_json()
94
+ data["Remote"] = self.remote.name
95
+ return json.dumps(data)
rclone_api/util.py CHANGED
@@ -1,9 +1,64 @@
1
+ import os
1
2
  import shutil
2
3
  import subprocess
3
4
  from pathlib import Path
4
5
  from tempfile import TemporaryDirectory
6
+ from typing import Any
5
7
 
6
- from rclone_api.types import Config
8
+ from rclone_api.config import Config
9
+ from rclone_api.dir import Dir
10
+ from rclone_api.remote import Remote
11
+ from rclone_api.rpath import RPath
12
+
13
+ # from .rclone import Rclone
14
+
15
+
16
+ def to_path(item: Dir | Remote | str, rclone: Any) -> RPath:
17
+ from rclone_api.rclone import Rclone
18
+
19
+ assert isinstance(rclone, Rclone)
20
+ # if str then it will be remote:path
21
+ if isinstance(item, str):
22
+ # return RPath(item)
23
+ # remote_name: str = item.split(":")[0]
24
+ parts = item.split(":")
25
+ remote_name = parts[0]
26
+ path = ":".join(parts[1:])
27
+ remote = Remote(name=remote_name, rclone=rclone)
28
+ out = RPath(
29
+ remote=remote,
30
+ path=path,
31
+ name="",
32
+ size=0,
33
+ mime_type="",
34
+ mod_time="",
35
+ is_dir=True,
36
+ )
37
+ out.set_rclone(rclone)
38
+ return out
39
+ elif isinstance(item, Dir):
40
+ return item.path
41
+ elif isinstance(item, Remote):
42
+ out = RPath(
43
+ remote=item,
44
+ path=str(item),
45
+ name=str(item),
46
+ size=0,
47
+ mime_type="inode/directory",
48
+ mod_time="",
49
+ is_dir=True,
50
+ )
51
+ out.set_rclone(rclone)
52
+ return out
53
+ else:
54
+ raise ValueError(f"Invalid type for item: {type(item)}")
55
+
56
+
57
+ def _get_verbose(verbose: bool | None) -> bool:
58
+ if verbose is not None:
59
+ return verbose
60
+ # get it from the environment
61
+ return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
7
62
 
8
63
 
9
64
  def get_rclone_exe(rclone_exe: Path | None) -> Path:
@@ -20,10 +75,11 @@ def rclone_execute(
20
75
  cmd: list[str],
21
76
  rclone_conf: Path | Config,
22
77
  rclone_exe: Path,
23
- verbose: bool = False,
78
+ verbose: bool | None = None,
24
79
  ) -> subprocess.CompletedProcess:
25
- print(subprocess.list2cmdline(cmd))
26
80
  tempdir: TemporaryDirectory | None = None
81
+ verbose = _get_verbose(verbose)
82
+ assert verbose is not None
27
83
 
28
84
  try:
29
85
  if isinstance(rclone_conf, Config):
@@ -38,8 +94,16 @@ def rclone_execute(
38
94
  cmd_str = subprocess.list2cmdline(cmd)
39
95
  print(f"Running: {cmd_str}")
40
96
  cp = subprocess.run(
41
- cmd, capture_output=True, encoding="utf-8", check=True, shell=False
97
+ cmd, capture_output=True, encoding="utf-8", check=False, shell=False
42
98
  )
99
+ if cp.returncode != 0:
100
+ cmd_str = subprocess.list2cmdline(cmd)
101
+ print(
102
+ f"Error running: {cmd_str}, returncode: {cp.returncode}\n{cp.stdout}\n{cp.stderr}"
103
+ )
104
+ raise subprocess.CalledProcessError(
105
+ cp.returncode, cmd, cp.stdout, cp.stderr
106
+ )
43
107
  return cp
44
108
  finally:
45
109
  if tempdir:
rclone_api/walk.py CHANGED
@@ -1,87 +1,70 @@
1
- from concurrent.futures import ThreadPoolExecutor
2
- from queue import Empty, Queue
1
+ from queue import Queue
2
+ from threading import Thread
3
3
  from typing import Generator
4
4
 
5
5
  from rclone_api import Dir
6
6
  from rclone_api.dir_listing import DirListing
7
7
  from rclone_api.remote import Remote
8
8
 
9
+ _MAX_OUT_QUEUE_SIZE = 50
9
10
 
10
- def walk(
11
- dir: Dir | Remote, max_depth: int = -1, max_workers: int = 4
12
- ) -> Generator[DirListing, None, None]:
11
+
12
+ def _walk_runner(
13
+ queue: Queue[Dir], max_depth: int, out_queue: Queue[DirListing | None]
14
+ ) -> None:
15
+ try:
16
+ while not queue.empty():
17
+ current_dir = queue.get()
18
+ dirlisting = current_dir.ls()
19
+ out_queue.put(dirlisting)
20
+ dirs = dirlisting.dirs
21
+
22
+ if max_depth != 0 and len(dirs) > 0:
23
+ for child in dirs:
24
+ queue.put(child)
25
+ if max_depth < 0:
26
+ continue
27
+ if max_depth > 0:
28
+ max_depth -= 1
29
+ out_queue.put(None)
30
+ except KeyboardInterrupt:
31
+ import _thread
32
+
33
+ out_queue.put(None)
34
+
35
+ _thread.interrupt_main()
36
+
37
+
38
+ def walk(dir: Dir | Remote, max_depth: int = -1) -> Generator[DirListing, None, None]:
13
39
  """Walk through the given directory recursively.
14
40
 
15
41
  Args:
16
42
  dir: Directory or Remote to walk through
17
43
  max_depth: Maximum depth to traverse (-1 for unlimited)
18
- max_workers: Maximum number of concurrent workers
19
44
 
20
45
  Yields:
21
46
  DirListing: Directory listing for each directory encountered
22
47
  """
23
- # Convert Remote to Dir if needed
24
- if isinstance(dir, Remote):
25
- dir = Dir(dir)
26
- pending: Queue[tuple[Dir | None, int]] = Queue()
27
- results: Queue[DirListing | Exception] = Queue()
48
+ try:
49
+ # Convert Remote to Dir if needed
50
+ if isinstance(dir, Remote):
51
+ dir = Dir(dir)
28
52
 
29
- def worker():
30
- while True:
31
- try:
32
- # Add timeout to allow checking for sentinel value
33
- try:
34
- current_dir, depth = pending.get(timeout=0.1)
35
- except Empty:
36
- continue
53
+ in_queue: Queue[Dir] = Queue()
54
+ out_queue: Queue[DirListing] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
55
+ in_queue.put(dir)
37
56
 
38
- # Check for sentinel value
39
- if current_dir is None:
40
- pending.task_done()
41
- break
57
+ # Start worker thread
58
+ worker = Thread(
59
+ target=_walk_runner, args=(in_queue, max_depth, out_queue), daemon=True
60
+ )
61
+ worker.start()
42
62
 
43
- listing = current_dir.ls()
44
- results.put(listing)
45
-
46
- if max_depth == -1 or depth < max_depth:
47
- for d in listing.dirs:
48
- pending.put((d, depth + 1))
49
-
50
- pending.task_done()
51
- except Exception as e:
52
- results.put(e)
53
- pending.task_done()
63
+ while dirlisting := out_queue.get():
64
+ if dirlisting is None:
54
65
  break
55
- return None
56
-
57
- # Start workers
58
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
59
- workers = [executor.submit(worker) for _ in range(max_workers)]
60
-
61
- # Start walking
62
- pending.put((dir, 0))
63
-
64
- # Process results while workers are running
65
- completed = 0
66
- while completed < max_workers:
67
- try:
68
- result = results.get(timeout=0.1)
69
- if isinstance(result, Exception):
70
- # Propagate exception
71
- raise result
72
- yield result
73
- except Empty:
74
- # Check if any workers have completed
75
- completed = sum(1 for w in workers if w.done())
76
- continue
77
-
78
- # Signal workers to stop
79
- for _ in range(max_workers):
80
- pending.put((None, 0))
66
+ yield dirlisting
81
67
 
82
- # Drain any remaining results
83
- while not results.empty():
84
- result = results.get()
85
- if isinstance(result, Exception):
86
- raise result
87
- yield result
68
+ worker.join()
69
+ except KeyboardInterrupt:
70
+ pass
@@ -0,0 +1,34 @@
1
+ Metadata-Version: 2.2
2
+ Name: rclone_api
3
+ Version: 1.0.14
4
+ Summary: rclone api in python
5
+ Home-page: https://github.com/zackees/rclone-api
6
+ Maintainer: Zachary Vorhies
7
+ License: BSD 3-Clause License
8
+ Keywords: template-python-cmd
9
+ Classifier: Programming Language :: Python :: 3
10
+ Requires-Python: >=3.10
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: python-dotenv>=1.0.0
14
+ Dynamic: home-page
15
+ Dynamic: maintainer
16
+
17
+ # rclone-api
18
+
19
+ [![Linting](https://github.com/zackees/rclone-api/actions/workflows/lint.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/lint.yml)
20
+ [![MacOS_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_macos.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_macos.yml)
21
+ [![Ubuntu_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
22
+ [![Win_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
23
+
24
+ Api version of rclone. It's well tested. It's just released so this readme is a little to be desired.
25
+
26
+ To develop software, run `. ./activate`
27
+
28
+ # Windows
29
+
30
+ This environment requires you to use `git-bash`.
31
+
32
+ # Linting
33
+
34
+ Run `./lint`
@@ -0,0 +1,19 @@
1
+ rclone_api/__init__.py,sha256=gca3IDhDdKBtyoS1ekGQ-xTa6zrShZ6Gb4XS8ChdlYo,266
2
+ rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
3
+ rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
4
+ rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
5
+ rclone_api/dir.py,sha256=xUV4i9_E7QHrqHld2IPBlon1CGaAp8qQYMKfJTyVvoQ,2088
6
+ rclone_api/dir_listing.py,sha256=8t5Jx9ZVOJPqGKTJbWaES6bjgogUT2bnpbPVWwK1Fcs,1124
7
+ rclone_api/exec.py,sha256=PgiNF5eQpsY8Pb62ZPwLli1WA9iMEJeMZwX43VEYKII,472
8
+ rclone_api/file.py,sha256=bb9FWQy54wKRydy0M12MQwNxycBwRSn_10ubYGAw0ag,1234
9
+ rclone_api/rclone.py,sha256=m2V3BQX0buK3mwImBiTSNTUh_JcdQJ27OeJWxqvgRZs,7455
10
+ rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
11
+ rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
12
+ rclone_api/util.py,sha256=uiTfOlKCFB5FdUkYR2-Hriz0fVyrWeuZtSgHgc9RrpA,3328
13
+ rclone_api/walk.py,sha256=J78-bY2AhNpt2ICsI5LqmXRE7oC6wVDoKoicIoU6XMg,1953
14
+ rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
15
+ rclone_api-1.0.14.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
16
+ rclone_api-1.0.14.dist-info/METADATA,sha256=pATg2JeY8MQ2I7kywlJj6x4KPdL0QKKcbxjxbvq1OCs,1375
17
+ rclone_api-1.0.14.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
18
+ rclone_api-1.0.14.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
19
+ rclone_api-1.0.14.dist-info/RECORD,,
@@ -1,36 +0,0 @@
1
- Metadata-Version: 2.2
2
- Name: rclone_api
3
- Version: 1.0.12
4
- Summary: rclone api in python
5
- Home-page: https://github.com/zackees/rclone-api
6
- Maintainer: Zachary Vorhies
7
- License: BSD 3-Clause License
8
- Keywords: template-python-cmd
9
- Classifier: Programming Language :: Python :: 3
10
- Requires-Python: >=3.7
11
- Description-Content-Type: text/markdown
12
- License-File: LICENSE
13
- Dynamic: home-page
14
- Dynamic: maintainer
15
-
16
- # template-python-cmd
17
- A template for quickly making a python lib that has a command line program attached
18
-
19
- [![Linting](../../actions/workflows/lint.yml/badge.svg)](../../actions/workflows/lint.yml)
20
-
21
- [![MacOS_Tests](../../actions/workflows/push_macos.yml/badge.svg)](../../actions/workflows/push_macos.yml)
22
- [![Ubuntu_Tests](../../actions/workflows/push_ubuntu.yml/badge.svg)](../../actions/workflows/push_ubuntu.yml)
23
- [![Win_Tests](../../actions/workflows/push_win.yml/badge.svg)](../../actions/workflows/push_win.yml)
24
-
25
- Replace `template-python-cmd` and `template_python_cmd` with your command. Run tox until it's
26
- correct.
27
-
28
- To develop software, run `. ./activate.sh`
29
-
30
- # Windows
31
-
32
- This environment requires you to use `git-bash`.
33
-
34
- # Linting
35
-
36
- Run `./lint.sh` to find linting errors using `pylint`, `flake8` and `mypy`.
@@ -1,18 +0,0 @@
1
- rclone_api/__init__.py,sha256=YH7KQaPwUiJWJiRf0NRKD7XHhMXsxWdXDjt9WLSwdjA,265
2
- rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
3
- rclone_api/config.py,sha256=UujjDNcpwgipgdyFPMhC3zikvlsrurUvvZiwXm5NlPg,471
4
- rclone_api/dir.py,sha256=rd40egerhoKGAdIKfeGh_P70fSDzMo-EiTl0euABGOU,1497
5
- rclone_api/dir_listing.py,sha256=NWleKHCCRW7_eh9JfRwE6r3QbjmiHD5ZEGQcd2vm4uY,458
6
- rclone_api/file.py,sha256=409neYPfCUKQX2w7Uw7_D0wR2PtH42srNe9u_nnOLxM,925
7
- rclone_api/rclone.py,sha256=EUsLW5qcipI5RzhpHUd0PQqQow43m2uKJmNgVR8ZBuU,2859
8
- rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
9
- rclone_api/rpath.py,sha256=_k59z-O75hAhpPyUSH4glPEoiYtIS3RUQb7ltU9Ianw,2009
10
- rclone_api/types.py,sha256=Yp15HrjwZonSQhE323R0WP7fA4NWqKjAfM7z3OwHpWI,518
11
- rclone_api/util.py,sha256=EXnijLLdFHqwoZvHrZdqeM8r6T7ad9-pN7qBN1b0I_g,1465
12
- rclone_api/walk.py,sha256=3GKnu9P5aPNiftfoi52U7wo1wixZxwxdqqS0_IlMGCE,2816
13
- rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
14
- rclone_api-1.0.12.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
15
- rclone_api-1.0.12.dist-info/METADATA,sha256=fkbaFAbZssh_WIKBJs-M1VNNUf3o3YKcEo3wIYDGO8o,1245
16
- rclone_api-1.0.12.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
17
- rclone_api-1.0.12.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
18
- rclone_api-1.0.12.dist-info/RECORD,,