rclone-api 1.0.12__py2.py3-none-any.whl → 1.0.14__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rclone_api/__init__.py CHANGED
@@ -1,9 +1,9 @@
1
+ from .config import Config
1
2
  from .dir import Dir
2
3
  from .dir_listing import DirListing
3
4
  from .file import File
4
5
  from .rclone import Rclone
5
6
  from .remote import Remote
6
7
  from .rpath import RPath
7
- from .types import Config
8
8
 
9
9
  __all__ = ["Rclone", "File", "Config", "Remote", "Dir", "RPath", "DirListing"]
rclone_api/config.py CHANGED
@@ -1,19 +1,8 @@
1
- import subprocess
2
1
  from dataclasses import dataclass
3
- from pathlib import Path
4
-
5
- from rclone_api.types import Config
6
2
 
7
3
 
8
4
  @dataclass
9
- class RcloneExec:
10
- """Rclone execution dataclass."""
11
-
12
- rclone_config: Path | Config
13
- rclone_exe: Path
14
-
15
- def execute(self, cmd: list[str]) -> subprocess.CompletedProcess:
16
- """Execute rclone command."""
17
- from rclone_api.util import rclone_execute
5
+ class Config:
6
+ """Rclone configuration dataclass."""
18
7
 
19
- return rclone_execute(cmd, self.rclone_config, self.rclone_exe)
8
+ text: str
rclone_api/convert.py ADDED
@@ -0,0 +1,31 @@
1
+ from rclone_api.dir import Dir
2
+ from rclone_api.file import File
3
+ from rclone_api.remote import Remote
4
+
5
+
6
+ def convert_to_filestr_list(files: str | File | list[str] | list[File]) -> list[str]:
7
+ out: list[str] = []
8
+ if isinstance(files, str):
9
+ out.append(files)
10
+ elif isinstance(files, File):
11
+ out.append(str(files.path))
12
+ elif isinstance(files, list):
13
+ for f in files:
14
+ if isinstance(f, File):
15
+ f = str(f.path)
16
+ out.append(f)
17
+ else:
18
+ raise ValueError(f"Invalid type for file: {type(files)}")
19
+ return out
20
+
21
+
22
+ def convert_to_str(file_or_dir: str | File | Dir | Remote) -> str:
23
+ if isinstance(file_or_dir, str):
24
+ return file_or_dir
25
+ if isinstance(file_or_dir, File):
26
+ return str(file_or_dir.path)
27
+ if isinstance(file_or_dir, Dir):
28
+ return str(file_or_dir.path)
29
+ if isinstance(file_or_dir, Remote):
30
+ return str(file_or_dir)
31
+ raise ValueError(f"Invalid type for file_or_dir: {type(file_or_dir)}")
rclone_api/dir.py CHANGED
@@ -1,3 +1,4 @@
1
+ import json
1
2
  from typing import Generator
2
3
 
3
4
  from rclone_api.dir_listing import DirListing
@@ -8,6 +9,14 @@ from rclone_api.rpath import RPath
8
9
  class Dir:
9
10
  """Remote file dataclass."""
10
11
 
12
+ @property
13
+ def remote(self) -> Remote:
14
+ return self.path.remote
15
+
16
+ @property
17
+ def name(self) -> str:
18
+ return self.path.name
19
+
11
20
  def __init__(self, path: RPath | Remote) -> None:
12
21
  """Initialize Dir with either an RPath or Remote.
13
22
 
@@ -17,6 +26,7 @@ class Dir:
17
26
  if isinstance(path, Remote):
18
27
  # Need to create an RPath for the Remote's root
19
28
  self.path = RPath(
29
+ remote=path,
20
30
  path=str(path),
21
31
  name=str(path),
22
32
  size=0,
@@ -28,11 +38,14 @@ class Dir:
28
38
  self.path.set_rclone(path.rclone)
29
39
  else:
30
40
  self.path = path
41
+ # self.path.set_rclone(self.path.remote.rclone)
42
+ assert self.path.rclone is not None
31
43
 
32
44
  def ls(self, max_depth: int = 0) -> DirListing:
33
45
  """List files and directories in the given path."""
34
46
  assert self.path.rclone is not None
35
- return self.path.rclone.ls(self.path.path, max_depth=max_depth)
47
+ dir = Dir(self.path)
48
+ return self.path.rclone.ls(dir, max_depth=max_depth)
36
49
 
37
50
  def walk(self, max_depth: int = -1) -> Generator[DirListing, None, None]:
38
51
  """List files and directories in the given path."""
@@ -41,5 +54,14 @@ class Dir:
41
54
  assert self.path.rclone is not None
42
55
  return walk(self, max_depth=max_depth)
43
56
 
57
+ def to_json(self) -> dict:
58
+ """Convert the Dir to a JSON serializable dictionary."""
59
+ return self.path.to_json()
60
+
44
61
  def __str__(self) -> str:
45
62
  return str(self.path)
63
+
64
+ def __repr__(self) -> str:
65
+ data = self.path.to_json()
66
+ data_str = json.dumps(data)
67
+ return data_str
rclone_api/dir_listing.py CHANGED
@@ -1,5 +1,5 @@
1
- # from rclone_api.dir import Dir
2
- # from rclone_api.file import File
1
+ import json
2
+
3
3
  from rclone_api.rpath import RPath
4
4
 
5
5
 
@@ -12,3 +12,29 @@ class DirListing:
12
12
 
13
13
  self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
14
14
  self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
15
+
16
+ def __str__(self) -> str:
17
+ n_files = len(self.files)
18
+ n_dirs = len(self.dirs)
19
+ msg = f"Files: {n_files}\n"
20
+ if n_files > 0:
21
+ for f in self.files:
22
+ msg += f" {f}\n"
23
+ msg += f"Dirs: {n_dirs}\n"
24
+ if n_dirs > 0:
25
+ for d in self.dirs:
26
+ msg += f" {d}\n"
27
+ return msg
28
+
29
+ def __repr__(self) -> str:
30
+ dirs: list = []
31
+ files: list = []
32
+ for d in self.dirs:
33
+ dirs.append(d.path.to_json())
34
+ for f in self.files:
35
+ files.append(f.path.to_json())
36
+ json_obj = {
37
+ "dirs": dirs,
38
+ "files": files,
39
+ }
40
+ return json.dumps(json_obj, indent=2)
@@ -2,12 +2,7 @@ import subprocess
2
2
  from dataclasses import dataclass
3
3
  from pathlib import Path
4
4
 
5
-
6
- @dataclass
7
- class Config:
8
- """Rclone configuration dataclass."""
9
-
10
- text: str
5
+ from rclone_api.config import Config
11
6
 
12
7
 
13
8
  @dataclass
rclone_api/file.py CHANGED
@@ -12,6 +12,10 @@ class File:
12
12
  ) -> None:
13
13
  self.path = path
14
14
 
15
+ @property
16
+ def name(self) -> str:
17
+ return self.path.name
18
+
15
19
  def read_text(self) -> str:
16
20
  """Read the file contents as bytes.
17
21
 
@@ -30,6 +34,14 @@ class File:
30
34
  result = self.path.rclone._run(["cat", self.path.path])
31
35
  return result.stdout
32
36
 
37
+ def to_json(self) -> dict:
38
+ """Convert the File to a JSON serializable dictionary."""
39
+ return self.path.to_json()
40
+
33
41
  def __str__(self) -> str:
34
- out = self.path.to_json()
35
- return json.dumps(out)
42
+ return str(self.path)
43
+
44
+ def __repr__(self) -> str:
45
+ data = self.path.to_json()
46
+ data_str = json.dumps(data)
47
+ return data_str
rclone_api/rclone.py CHANGED
@@ -3,15 +3,20 @@ Unit test file.
3
3
  """
4
4
 
5
5
  import subprocess
6
+ from concurrent.futures import ThreadPoolExecutor
7
+ from fnmatch import fnmatch
6
8
  from pathlib import Path
7
9
  from typing import Generator
8
10
 
9
11
  from rclone_api import Dir
12
+ from rclone_api.config import Config
13
+ from rclone_api.convert import convert_to_filestr_list, convert_to_str
10
14
  from rclone_api.dir_listing import DirListing
15
+ from rclone_api.exec import RcloneExec
16
+ from rclone_api.file import File
11
17
  from rclone_api.remote import Remote
12
18
  from rclone_api.rpath import RPath
13
- from rclone_api.types import Config, RcloneExec
14
- from rclone_api.util import get_rclone_exe
19
+ from rclone_api.util import get_rclone_exe, to_path
15
20
  from rclone_api.walk import walk
16
21
 
17
22
 
@@ -27,7 +32,12 @@ class Rclone:
27
32
  def _run(self, cmd: list[str]) -> subprocess.CompletedProcess:
28
33
  return self._exec.execute(cmd)
29
34
 
30
- def ls(self, path: str | Remote, max_depth: int = 0) -> DirListing:
35
+ def ls(
36
+ self,
37
+ path: Dir | Remote | str,
38
+ max_depth: int | None = None,
39
+ glob: str | None = None,
40
+ ) -> DirListing:
31
41
  """List files in the given path.
32
42
 
33
43
  Args:
@@ -37,16 +47,35 @@ class Rclone:
37
47
  Returns:
38
48
  List of File objects found at the path
39
49
  """
50
+
51
+ if isinstance(path, str):
52
+ path = Dir(
53
+ to_path(path, self)
54
+ ) # assume it's a directory if ls is being called.
55
+
40
56
  cmd = ["lsjson"]
41
- if max_depth > 0:
42
- cmd.extend(["--recursive", "--max-depth", str(max_depth)])
57
+ if max_depth is not None:
58
+ cmd.append("--recursive")
59
+ if max_depth > -1:
60
+ cmd.append("--max-depth")
61
+ cmd.append(str(max_depth))
43
62
  cmd.append(str(path))
63
+ remote = path.remote if isinstance(path, Dir) else path
64
+ assert isinstance(remote, Remote)
44
65
 
45
66
  cp = self._run(cmd)
46
67
  text = cp.stdout
47
- paths: list[RPath] = RPath.from_json_str(text)
68
+ parent_path: str | None = None
69
+ if isinstance(path, Dir):
70
+ parent_path = path.path.path
71
+ paths: list[RPath] = RPath.from_json_str(text, remote, parent_path=parent_path)
72
+ # print(parent_path)
48
73
  for o in paths:
49
74
  o.set_rclone(self)
75
+
76
+ # do we have a glob pattern?
77
+ if glob is not None:
78
+ paths = [p for p in paths if fnmatch(p.path, glob)]
50
79
  return DirListing(paths)
51
80
 
52
81
  def listremotes(self) -> list[Remote]:
@@ -61,7 +90,7 @@ class Rclone:
61
90
  return out
62
91
 
63
92
  def walk(
64
- self, path: str | Remote, max_depth: int = -1
93
+ self, path: Dir | Remote | str, max_depth: int = -1
65
94
  ) -> Generator[DirListing, None, None]:
66
95
  """Walk through the given path recursively.
67
96
 
@@ -72,11 +101,13 @@ class Rclone:
72
101
  Yields:
73
102
  DirListing: Directory listing for each directory encountered
74
103
  """
75
- if isinstance(path, str):
104
+ if isinstance(path, Dir):
76
105
  # Create a Remote object for the path
106
+ remote = path.remote
77
107
  rpath = RPath(
78
- path=path,
79
- name=path,
108
+ remote=remote,
109
+ path=path.path.path,
110
+ name=path.path.name,
80
111
  size=0,
81
112
  mime_type="inode/directory",
82
113
  mod_time="",
@@ -84,7 +115,100 @@ class Rclone:
84
115
  )
85
116
  rpath.set_rclone(self)
86
117
  dir_obj = Dir(rpath)
87
- else:
118
+ elif isinstance(path, str):
119
+ dir_obj = Dir(to_path(path, self))
120
+ elif isinstance(path, Remote):
88
121
  dir_obj = Dir(path)
122
+ else:
123
+ assert f"Invalid type for path: {type(path)}"
89
124
 
90
125
  yield from walk(dir_obj, max_depth=max_depth)
126
+
127
+ def copyfile(self, src: File | str, dst: File | str) -> None:
128
+ """Copy a single file from source to destination.
129
+
130
+ Args:
131
+ src: Source file path (including remote if applicable)
132
+ dst: Destination file path (including remote if applicable)
133
+
134
+ Raises:
135
+ subprocess.CalledProcessError: If the copy operation fails
136
+ """
137
+ src = src if isinstance(src, str) else str(src.path)
138
+ dst = dst if isinstance(dst, str) else str(dst.path)
139
+ cmd_list: list[str] = ["copyto", src, dst]
140
+ self._run(cmd_list)
141
+
142
+ def copyfiles(self, filelist: dict[File, File] | dict[str, str]) -> None:
143
+ """Copy multiple files from source to destination.
144
+
145
+ Warning - slow.
146
+
147
+ Args:
148
+ payload: Dictionary of source and destination file paths
149
+ """
150
+ str_dict: dict[str, str] = {}
151
+ for src, dst in filelist.items():
152
+ src = src if isinstance(src, str) else str(src.path)
153
+ dst = dst if isinstance(dst, str) else str(dst.path)
154
+ str_dict[src] = dst
155
+
156
+ with ThreadPoolExecutor(max_workers=64) as executor:
157
+ for src, dst in str_dict.items(): # warning - slow
158
+ cmd_list: list[str] = ["copyto", src, dst]
159
+ # self._run(cmd_list)
160
+ executor.submit(self._run, cmd_list)
161
+
162
+ def copy(self, src: Dir, dst: Dir) -> None:
163
+ """Copy files from source to destination.
164
+
165
+ Args:
166
+ src: Source directory
167
+ dst: Destination directory
168
+ """
169
+ src_dir = src.path.path
170
+ dst_dir = dst.path.path
171
+ cmd_list: list[str] = ["copy", src_dir, dst_dir]
172
+ self._run(cmd_list)
173
+
174
+ def purge(self, path: Dir | str) -> None:
175
+ """Purge a directory"""
176
+ # path should always be a string
177
+ path = path if isinstance(path, str) else str(path.path)
178
+ cmd_list: list[str] = ["purge", str(path)]
179
+ self._run(cmd_list)
180
+
181
+ def deletefiles(self, files: str | File | list[str] | list[File]) -> None:
182
+ """Delete a directory"""
183
+ payload: list[str] = convert_to_filestr_list(files)
184
+ cmd_list: list[str] = ["delete"] + payload
185
+ self._run(cmd_list)
186
+
187
+ def exists(self, path: Dir | Remote | str | File) -> bool:
188
+ """Check if a file or directory exists."""
189
+ arg: str = convert_to_str(path)
190
+ assert isinstance(arg, str)
191
+ try:
192
+ self.ls(arg)
193
+ return True
194
+ except subprocess.CalledProcessError:
195
+ return False
196
+
197
+ def is_synced(self, src: str | Dir, dst: str | Dir) -> bool:
198
+ """Check if two directories are in sync."""
199
+ src = convert_to_str(src)
200
+ dst = convert_to_str(dst)
201
+ cmd_list: list[str] = ["check", str(src), str(dst)]
202
+ try:
203
+ self._run(cmd_list)
204
+ return True
205
+ except subprocess.CalledProcessError:
206
+ return False
207
+
208
+ def copy_dir(self, src: str | Dir, dst: str | Dir) -> None:
209
+ """Copy a directory from source to destination."""
210
+ # convert src to str, also dst
211
+ src = convert_to_str(src)
212
+ dst = convert_to_str(dst)
213
+ cmd_list: list[str] = ["copy", src, dst]
214
+ self._run(cmd_list)
rclone_api/rpath.py CHANGED
@@ -1,12 +1,15 @@
1
1
  import json
2
2
  from typing import Any
3
3
 
4
+ from rclone_api.remote import Remote
5
+
4
6
 
5
7
  class RPath:
6
8
  """Remote file dataclass."""
7
9
 
8
10
  def __init__(
9
11
  self,
12
+ remote: Remote,
10
13
  path: str,
11
14
  name: str,
12
15
  size: int,
@@ -16,6 +19,7 @@ class RPath:
16
19
  ) -> None:
17
20
  from rclone_api.rclone import Rclone
18
21
 
22
+ self.remote = remote
19
23
  self.path = path
20
24
  self.name = name
21
25
  self.size = size
@@ -32,10 +36,16 @@ class RPath:
32
36
  self.rclone = rclone
33
37
 
34
38
  @staticmethod
35
- def from_dict(data: dict) -> "RPath":
39
+ def from_dict(
40
+ data: dict, remote: Remote, parent_path: str | None = None
41
+ ) -> "RPath":
36
42
  """Create a File from a dictionary."""
43
+ path = data["Path"]
44
+ if parent_path is not None:
45
+ path = f"{parent_path}/{path}"
37
46
  return RPath(
38
- data["Path"],
47
+ remote,
48
+ path,
39
49
  data["Name"],
40
50
  data["Size"],
41
51
  data["MimeType"],
@@ -45,21 +55,25 @@ class RPath:
45
55
  )
46
56
 
47
57
  @staticmethod
48
- def from_array(data: list[dict]) -> list["RPath"]:
58
+ def from_array(
59
+ data: list[dict], remote: Remote, parent_path: str | None = None
60
+ ) -> list["RPath"]:
49
61
  """Create a File from a dictionary."""
50
62
  out: list[RPath] = []
51
63
  for d in data:
52
- file: RPath = RPath.from_dict(d)
64
+ file: RPath = RPath.from_dict(d, remote, parent_path)
53
65
  out.append(file)
54
66
  return out
55
67
 
56
68
  @staticmethod
57
- def from_json_str(json_str: str) -> list["RPath"]:
69
+ def from_json_str(
70
+ json_str: str, remote: Remote, parent_path: str | None = None
71
+ ) -> list["RPath"]:
58
72
  """Create a File from a JSON string."""
59
73
  json_obj = json.loads(json_str)
60
74
  if isinstance(json_obj, dict):
61
- return [RPath.from_dict(json_obj)]
62
- return RPath.from_array(json_obj)
75
+ return [RPath.from_dict(json_obj, remote, parent_path)]
76
+ return RPath.from_array(json_obj, remote, parent_path)
63
77
 
64
78
  def to_json(self) -> dict:
65
79
  return {
@@ -73,5 +87,9 @@ class RPath:
73
87
  }
74
88
 
75
89
  def __str__(self) -> str:
76
- out = self.to_json()
77
- return json.dumps(out)
90
+ return f"{self.remote.name}:{self.path}"
91
+
92
+ def __repr__(self):
93
+ data = self.to_json()
94
+ data["Remote"] = self.remote.name
95
+ return json.dumps(data)
rclone_api/util.py CHANGED
@@ -1,9 +1,64 @@
1
+ import os
1
2
  import shutil
2
3
  import subprocess
3
4
  from pathlib import Path
4
5
  from tempfile import TemporaryDirectory
6
+ from typing import Any
5
7
 
6
- from rclone_api.types import Config
8
+ from rclone_api.config import Config
9
+ from rclone_api.dir import Dir
10
+ from rclone_api.remote import Remote
11
+ from rclone_api.rpath import RPath
12
+
13
+ # from .rclone import Rclone
14
+
15
+
16
+ def to_path(item: Dir | Remote | str, rclone: Any) -> RPath:
17
+ from rclone_api.rclone import Rclone
18
+
19
+ assert isinstance(rclone, Rclone)
20
+ # if str then it will be remote:path
21
+ if isinstance(item, str):
22
+ # return RPath(item)
23
+ # remote_name: str = item.split(":")[0]
24
+ parts = item.split(":")
25
+ remote_name = parts[0]
26
+ path = ":".join(parts[1:])
27
+ remote = Remote(name=remote_name, rclone=rclone)
28
+ out = RPath(
29
+ remote=remote,
30
+ path=path,
31
+ name="",
32
+ size=0,
33
+ mime_type="",
34
+ mod_time="",
35
+ is_dir=True,
36
+ )
37
+ out.set_rclone(rclone)
38
+ return out
39
+ elif isinstance(item, Dir):
40
+ return item.path
41
+ elif isinstance(item, Remote):
42
+ out = RPath(
43
+ remote=item,
44
+ path=str(item),
45
+ name=str(item),
46
+ size=0,
47
+ mime_type="inode/directory",
48
+ mod_time="",
49
+ is_dir=True,
50
+ )
51
+ out.set_rclone(rclone)
52
+ return out
53
+ else:
54
+ raise ValueError(f"Invalid type for item: {type(item)}")
55
+
56
+
57
+ def _get_verbose(verbose: bool | None) -> bool:
58
+ if verbose is not None:
59
+ return verbose
60
+ # get it from the environment
61
+ return bool(int(os.getenv("RCLONE_API_VERBOSE", "0")))
7
62
 
8
63
 
9
64
  def get_rclone_exe(rclone_exe: Path | None) -> Path:
@@ -20,10 +75,11 @@ def rclone_execute(
20
75
  cmd: list[str],
21
76
  rclone_conf: Path | Config,
22
77
  rclone_exe: Path,
23
- verbose: bool = False,
78
+ verbose: bool | None = None,
24
79
  ) -> subprocess.CompletedProcess:
25
- print(subprocess.list2cmdline(cmd))
26
80
  tempdir: TemporaryDirectory | None = None
81
+ verbose = _get_verbose(verbose)
82
+ assert verbose is not None
27
83
 
28
84
  try:
29
85
  if isinstance(rclone_conf, Config):
@@ -38,8 +94,16 @@ def rclone_execute(
38
94
  cmd_str = subprocess.list2cmdline(cmd)
39
95
  print(f"Running: {cmd_str}")
40
96
  cp = subprocess.run(
41
- cmd, capture_output=True, encoding="utf-8", check=True, shell=False
97
+ cmd, capture_output=True, encoding="utf-8", check=False, shell=False
42
98
  )
99
+ if cp.returncode != 0:
100
+ cmd_str = subprocess.list2cmdline(cmd)
101
+ print(
102
+ f"Error running: {cmd_str}, returncode: {cp.returncode}\n{cp.stdout}\n{cp.stderr}"
103
+ )
104
+ raise subprocess.CalledProcessError(
105
+ cp.returncode, cmd, cp.stdout, cp.stderr
106
+ )
43
107
  return cp
44
108
  finally:
45
109
  if tempdir:
rclone_api/walk.py CHANGED
@@ -1,87 +1,70 @@
1
- from concurrent.futures import ThreadPoolExecutor
2
- from queue import Empty, Queue
1
+ from queue import Queue
2
+ from threading import Thread
3
3
  from typing import Generator
4
4
 
5
5
  from rclone_api import Dir
6
6
  from rclone_api.dir_listing import DirListing
7
7
  from rclone_api.remote import Remote
8
8
 
9
+ _MAX_OUT_QUEUE_SIZE = 50
9
10
 
10
- def walk(
11
- dir: Dir | Remote, max_depth: int = -1, max_workers: int = 4
12
- ) -> Generator[DirListing, None, None]:
11
+
12
+ def _walk_runner(
13
+ queue: Queue[Dir], max_depth: int, out_queue: Queue[DirListing | None]
14
+ ) -> None:
15
+ try:
16
+ while not queue.empty():
17
+ current_dir = queue.get()
18
+ dirlisting = current_dir.ls()
19
+ out_queue.put(dirlisting)
20
+ dirs = dirlisting.dirs
21
+
22
+ if max_depth != 0 and len(dirs) > 0:
23
+ for child in dirs:
24
+ queue.put(child)
25
+ if max_depth < 0:
26
+ continue
27
+ if max_depth > 0:
28
+ max_depth -= 1
29
+ out_queue.put(None)
30
+ except KeyboardInterrupt:
31
+ import _thread
32
+
33
+ out_queue.put(None)
34
+
35
+ _thread.interrupt_main()
36
+
37
+
38
+ def walk(dir: Dir | Remote, max_depth: int = -1) -> Generator[DirListing, None, None]:
13
39
  """Walk through the given directory recursively.
14
40
 
15
41
  Args:
16
42
  dir: Directory or Remote to walk through
17
43
  max_depth: Maximum depth to traverse (-1 for unlimited)
18
- max_workers: Maximum number of concurrent workers
19
44
 
20
45
  Yields:
21
46
  DirListing: Directory listing for each directory encountered
22
47
  """
23
- # Convert Remote to Dir if needed
24
- if isinstance(dir, Remote):
25
- dir = Dir(dir)
26
- pending: Queue[tuple[Dir | None, int]] = Queue()
27
- results: Queue[DirListing | Exception] = Queue()
48
+ try:
49
+ # Convert Remote to Dir if needed
50
+ if isinstance(dir, Remote):
51
+ dir = Dir(dir)
28
52
 
29
- def worker():
30
- while True:
31
- try:
32
- # Add timeout to allow checking for sentinel value
33
- try:
34
- current_dir, depth = pending.get(timeout=0.1)
35
- except Empty:
36
- continue
53
+ in_queue: Queue[Dir] = Queue()
54
+ out_queue: Queue[DirListing] = Queue(maxsize=_MAX_OUT_QUEUE_SIZE)
55
+ in_queue.put(dir)
37
56
 
38
- # Check for sentinel value
39
- if current_dir is None:
40
- pending.task_done()
41
- break
57
+ # Start worker thread
58
+ worker = Thread(
59
+ target=_walk_runner, args=(in_queue, max_depth, out_queue), daemon=True
60
+ )
61
+ worker.start()
42
62
 
43
- listing = current_dir.ls()
44
- results.put(listing)
45
-
46
- if max_depth == -1 or depth < max_depth:
47
- for d in listing.dirs:
48
- pending.put((d, depth + 1))
49
-
50
- pending.task_done()
51
- except Exception as e:
52
- results.put(e)
53
- pending.task_done()
63
+ while dirlisting := out_queue.get():
64
+ if dirlisting is None:
54
65
  break
55
- return None
56
-
57
- # Start workers
58
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
59
- workers = [executor.submit(worker) for _ in range(max_workers)]
60
-
61
- # Start walking
62
- pending.put((dir, 0))
63
-
64
- # Process results while workers are running
65
- completed = 0
66
- while completed < max_workers:
67
- try:
68
- result = results.get(timeout=0.1)
69
- if isinstance(result, Exception):
70
- # Propagate exception
71
- raise result
72
- yield result
73
- except Empty:
74
- # Check if any workers have completed
75
- completed = sum(1 for w in workers if w.done())
76
- continue
77
-
78
- # Signal workers to stop
79
- for _ in range(max_workers):
80
- pending.put((None, 0))
66
+ yield dirlisting
81
67
 
82
- # Drain any remaining results
83
- while not results.empty():
84
- result = results.get()
85
- if isinstance(result, Exception):
86
- raise result
87
- yield result
68
+ worker.join()
69
+ except KeyboardInterrupt:
70
+ pass
@@ -0,0 +1,34 @@
1
+ Metadata-Version: 2.2
2
+ Name: rclone_api
3
+ Version: 1.0.14
4
+ Summary: rclone api in python
5
+ Home-page: https://github.com/zackees/rclone-api
6
+ Maintainer: Zachary Vorhies
7
+ License: BSD 3-Clause License
8
+ Keywords: template-python-cmd
9
+ Classifier: Programming Language :: Python :: 3
10
+ Requires-Python: >=3.10
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: python-dotenv>=1.0.0
14
+ Dynamic: home-page
15
+ Dynamic: maintainer
16
+
17
+ # rclone-api
18
+
19
+ [![Linting](https://github.com/zackees/rclone-api/actions/workflows/lint.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/lint.yml)
20
+ [![MacOS_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_macos.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_macos.yml)
21
+ [![Ubuntu_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_ubuntu.yml)
22
+ [![Win_Tests](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml/badge.svg)](https://github.com/zackees/rclone-api/actions/workflows/push_win.yml)
23
+
24
+ Api version of rclone. It's well tested. It's just released so this readme is a little to be desired.
25
+
26
+ To develop software, run `. ./activate`
27
+
28
+ # Windows
29
+
30
+ This environment requires you to use `git-bash`.
31
+
32
+ # Linting
33
+
34
+ Run `./lint`
@@ -0,0 +1,19 @@
1
+ rclone_api/__init__.py,sha256=gca3IDhDdKBtyoS1ekGQ-xTa6zrShZ6Gb4XS8ChdlYo,266
2
+ rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
3
+ rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
4
+ rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
5
+ rclone_api/dir.py,sha256=xUV4i9_E7QHrqHld2IPBlon1CGaAp8qQYMKfJTyVvoQ,2088
6
+ rclone_api/dir_listing.py,sha256=8t5Jx9ZVOJPqGKTJbWaES6bjgogUT2bnpbPVWwK1Fcs,1124
7
+ rclone_api/exec.py,sha256=PgiNF5eQpsY8Pb62ZPwLli1WA9iMEJeMZwX43VEYKII,472
8
+ rclone_api/file.py,sha256=bb9FWQy54wKRydy0M12MQwNxycBwRSn_10ubYGAw0ag,1234
9
+ rclone_api/rclone.py,sha256=m2V3BQX0buK3mwImBiTSNTUh_JcdQJ27OeJWxqvgRZs,7455
10
+ rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
11
+ rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
12
+ rclone_api/util.py,sha256=uiTfOlKCFB5FdUkYR2-Hriz0fVyrWeuZtSgHgc9RrpA,3328
13
+ rclone_api/walk.py,sha256=J78-bY2AhNpt2ICsI5LqmXRE7oC6wVDoKoicIoU6XMg,1953
14
+ rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
15
+ rclone_api-1.0.14.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
16
+ rclone_api-1.0.14.dist-info/METADATA,sha256=pATg2JeY8MQ2I7kywlJj6x4KPdL0QKKcbxjxbvq1OCs,1375
17
+ rclone_api-1.0.14.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
18
+ rclone_api-1.0.14.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
19
+ rclone_api-1.0.14.dist-info/RECORD,,
@@ -1,36 +0,0 @@
1
- Metadata-Version: 2.2
2
- Name: rclone_api
3
- Version: 1.0.12
4
- Summary: rclone api in python
5
- Home-page: https://github.com/zackees/rclone-api
6
- Maintainer: Zachary Vorhies
7
- License: BSD 3-Clause License
8
- Keywords: template-python-cmd
9
- Classifier: Programming Language :: Python :: 3
10
- Requires-Python: >=3.7
11
- Description-Content-Type: text/markdown
12
- License-File: LICENSE
13
- Dynamic: home-page
14
- Dynamic: maintainer
15
-
16
- # template-python-cmd
17
- A template for quickly making a python lib that has a command line program attached
18
-
19
- [![Linting](../../actions/workflows/lint.yml/badge.svg)](../../actions/workflows/lint.yml)
20
-
21
- [![MacOS_Tests](../../actions/workflows/push_macos.yml/badge.svg)](../../actions/workflows/push_macos.yml)
22
- [![Ubuntu_Tests](../../actions/workflows/push_ubuntu.yml/badge.svg)](../../actions/workflows/push_ubuntu.yml)
23
- [![Win_Tests](../../actions/workflows/push_win.yml/badge.svg)](../../actions/workflows/push_win.yml)
24
-
25
- Replace `template-python-cmd` and `template_python_cmd` with your command. Run tox until it's
26
- correct.
27
-
28
- To develop software, run `. ./activate.sh`
29
-
30
- # Windows
31
-
32
- This environment requires you to use `git-bash`.
33
-
34
- # Linting
35
-
36
- Run `./lint.sh` to find linting errors using `pylint`, `flake8` and `mypy`.
@@ -1,18 +0,0 @@
1
- rclone_api/__init__.py,sha256=YH7KQaPwUiJWJiRf0NRKD7XHhMXsxWdXDjt9WLSwdjA,265
2
- rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
3
- rclone_api/config.py,sha256=UujjDNcpwgipgdyFPMhC3zikvlsrurUvvZiwXm5NlPg,471
4
- rclone_api/dir.py,sha256=rd40egerhoKGAdIKfeGh_P70fSDzMo-EiTl0euABGOU,1497
5
- rclone_api/dir_listing.py,sha256=NWleKHCCRW7_eh9JfRwE6r3QbjmiHD5ZEGQcd2vm4uY,458
6
- rclone_api/file.py,sha256=409neYPfCUKQX2w7Uw7_D0wR2PtH42srNe9u_nnOLxM,925
7
- rclone_api/rclone.py,sha256=EUsLW5qcipI5RzhpHUd0PQqQow43m2uKJmNgVR8ZBuU,2859
8
- rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
9
- rclone_api/rpath.py,sha256=_k59z-O75hAhpPyUSH4glPEoiYtIS3RUQb7ltU9Ianw,2009
10
- rclone_api/types.py,sha256=Yp15HrjwZonSQhE323R0WP7fA4NWqKjAfM7z3OwHpWI,518
11
- rclone_api/util.py,sha256=EXnijLLdFHqwoZvHrZdqeM8r6T7ad9-pN7qBN1b0I_g,1465
12
- rclone_api/walk.py,sha256=3GKnu9P5aPNiftfoi52U7wo1wixZxwxdqqS0_IlMGCE,2816
13
- rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
14
- rclone_api-1.0.12.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
15
- rclone_api-1.0.12.dist-info/METADATA,sha256=fkbaFAbZssh_WIKBJs-M1VNNUf3o3YKcEo3wIYDGO8o,1245
16
- rclone_api-1.0.12.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
17
- rclone_api-1.0.12.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
18
- rclone_api-1.0.12.dist-info/RECORD,,