rclone-api 1.0.76__py2.py3-none-any.whl → 1.0.78__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rclone_api/__init__.py CHANGED
@@ -9,7 +9,7 @@ from .process import Process
9
9
  from .rclone import Rclone, rclone_verbose
10
10
  from .remote import Remote
11
11
  from .rpath import RPath
12
- from .types import ListingOption, Order
12
+ from .types import ListingOption, Order, SizeResult
13
13
 
14
14
  __all__ = [
15
15
  "Rclone",
@@ -29,4 +29,5 @@ __all__ = [
29
29
  "ListingOption",
30
30
  "Order",
31
31
  "ListingOption",
32
+ "SizeResult",
32
33
  ]
@@ -47,3 +47,14 @@ class CompletedProcess:
47
47
  if rtn != 0:
48
48
  return rtn
49
49
  return 0
50
+
51
+ def __str__(self) -> str:
52
+
53
+ cmd_strs: list[str] = []
54
+ rtn_cods: list[int] = []
55
+ for cp in self.completed:
56
+ cmd_strs.append(subprocess.list2cmdline(cp.args))
57
+ rtn_cods.append(cp.returncode)
58
+ msg = f"CompletedProcess: {len(cmd_strs)} commands\n"
59
+ msg += "\n".join([f"{cmd} -> {rtn}" for cmd, rtn in zip(cmd_strs, rtn_cods)])
60
+ return msg
rclone_api/dir_listing.py CHANGED
@@ -29,6 +29,16 @@ class DirListing:
29
29
  self.dirs: list[Dir] = [Dir(d) for d in dirs_and_files if d.is_dir]
30
30
  self.files: list[File] = [File(f) for f in dirs_and_files if not f.is_dir]
31
31
 
32
+ def files_relative(self, prefix: str) -> list[str]:
33
+ """Return a list of file paths relative to the root directory."""
34
+ from rclone_api.file import File
35
+
36
+ out: list[str] = []
37
+ f: File
38
+ for f in self.files:
39
+ out.append(f.relative_to(prefix))
40
+ return out
41
+
32
42
  def __str__(self) -> str:
33
43
  n_files = len(self.files)
34
44
  n_dirs = len(self.dirs)
rclone_api/file.py CHANGED
@@ -1,4 +1,5 @@
1
1
  import json
2
+ from pathlib import Path
2
3
 
3
4
  from rclone_api.rpath import RPath
4
5
 
@@ -40,10 +41,23 @@ class File:
40
41
 
41
42
  def to_string(self, include_remote: bool = True) -> str:
42
43
  """Convert the File to a string."""
43
- out = str(self.path)
44
- if not include_remote:
45
- _, out = out.split(":", 1)
46
- return out
44
+ # out = str(self.path)
45
+ remote = self.path.remote
46
+ rest = self.path.path
47
+ if include_remote:
48
+ return f"{remote.name}:{rest}"
49
+ return rest
50
+
51
+ def relative_to(self, prefix: str) -> str:
52
+ """Return the relative path to the other directory."""
53
+ self_path = Path(str(self))
54
+ rel_path = self_path.relative_to(prefix)
55
+ return str(rel_path.as_posix())
56
+
57
+ @property
58
+ def size(self) -> int:
59
+ """Get the size of the file."""
60
+ return self.path.size
47
61
 
48
62
  def __str__(self) -> str:
49
63
  return str(self.path)
rclone_api/group_files.py CHANGED
@@ -9,6 +9,19 @@ class FilePathParts:
9
9
  parents: list[str]
10
10
  name: str
11
11
 
12
+ def to_string(self, include_remote: bool, include_bucket: bool) -> str:
13
+ """Convert to string, may throw for not include_bucket=False."""
14
+ parents = list(self.parents)
15
+ if not include_bucket:
16
+ parents.pop(0)
17
+ path = "/".join(parents)
18
+ if path:
19
+ path += "/"
20
+ path += self.name
21
+ if include_remote:
22
+ return f"{self.remote}{path}"
23
+ return path
24
+
12
25
 
13
26
  def parse_file(file_path: str) -> FilePathParts:
14
27
  """Parse file path into parts."""
@@ -165,4 +178,38 @@ def group_files(files: list[str], fully_qualified: bool = True) -> dict[str, lis
165
178
  return out
166
179
 
167
180
 
168
- __all__ = ["group_files"]
181
+ def group_under_remote(
182
+ files: list[str], fully_qualified: bool = True
183
+ ) -> dict[str, list[str]]:
184
+ """split between filename and remote"""
185
+
186
+ #### DOE STHIS NEED TO BE REMOVEDD????? #####
187
+
188
+ assert fully_qualified is True, "Not implemented for fully_qualified=False"
189
+ out: dict[str, list[str]] = {}
190
+ for file in files:
191
+ parsed = parse_file(file)
192
+ remote = f"{parsed.remote}:"
193
+ file_list = out.setdefault(remote, [])
194
+ file_list.append(parsed.to_string(include_remote=False, include_bucket=True))
195
+ return out
196
+
197
+
198
+ def group_under_remote_bucket(
199
+ files: list[str], fully_qualified: bool = True
200
+ ) -> dict[str, list[str]]:
201
+ """split between filename and bucket"""
202
+ assert fully_qualified is True, "Not implemented for fully_qualified=False"
203
+ out: dict[str, list[str]] = {}
204
+ for file in files:
205
+ parsed = parse_file(file)
206
+ remote = f"{parsed.remote}:"
207
+ parts = parsed.parents
208
+ bucket = parts[0]
209
+ remote_bucket = f"{remote}{bucket}"
210
+ file_list = out.setdefault(remote_bucket, [])
211
+ file_list.append(parsed.to_string(include_remote=False, include_bucket=False))
212
+ return out
213
+
214
+
215
+ __all__ = ["group_files", "group_under_remote", "group_under_remote_bucket"]
rclone_api/rclone.py CHANGED
@@ -22,11 +22,19 @@ from rclone_api.diff import DiffItem, DiffOption, diff_stream_from_running_proce
22
22
  from rclone_api.dir_listing import DirListing
23
23
  from rclone_api.exec import RcloneExec
24
24
  from rclone_api.file import File
25
- from rclone_api.group_files import group_files
25
+ from rclone_api.group_files import (
26
+ group_files,
27
+ group_under_remote_bucket,
28
+ )
26
29
  from rclone_api.process import Process
27
30
  from rclone_api.remote import Remote
28
31
  from rclone_api.rpath import RPath
29
- from rclone_api.types import ListingOption, ModTimeStrategy, Order
32
+ from rclone_api.types import (
33
+ ListingOption,
34
+ ModTimeStrategy,
35
+ Order,
36
+ SizeResult,
37
+ )
30
38
  from rclone_api.util import (
31
39
  get_check,
32
40
  get_rclone_exe,
@@ -827,3 +835,77 @@ class Rclone:
827
835
  if proc.poll() is not None:
828
836
  raise ValueError("NFS serve process failed to start")
829
837
  return proc
838
+
839
+ def size_files(
840
+ self,
841
+ src: str,
842
+ files: list[str],
843
+ fast_list: bool = True,
844
+ other_args: list[str] | None = None,
845
+ check: bool | None = False,
846
+ verbose: bool | None = None,
847
+ ) -> SizeResult:
848
+ """Get the size of a list of files. Example of files items: "remote:bucket/to/file"."""
849
+ verbose = get_verbose(verbose)
850
+ check = get_check(check)
851
+ files = list(files)
852
+ prefix = src if src.endswith(":") else f"{src}/"
853
+ if src:
854
+ files = [f"{prefix}{f}" for f in files]
855
+ file_list: dict[str, list[str]]
856
+ file_list = group_under_remote_bucket(files)
857
+ all_files: list[File] = []
858
+ for src_path, files in file_list.items():
859
+ cmd = ["lsjson", src_path, "--files-only", "-R"]
860
+ with TemporaryDirectory() as tmpdir:
861
+ # print("files: " + ",".join(files))
862
+ include_files_txt = Path(tmpdir) / "include_files.txt"
863
+ include_files_txt.write_text("\n".join(files), encoding="utf-8")
864
+ cmd += ["--files-from", str(include_files_txt)]
865
+ if fast_list:
866
+ cmd.append("--fast-list")
867
+ if other_args:
868
+ cmd += other_args
869
+ cp = self._run(cmd, check=check)
870
+
871
+ if cp.returncode != 0:
872
+ if check:
873
+ raise ValueError(f"Error getting file sizes: {cp.stderr}")
874
+ else:
875
+ warnings.warn(f"Error getting file sizes: {cp.stderr}")
876
+ stdout = cp.stdout
877
+ pieces = src_path.split(":", 1)
878
+ remote_name = pieces[0]
879
+ parent_path: str | None
880
+ if len(pieces) > 1:
881
+ parent_path = pieces[1]
882
+ else:
883
+ parent_path = None
884
+ remote = Remote(name=remote_name, rclone=self)
885
+ paths: list[RPath] = RPath.from_json_str(
886
+ stdout, remote, parent_path=parent_path
887
+ )
888
+ # print(paths)
889
+ all_files += [File(p) for p in paths]
890
+ file_sizes: dict[str, int] = {}
891
+ f: File
892
+ for f in all_files:
893
+ p = f.to_string(include_remote=True)
894
+ if p in file_sizes:
895
+ warnings.warn(f"Duplicate file found: {p}")
896
+ continue
897
+ size = f.size
898
+ if size == 0:
899
+ warnings.warn(f"File size is 0: {p}")
900
+ file_sizes[p] = f.size
901
+ total_size = sum(file_sizes.values())
902
+ file_sizes_path_corrected: dict[str, int] = {}
903
+ for path, size in file_sizes.items():
904
+ # remove the prefix
905
+ path_path = Path(path)
906
+ path_str = path_path.relative_to(prefix).as_posix()
907
+ file_sizes_path_corrected[path_str] = size
908
+ out: SizeResult = SizeResult(
909
+ prefix=prefix, total_size=total_size, file_sizes=file_sizes_path_corrected
910
+ )
911
+ return out
rclone_api/remote.py CHANGED
@@ -7,6 +7,9 @@ class Remote:
7
7
  def __init__(self, name: str, rclone: Any) -> None:
8
8
  from rclone_api.rclone import Rclone
9
9
 
10
+ if ":" in name:
11
+ raise ValueError("Remote name cannot contain ':'")
12
+
10
13
  assert isinstance(rclone, Rclone)
11
14
  self.name = name
12
15
  self.rclone: Rclone = rclone
rclone_api/types.py CHANGED
@@ -1,3 +1,4 @@
1
+ from dataclasses import dataclass
1
2
  from enum import Enum
2
3
 
3
4
 
@@ -16,3 +17,17 @@ class Order(Enum):
16
17
  NORMAL = "normal"
17
18
  REVERSE = "reverse"
18
19
  RANDOM = "random"
20
+
21
+
22
+ # class GroupingOption(Enum):
23
+ # BUCKET = "bucket"
24
+ # REMOTE = "remote"
25
+
26
+
27
+ @dataclass
28
+ class SizeResult:
29
+ """Size result dataclass."""
30
+
31
+ prefix: str
32
+ total_size: int
33
+ file_sizes: dict[str, int]
rclone_api/util.py CHANGED
@@ -104,7 +104,7 @@ def rclone_execute(
104
104
  )
105
105
  if verbose:
106
106
  cmd_str = subprocess.list2cmdline(cmd)
107
- print(f"Running: {cmd_str}")
107
+ print(f"\nRunning: {cmd_str}")
108
108
  cp = subprocess.run(
109
109
  cmd, capture_output=capture, encoding="utf-8", check=False, shell=False
110
110
  )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: rclone_api
3
- Version: 1.0.76
3
+ Version: 1.0.78
4
4
  Summary: rclone api in python
5
5
  Home-page: https://github.com/zackees/rclone-api
6
6
  Maintainer: Zachary Vorhies
@@ -1,29 +1,29 @@
1
- rclone_api/__init__.py,sha256=kJUk9KAxQ2AEms9lM5yqw-dqlabarhU69b2FAxJVBlY,692
1
+ rclone_api/__init__.py,sha256=UH1aIcovOSoiWwubbgBs_fvX3YcMhHnLTYJzLnCPKVs,722
2
2
  rclone_api/cli.py,sha256=dibfAZIh0kXWsBbfp3onKLjyZXo54mTzDjUdzJlDlWo,231
3
- rclone_api/completed_process.py,sha256=Pp-hXnLgej0IGO5ee9Fmx64dGzIofbQFEUyXdFCvO54,1371
3
+ rclone_api/completed_process.py,sha256=_IZ8IWK7DM1_tsbDEkH6wPZ-bbcrgf7A7smls854pmg,1775
4
4
  rclone_api/config.py,sha256=tP6cU9DnCCEIRc_KP9HPur1jFLLg2QGFSxNwFm6_MVw,118
5
5
  rclone_api/convert.py,sha256=Mx9Qo7zhkOedJd8LdhPvNGHp8znJzOk4f_2KWnoGc78,1012
6
6
  rclone_api/deprecated.py,sha256=qWKpnZdYcBK7YQZKuVoWWXDwi-uqiAtbjgPcci_efow,590
7
7
  rclone_api/diff.py,sha256=tMoJMAGmLSE6Q_7QhPf6PnCzb840djxMZtDmhc2GlGQ,5227
8
8
  rclone_api/dir.py,sha256=i4h7LX5hB_WmVixxDRWL_l1nifvscrdWct_8Wx7wHZc,3540
9
- rclone_api/dir_listing.py,sha256=9Qqf2SUswrOEkyqmaH23V51I18X6ePiXb9B1vUwRF5o,1571
9
+ rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,1882
10
10
  rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
11
- rclone_api/file.py,sha256=YtR5Y6c0YfXTS-sReOy2UgiSnafcAeO6b2hnbojBQD4,1423
11
+ rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
12
12
  rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
13
- rclone_api/group_files.py,sha256=kOHh6ysFDkxjldSwvW6KqmiADUC1yFCdrZRY57TvbGY,5328
13
+ rclone_api/group_files.py,sha256=CiD2eRVyBn7_xumU0WvPW1268H3VTSy4m_7ZnOy-abg,6991
14
14
  rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
15
- rclone_api/rclone.py,sha256=JeeqJsJH8rBZW895rkT3HMCkaWeKllnG-6xj_PzLwXY,29738
16
- rclone_api/remote.py,sha256=c9hlRKBCg1BFB9MCINaQIoCg10qyAkeqiS4brl8ce-8,343
15
+ rclone_api/rclone.py,sha256=e-SVNjxEGSvyyDXOv7AnLek7Wjt0NYDfBtvWSOA5F78,32858
16
+ rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
17
17
  rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
18
18
  rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
19
- rclone_api/types.py,sha256=DcbNw1R6j2f_1zHrhqEJcpCR-8kJfFJawMY0AmPsCnM,321
20
- rclone_api/util.py,sha256=_cvmHcJPRl2yXw4zgZiop3z-riA_8Ek6S5NDPw8cqSY,4198
19
+ rclone_api/types.py,sha256=9-qY0Z1NFwiIrvV3e4Ty-dVfed1I_482jPohJ-2hs-o,567
20
+ rclone_api/util.py,sha256=XMrA2m_di4h8JTM-qyx2iyrFZn-l-or_SJOa5tEsDsI,4200
21
21
  rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
22
22
  rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
23
23
  rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
24
- rclone_api-1.0.76.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
25
- rclone_api-1.0.76.dist-info/METADATA,sha256=13_xQBLBt8rYCmVYCXjHahKRMdC8p1Ao0lKOY8xhlVU,4489
26
- rclone_api-1.0.76.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
27
- rclone_api-1.0.76.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
28
- rclone_api-1.0.76.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
29
- rclone_api-1.0.76.dist-info/RECORD,,
24
+ rclone_api-1.0.78.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
25
+ rclone_api-1.0.78.dist-info/METADATA,sha256=DRNqEuwtsDKLk4yejLHUugWmD2oKBCSUGBxwZmSmYMw,4489
26
+ rclone_api-1.0.78.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
27
+ rclone_api-1.0.78.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
28
+ rclone_api-1.0.78.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
29
+ rclone_api-1.0.78.dist-info/RECORD,,