rclone-api 1.0.78__py2.py3-none-any.whl → 1.0.80__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rclone_api/group_files.py CHANGED
@@ -1,4 +1,11 @@
1
1
  from dataclasses import dataclass
2
+ from pathlib import Path
3
+
4
+
5
+ @dataclass
6
+ class PrefixResult:
7
+ prefix: str
8
+ files: list[str]
2
9
 
3
10
 
4
11
  @dataclass
@@ -178,23 +185,6 @@ def group_files(files: list[str], fully_qualified: bool = True) -> dict[str, lis
178
185
  return out
179
186
 
180
187
 
181
- def group_under_remote(
182
- files: list[str], fully_qualified: bool = True
183
- ) -> dict[str, list[str]]:
184
- """split between filename and remote"""
185
-
186
- #### DOE STHIS NEED TO BE REMOVEDD????? #####
187
-
188
- assert fully_qualified is True, "Not implemented for fully_qualified=False"
189
- out: dict[str, list[str]] = {}
190
- for file in files:
191
- parsed = parse_file(file)
192
- remote = f"{parsed.remote}:"
193
- file_list = out.setdefault(remote, [])
194
- file_list.append(parsed.to_string(include_remote=False, include_bucket=True))
195
- return out
196
-
197
-
198
188
  def group_under_remote_bucket(
199
189
  files: list[str], fully_qualified: bool = True
200
190
  ) -> dict[str, list[str]]:
@@ -212,4 +202,52 @@ def group_under_remote_bucket(
212
202
  return out
213
203
 
214
204
 
215
- __all__ = ["group_files", "group_under_remote", "group_under_remote_bucket"]
205
+ def _get_prefix(path: str) -> tuple[str, str] | None:
206
+ path_path: Path = Path(path)
207
+ parts = path_path.parts
208
+ if len(parts) == 1:
209
+ return None
210
+ return parts[0], "/".join(parts[1:])
211
+
212
+
213
+ def _common_prefix(prefix: str, files: list[str]) -> PrefixResult:
214
+ if not files:
215
+ return PrefixResult(prefix=prefix, files=[])
216
+ prefix = prefix
217
+ tmp: list[str] = list(files)
218
+ while True:
219
+ if not tmp:
220
+ break
221
+ prefix_set: set[str | None] = set()
222
+ for file in tmp:
223
+ pair = _get_prefix(file)
224
+ if pair is None:
225
+ break
226
+ _prefix, _ = pair
227
+ prefix_set.add(_prefix)
228
+ if len(prefix_set) > 1 or len(prefix_set) == 0:
229
+ break
230
+ next_prefix: str | None = prefix_set.pop()
231
+ if next_prefix is None:
232
+ break
233
+ prefix += f"/{next_prefix}"
234
+ new_tmp: list[str] = []
235
+ for file in tmp:
236
+ pair = _get_prefix(file)
237
+ assert pair is not None
238
+ _, path = pair
239
+ new_tmp.append(path)
240
+ tmp = new_tmp
241
+ return PrefixResult(prefix=prefix, files=tmp)
242
+
243
+
244
+ def group_under_one_prefix(prefix: str, files: list[str]) -> tuple[str, list[str]]:
245
+ """Group files under one prefix."""
246
+ if not files:
247
+ return prefix, []
248
+ prefix = prefix
249
+ result = _common_prefix(prefix, files)
250
+ return result.prefix, result.files
251
+
252
+
253
+ __all__ = ["group_files", "group_under_remote_bucket", "group_under_one_prefix"]
rclone_api/rclone.py CHANGED
@@ -22,10 +22,7 @@ from rclone_api.diff import DiffItem, DiffOption, diff_stream_from_running_proce
22
22
  from rclone_api.dir_listing import DirListing
23
23
  from rclone_api.exec import RcloneExec
24
24
  from rclone_api.file import File
25
- from rclone_api.group_files import (
26
- group_files,
27
- group_under_remote_bucket,
28
- )
25
+ from rclone_api.group_files import group_files, group_under_one_prefix
29
26
  from rclone_api.process import Process
30
27
  from rclone_api.remote import Remote
31
28
  from rclone_api.rpath import RPath
@@ -849,44 +846,39 @@ class Rclone:
849
846
  verbose = get_verbose(verbose)
850
847
  check = get_check(check)
851
848
  files = list(files)
852
- prefix = src if src.endswith(":") else f"{src}/"
853
- if src:
854
- files = [f"{prefix}{f}" for f in files]
855
- file_list: dict[str, list[str]]
856
- file_list = group_under_remote_bucket(files)
857
849
  all_files: list[File] = []
858
- for src_path, files in file_list.items():
859
- cmd = ["lsjson", src_path, "--files-only", "-R"]
860
- with TemporaryDirectory() as tmpdir:
861
- # print("files: " + ",".join(files))
862
- include_files_txt = Path(tmpdir) / "include_files.txt"
863
- include_files_txt.write_text("\n".join(files), encoding="utf-8")
864
- cmd += ["--files-from", str(include_files_txt)]
865
- if fast_list:
866
- cmd.append("--fast-list")
867
- if other_args:
868
- cmd += other_args
869
- cp = self._run(cmd, check=check)
870
-
871
- if cp.returncode != 0:
872
- if check:
873
- raise ValueError(f"Error getting file sizes: {cp.stderr}")
874
- else:
875
- warnings.warn(f"Error getting file sizes: {cp.stderr}")
876
- stdout = cp.stdout
877
- pieces = src_path.split(":", 1)
878
- remote_name = pieces[0]
879
- parent_path: str | None
880
- if len(pieces) > 1:
881
- parent_path = pieces[1]
850
+ src, files = group_under_one_prefix(src, files)
851
+ cmd = ["lsjson", src, "--files-only", "-R"]
852
+ with TemporaryDirectory() as tmpdir:
853
+ # print("files: " + ",".join(files))
854
+ include_files_txt = Path(tmpdir) / "include_files.txt"
855
+ include_files_txt.write_text("\n".join(files), encoding="utf-8")
856
+ cmd += ["--files-from", str(include_files_txt)]
857
+ if fast_list:
858
+ cmd.append("--fast-list")
859
+ if other_args:
860
+ cmd += other_args
861
+ cp = self._run(cmd, check=check)
862
+
863
+ if cp.returncode != 0:
864
+ if check:
865
+ raise ValueError(f"Error getting file sizes: {cp.stderr}")
882
866
  else:
883
- parent_path = None
884
- remote = Remote(name=remote_name, rclone=self)
885
- paths: list[RPath] = RPath.from_json_str(
886
- stdout, remote, parent_path=parent_path
887
- )
888
- # print(paths)
889
- all_files += [File(p) for p in paths]
867
+ warnings.warn(f"Error getting file sizes: {cp.stderr}")
868
+ stdout = cp.stdout
869
+ pieces = src.split(":", 1)
870
+ remote_name = pieces[0]
871
+ parent_path: str | None
872
+ if len(pieces) > 1:
873
+ parent_path = pieces[1]
874
+ else:
875
+ parent_path = None
876
+ remote = Remote(name=remote_name, rclone=self)
877
+ paths: list[RPath] = RPath.from_json_str(
878
+ stdout, remote, parent_path=parent_path
879
+ )
880
+ # print(paths)
881
+ all_files += [File(p) for p in paths]
890
882
  file_sizes: dict[str, int] = {}
891
883
  f: File
892
884
  for f in all_files:
@@ -903,9 +895,9 @@ class Rclone:
903
895
  for path, size in file_sizes.items():
904
896
  # remove the prefix
905
897
  path_path = Path(path)
906
- path_str = path_path.relative_to(prefix).as_posix()
898
+ path_str = path_path.relative_to(src).as_posix()
907
899
  file_sizes_path_corrected[path_str] = size
908
900
  out: SizeResult = SizeResult(
909
- prefix=prefix, total_size=total_size, file_sizes=file_sizes_path_corrected
901
+ prefix=src, total_size=total_size, file_sizes=file_sizes_path_corrected
910
902
  )
911
903
  return out
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: rclone_api
3
- Version: 1.0.78
3
+ Version: 1.0.80
4
4
  Summary: rclone api in python
5
5
  Home-page: https://github.com/zackees/rclone-api
6
6
  Maintainer: Zachary Vorhies
@@ -10,9 +10,9 @@ rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,188
10
10
  rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
11
11
  rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
12
12
  rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
13
- rclone_api/group_files.py,sha256=CiD2eRVyBn7_xumU0WvPW1268H3VTSy4m_7ZnOy-abg,6991
13
+ rclone_api/group_files.py,sha256=nCNDS_V0M-qK_ydZHvbz0LAOWcW0CcWnst86JBmuhsU,7950
14
14
  rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
15
- rclone_api/rclone.py,sha256=e-SVNjxEGSvyyDXOv7AnLek7Wjt0NYDfBtvWSOA5F78,32858
15
+ rclone_api/rclone.py,sha256=L12NGR8f8AlHGZ7qrHZuKQsF0XXSU023RTIDpu5ntkM,32494
16
16
  rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
17
17
  rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
18
18
  rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
@@ -21,9 +21,9 @@ rclone_api/util.py,sha256=XMrA2m_di4h8JTM-qyx2iyrFZn-l-or_SJOa5tEsDsI,4200
21
21
  rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
22
22
  rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
23
23
  rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
24
- rclone_api-1.0.78.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
25
- rclone_api-1.0.78.dist-info/METADATA,sha256=DRNqEuwtsDKLk4yejLHUugWmD2oKBCSUGBxwZmSmYMw,4489
26
- rclone_api-1.0.78.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
27
- rclone_api-1.0.78.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
28
- rclone_api-1.0.78.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
29
- rclone_api-1.0.78.dist-info/RECORD,,
24
+ rclone_api-1.0.80.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
25
+ rclone_api-1.0.80.dist-info/METADATA,sha256=N3sS1E53Jk21CoeXsSqTd9TpSFNlEZt2WfV7KQS49b0,4489
26
+ rclone_api-1.0.80.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
27
+ rclone_api-1.0.80.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
28
+ rclone_api-1.0.80.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
29
+ rclone_api-1.0.80.dist-info/RECORD,,