rclone-api 1.0.78__py2.py3-none-any.whl → 1.0.80__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rclone_api/group_files.py +56 -18
- rclone_api/rclone.py +34 -42
- {rclone_api-1.0.78.dist-info → rclone_api-1.0.80.dist-info}/METADATA +1 -1
- {rclone_api-1.0.78.dist-info → rclone_api-1.0.80.dist-info}/RECORD +8 -8
- {rclone_api-1.0.78.dist-info → rclone_api-1.0.80.dist-info}/LICENSE +0 -0
- {rclone_api-1.0.78.dist-info → rclone_api-1.0.80.dist-info}/WHEEL +0 -0
- {rclone_api-1.0.78.dist-info → rclone_api-1.0.80.dist-info}/entry_points.txt +0 -0
- {rclone_api-1.0.78.dist-info → rclone_api-1.0.80.dist-info}/top_level.txt +0 -0
rclone_api/group_files.py
CHANGED
@@ -1,4 +1,11 @@
|
|
1
1
|
from dataclasses import dataclass
|
2
|
+
from pathlib import Path
|
3
|
+
|
4
|
+
|
5
|
+
@dataclass
|
6
|
+
class PrefixResult:
|
7
|
+
prefix: str
|
8
|
+
files: list[str]
|
2
9
|
|
3
10
|
|
4
11
|
@dataclass
|
@@ -178,23 +185,6 @@ def group_files(files: list[str], fully_qualified: bool = True) -> dict[str, lis
|
|
178
185
|
return out
|
179
186
|
|
180
187
|
|
181
|
-
def group_under_remote(
|
182
|
-
files: list[str], fully_qualified: bool = True
|
183
|
-
) -> dict[str, list[str]]:
|
184
|
-
"""split between filename and remote"""
|
185
|
-
|
186
|
-
#### DOE STHIS NEED TO BE REMOVEDD????? #####
|
187
|
-
|
188
|
-
assert fully_qualified is True, "Not implemented for fully_qualified=False"
|
189
|
-
out: dict[str, list[str]] = {}
|
190
|
-
for file in files:
|
191
|
-
parsed = parse_file(file)
|
192
|
-
remote = f"{parsed.remote}:"
|
193
|
-
file_list = out.setdefault(remote, [])
|
194
|
-
file_list.append(parsed.to_string(include_remote=False, include_bucket=True))
|
195
|
-
return out
|
196
|
-
|
197
|
-
|
198
188
|
def group_under_remote_bucket(
|
199
189
|
files: list[str], fully_qualified: bool = True
|
200
190
|
) -> dict[str, list[str]]:
|
@@ -212,4 +202,52 @@ def group_under_remote_bucket(
|
|
212
202
|
return out
|
213
203
|
|
214
204
|
|
215
|
-
|
205
|
+
def _get_prefix(path: str) -> tuple[str, str] | None:
|
206
|
+
path_path: Path = Path(path)
|
207
|
+
parts = path_path.parts
|
208
|
+
if len(parts) == 1:
|
209
|
+
return None
|
210
|
+
return parts[0], "/".join(parts[1:])
|
211
|
+
|
212
|
+
|
213
|
+
def _common_prefix(prefix: str, files: list[str]) -> PrefixResult:
|
214
|
+
if not files:
|
215
|
+
return PrefixResult(prefix=prefix, files=[])
|
216
|
+
prefix = prefix
|
217
|
+
tmp: list[str] = list(files)
|
218
|
+
while True:
|
219
|
+
if not tmp:
|
220
|
+
break
|
221
|
+
prefix_set: set[str | None] = set()
|
222
|
+
for file in tmp:
|
223
|
+
pair = _get_prefix(file)
|
224
|
+
if pair is None:
|
225
|
+
break
|
226
|
+
_prefix, _ = pair
|
227
|
+
prefix_set.add(_prefix)
|
228
|
+
if len(prefix_set) > 1 or len(prefix_set) == 0:
|
229
|
+
break
|
230
|
+
next_prefix: str | None = prefix_set.pop()
|
231
|
+
if next_prefix is None:
|
232
|
+
break
|
233
|
+
prefix += f"/{next_prefix}"
|
234
|
+
new_tmp: list[str] = []
|
235
|
+
for file in tmp:
|
236
|
+
pair = _get_prefix(file)
|
237
|
+
assert pair is not None
|
238
|
+
_, path = pair
|
239
|
+
new_tmp.append(path)
|
240
|
+
tmp = new_tmp
|
241
|
+
return PrefixResult(prefix=prefix, files=tmp)
|
242
|
+
|
243
|
+
|
244
|
+
def group_under_one_prefix(prefix: str, files: list[str]) -> tuple[str, list[str]]:
|
245
|
+
"""Group files under one prefix."""
|
246
|
+
if not files:
|
247
|
+
return prefix, []
|
248
|
+
prefix = prefix
|
249
|
+
result = _common_prefix(prefix, files)
|
250
|
+
return result.prefix, result.files
|
251
|
+
|
252
|
+
|
253
|
+
__all__ = ["group_files", "group_under_remote_bucket", "group_under_one_prefix"]
|
rclone_api/rclone.py
CHANGED
@@ -22,10 +22,7 @@ from rclone_api.diff import DiffItem, DiffOption, diff_stream_from_running_proce
|
|
22
22
|
from rclone_api.dir_listing import DirListing
|
23
23
|
from rclone_api.exec import RcloneExec
|
24
24
|
from rclone_api.file import File
|
25
|
-
from rclone_api.group_files import
|
26
|
-
group_files,
|
27
|
-
group_under_remote_bucket,
|
28
|
-
)
|
25
|
+
from rclone_api.group_files import group_files, group_under_one_prefix
|
29
26
|
from rclone_api.process import Process
|
30
27
|
from rclone_api.remote import Remote
|
31
28
|
from rclone_api.rpath import RPath
|
@@ -849,44 +846,39 @@ class Rclone:
|
|
849
846
|
verbose = get_verbose(verbose)
|
850
847
|
check = get_check(check)
|
851
848
|
files = list(files)
|
852
|
-
prefix = src if src.endswith(":") else f"{src}/"
|
853
|
-
if src:
|
854
|
-
files = [f"{prefix}{f}" for f in files]
|
855
|
-
file_list: dict[str, list[str]]
|
856
|
-
file_list = group_under_remote_bucket(files)
|
857
849
|
all_files: list[File] = []
|
858
|
-
|
859
|
-
|
860
|
-
|
861
|
-
|
862
|
-
|
863
|
-
|
864
|
-
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
else:
|
875
|
-
warnings.warn(f"Error getting file sizes: {cp.stderr}")
|
876
|
-
stdout = cp.stdout
|
877
|
-
pieces = src_path.split(":", 1)
|
878
|
-
remote_name = pieces[0]
|
879
|
-
parent_path: str | None
|
880
|
-
if len(pieces) > 1:
|
881
|
-
parent_path = pieces[1]
|
850
|
+
src, files = group_under_one_prefix(src, files)
|
851
|
+
cmd = ["lsjson", src, "--files-only", "-R"]
|
852
|
+
with TemporaryDirectory() as tmpdir:
|
853
|
+
# print("files: " + ",".join(files))
|
854
|
+
include_files_txt = Path(tmpdir) / "include_files.txt"
|
855
|
+
include_files_txt.write_text("\n".join(files), encoding="utf-8")
|
856
|
+
cmd += ["--files-from", str(include_files_txt)]
|
857
|
+
if fast_list:
|
858
|
+
cmd.append("--fast-list")
|
859
|
+
if other_args:
|
860
|
+
cmd += other_args
|
861
|
+
cp = self._run(cmd, check=check)
|
862
|
+
|
863
|
+
if cp.returncode != 0:
|
864
|
+
if check:
|
865
|
+
raise ValueError(f"Error getting file sizes: {cp.stderr}")
|
882
866
|
else:
|
883
|
-
|
884
|
-
|
885
|
-
|
886
|
-
|
887
|
-
|
888
|
-
|
889
|
-
|
867
|
+
warnings.warn(f"Error getting file sizes: {cp.stderr}")
|
868
|
+
stdout = cp.stdout
|
869
|
+
pieces = src.split(":", 1)
|
870
|
+
remote_name = pieces[0]
|
871
|
+
parent_path: str | None
|
872
|
+
if len(pieces) > 1:
|
873
|
+
parent_path = pieces[1]
|
874
|
+
else:
|
875
|
+
parent_path = None
|
876
|
+
remote = Remote(name=remote_name, rclone=self)
|
877
|
+
paths: list[RPath] = RPath.from_json_str(
|
878
|
+
stdout, remote, parent_path=parent_path
|
879
|
+
)
|
880
|
+
# print(paths)
|
881
|
+
all_files += [File(p) for p in paths]
|
890
882
|
file_sizes: dict[str, int] = {}
|
891
883
|
f: File
|
892
884
|
for f in all_files:
|
@@ -903,9 +895,9 @@ class Rclone:
|
|
903
895
|
for path, size in file_sizes.items():
|
904
896
|
# remove the prefix
|
905
897
|
path_path = Path(path)
|
906
|
-
path_str = path_path.relative_to(
|
898
|
+
path_str = path_path.relative_to(src).as_posix()
|
907
899
|
file_sizes_path_corrected[path_str] = size
|
908
900
|
out: SizeResult = SizeResult(
|
909
|
-
prefix=
|
901
|
+
prefix=src, total_size=total_size, file_sizes=file_sizes_path_corrected
|
910
902
|
)
|
911
903
|
return out
|
@@ -10,9 +10,9 @@ rclone_api/dir_listing.py,sha256=GoziW8Sne6FY90MLNcb2aO3aaa3jphB6H8ExYrV0Ryo,188
|
|
10
10
|
rclone_api/exec.py,sha256=1ovvaMXDEfLiT7BrYZyE85u_yFhEUwUNW3jPOzqknR8,1023
|
11
11
|
rclone_api/file.py,sha256=EP5yT2dZ0H2p7CY5n0y5k5pHhIliV25pm8KOwBklUTk,1863
|
12
12
|
rclone_api/filelist.py,sha256=xbiusvNgaB_b_kQOZoHMJJxn6TWGtPrWd2J042BI28o,767
|
13
|
-
rclone_api/group_files.py,sha256=
|
13
|
+
rclone_api/group_files.py,sha256=nCNDS_V0M-qK_ydZHvbz0LAOWcW0CcWnst86JBmuhsU,7950
|
14
14
|
rclone_api/process.py,sha256=RrMfTe0bndmJ6gBK67ioqNvCstJ8aTC8RlGX1XBLlcw,4191
|
15
|
-
rclone_api/rclone.py,sha256=
|
15
|
+
rclone_api/rclone.py,sha256=L12NGR8f8AlHGZ7qrHZuKQsF0XXSU023RTIDpu5ntkM,32494
|
16
16
|
rclone_api/remote.py,sha256=O9WDUFQy9f6oT1HdUbTixK2eg0xtBBm8k4Xl6aa6K00,431
|
17
17
|
rclone_api/rpath.py,sha256=8ZA_1wxWtskwcy0I8V2VbjKDmzPkiWd8Q2JQSvh-sYE,2586
|
18
18
|
rclone_api/scan_missing_folders.py,sha256=Kulca2Q6WZodt00ATFHkmqqInuoPvBkhTcS9703y6po,4740
|
@@ -21,9 +21,9 @@ rclone_api/util.py,sha256=XMrA2m_di4h8JTM-qyx2iyrFZn-l-or_SJOa5tEsDsI,4200
|
|
21
21
|
rclone_api/walk.py,sha256=-54NVE8EJcCstwDoaC_UtHm73R2HrZwVwQmsnv55xNU,3369
|
22
22
|
rclone_api/assets/example.txt,sha256=lTBovRjiz0_TgtAtbA1C5hNi2ffbqnNPqkKg6UiKCT8,54
|
23
23
|
rclone_api/cmd/list_files.py,sha256=x8FHODEilwKqwdiU1jdkeJbLwOqUkUQuDWPo2u_zpf0,741
|
24
|
-
rclone_api-1.0.
|
25
|
-
rclone_api-1.0.
|
26
|
-
rclone_api-1.0.
|
27
|
-
rclone_api-1.0.
|
28
|
-
rclone_api-1.0.
|
29
|
-
rclone_api-1.0.
|
24
|
+
rclone_api-1.0.80.dist-info/LICENSE,sha256=b6pOoifSXiUaz_lDS84vWlG3fr4yUKwB8fzkrH9R8bQ,1064
|
25
|
+
rclone_api-1.0.80.dist-info/METADATA,sha256=N3sS1E53Jk21CoeXsSqTd9TpSFNlEZt2WfV7KQS49b0,4489
|
26
|
+
rclone_api-1.0.80.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
|
27
|
+
rclone_api-1.0.80.dist-info/entry_points.txt,sha256=XUoTX3m7CWxdj2VAKhEuO0NMOfX2qf-OcEDFwdyk9ZE,72
|
28
|
+
rclone_api-1.0.80.dist-info/top_level.txt,sha256=EvZ7uuruUpe9RiUyEp25d1Keq7PWYNT0O_-mr8FCG5g,11
|
29
|
+
rclone_api-1.0.80.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|