dvc-objects 0.20.0__tar.gz → 0.21.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/.cruft.json +1 -1
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/.pre-commit-config.yaml +5 -6
- {dvc-objects-0.20.0/src/dvc_objects.egg-info → dvc-objects-0.21.1}/PKG-INFO +1 -1
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/pyproject.toml +5 -2
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/setup.cfg +5 -5
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/_tqdm.py +3 -9
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/db.py +7 -21
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/executors.py +1 -3
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/__init__.py +4 -12
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/base.py +11 -33
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/callbacks.py +1 -3
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/generic.py +5 -8
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/local.py +1 -3
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/memory.py +1 -3
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/path.py +2 -6
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/utils.py +1 -10
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/obj.py +1 -5
- {dvc-objects-0.20.0 → dvc-objects-0.21.1/src/dvc_objects.egg-info}/PKG-INFO +1 -1
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/fs/test_callbacks.py +1 -3
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/fs/test_localfs.py +4 -11
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/fs/test_system.py +1 -7
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/test_odb.py +2 -6
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/test_reflink.py +1 -3
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/.gitattributes +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/.github/dependabot.yml +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/.github/workflows/release.yml +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/.github/workflows/tests.yml +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/.github/workflows/update-template.yaml +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/.gitignore +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/CODE_OF_CONDUCT.rst +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/CONTRIBUTING.rst +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/LICENSE +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/README.rst +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/noxfile.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/__init__.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/errors.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/errors.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/scheme.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/fs/system.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/py.typed +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects/transfer.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects.egg-info/SOURCES.txt +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects.egg-info/dependency_links.txt +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects.egg-info/not-zip-safe +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects.egg-info/requires.txt +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/src/dvc_objects.egg-info/top_level.txt +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/__init__.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/conftest.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/fs/__init__.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/fs/test_utils.py +0 -0
- {dvc-objects-0.20.0 → dvc-objects-0.21.1}/tests/test_transfer.py +0 -0
|
@@ -2,11 +2,11 @@ default_language_version:
|
|
|
2
2
|
python: python3
|
|
3
3
|
repos:
|
|
4
4
|
- repo: https://github.com/psf/black
|
|
5
|
-
rev:
|
|
5
|
+
rev: 23.1.0
|
|
6
6
|
hooks:
|
|
7
7
|
- id: black
|
|
8
8
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
9
|
-
rev: v4.
|
|
9
|
+
rev: v4.4.0
|
|
10
10
|
hooks:
|
|
11
11
|
- id: check-added-large-files
|
|
12
12
|
- id: check-case-conflict
|
|
@@ -29,7 +29,7 @@ repos:
|
|
|
29
29
|
- id: codespell
|
|
30
30
|
additional_dependencies: ["tomli"]
|
|
31
31
|
- repo: https://github.com/asottile/pyupgrade
|
|
32
|
-
rev: v3.
|
|
32
|
+
rev: v3.3.1
|
|
33
33
|
hooks:
|
|
34
34
|
- id: pyupgrade
|
|
35
35
|
args: [--py38-plus]
|
|
@@ -38,12 +38,11 @@ repos:
|
|
|
38
38
|
hooks:
|
|
39
39
|
- id: isort
|
|
40
40
|
- repo: https://github.com/pycqa/flake8
|
|
41
|
-
rev:
|
|
41
|
+
rev: 6.0.0
|
|
42
42
|
hooks:
|
|
43
43
|
- id: flake8
|
|
44
44
|
additional_dependencies:
|
|
45
|
-
- flake8-
|
|
46
|
-
- flake8-bugbear==22.10.27
|
|
45
|
+
- flake8-bugbear==23.1.20
|
|
47
46
|
- flake8-comprehensions==3.10.1
|
|
48
47
|
- flake8-debugger==4.1.2
|
|
49
48
|
- flake8-string-format==0.3.0
|
|
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
[tool.setuptools_scm]
|
|
6
6
|
|
|
7
7
|
[tool.black]
|
|
8
|
-
line-length =
|
|
8
|
+
line-length = 88
|
|
9
9
|
include = '\.pyi?$'
|
|
10
10
|
exclude = '''
|
|
11
11
|
/(
|
|
@@ -25,7 +25,7 @@ exclude = '''
|
|
|
25
25
|
[tool.isort]
|
|
26
26
|
profile = "black"
|
|
27
27
|
known_first_party = ["dvc_objects"]
|
|
28
|
-
line_length =
|
|
28
|
+
line_length = 88
|
|
29
29
|
|
|
30
30
|
[tool.pytest.ini_options]
|
|
31
31
|
addopts = "-ra -Werror"
|
|
@@ -64,6 +64,9 @@ warn_unreachable = true
|
|
|
64
64
|
ignore_missing_imports = true
|
|
65
65
|
files = ["src", "tests"]
|
|
66
66
|
|
|
67
|
+
[tool.pylint.format]
|
|
68
|
+
max-line-length = 88
|
|
69
|
+
|
|
67
70
|
[tool.pylint.message_control]
|
|
68
71
|
enable = ["no-else-return"]
|
|
69
72
|
disable = [
|
|
@@ -53,11 +53,11 @@ dvc_objects = py.typed
|
|
|
53
53
|
|
|
54
54
|
[flake8]
|
|
55
55
|
ignore =
|
|
56
|
-
E203
|
|
57
|
-
E266
|
|
58
|
-
W503
|
|
59
|
-
P1
|
|
60
|
-
max_line_length =
|
|
56
|
+
E203
|
|
57
|
+
E266
|
|
58
|
+
W503
|
|
59
|
+
P1
|
|
60
|
+
max_line_length = 88
|
|
61
61
|
max-complexity = 15
|
|
62
62
|
select = B,C,E,F,W,T4,B902,T,P
|
|
63
63
|
show_source = true
|
|
@@ -37,9 +37,7 @@ class Tqdm(tqdm):
|
|
|
37
37
|
"{postfix[info]}{n_fmt}/{total_fmt}"
|
|
38
38
|
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
|
|
39
39
|
)
|
|
40
|
-
BAR_FMT_NOTOTAL =
|
|
41
|
-
"{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
|
|
42
|
-
)
|
|
40
|
+
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
|
|
43
41
|
BYTES_DEFAULTS = {
|
|
44
42
|
"unit": "B",
|
|
45
43
|
"unit_scale": True,
|
|
@@ -104,9 +102,7 @@ class Tqdm(tqdm):
|
|
|
104
102
|
if bar_format is None:
|
|
105
103
|
if self.__len__():
|
|
106
104
|
self.bar_format = (
|
|
107
|
-
self.BAR_FMT_DEFAULT_NESTED
|
|
108
|
-
if self.pos
|
|
109
|
-
else self.BAR_FMT_DEFAULT
|
|
105
|
+
self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT
|
|
110
106
|
)
|
|
111
107
|
else:
|
|
112
108
|
self.bar_format = self.BAR_FMT_NOTOTAL
|
|
@@ -148,9 +144,7 @@ class Tqdm(tqdm):
|
|
|
148
144
|
d = super().format_dict
|
|
149
145
|
ncols = d["ncols"] or 80
|
|
150
146
|
# assumes `bar_format` has max one of ("ncols_desc" & "ncols_info")
|
|
151
|
-
ncols_left = (
|
|
152
|
-
ncols - len(self.format_meter(ncols_desc=1, ncols_info=1, **d)) + 1
|
|
153
|
-
)
|
|
147
|
+
ncols_left = ncols - len(self.format_meter(ncols_desc=1, ncols_info=1, **d)) + 1
|
|
154
148
|
ncols_left = max(ncols_left, 0)
|
|
155
149
|
if ncols_left:
|
|
156
150
|
d["ncols_desc"] = d["ncols_info"] = ncols_left
|
|
@@ -203,9 +203,7 @@ class ObjectDB:
|
|
|
203
203
|
jobs: Optional[int] = None,
|
|
204
204
|
) -> Iterator[str]:
|
|
205
205
|
if prefixes:
|
|
206
|
-
paths: Union[str, List[str]] = list(
|
|
207
|
-
map(self.oid_to_path, prefixes)
|
|
208
|
-
)
|
|
206
|
+
paths: Union[str, List[str]] = list(map(self.oid_to_path, prefixes))
|
|
209
207
|
if len(paths) == 1:
|
|
210
208
|
paths = paths[0]
|
|
211
209
|
prefix = True
|
|
@@ -236,9 +234,7 @@ class ObjectDB:
|
|
|
236
234
|
try:
|
|
237
235
|
yield self.path_to_oid(path)
|
|
238
236
|
except ValueError:
|
|
239
|
-
logger.debug(
|
|
240
|
-
"'%s' doesn't look like a cache file, skipping", path
|
|
241
|
-
)
|
|
237
|
+
logger.debug("'%s' doesn't look like a cache file, skipping", path)
|
|
242
238
|
|
|
243
239
|
def _oids_with_limit(
|
|
244
240
|
self,
|
|
@@ -287,9 +283,7 @@ class ObjectDB:
|
|
|
287
283
|
yield oid
|
|
288
284
|
|
|
289
285
|
if max_oids:
|
|
290
|
-
oids = self._oids_with_limit(
|
|
291
|
-
max_oids / total_prefixes, prefixes=[prefix]
|
|
292
|
-
)
|
|
286
|
+
oids = self._oids_with_limit(max_oids / total_prefixes, prefixes=[prefix])
|
|
293
287
|
else:
|
|
294
288
|
oids = self._list_oids(prefixes=[prefix])
|
|
295
289
|
|
|
@@ -403,9 +397,7 @@ class ObjectDB:
|
|
|
403
397
|
always_traverse = getattr(self.fs, "_ALWAYS_TRAVERSE", False)
|
|
404
398
|
|
|
405
399
|
oids = set(oids)
|
|
406
|
-
if (
|
|
407
|
-
len(oids) == 1 or not self.fs.CAN_TRAVERSE
|
|
408
|
-
) and not always_traverse:
|
|
400
|
+
if (len(oids) == 1 or not self.fs.CAN_TRAVERSE) and not always_traverse:
|
|
409
401
|
remote_oids = self.list_oids_exists(oids, jobs=jobs)
|
|
410
402
|
callback = partial(progress, "querying", len(oids))
|
|
411
403
|
return list(wrap_iter(remote_oids, callback))
|
|
@@ -423,9 +415,7 @@ class ObjectDB:
|
|
|
423
415
|
# From testing with S3, for remotes with 1M+ files, object_exists is
|
|
424
416
|
# faster until len(oids) is at least 10k~100k
|
|
425
417
|
if remote_size > self.fs.TRAVERSE_THRESHOLD_SIZE:
|
|
426
|
-
traverse_weight =
|
|
427
|
-
traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER
|
|
428
|
-
)
|
|
418
|
+
traverse_weight = traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER
|
|
429
419
|
else:
|
|
430
420
|
traverse_weight = traverse_pages
|
|
431
421
|
if len(oids) < traverse_weight and not always_traverse:
|
|
@@ -439,15 +429,11 @@ class ObjectDB:
|
|
|
439
429
|
ret = list(oids & remote_oids)
|
|
440
430
|
callback = partial(progress, "querying", len(remaining_oids))
|
|
441
431
|
ret.extend(
|
|
442
|
-
wrap_iter(
|
|
443
|
-
self.list_oids_exists(remaining_oids, jobs=jobs), callback
|
|
444
|
-
)
|
|
432
|
+
wrap_iter(self.list_oids_exists(remaining_oids, jobs=jobs), callback)
|
|
445
433
|
)
|
|
446
434
|
return ret
|
|
447
435
|
|
|
448
436
|
logger.debug(f"Querying '{len(oids)}' oids via traverse")
|
|
449
|
-
remote_oids = self._list_oids_traverse(
|
|
450
|
-
remote_size, remote_oids, jobs=jobs
|
|
451
|
-
)
|
|
437
|
+
remote_oids = self._list_oids_traverse(remote_size, remote_oids, jobs=jobs)
|
|
452
438
|
callback = partial(progress, "querying", remote_size)
|
|
453
439
|
return list(oids & set(wrap_iter(remote_oids, callback)))
|
|
@@ -51,9 +51,7 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor):
|
|
|
51
51
|
it = zip(*iterables)
|
|
52
52
|
tasks = create_taskset(self.max_workers * 5)
|
|
53
53
|
while tasks:
|
|
54
|
-
done, tasks = futures.wait(
|
|
55
|
-
tasks, return_when=futures.FIRST_COMPLETED
|
|
56
|
-
)
|
|
54
|
+
done, tasks = futures.wait(tasks, return_when=futures.FIRST_COMPLETED)
|
|
57
55
|
for fut in done:
|
|
58
56
|
yield fut.result()
|
|
59
57
|
tasks.update(create_taskset(len(done)))
|
|
@@ -22,9 +22,7 @@ known_implementations = {
|
|
|
22
22
|
},
|
|
23
23
|
Schemes.GDRIVE: {
|
|
24
24
|
"class": "dvc_gdrive.GDriveFileSystem",
|
|
25
|
-
"err": (
|
|
26
|
-
"gdrive is supported, but requires 'dvc-gdrive' to be installed"
|
|
27
|
-
),
|
|
25
|
+
"err": ("gdrive is supported, but requires 'dvc-gdrive' to be installed"),
|
|
28
26
|
},
|
|
29
27
|
Schemes.GS: {
|
|
30
28
|
"class": "dvc_gs.GSFileSystem",
|
|
@@ -56,21 +54,15 @@ known_implementations = {
|
|
|
56
54
|
},
|
|
57
55
|
Schemes.WEBDAV: {
|
|
58
56
|
"class": "dvc_webdav.WebDAVFileSystem",
|
|
59
|
-
"err": (
|
|
60
|
-
"webdav is supported, but requires 'dvc-webdav' to be installed"
|
|
61
|
-
),
|
|
57
|
+
"err": ("webdav is supported, but requires 'dvc-webdav' to be installed"),
|
|
62
58
|
},
|
|
63
59
|
Schemes.WEBDAVS: {
|
|
64
60
|
"class": "dvc_webdav.WebDAVSFileSystem",
|
|
65
|
-
"err": (
|
|
66
|
-
"webdavs is supported, but requires 'dvc-webdav' to be installed"
|
|
67
|
-
),
|
|
61
|
+
"err": ("webdavs is supported, but requires 'dvc-webdav' to be installed"),
|
|
68
62
|
},
|
|
69
63
|
Schemes.WEBHDFS: {
|
|
70
64
|
"class": "dvc_webhdfs.WebHDFSFileSystem",
|
|
71
|
-
"err": (
|
|
72
|
-
"webhdfs is supported, but requires 'dvc-webhdfs' to be installed"
|
|
73
|
-
),
|
|
65
|
+
"err": ("webhdfs is supported, but requires 'dvc-webhdfs' to be installed"),
|
|
74
66
|
},
|
|
75
67
|
}
|
|
76
68
|
|
|
@@ -209,9 +209,7 @@ class FileSystem:
|
|
|
209
209
|
on_error: Literal["raise", "omit", "return"] = "raise",
|
|
210
210
|
**kwargs: Any,
|
|
211
211
|
) -> Union[bytes, Dict[AnyFSPath, bytes]]:
|
|
212
|
-
return self.fs.cat(
|
|
213
|
-
path, recursive=recursive, on_error=on_error, **kwargs
|
|
214
|
-
)
|
|
212
|
+
return self.fs.cat(path, recursive=recursive, on_error=on_error, **kwargs)
|
|
215
213
|
|
|
216
214
|
def cat_ranges(
|
|
217
215
|
self,
|
|
@@ -221,9 +219,7 @@ class FileSystem:
|
|
|
221
219
|
max_gap: int = None,
|
|
222
220
|
**kwargs,
|
|
223
221
|
) -> List[bytes]:
|
|
224
|
-
return self.fs.cat_ranges(
|
|
225
|
-
paths, starts, ends, max_gap=max_gap, **kwargs
|
|
226
|
-
)
|
|
222
|
+
return self.fs.cat_ranges(paths, starts, ends, max_gap=max_gap, **kwargs)
|
|
227
223
|
|
|
228
224
|
def cat_file(
|
|
229
225
|
self,
|
|
@@ -284,9 +280,7 @@ class FileSystem:
|
|
|
284
280
|
) -> None:
|
|
285
281
|
return self.fs.pipe(path, value=value, **kwargs)
|
|
286
282
|
|
|
287
|
-
def touch(
|
|
288
|
-
self, path: AnyFSPath, truncate: bool = True, **kwargs: Any
|
|
289
|
-
) -> None:
|
|
283
|
+
def touch(self, path: AnyFSPath, truncate: bool = True, **kwargs: Any) -> None:
|
|
290
284
|
return self.fs.touch(path, truncate=truncate, **kwargs)
|
|
291
285
|
|
|
292
286
|
def checksum(self, path: AnyFSPath) -> str:
|
|
@@ -296,9 +290,7 @@ class FileSystem:
|
|
|
296
290
|
self.makedirs(self.path.parent(to_info))
|
|
297
291
|
self.fs.copy(from_info, to_info)
|
|
298
292
|
|
|
299
|
-
def cp_file(
|
|
300
|
-
self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any
|
|
301
|
-
) -> None:
|
|
293
|
+
def cp_file(self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any) -> None:
|
|
302
294
|
self.fs.cp_file(from_info, to_info, **kwargs)
|
|
303
295
|
|
|
304
296
|
@overload
|
|
@@ -388,15 +380,11 @@ class FileSystem:
|
|
|
388
380
|
return not (self.is_symlink(path) or self.is_hardlink(path))
|
|
389
381
|
|
|
390
382
|
@overload
|
|
391
|
-
def ls(
|
|
392
|
-
self, path: AnyFSPath, detail: Literal[True], **kwargs
|
|
393
|
-
) -> "Iterator[Entry]":
|
|
383
|
+
def ls(self, path: AnyFSPath, detail: Literal[True], **kwargs) -> "Iterator[Entry]":
|
|
394
384
|
...
|
|
395
385
|
|
|
396
386
|
@overload
|
|
397
|
-
def ls(
|
|
398
|
-
self, path: AnyFSPath, detail: Literal[False], **kwargs
|
|
399
|
-
) -> Iterator[str]:
|
|
387
|
+
def ls(self, path: AnyFSPath, detail: Literal[False], **kwargs) -> Iterator[str]:
|
|
400
388
|
...
|
|
401
389
|
|
|
402
390
|
def ls(self, path, detail=False, **kwargs):
|
|
@@ -434,9 +422,7 @@ class FileSystem:
|
|
|
434
422
|
for result in executor.imap_unordered(find, path):
|
|
435
423
|
yield from result
|
|
436
424
|
|
|
437
|
-
def mv(
|
|
438
|
-
self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any
|
|
439
|
-
) -> None:
|
|
425
|
+
def mv(self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any) -> None:
|
|
440
426
|
self.fs.mv(from_info, to_info)
|
|
441
427
|
|
|
442
428
|
move = mv
|
|
@@ -486,9 +472,7 @@ class FileSystem:
|
|
|
486
472
|
fut = asyncio.run_coroutine_threadsafe(
|
|
487
473
|
batch_coros(
|
|
488
474
|
[
|
|
489
|
-
self.fs._info( # pylint: disable=protected-access
|
|
490
|
-
p, **kwargs
|
|
491
|
-
)
|
|
475
|
+
self.fs._info(p, **kwargs) # pylint: disable=protected-access
|
|
492
476
|
for p in path
|
|
493
477
|
],
|
|
494
478
|
batch_size=jobs,
|
|
@@ -525,9 +509,7 @@ class FileSystem:
|
|
|
525
509
|
self.upload_fobj(stream, to_info, size=size)
|
|
526
510
|
else:
|
|
527
511
|
assert isinstance(from_file, str)
|
|
528
|
-
self.fs.put_file(
|
|
529
|
-
os.fspath(from_file), to_info, callback=callback, **kwargs
|
|
530
|
-
)
|
|
512
|
+
self.fs.put_file(os.fspath(from_file), to_info, callback=callback, **kwargs)
|
|
531
513
|
self.fs.invalidate_cache(self.path.parent(to_info))
|
|
532
514
|
|
|
533
515
|
def get_file(
|
|
@@ -631,9 +613,7 @@ class FileSystem:
|
|
|
631
613
|
return localfs.makedirs(to_info, exist_ok=True)
|
|
632
614
|
|
|
633
615
|
to_infos = [
|
|
634
|
-
localfs.path.join(
|
|
635
|
-
to_info, *self.path.relparts(info, from_info)
|
|
636
|
-
)
|
|
616
|
+
localfs.path.join(to_info, *self.path.relparts(info, from_info))
|
|
637
617
|
for info in from_infos
|
|
638
618
|
]
|
|
639
619
|
|
|
@@ -660,9 +640,7 @@ class FileSystem:
|
|
|
660
640
|
def modified(self, path: AnyFSPath) -> datetime.datetime:
|
|
661
641
|
return self.fs.modified(path)
|
|
662
642
|
|
|
663
|
-
def sign(
|
|
664
|
-
self, path: AnyFSPath, expiration: int = 100, **kwargs: Any
|
|
665
|
-
) -> str:
|
|
643
|
+
def sign(self, path: AnyFSPath, expiration: int = 100, **kwargs: Any) -> str:
|
|
666
644
|
return self.fs.sign(path, expiration=expiration, **kwargs)
|
|
667
645
|
|
|
668
646
|
|
|
@@ -103,9 +103,7 @@ class Callback(fsspec.Callback):
|
|
|
103
103
|
return super().absolute_update(value)
|
|
104
104
|
|
|
105
105
|
@classmethod
|
|
106
|
-
def as_callback(
|
|
107
|
-
cls, maybe_callback: Optional["Callback"] = None
|
|
108
|
-
) -> "Callback":
|
|
106
|
+
def as_callback(cls, maybe_callback: Optional["Callback"] = None) -> "Callback":
|
|
109
107
|
if maybe_callback is None:
|
|
110
108
|
return DEFAULT_CALLBACK
|
|
111
109
|
return maybe_callback
|
|
@@ -20,9 +20,7 @@ if TYPE_CHECKING:
|
|
|
20
20
|
logger = logging.getLogger(__name__)
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
TransferErrorHandler = Callable[
|
|
24
|
-
["AnyFSPath", "AnyFSPath", BaseException], None
|
|
25
|
-
]
|
|
23
|
+
TransferErrorHandler = Callable[["AnyFSPath", "AnyFSPath", BaseException], None]
|
|
26
24
|
|
|
27
25
|
|
|
28
26
|
def log_exceptions(func: Callable) -> Callable:
|
|
@@ -264,9 +262,7 @@ def _try_links(
|
|
|
264
262
|
|
|
265
263
|
del links[0]
|
|
266
264
|
|
|
267
|
-
raise OSError(
|
|
268
|
-
errno.ENOTSUP, "no more link types left to try out"
|
|
269
|
-
) from error
|
|
265
|
+
raise OSError(errno.ENOTSUP, "no more link types left to try out") from error
|
|
270
266
|
|
|
271
267
|
|
|
272
268
|
def transfer(
|
|
@@ -297,7 +293,7 @@ def transfer(
|
|
|
297
293
|
# copy, the remaining copy operations will be batched.
|
|
298
294
|
for i, (from_p, to_p) in enumerate(zip(from_path, to_path)):
|
|
299
295
|
if links[0] == "copy":
|
|
300
|
-
|
|
296
|
+
copy(
|
|
301
297
|
from_fs,
|
|
302
298
|
from_path[i:],
|
|
303
299
|
to_fs,
|
|
@@ -306,6 +302,7 @@ def transfer(
|
|
|
306
302
|
batch_size=batch_size,
|
|
307
303
|
on_error=on_error,
|
|
308
304
|
)
|
|
305
|
+
return
|
|
309
306
|
try:
|
|
310
307
|
_try_links(
|
|
311
308
|
links,
|
|
@@ -329,7 +326,7 @@ def transfer(
|
|
|
329
326
|
and isinstance(exc.__context__, FileExistsError)
|
|
330
327
|
):
|
|
331
328
|
logger.debug("'%s' file already exists, skipping", to_path)
|
|
332
|
-
|
|
329
|
+
continue
|
|
333
330
|
|
|
334
331
|
if on_error is not None:
|
|
335
332
|
on_error(from_p, to_p, exc)
|
|
@@ -61,9 +61,7 @@ class FsspecLocalFileSystem(fsspec.AbstractFileSystem):
|
|
|
61
61
|
topdown=topdown,
|
|
62
62
|
):
|
|
63
63
|
if detail:
|
|
64
|
-
dirs_dict = {
|
|
65
|
-
name: self.info(os.path.join(root, name)) for name in dirs
|
|
66
|
-
}
|
|
64
|
+
dirs_dict = {name: self.info(os.path.join(root, name)) for name in dirs}
|
|
67
65
|
files_dict = {
|
|
68
66
|
name: self.info(os.path.join(root, name)) for name in files
|
|
69
67
|
}
|
|
@@ -16,8 +16,6 @@ class MemoryFileSystem(FileSystem): # pylint:disable=abstract-method
|
|
|
16
16
|
self.fs.pseudo_dirs = [""]
|
|
17
17
|
|
|
18
18
|
def __eq__(self, other):
|
|
19
|
-
return (
|
|
20
|
-
isinstance(other, type(self)) and self.fs.store is other.fs.store
|
|
21
|
-
)
|
|
19
|
+
return isinstance(other, type(self)) and self.fs.store is other.fs.store
|
|
22
20
|
|
|
23
21
|
__hash__ = FileSystem.__hash__
|
|
@@ -133,13 +133,9 @@ class Path:
|
|
|
133
133
|
def relpath(self, path: str, start: Optional[str] = None) -> str:
|
|
134
134
|
if start is None:
|
|
135
135
|
start = "."
|
|
136
|
-
return self.flavour.relpath(
|
|
137
|
-
self.abspath(path), start=self.abspath(start)
|
|
138
|
-
)
|
|
136
|
+
return self.flavour.relpath(self.abspath(path), start=self.abspath(start))
|
|
139
137
|
|
|
140
|
-
def relparts(
|
|
141
|
-
self, path: str, start: Optional[str] = None
|
|
142
|
-
) -> Tuple[str, ...]:
|
|
138
|
+
def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]:
|
|
143
139
|
return self.parts(self.relpath(path, start=start))
|
|
144
140
|
|
|
145
141
|
def as_posix(self, path: str) -> str:
|
|
@@ -7,16 +7,7 @@ import sys
|
|
|
7
7
|
import threading
|
|
8
8
|
from concurrent import futures
|
|
9
9
|
from contextlib import contextmanager, suppress
|
|
10
|
-
from typing import
|
|
11
|
-
TYPE_CHECKING,
|
|
12
|
-
Any,
|
|
13
|
-
Collection,
|
|
14
|
-
Dict,
|
|
15
|
-
Iterator,
|
|
16
|
-
Optional,
|
|
17
|
-
Set,
|
|
18
|
-
Union,
|
|
19
|
-
)
|
|
10
|
+
from typing import TYPE_CHECKING, Any, Collection, Dict, Iterator, Optional, Set, Union
|
|
20
11
|
|
|
21
12
|
from ..executors import ThreadPoolExecutor
|
|
22
13
|
from . import system
|
|
@@ -29,11 +29,7 @@ class Object:
|
|
|
29
29
|
def __eq__(self, other):
|
|
30
30
|
if not isinstance(other, type(self)):
|
|
31
31
|
return False
|
|
32
|
-
return
|
|
33
|
-
self.path == other.path
|
|
34
|
-
and self.fs == other.fs
|
|
35
|
-
and self.oid == other.oid
|
|
36
|
-
)
|
|
32
|
+
return self.path == other.path and self.fs == other.fs and self.oid == other.oid
|
|
37
33
|
|
|
38
34
|
def __hash__(self):
|
|
39
35
|
return hash(
|
|
@@ -3,9 +3,7 @@ import pytest
|
|
|
3
3
|
from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
@pytest.mark.parametrize(
|
|
7
|
-
"api", ["set_size", "relative_update", "absolute_update"]
|
|
8
|
-
)
|
|
6
|
+
@pytest.mark.parametrize("api", ["set_size", "relative_update", "absolute_update"])
|
|
9
7
|
@pytest.mark.parametrize(
|
|
10
8
|
"callback_factory, kwargs",
|
|
11
9
|
[
|
|
@@ -6,9 +6,7 @@ import pytest
|
|
|
6
6
|
from dvc_objects.fs.local import LocalFileSystem
|
|
7
7
|
|
|
8
8
|
|
|
9
|
-
@pytest.mark.parametrize(
|
|
10
|
-
"path, contents", [("file", "foo"), ("тест", "проверка")]
|
|
11
|
-
)
|
|
9
|
+
@pytest.mark.parametrize("path, contents", [("file", "foo"), ("тест", "проверка")])
|
|
12
10
|
def test_local_fs_open(tmp_path, path, contents):
|
|
13
11
|
file = tmp_path / path
|
|
14
12
|
file.write_text(contents, encoding="utf8")
|
|
@@ -61,9 +59,7 @@ def test_local_fs_rm(tmp_path):
|
|
|
61
59
|
|
|
62
60
|
|
|
63
61
|
def convert_to_sets(walk_results):
|
|
64
|
-
return [
|
|
65
|
-
(root, set(dirs), set(nondirs)) for root, dirs, nondirs in walk_results
|
|
66
|
-
]
|
|
62
|
+
return [(root, set(dirs), set(nondirs)) for root, dirs, nondirs in walk_results]
|
|
67
63
|
|
|
68
64
|
|
|
69
65
|
@pytest.fixture
|
|
@@ -74,16 +70,13 @@ def dir_path(tmp_path):
|
|
|
74
70
|
("тест", "проверка"),
|
|
75
71
|
(
|
|
76
72
|
"code.py",
|
|
77
|
-
"import sys\nimport shutil\n"
|
|
78
|
-
"shutil.copyfile(sys.argv[1], sys.argv[2])",
|
|
73
|
+
"import sys\nimport shutil\n" "shutil.copyfile(sys.argv[1], sys.argv[2])",
|
|
79
74
|
),
|
|
80
75
|
]:
|
|
81
76
|
(tmp_path / file).write_text(contents, encoding="utf8")
|
|
82
77
|
(tmp_path / "data" / "sub").mkdir(parents=True)
|
|
83
78
|
(tmp_path / "data" / "file").write_text("file", encoding="utf8")
|
|
84
|
-
(tmp_path / "data" / "sub" / "file").write_text(
|
|
85
|
-
"sub_file", encoding="utf8"
|
|
86
|
-
)
|
|
79
|
+
(tmp_path / "data" / "sub" / "file").write_text("sub_file", encoding="utf8")
|
|
87
80
|
return tmp_path
|
|
88
81
|
|
|
89
82
|
|
|
@@ -1,13 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from os import fspath
|
|
3
3
|
|
|
4
|
-
from dvc_objects.fs.system import
|
|
5
|
-
hardlink,
|
|
6
|
-
inode,
|
|
7
|
-
is_hardlink,
|
|
8
|
-
is_symlink,
|
|
9
|
-
symlink,
|
|
10
|
-
)
|
|
4
|
+
from dvc_objects.fs.system import hardlink, inode, is_hardlink, is_symlink, symlink
|
|
11
5
|
|
|
12
6
|
|
|
13
7
|
def test_inode(tmp_path):
|
|
@@ -206,9 +206,7 @@ def test_oids_exist_less_oids_larger_fs(mocker):
|
|
|
206
206
|
odb._max_estimation_size(oids) / pow(16, odb.fs.TRAVERSE_PREFIX_LEN)
|
|
207
207
|
)
|
|
208
208
|
assert max_oids < 2048
|
|
209
|
-
object_exists.assert_called_with(
|
|
210
|
-
frozenset(range(max_oids, 1000)), jobs=None
|
|
211
|
-
)
|
|
209
|
+
object_exists.assert_called_with(frozenset(range(max_oids, 1000)), jobs=None)
|
|
212
210
|
traverse.assert_not_called()
|
|
213
211
|
|
|
214
212
|
|
|
@@ -264,9 +262,7 @@ def test_list_oids_traverse(mocker, prefix_len, extra_prefixes):
|
|
|
264
262
|
odb = ObjectDB(FileSystem(), "/odb")
|
|
265
263
|
|
|
266
264
|
list_oids = mocker.patch.object(odb, "_list_oids", return_value=[])
|
|
267
|
-
mocker.patch.object(
|
|
268
|
-
odb, "path_to_oid", side_effect=lambda x: x
|
|
269
|
-
) # pragma: no cover
|
|
265
|
+
mocker.patch.object(odb, "path_to_oid", side_effect=lambda x: x) # pragma: no cover
|
|
270
266
|
mocker.patch.object(odb.fs, "TRAVERSE_PREFIX_LEN", prefix_len)
|
|
271
267
|
|
|
272
268
|
# parallel traverse
|
|
@@ -33,9 +33,7 @@ def test_reflink(test_dir):
|
|
|
33
33
|
assert dest.is_file()
|
|
34
34
|
assert dest.read_bytes() == b"content"
|
|
35
35
|
|
|
36
|
-
stat_mode = src.stat().st_mode & (
|
|
37
|
-
stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
|
|
38
|
-
)
|
|
36
|
+
stat_mode = src.stat().st_mode & (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
|
|
39
37
|
assert stat_mode == (0o666 & ~umask(0))
|
|
40
38
|
|
|
41
39
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|