rms-filecache 2.1.0__tar.gz → 2.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-2.1.0/rms_filecache.egg-info → rms_filecache-2.2.1}/PKG-INFO +1 -1
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/docs/module.rst +3 -3
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/filecache/_version.py +2 -2
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/filecache/file_cache.py +263 -7
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/filecache/file_cache_path.py +964 -198
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/filecache/file_cache_source.py +329 -11
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/mypy.ini +1 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/pyproject.toml +1 -1
- {rms_filecache-2.1.0 → rms_filecache-2.2.1/rms_filecache.egg-info}/PKG-INFO +1 -1
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/rms_filecache.egg-info/SOURCES.txt +0 -1
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/s3_bucket_policy_write.txt +8 -3
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/test_file_cache.py +487 -7
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/test_file_cache_path.py +356 -60
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/test_file_cache_source.py +8 -0
- rms_filecache-2.1.0/filecache/my_glob.py +0 -639
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/.coveragerc +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/.flake8 +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/.github/pull_request_template.md +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/.github/workflows/publish_to_pypi.yml +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/.github/workflows/publish_to_test_pypi.yml +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/.github/workflows/run-tests.yml +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/.gitignore +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/.readthedocs.yaml +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/LICENSE +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/README.md +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/docs/Makefile +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/docs/conf.py +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/docs/index.rst +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/docs/make.bat +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/filecache/__init__.py +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/filecache/file_cache_types.py +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/filecache/py.typed +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/requirements.txt +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/setup.cfg +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/test_programs/locking.py +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/test_programs/multi_download.py +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/__init__.py +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/s3_lifecycle_policy.txt +0 -0
- {rms_filecache-2.1.0 → rms_filecache-2.2.1}/tests/test_url_to_path.py +0 -0
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
:undoc-members:
|
|
8
8
|
:special-members:
|
|
9
9
|
:show-inheritance:
|
|
10
|
-
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
|
|
10
|
+
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__, __firstlineno__, __static_attributes__, __abstractmethods__
|
|
11
11
|
|
|
12
12
|
.. automodule:: filecache.file_cache_path
|
|
13
13
|
:member-order: bysource
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
:undoc-members:
|
|
16
16
|
:special-members:
|
|
17
17
|
:show-inheritance:
|
|
18
|
-
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
|
|
18
|
+
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__, __firstlineno__, __static_attributes__, __abstractmethods__
|
|
19
19
|
|
|
20
20
|
.. automodule:: filecache.file_cache_source
|
|
21
21
|
:member-order: bysource
|
|
@@ -23,4 +23,4 @@
|
|
|
23
23
|
:undoc-members:
|
|
24
24
|
:special-members:
|
|
25
25
|
:show-inheritance:
|
|
26
|
-
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
|
|
26
|
+
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__, __firstlineno__, __static_attributes__, __abstractmethods__
|
|
@@ -17,8 +17,8 @@ import time
|
|
|
17
17
|
from types import TracebackType
|
|
18
18
|
from typing import (cast,
|
|
19
19
|
Any,
|
|
20
|
-
Generator,
|
|
21
20
|
IO,
|
|
21
|
+
Iterator,
|
|
22
22
|
Literal,
|
|
23
23
|
Optional,
|
|
24
24
|
Type,
|
|
@@ -50,6 +50,7 @@ _SCHEME_CLASSES: dict[str, type[FileCacheSource]] = {}
|
|
|
50
50
|
|
|
51
51
|
def register_filecachesource(cls: type[FileCacheSource]) -> None:
|
|
52
52
|
"""Register one or more URL FileCacheSource subclasses as URL schemes."""
|
|
53
|
+
|
|
53
54
|
for s in cls.schemes():
|
|
54
55
|
_SCHEME_CLASSES[s] = cls
|
|
55
56
|
|
|
@@ -66,12 +67,14 @@ _GLOBAL_LOGGER: Logger | None = None
|
|
|
66
67
|
|
|
67
68
|
def set_global_logger(logger: Logger | None) -> None:
|
|
68
69
|
"""Set the global logger for all FileCache instances that don't specify one."""
|
|
70
|
+
|
|
69
71
|
global _GLOBAL_LOGGER
|
|
70
72
|
_GLOBAL_LOGGER = logger
|
|
71
73
|
|
|
72
74
|
|
|
73
75
|
def set_easy_logger() -> None:
|
|
74
76
|
"""Set a default logger that outputs all messages to stdout."""
|
|
77
|
+
|
|
75
78
|
easy_logger = logging.getLogger(__name__)
|
|
76
79
|
easy_logger.setLevel(logging.DEBUG)
|
|
77
80
|
|
|
@@ -89,6 +92,7 @@ def set_easy_logger() -> None:
|
|
|
89
92
|
|
|
90
93
|
def get_global_logger() -> Logger | None:
|
|
91
94
|
"""Return the current global logger."""
|
|
95
|
+
|
|
92
96
|
return _GLOBAL_LOGGER
|
|
93
97
|
|
|
94
98
|
|
|
@@ -264,41 +268,49 @@ class FileCache:
|
|
|
264
268
|
@property
|
|
265
269
|
def cache_dir(self) -> Path:
|
|
266
270
|
"""The top-level directory of the cache as a Path object."""
|
|
271
|
+
|
|
267
272
|
return self._cache_dir
|
|
268
273
|
|
|
269
274
|
@property
|
|
270
275
|
def download_counter(self) -> int:
|
|
271
276
|
"""The number of actual file downloads that have taken place."""
|
|
277
|
+
|
|
272
278
|
return self._download_counter
|
|
273
279
|
|
|
274
280
|
@property
|
|
275
281
|
def upload_counter(self) -> int:
|
|
276
282
|
"""The number of actual file uploads that have taken place."""
|
|
283
|
+
|
|
277
284
|
return self._upload_counter
|
|
278
285
|
|
|
279
286
|
@property
|
|
280
287
|
def anonymous(self) -> bool:
|
|
281
288
|
"""The default bool indicating whether to make all cloud accesses anonymous."""
|
|
289
|
+
|
|
282
290
|
return self._anonymous
|
|
283
291
|
|
|
284
292
|
@property
|
|
285
293
|
def lock_timeout(self) -> int:
|
|
286
294
|
"""The default timeout in seconds while waiting for a file lock."""
|
|
295
|
+
|
|
287
296
|
return self._lock_timeout
|
|
288
297
|
|
|
289
298
|
@property
|
|
290
299
|
def nthreads(self) -> int:
|
|
291
300
|
"""The default number of threads to use for multiple-file operations."""
|
|
301
|
+
|
|
292
302
|
return self._nthreads
|
|
293
303
|
|
|
294
304
|
@property
|
|
295
305
|
def delete_on_exit(self) -> bool:
|
|
296
306
|
"""A bool indicating whether this FileCache will be deleted on exit."""
|
|
307
|
+
|
|
297
308
|
return self._delete_on_exit
|
|
298
309
|
|
|
299
310
|
@property
|
|
300
311
|
def is_mp_safe(self) -> bool:
|
|
301
312
|
"""A bool indicating whether this FileCache is multi-processor safe."""
|
|
313
|
+
|
|
302
314
|
return self._is_mp_safe
|
|
303
315
|
|
|
304
316
|
def _log_debug(self, msg: str) -> None:
|
|
@@ -333,8 +345,10 @@ class FileCache:
|
|
|
333
345
|
elif len(parts) == 2:
|
|
334
346
|
slash_split = parts[1].split('/', maxsplit=1)
|
|
335
347
|
if len(slash_split) != 2:
|
|
336
|
-
|
|
337
|
-
|
|
348
|
+
remote = parts[1]
|
|
349
|
+
sub_path = ''
|
|
350
|
+
else:
|
|
351
|
+
remote, sub_path = slash_split
|
|
338
352
|
scheme = parts[0].lower()
|
|
339
353
|
if scheme == 'file':
|
|
340
354
|
# All file accesses are absolute
|
|
@@ -348,7 +362,8 @@ class FileCache:
|
|
|
348
362
|
else:
|
|
349
363
|
# file:///dir/file
|
|
350
364
|
# We have to add the / back on the beginning
|
|
351
|
-
|
|
365
|
+
if sub_path:
|
|
366
|
+
sub_path = f'/{sub_path}'
|
|
352
367
|
sub_path = str(Path(sub_path))
|
|
353
368
|
if scheme not in _SCHEME_CLASSES:
|
|
354
369
|
raise ValueError(f'Unknown scheme {scheme} in {url}')
|
|
@@ -390,7 +405,8 @@ class FileCache:
|
|
|
390
405
|
|
|
391
406
|
key = (scheme, remote, anonymous)
|
|
392
407
|
if key not in _SOURCE_CACHE:
|
|
393
|
-
_SOURCE_CACHE[key] = _SCHEME_CLASSES[scheme](scheme, remote,
|
|
408
|
+
_SOURCE_CACHE[key] = _SCHEME_CLASSES[scheme](scheme, remote,
|
|
409
|
+
anonymous=anonymous)
|
|
394
410
|
|
|
395
411
|
source = _SOURCE_CACHE[key]
|
|
396
412
|
|
|
@@ -1178,7 +1194,7 @@ class FileCache:
|
|
|
1178
1194
|
source_dict: dict[str, list[tuple[int, FileCacheSource, str, Path]]] = {}
|
|
1179
1195
|
|
|
1180
1196
|
# First find all the files that are either local or that we have already cached.
|
|
1181
|
-
# For other files, create a list of just the files we need to
|
|
1197
|
+
# For other files, create a list of just the files we need to upload and
|
|
1182
1198
|
# organize them by source; we use the source prefix to distinguish among them.
|
|
1183
1199
|
self._log_debug('Performing multi-file upload of:')
|
|
1184
1200
|
for idx, (source, sub_path, local_path) in enumerate(zip(sources,
|
|
@@ -1248,7 +1264,7 @@ class FileCache:
|
|
|
1248
1264
|
anonymous: Optional[bool] = None,
|
|
1249
1265
|
lock_timeout: Optional[int] = None,
|
|
1250
1266
|
url_to_path: Optional[UrlToPathFuncOrSeqType] = None,
|
|
1251
|
-
**kwargs: Any) ->
|
|
1267
|
+
**kwargs: Any) -> Iterator[IO[Any]]:
|
|
1252
1268
|
"""Retrieve+open or open+upload a file as a context manager.
|
|
1253
1269
|
|
|
1254
1270
|
If `mode` is a read mode (like ``'r'`` or ``'rb'``) then the file will be first
|
|
@@ -1259,6 +1275,7 @@ class FileCache:
|
|
|
1259
1275
|
Parameters:
|
|
1260
1276
|
url: The filename to open.
|
|
1261
1277
|
mode: The mode string as you would specify to Python's `open()` function.
|
|
1278
|
+
**args: Any additional arguments are passed to the Python ``open()`` function.
|
|
1262
1279
|
anonymous: If True, access cloud resources without specifying credentials. If
|
|
1263
1280
|
False, credentials must be initialized in the program's environment. If
|
|
1264
1281
|
None, use the default setting for this :class:`FileCache` instance.
|
|
@@ -1292,6 +1309,8 @@ class FileCache:
|
|
|
1292
1309
|
If this parameter is specified, it replaces the default translators for
|
|
1293
1310
|
this :class:`FileCache` instance. If this parameter is omitted, the
|
|
1294
1311
|
default translators are used.
|
|
1312
|
+
**kwargs: Any additional arguments are passed to the Python ``open()``
|
|
1313
|
+
function.
|
|
1295
1314
|
|
|
1296
1315
|
Returns:
|
|
1297
1316
|
The same object as would be returned by the normal `open()` function.
|
|
@@ -1310,6 +1329,243 @@ class FileCache:
|
|
|
1310
1329
|
yield fp
|
|
1311
1330
|
self.upload(url, anonymous=anonymous, url_to_path=url_to_path)
|
|
1312
1331
|
|
|
1332
|
+
def iterdir(self,
|
|
1333
|
+
url: str | Path,
|
|
1334
|
+
*,
|
|
1335
|
+
anonymous: Optional[bool] = None,
|
|
1336
|
+
) -> Iterator[str]:
|
|
1337
|
+
"""Enumerate the files and sub-directories in a directory.
|
|
1338
|
+
|
|
1339
|
+
This function always accesses a remote location (ignoring the local cache),
|
|
1340
|
+
if appropriate, because there is no way to know if the local cache contains
|
|
1341
|
+
all of the files and sub-directories present in the remote.
|
|
1342
|
+
|
|
1343
|
+
Parameters:
|
|
1344
|
+
url: The URL of the directory, including any source prefix.
|
|
1345
|
+
anonymous: If specified, override the default setting for anonymous access.
|
|
1346
|
+
If True, access cloud resources without specifying credentials. If False,
|
|
1347
|
+
credentials must be initialized in the program's environment.
|
|
1348
|
+
|
|
1349
|
+
Yields:
|
|
1350
|
+
All files and sub-directories in the directory given by the url, in no
|
|
1351
|
+
particular order. Special directories ``.`` and ``..`` are ignored.
|
|
1352
|
+
"""
|
|
1353
|
+
|
|
1354
|
+
self._log_debug(f'Iterating directory contents: {url}')
|
|
1355
|
+
|
|
1356
|
+
source, sub_path, _ = self._get_source_and_paths(url, anonymous, None)
|
|
1357
|
+
|
|
1358
|
+
for obj_name, _ in source.iterdir_type(sub_path):
|
|
1359
|
+
yield obj_name
|
|
1360
|
+
|
|
1361
|
+
def iterdir_type(self,
|
|
1362
|
+
url: str | Path,
|
|
1363
|
+
*,
|
|
1364
|
+
anonymous: Optional[bool] = None,
|
|
1365
|
+
) -> Iterator[tuple[str, bool]]:
|
|
1366
|
+
"""Enumerate the files and sub-dirs in a directory indicating which is a dir.
|
|
1367
|
+
|
|
1368
|
+
This function always accesses a remote location (ignoring the local cache),
|
|
1369
|
+
if appropriate, because there is no way to know if the local cache contains
|
|
1370
|
+
all of the files and sub-directories present in the remote.
|
|
1371
|
+
|
|
1372
|
+
Parameters:
|
|
1373
|
+
url: The URL of the directory, including any source prefix.
|
|
1374
|
+
anonymous: If specified, override the default setting for anonymous access.
|
|
1375
|
+
If True, access cloud resources without specifying credentials. If False,
|
|
1376
|
+
credentials must be initialized in the program's environment.
|
|
1377
|
+
|
|
1378
|
+
Yields:
|
|
1379
|
+
All files and sub-directories in the directory given by the url, in no
|
|
1380
|
+
particular order. Special directories ``.`` and ``..`` are ignored. The bool
|
|
1381
|
+
is True if the returned name is a directory, False if it is a file.
|
|
1382
|
+
"""
|
|
1383
|
+
|
|
1384
|
+
self._log_debug(f'Iterating directory contents: {url}')
|
|
1385
|
+
|
|
1386
|
+
source, sub_path, _ = self._get_source_and_paths(url, anonymous, None)
|
|
1387
|
+
|
|
1388
|
+
for obj_name, is_dir in source.iterdir_type(sub_path):
|
|
1389
|
+
yield obj_name, is_dir
|
|
1390
|
+
|
|
1391
|
+
def unlink(self,
|
|
1392
|
+
url: StrOrPathOrSeqType,
|
|
1393
|
+
*,
|
|
1394
|
+
missing_ok: bool = False,
|
|
1395
|
+
anonymous: Optional[bool] = None,
|
|
1396
|
+
nthreads: Optional[int] = None,
|
|
1397
|
+
exception_on_fail: bool = True,
|
|
1398
|
+
url_to_path: Optional[UrlToPathFuncOrSeqType] = None
|
|
1399
|
+
) -> str | Exception | list[str | Exception]:
|
|
1400
|
+
"""Remove a file, including any locally cached copy.
|
|
1401
|
+
|
|
1402
|
+
Parameters:
|
|
1403
|
+
url: The URL of the file, including any source prefix. If `url` is a list or
|
|
1404
|
+
tuple, all URLs are unlinked.
|
|
1405
|
+
missing_ok: True if it is OK to unlink a file that doesn't exist; False to
|
|
1406
|
+
raise a FileNotFoundError in this case.
|
|
1407
|
+
anonymous: If specified, override the default setting for anonymous access.
|
|
1408
|
+
If True, access cloud resources without specifying credentials. If False,
|
|
1409
|
+
credentials must be initialized in the program's environment.
|
|
1410
|
+
nthreads: The maximum number of threads to use when doing multiple-file
|
|
1411
|
+
retrieval or upload. If None, use the default value for this
|
|
1412
|
+
:class:`FileCache` instance.
|
|
1413
|
+
exception_on_fail: If True, if any file does not exist or upload fails an
|
|
1414
|
+
exception is raised. If False, the function returns normally and any
|
|
1415
|
+
failed upload is marked with the Exception that caused the failure in
|
|
1416
|
+
place of the returned path.
|
|
1417
|
+
url_to_path: The function (or list of functions) that is used to translate
|
|
1418
|
+
URLs into local paths. By default, :class:`FileCache` uses a directory
|
|
1419
|
+
hierarchy consisting of ``<cache_dir>/<cache_name>/<source>/<path>``,
|
|
1420
|
+
where ``source`` is the URL prefix converted to a filesystem-friendly
|
|
1421
|
+
format (e.g. ``gs://bucket`` is converted to ``gs_bucket``). A
|
|
1422
|
+
user-specified translator function takes five arguments::
|
|
1423
|
+
|
|
1424
|
+
func(scheme: str, remote: str, path: str, cache_dir: Path,
|
|
1425
|
+
cache_subdir: str) -> str | Path
|
|
1426
|
+
|
|
1427
|
+
where `scheme` is the URL scheme (like ``"gs"`` or ``"file"``), `remote`
|
|
1428
|
+
is the name of the bucket or webserver or the empty string for a local
|
|
1429
|
+
file, `path` is the rest of the URL, `cache_dir` is the top-level
|
|
1430
|
+
directory of the cache (``<cache_dir>/<cache_name>``), and `cache_subdir`
|
|
1431
|
+
is the subdirectory specific to this scheme and remote. If the translator
|
|
1432
|
+
wants to override the default translation, it can return a Path.
|
|
1433
|
+
Otherwise, it returns None. If the returned Path is relative, if will be
|
|
1434
|
+
appended to `cache_dir`; if it is absolute, it will be used directly (be
|
|
1435
|
+
very careful with this, as it has the ability to access files outside of
|
|
1436
|
+
the cache directory). If more than one translator is specified, they are
|
|
1437
|
+
called in order until one returns a Path, or it falls through to the
|
|
1438
|
+
default.
|
|
1439
|
+
|
|
1440
|
+
If this parameter is specified, it replaces the default translators for
|
|
1441
|
+
this :class:`FileCache` instance. If this parameter is omitted, the
|
|
1442
|
+
default translators are used.
|
|
1443
|
+
|
|
1444
|
+
Returns:
|
|
1445
|
+
The Path of the filename in the cache directory (or the original absolute path
|
|
1446
|
+
if local). If `url` was a list or tuple of paths, then instead return a list
|
|
1447
|
+
of Paths of the filenames in the temporary directory (or the original full
|
|
1448
|
+
path if local). If `exception_on_fail` is False, any Path may be an Exception
|
|
1449
|
+
if that file does not exist and missing_ok is True.
|
|
1450
|
+
|
|
1451
|
+
Notes:
|
|
1452
|
+
If a URL points to a remote location, the locally cached version (if any) is
|
|
1453
|
+
only removed if the unlink of the remote location succeeded.
|
|
1454
|
+
|
|
1455
|
+
Raises:
|
|
1456
|
+
FileNotFoundError: If a file to unlink does not exist or the unlink failed,
|
|
1457
|
+
and exception_on_fail is True.
|
|
1458
|
+
"""
|
|
1459
|
+
|
|
1460
|
+
nthreads = self._validate_nthreads(nthreads)
|
|
1461
|
+
|
|
1462
|
+
if isinstance(url, (list, tuple)):
|
|
1463
|
+
sources = []
|
|
1464
|
+
sub_paths = []
|
|
1465
|
+
local_paths = []
|
|
1466
|
+
url2 = [str(x) for x in url]
|
|
1467
|
+
for one_url in url2:
|
|
1468
|
+
source, sub_path, local_path = self._get_source_and_paths(one_url,
|
|
1469
|
+
anonymous,
|
|
1470
|
+
url_to_path)
|
|
1471
|
+
sources.append(source)
|
|
1472
|
+
sub_paths.append(sub_path)
|
|
1473
|
+
local_paths.append(local_path)
|
|
1474
|
+
return self._unlink_multi(url2, sources, sub_paths, local_paths,
|
|
1475
|
+
missing_ok, nthreads, exception_on_fail)
|
|
1476
|
+
|
|
1477
|
+
url3 = str(url)
|
|
1478
|
+
source, sub_path, local_path = self._get_source_and_paths(url3, anonymous,
|
|
1479
|
+
url_to_path)
|
|
1480
|
+
|
|
1481
|
+
self._log_debug(f'Unlinking {url3}')
|
|
1482
|
+
try:
|
|
1483
|
+
source.unlink(sub_path, missing_ok=missing_ok)
|
|
1484
|
+
local_path.unlink(missing_ok=True) # Don't care if it's cached or not
|
|
1485
|
+
except Exception as e:
|
|
1486
|
+
if exception_on_fail:
|
|
1487
|
+
raise e
|
|
1488
|
+
else:
|
|
1489
|
+
return e
|
|
1490
|
+
|
|
1491
|
+
return sub_path
|
|
1492
|
+
|
|
1493
|
+
def _unlink_multi(self,
|
|
1494
|
+
urls: list[str],
|
|
1495
|
+
sources: list[FileCacheSource],
|
|
1496
|
+
sub_paths: list[str],
|
|
1497
|
+
local_paths: list[Path],
|
|
1498
|
+
missing_ok: bool,
|
|
1499
|
+
nthreads: int,
|
|
1500
|
+
exception_on_fail: bool) -> list[str | Exception]:
|
|
1501
|
+
"""Unlink multiple files."""
|
|
1502
|
+
|
|
1503
|
+
func_ret: list[str | BaseException | None] = [None] * len(sources)
|
|
1504
|
+
|
|
1505
|
+
files_not_exist = []
|
|
1506
|
+
|
|
1507
|
+
source_dict: dict[str, list[tuple[int, FileCacheSource, str, Path]]] = {}
|
|
1508
|
+
|
|
1509
|
+
# First find all the files that are either local or that we have already cached.
|
|
1510
|
+
# For other files, create a list of just the files we need to retrieve and
|
|
1511
|
+
# organize them by source; we use the source prefix to distinguish among them.
|
|
1512
|
+
self._log_debug('Performing multi-file unlink of:')
|
|
1513
|
+
for idx, (url, source, sub_path, local_path) in enumerate(zip(urls, sources,
|
|
1514
|
+
sub_paths,
|
|
1515
|
+
local_paths)):
|
|
1516
|
+
pfx = source._src_prefix_
|
|
1517
|
+
if source.primary_scheme() == 'file':
|
|
1518
|
+
try:
|
|
1519
|
+
source.unlink(sub_path, missing_ok=missing_ok)
|
|
1520
|
+
func_ret[idx] = str(url)
|
|
1521
|
+
self._log_debug(f' Local file {pfx}{sub_path}')
|
|
1522
|
+
except FileNotFoundError as e:
|
|
1523
|
+
self._log_debug(f' LOCAL FILE DOES NOT EXIST {pfx}{sub_path}')
|
|
1524
|
+
files_not_exist.append(sub_path)
|
|
1525
|
+
func_ret[idx] = e
|
|
1526
|
+
continue
|
|
1527
|
+
assert '://' in pfx
|
|
1528
|
+
if pfx not in source_dict:
|
|
1529
|
+
source_dict[pfx] = []
|
|
1530
|
+
source_dict[pfx].append((idx, source, sub_path, local_path))
|
|
1531
|
+
|
|
1532
|
+
# Now go through the sources, package up the paths to unlink, and unlink
|
|
1533
|
+
# them all at once
|
|
1534
|
+
for source_pfx in source_dict:
|
|
1535
|
+
source = source_dict[source_pfx][0][1] # All the same
|
|
1536
|
+
source_idxes, _, source_sub_paths, source_local_paths = list(
|
|
1537
|
+
zip(*source_dict[source_pfx]))
|
|
1538
|
+
self._log_debug(
|
|
1539
|
+
f'Performing multi-file unlink for prefix {source_pfx}:')
|
|
1540
|
+
for sub_path in source_sub_paths:
|
|
1541
|
+
self._log_debug(f' {sub_path}')
|
|
1542
|
+
rets = source.unlink_multi(source_sub_paths, missing_ok=missing_ok,
|
|
1543
|
+
nthreads=nthreads)
|
|
1544
|
+
assert len(source_idxes) == len(rets)
|
|
1545
|
+
for ret2, local_path, url in zip(rets, source_local_paths, urls):
|
|
1546
|
+
if isinstance(ret2, Exception):
|
|
1547
|
+
self._log_debug(f' Unlink failed: {url} {ret2}')
|
|
1548
|
+
files_not_exist.append(str(url))
|
|
1549
|
+
else:
|
|
1550
|
+
local_path.unlink(missing_ok=True) # Remove from cache
|
|
1551
|
+
|
|
1552
|
+
for source_ret, source_idx in zip(rets, source_idxes):
|
|
1553
|
+
if isinstance(source_ret, Exception):
|
|
1554
|
+
func_ret[source_idx] = source_ret
|
|
1555
|
+
else:
|
|
1556
|
+
# We want to return the entire URL, not just the sub_path,
|
|
1557
|
+
# so we have to add the prefix back on
|
|
1558
|
+
func_ret[source_idx] = str(urls[source_idx])
|
|
1559
|
+
|
|
1560
|
+
if exception_on_fail and not missing_ok:
|
|
1561
|
+
exc_str = ''
|
|
1562
|
+
if files_not_exist:
|
|
1563
|
+
exc_str += f"File(s) do not exist: {', '.join(files_not_exist)}"
|
|
1564
|
+
if exc_str:
|
|
1565
|
+
raise FileNotFoundError(exc_str)
|
|
1566
|
+
|
|
1567
|
+
return cast(list[Union[str, Exception]], func_ret)
|
|
1568
|
+
|
|
1313
1569
|
def new_path(self,
|
|
1314
1570
|
path: str | Path | FCPath,
|
|
1315
1571
|
*,
|