rms-filecache 2.1.0__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {rms_filecache-2.1.0/rms_filecache.egg-info → rms_filecache-2.2.0}/PKG-INFO +1 -1
  2. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/docs/module.rst +3 -3
  3. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/filecache/_version.py +2 -2
  4. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/filecache/file_cache.py +263 -7
  5. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/filecache/file_cache_path.py +963 -198
  6. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/filecache/file_cache_source.py +329 -11
  7. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/mypy.ini +1 -0
  8. {rms_filecache-2.1.0 → rms_filecache-2.2.0/rms_filecache.egg-info}/PKG-INFO +1 -1
  9. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/rms_filecache.egg-info/SOURCES.txt +0 -1
  10. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/s3_bucket_policy_write.txt +8 -3
  11. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/test_file_cache.py +487 -7
  12. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/test_file_cache_path.py +356 -60
  13. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/test_file_cache_source.py +8 -0
  14. rms_filecache-2.1.0/filecache/my_glob.py +0 -639
  15. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/.coveragerc +0 -0
  16. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/.flake8 +0 -0
  17. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/.github/pull_request_template.md +0 -0
  18. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/.github/workflows/publish_to_pypi.yml +0 -0
  19. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/.github/workflows/publish_to_test_pypi.yml +0 -0
  20. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/.github/workflows/run-tests.yml +0 -0
  21. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/.gitignore +0 -0
  22. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/.readthedocs.yaml +0 -0
  23. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/LICENSE +0 -0
  24. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/README.md +0 -0
  25. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/docs/Makefile +0 -0
  26. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/docs/conf.py +0 -0
  27. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/docs/index.rst +0 -0
  28. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/docs/make.bat +0 -0
  29. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/filecache/__init__.py +0 -0
  30. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/filecache/file_cache_types.py +0 -0
  31. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/filecache/py.typed +0 -0
  32. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/pyproject.toml +0 -0
  33. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/requirements.txt +0 -0
  34. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
  35. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/rms_filecache.egg-info/requires.txt +0 -0
  36. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/rms_filecache.egg-info/top_level.txt +0 -0
  37. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/setup.cfg +0 -0
  38. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/test_files/expected/lorem1.txt +0 -0
  39. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/test_files/expected/subdir1/lorem1.txt +0 -0
  40. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
  41. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
  42. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/test_programs/locking.py +0 -0
  43. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/test_programs/multi_download.py +0 -0
  44. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/__init__.py +0 -0
  45. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/gs_lifecycle_config.json +0 -0
  46. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/gs_lifecycle_config_info.txt +0 -0
  47. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/s3_bucket_policy_read.txt +0 -0
  48. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/s3_lifecycle_policy.txt +0 -0
  49. {rms_filecache-2.1.0 → rms_filecache-2.2.0}/tests/test_url_to_path.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: rms-filecache
3
- Version: 2.1.0
3
+ Version: 2.2.0
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -7,7 +7,7 @@
7
7
  :undoc-members:
8
8
  :special-members:
9
9
  :show-inheritance:
10
- :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
10
+ :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__, __firstlineno__, __static_attributes__, __abstractmethods__
11
11
 
12
12
  .. automodule:: filecache.file_cache_path
13
13
  :member-order: bysource
@@ -15,7 +15,7 @@
15
15
  :undoc-members:
16
16
  :special-members:
17
17
  :show-inheritance:
18
- :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
18
+ :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__, __firstlineno__, __static_attributes__, __abstractmethods__
19
19
 
20
20
  .. automodule:: filecache.file_cache_source
21
21
  :member-order: bysource
@@ -23,4 +23,4 @@
23
23
  :undoc-members:
24
24
  :special-members:
25
25
  :show-inheritance:
26
- :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
26
+ :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__, __firstlineno__, __static_attributes__, __abstractmethods__
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '2.1.0'
16
- __version_tuple__ = version_tuple = (2, 1, 0)
15
+ __version__ = version = '2.2.0'
16
+ __version_tuple__ = version_tuple = (2, 2, 0)
@@ -17,8 +17,8 @@ import time
17
17
  from types import TracebackType
18
18
  from typing import (cast,
19
19
  Any,
20
- Generator,
21
20
  IO,
21
+ Iterator,
22
22
  Literal,
23
23
  Optional,
24
24
  Type,
@@ -50,6 +50,7 @@ _SCHEME_CLASSES: dict[str, type[FileCacheSource]] = {}
50
50
 
51
51
  def register_filecachesource(cls: type[FileCacheSource]) -> None:
52
52
  """Register one or more URL FileCacheSource subclasses as URL schemes."""
53
+
53
54
  for s in cls.schemes():
54
55
  _SCHEME_CLASSES[s] = cls
55
56
 
@@ -66,12 +67,14 @@ _GLOBAL_LOGGER: Logger | None = None
66
67
 
67
68
  def set_global_logger(logger: Logger | None) -> None:
68
69
  """Set the global logger for all FileCache instances that don't specify one."""
70
+
69
71
  global _GLOBAL_LOGGER
70
72
  _GLOBAL_LOGGER = logger
71
73
 
72
74
 
73
75
  def set_easy_logger() -> None:
74
76
  """Set a default logger that outputs all messages to stdout."""
77
+
75
78
  easy_logger = logging.getLogger(__name__)
76
79
  easy_logger.setLevel(logging.DEBUG)
77
80
 
@@ -89,6 +92,7 @@ def set_easy_logger() -> None:
89
92
 
90
93
  def get_global_logger() -> Logger | None:
91
94
  """Return the current global logger."""
95
+
92
96
  return _GLOBAL_LOGGER
93
97
 
94
98
 
@@ -264,41 +268,49 @@ class FileCache:
264
268
  @property
265
269
  def cache_dir(self) -> Path:
266
270
  """The top-level directory of the cache as a Path object."""
271
+
267
272
  return self._cache_dir
268
273
 
269
274
  @property
270
275
  def download_counter(self) -> int:
271
276
  """The number of actual file downloads that have taken place."""
277
+
272
278
  return self._download_counter
273
279
 
274
280
  @property
275
281
  def upload_counter(self) -> int:
276
282
  """The number of actual file uploads that have taken place."""
283
+
277
284
  return self._upload_counter
278
285
 
279
286
  @property
280
287
  def anonymous(self) -> bool:
281
288
  """The default bool indicating whether to make all cloud accesses anonymous."""
289
+
282
290
  return self._anonymous
283
291
 
284
292
  @property
285
293
  def lock_timeout(self) -> int:
286
294
  """The default timeout in seconds while waiting for a file lock."""
295
+
287
296
  return self._lock_timeout
288
297
 
289
298
  @property
290
299
  def nthreads(self) -> int:
291
300
  """The default number of threads to use for multiple-file operations."""
301
+
292
302
  return self._nthreads
293
303
 
294
304
  @property
295
305
  def delete_on_exit(self) -> bool:
296
306
  """A bool indicating whether this FileCache will be deleted on exit."""
307
+
297
308
  return self._delete_on_exit
298
309
 
299
310
  @property
300
311
  def is_mp_safe(self) -> bool:
301
312
  """A bool indicating whether this FileCache is multi-processor safe."""
313
+
302
314
  return self._is_mp_safe
303
315
 
304
316
  def _log_debug(self, msg: str) -> None:
@@ -333,8 +345,10 @@ class FileCache:
333
345
  elif len(parts) == 2:
334
346
  slash_split = parts[1].split('/', maxsplit=1)
335
347
  if len(slash_split) != 2:
336
- raise ValueError(f'No path after remote: {url}')
337
- remote, sub_path = slash_split
348
+ remote = parts[1]
349
+ sub_path = ''
350
+ else:
351
+ remote, sub_path = slash_split
338
352
  scheme = parts[0].lower()
339
353
  if scheme == 'file':
340
354
  # All file accesses are absolute
@@ -348,7 +362,8 @@ class FileCache:
348
362
  else:
349
363
  # file:///dir/file
350
364
  # We have to add the / back on the beginning
351
- sub_path = f'/{sub_path}'
365
+ if sub_path:
366
+ sub_path = f'/{sub_path}'
352
367
  sub_path = str(Path(sub_path))
353
368
  if scheme not in _SCHEME_CLASSES:
354
369
  raise ValueError(f'Unknown scheme {scheme} in {url}')
@@ -390,7 +405,8 @@ class FileCache:
390
405
 
391
406
  key = (scheme, remote, anonymous)
392
407
  if key not in _SOURCE_CACHE:
393
- _SOURCE_CACHE[key] = _SCHEME_CLASSES[scheme](scheme, remote, anonymous)
408
+ _SOURCE_CACHE[key] = _SCHEME_CLASSES[scheme](scheme, remote,
409
+ anonymous=anonymous)
394
410
 
395
411
  source = _SOURCE_CACHE[key]
396
412
 
@@ -1178,7 +1194,7 @@ class FileCache:
1178
1194
  source_dict: dict[str, list[tuple[int, FileCacheSource, str, Path]]] = {}
1179
1195
 
1180
1196
  # First find all the files that are either local or that we have already cached.
1181
- # For other files, create a list of just the files we need to retrieve and
1197
+ # For other files, create a list of just the files we need to upload and
1182
1198
  # organize them by source; we use the source prefix to distinguish among them.
1183
1199
  self._log_debug('Performing multi-file upload of:')
1184
1200
  for idx, (source, sub_path, local_path) in enumerate(zip(sources,
@@ -1248,7 +1264,7 @@ class FileCache:
1248
1264
  anonymous: Optional[bool] = None,
1249
1265
  lock_timeout: Optional[int] = None,
1250
1266
  url_to_path: Optional[UrlToPathFuncOrSeqType] = None,
1251
- **kwargs: Any) -> Generator[IO[Any]]:
1267
+ **kwargs: Any) -> Iterator[IO[Any]]:
1252
1268
  """Retrieve+open or open+upload a file as a context manager.
1253
1269
 
1254
1270
  If `mode` is a read mode (like ``'r'`` or ``'rb'``) then the file will be first
@@ -1259,6 +1275,7 @@ class FileCache:
1259
1275
  Parameters:
1260
1276
  url: The filename to open.
1261
1277
  mode: The mode string as you would specify to Python's `open()` function.
1278
+ **args: Any additional arguments are passed to the Python ``open()`` function.
1262
1279
  anonymous: If True, access cloud resources without specifying credentials. If
1263
1280
  False, credentials must be initialized in the program's environment. If
1264
1281
  None, use the default setting for this :class:`FileCache` instance.
@@ -1292,6 +1309,8 @@ class FileCache:
1292
1309
  If this parameter is specified, it replaces the default translators for
1293
1310
  this :class:`FileCache` instance. If this parameter is omitted, the
1294
1311
  default translators are used.
1312
+ **kwargs: Any additional arguments are passed to the Python ``open()``
1313
+ function.
1295
1314
 
1296
1315
  Returns:
1297
1316
  The same object as would be returned by the normal `open()` function.
@@ -1310,6 +1329,243 @@ class FileCache:
1310
1329
  yield fp
1311
1330
  self.upload(url, anonymous=anonymous, url_to_path=url_to_path)
1312
1331
 
1332
+ def iterdir(self,
1333
+ url: str | Path,
1334
+ *,
1335
+ anonymous: Optional[bool] = None,
1336
+ ) -> Iterator[str]:
1337
+ """Enumerate the files and sub-directories in a directory.
1338
+
1339
+ This function always accesses a remote location (ignoring the local cache),
1340
+ if appropriate, because there is no way to know if the local cache contains
1341
+ all of the files and sub-directories present in the remote.
1342
+
1343
+ Parameters:
1344
+ url: The URL of the directory, including any source prefix.
1345
+ anonymous: If specified, override the default setting for anonymous access.
1346
+ If True, access cloud resources without specifying credentials. If False,
1347
+ credentials must be initialized in the program's environment.
1348
+
1349
+ Yields:
1350
+ All files and sub-directories in the directory given by the url, in no
1351
+ particular order. Special directories ``.`` and ``..`` are ignored.
1352
+ """
1353
+
1354
+ self._log_debug(f'Iterating directory contents: {url}')
1355
+
1356
+ source, sub_path, _ = self._get_source_and_paths(url, anonymous, None)
1357
+
1358
+ for obj_name, _ in source.iterdir_type(sub_path):
1359
+ yield obj_name
1360
+
1361
+ def iterdir_type(self,
1362
+ url: str | Path,
1363
+ *,
1364
+ anonymous: Optional[bool] = None,
1365
+ ) -> Iterator[tuple[str, bool]]:
1366
+ """Enumerate the files and sub-dirs in a directory indicating which is a dir.
1367
+
1368
+ This function always accesses a remote location (ignoring the local cache),
1369
+ if appropriate, because there is no way to know if the local cache contains
1370
+ all of the files and sub-directories present in the remote.
1371
+
1372
+ Parameters:
1373
+ url: The URL of the directory, including any source prefix.
1374
+ anonymous: If specified, override the default setting for anonymous access.
1375
+ If True, access cloud resources without specifying credentials. If False,
1376
+ credentials must be initialized in the program's environment.
1377
+
1378
+ Yields:
1379
+ All files and sub-directories in the directory given by the url, in no
1380
+ particular order. Special directories ``.`` and ``..`` are ignored. The bool
1381
+ is True if the returned name is a directory, False if it is a file.
1382
+ """
1383
+
1384
+ self._log_debug(f'Iterating directory contents: {url}')
1385
+
1386
+ source, sub_path, _ = self._get_source_and_paths(url, anonymous, None)
1387
+
1388
+ for obj_name, is_dir in source.iterdir_type(sub_path):
1389
+ yield obj_name, is_dir
1390
+
1391
+ def unlink(self,
1392
+ url: StrOrPathOrSeqType,
1393
+ *,
1394
+ missing_ok: bool = False,
1395
+ anonymous: Optional[bool] = None,
1396
+ nthreads: Optional[int] = None,
1397
+ exception_on_fail: bool = True,
1398
+ url_to_path: Optional[UrlToPathFuncOrSeqType] = None
1399
+ ) -> str | Exception | list[str | Exception]:
1400
+ """Remove a file, including any locally cached copy.
1401
+
1402
+ Parameters:
1403
+ url: The URL of the file, including any source prefix. If `url` is a list or
1404
+ tuple, all URLs are unlinked.
1405
+ missing_ok: True if it is OK to unlink a file that doesn't exist; False to
1406
+ raise a FileNotFoundError in this case.
1407
+ anonymous: If specified, override the default setting for anonymous access.
1408
+ If True, access cloud resources without specifying credentials. If False,
1409
+ credentials must be initialized in the program's environment.
1410
+ nthreads: The maximum number of threads to use when doing multiple-file
1411
+ retrieval or upload. If None, use the default value for this
1412
+ :class:`FileCache` instance.
1413
+ exception_on_fail: If True, if any file does not exist or upload fails an
1414
+ exception is raised. If False, the function returns normally and any
1415
+ failed upload is marked with the Exception that caused the failure in
1416
+ place of the returned path.
1417
+ url_to_path: The function (or list of functions) that is used to translate
1418
+ URLs into local paths. By default, :class:`FileCache` uses a directory
1419
+ hierarchy consisting of ``<cache_dir>/<cache_name>/<source>/<path>``,
1420
+ where ``source`` is the URL prefix converted to a filesystem-friendly
1421
+ format (e.g. ``gs://bucket`` is converted to ``gs_bucket``). A
1422
+ user-specified translator function takes five arguments::
1423
+
1424
+ func(scheme: str, remote: str, path: str, cache_dir: Path,
1425
+ cache_subdir: str) -> str | Path
1426
+
1427
+ where `scheme` is the URL scheme (like ``"gs"`` or ``"file"``), `remote`
1428
+ is the name of the bucket or webserver or the empty string for a local
1429
+ file, `path` is the rest of the URL, `cache_dir` is the top-level
1430
+ directory of the cache (``<cache_dir>/<cache_name>``), and `cache_subdir`
1431
+ is the subdirectory specific to this scheme and remote. If the translator
1432
+ wants to override the default translation, it can return a Path.
1433
+ Otherwise, it returns None. If the returned Path is relative, if will be
1434
+ appended to `cache_dir`; if it is absolute, it will be used directly (be
1435
+ very careful with this, as it has the ability to access files outside of
1436
+ the cache directory). If more than one translator is specified, they are
1437
+ called in order until one returns a Path, or it falls through to the
1438
+ default.
1439
+
1440
+ If this parameter is specified, it replaces the default translators for
1441
+ this :class:`FileCache` instance. If this parameter is omitted, the
1442
+ default translators are used.
1443
+
1444
+ Returns:
1445
+ The Path of the filename in the cache directory (or the original absolute path
1446
+ if local). If `url` was a list or tuple of paths, then instead return a list
1447
+ of Paths of the filenames in the temporary directory (or the original full
1448
+ path if local). If `exception_on_fail` is False, any Path may be an Exception
1449
+ if that file does not exist and missing_ok is True.
1450
+
1451
+ Notes:
1452
+ If a URL points to a remote location, the locally cached version (if any) is
1453
+ only removed if the unlink of the remote location succeeded.
1454
+
1455
+ Raises:
1456
+ FileNotFoundError: If a file to unlink does not exist or the unlink failed,
1457
+ and exception_on_fail is True.
1458
+ """
1459
+
1460
+ nthreads = self._validate_nthreads(nthreads)
1461
+
1462
+ if isinstance(url, (list, tuple)):
1463
+ sources = []
1464
+ sub_paths = []
1465
+ local_paths = []
1466
+ url2 = [str(x) for x in url]
1467
+ for one_url in url2:
1468
+ source, sub_path, local_path = self._get_source_and_paths(one_url,
1469
+ anonymous,
1470
+ url_to_path)
1471
+ sources.append(source)
1472
+ sub_paths.append(sub_path)
1473
+ local_paths.append(local_path)
1474
+ return self._unlink_multi(url2, sources, sub_paths, local_paths,
1475
+ missing_ok, nthreads, exception_on_fail)
1476
+
1477
+ url3 = str(url)
1478
+ source, sub_path, local_path = self._get_source_and_paths(url3, anonymous,
1479
+ url_to_path)
1480
+
1481
+ self._log_debug(f'Unlinking {url3}')
1482
+ try:
1483
+ source.unlink(sub_path, missing_ok=missing_ok)
1484
+ local_path.unlink(missing_ok=True) # Don't care if it's cached or not
1485
+ except Exception as e:
1486
+ if exception_on_fail:
1487
+ raise e
1488
+ else:
1489
+ return e
1490
+
1491
+ return sub_path
1492
+
1493
+ def _unlink_multi(self,
1494
+ urls: list[str],
1495
+ sources: list[FileCacheSource],
1496
+ sub_paths: list[str],
1497
+ local_paths: list[Path],
1498
+ missing_ok: bool,
1499
+ nthreads: int,
1500
+ exception_on_fail: bool) -> list[str | Exception]:
1501
+ """Unlink multiple files."""
1502
+
1503
+ func_ret: list[str | BaseException | None] = [None] * len(sources)
1504
+
1505
+ files_not_exist = []
1506
+
1507
+ source_dict: dict[str, list[tuple[int, FileCacheSource, str, Path]]] = {}
1508
+
1509
+ # First find all the files that are either local or that we have already cached.
1510
+ # For other files, create a list of just the files we need to retrieve and
1511
+ # organize them by source; we use the source prefix to distinguish among them.
1512
+ self._log_debug('Performing multi-file unlink of:')
1513
+ for idx, (url, source, sub_path, local_path) in enumerate(zip(urls, sources,
1514
+ sub_paths,
1515
+ local_paths)):
1516
+ pfx = source._src_prefix_
1517
+ if source.primary_scheme() == 'file':
1518
+ try:
1519
+ source.unlink(sub_path, missing_ok=missing_ok)
1520
+ func_ret[idx] = str(url)
1521
+ self._log_debug(f' Local file {pfx}{sub_path}')
1522
+ except FileNotFoundError as e:
1523
+ self._log_debug(f' LOCAL FILE DOES NOT EXIST {pfx}{sub_path}')
1524
+ files_not_exist.append(sub_path)
1525
+ func_ret[idx] = e
1526
+ continue
1527
+ assert '://' in pfx
1528
+ if pfx not in source_dict:
1529
+ source_dict[pfx] = []
1530
+ source_dict[pfx].append((idx, source, sub_path, local_path))
1531
+
1532
+ # Now go through the sources, package up the paths to unlink, and unlink
1533
+ # them all at once
1534
+ for source_pfx in source_dict:
1535
+ source = source_dict[source_pfx][0][1] # All the same
1536
+ source_idxes, _, source_sub_paths, source_local_paths = list(
1537
+ zip(*source_dict[source_pfx]))
1538
+ self._log_debug(
1539
+ f'Performing multi-file unlink for prefix {source_pfx}:')
1540
+ for sub_path in source_sub_paths:
1541
+ self._log_debug(f' {sub_path}')
1542
+ rets = source.unlink_multi(source_sub_paths, missing_ok=missing_ok,
1543
+ nthreads=nthreads)
1544
+ assert len(source_idxes) == len(rets)
1545
+ for ret2, local_path, url in zip(rets, source_local_paths, urls):
1546
+ if isinstance(ret2, Exception):
1547
+ self._log_debug(f' Unlink failed: {url} {ret2}')
1548
+ files_not_exist.append(str(url))
1549
+ else:
1550
+ local_path.unlink(missing_ok=True) # Remove from cache
1551
+
1552
+ for source_ret, source_idx in zip(rets, source_idxes):
1553
+ if isinstance(source_ret, Exception):
1554
+ func_ret[source_idx] = source_ret
1555
+ else:
1556
+ # We want to return the entire URL, not just the sub_path,
1557
+ # so we have to add the prefix back on
1558
+ func_ret[source_idx] = str(urls[source_idx])
1559
+
1560
+ if exception_on_fail and not missing_ok:
1561
+ exc_str = ''
1562
+ if files_not_exist:
1563
+ exc_str += f"File(s) do not exist: {', '.join(files_not_exist)}"
1564
+ if exc_str:
1565
+ raise FileNotFoundError(exc_str)
1566
+
1567
+ return cast(list[Union[str, Exception]], func_ret)
1568
+
1313
1569
  def new_path(self,
1314
1570
  path: str | Path | FCPath,
1315
1571
  *,