rms-filecache 3.1.0__tar.gz → 3.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.github/workflows/publish_to_pypi.yml +2 -2
  2. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.github/workflows/publish_to_test_pypi.yml +2 -2
  3. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.github/workflows/run-tests.yml +7 -7
  4. {rms_filecache-3.1.0/rms_filecache.egg-info → rms_filecache-3.2.0}/PKG-INFO +4 -3
  5. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/__init__.py +0 -1
  6. rms_filecache-3.2.0/filecache/_version.py +24 -0
  7. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/file_cache.py +91 -15
  8. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/file_cache_path.py +14 -1
  9. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/file_cache_source.py +7 -0
  10. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/pyproject.toml +3 -2
  11. {rms_filecache-3.1.0 → rms_filecache-3.2.0/rms_filecache.egg-info}/PKG-INFO +4 -3
  12. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/rms_filecache.egg-info/SOURCES.txt +2 -0
  13. rms_filecache-3.2.0/rms_filecache.egg-info/scm_file_list.json +47 -0
  14. rms_filecache-3.2.0/rms_filecache.egg-info/scm_version.json +8 -0
  15. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/test_file_cache.py +211 -0
  16. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/test_file_cache_path.py +34 -0
  17. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/test_file_cache_source.py +29 -0
  18. rms_filecache-3.1.0/filecache/_version.py +0 -34
  19. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.coveragerc +0 -0
  20. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.flake8 +0 -0
  21. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.github/pull_request_template.md +0 -0
  22. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.gitignore +0 -0
  23. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.readthedocs.yaml +0 -0
  24. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/LICENSE +0 -0
  25. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/README.md +0 -0
  26. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/Makefile +0 -0
  27. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/conf.py +0 -0
  28. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/examples.rst +0 -0
  29. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/index.rst +0 -0
  30. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/introduction.rst +0 -0
  31. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/make.bat +0 -0
  32. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/module.rst +0 -0
  33. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/file_cache_types.py +0 -0
  34. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/py.typed +0 -0
  35. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/mypy.ini +0 -0
  36. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/requirements.txt +0 -0
  37. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
  38. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/rms_filecache.egg-info/requires.txt +0 -0
  39. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/rms_filecache.egg-info/top_level.txt +0 -0
  40. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/setup.cfg +0 -0
  41. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/lorem1.txt +0 -0
  42. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/subdir1/lorem1.txt +0 -0
  43. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
  44. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
  45. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_programs/locking.py +0 -0
  46. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_programs/multi_download.py +0 -0
  47. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/__init__.py +0 -0
  48. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/gs_lifecycle_config.json +0 -0
  49. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/gs_lifecycle_config_info.txt +0 -0
  50. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/s3_bucket_policy_read.txt +0 -0
  51. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/s3_bucket_policy_write.txt +0 -0
  52. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/s3_lifecycle_policy.txt +0 -0
  53. {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/test_url_mapping.py +0 -0
@@ -10,12 +10,12 @@ jobs:
10
10
  runs-on: ubuntu-latest
11
11
  steps:
12
12
  - name: Checkout
13
- uses: actions/checkout@v4
13
+ uses: actions/checkout@v7
14
14
  with:
15
15
  fetch-depth: 0
16
16
 
17
17
  - name: Set up Python 3.12
18
- uses: actions/setup-python@v5
18
+ uses: actions/setup-python@v7
19
19
  with:
20
20
  python-version: 3.12
21
21
 
@@ -9,12 +9,12 @@ jobs:
9
9
  runs-on: ubuntu-latest
10
10
  steps:
11
11
  - name: Checkout
12
- uses: actions/checkout@v4
12
+ uses: actions/checkout@v7
13
13
  with:
14
14
  fetch-depth: 0
15
15
 
16
16
  - name: Set up Python 3.12
17
- uses: actions/setup-python@v5
17
+ uses: actions/setup-python@v7
18
18
  with:
19
19
  python-version: 3.12
20
20
 
@@ -16,12 +16,12 @@ jobs:
16
16
  runs-on: ubuntu-latest
17
17
  steps:
18
18
  - name: Checkout
19
- uses: actions/checkout@v4
19
+ uses: actions/checkout@v7
20
20
  with:
21
21
  ref: ${{ github.event.pull_request.head.sha }}
22
22
 
23
23
  - name: Set up Python ${{ matrix.python-version }}
24
- uses: actions/setup-python@v5
24
+ uses: actions/setup-python@v7
25
25
  with:
26
26
  python-version: ${{ matrix.python-version }}
27
27
 
@@ -43,19 +43,19 @@ jobs:
43
43
  strategy:
44
44
  matrix:
45
45
  os: [ubuntu-latest, macos-latest, windows-latest]
46
- python-version: ['3.10', '3.11', '3.12', '3.13']
46
+ python-version: ['3.11', '3.12', '3.13', '3.14', '3.15']
47
47
  fail-fast: false
48
48
  permissions:
49
49
  contents: read
50
50
  id-token: write
51
51
  steps:
52
52
  - name: Checkout
53
- uses: actions/checkout@v4
53
+ uses: actions/checkout@v7
54
54
  with:
55
55
  ref: ${{ github.event.pull_request.head.sha }}
56
56
 
57
57
  - name: Set up Python ${{ matrix.python-version }}
58
- uses: actions/setup-python@v5
58
+ uses: actions/setup-python@v7
59
59
  with:
60
60
  python-version: ${{ matrix.python-version }}
61
61
 
@@ -66,7 +66,7 @@ jobs:
66
66
  # See: https://www.youtube.com/watch?v=ZgVhU5qvK1M
67
67
  # for a walkthrough on how to create the workload_identity_provider
68
68
  - name: Authenticate to Google Cloud
69
- uses: google-github-actions/auth@v2
69
+ uses: google-github-actions/auth@v3
70
70
  with:
71
71
  project_id: rms-node
72
72
  workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
@@ -80,7 +80,7 @@ jobs:
80
80
  coverage report -m
81
81
 
82
82
  - name: Upload coverage report to codecov
83
- uses: codecov/codecov-action@v5
83
+ uses: codecov/codecov-action@v7
84
84
  if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
85
85
  with:
86
86
  token: ${{ secrets.CODECOV_TOKEN }}
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rms-filecache
3
- Version: 3.1.0
3
+ Version: 3.2.0
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -15,14 +15,15 @@ Classifier: Natural Language :: English
15
15
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
16
  Classifier: Topic :: Utilities
17
17
  Classifier: License :: OSI Approved :: Apache Software License
18
- Classifier: Programming Language :: Python :: 3.10
19
18
  Classifier: Programming Language :: Python :: 3.11
20
19
  Classifier: Programming Language :: Python :: 3.12
21
20
  Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Programming Language :: Python :: 3.14
22
+ Classifier: Programming Language :: Python :: 3.15
22
23
  Classifier: Operating System :: MacOS :: MacOS X
23
24
  Classifier: Operating System :: POSIX :: Linux
24
25
  Classifier: Operating System :: Microsoft :: Windows
25
- Requires-Python: >=3.10
26
+ Requires-Python: >=3.11
26
27
  Description-Content-Type: text/markdown
27
28
  License-File: LICENSE
28
29
  Requires-Dist: boto3
@@ -174,7 +174,6 @@ from .file_cache_source import (FileCacheSource, # noqa: ignore E401
174
174
  FileCacheSourceGS,
175
175
  FileCacheSourceS3,
176
176
  FileCacheSourceFake)
177
-
178
177
  __all__ = ['get_global_logger',
179
178
  'register_filecachesource',
180
179
  'set_easy_logger',
@@ -0,0 +1,24 @@
1
+ # file generated by vcs-versioning
2
+ # don't change, don't track in version control
3
+ from __future__ import annotations
4
+
5
+ __all__ = [
6
+ "__version__",
7
+ "__version_tuple__",
8
+ "version",
9
+ "version_tuple",
10
+ "__commit_id__",
11
+ "commit_id",
12
+ ]
13
+
14
+ version: str
15
+ __version__: str
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
20
+
21
+ __version__ = version = '3.2.0'
22
+ __version_tuple__ = version_tuple = (3, 2, 0)
23
+
24
+ __commit_id__ = commit_id = 'g521391624'
@@ -264,6 +264,7 @@ class FileCache:
264
264
  raise TypeError(f'cache_name argument {cache_name} is of improper type')
265
265
 
266
266
  is_shared = (cache_name is not None)
267
+ self._cache_name = cache_name
267
268
 
268
269
  self._delete_on_exit = (delete_on_exit if delete_on_exit is not None
269
270
  else not is_shared)
@@ -418,6 +419,15 @@ class FileCache:
418
419
  return _GLOBAL_LOGGER
419
420
  return cast(Logger, self._logger)
420
421
 
422
+ def __repr__(self) -> str:
423
+ return (f'FileCache({self._cache_name!r}, '
424
+ f'anonymous={self._anonymous!r}, '
425
+ f'lock_timeout={self._lock_timeout!r}, '
426
+ f'nthreads={self._nthreads!r})')
427
+
428
+ def __str__(self) -> str:
429
+ return str(self._cache_dir)
430
+
421
431
  def _log_debug(self, msg: str) -> None:
422
432
  logger = self.logger
423
433
  if logger:
@@ -1600,7 +1610,7 @@ class FileCache:
1600
1610
  except filelock._error.Timeout:
1601
1611
  self._log_debug(f' Failed to lock: {sub_path}')
1602
1612
  wait_to_appear.append((idx, f'{source_pfx}{sub_path}', local_path,
1603
- lock_path))
1613
+ lock_path, source, sub_path))
1604
1614
  continue
1605
1615
  lock_list.append((lock_path, lock))
1606
1616
  source_idxes.append(idx)
@@ -1644,12 +1654,29 @@ class FileCache:
1644
1654
  # If wait_to_appear is not empty, then we failed to acquire at least one lock,
1645
1655
  # which means that another process was downloading the file. So now we just
1646
1656
  # sit here and wait for all of the missing files to magically show up, or for
1647
- # us to time out. If the lock file disappears but the destination file isn't
1648
- # present, that means the other process failed in its download.
1657
+ # us to time out.
1658
+ #
1659
+ # In each iteration we also check for stale locks: a stale lock exists when the
1660
+ # lock file is present but the process that created it has died (the OS released
1661
+ # the advisory flock, but did not delete the file). We detect a stale lock by
1662
+ # attempting a non-blocking acquire -- if it succeeds, no live process holds the
1663
+ # lock, so we steal it and initiate the download ourselves.
1664
+ #
1665
+ # Race-condition guarantee: multiple waiting processes may all notice the same
1666
+ # stale lock and all attempt `lock.acquire(timeout=0)` simultaneously. Because
1667
+ # the underlying flock(2) call is atomic, exactly one process wins the race; the
1668
+ # others receive a Timeout and remain in the wait list. The winner downloads the
1669
+ # file atomically (write to temp path + rename), so the losers will find the
1670
+ # completed file on their next poll iteration.
1671
+ #
1672
+ # If the lock file disappears without the destination file appearing, that means
1673
+ # the other process failed (or cleaned up after itself on error).
1649
1674
  timed_out = False
1650
1675
  while wait_to_appear:
1651
1676
  new_wait_to_appear = []
1652
- for idx, url, local_path, lock_path in wait_to_appear:
1677
+ stale_lock_items = [] # Items whose locks we have stolen
1678
+
1679
+ for idx, url, local_path, lock_path, source, sub_path in wait_to_appear:
1653
1680
  if local_path.is_file():
1654
1681
  func_ret[idx] = local_path
1655
1682
  self._log_debug(f' Downloaded elsewhere: {url}')
@@ -1659,18 +1686,67 @@ class FileCache:
1659
1686
  f'Another process failed to download {url}')
1660
1687
  self._log_debug(f' Download elsewhere failed: {url}')
1661
1688
  continue
1662
- new_wait_to_appear.append((idx, url, local_path, lock_path))
1689
+ # Lock file exists and destination is absent. Check whether the lock is
1690
+ # stale (flock released by a crashed process but file not cleaned up).
1691
+ stale_lock = filelock.FileLock(lock_path, timeout=0)
1692
+ try:
1693
+ stale_lock.acquire()
1694
+ # Acquired with timeout=0 → no live process holds this lock.
1695
+ stale_lock_items.append(
1696
+ (idx, url, local_path, lock_path, stale_lock, source, sub_path))
1697
+ self._log_debug(f' Stale lock detected for {url}, will re-download')
1698
+ except filelock._error.Timeout:
1699
+ # Lock is actively held -- another process is still downloading.
1700
+ new_wait_to_appear.append(
1701
+ (idx, url, local_path, lock_path, source, sub_path))
1702
+
1703
+ # Download all files whose stale locks we have just acquired, grouped by
1704
+ # source so that retrieve_multi can fetch them in parallel.
1705
+ stale_by_source: dict[str, list[tuple[
1706
+ int, str, Path, Path, filelock.FileLock, FileCacheSource, str]]] = {}
1707
+ for item in stale_lock_items:
1708
+ pfx = item[5]._src_prefix_
1709
+ if pfx not in stale_by_source:
1710
+ stale_by_source[pfx] = []
1711
+ stale_by_source[pfx].append(item)
1712
+
1713
+ for items in stale_by_source.values():
1714
+ s_idxes = [it[0] for it in items]
1715
+ s_urls = [it[1] for it in items]
1716
+ s_local_paths = [it[2] for it in items]
1717
+ s_lock_paths = [it[3] for it in items]
1718
+ s_locks = [it[4] for it in items]
1719
+ s_source = items[0][5]
1720
+ s_sub_paths = [it[6] for it in items]
1721
+
1722
+ rets = s_source.retrieve_multi(s_sub_paths, s_local_paths,
1723
+ preserve_mtime=False, nthreads=nthreads)
1724
+ for idx, url, ret in zip(s_idxes, s_urls, rets):
1725
+ func_ret[idx] = ret
1726
+ if isinstance(ret, Exception):
1727
+ files_not_exist.append(url)
1728
+ self._log_debug(
1729
+ f' Stale lock recovery download failed: '
1730
+ f'{url}: {ret!r}')
1731
+ else:
1732
+ self._download_counter += 1
1733
+ self._log_debug(
1734
+ f' Re-downloaded after stale lock recovery: {url}')
1735
+
1736
+ for s_lock, s_lock_path in zip(s_locks, s_lock_paths):
1737
+ s_lock.release()
1738
+ s_lock_path.unlink(missing_ok=True)
1663
1739
 
1664
1740
  if not new_wait_to_appear:
1665
1741
  break
1666
1742
 
1667
1743
  wait_to_appear = new_wait_to_appear
1668
- if time.time() - start_time > lock_timeout:
1744
+ if lock_timeout >= 0 and time.time() - start_time > lock_timeout:
1669
1745
  exc = TimeoutError(
1670
1746
  'Timeout while waiting for another process to finish downloading')
1671
1747
  self._log_debug(
1672
1748
  ' Timeout while waiting for another process to finish downloading:')
1673
- for idx, url, local_path, lock_path in wait_to_appear:
1749
+ for idx, url, local_path, lock_path, source, sub_path in wait_to_appear:
1674
1750
  func_ret[idx] = exc
1675
1751
  self._log_debug(f' {url}')
1676
1752
  if exception_on_fail:
@@ -1923,14 +1999,14 @@ class FileCache:
1923
1999
  pass
1924
2000
 
1925
2001
  if exception_on_fail:
1926
- exc_str = ''
1927
- if files_not_exist:
1928
- exc_str += f"File(s) do not exist: {', '.join(files_not_exist)}"
1929
- if files_failed:
1930
- if exc_str:
1931
- exc_str += ' AND '
1932
- exc_str += f"Failed to upload file(s): {', '.join(files_failed)}"
1933
- if exc_str:
2002
+ if files_not_exist and not files_failed:
2003
+ raise FileNotFoundError(
2004
+ f"File(s) do not exist: {', '.join(files_not_exist)}")
2005
+ elif files_failed:
2006
+ exc_str = f"Failed to upload file(s): {', '.join(files_failed)}"
2007
+ if files_not_exist:
2008
+ exc_str = (f"File(s) do not exist: {', '.join(files_not_exist)}"
2009
+ f' AND {exc_str}')
1934
2010
  raise FileNotFoundError(exc_str)
1935
2011
 
1936
2012
  return cast(list[Union[Path, Exception]], func_ret)
@@ -539,7 +539,20 @@ class FCPath:
539
539
  for i, j in zip(indices[:-1], indices[1:]))
540
540
 
541
541
  def __repr__(self) -> str:
542
- return f'FCPath({self._path!r})'
542
+ parts = [repr(self._path)]
543
+ if self._filecache is not None:
544
+ parts.append(f'filecache={self._filecache!r}')
545
+ if self._anonymous is not None:
546
+ parts.append(f'anonymous={self._anonymous!r}')
547
+ if self._lock_timeout is not None:
548
+ parts.append(f'lock_timeout={self._lock_timeout!r}')
549
+ if self._nthreads is not None:
550
+ parts.append(f'nthreads={self._nthreads!r}')
551
+ if self._url_to_url is not None:
552
+ parts.append(f'url_to_url={self._url_to_url!r}')
553
+ if self._url_to_path is not None:
554
+ parts.append(f'url_to_path={self._url_to_path!r}')
555
+ return f'FCPath({", ".join(parts)})'
543
556
 
544
557
  def __eq__(self, other: object) -> bool:
545
558
  if not isinstance(other, FCPath):
@@ -72,6 +72,13 @@ class FileCacheSource(ABC):
72
72
  # The _cache_subdir attribute is only used by the FileCache class
73
73
  self._cache_subdir = ''
74
74
 
75
+ def __repr__(self) -> str:
76
+ return (f'{type(self).__name__}({self._scheme!r}, {self._remote!r}, '
77
+ f'anonymous={self._anonymous!r})')
78
+
79
+ def __str__(self) -> str:
80
+ return self._src_prefix
81
+
75
82
  @classmethod
76
83
  @abstractmethod
77
84
  def schemes(self) -> tuple[str, ...]:
@@ -7,7 +7,7 @@ name = "rms-filecache"
7
7
  dynamic = ["version"]
8
8
  description = "File cache for files retrieved from the cloud"
9
9
  readme = "README.md"
10
- requires-python = ">=3.10"
10
+ requires-python = ">=3.11"
11
11
  dependencies = [
12
12
  "boto3",
13
13
  "filelock",
@@ -32,10 +32,11 @@ classifiers = [
32
32
  "Topic :: Software Development :: Libraries :: Python Modules",
33
33
  "Topic :: Utilities",
34
34
  "License :: OSI Approved :: Apache Software License",
35
- "Programming Language :: Python :: 3.10",
36
35
  "Programming Language :: Python :: 3.11",
37
36
  "Programming Language :: Python :: 3.12",
38
37
  "Programming Language :: Python :: 3.13",
38
+ "Programming Language :: Python :: 3.14",
39
+ "Programming Language :: Python :: 3.15",
39
40
  "Operating System :: MacOS :: MacOS X",
40
41
  "Operating System :: POSIX :: Linux",
41
42
  "Operating System :: Microsoft :: Windows"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rms-filecache
3
- Version: 3.1.0
3
+ Version: 3.2.0
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -15,14 +15,15 @@ Classifier: Natural Language :: English
15
15
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
16
  Classifier: Topic :: Utilities
17
17
  Classifier: License :: OSI Approved :: Apache Software License
18
- Classifier: Programming Language :: Python :: 3.10
19
18
  Classifier: Programming Language :: Python :: 3.11
20
19
  Classifier: Programming Language :: Python :: 3.12
21
20
  Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Programming Language :: Python :: 3.14
22
+ Classifier: Programming Language :: Python :: 3.15
22
23
  Classifier: Operating System :: MacOS :: MacOS X
23
24
  Classifier: Operating System :: POSIX :: Linux
24
25
  Classifier: Operating System :: Microsoft :: Windows
25
- Requires-Python: >=3.10
26
+ Requires-Python: >=3.11
26
27
  Description-Content-Type: text/markdown
27
28
  License-File: LICENSE
28
29
  Requires-Dist: boto3
@@ -30,6 +30,8 @@ rms_filecache.egg-info/PKG-INFO
30
30
  rms_filecache.egg-info/SOURCES.txt
31
31
  rms_filecache.egg-info/dependency_links.txt
32
32
  rms_filecache.egg-info/requires.txt
33
+ rms_filecache.egg-info/scm_file_list.json
34
+ rms_filecache.egg-info/scm_version.json
33
35
  rms_filecache.egg-info/top_level.txt
34
36
  test_files/expected/lorem1.txt
35
37
  test_files/expected/subdir1/lorem1.txt
@@ -0,0 +1,47 @@
1
+ {
2
+ "files": [
3
+ ".coveragerc",
4
+ ".flake8",
5
+ ".github/pull_request_template.md",
6
+ ".github/workflows/publish_to_pypi.yml",
7
+ ".github/workflows/publish_to_test_pypi.yml",
8
+ ".github/workflows/run-tests.yml",
9
+ ".gitignore",
10
+ ".readthedocs.yaml",
11
+ "LICENSE",
12
+ "README.md",
13
+ "docs/Makefile",
14
+ "docs/conf.py",
15
+ "docs/examples.rst",
16
+ "docs/index.rst",
17
+ "docs/introduction.rst",
18
+ "docs/make.bat",
19
+ "docs/module.rst",
20
+ "filecache/__init__.py",
21
+ "filecache/file_cache.py",
22
+ "filecache/file_cache_path.py",
23
+ "filecache/file_cache_source.py",
24
+ "filecache/file_cache_types.py",
25
+ "filecache/py.typed",
26
+ "mypy.ini",
27
+ "pyproject.toml",
28
+ "requirements.txt",
29
+ "setup.cfg",
30
+ "test_files/expected/lorem1.txt",
31
+ "test_files/expected/subdir1/lorem1.txt",
32
+ "test_files/expected/subdir1/subdir2a/binary1.bin",
33
+ "test_files/expected/subdir1/subdir2b/binary1.bin",
34
+ "test_programs/locking.py",
35
+ "test_programs/multi_download.py",
36
+ "tests/__init__.py",
37
+ "tests/gs_lifecycle_config.json",
38
+ "tests/gs_lifecycle_config_info.txt",
39
+ "tests/s3_bucket_policy_read.txt",
40
+ "tests/s3_bucket_policy_write.txt",
41
+ "tests/s3_lifecycle_policy.txt",
42
+ "tests/test_file_cache.py",
43
+ "tests/test_file_cache_path.py",
44
+ "tests/test_file_cache_source.py",
45
+ "tests/test_url_mapping.py"
46
+ ]
47
+ }
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "3.2.0",
3
+ "distance": 0,
4
+ "node": "g521391624fb1be045c1b5fe632c3eee1c6801baf",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-10-21"
8
+ }
@@ -10,6 +10,7 @@ import os
10
10
  from pathlib import Path
11
11
  import platform
12
12
  import tempfile
13
+ import threading
13
14
  import time
14
15
  import uuid
15
16
 
@@ -1055,6 +1056,197 @@ def test_locking_multi_pfx_2():
1055
1056
  assert not fc.cache_dir.exists()
1056
1057
 
1057
1058
 
1059
+ @pytest.mark.skipif(platform.system() == 'Windows',
1060
+ reason='Uses POSIX fcntl for stale-lock simulation')
1061
+ def test_stale_lock_single():
1062
+ """A stale lock (file exists, no flock held) must not block a single-file retrieve.
1063
+
1064
+ We simulate a stale lock by acquiring the flock with raw fcntl, then releasing
1065
+ only the flock (leaving the lock file on disk) before the retrieve. On the
1066
+ next call, filelock should acquire the lock immediately because no process
1067
+ holds it.
1068
+ """
1069
+ import fcntl
1070
+
1071
+ with FileCache(delete_on_exit=True) as fc:
1072
+ filename = EXPECTED_FILENAMES[0]
1073
+ local_path = (fc.cache_dir /
1074
+ (HTTP_TEST_ROOT.replace('https://', 'http_') + '/' + filename))
1075
+ lock_path = fc._lock_path(local_path)
1076
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
1077
+
1078
+ # Create and hold the lock via raw fcntl so we can release the flock without
1079
+ # deleting the file (simulating a process that crashed).
1080
+ fd = os.open(str(lock_path), os.O_RDWR | os.O_CREAT, 0o644)
1081
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1082
+
1083
+ # Release the flock but keep the file -- this is the "stale lock" state.
1084
+ fcntl.flock(fd, fcntl.LOCK_UN)
1085
+ os.close(fd)
1086
+
1087
+ # The lock file exists, but nobody holds the flock. The retrieve must
1088
+ # succeed rather than raising a TimeoutError.
1089
+ path = fc.retrieve(f'{HTTP_TEST_ROOT}/{filename}', lock_timeout=0)
1090
+ assert isinstance(path, Path)
1091
+ assert path.is_file()
1092
+ _compare_to_expected_path(path, filename)
1093
+ assert fc.download_counter == 1
1094
+ assert fc.upload_counter == 0
1095
+
1096
+
1097
+ @pytest.mark.skipif(platform.system() == 'Windows',
1098
+ reason='Uses POSIX fcntl for stale-lock simulation')
1099
+ def test_stale_lock_multi():
1100
+ """A stale lock must not permanently block multi-file retrieval.
1101
+
1102
+ We hold the flock for the first file from a background thread, trigger the
1103
+ multi-file download (which puts file[0] into wait_to_appear), then simulate
1104
+ a crash by releasing only the flock (but not deleting the lock file) while
1105
+ the wait loop is running. The stale-lock detection code must steal the lock
1106
+ and complete the download without timing out.
1107
+
1108
+ Race-condition guarantee: the background thread uses a threading.Event to
1109
+ synchronise so that the flock is released only after the main thread's
1110
+ retrieve call has entered the wait_to_appear loop. Because only one process
1111
+ can win the flock race, the main thread is guaranteed to be the sole
1112
+ downloader of file[0] after stealing the stale lock.
1113
+ """
1114
+ import fcntl
1115
+
1116
+ with FileCache(delete_on_exit=True) as fc:
1117
+ filename0 = EXPECTED_FILENAMES[0]
1118
+ local_path0 = (fc.cache_dir /
1119
+ (HTTP_TEST_ROOT.replace('https://', 'http_') + '/' + filename0))
1120
+ lock_path0 = fc._lock_path(local_path0)
1121
+ lock_path0.parent.mkdir(parents=True, exist_ok=True)
1122
+
1123
+ # Phase-control events
1124
+ flock_held = threading.Event() # signalled once the raw flock is held
1125
+ flock_release = threading.Event() # signalled when the "crash" should happen
1126
+
1127
+ def holder_thread() -> None:
1128
+ """Hold the flock until told to "crash", then release without cleanup."""
1129
+ fd = os.open(str(lock_path0), os.O_RDWR | os.O_CREAT, 0o644)
1130
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1131
+ flock_held.set()
1132
+ flock_release.wait()
1133
+ # Simulate crash: release flock, leave file on disk.
1134
+ fcntl.flock(fd, fcntl.LOCK_UN)
1135
+ os.close(fd)
1136
+
1137
+ t = threading.Thread(target=holder_thread, daemon=True)
1138
+ t.start()
1139
+ assert flock_held.wait(timeout=5), 'holder thread never acquired flock'
1140
+
1141
+ results: list[list[Path]] = []
1142
+ exc_holder: list[Exception] = []
1143
+
1144
+ def do_retrieve() -> None:
1145
+ try:
1146
+ filenames = [f'{HTTP_TEST_ROOT}/{f}' for f in EXPECTED_FILENAMES]
1147
+ ret = fc.retrieve(filenames, lock_timeout=15)
1148
+ results.append(ret) # type: ignore[arg-type]
1149
+ except Exception as e:
1150
+ exc_holder.append(e)
1151
+
1152
+ retrieve_thread = threading.Thread(target=do_retrieve, daemon=True)
1153
+ retrieve_thread.start()
1154
+
1155
+ # Wait until all non-blocked files have downloaded and retrieve_thread is
1156
+ # still alive — at that point it must be blocked in the wait_to_appear loop.
1157
+ deadline = time.monotonic() + 15
1158
+ while time.monotonic() < deadline:
1159
+ if (fc.download_counter == len(EXPECTED_FILENAMES) - 1 and
1160
+ retrieve_thread.is_alive()):
1161
+ break
1162
+ time.sleep(0.01)
1163
+ else:
1164
+ pytest.fail('retrieve never reached the blocked/stale-lock state')
1165
+
1166
+ # Signal the holder to "crash" -- releases flock, keeps the lock file.
1167
+ flock_release.set()
1168
+ t.join(timeout=5)
1169
+ assert not t.is_alive(), 'holder thread did not finish'
1170
+
1171
+ retrieve_thread.join(timeout=20)
1172
+ assert not retrieve_thread.is_alive(), 'retrieve thread timed out (stale lock not recovered)'
1173
+
1174
+ assert not exc_holder, f'retrieve raised: {exc_holder[0]}'
1175
+ assert results, 'retrieve returned no result'
1176
+ for r in results[0]:
1177
+ assert isinstance(r, Path), f'Expected Path, got {r!r}'
1178
+ assert fc.download_counter == len(EXPECTED_FILENAMES)
1179
+ assert fc.upload_counter == 0
1180
+
1181
+
1182
+ @pytest.mark.skipif(platform.system() == 'Windows',
1183
+ reason='Uses POSIX fcntl for stale-lock simulation')
1184
+ def test_stale_lock_multi_pfx():
1185
+ """Same as test_stale_lock_multi but exercised through the FCPath interface."""
1186
+ import fcntl
1187
+
1188
+ with FileCache(delete_on_exit=True) as fc:
1189
+ pfx = fc.new_path(HTTP_TEST_ROOT, lock_timeout=15)
1190
+ filename0 = EXPECTED_FILENAMES[0]
1191
+ local_path0 = (fc.cache_dir /
1192
+ (HTTP_TEST_ROOT.replace('https://', 'http_') + '/' + filename0))
1193
+ lock_path0 = fc._lock_path(local_path0)
1194
+ lock_path0.parent.mkdir(parents=True, exist_ok=True)
1195
+
1196
+ flock_held = threading.Event()
1197
+ flock_release = threading.Event()
1198
+
1199
+ def holder_thread() -> None:
1200
+ fd = os.open(str(lock_path0), os.O_RDWR | os.O_CREAT, 0o644)
1201
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1202
+ flock_held.set()
1203
+ flock_release.wait()
1204
+ fcntl.flock(fd, fcntl.LOCK_UN)
1205
+ os.close(fd)
1206
+
1207
+ t = threading.Thread(target=holder_thread, daemon=True)
1208
+ t.start()
1209
+ assert flock_held.wait(timeout=5), 'holder thread never acquired flock'
1210
+
1211
+ results: list[list[Path]] = []
1212
+ exc_holder: list[Exception] = []
1213
+
1214
+ def do_retrieve() -> None:
1215
+ try:
1216
+ ret = pfx.retrieve(EXPECTED_FILENAMES)
1217
+ results.append(ret) # type: ignore[arg-type]
1218
+ except Exception as e:
1219
+ exc_holder.append(e)
1220
+
1221
+ retrieve_thread = threading.Thread(target=do_retrieve, daemon=True)
1222
+ retrieve_thread.start()
1223
+
1224
+ # Wait until all non-blocked files have downloaded and retrieve_thread is
1225
+ # still alive — at that point it must be blocked in the wait_to_appear loop.
1226
+ deadline = time.monotonic() + 15
1227
+ while time.monotonic() < deadline:
1228
+ if (fc.download_counter == len(EXPECTED_FILENAMES) - 1 and
1229
+ retrieve_thread.is_alive()):
1230
+ break
1231
+ time.sleep(0.01)
1232
+ else:
1233
+ pytest.fail('retrieve never reached the blocked/stale-lock state')
1234
+
1235
+ flock_release.set()
1236
+ t.join(timeout=5)
1237
+ assert not t.is_alive()
1238
+
1239
+ retrieve_thread.join(timeout=20)
1240
+ assert not retrieve_thread.is_alive(), 'retrieve thread timed out (stale lock not recovered)'
1241
+
1242
+ assert not exc_holder, f'retrieve raised: {exc_holder[0]}'
1243
+ assert results
1244
+ for r in results[0]:
1245
+ assert isinstance(r, Path)
1246
+ assert fc.download_counter == len(EXPECTED_FILENAMES)
1247
+ assert fc.upload_counter == 0
1248
+
1249
+
1058
1250
  def test_cleanup_locking_bad():
1059
1251
  logger = MyLogger()
1060
1252
  with FileCache(cache_name=None, logger=logger) as fc:
@@ -2948,3 +3140,22 @@ def test_modification_time_caching_multi(time_sensitive, cache_metadata, mp_safe
2948
3140
  assert lp[0].stat().st_mtime == mtime_lp_orig[0]
2949
3141
  assert lp[1].stat().st_mtime == mtime_lp_orig[1]
2950
3142
  assert lp[2].stat().st_mtime == mtime_lp_orig[2]
3143
+
3144
+
3145
+ def test_filecache_repr_str():
3146
+ with FileCache(cache_name=None) as fc:
3147
+ r = repr(fc)
3148
+ assert r.startswith('FileCache(None,')
3149
+ assert 'anonymous=False' in r
3150
+ assert 'lock_timeout=60' in r
3151
+ assert 'nthreads=8' in r
3152
+ assert str(fc) == str(fc.cache_dir)
3153
+
3154
+ # Named cache with non-default options
3155
+ with FileCache('repr-test', anonymous=True, lock_timeout=30, nthreads=4,
3156
+ delete_on_exit=True) as fc:
3157
+ r = repr(fc)
3158
+ assert r.startswith("FileCache('repr-test',")
3159
+ assert 'anonymous=True' in r
3160
+ assert 'lock_timeout=30' in r
3161
+ assert 'nthreads=4' in r
@@ -1308,3 +1308,37 @@ def test_splitpath():
1308
1308
  FCPath('d'))
1309
1309
  assert FCPath('gs://bucket/a/c/b/b1/c/d/d1').splitpath('c') == \
1310
1310
  (FCPath('gs://bucket/a'), FCPath('b/b1'), FCPath('d/d1'))
1311
+
1312
+
1313
+ def test_fcpath_repr_str():
1314
+ # Default FCPath — only the path
1315
+ p = FCPath('gs://bucket/path/to/file')
1316
+ assert repr(p) == "FCPath('gs://bucket/path/to/file')"
1317
+ assert str(p) == 'gs://bucket/path/to/file'
1318
+
1319
+ # FCPath with extra options — they appear in repr
1320
+ p2 = FCPath('s3://bucket/path', anonymous=True, lock_timeout=10, nthreads=2)
1321
+ r = repr(p2)
1322
+ assert r.startswith('FCPath(')
1323
+ assert 'anonymous=True' in r
1324
+ assert 'lock_timeout=10' in r
1325
+ assert 'nthreads=2' in r
1326
+
1327
+ # filecache kwarg appears in repr when set
1328
+ with FileCache(cache_name=None, delete_on_exit=True) as fc:
1329
+ p3 = FCPath('gs://bucket/x', filecache=fc)
1330
+ r3 = repr(p3)
1331
+ assert 'filecache=' in r3
1332
+
1333
+ # url_to_url and url_to_path appear in repr when set
1334
+ def _identity_url(scheme, remote, path):
1335
+ return None
1336
+
1337
+ def _identity_path(scheme, remote, path, cache_dir, cache_subdir):
1338
+ return None
1339
+
1340
+ p4 = FCPath('gs://bucket/y',
1341
+ url_to_url=_identity_url, url_to_path=_identity_path)
1342
+ r4 = repr(p4)
1343
+ assert 'url_to_url=' in r4
1344
+ assert 'url_to_path=' in r4
@@ -404,3 +404,32 @@ def test_fake_source_atomic_operations(tmp_path: Path):
404
404
  pathlib.Path.rename = original_rename
405
405
  finally:
406
406
  FileCacheSourceFake.delete_default_storage_dir()
407
+
408
+
409
+ def test_filecachesource_repr_str():
410
+ src_gs = FileCacheSourceGS('gs', 'rms-filecache-tests', anonymous=True)
411
+ r = repr(src_gs)
412
+ assert r == "FileCacheSourceGS('gs', 'rms-filecache-tests', anonymous=True)"
413
+ assert str(src_gs) == 'gs://rms-filecache-tests'
414
+
415
+ src_s3 = FileCacheSourceS3('s3', 'rms-filecache-tests', anonymous=True)
416
+ r = repr(src_s3)
417
+ assert r == "FileCacheSourceS3('s3', 'rms-filecache-tests', anonymous=True)"
418
+ assert str(src_s3) == 's3://rms-filecache-tests'
419
+
420
+ src_http = FileCacheSourceHTTP('https', 'storage.googleapis.com',
421
+ anonymous=False)
422
+ r = repr(src_http)
423
+ assert r == ("FileCacheSourceHTTP('https', 'storage.googleapis.com', "
424
+ "anonymous=False)")
425
+ assert str(src_http) == 'https://storage.googleapis.com'
426
+
427
+ src_file = FileCacheSourceFile('file', '', anonymous=False)
428
+ r = repr(src_file)
429
+ assert r == "FileCacheSourceFile('file', '', anonymous=False)"
430
+ assert str(src_file) == 'file://'
431
+
432
+ src_fake = FileCacheSourceFake('fake', 'fake-bucket', anonymous=False)
433
+ r = repr(src_fake)
434
+ assert r == "FileCacheSourceFake('fake', 'fake-bucket', anonymous=False)"
435
+ assert str(src_fake) == 'fake://fake-bucket'
@@ -1,34 +0,0 @@
1
- # file generated by setuptools-scm
2
- # don't change, don't track in version control
3
-
4
- __all__ = [
5
- "__version__",
6
- "__version_tuple__",
7
- "version",
8
- "version_tuple",
9
- "__commit_id__",
10
- "commit_id",
11
- ]
12
-
13
- TYPE_CHECKING = False
14
- if TYPE_CHECKING:
15
- from typing import Tuple
16
- from typing import Union
17
-
18
- VERSION_TUPLE = Tuple[Union[int, str], ...]
19
- COMMIT_ID = Union[str, None]
20
- else:
21
- VERSION_TUPLE = object
22
- COMMIT_ID = object
23
-
24
- version: str
25
- __version__: str
26
- __version_tuple__: VERSION_TUPLE
27
- version_tuple: VERSION_TUPLE
28
- commit_id: COMMIT_ID
29
- __commit_id__: COMMIT_ID
30
-
31
- __version__ = version = '3.1.0'
32
- __version_tuple__ = version_tuple = (3, 1, 0)
33
-
34
- __commit_id__ = commit_id = 'g3ecac5a3d'
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes