rms-filecache 3.1.0__tar.gz → 3.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.github/workflows/publish_to_pypi.yml +2 -2
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.github/workflows/publish_to_test_pypi.yml +2 -2
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.github/workflows/run-tests.yml +7 -7
- {rms_filecache-3.1.0/rms_filecache.egg-info → rms_filecache-3.2.0}/PKG-INFO +4 -3
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/__init__.py +0 -1
- rms_filecache-3.2.0/filecache/_version.py +24 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/file_cache.py +91 -15
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/file_cache_path.py +14 -1
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/file_cache_source.py +7 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/pyproject.toml +3 -2
- {rms_filecache-3.1.0 → rms_filecache-3.2.0/rms_filecache.egg-info}/PKG-INFO +4 -3
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/rms_filecache.egg-info/SOURCES.txt +2 -0
- rms_filecache-3.2.0/rms_filecache.egg-info/scm_file_list.json +47 -0
- rms_filecache-3.2.0/rms_filecache.egg-info/scm_version.json +8 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/test_file_cache.py +211 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/test_file_cache_path.py +34 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/test_file_cache_source.py +29 -0
- rms_filecache-3.1.0/filecache/_version.py +0 -34
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.coveragerc +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.flake8 +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.github/pull_request_template.md +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.gitignore +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/.readthedocs.yaml +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/LICENSE +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/README.md +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/Makefile +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/conf.py +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/examples.rst +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/index.rst +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/introduction.rst +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/make.bat +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/docs/module.rst +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/file_cache_types.py +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/filecache/py.typed +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/mypy.ini +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/requirements.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/setup.cfg +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_programs/locking.py +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_programs/multi_download.py +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/__init__.py +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/s3_bucket_policy_write.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/s3_lifecycle_policy.txt +0 -0
- {rms_filecache-3.1.0 → rms_filecache-3.2.0}/tests/test_url_mapping.py +0 -0
|
@@ -10,12 +10,12 @@ jobs:
|
|
|
10
10
|
runs-on: ubuntu-latest
|
|
11
11
|
steps:
|
|
12
12
|
- name: Checkout
|
|
13
|
-
uses: actions/checkout@
|
|
13
|
+
uses: actions/checkout@v7
|
|
14
14
|
with:
|
|
15
15
|
fetch-depth: 0
|
|
16
16
|
|
|
17
17
|
- name: Set up Python 3.12
|
|
18
|
-
uses: actions/setup-python@
|
|
18
|
+
uses: actions/setup-python@v7
|
|
19
19
|
with:
|
|
20
20
|
python-version: 3.12
|
|
21
21
|
|
|
@@ -9,12 +9,12 @@ jobs:
|
|
|
9
9
|
runs-on: ubuntu-latest
|
|
10
10
|
steps:
|
|
11
11
|
- name: Checkout
|
|
12
|
-
uses: actions/checkout@
|
|
12
|
+
uses: actions/checkout@v7
|
|
13
13
|
with:
|
|
14
14
|
fetch-depth: 0
|
|
15
15
|
|
|
16
16
|
- name: Set up Python 3.12
|
|
17
|
-
uses: actions/setup-python@
|
|
17
|
+
uses: actions/setup-python@v7
|
|
18
18
|
with:
|
|
19
19
|
python-version: 3.12
|
|
20
20
|
|
|
@@ -16,12 +16,12 @@ jobs:
|
|
|
16
16
|
runs-on: ubuntu-latest
|
|
17
17
|
steps:
|
|
18
18
|
- name: Checkout
|
|
19
|
-
uses: actions/checkout@
|
|
19
|
+
uses: actions/checkout@v7
|
|
20
20
|
with:
|
|
21
21
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
22
22
|
|
|
23
23
|
- name: Set up Python ${{ matrix.python-version }}
|
|
24
|
-
uses: actions/setup-python@
|
|
24
|
+
uses: actions/setup-python@v7
|
|
25
25
|
with:
|
|
26
26
|
python-version: ${{ matrix.python-version }}
|
|
27
27
|
|
|
@@ -43,19 +43,19 @@ jobs:
|
|
|
43
43
|
strategy:
|
|
44
44
|
matrix:
|
|
45
45
|
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
46
|
-
python-version: ['3.
|
|
46
|
+
python-version: ['3.11', '3.12', '3.13', '3.14', '3.15']
|
|
47
47
|
fail-fast: false
|
|
48
48
|
permissions:
|
|
49
49
|
contents: read
|
|
50
50
|
id-token: write
|
|
51
51
|
steps:
|
|
52
52
|
- name: Checkout
|
|
53
|
-
uses: actions/checkout@
|
|
53
|
+
uses: actions/checkout@v7
|
|
54
54
|
with:
|
|
55
55
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
56
56
|
|
|
57
57
|
- name: Set up Python ${{ matrix.python-version }}
|
|
58
|
-
uses: actions/setup-python@
|
|
58
|
+
uses: actions/setup-python@v7
|
|
59
59
|
with:
|
|
60
60
|
python-version: ${{ matrix.python-version }}
|
|
61
61
|
|
|
@@ -66,7 +66,7 @@ jobs:
|
|
|
66
66
|
# See: https://www.youtube.com/watch?v=ZgVhU5qvK1M
|
|
67
67
|
# for a walkthrough on how to create the workload_identity_provider
|
|
68
68
|
- name: Authenticate to Google Cloud
|
|
69
|
-
uses: google-github-actions/auth@
|
|
69
|
+
uses: google-github-actions/auth@v3
|
|
70
70
|
with:
|
|
71
71
|
project_id: rms-node
|
|
72
72
|
workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
|
|
@@ -80,7 +80,7 @@ jobs:
|
|
|
80
80
|
coverage report -m
|
|
81
81
|
|
|
82
82
|
- name: Upload coverage report to codecov
|
|
83
|
-
uses: codecov/codecov-action@
|
|
83
|
+
uses: codecov/codecov-action@v7
|
|
84
84
|
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
|
|
85
85
|
with:
|
|
86
86
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rms-filecache
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: File cache for files retrieved from the cloud
|
|
5
5
|
Maintainer-email: "Robert S. French" <rfrench@seti.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -15,14 +15,15 @@ Classifier: Natural Language :: English
|
|
|
15
15
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
16
16
|
Classifier: Topic :: Utilities
|
|
17
17
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
23
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
23
24
|
Classifier: Operating System :: POSIX :: Linux
|
|
24
25
|
Classifier: Operating System :: Microsoft :: Windows
|
|
25
|
-
Requires-Python: >=3.
|
|
26
|
+
Requires-Python: >=3.11
|
|
26
27
|
Description-Content-Type: text/markdown
|
|
27
28
|
License-File: LICENSE
|
|
28
29
|
Requires-Dist: boto3
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '3.2.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (3, 2, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'g521391624'
|
|
@@ -264,6 +264,7 @@ class FileCache:
|
|
|
264
264
|
raise TypeError(f'cache_name argument {cache_name} is of improper type')
|
|
265
265
|
|
|
266
266
|
is_shared = (cache_name is not None)
|
|
267
|
+
self._cache_name = cache_name
|
|
267
268
|
|
|
268
269
|
self._delete_on_exit = (delete_on_exit if delete_on_exit is not None
|
|
269
270
|
else not is_shared)
|
|
@@ -418,6 +419,15 @@ class FileCache:
|
|
|
418
419
|
return _GLOBAL_LOGGER
|
|
419
420
|
return cast(Logger, self._logger)
|
|
420
421
|
|
|
422
|
+
def __repr__(self) -> str:
|
|
423
|
+
return (f'FileCache({self._cache_name!r}, '
|
|
424
|
+
f'anonymous={self._anonymous!r}, '
|
|
425
|
+
f'lock_timeout={self._lock_timeout!r}, '
|
|
426
|
+
f'nthreads={self._nthreads!r})')
|
|
427
|
+
|
|
428
|
+
def __str__(self) -> str:
|
|
429
|
+
return str(self._cache_dir)
|
|
430
|
+
|
|
421
431
|
def _log_debug(self, msg: str) -> None:
|
|
422
432
|
logger = self.logger
|
|
423
433
|
if logger:
|
|
@@ -1600,7 +1610,7 @@ class FileCache:
|
|
|
1600
1610
|
except filelock._error.Timeout:
|
|
1601
1611
|
self._log_debug(f' Failed to lock: {sub_path}')
|
|
1602
1612
|
wait_to_appear.append((idx, f'{source_pfx}{sub_path}', local_path,
|
|
1603
|
-
lock_path))
|
|
1613
|
+
lock_path, source, sub_path))
|
|
1604
1614
|
continue
|
|
1605
1615
|
lock_list.append((lock_path, lock))
|
|
1606
1616
|
source_idxes.append(idx)
|
|
@@ -1644,12 +1654,29 @@ class FileCache:
|
|
|
1644
1654
|
# If wait_to_appear is not empty, then we failed to acquire at least one lock,
|
|
1645
1655
|
# which means that another process was downloading the file. So now we just
|
|
1646
1656
|
# sit here and wait for all of the missing files to magically show up, or for
|
|
1647
|
-
# us to time out.
|
|
1648
|
-
#
|
|
1657
|
+
# us to time out.
|
|
1658
|
+
#
|
|
1659
|
+
# In each iteration we also check for stale locks: a stale lock exists when the
|
|
1660
|
+
# lock file is present but the process that created it has died (the OS released
|
|
1661
|
+
# the advisory flock, but did not delete the file). We detect a stale lock by
|
|
1662
|
+
# attempting a non-blocking acquire -- if it succeeds, no live process holds the
|
|
1663
|
+
# lock, so we steal it and initiate the download ourselves.
|
|
1664
|
+
#
|
|
1665
|
+
# Race-condition guarantee: multiple waiting processes may all notice the same
|
|
1666
|
+
# stale lock and all attempt `lock.acquire(timeout=0)` simultaneously. Because
|
|
1667
|
+
# the underlying flock(2) call is atomic, exactly one process wins the race; the
|
|
1668
|
+
# others receive a Timeout and remain in the wait list. The winner downloads the
|
|
1669
|
+
# file atomically (write to temp path + rename), so the losers will find the
|
|
1670
|
+
# completed file on their next poll iteration.
|
|
1671
|
+
#
|
|
1672
|
+
# If the lock file disappears without the destination file appearing, that means
|
|
1673
|
+
# the other process failed (or cleaned up after itself on error).
|
|
1649
1674
|
timed_out = False
|
|
1650
1675
|
while wait_to_appear:
|
|
1651
1676
|
new_wait_to_appear = []
|
|
1652
|
-
|
|
1677
|
+
stale_lock_items = [] # Items whose locks we have stolen
|
|
1678
|
+
|
|
1679
|
+
for idx, url, local_path, lock_path, source, sub_path in wait_to_appear:
|
|
1653
1680
|
if local_path.is_file():
|
|
1654
1681
|
func_ret[idx] = local_path
|
|
1655
1682
|
self._log_debug(f' Downloaded elsewhere: {url}')
|
|
@@ -1659,18 +1686,67 @@ class FileCache:
|
|
|
1659
1686
|
f'Another process failed to download {url}')
|
|
1660
1687
|
self._log_debug(f' Download elsewhere failed: {url}')
|
|
1661
1688
|
continue
|
|
1662
|
-
|
|
1689
|
+
# Lock file exists and destination is absent. Check whether the lock is
|
|
1690
|
+
# stale (flock released by a crashed process but file not cleaned up).
|
|
1691
|
+
stale_lock = filelock.FileLock(lock_path, timeout=0)
|
|
1692
|
+
try:
|
|
1693
|
+
stale_lock.acquire()
|
|
1694
|
+
# Acquired with timeout=0 → no live process holds this lock.
|
|
1695
|
+
stale_lock_items.append(
|
|
1696
|
+
(idx, url, local_path, lock_path, stale_lock, source, sub_path))
|
|
1697
|
+
self._log_debug(f' Stale lock detected for {url}, will re-download')
|
|
1698
|
+
except filelock._error.Timeout:
|
|
1699
|
+
# Lock is actively held -- another process is still downloading.
|
|
1700
|
+
new_wait_to_appear.append(
|
|
1701
|
+
(idx, url, local_path, lock_path, source, sub_path))
|
|
1702
|
+
|
|
1703
|
+
# Download all files whose stale locks we have just acquired, grouped by
|
|
1704
|
+
# source so that retrieve_multi can fetch them in parallel.
|
|
1705
|
+
stale_by_source: dict[str, list[tuple[
|
|
1706
|
+
int, str, Path, Path, filelock.FileLock, FileCacheSource, str]]] = {}
|
|
1707
|
+
for item in stale_lock_items:
|
|
1708
|
+
pfx = item[5]._src_prefix_
|
|
1709
|
+
if pfx not in stale_by_source:
|
|
1710
|
+
stale_by_source[pfx] = []
|
|
1711
|
+
stale_by_source[pfx].append(item)
|
|
1712
|
+
|
|
1713
|
+
for items in stale_by_source.values():
|
|
1714
|
+
s_idxes = [it[0] for it in items]
|
|
1715
|
+
s_urls = [it[1] for it in items]
|
|
1716
|
+
s_local_paths = [it[2] for it in items]
|
|
1717
|
+
s_lock_paths = [it[3] for it in items]
|
|
1718
|
+
s_locks = [it[4] for it in items]
|
|
1719
|
+
s_source = items[0][5]
|
|
1720
|
+
s_sub_paths = [it[6] for it in items]
|
|
1721
|
+
|
|
1722
|
+
rets = s_source.retrieve_multi(s_sub_paths, s_local_paths,
|
|
1723
|
+
preserve_mtime=False, nthreads=nthreads)
|
|
1724
|
+
for idx, url, ret in zip(s_idxes, s_urls, rets):
|
|
1725
|
+
func_ret[idx] = ret
|
|
1726
|
+
if isinstance(ret, Exception):
|
|
1727
|
+
files_not_exist.append(url)
|
|
1728
|
+
self._log_debug(
|
|
1729
|
+
f' Stale lock recovery download failed: '
|
|
1730
|
+
f'{url}: {ret!r}')
|
|
1731
|
+
else:
|
|
1732
|
+
self._download_counter += 1
|
|
1733
|
+
self._log_debug(
|
|
1734
|
+
f' Re-downloaded after stale lock recovery: {url}')
|
|
1735
|
+
|
|
1736
|
+
for s_lock, s_lock_path in zip(s_locks, s_lock_paths):
|
|
1737
|
+
s_lock.release()
|
|
1738
|
+
s_lock_path.unlink(missing_ok=True)
|
|
1663
1739
|
|
|
1664
1740
|
if not new_wait_to_appear:
|
|
1665
1741
|
break
|
|
1666
1742
|
|
|
1667
1743
|
wait_to_appear = new_wait_to_appear
|
|
1668
|
-
if time.time() - start_time > lock_timeout:
|
|
1744
|
+
if lock_timeout >= 0 and time.time() - start_time > lock_timeout:
|
|
1669
1745
|
exc = TimeoutError(
|
|
1670
1746
|
'Timeout while waiting for another process to finish downloading')
|
|
1671
1747
|
self._log_debug(
|
|
1672
1748
|
' Timeout while waiting for another process to finish downloading:')
|
|
1673
|
-
for idx, url, local_path, lock_path in wait_to_appear:
|
|
1749
|
+
for idx, url, local_path, lock_path, source, sub_path in wait_to_appear:
|
|
1674
1750
|
func_ret[idx] = exc
|
|
1675
1751
|
self._log_debug(f' {url}')
|
|
1676
1752
|
if exception_on_fail:
|
|
@@ -1923,14 +1999,14 @@ class FileCache:
|
|
|
1923
1999
|
pass
|
|
1924
2000
|
|
|
1925
2001
|
if exception_on_fail:
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
2002
|
+
if files_not_exist and not files_failed:
|
|
2003
|
+
raise FileNotFoundError(
|
|
2004
|
+
f"File(s) do not exist: {', '.join(files_not_exist)}")
|
|
2005
|
+
elif files_failed:
|
|
2006
|
+
exc_str = f"Failed to upload file(s): {', '.join(files_failed)}"
|
|
2007
|
+
if files_not_exist:
|
|
2008
|
+
exc_str = (f"File(s) do not exist: {', '.join(files_not_exist)}"
|
|
2009
|
+
f' AND {exc_str}')
|
|
1934
2010
|
raise FileNotFoundError(exc_str)
|
|
1935
2011
|
|
|
1936
2012
|
return cast(list[Union[Path, Exception]], func_ret)
|
|
@@ -539,7 +539,20 @@ class FCPath:
|
|
|
539
539
|
for i, j in zip(indices[:-1], indices[1:]))
|
|
540
540
|
|
|
541
541
|
def __repr__(self) -> str:
|
|
542
|
-
|
|
542
|
+
parts = [repr(self._path)]
|
|
543
|
+
if self._filecache is not None:
|
|
544
|
+
parts.append(f'filecache={self._filecache!r}')
|
|
545
|
+
if self._anonymous is not None:
|
|
546
|
+
parts.append(f'anonymous={self._anonymous!r}')
|
|
547
|
+
if self._lock_timeout is not None:
|
|
548
|
+
parts.append(f'lock_timeout={self._lock_timeout!r}')
|
|
549
|
+
if self._nthreads is not None:
|
|
550
|
+
parts.append(f'nthreads={self._nthreads!r}')
|
|
551
|
+
if self._url_to_url is not None:
|
|
552
|
+
parts.append(f'url_to_url={self._url_to_url!r}')
|
|
553
|
+
if self._url_to_path is not None:
|
|
554
|
+
parts.append(f'url_to_path={self._url_to_path!r}')
|
|
555
|
+
return f'FCPath({", ".join(parts)})'
|
|
543
556
|
|
|
544
557
|
def __eq__(self, other: object) -> bool:
|
|
545
558
|
if not isinstance(other, FCPath):
|
|
@@ -72,6 +72,13 @@ class FileCacheSource(ABC):
|
|
|
72
72
|
# The _cache_subdir attribute is only used by the FileCache class
|
|
73
73
|
self._cache_subdir = ''
|
|
74
74
|
|
|
75
|
+
def __repr__(self) -> str:
|
|
76
|
+
return (f'{type(self).__name__}({self._scheme!r}, {self._remote!r}, '
|
|
77
|
+
f'anonymous={self._anonymous!r})')
|
|
78
|
+
|
|
79
|
+
def __str__(self) -> str:
|
|
80
|
+
return self._src_prefix
|
|
81
|
+
|
|
75
82
|
@classmethod
|
|
76
83
|
@abstractmethod
|
|
77
84
|
def schemes(self) -> tuple[str, ...]:
|
|
@@ -7,7 +7,7 @@ name = "rms-filecache"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
description = "File cache for files retrieved from the cloud"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
11
|
dependencies = [
|
|
12
12
|
"boto3",
|
|
13
13
|
"filelock",
|
|
@@ -32,10 +32,11 @@ classifiers = [
|
|
|
32
32
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
33
33
|
"Topic :: Utilities",
|
|
34
34
|
"License :: OSI Approved :: Apache Software License",
|
|
35
|
-
"Programming Language :: Python :: 3.10",
|
|
36
35
|
"Programming Language :: Python :: 3.11",
|
|
37
36
|
"Programming Language :: Python :: 3.12",
|
|
38
37
|
"Programming Language :: Python :: 3.13",
|
|
38
|
+
"Programming Language :: Python :: 3.14",
|
|
39
|
+
"Programming Language :: Python :: 3.15",
|
|
39
40
|
"Operating System :: MacOS :: MacOS X",
|
|
40
41
|
"Operating System :: POSIX :: Linux",
|
|
41
42
|
"Operating System :: Microsoft :: Windows"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rms-filecache
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: File cache for files retrieved from the cloud
|
|
5
5
|
Maintainer-email: "Robert S. French" <rfrench@seti.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -15,14 +15,15 @@ Classifier: Natural Language :: English
|
|
|
15
15
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
16
16
|
Classifier: Topic :: Utilities
|
|
17
17
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
23
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
23
24
|
Classifier: Operating System :: POSIX :: Linux
|
|
24
25
|
Classifier: Operating System :: Microsoft :: Windows
|
|
25
|
-
Requires-Python: >=3.
|
|
26
|
+
Requires-Python: >=3.11
|
|
26
27
|
Description-Content-Type: text/markdown
|
|
27
28
|
License-File: LICENSE
|
|
28
29
|
Requires-Dist: boto3
|
|
@@ -30,6 +30,8 @@ rms_filecache.egg-info/PKG-INFO
|
|
|
30
30
|
rms_filecache.egg-info/SOURCES.txt
|
|
31
31
|
rms_filecache.egg-info/dependency_links.txt
|
|
32
32
|
rms_filecache.egg-info/requires.txt
|
|
33
|
+
rms_filecache.egg-info/scm_file_list.json
|
|
34
|
+
rms_filecache.egg-info/scm_version.json
|
|
33
35
|
rms_filecache.egg-info/top_level.txt
|
|
34
36
|
test_files/expected/lorem1.txt
|
|
35
37
|
test_files/expected/subdir1/lorem1.txt
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"files": [
|
|
3
|
+
".coveragerc",
|
|
4
|
+
".flake8",
|
|
5
|
+
".github/pull_request_template.md",
|
|
6
|
+
".github/workflows/publish_to_pypi.yml",
|
|
7
|
+
".github/workflows/publish_to_test_pypi.yml",
|
|
8
|
+
".github/workflows/run-tests.yml",
|
|
9
|
+
".gitignore",
|
|
10
|
+
".readthedocs.yaml",
|
|
11
|
+
"LICENSE",
|
|
12
|
+
"README.md",
|
|
13
|
+
"docs/Makefile",
|
|
14
|
+
"docs/conf.py",
|
|
15
|
+
"docs/examples.rst",
|
|
16
|
+
"docs/index.rst",
|
|
17
|
+
"docs/introduction.rst",
|
|
18
|
+
"docs/make.bat",
|
|
19
|
+
"docs/module.rst",
|
|
20
|
+
"filecache/__init__.py",
|
|
21
|
+
"filecache/file_cache.py",
|
|
22
|
+
"filecache/file_cache_path.py",
|
|
23
|
+
"filecache/file_cache_source.py",
|
|
24
|
+
"filecache/file_cache_types.py",
|
|
25
|
+
"filecache/py.typed",
|
|
26
|
+
"mypy.ini",
|
|
27
|
+
"pyproject.toml",
|
|
28
|
+
"requirements.txt",
|
|
29
|
+
"setup.cfg",
|
|
30
|
+
"test_files/expected/lorem1.txt",
|
|
31
|
+
"test_files/expected/subdir1/lorem1.txt",
|
|
32
|
+
"test_files/expected/subdir1/subdir2a/binary1.bin",
|
|
33
|
+
"test_files/expected/subdir1/subdir2b/binary1.bin",
|
|
34
|
+
"test_programs/locking.py",
|
|
35
|
+
"test_programs/multi_download.py",
|
|
36
|
+
"tests/__init__.py",
|
|
37
|
+
"tests/gs_lifecycle_config.json",
|
|
38
|
+
"tests/gs_lifecycle_config_info.txt",
|
|
39
|
+
"tests/s3_bucket_policy_read.txt",
|
|
40
|
+
"tests/s3_bucket_policy_write.txt",
|
|
41
|
+
"tests/s3_lifecycle_policy.txt",
|
|
42
|
+
"tests/test_file_cache.py",
|
|
43
|
+
"tests/test_file_cache_path.py",
|
|
44
|
+
"tests/test_file_cache_source.py",
|
|
45
|
+
"tests/test_url_mapping.py"
|
|
46
|
+
]
|
|
47
|
+
}
|
|
@@ -10,6 +10,7 @@ import os
|
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
import platform
|
|
12
12
|
import tempfile
|
|
13
|
+
import threading
|
|
13
14
|
import time
|
|
14
15
|
import uuid
|
|
15
16
|
|
|
@@ -1055,6 +1056,197 @@ def test_locking_multi_pfx_2():
|
|
|
1055
1056
|
assert not fc.cache_dir.exists()
|
|
1056
1057
|
|
|
1057
1058
|
|
|
1059
|
+
@pytest.mark.skipif(platform.system() == 'Windows',
|
|
1060
|
+
reason='Uses POSIX fcntl for stale-lock simulation')
|
|
1061
|
+
def test_stale_lock_single():
|
|
1062
|
+
"""A stale lock (file exists, no flock held) must not block a single-file retrieve.
|
|
1063
|
+
|
|
1064
|
+
We simulate a stale lock by acquiring the flock with raw fcntl, then releasing
|
|
1065
|
+
only the flock (leaving the lock file on disk) before the retrieve. On the
|
|
1066
|
+
next call, filelock should acquire the lock immediately because no process
|
|
1067
|
+
holds it.
|
|
1068
|
+
"""
|
|
1069
|
+
import fcntl
|
|
1070
|
+
|
|
1071
|
+
with FileCache(delete_on_exit=True) as fc:
|
|
1072
|
+
filename = EXPECTED_FILENAMES[0]
|
|
1073
|
+
local_path = (fc.cache_dir /
|
|
1074
|
+
(HTTP_TEST_ROOT.replace('https://', 'http_') + '/' + filename))
|
|
1075
|
+
lock_path = fc._lock_path(local_path)
|
|
1076
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1077
|
+
|
|
1078
|
+
# Create and hold the lock via raw fcntl so we can release the flock without
|
|
1079
|
+
# deleting the file (simulating a process that crashed).
|
|
1080
|
+
fd = os.open(str(lock_path), os.O_RDWR | os.O_CREAT, 0o644)
|
|
1081
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1082
|
+
|
|
1083
|
+
# Release the flock but keep the file -- this is the "stale lock" state.
|
|
1084
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
1085
|
+
os.close(fd)
|
|
1086
|
+
|
|
1087
|
+
# The lock file exists, but nobody holds the flock. The retrieve must
|
|
1088
|
+
# succeed rather than raising a TimeoutError.
|
|
1089
|
+
path = fc.retrieve(f'{HTTP_TEST_ROOT}/{filename}', lock_timeout=0)
|
|
1090
|
+
assert isinstance(path, Path)
|
|
1091
|
+
assert path.is_file()
|
|
1092
|
+
_compare_to_expected_path(path, filename)
|
|
1093
|
+
assert fc.download_counter == 1
|
|
1094
|
+
assert fc.upload_counter == 0
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
@pytest.mark.skipif(platform.system() == 'Windows',
|
|
1098
|
+
reason='Uses POSIX fcntl for stale-lock simulation')
|
|
1099
|
+
def test_stale_lock_multi():
|
|
1100
|
+
"""A stale lock must not permanently block multi-file retrieval.
|
|
1101
|
+
|
|
1102
|
+
We hold the flock for the first file from a background thread, trigger the
|
|
1103
|
+
multi-file download (which puts file[0] into wait_to_appear), then simulate
|
|
1104
|
+
a crash by releasing only the flock (but not deleting the lock file) while
|
|
1105
|
+
the wait loop is running. The stale-lock detection code must steal the lock
|
|
1106
|
+
and complete the download without timing out.
|
|
1107
|
+
|
|
1108
|
+
Race-condition guarantee: the background thread uses a threading.Event to
|
|
1109
|
+
synchronise so that the flock is released only after the main thread's
|
|
1110
|
+
retrieve call has entered the wait_to_appear loop. Because only one process
|
|
1111
|
+
can win the flock race, the main thread is guaranteed to be the sole
|
|
1112
|
+
downloader of file[0] after stealing the stale lock.
|
|
1113
|
+
"""
|
|
1114
|
+
import fcntl
|
|
1115
|
+
|
|
1116
|
+
with FileCache(delete_on_exit=True) as fc:
|
|
1117
|
+
filename0 = EXPECTED_FILENAMES[0]
|
|
1118
|
+
local_path0 = (fc.cache_dir /
|
|
1119
|
+
(HTTP_TEST_ROOT.replace('https://', 'http_') + '/' + filename0))
|
|
1120
|
+
lock_path0 = fc._lock_path(local_path0)
|
|
1121
|
+
lock_path0.parent.mkdir(parents=True, exist_ok=True)
|
|
1122
|
+
|
|
1123
|
+
# Phase-control events
|
|
1124
|
+
flock_held = threading.Event() # signalled once the raw flock is held
|
|
1125
|
+
flock_release = threading.Event() # signalled when the "crash" should happen
|
|
1126
|
+
|
|
1127
|
+
def holder_thread() -> None:
|
|
1128
|
+
"""Hold the flock until told to "crash", then release without cleanup."""
|
|
1129
|
+
fd = os.open(str(lock_path0), os.O_RDWR | os.O_CREAT, 0o644)
|
|
1130
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1131
|
+
flock_held.set()
|
|
1132
|
+
flock_release.wait()
|
|
1133
|
+
# Simulate crash: release flock, leave file on disk.
|
|
1134
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
1135
|
+
os.close(fd)
|
|
1136
|
+
|
|
1137
|
+
t = threading.Thread(target=holder_thread, daemon=True)
|
|
1138
|
+
t.start()
|
|
1139
|
+
assert flock_held.wait(timeout=5), 'holder thread never acquired flock'
|
|
1140
|
+
|
|
1141
|
+
results: list[list[Path]] = []
|
|
1142
|
+
exc_holder: list[Exception] = []
|
|
1143
|
+
|
|
1144
|
+
def do_retrieve() -> None:
|
|
1145
|
+
try:
|
|
1146
|
+
filenames = [f'{HTTP_TEST_ROOT}/{f}' for f in EXPECTED_FILENAMES]
|
|
1147
|
+
ret = fc.retrieve(filenames, lock_timeout=15)
|
|
1148
|
+
results.append(ret) # type: ignore[arg-type]
|
|
1149
|
+
except Exception as e:
|
|
1150
|
+
exc_holder.append(e)
|
|
1151
|
+
|
|
1152
|
+
retrieve_thread = threading.Thread(target=do_retrieve, daemon=True)
|
|
1153
|
+
retrieve_thread.start()
|
|
1154
|
+
|
|
1155
|
+
# Wait until all non-blocked files have downloaded and retrieve_thread is
|
|
1156
|
+
# still alive — at that point it must be blocked in the wait_to_appear loop.
|
|
1157
|
+
deadline = time.monotonic() + 15
|
|
1158
|
+
while time.monotonic() < deadline:
|
|
1159
|
+
if (fc.download_counter == len(EXPECTED_FILENAMES) - 1 and
|
|
1160
|
+
retrieve_thread.is_alive()):
|
|
1161
|
+
break
|
|
1162
|
+
time.sleep(0.01)
|
|
1163
|
+
else:
|
|
1164
|
+
pytest.fail('retrieve never reached the blocked/stale-lock state')
|
|
1165
|
+
|
|
1166
|
+
# Signal the holder to "crash" -- releases flock, keeps the lock file.
|
|
1167
|
+
flock_release.set()
|
|
1168
|
+
t.join(timeout=5)
|
|
1169
|
+
assert not t.is_alive(), 'holder thread did not finish'
|
|
1170
|
+
|
|
1171
|
+
retrieve_thread.join(timeout=20)
|
|
1172
|
+
assert not retrieve_thread.is_alive(), 'retrieve thread timed out (stale lock not recovered)'
|
|
1173
|
+
|
|
1174
|
+
assert not exc_holder, f'retrieve raised: {exc_holder[0]}'
|
|
1175
|
+
assert results, 'retrieve returned no result'
|
|
1176
|
+
for r in results[0]:
|
|
1177
|
+
assert isinstance(r, Path), f'Expected Path, got {r!r}'
|
|
1178
|
+
assert fc.download_counter == len(EXPECTED_FILENAMES)
|
|
1179
|
+
assert fc.upload_counter == 0
|
|
1180
|
+
|
|
1181
|
+
|
|
1182
|
+
@pytest.mark.skipif(platform.system() == 'Windows',
|
|
1183
|
+
reason='Uses POSIX fcntl for stale-lock simulation')
|
|
1184
|
+
def test_stale_lock_multi_pfx():
|
|
1185
|
+
"""Same as test_stale_lock_multi but exercised through the FCPath interface."""
|
|
1186
|
+
import fcntl
|
|
1187
|
+
|
|
1188
|
+
with FileCache(delete_on_exit=True) as fc:
|
|
1189
|
+
pfx = fc.new_path(HTTP_TEST_ROOT, lock_timeout=15)
|
|
1190
|
+
filename0 = EXPECTED_FILENAMES[0]
|
|
1191
|
+
local_path0 = (fc.cache_dir /
|
|
1192
|
+
(HTTP_TEST_ROOT.replace('https://', 'http_') + '/' + filename0))
|
|
1193
|
+
lock_path0 = fc._lock_path(local_path0)
|
|
1194
|
+
lock_path0.parent.mkdir(parents=True, exist_ok=True)
|
|
1195
|
+
|
|
1196
|
+
flock_held = threading.Event()
|
|
1197
|
+
flock_release = threading.Event()
|
|
1198
|
+
|
|
1199
|
+
def holder_thread() -> None:
|
|
1200
|
+
fd = os.open(str(lock_path0), os.O_RDWR | os.O_CREAT, 0o644)
|
|
1201
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1202
|
+
flock_held.set()
|
|
1203
|
+
flock_release.wait()
|
|
1204
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
1205
|
+
os.close(fd)
|
|
1206
|
+
|
|
1207
|
+
t = threading.Thread(target=holder_thread, daemon=True)
|
|
1208
|
+
t.start()
|
|
1209
|
+
assert flock_held.wait(timeout=5), 'holder thread never acquired flock'
|
|
1210
|
+
|
|
1211
|
+
results: list[list[Path]] = []
|
|
1212
|
+
exc_holder: list[Exception] = []
|
|
1213
|
+
|
|
1214
|
+
def do_retrieve() -> None:
|
|
1215
|
+
try:
|
|
1216
|
+
ret = pfx.retrieve(EXPECTED_FILENAMES)
|
|
1217
|
+
results.append(ret) # type: ignore[arg-type]
|
|
1218
|
+
except Exception as e:
|
|
1219
|
+
exc_holder.append(e)
|
|
1220
|
+
|
|
1221
|
+
retrieve_thread = threading.Thread(target=do_retrieve, daemon=True)
|
|
1222
|
+
retrieve_thread.start()
|
|
1223
|
+
|
|
1224
|
+
# Wait until all non-blocked files have downloaded and retrieve_thread is
|
|
1225
|
+
# still alive — at that point it must be blocked in the wait_to_appear loop.
|
|
1226
|
+
deadline = time.monotonic() + 15
|
|
1227
|
+
while time.monotonic() < deadline:
|
|
1228
|
+
if (fc.download_counter == len(EXPECTED_FILENAMES) - 1 and
|
|
1229
|
+
retrieve_thread.is_alive()):
|
|
1230
|
+
break
|
|
1231
|
+
time.sleep(0.01)
|
|
1232
|
+
else:
|
|
1233
|
+
pytest.fail('retrieve never reached the blocked/stale-lock state')
|
|
1234
|
+
|
|
1235
|
+
flock_release.set()
|
|
1236
|
+
t.join(timeout=5)
|
|
1237
|
+
assert not t.is_alive()
|
|
1238
|
+
|
|
1239
|
+
retrieve_thread.join(timeout=20)
|
|
1240
|
+
assert not retrieve_thread.is_alive(), 'retrieve thread timed out (stale lock not recovered)'
|
|
1241
|
+
|
|
1242
|
+
assert not exc_holder, f'retrieve raised: {exc_holder[0]}'
|
|
1243
|
+
assert results
|
|
1244
|
+
for r in results[0]:
|
|
1245
|
+
assert isinstance(r, Path)
|
|
1246
|
+
assert fc.download_counter == len(EXPECTED_FILENAMES)
|
|
1247
|
+
assert fc.upload_counter == 0
|
|
1248
|
+
|
|
1249
|
+
|
|
1058
1250
|
def test_cleanup_locking_bad():
|
|
1059
1251
|
logger = MyLogger()
|
|
1060
1252
|
with FileCache(cache_name=None, logger=logger) as fc:
|
|
@@ -2948,3 +3140,22 @@ def test_modification_time_caching_multi(time_sensitive, cache_metadata, mp_safe
|
|
|
2948
3140
|
assert lp[0].stat().st_mtime == mtime_lp_orig[0]
|
|
2949
3141
|
assert lp[1].stat().st_mtime == mtime_lp_orig[1]
|
|
2950
3142
|
assert lp[2].stat().st_mtime == mtime_lp_orig[2]
|
|
3143
|
+
|
|
3144
|
+
|
|
3145
|
+
def test_filecache_repr_str():
|
|
3146
|
+
with FileCache(cache_name=None) as fc:
|
|
3147
|
+
r = repr(fc)
|
|
3148
|
+
assert r.startswith('FileCache(None,')
|
|
3149
|
+
assert 'anonymous=False' in r
|
|
3150
|
+
assert 'lock_timeout=60' in r
|
|
3151
|
+
assert 'nthreads=8' in r
|
|
3152
|
+
assert str(fc) == str(fc.cache_dir)
|
|
3153
|
+
|
|
3154
|
+
# Named cache with non-default options
|
|
3155
|
+
with FileCache('repr-test', anonymous=True, lock_timeout=30, nthreads=4,
|
|
3156
|
+
delete_on_exit=True) as fc:
|
|
3157
|
+
r = repr(fc)
|
|
3158
|
+
assert r.startswith("FileCache('repr-test',")
|
|
3159
|
+
assert 'anonymous=True' in r
|
|
3160
|
+
assert 'lock_timeout=30' in r
|
|
3161
|
+
assert 'nthreads=4' in r
|
|
@@ -1308,3 +1308,37 @@ def test_splitpath():
|
|
|
1308
1308
|
FCPath('d'))
|
|
1309
1309
|
assert FCPath('gs://bucket/a/c/b/b1/c/d/d1').splitpath('c') == \
|
|
1310
1310
|
(FCPath('gs://bucket/a'), FCPath('b/b1'), FCPath('d/d1'))
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
def test_fcpath_repr_str():
|
|
1314
|
+
# Default FCPath — only the path
|
|
1315
|
+
p = FCPath('gs://bucket/path/to/file')
|
|
1316
|
+
assert repr(p) == "FCPath('gs://bucket/path/to/file')"
|
|
1317
|
+
assert str(p) == 'gs://bucket/path/to/file'
|
|
1318
|
+
|
|
1319
|
+
# FCPath with extra options — they appear in repr
|
|
1320
|
+
p2 = FCPath('s3://bucket/path', anonymous=True, lock_timeout=10, nthreads=2)
|
|
1321
|
+
r = repr(p2)
|
|
1322
|
+
assert r.startswith('FCPath(')
|
|
1323
|
+
assert 'anonymous=True' in r
|
|
1324
|
+
assert 'lock_timeout=10' in r
|
|
1325
|
+
assert 'nthreads=2' in r
|
|
1326
|
+
|
|
1327
|
+
# filecache kwarg appears in repr when set
|
|
1328
|
+
with FileCache(cache_name=None, delete_on_exit=True) as fc:
|
|
1329
|
+
p3 = FCPath('gs://bucket/x', filecache=fc)
|
|
1330
|
+
r3 = repr(p3)
|
|
1331
|
+
assert 'filecache=' in r3
|
|
1332
|
+
|
|
1333
|
+
# url_to_url and url_to_path appear in repr when set
|
|
1334
|
+
def _identity_url(scheme, remote, path):
|
|
1335
|
+
return None
|
|
1336
|
+
|
|
1337
|
+
def _identity_path(scheme, remote, path, cache_dir, cache_subdir):
|
|
1338
|
+
return None
|
|
1339
|
+
|
|
1340
|
+
p4 = FCPath('gs://bucket/y',
|
|
1341
|
+
url_to_url=_identity_url, url_to_path=_identity_path)
|
|
1342
|
+
r4 = repr(p4)
|
|
1343
|
+
assert 'url_to_url=' in r4
|
|
1344
|
+
assert 'url_to_path=' in r4
|
|
@@ -404,3 +404,32 @@ def test_fake_source_atomic_operations(tmp_path: Path):
|
|
|
404
404
|
pathlib.Path.rename = original_rename
|
|
405
405
|
finally:
|
|
406
406
|
FileCacheSourceFake.delete_default_storage_dir()
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def test_filecachesource_repr_str():
|
|
410
|
+
src_gs = FileCacheSourceGS('gs', 'rms-filecache-tests', anonymous=True)
|
|
411
|
+
r = repr(src_gs)
|
|
412
|
+
assert r == "FileCacheSourceGS('gs', 'rms-filecache-tests', anonymous=True)"
|
|
413
|
+
assert str(src_gs) == 'gs://rms-filecache-tests'
|
|
414
|
+
|
|
415
|
+
src_s3 = FileCacheSourceS3('s3', 'rms-filecache-tests', anonymous=True)
|
|
416
|
+
r = repr(src_s3)
|
|
417
|
+
assert r == "FileCacheSourceS3('s3', 'rms-filecache-tests', anonymous=True)"
|
|
418
|
+
assert str(src_s3) == 's3://rms-filecache-tests'
|
|
419
|
+
|
|
420
|
+
src_http = FileCacheSourceHTTP('https', 'storage.googleapis.com',
|
|
421
|
+
anonymous=False)
|
|
422
|
+
r = repr(src_http)
|
|
423
|
+
assert r == ("FileCacheSourceHTTP('https', 'storage.googleapis.com', "
|
|
424
|
+
"anonymous=False)")
|
|
425
|
+
assert str(src_http) == 'https://storage.googleapis.com'
|
|
426
|
+
|
|
427
|
+
src_file = FileCacheSourceFile('file', '', anonymous=False)
|
|
428
|
+
r = repr(src_file)
|
|
429
|
+
assert r == "FileCacheSourceFile('file', '', anonymous=False)"
|
|
430
|
+
assert str(src_file) == 'file://'
|
|
431
|
+
|
|
432
|
+
src_fake = FileCacheSourceFake('fake', 'fake-bucket', anonymous=False)
|
|
433
|
+
r = repr(src_fake)
|
|
434
|
+
assert r == "FileCacheSourceFake('fake', 'fake-bucket', anonymous=False)"
|
|
435
|
+
assert str(src_fake) == 'fake://fake-bucket'
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
# file generated by setuptools-scm
|
|
2
|
-
# don't change, don't track in version control
|
|
3
|
-
|
|
4
|
-
__all__ = [
|
|
5
|
-
"__version__",
|
|
6
|
-
"__version_tuple__",
|
|
7
|
-
"version",
|
|
8
|
-
"version_tuple",
|
|
9
|
-
"__commit_id__",
|
|
10
|
-
"commit_id",
|
|
11
|
-
]
|
|
12
|
-
|
|
13
|
-
TYPE_CHECKING = False
|
|
14
|
-
if TYPE_CHECKING:
|
|
15
|
-
from typing import Tuple
|
|
16
|
-
from typing import Union
|
|
17
|
-
|
|
18
|
-
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
-
COMMIT_ID = Union[str, None]
|
|
20
|
-
else:
|
|
21
|
-
VERSION_TUPLE = object
|
|
22
|
-
COMMIT_ID = object
|
|
23
|
-
|
|
24
|
-
version: str
|
|
25
|
-
__version__: str
|
|
26
|
-
__version_tuple__: VERSION_TUPLE
|
|
27
|
-
version_tuple: VERSION_TUPLE
|
|
28
|
-
commit_id: COMMIT_ID
|
|
29
|
-
__commit_id__: COMMIT_ID
|
|
30
|
-
|
|
31
|
-
__version__ = version = '3.1.0'
|
|
32
|
-
__version_tuple__ = version_tuple = (3, 1, 0)
|
|
33
|
-
|
|
34
|
-
__commit_id__ = commit_id = 'g3ecac5a3d'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/subdir1/subdir2a/binary1.bin
RENAMED
|
File without changes
|
{rms_filecache-3.1.0 → rms_filecache-3.2.0}/test_files/expected/subdir1/subdir2b/binary1.bin
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|