rms-filecache 2.3.0__tar.gz → 2.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.gitignore +2 -0
- {rms_filecache-2.3.0/rms_filecache.egg-info → rms_filecache-2.3.1}/PKG-INFO +23 -16
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/README.md +20 -14
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/__init__.py +29 -23
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/_version.py +9 -4
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/file_cache.py +2 -1
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/file_cache_source.py +215 -3
- {rms_filecache-2.3.0 → rms_filecache-2.3.1/rms_filecache.egg-info}/PKG-INFO +23 -16
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/test_file_cache_path.py +49 -1
- rms_filecache-2.3.1/tests/test_file_cache_source.py +347 -0
- rms_filecache-2.3.0/tests/test_file_cache_source.py +0 -81
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.coveragerc +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.flake8 +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.github/pull_request_template.md +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.github/workflows/publish_to_pypi.yml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.github/workflows/publish_to_test_pypi.yml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.github/workflows/run-tests.yml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.readthedocs.yaml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/LICENSE +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/Makefile +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/conf.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/index.rst +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/make.bat +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/module.rst +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/file_cache_path.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/file_cache_types.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/py.typed +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/mypy.ini +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/pyproject.toml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/requirements.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/rms_filecache.egg-info/SOURCES.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/setup.cfg +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_programs/locking.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_programs/multi_download.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/__init__.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/s3_bucket_policy_write.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/s3_lifecycle_policy.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/test_file_cache.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/test_url_to_path.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: rms-filecache
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.1
|
|
4
4
|
Summary: File cache for files retrieved from the cloud
|
|
5
5
|
Maintainer-email: "Robert S. French" <rfrench@seti.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -31,6 +31,7 @@ Requires-Dist: filelock
|
|
|
31
31
|
Requires-Dist: google-cloud-storage
|
|
32
32
|
Requires-Dist: requests
|
|
33
33
|
Requires-Dist: typing_extensions
|
|
34
|
+
Dynamic: license-file
|
|
34
35
|
|
|
35
36
|
[](https://github.com/SETI/rms-filecache/releases)
|
|
36
37
|
[](https://github.com/SETI/rms-filecache/releases)
|
|
@@ -59,14 +60,16 @@ Requires-Dist: typing_extensions
|
|
|
59
60
|
# Introduction
|
|
60
61
|
|
|
61
62
|
`filecache` is a Python module that abstracts away the location where files used or
|
|
62
|
-
generated by a program are stored. Files can be on the local file system
|
|
63
|
-
Storage, on Amazon Web Services S3, or on a
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
local file system, they are simply
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
63
|
+
generated by a program are stored. Files can be on the local file system (`/` or
|
|
64
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
|
|
65
|
+
webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
|
|
66
|
+
a remote using the local filesystem as a file repository (`fake://`). When files to be read
|
|
67
|
+
are on the local file system, they are simply accessed in-place. Otherwise, they are
|
|
68
|
+
downloaded from the remote source to a local temporary directory. When files to be written
|
|
69
|
+
are on the local file system, they are simply written in-place. Otherwise, they are
|
|
70
|
+
written to a local temporary directory and then uploaded to the remote location (it is not
|
|
71
|
+
possible to upload to a webserver). When a cache is no longer needed, it is deleted from
|
|
72
|
+
the local disk.
|
|
70
73
|
|
|
71
74
|
`filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
|
|
72
75
|
|
|
@@ -218,14 +221,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
|
218
221
|
contents = voldesc_path.read_text()
|
|
219
222
|
```
|
|
220
223
|
|
|
221
|
-
Finally, there are
|
|
222
|
-
locations without invoking any caching behavior: `
|
|
223
|
-
`FileCacheSourceHTTP`, `FileCacheSourceGS`, and `
|
|
224
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
225
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
226
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
|
|
227
|
+
The first four provide access to real storage locations, while `FileCacheSourceFake`
|
|
228
|
+
simulates a remote source for testing purposes:
|
|
224
229
|
|
|
225
230
|
```python
|
|
226
|
-
from filecache import FileCacheSourceGS
|
|
227
|
-
|
|
228
|
-
|
|
231
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake
|
|
232
|
+
src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
233
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
|
|
234
|
+
src2 = FileCacheSourceFake('fake://fake-bucket')
|
|
235
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
229
236
|
```
|
|
230
237
|
|
|
231
238
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -25,14 +25,16 @@
|
|
|
25
25
|
# Introduction
|
|
26
26
|
|
|
27
27
|
`filecache` is a Python module that abstracts away the location where files used or
|
|
28
|
-
generated by a program are stored. Files can be on the local file system
|
|
29
|
-
Storage, on Amazon Web Services S3, or on a
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
local file system, they are simply
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
generated by a program are stored. Files can be on the local file system (`/` or
|
|
29
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
|
|
30
|
+
webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
|
|
31
|
+
a remote using the local filesystem as a file repository (`fake://`). When files to be read
|
|
32
|
+
are on the local file system, they are simply accessed in-place. Otherwise, they are
|
|
33
|
+
downloaded from the remote source to a local temporary directory. When files to be written
|
|
34
|
+
are on the local file system, they are simply written in-place. Otherwise, they are
|
|
35
|
+
written to a local temporary directory and then uploaded to the remote location (it is not
|
|
36
|
+
possible to upload to a webserver). When a cache is no longer needed, it is deleted from
|
|
37
|
+
the local disk.
|
|
36
38
|
|
|
37
39
|
`filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
|
|
38
40
|
|
|
@@ -184,14 +186,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
|
184
186
|
contents = voldesc_path.read_text()
|
|
185
187
|
```
|
|
186
188
|
|
|
187
|
-
Finally, there are
|
|
188
|
-
locations without invoking any caching behavior: `
|
|
189
|
-
`FileCacheSourceHTTP`, `FileCacheSourceGS`, and `
|
|
189
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
190
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
191
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
|
|
192
|
+
The first four provide access to real storage locations, while `FileCacheSourceFake`
|
|
193
|
+
simulates a remote source for testing purposes:
|
|
190
194
|
|
|
191
195
|
```python
|
|
192
|
-
from filecache import FileCacheSourceGS
|
|
193
|
-
|
|
194
|
-
|
|
196
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake
|
|
197
|
+
src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
198
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
|
|
199
|
+
src2 = FileCacheSourceFake('fake://fake-bucket')
|
|
200
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
195
201
|
```
|
|
196
202
|
|
|
197
203
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -3,14 +3,16 @@
|
|
|
3
3
|
##########################################################################################
|
|
4
4
|
|
|
5
5
|
"""The purpose of the `filecache` module is to abstract away the location where files
|
|
6
|
-
used or generated by a program are stored. Files can be on the local file system
|
|
7
|
-
Google Cloud Storage, on Amazon Web Services S3, or on
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
are on the local file system, they are simply
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
6
|
+
used or generated by a program are stored. Files can be on the local file system (`/` or
|
|
7
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on
|
|
8
|
+
a webserver (`http://` or `https://`). There is also a fake remote that can be used to
|
|
9
|
+
simulate a remote using the local filesystem as a file repository (`fake://`). When files
|
|
10
|
+
to be read are on the local file system, they are simply accessed in-place. Otherwise,
|
|
11
|
+
they are downloaded from the remote source to a local temporary directory. When files to
|
|
12
|
+
be written are on the local file system, they are simply written in-place. Otherwise, they
|
|
13
|
+
are written to a local temporary directory and then uploaded to the remote location (it is
|
|
14
|
+
not possible to upload to a webserver). When a cache is no longer needed, it is deleted
|
|
15
|
+
from the local disk.
|
|
14
16
|
|
|
15
17
|
The top-level file organization is provided by the :class:`FileCache` class. A
|
|
16
18
|
:class:`FileCache` instance is used to specify a particular **sharing policy** and
|
|
@@ -135,19 +137,21 @@ was run. A shared cache is indicated by giving the cache a name (or no argument,
|
|
|
135
137
|
defaults to ``"global"``); also :class:`FCPath` defaults to using the global cache if no
|
|
136
138
|
:class:`FileCache` is specified. This results in the simplest form of the program:
|
|
137
139
|
|
|
138
|
-
from filecache import FCPath
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
from filecache import FileCacheSourceGS
|
|
149
|
-
|
|
150
|
-
|
|
140
|
+
from filecache import FCPath import os p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
|
|
141
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat' contents =
|
|
142
|
+
voldesc_path.read_text()
|
|
143
|
+
|
|
144
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
145
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
146
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and
|
|
147
|
+
`FileCacheSourceFake`. The first four provide access to real storage locations, while
|
|
148
|
+
`FileCacheSourceFake` simulates a remote source for testing purposes:
|
|
149
|
+
|
|
150
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake src1 =
|
|
151
|
+
FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
152
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin') src2 =
|
|
153
|
+
FileCacheSourceFake('fake://fake-bucket')
|
|
154
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
151
155
|
"""
|
|
152
156
|
|
|
153
157
|
import typing
|
|
@@ -168,7 +172,8 @@ from .file_cache_source import (FileCacheSource, # noqa: ignore E401
|
|
|
168
172
|
FileCacheSourceFile,
|
|
169
173
|
FileCacheSourceHTTP,
|
|
170
174
|
FileCacheSourceGS,
|
|
171
|
-
FileCacheSourceS3
|
|
175
|
+
FileCacheSourceS3,
|
|
176
|
+
FileCacheSourceFake)
|
|
172
177
|
|
|
173
178
|
__all__ = ['get_global_logger',
|
|
174
179
|
'register_filecachesource',
|
|
@@ -180,4 +185,5 @@ __all__ = ['get_global_logger',
|
|
|
180
185
|
'FileCacheSourceFile',
|
|
181
186
|
'FileCacheSourceHTTP',
|
|
182
187
|
'FileCacheSourceGS',
|
|
183
|
-
'FileCacheSourceS3'
|
|
188
|
+
'FileCacheSourceS3',
|
|
189
|
+
'FileCacheSourceFake']
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
# file generated by
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
2
|
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
|
|
5
|
+
|
|
3
6
|
TYPE_CHECKING = False
|
|
4
7
|
if TYPE_CHECKING:
|
|
5
|
-
from typing import Tuple
|
|
8
|
+
from typing import Tuple
|
|
9
|
+
from typing import Union
|
|
10
|
+
|
|
6
11
|
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
7
12
|
else:
|
|
8
13
|
VERSION_TUPLE = object
|
|
@@ -12,5 +17,5 @@ __version__: str
|
|
|
12
17
|
__version_tuple__: VERSION_TUPLE
|
|
13
18
|
version_tuple: VERSION_TUPLE
|
|
14
19
|
|
|
15
|
-
__version__ = version = '2.3.
|
|
16
|
-
__version_tuple__ = version_tuple = (2, 3,
|
|
20
|
+
__version__ = version = '2.3.1'
|
|
21
|
+
__version_tuple__ = version_tuple = (2, 3, 1)
|
|
@@ -33,7 +33,7 @@ from .file_cache_source import (FileCacheSource,
|
|
|
33
33
|
FileCacheSourceHTTP,
|
|
34
34
|
FileCacheSourceGS,
|
|
35
35
|
FileCacheSourceS3,
|
|
36
|
-
)
|
|
36
|
+
FileCacheSourceFake)
|
|
37
37
|
from .file_cache_path import FCPath
|
|
38
38
|
from .file_cache_types import (StrOrPathOrSeqType,
|
|
39
39
|
UrlToPathFuncOrSeqType)
|
|
@@ -59,6 +59,7 @@ register_filecachesource(FileCacheSourceFile)
|
|
|
59
59
|
register_filecachesource(FileCacheSourceHTTP)
|
|
60
60
|
register_filecachesource(FileCacheSourceGS)
|
|
61
61
|
register_filecachesource(FileCacheSourceS3)
|
|
62
|
+
register_filecachesource(FileCacheSourceFake)
|
|
62
63
|
|
|
63
64
|
|
|
64
65
|
# Default logger for all FileCache instances that didn't specify one explicitly
|
|
@@ -10,6 +10,8 @@ from concurrent import futures
|
|
|
10
10
|
from concurrent.futures import ThreadPoolExecutor
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
import requests
|
|
13
|
+
import shutil
|
|
14
|
+
import tempfile
|
|
13
15
|
from typing import Iterator
|
|
14
16
|
import uuid
|
|
15
17
|
|
|
@@ -606,7 +608,7 @@ class FileCacheSourceHTTP(FileCacheSource):
|
|
|
606
608
|
except requests.exceptions.RequestException as e:
|
|
607
609
|
raise FileNotFoundError(f'Failed to download file: {url}') from e
|
|
608
610
|
|
|
609
|
-
temp_local_path = local_path.with_suffix(local_path.suffix
|
|
611
|
+
temp_local_path = local_path.with_suffix(f'.{local_path.suffix}_{uuid.uuid4()}')
|
|
610
612
|
try:
|
|
611
613
|
with open(temp_local_path, 'wb') as f:
|
|
612
614
|
for chunk in response.iter_content(chunk_size=1024*1024):
|
|
@@ -758,7 +760,7 @@ class FileCacheSourceGS(FileCacheSource):
|
|
|
758
760
|
|
|
759
761
|
blob = self._bucket.blob(sub_path)
|
|
760
762
|
|
|
761
|
-
temp_local_path = local_path.with_suffix(f'{local_path.suffix}
|
|
763
|
+
temp_local_path = local_path.with_suffix(f'.{local_path.suffix}_{uuid.uuid4()}')
|
|
762
764
|
try:
|
|
763
765
|
blob.download_to_filename(str(temp_local_path))
|
|
764
766
|
temp_local_path.rename(local_path)
|
|
@@ -963,7 +965,7 @@ class FileCacheSourceS3(FileCacheSource):
|
|
|
963
965
|
|
|
964
966
|
local_path.parent.mkdir(parents=True, exist_ok=True)
|
|
965
967
|
|
|
966
|
-
temp_local_path = local_path.with_suffix(f'{local_path.suffix}
|
|
968
|
+
temp_local_path = local_path.with_suffix(f'.{local_path.suffix}_{uuid.uuid4()}')
|
|
967
969
|
try:
|
|
968
970
|
self._client.download_file(self._bucket_name, sub_path,
|
|
969
971
|
str(temp_local_path))
|
|
@@ -1068,3 +1070,213 @@ class FileCacheSourceS3(FileCacheSource):
|
|
|
1068
1070
|
self._client.delete_object(Bucket=self._bucket_name, Key=sub_path)
|
|
1069
1071
|
|
|
1070
1072
|
return sub_path
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
class FileCacheSourceFake(FileCacheSource):
|
|
1076
|
+
"""Class that simulates a remote file source using a local directory structure.
|
|
1077
|
+
|
|
1078
|
+
This class is useful for testing file operations without requiring actual remote
|
|
1079
|
+
connections. Files are stored in a local directory that simulates the remote storage,
|
|
1080
|
+
including the need for uploads and downloads. By default, the storage directory is
|
|
1081
|
+
``<TEMPDIR>/.filecache_fake_remote`` and persists across program runs.
|
|
1082
|
+
"""
|
|
1083
|
+
|
|
1084
|
+
_DEFAULT_STORAGE_DIR = Path(tempfile.gettempdir()) / '.filecache_fake_remote'
|
|
1085
|
+
|
|
1086
|
+
@classmethod
|
|
1087
|
+
def get_default_storage_dir(cls) -> Path:
|
|
1088
|
+
"""Get the current default storage directory for fake remote files.
|
|
1089
|
+
|
|
1090
|
+
Returns:
|
|
1091
|
+
The current default storage directory Path.
|
|
1092
|
+
"""
|
|
1093
|
+
|
|
1094
|
+
return cls._DEFAULT_STORAGE_DIR
|
|
1095
|
+
|
|
1096
|
+
@classmethod
|
|
1097
|
+
def set_default_storage_dir(cls, directory: str | Path) -> None:
|
|
1098
|
+
"""Set the default storage directory for fake remote files.
|
|
1099
|
+
|
|
1100
|
+
Parameters:
|
|
1101
|
+
directory: The directory to use as the default storage location. The directory
|
|
1102
|
+
is expanded and resolved to an absolute path.
|
|
1103
|
+
"""
|
|
1104
|
+
|
|
1105
|
+
cls._DEFAULT_STORAGE_DIR = Path(directory).expanduser().resolve()
|
|
1106
|
+
|
|
1107
|
+
@classmethod
|
|
1108
|
+
def delete_default_storage_dir(cls) -> None:
|
|
1109
|
+
"""Delete the current default storage directory and all its contents.
|
|
1110
|
+
|
|
1111
|
+
This is useful for cleanup after testing.
|
|
1112
|
+
"""
|
|
1113
|
+
|
|
1114
|
+
if cls._DEFAULT_STORAGE_DIR.exists():
|
|
1115
|
+
shutil.rmtree(cls._DEFAULT_STORAGE_DIR)
|
|
1116
|
+
|
|
1117
|
+
def __init__(self,
|
|
1118
|
+
scheme: str,
|
|
1119
|
+
remote: str,
|
|
1120
|
+
*,
|
|
1121
|
+
anonymous: bool = False,
|
|
1122
|
+
storage_dir: str | Path | None = None):
|
|
1123
|
+
"""Initialize the FileCacheSourceFake class.
|
|
1124
|
+
|
|
1125
|
+
Parameters:
|
|
1126
|
+
scheme: The scheme of the source. Must be "fake".
|
|
1127
|
+
remote: The simulated remote/bucket name.
|
|
1128
|
+
anonymous: Not used for this class.
|
|
1129
|
+
storage_dir: Base directory in which to store the fake remote files. If None,
|
|
1130
|
+
uses the class default storage directory.
|
|
1131
|
+
"""
|
|
1132
|
+
|
|
1133
|
+
if scheme != 'fake':
|
|
1134
|
+
raise ValueError('Scheme must be "fake"')
|
|
1135
|
+
|
|
1136
|
+
if remote == '':
|
|
1137
|
+
raise ValueError('remote parameter must have a value')
|
|
1138
|
+
|
|
1139
|
+
super().__init__(scheme, remote, anonymous=anonymous)
|
|
1140
|
+
|
|
1141
|
+
self._storage_base = (Path(storage_dir).expanduser().resolve()
|
|
1142
|
+
if storage_dir is not None
|
|
1143
|
+
else self._DEFAULT_STORAGE_DIR)
|
|
1144
|
+
self._storage_dir = self._storage_base / remote
|
|
1145
|
+
self._storage_dir.mkdir(parents=True, exist_ok=True)
|
|
1146
|
+
self._cache_subdir = self._src_prefix.replace('fake://', 'fake_')
|
|
1147
|
+
|
|
1148
|
+
@classmethod
|
|
1149
|
+
def schemes(cls) -> tuple[str, ...]:
|
|
1150
|
+
"""The URL schemes supported by this class."""
|
|
1151
|
+
|
|
1152
|
+
return ('fake',)
|
|
1153
|
+
|
|
1154
|
+
@classmethod
|
|
1155
|
+
def uses_anonymous(cls) -> bool:
|
|
1156
|
+
"""Whether this class has the concept of anonymous accesses."""
|
|
1157
|
+
|
|
1158
|
+
return False
|
|
1159
|
+
|
|
1160
|
+
def exists(self, sub_path: str) -> bool:
|
|
1161
|
+
"""Check if a file exists in the fake remote storage.
|
|
1162
|
+
|
|
1163
|
+
Parameters:
|
|
1164
|
+
sub_path: The path of the file relative to the storage directory.
|
|
1165
|
+
|
|
1166
|
+
Returns:
|
|
1167
|
+
True if the file exists, False otherwise.
|
|
1168
|
+
"""
|
|
1169
|
+
|
|
1170
|
+
return (self._storage_dir / sub_path).is_file()
|
|
1171
|
+
|
|
1172
|
+
def retrieve(self,
|
|
1173
|
+
sub_path: str,
|
|
1174
|
+
local_path: str | Path) -> Path:
|
|
1175
|
+
"""Retrieve a file from the fake remote storage.
|
|
1176
|
+
|
|
1177
|
+
Parameters:
|
|
1178
|
+
sub_path: The path of the file relative to the storage directory.
|
|
1179
|
+
local_path: The path where the file should be copied to.
|
|
1180
|
+
|
|
1181
|
+
Returns:
|
|
1182
|
+
The Path where the file was stored (same as local_path).
|
|
1183
|
+
|
|
1184
|
+
Raises:
|
|
1185
|
+
FileNotFoundError: If the remote file does not exist.
|
|
1186
|
+
"""
|
|
1187
|
+
|
|
1188
|
+
source_path = self._storage_dir / sub_path
|
|
1189
|
+
local_path = Path(local_path)
|
|
1190
|
+
|
|
1191
|
+
if not source_path.is_file():
|
|
1192
|
+
raise FileNotFoundError(f'File does not exist: {self._src_prefix_}{sub_path}')
|
|
1193
|
+
|
|
1194
|
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1195
|
+
|
|
1196
|
+
temp_local_path = local_path.with_suffix(f'.{local_path.suffix}_{uuid.uuid4()}')
|
|
1197
|
+
try:
|
|
1198
|
+
shutil.copy2(source_path, temp_local_path)
|
|
1199
|
+
temp_local_path.rename(local_path)
|
|
1200
|
+
except Exception:
|
|
1201
|
+
temp_local_path.unlink(missing_ok=True)
|
|
1202
|
+
raise
|
|
1203
|
+
|
|
1204
|
+
return local_path
|
|
1205
|
+
|
|
1206
|
+
def upload(self,
|
|
1207
|
+
sub_path: str,
|
|
1208
|
+
local_path: str | Path) -> Path:
|
|
1209
|
+
"""Upload a file to the fake remote storage.
|
|
1210
|
+
|
|
1211
|
+
Parameters:
|
|
1212
|
+
sub_path: The destination path relative to the storage directory.
|
|
1213
|
+
local_path: The path of the local file to upload.
|
|
1214
|
+
|
|
1215
|
+
Returns:
|
|
1216
|
+
The Path of the local file that was uploaded.
|
|
1217
|
+
|
|
1218
|
+
Raises:
|
|
1219
|
+
FileNotFoundError: If the local file does not exist.
|
|
1220
|
+
"""
|
|
1221
|
+
|
|
1222
|
+
local_path = Path(local_path)
|
|
1223
|
+
if not local_path.is_file():
|
|
1224
|
+
raise FileNotFoundError(f'File does not exist: {local_path}')
|
|
1225
|
+
|
|
1226
|
+
dest_path = self._storage_dir / sub_path
|
|
1227
|
+
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1228
|
+
|
|
1229
|
+
temp_dest_path = dest_path.with_suffix(f'.{dest_path.suffix}_{uuid.uuid4()}')
|
|
1230
|
+
try:
|
|
1231
|
+
shutil.copy2(local_path, temp_dest_path)
|
|
1232
|
+
temp_dest_path.rename(dest_path)
|
|
1233
|
+
except Exception:
|
|
1234
|
+
temp_dest_path.unlink(missing_ok=True)
|
|
1235
|
+
raise
|
|
1236
|
+
|
|
1237
|
+
return local_path
|
|
1238
|
+
|
|
1239
|
+
def iterdir_type(self, sub_path: str) -> Iterator[tuple[str, bool]]:
|
|
1240
|
+
"""Iterate over the contents of a directory in the fake remote storage.
|
|
1241
|
+
|
|
1242
|
+
Parameters:
|
|
1243
|
+
sub_path: The path of the directory relative to the storage directory.
|
|
1244
|
+
|
|
1245
|
+
Yields:
|
|
1246
|
+
Tuples of (path, is_dir) for each item in the directory.
|
|
1247
|
+
"""
|
|
1248
|
+
|
|
1249
|
+
dir_path = self._storage_dir / sub_path if sub_path else self._storage_dir
|
|
1250
|
+
|
|
1251
|
+
if not dir_path.is_dir():
|
|
1252
|
+
return
|
|
1253
|
+
|
|
1254
|
+
for item in dir_path.iterdir():
|
|
1255
|
+
relative_path = str(item.relative_to(self._storage_dir)).replace('\\', '/')
|
|
1256
|
+
yield f'{self._src_prefix_}{relative_path}', item.is_dir()
|
|
1257
|
+
|
|
1258
|
+
def unlink(self,
|
|
1259
|
+
sub_path: str,
|
|
1260
|
+
*,
|
|
1261
|
+
missing_ok: bool = False) -> str:
|
|
1262
|
+
"""Remove a file from the fake remote storage.
|
|
1263
|
+
|
|
1264
|
+
Parameters:
|
|
1265
|
+
sub_path: The path of the file relative to the storage directory.
|
|
1266
|
+
missing_ok: If True, don't raise an error if the file doesn't exist.
|
|
1267
|
+
|
|
1268
|
+
Returns:
|
|
1269
|
+
The sub_path that was removed.
|
|
1270
|
+
|
|
1271
|
+
Raises:
|
|
1272
|
+
FileNotFoundError: If the file doesn't exist and missing_ok is False.
|
|
1273
|
+
"""
|
|
1274
|
+
|
|
1275
|
+
file_path = self._storage_dir / sub_path
|
|
1276
|
+
try:
|
|
1277
|
+
file_path.unlink()
|
|
1278
|
+
except FileNotFoundError:
|
|
1279
|
+
if not missing_ok:
|
|
1280
|
+
raise
|
|
1281
|
+
|
|
1282
|
+
return sub_path
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: rms-filecache
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.1
|
|
4
4
|
Summary: File cache for files retrieved from the cloud
|
|
5
5
|
Maintainer-email: "Robert S. French" <rfrench@seti.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -31,6 +31,7 @@ Requires-Dist: filelock
|
|
|
31
31
|
Requires-Dist: google-cloud-storage
|
|
32
32
|
Requires-Dist: requests
|
|
33
33
|
Requires-Dist: typing_extensions
|
|
34
|
+
Dynamic: license-file
|
|
34
35
|
|
|
35
36
|
[](https://github.com/SETI/rms-filecache/releases)
|
|
36
37
|
[](https://github.com/SETI/rms-filecache/releases)
|
|
@@ -59,14 +60,16 @@ Requires-Dist: typing_extensions
|
|
|
59
60
|
# Introduction
|
|
60
61
|
|
|
61
62
|
`filecache` is a Python module that abstracts away the location where files used or
|
|
62
|
-
generated by a program are stored. Files can be on the local file system
|
|
63
|
-
Storage, on Amazon Web Services S3, or on a
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
local file system, they are simply
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
63
|
+
generated by a program are stored. Files can be on the local file system (`/` or
|
|
64
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
|
|
65
|
+
webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
|
|
66
|
+
a remote using the local filesystem as a file repository (`fake://`). When files to be read
|
|
67
|
+
are on the local file system, they are simply accessed in-place. Otherwise, they are
|
|
68
|
+
downloaded from the remote source to a local temporary directory. When files to be written
|
|
69
|
+
are on the local file system, they are simply written in-place. Otherwise, they are
|
|
70
|
+
written to a local temporary directory and then uploaded to the remote location (it is not
|
|
71
|
+
possible to upload to a webserver). When a cache is no longer needed, it is deleted from
|
|
72
|
+
the local disk.
|
|
70
73
|
|
|
71
74
|
`filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
|
|
72
75
|
|
|
@@ -218,14 +221,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
|
218
221
|
contents = voldesc_path.read_text()
|
|
219
222
|
```
|
|
220
223
|
|
|
221
|
-
Finally, there are
|
|
222
|
-
locations without invoking any caching behavior: `
|
|
223
|
-
`FileCacheSourceHTTP`, `FileCacheSourceGS`, and `
|
|
224
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
225
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
226
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
|
|
227
|
+
The first four provide access to real storage locations, while `FileCacheSourceFake`
|
|
228
|
+
simulates a remote source for testing purposes:
|
|
224
229
|
|
|
225
230
|
```python
|
|
226
|
-
from filecache import FileCacheSourceGS
|
|
227
|
-
|
|
228
|
-
|
|
231
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake
|
|
232
|
+
src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
233
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
|
|
234
|
+
src2 = FileCacheSourceFake('fake://fake-bucket')
|
|
235
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
229
236
|
```
|
|
230
237
|
|
|
231
238
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -11,7 +11,7 @@ import uuid
|
|
|
11
11
|
|
|
12
12
|
import pytest
|
|
13
13
|
|
|
14
|
-
from filecache import FileCache, FCPath
|
|
14
|
+
from filecache import FileCache, FileCacheSourceFake, FCPath
|
|
15
15
|
|
|
16
16
|
from tests.test_file_cache import (GS_TEST_BUCKET_ROOT,
|
|
17
17
|
S3_TEST_BUCKET_ROOT,
|
|
@@ -667,6 +667,45 @@ def test_read_write():
|
|
|
667
667
|
p = FCPath('a')
|
|
668
668
|
p.write_text(5)
|
|
669
669
|
|
|
670
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
671
|
+
try:
|
|
672
|
+
with FileCache(cache_name=None) as fc:
|
|
673
|
+
fake_prefix = 'fake://test-bucket'
|
|
674
|
+
pfx = fc.new_path(fake_prefix)
|
|
675
|
+
p1 = pfx / 'test_file.bin'
|
|
676
|
+
p2 = pfx / 'test_file.txt'
|
|
677
|
+
|
|
678
|
+
p1.write_bytes(b'A')
|
|
679
|
+
p2.write_text('ABC\n')
|
|
680
|
+
assert p1.read_bytes() == b'A'
|
|
681
|
+
assert p2.read_text() == 'ABC\n'
|
|
682
|
+
|
|
683
|
+
# Should behave like a remote source with uploads
|
|
684
|
+
assert fc.download_counter == 0
|
|
685
|
+
assert fc.upload_counter == 2
|
|
686
|
+
assert p1.download_counter == 0
|
|
687
|
+
assert p1.upload_counter == 1
|
|
688
|
+
assert p2.download_counter == 0
|
|
689
|
+
assert p2.upload_counter == 1
|
|
690
|
+
|
|
691
|
+
# Test in new cache to verify persistence
|
|
692
|
+
with FileCache(cache_name=None) as fc:
|
|
693
|
+
pfx = fc.new_path(fake_prefix)
|
|
694
|
+
p1 = pfx / 'test_file.bin'
|
|
695
|
+
p2 = pfx / 'test_file.txt'
|
|
696
|
+
|
|
697
|
+
# Reading should require downloads from "remote" storage
|
|
698
|
+
assert p1.read_bytes() == b'A'
|
|
699
|
+
assert p2.read_text() == 'ABC\n'
|
|
700
|
+
assert fc.download_counter == 2
|
|
701
|
+
assert fc.upload_counter == 0
|
|
702
|
+
assert p1.download_counter == 1
|
|
703
|
+
assert p1.upload_counter == 0
|
|
704
|
+
assert p2.download_counter == 1
|
|
705
|
+
assert p2.upload_counter == 0
|
|
706
|
+
finally:
|
|
707
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
708
|
+
|
|
670
709
|
|
|
671
710
|
def test_default_filecache():
|
|
672
711
|
with FileCache(delete_on_exit=True) as fc:
|
|
@@ -676,6 +715,15 @@ def test_default_filecache():
|
|
|
676
715
|
p3 = FCPath(GS_WRITABLE_TEST_BUCKET_ROOT) / EXPECTED_FILENAMES[0]
|
|
677
716
|
assert p2.get_local_path() == p3.get_local_path()
|
|
678
717
|
|
|
718
|
+
fake_path = 'fake://test-bucket/test.txt'
|
|
719
|
+
p1 = FCPath(fake_path)
|
|
720
|
+
p2 = fc.new_path(fake_path)
|
|
721
|
+
p1.write_text('test content') # Write using default FileCache
|
|
722
|
+
assert p2.read_text() == 'test content' # Read using explicit FileCache
|
|
723
|
+
assert p1.get_local_path() == p2.get_local_path()
|
|
724
|
+
p3 = FCPath(fake_path) # Create new path with default FileCache
|
|
725
|
+
assert p2.get_local_path() == p3.get_local_path()
|
|
726
|
+
|
|
679
727
|
|
|
680
728
|
def test_relative():
|
|
681
729
|
assert (FCPath(f'{GS_TEST_BUCKET_ROOT}/a/b/c.txt')
|
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
################################################################################
|
|
2
|
+
# tests/test_file_cache_source.py
|
|
3
|
+
################################################################################
|
|
4
|
+
|
|
5
|
+
import pathlib
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import pytest
|
|
8
|
+
import shutil
|
|
9
|
+
import os
|
|
10
|
+
|
|
11
|
+
from filecache import (FileCacheSource,
|
|
12
|
+
FileCacheSourceFile,
|
|
13
|
+
FileCacheSourceHTTP,
|
|
14
|
+
FileCacheSourceGS,
|
|
15
|
+
FileCacheSourceS3,
|
|
16
|
+
FileCacheSourceFake)
|
|
17
|
+
|
|
18
|
+
from .test_file_cache import EXPECTED_DIR, EXPECTED_FILENAMES
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_source_bad():
|
|
22
|
+
with pytest.raises(ValueError):
|
|
23
|
+
FileCacheSourceFile('fred', 'hi')
|
|
24
|
+
|
|
25
|
+
with pytest.raises(ValueError):
|
|
26
|
+
FileCacheSourceHTTP('fred', 'hi')
|
|
27
|
+
with pytest.raises(ValueError):
|
|
28
|
+
FileCacheSourceHTTP('http', 'hi/hi')
|
|
29
|
+
with pytest.raises(ValueError):
|
|
30
|
+
FileCacheSourceHTTP('https', '')
|
|
31
|
+
|
|
32
|
+
with pytest.raises(ValueError):
|
|
33
|
+
FileCacheSourceGS('fred', 'hi')
|
|
34
|
+
with pytest.raises(ValueError):
|
|
35
|
+
FileCacheSourceGS('gs', 'hi/hi')
|
|
36
|
+
with pytest.raises(ValueError):
|
|
37
|
+
FileCacheSourceGS('gs', '')
|
|
38
|
+
|
|
39
|
+
with pytest.raises(ValueError):
|
|
40
|
+
FileCacheSourceS3('fred', 'hi')
|
|
41
|
+
with pytest.raises(ValueError):
|
|
42
|
+
FileCacheSourceS3('s3', 'hi/hi')
|
|
43
|
+
with pytest.raises(ValueError):
|
|
44
|
+
FileCacheSourceS3('s3', '')
|
|
45
|
+
|
|
46
|
+
with pytest.raises(ValueError):
|
|
47
|
+
FileCacheSourceFake('not-fake', 'test-bucket')
|
|
48
|
+
with pytest.raises(ValueError):
|
|
49
|
+
FileCacheSourceFake('fake', 'hi/hi')
|
|
50
|
+
with pytest.raises(ValueError):
|
|
51
|
+
FileCacheSourceFake('fake', '')
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_filesource_bad():
|
|
55
|
+
sl = FileCacheSourceFile('file', '')
|
|
56
|
+
with pytest.raises(FileNotFoundError):
|
|
57
|
+
sl.upload('non-existent.txt', 'non-existent.txt')
|
|
58
|
+
assert not sl.exists('non-existent.txt')
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_filesource_good():
|
|
62
|
+
sl = FileCacheSourceFile('file', '')
|
|
63
|
+
assert sl.exists(EXPECTED_DIR / EXPECTED_FILENAMES[1])
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_source_notimp():
|
|
67
|
+
with pytest.raises(TypeError):
|
|
68
|
+
FileCacheSource('', '').exists('')
|
|
69
|
+
with pytest.raises(NotImplementedError):
|
|
70
|
+
FileCacheSourceHTTP('http', 'fred').upload('', '')
|
|
71
|
+
with pytest.raises(NotImplementedError):
|
|
72
|
+
FileCacheSourceHTTP('http', 'fred').iterdir_type('')
|
|
73
|
+
with pytest.raises(NotImplementedError):
|
|
74
|
+
FileCacheSourceHTTP('http', 'fred').unlink('')
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_source_nthreads_bad():
|
|
78
|
+
with pytest.raises(ValueError):
|
|
79
|
+
FileCacheSourceFile('file', '').retrieve_multi(['/test'], ['/test'], nthreads=-1)
|
|
80
|
+
with pytest.raises(ValueError):
|
|
81
|
+
FileCacheSourceFile('file', '').retrieve_multi(['/test'], ['/test'], nthreads=4.5)
|
|
82
|
+
with pytest.raises(ValueError):
|
|
83
|
+
FileCacheSourceFile('file', '').upload_multi(['/test'], ['/test'], nthreads=-1)
|
|
84
|
+
with pytest.raises(ValueError):
|
|
85
|
+
FileCacheSourceFile('file', '').upload_multi(['/test'], ['/test'], nthreads=4.5)
|
|
86
|
+
with pytest.raises(ValueError):
|
|
87
|
+
FileCacheSourceFile('file', '').exists_multi(['/test'], nthreads=-1)
|
|
88
|
+
with pytest.raises(ValueError):
|
|
89
|
+
FileCacheSourceFile('file', '').exists_multi(['/test'], nthreads=4.5)
|
|
90
|
+
with pytest.raises(ValueError):
|
|
91
|
+
FileCacheSourceFile('file', '').unlink_multi(['/test'], nthreads=-1)
|
|
92
|
+
with pytest.raises(ValueError):
|
|
93
|
+
FileCacheSourceFile('file', '').unlink_multi(['/test'], nthreads=4.5)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_fake_source_init():
|
|
97
|
+
"""Test FileCacheSourceFake initialization."""
|
|
98
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
99
|
+
try:
|
|
100
|
+
# Test valid initialization
|
|
101
|
+
fake = FileCacheSourceFake('fake', 'test-bucket')
|
|
102
|
+
assert fake.schemes() == ('fake',)
|
|
103
|
+
assert not fake.uses_anonymous()
|
|
104
|
+
|
|
105
|
+
# Test invalid scheme
|
|
106
|
+
with pytest.raises(ValueError, match='Scheme must be "fake"'):
|
|
107
|
+
FileCacheSourceFake('not-fake', 'test-bucket')
|
|
108
|
+
|
|
109
|
+
# Test empty remote
|
|
110
|
+
with pytest.raises(ValueError, match='remote parameter must have a value'):
|
|
111
|
+
FileCacheSourceFake('fake', '')
|
|
112
|
+
finally:
|
|
113
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_fake_source_default_storage_dir(tmp_path: Path):
|
|
117
|
+
"""Test the default storage directory management."""
|
|
118
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
119
|
+
try:
|
|
120
|
+
# Test getting default directory
|
|
121
|
+
default_dir = FileCacheSourceFake.get_default_storage_dir()
|
|
122
|
+
assert '.filecache_fake_remote' in str(default_dir)
|
|
123
|
+
|
|
124
|
+
# Test setting default directory
|
|
125
|
+
FileCacheSourceFake.set_default_storage_dir(tmp_path)
|
|
126
|
+
assert FileCacheSourceFake.get_default_storage_dir() == tmp_path
|
|
127
|
+
|
|
128
|
+
# Test creating a source uses the default directory
|
|
129
|
+
fake = FileCacheSourceFake('fake', 'test-bucket')
|
|
130
|
+
assert fake._storage_base == tmp_path
|
|
131
|
+
|
|
132
|
+
# Test deleting default directory
|
|
133
|
+
path = tmp_path / 'test.txt'
|
|
134
|
+
path.write_text('test')
|
|
135
|
+
assert path.exists()
|
|
136
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
137
|
+
assert not path.exists()
|
|
138
|
+
finally:
|
|
139
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_fake_source_file_operations(tmp_path: Path):
|
|
143
|
+
"""Test basic file operations with FileCacheSourceFake."""
|
|
144
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
145
|
+
try:
|
|
146
|
+
fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
|
|
147
|
+
|
|
148
|
+
# Test file that doesn't exist
|
|
149
|
+
assert not fake.exists('test.txt')
|
|
150
|
+
|
|
151
|
+
# Test upload
|
|
152
|
+
source_file = tmp_path / 'source.txt'
|
|
153
|
+
source_file.write_text('test content')
|
|
154
|
+
|
|
155
|
+
fake.upload('test.txt', source_file)
|
|
156
|
+
assert fake.exists('test.txt')
|
|
157
|
+
|
|
158
|
+
# Test retrieve
|
|
159
|
+
dest_file = tmp_path / 'dest.txt'
|
|
160
|
+
fake.retrieve('test.txt', dest_file)
|
|
161
|
+
assert dest_file.read_text() == 'test content'
|
|
162
|
+
|
|
163
|
+
# Test unlink
|
|
164
|
+
fake.unlink('test.txt')
|
|
165
|
+
assert not fake.exists('test.txt')
|
|
166
|
+
|
|
167
|
+
# Test unlink missing file
|
|
168
|
+
with pytest.raises(FileNotFoundError):
|
|
169
|
+
fake.unlink('missing.txt')
|
|
170
|
+
|
|
171
|
+
# Test unlink missing file with missing_ok=True
|
|
172
|
+
fake.unlink('missing.txt', missing_ok=True)
|
|
173
|
+
finally:
|
|
174
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def test_fake_source_directory_operations(tmp_path: Path):
|
|
178
|
+
"""Test directory operations with FileCacheSourceFake."""
|
|
179
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
180
|
+
try:
|
|
181
|
+
fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
|
|
182
|
+
|
|
183
|
+
# Create some test files and directories
|
|
184
|
+
(tmp_path / 'test-bucket/dir1').mkdir(parents=True)
|
|
185
|
+
(tmp_path / 'test-bucket/dir1/file1.txt').write_text('content1')
|
|
186
|
+
(tmp_path / 'test-bucket/dir2').mkdir(parents=True)
|
|
187
|
+
(tmp_path / 'test-bucket/dir2/file2.txt').write_text('content2')
|
|
188
|
+
(tmp_path / 'test-bucket/file3.txt').write_text('content3')
|
|
189
|
+
|
|
190
|
+
# Test iterdir_type
|
|
191
|
+
entries = list(fake.iterdir_type(''))
|
|
192
|
+
assert len(entries) == 3
|
|
193
|
+
|
|
194
|
+
# Sort entries for consistent testing
|
|
195
|
+
entries.sort()
|
|
196
|
+
|
|
197
|
+
# Test full paths and types
|
|
198
|
+
assert entries[0] == ('fake://test-bucket/dir1', True)
|
|
199
|
+
assert entries[1] == ('fake://test-bucket/dir2', True)
|
|
200
|
+
assert entries[2] == ('fake://test-bucket/file3.txt', False)
|
|
201
|
+
|
|
202
|
+
# Test subdirectory listing
|
|
203
|
+
entries = list(fake.iterdir_type('dir1'))
|
|
204
|
+
assert len(entries) == 1
|
|
205
|
+
assert entries[0] == ('fake://test-bucket/dir1/file1.txt', False)
|
|
206
|
+
finally:
|
|
207
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def test_fake_source_error_cases(tmp_path: Path):
|
|
211
|
+
"""Test error cases with FileCacheSourceFake."""
|
|
212
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
213
|
+
try:
|
|
214
|
+
fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
|
|
215
|
+
|
|
216
|
+
# Test retrieve non-existent file
|
|
217
|
+
with pytest.raises(FileNotFoundError):
|
|
218
|
+
fake.retrieve('missing.txt', tmp_path / 'dest.txt')
|
|
219
|
+
|
|
220
|
+
# Test upload non-existent file
|
|
221
|
+
with pytest.raises(FileNotFoundError):
|
|
222
|
+
fake.upload('dest.txt', tmp_path / 'missing.txt')
|
|
223
|
+
|
|
224
|
+
# Test iterdir_type on non-existent directory
|
|
225
|
+
assert list(fake.iterdir_type('missing-dir')) == []
|
|
226
|
+
|
|
227
|
+
# Create a test file
|
|
228
|
+
source_path = tmp_path / 'test-bucket/test.txt'
|
|
229
|
+
source_path.parent.mkdir(parents=True, exist_ok=True)
|
|
230
|
+
source_path.write_text('test content')
|
|
231
|
+
|
|
232
|
+
# Create a destination directory that exists but is not writable
|
|
233
|
+
dest_dir = tmp_path / 'dest'
|
|
234
|
+
dest_dir.mkdir()
|
|
235
|
+
dest_path = dest_dir / 'test.txt'
|
|
236
|
+
|
|
237
|
+
# Mock shutil.copy2 to raise an error after creating the temp file
|
|
238
|
+
original_copy2 = shutil.copy2
|
|
239
|
+
|
|
240
|
+
def mock_copy2(src, dst):
|
|
241
|
+
# Create the temp file
|
|
242
|
+
with open(dst, 'w') as f:
|
|
243
|
+
f.write('partial content')
|
|
244
|
+
# Then raise an error
|
|
245
|
+
raise OSError('Mock copy error')
|
|
246
|
+
|
|
247
|
+
shutil.copy2 = mock_copy2
|
|
248
|
+
try:
|
|
249
|
+
with pytest.raises(OSError, match="Mock copy error"):
|
|
250
|
+
fake.retrieve('test.txt', dest_path)
|
|
251
|
+
finally:
|
|
252
|
+
shutil.copy2 = original_copy2
|
|
253
|
+
finally:
|
|
254
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def test_fake_source_multi_operations(tmp_path: Path):
|
|
258
|
+
"""Test multi-file operations with FileCacheSourceFake."""
|
|
259
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
260
|
+
try:
|
|
261
|
+
fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
|
|
262
|
+
|
|
263
|
+
# Create test files
|
|
264
|
+
files = []
|
|
265
|
+
for i in range(3):
|
|
266
|
+
path = tmp_path / f'source{i}.txt'
|
|
267
|
+
path.write_text(f'content{i}')
|
|
268
|
+
files.append(path)
|
|
269
|
+
|
|
270
|
+
# Test exists_multi
|
|
271
|
+
results = fake.exists_multi(['test1.txt', 'test2.txt', 'test3.txt'])
|
|
272
|
+
assert results == [False, False, False]
|
|
273
|
+
|
|
274
|
+
# Test upload_multi
|
|
275
|
+
sub_paths = [f'test{i}.txt' for i in range(3)]
|
|
276
|
+
results = fake.upload_multi(sub_paths[:2], files[:2])
|
|
277
|
+
assert all(isinstance(r, Path) for r in results)
|
|
278
|
+
|
|
279
|
+
# Test exists_multi after upload
|
|
280
|
+
results = fake.exists_multi(sub_paths)
|
|
281
|
+
assert results == [True, True, False]
|
|
282
|
+
|
|
283
|
+
# Test retrieve_multi
|
|
284
|
+
dest_files = [tmp_path / f'dest{i}.txt' for i in range(3)]
|
|
285
|
+
results = fake.retrieve_multi(sub_paths, dest_files)
|
|
286
|
+
assert isinstance(results[0], Path)
|
|
287
|
+
assert isinstance(results[1], Path)
|
|
288
|
+
assert isinstance(results[2], BaseException)
|
|
289
|
+
assert dest_files[0].exists()
|
|
290
|
+
assert dest_files[1].exists()
|
|
291
|
+
assert not dest_files[2].exists()
|
|
292
|
+
|
|
293
|
+
# Test unlink_multi
|
|
294
|
+
results = fake.unlink_multi(sub_paths)
|
|
295
|
+
assert isinstance(results[0], str)
|
|
296
|
+
assert isinstance(results[1], str)
|
|
297
|
+
assert isinstance(results[2], BaseException)
|
|
298
|
+
assert not any(fake.exists(p) for p in sub_paths)
|
|
299
|
+
finally:
|
|
300
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def test_fake_source_atomic_operations(tmp_path: Path):
|
|
304
|
+
"""Test that uploads and downloads are atomic operations."""
|
|
305
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
306
|
+
try:
|
|
307
|
+
fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
|
|
308
|
+
|
|
309
|
+
# Test atomic upload
|
|
310
|
+
source_file = tmp_path / 'source.txt'
|
|
311
|
+
source_file.write_text('test content')
|
|
312
|
+
|
|
313
|
+
# Verify upload creates temp file and then renames
|
|
314
|
+
def mock_rename(src, dst):
|
|
315
|
+
# Verify temp file exists and destination doesn't
|
|
316
|
+
assert src.exists()
|
|
317
|
+
assert not dst.exists()
|
|
318
|
+
# Just verify it's a temp file with a UUID-like pattern
|
|
319
|
+
assert '.txt_' in str(src)
|
|
320
|
+
os.rename(str(src), str(dst)) # Use os.rename to avoid recursion
|
|
321
|
+
|
|
322
|
+
original_rename = pathlib.Path.rename
|
|
323
|
+
pathlib.Path.rename = mock_rename
|
|
324
|
+
try:
|
|
325
|
+
fake.upload('test.txt', source_file)
|
|
326
|
+
finally:
|
|
327
|
+
pathlib.Path.rename = original_rename
|
|
328
|
+
|
|
329
|
+
# Test atomic download
|
|
330
|
+
dest_file = tmp_path / 'dest.txt'
|
|
331
|
+
|
|
332
|
+
# Verify download creates temp file and then renames
|
|
333
|
+
def mock_rename2(src, dst):
|
|
334
|
+
# Verify temp file exists and destination doesn't
|
|
335
|
+
assert src.exists()
|
|
336
|
+
assert not dst.exists()
|
|
337
|
+
# Just verify it's a temp file with a UUID-like pattern
|
|
338
|
+
assert '.txt_' in str(src)
|
|
339
|
+
os.rename(str(src), str(dst)) # Use os.rename to avoid recursion
|
|
340
|
+
|
|
341
|
+
pathlib.Path.rename = mock_rename2
|
|
342
|
+
try:
|
|
343
|
+
fake.retrieve('test.txt', dest_file)
|
|
344
|
+
finally:
|
|
345
|
+
pathlib.Path.rename = original_rename
|
|
346
|
+
finally:
|
|
347
|
+
FileCacheSourceFake.delete_default_storage_dir()
|
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
################################################################################
|
|
2
|
-
# tests/test_file_cache_source.py
|
|
3
|
-
################################################################################
|
|
4
|
-
|
|
5
|
-
import pytest
|
|
6
|
-
|
|
7
|
-
from filecache import (FileCacheSource,
|
|
8
|
-
FileCacheSourceFile,
|
|
9
|
-
FileCacheSourceHTTP,
|
|
10
|
-
FileCacheSourceGS,
|
|
11
|
-
FileCacheSourceS3)
|
|
12
|
-
|
|
13
|
-
from .test_file_cache import EXPECTED_DIR, EXPECTED_FILENAMES
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def test_source_bad():
|
|
17
|
-
with pytest.raises(ValueError):
|
|
18
|
-
FileCacheSourceFile('fred', 'hi')
|
|
19
|
-
|
|
20
|
-
with pytest.raises(ValueError):
|
|
21
|
-
FileCacheSourceHTTP('fred', 'hi')
|
|
22
|
-
with pytest.raises(ValueError):
|
|
23
|
-
FileCacheSourceHTTP('http', 'hi/hi')
|
|
24
|
-
with pytest.raises(ValueError):
|
|
25
|
-
FileCacheSourceHTTP('https', '')
|
|
26
|
-
|
|
27
|
-
with pytest.raises(ValueError):
|
|
28
|
-
FileCacheSourceGS('fred', 'hi')
|
|
29
|
-
with pytest.raises(ValueError):
|
|
30
|
-
FileCacheSourceGS('gs', 'hi/hi')
|
|
31
|
-
with pytest.raises(ValueError):
|
|
32
|
-
FileCacheSourceGS('gs', '')
|
|
33
|
-
|
|
34
|
-
with pytest.raises(ValueError):
|
|
35
|
-
FileCacheSourceS3('fred', 'hi')
|
|
36
|
-
with pytest.raises(ValueError):
|
|
37
|
-
FileCacheSourceS3('s3', 'hi/hi')
|
|
38
|
-
with pytest.raises(ValueError):
|
|
39
|
-
FileCacheSourceS3('s3', '')
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def test_filesource_bad():
|
|
43
|
-
sl = FileCacheSourceFile('file', '')
|
|
44
|
-
with pytest.raises(FileNotFoundError):
|
|
45
|
-
sl.upload('non-existent.txt', 'non-existent.txt')
|
|
46
|
-
assert not sl.exists('non-existent.txt')
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def test_filesource_good():
|
|
50
|
-
sl = FileCacheSourceFile('file', '')
|
|
51
|
-
assert sl.exists(EXPECTED_DIR / EXPECTED_FILENAMES[1])
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
def test_source_notimp():
|
|
55
|
-
with pytest.raises(TypeError):
|
|
56
|
-
FileCacheSource('', '').exists('')
|
|
57
|
-
with pytest.raises(NotImplementedError):
|
|
58
|
-
FileCacheSourceHTTP('http', 'fred').upload('', '')
|
|
59
|
-
with pytest.raises(NotImplementedError):
|
|
60
|
-
FileCacheSourceHTTP('http', 'fred').iterdir_type('')
|
|
61
|
-
with pytest.raises(NotImplementedError):
|
|
62
|
-
FileCacheSourceHTTP('http', 'fred').unlink('')
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def test_source_nthreads_bad():
|
|
66
|
-
with pytest.raises(ValueError):
|
|
67
|
-
FileCacheSourceFile('file', '').retrieve_multi(['/test'], ['/test'], nthreads=-1)
|
|
68
|
-
with pytest.raises(ValueError):
|
|
69
|
-
FileCacheSourceFile('file', '').retrieve_multi(['/test'], ['/test'], nthreads=4.5)
|
|
70
|
-
with pytest.raises(ValueError):
|
|
71
|
-
FileCacheSourceFile('file', '').upload_multi(['/test'], ['/test'], nthreads=-1)
|
|
72
|
-
with pytest.raises(ValueError):
|
|
73
|
-
FileCacheSourceFile('file', '').upload_multi(['/test'], ['/test'], nthreads=4.5)
|
|
74
|
-
with pytest.raises(ValueError):
|
|
75
|
-
FileCacheSourceFile('file', '').exists_multi(['/test'], nthreads=-1)
|
|
76
|
-
with pytest.raises(ValueError):
|
|
77
|
-
FileCacheSourceFile('file', '').exists_multi(['/test'], nthreads=4.5)
|
|
78
|
-
with pytest.raises(ValueError):
|
|
79
|
-
FileCacheSourceFile('file', '').unlink_multi(['/test'], nthreads=-1)
|
|
80
|
-
with pytest.raises(ValueError):
|
|
81
|
-
FileCacheSourceFile('file', '').unlink_multi(['/test'], nthreads=4.5)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2a/binary1.bin
RENAMED
|
File without changes
|
{rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2b/binary1.bin
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|