rms-filecache 2.3.0__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.gitignore +2 -0
- {rms_filecache-2.3.0/rms_filecache.egg-info → rms_filecache-3.0.0}/PKG-INFO +27 -25
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/README.md +24 -23
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/__init__.py +29 -23
- rms_filecache-3.0.0/filecache/_version.py +34 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/file_cache.py +920 -122
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/file_cache_path.py +345 -59
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/file_cache_source.py +835 -39
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/file_cache_types.py +8 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/requirements.txt +1 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0/rms_filecache.egg-info}/PKG-INFO +27 -25
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/rms_filecache.egg-info/SOURCES.txt +1 -1
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/test_file_cache.py +594 -61
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/test_file_cache_path.py +315 -72
- rms_filecache-3.0.0/tests/test_file_cache_source.py +406 -0
- rms_filecache-2.3.0/tests/test_url_to_path.py → rms_filecache-3.0.0/tests/test_url_mapping.py +265 -24
- rms_filecache-2.3.0/filecache/_version.py +0 -16
- rms_filecache-2.3.0/tests/test_file_cache_source.py +0 -81
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.coveragerc +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.flake8 +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.github/pull_request_template.md +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.github/workflows/publish_to_pypi.yml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.github/workflows/publish_to_test_pypi.yml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.github/workflows/run-tests.yml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.readthedocs.yaml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/LICENSE +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/Makefile +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/conf.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/index.rst +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/make.bat +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/module.rst +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/py.typed +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/mypy.ini +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/pyproject.toml +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/setup.cfg +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_programs/locking.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_programs/multi_download.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/__init__.py +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/s3_bucket_policy_write.txt +0 -0
- {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/s3_lifecycle_policy.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: rms-filecache
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: File cache for files retrieved from the cloud
|
|
5
5
|
Maintainer-email: "Robert S. French" <rfrench@seti.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -31,6 +31,7 @@ Requires-Dist: filelock
|
|
|
31
31
|
Requires-Dist: google-cloud-storage
|
|
32
32
|
Requires-Dist: requests
|
|
33
33
|
Requires-Dist: typing_extensions
|
|
34
|
+
Dynamic: license-file
|
|
34
35
|
|
|
35
36
|
[](https://github.com/SETI/rms-filecache/releases)
|
|
36
37
|
[](https://github.com/SETI/rms-filecache/releases)
|
|
@@ -59,14 +60,16 @@ Requires-Dist: typing_extensions
|
|
|
59
60
|
# Introduction
|
|
60
61
|
|
|
61
62
|
`filecache` is a Python module that abstracts away the location where files used or
|
|
62
|
-
generated by a program are stored. Files can be on the local file system
|
|
63
|
-
Storage, on Amazon Web Services S3, or on a
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
local file system, they are simply
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
63
|
+
generated by a program are stored. Files can be on the local file system (`/` or
|
|
64
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
|
|
65
|
+
webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
|
|
66
|
+
a remote using the local filesystem as a file repository (`fake://`). When files to be read
|
|
67
|
+
are on the local file system, they are simply accessed in-place. Otherwise, they are
|
|
68
|
+
downloaded from the remote source to a local temporary directory. When files to be written
|
|
69
|
+
are on the local file system, they are simply written in-place. Otherwise, they are
|
|
70
|
+
written to a local temporary directory and then uploaded to the remote location (it is not
|
|
71
|
+
possible to upload to a webserver). When a cache is no longer needed, it is deleted from
|
|
72
|
+
the local disk.
|
|
70
73
|
|
|
71
74
|
`filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
|
|
72
75
|
|
|
@@ -153,21 +156,16 @@ from filecache import FileCache, FCPath
|
|
|
153
156
|
with FileCache(None) as fc: # Use as context manager
|
|
154
157
|
# Use GS by specifying the bucket name and one directory level
|
|
155
158
|
p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
|
|
156
|
-
# Use S3 by specifying the bucket name and two directory levels
|
|
157
|
-
# Alternative creation method
|
|
158
|
-
p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
|
|
159
|
-
anonymous=True)
|
|
160
159
|
# Access GS using a directory + filename (since only one directory level
|
|
161
160
|
# was specified by the FCPath)
|
|
162
|
-
# The additional
|
|
161
|
+
# The additional filename is specified by using the / operator to create a new
|
|
162
|
+
# FCPath instance; anonymous=True is inherited
|
|
163
163
|
# Also use open() as a context manager
|
|
164
|
-
with p1
|
|
164
|
+
with (p1 / 'subdir2a/binary1.bin').open('rb') as fp:
|
|
165
165
|
bin1 = fp.read()
|
|
166
166
|
# Access S3 using a filename only (since two directory levels were already
|
|
167
167
|
# specified by the FCPath)
|
|
168
|
-
|
|
169
|
-
# FCPath instance; anonymous=True is inherited
|
|
170
|
-
with (p2 / 'binary1.bin').open(mode='rb') as fp:
|
|
168
|
+
with (p2 / 'binary1.bin').open('rb') as fp:
|
|
171
169
|
bin2 = fp.read()
|
|
172
170
|
assert bin1 == bin2
|
|
173
171
|
# Cache automatically deleted here
|
|
@@ -218,14 +216,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
|
218
216
|
contents = voldesc_path.read_text()
|
|
219
217
|
```
|
|
220
218
|
|
|
221
|
-
Finally, there are
|
|
222
|
-
locations without invoking any caching behavior: `
|
|
223
|
-
`FileCacheSourceHTTP`, `FileCacheSourceGS`, and `
|
|
219
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
220
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
221
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
|
|
222
|
+
The first four provide access to real storage locations, while `FileCacheSourceFake`
|
|
223
|
+
simulates a remote source for testing purposes:
|
|
224
224
|
|
|
225
225
|
```python
|
|
226
|
-
from filecache import FileCacheSourceGS
|
|
227
|
-
|
|
228
|
-
|
|
226
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake
|
|
227
|
+
src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
228
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
|
|
229
|
+
src2 = FileCacheSourceFake('fake://fake-bucket')
|
|
230
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
229
231
|
```
|
|
230
232
|
|
|
231
233
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -25,14 +25,16 @@
|
|
|
25
25
|
# Introduction
|
|
26
26
|
|
|
27
27
|
`filecache` is a Python module that abstracts away the location where files used or
|
|
28
|
-
generated by a program are stored. Files can be on the local file system
|
|
29
|
-
Storage, on Amazon Web Services S3, or on a
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
local file system, they are simply
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
generated by a program are stored. Files can be on the local file system (`/` or
|
|
29
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
|
|
30
|
+
webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
|
|
31
|
+
a remote using the local filesystem as a file repository (`fake://`). When files to be read
|
|
32
|
+
are on the local file system, they are simply accessed in-place. Otherwise, they are
|
|
33
|
+
downloaded from the remote source to a local temporary directory. When files to be written
|
|
34
|
+
are on the local file system, they are simply written in-place. Otherwise, they are
|
|
35
|
+
written to a local temporary directory and then uploaded to the remote location (it is not
|
|
36
|
+
possible to upload to a webserver). When a cache is no longer needed, it is deleted from
|
|
37
|
+
the local disk.
|
|
36
38
|
|
|
37
39
|
`filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
|
|
38
40
|
|
|
@@ -119,21 +121,16 @@ from filecache import FileCache, FCPath
|
|
|
119
121
|
with FileCache(None) as fc: # Use as context manager
|
|
120
122
|
# Use GS by specifying the bucket name and one directory level
|
|
121
123
|
p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
|
|
122
|
-
# Use S3 by specifying the bucket name and two directory levels
|
|
123
|
-
# Alternative creation method
|
|
124
|
-
p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
|
|
125
|
-
anonymous=True)
|
|
126
124
|
# Access GS using a directory + filename (since only one directory level
|
|
127
125
|
# was specified by the FCPath)
|
|
128
|
-
# The additional
|
|
126
|
+
# The additional filename is specified by using the / operator to create a new
|
|
127
|
+
# FCPath instance; anonymous=True is inherited
|
|
129
128
|
# Also use open() as a context manager
|
|
130
|
-
with p1
|
|
129
|
+
with (p1 / 'subdir2a/binary1.bin').open('rb') as fp:
|
|
131
130
|
bin1 = fp.read()
|
|
132
131
|
# Access S3 using a filename only (since two directory levels were already
|
|
133
132
|
# specified by the FCPath)
|
|
134
|
-
|
|
135
|
-
# FCPath instance; anonymous=True is inherited
|
|
136
|
-
with (p2 / 'binary1.bin').open(mode='rb') as fp:
|
|
133
|
+
with (p2 / 'binary1.bin').open('rb') as fp:
|
|
137
134
|
bin2 = fp.read()
|
|
138
135
|
assert bin1 == bin2
|
|
139
136
|
# Cache automatically deleted here
|
|
@@ -184,14 +181,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
|
184
181
|
contents = voldesc_path.read_text()
|
|
185
182
|
```
|
|
186
183
|
|
|
187
|
-
Finally, there are
|
|
188
|
-
locations without invoking any caching behavior: `
|
|
189
|
-
`FileCacheSourceHTTP`, `FileCacheSourceGS`, and `
|
|
184
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
185
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
186
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
|
|
187
|
+
The first four provide access to real storage locations, while `FileCacheSourceFake`
|
|
188
|
+
simulates a remote source for testing purposes:
|
|
190
189
|
|
|
191
190
|
```python
|
|
192
|
-
from filecache import FileCacheSourceGS
|
|
193
|
-
|
|
194
|
-
|
|
191
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake
|
|
192
|
+
src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
193
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
|
|
194
|
+
src2 = FileCacheSourceFake('fake://fake-bucket')
|
|
195
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
195
196
|
```
|
|
196
197
|
|
|
197
198
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -3,14 +3,16 @@
|
|
|
3
3
|
##########################################################################################
|
|
4
4
|
|
|
5
5
|
"""The purpose of the `filecache` module is to abstract away the location where files
|
|
6
|
-
used or generated by a program are stored. Files can be on the local file system
|
|
7
|
-
Google Cloud Storage, on Amazon Web Services S3, or on
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
are on the local file system, they are simply
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
6
|
+
used or generated by a program are stored. Files can be on the local file system (`/` or
|
|
7
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on
|
|
8
|
+
a webserver (`http://` or `https://`). There is also a fake remote that can be used to
|
|
9
|
+
simulate a remote using the local filesystem as a file repository (`fake://`). When files
|
|
10
|
+
to be read are on the local file system, they are simply accessed in-place. Otherwise,
|
|
11
|
+
they are downloaded from the remote source to a local temporary directory. When files to
|
|
12
|
+
be written are on the local file system, they are simply written in-place. Otherwise, they
|
|
13
|
+
are written to a local temporary directory and then uploaded to the remote location (it is
|
|
14
|
+
not possible to upload to a webserver). When a cache is no longer needed, it is deleted
|
|
15
|
+
from the local disk.
|
|
14
16
|
|
|
15
17
|
The top-level file organization is provided by the :class:`FileCache` class. A
|
|
16
18
|
:class:`FileCache` instance is used to specify a particular **sharing policy** and
|
|
@@ -135,19 +137,21 @@ was run. A shared cache is indicated by giving the cache a name (or no argument,
|
|
|
135
137
|
defaults to ``"global"``); also :class:`FCPath` defaults to using the global cache if no
|
|
136
138
|
:class:`FileCache` is specified. This results in the simplest form of the program:
|
|
137
139
|
|
|
138
|
-
from filecache import FCPath
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
from filecache import FileCacheSourceGS
|
|
149
|
-
|
|
150
|
-
|
|
140
|
+
from filecache import FCPath import os p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
|
|
141
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat' contents =
|
|
142
|
+
voldesc_path.read_text()
|
|
143
|
+
|
|
144
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
145
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
146
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and
|
|
147
|
+
`FileCacheSourceFake`. The first four provide access to real storage locations, while
|
|
148
|
+
`FileCacheSourceFake` simulates a remote source for testing purposes:
|
|
149
|
+
|
|
150
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake src1 =
|
|
151
|
+
FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
152
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin') src2 =
|
|
153
|
+
FileCacheSourceFake('fake://fake-bucket')
|
|
154
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
151
155
|
"""
|
|
152
156
|
|
|
153
157
|
import typing
|
|
@@ -168,7 +172,8 @@ from .file_cache_source import (FileCacheSource, # noqa: ignore E401
|
|
|
168
172
|
FileCacheSourceFile,
|
|
169
173
|
FileCacheSourceHTTP,
|
|
170
174
|
FileCacheSourceGS,
|
|
171
|
-
FileCacheSourceS3
|
|
175
|
+
FileCacheSourceS3,
|
|
176
|
+
FileCacheSourceFake)
|
|
172
177
|
|
|
173
178
|
__all__ = ['get_global_logger',
|
|
174
179
|
'register_filecachesource',
|
|
@@ -180,4 +185,5 @@ __all__ = ['get_global_logger',
|
|
|
180
185
|
'FileCacheSourceFile',
|
|
181
186
|
'FileCacheSourceHTTP',
|
|
182
187
|
'FileCacheSourceGS',
|
|
183
|
-
'FileCacheSourceS3'
|
|
188
|
+
'FileCacheSourceS3',
|
|
189
|
+
'FileCacheSourceFake']
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = [
|
|
5
|
+
"__version__",
|
|
6
|
+
"__version_tuple__",
|
|
7
|
+
"version",
|
|
8
|
+
"version_tuple",
|
|
9
|
+
"__commit_id__",
|
|
10
|
+
"commit_id",
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
TYPE_CHECKING = False
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from typing import Tuple
|
|
16
|
+
from typing import Union
|
|
17
|
+
|
|
18
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
+
COMMIT_ID = Union[str, None]
|
|
20
|
+
else:
|
|
21
|
+
VERSION_TUPLE = object
|
|
22
|
+
COMMIT_ID = object
|
|
23
|
+
|
|
24
|
+
version: str
|
|
25
|
+
__version__: str
|
|
26
|
+
__version_tuple__: VERSION_TUPLE
|
|
27
|
+
version_tuple: VERSION_TUPLE
|
|
28
|
+
commit_id: COMMIT_ID
|
|
29
|
+
__commit_id__: COMMIT_ID
|
|
30
|
+
|
|
31
|
+
__version__ = version = '3.0.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (3, 0, 0)
|
|
33
|
+
|
|
34
|
+
__commit_id__ = commit_id = 'g8dae7f090'
|