rms-filecache 1.2.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.coveragerc +1 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.flake8 +3 -1
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.github/workflows/run-tests.yml +14 -2
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/PKG-INFO +58 -52
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/README.md +57 -51
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/docs/conf.py +1 -0
- rms_filecache-2.0.0/docs/module.rst +26 -0
- rms_filecache-2.0.0/filecache/__init__.py +183 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/filecache/_version.py +2 -2
- rms_filecache-2.0.0/filecache/file_cache.py +1445 -0
- rms_filecache-2.0.0/filecache/file_cache_path.py +1258 -0
- rms_filecache-2.0.0/filecache/file_cache_source.py +752 -0
- rms_filecache-2.0.0/filecache/file_cache_types.py +21 -0
- rms_filecache-2.0.0/filecache/my_glob.py +639 -0
- rms_filecache-2.0.0/mypy.ini +3 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/requirements.txt +5 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/PKG-INFO +58 -52
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/SOURCES.txt +11 -1
- rms_filecache-2.0.0/tests/__init__.py +0 -0
- rms_filecache-1.2.0/tests/test_filecache.py → rms_filecache-2.0.0/tests/test_file_cache.py +551 -414
- rms_filecache-2.0.0/tests/test_file_cache_path.py +703 -0
- rms_filecache-2.0.0/tests/test_file_cache_source.py +73 -0
- rms_filecache-2.0.0/tests/test_url_to_path.py +306 -0
- rms_filecache-1.2.0/docs/module.rst +0 -10
- rms_filecache-1.2.0/filecache/__init__.py +0 -2003
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.github/pull_request_template.md +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.github/workflows/publish_to_pypi.yml +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.github/workflows/publish_to_test_pypi.yml +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.gitignore +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.readthedocs.yaml +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/LICENSE +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/docs/Makefile +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/docs/index.rst +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/docs/make.bat +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/pyproject.toml +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/setup.cfg +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_programs/locking.py +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_programs/multi_download.py +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/s3_bucket_policy_write.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/s3_lifecycle_policy.txt +0 -0
|
@@ -12,7 +12,7 @@ on:
|
|
|
12
12
|
|
|
13
13
|
jobs:
|
|
14
14
|
flake8:
|
|
15
|
-
name:
|
|
15
|
+
name: Lint filecache
|
|
16
16
|
runs-on: ubuntu-latest
|
|
17
17
|
steps:
|
|
18
18
|
- name: Checkout
|
|
@@ -20,11 +20,23 @@ jobs:
|
|
|
20
20
|
with:
|
|
21
21
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
22
22
|
|
|
23
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
24
|
+
uses: actions/setup-python@v5
|
|
25
|
+
with:
|
|
26
|
+
python-version: ${{ matrix.python-version }}
|
|
27
|
+
|
|
28
|
+
- name: Install dependencies
|
|
29
|
+
run: |
|
|
30
|
+
python -m pip install -r requirements.txt
|
|
31
|
+
|
|
23
32
|
- name: Flake8
|
|
24
33
|
run: |
|
|
25
|
-
pip install flake8
|
|
26
34
|
flake8 filecache tests
|
|
27
35
|
|
|
36
|
+
- name: Mypy
|
|
37
|
+
run: |
|
|
38
|
+
mypy filecache
|
|
39
|
+
|
|
28
40
|
test:
|
|
29
41
|
name: Test filecache
|
|
30
42
|
runs-on: ${{ matrix.os }}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: rms-filecache
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: File cache for files retrieved from the cloud
|
|
5
5
|
Maintainer-email: "Robert S. French" <rfrench@seti.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -79,28 +79,26 @@ pip install rms-filecache
|
|
|
79
79
|
|
|
80
80
|
# Getting Started
|
|
81
81
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
allowing them to be downloaded from a remote source only one time, saving time and
|
|
90
|
-
bandwidth.
|
|
82
|
+
The top-level file organization is provided by the `FileCache` class. A `FileCache`
|
|
83
|
+
instance is used to specify a particular **sharing policy** and **lifetime**. For example,
|
|
84
|
+
a cache could be private to the current process and group a set of files that all have the
|
|
85
|
+
same basic purpose. Once these files have been (downloaded and) read, they are deleted as
|
|
86
|
+
a group. Another cache could be shared among all processes on the current machine and
|
|
87
|
+
group a set of files that are needed by multiple processes, thus allowing them to be
|
|
88
|
+
downloaded from a remote source only one time, saving time and bandwidth.
|
|
91
89
|
|
|
92
90
|
A `FileCache` can be instantiated either directly or as a context manager. When
|
|
93
|
-
instantiated directly, the programmer is responsible for calling
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
full details.
|
|
91
|
+
instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
|
|
92
|
+
directly to delete the cache when finished (a non-shared cache will be automatically
|
|
93
|
+
deleted on program exit). When instantiated as a context manager, a non-shared cache is
|
|
94
|
+
deleted on exit from the context. See the class documentation for full details.
|
|
98
95
|
|
|
99
96
|
Usage examples:
|
|
100
97
|
|
|
101
98
|
```python
|
|
102
99
|
from filecache import FileCache
|
|
103
|
-
with
|
|
100
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
101
|
+
with FileCache(None) as fc: # Use as context manager
|
|
104
102
|
# Also use open() as a context manager
|
|
105
103
|
with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
|
|
106
104
|
anonymous=True) as fp:
|
|
@@ -111,7 +109,7 @@ with FileCache() as fc: # Use as context manager
|
|
|
111
109
|
assert bin1 == bin2
|
|
112
110
|
# Cache automatically deleted here
|
|
113
111
|
|
|
114
|
-
fc = FileCache() # Use without context manager
|
|
112
|
+
fc = FileCache(None) # Use without context manager
|
|
115
113
|
# Also retrieve file without using open context manager
|
|
116
114
|
path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
117
115
|
anonymous=True)
|
|
@@ -121,43 +119,53 @@ path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
|
121
119
|
anonymous=True)
|
|
122
120
|
with open(path2, 'rb') as fp:
|
|
123
121
|
bin2 = fp.read()
|
|
124
|
-
fc.
|
|
122
|
+
fc.delete_cache() # Cache manually deleted here
|
|
125
123
|
assert bin1 == bin2
|
|
126
124
|
|
|
127
125
|
# Write a file to a bucket and read it back
|
|
128
|
-
with FileCache() as fc:
|
|
126
|
+
with FileCache(None) as fc:
|
|
129
127
|
with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
|
|
130
128
|
fp.write('A')
|
|
131
129
|
# The cache will be deleted here so the file will have to be downloaded
|
|
132
|
-
with FileCache() as fc:
|
|
130
|
+
with FileCache(None) as fc:
|
|
133
131
|
with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
|
|
134
132
|
print(fp.read())
|
|
135
133
|
```
|
|
136
134
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
135
|
+
The `FCPath` class is a reimplementation of the Python `Path` class to support remote
|
|
136
|
+
acess using an associated `FileCache`. Like `Path`, an `FCPath` instance can contain any
|
|
137
|
+
part of a URI, but only an absolute URI can be used when actually accessing the file
|
|
138
|
+
specified by the `FCPath`. In addition, an `FCPath` can encapsulate various arguments such
|
|
139
|
+
as `anonymous` and `time_out` so that they do not need to be specified to each access
|
|
140
|
+
method. Thus, using this class can simplify the use of a `FileCache` by allowing the user
|
|
141
|
+
to operate on paths using the simpler syntax provided by `Path`, and to not specify
|
|
142
|
+
various other parameters at each method call site. If an `FCPath` instance is created
|
|
143
|
+
without an explicitly-associated `FileCache`, then the default `FileCache()` is used,
|
|
144
|
+
which specifies a shared cache named `"global"` that will persist after the program exits.
|
|
143
145
|
|
|
144
146
|
Compare this example to the one above:
|
|
145
147
|
|
|
146
148
|
```python
|
|
147
|
-
from filecache import FileCache
|
|
148
|
-
with
|
|
149
|
+
from filecache import FileCache, FCPath
|
|
150
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
151
|
+
with FileCache(None) as fc: # Use as context manager
|
|
149
152
|
# Use GS by specifying the bucket name and one directory level
|
|
150
|
-
|
|
153
|
+
p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
|
|
151
154
|
# Use S3 by specifying the bucket name and two directory levels
|
|
152
|
-
|
|
155
|
+
# Alternative creation method
|
|
156
|
+
p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
|
|
157
|
+
anonymous=True)
|
|
153
158
|
# Access GS using a directory + filename (since only one directory level
|
|
154
|
-
# was specified by the
|
|
159
|
+
# was specified by the FCPath)
|
|
160
|
+
# The additional directory and filename are specified as an argument to open()
|
|
155
161
|
# Also use open() as a context manager
|
|
156
|
-
with
|
|
162
|
+
with p1.open('subdir2a/binary1.bin', 'rb') as fp:
|
|
157
163
|
bin1 = fp.read()
|
|
158
164
|
# Access S3 using a filename only (since two directory levels were already
|
|
159
|
-
# specified by the
|
|
160
|
-
|
|
165
|
+
# specified by the FCPath)
|
|
166
|
+
# The additional filename is specified by using the / operator to create a new
|
|
167
|
+
# FCPath instance; anonymous=True is inherited
|
|
168
|
+
with (p2 / 'binary1.bin').open(mode='rb') as fp:
|
|
161
169
|
bin2 = fp.read()
|
|
162
170
|
assert bin1 == bin2
|
|
163
171
|
# Cache automatically deleted here
|
|
@@ -186,28 +194,26 @@ Then the program could be written as:
|
|
|
186
194
|
```python
|
|
187
195
|
from filecache import FileCache
|
|
188
196
|
import os
|
|
189
|
-
with FileCache() as fc:
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
197
|
+
with FileCache(None) as fc:
|
|
198
|
+
p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
|
|
199
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
200
|
+
contents = voldesc_path.read_text()
|
|
193
201
|
# Cache automatically deleted here
|
|
194
202
|
```
|
|
195
203
|
|
|
196
204
|
If the program was going to be run multiple times in a row, or multiple copies were going
|
|
197
|
-
to be run simultaneously,
|
|
198
|
-
|
|
199
|
-
|
|
205
|
+
to be run simultaneously, using a shared cache would allow all of the processes to share
|
|
206
|
+
the same copy, thus requiring only a single download no matter how many times the program
|
|
207
|
+
was run. A shared cache is indicated by giving the cache a name (or no argument, which
|
|
208
|
+
defaults to ``"global"``); also `FCPath` defaults to using the global cache if no
|
|
209
|
+
`FileCache` is specified. This results in the simplest form of the program:
|
|
200
210
|
|
|
201
211
|
```python
|
|
202
|
-
from filecache import
|
|
212
|
+
from filecache import FCPath
|
|
203
213
|
import os
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
contents = fp.read()
|
|
208
|
-
# Cache not deleted here; must be deleted manually using fc.clean_up(final=True)
|
|
209
|
-
# If not deleted manually, the shared cache will persist until the temporary
|
|
210
|
-
# directory is purged by the operating system (which may be never)
|
|
214
|
+
p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
|
|
215
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
216
|
+
contents = voldesc_path.read_text()
|
|
211
217
|
```
|
|
212
218
|
|
|
213
219
|
Finally, there are four classes that allow direct access to the four possible storage
|
|
@@ -215,9 +221,9 @@ locations without invoking any caching behavior: :class:`FileCacheSourceLocal`,
|
|
|
215
221
|
:class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`:
|
|
216
222
|
|
|
217
223
|
```python
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
224
|
+
from filecache import FileCacheSourceGS
|
|
225
|
+
src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
226
|
+
src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
|
|
221
227
|
```
|
|
222
228
|
|
|
223
229
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -47,28 +47,26 @@ pip install rms-filecache
|
|
|
47
47
|
|
|
48
48
|
# Getting Started
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
allowing them to be downloaded from a remote source only one time, saving time and
|
|
58
|
-
bandwidth.
|
|
50
|
+
The top-level file organization is provided by the `FileCache` class. A `FileCache`
|
|
51
|
+
instance is used to specify a particular **sharing policy** and **lifetime**. For example,
|
|
52
|
+
a cache could be private to the current process and group a set of files that all have the
|
|
53
|
+
same basic purpose. Once these files have been (downloaded and) read, they are deleted as
|
|
54
|
+
a group. Another cache could be shared among all processes on the current machine and
|
|
55
|
+
group a set of files that are needed by multiple processes, thus allowing them to be
|
|
56
|
+
downloaded from a remote source only one time, saving time and bandwidth.
|
|
59
57
|
|
|
60
58
|
A `FileCache` can be instantiated either directly or as a context manager. When
|
|
61
|
-
instantiated directly, the programmer is responsible for calling
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
full details.
|
|
59
|
+
instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
|
|
60
|
+
directly to delete the cache when finished (a non-shared cache will be automatically
|
|
61
|
+
deleted on program exit). When instantiated as a context manager, a non-shared cache is
|
|
62
|
+
deleted on exit from the context. See the class documentation for full details.
|
|
66
63
|
|
|
67
64
|
Usage examples:
|
|
68
65
|
|
|
69
66
|
```python
|
|
70
67
|
from filecache import FileCache
|
|
71
|
-
with
|
|
68
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
69
|
+
with FileCache(None) as fc: # Use as context manager
|
|
72
70
|
# Also use open() as a context manager
|
|
73
71
|
with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
|
|
74
72
|
anonymous=True) as fp:
|
|
@@ -79,7 +77,7 @@ with FileCache() as fc: # Use as context manager
|
|
|
79
77
|
assert bin1 == bin2
|
|
80
78
|
# Cache automatically deleted here
|
|
81
79
|
|
|
82
|
-
fc = FileCache() # Use without context manager
|
|
80
|
+
fc = FileCache(None) # Use without context manager
|
|
83
81
|
# Also retrieve file without using open context manager
|
|
84
82
|
path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
85
83
|
anonymous=True)
|
|
@@ -89,43 +87,53 @@ path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
|
89
87
|
anonymous=True)
|
|
90
88
|
with open(path2, 'rb') as fp:
|
|
91
89
|
bin2 = fp.read()
|
|
92
|
-
fc.
|
|
90
|
+
fc.delete_cache() # Cache manually deleted here
|
|
93
91
|
assert bin1 == bin2
|
|
94
92
|
|
|
95
93
|
# Write a file to a bucket and read it back
|
|
96
|
-
with FileCache() as fc:
|
|
94
|
+
with FileCache(None) as fc:
|
|
97
95
|
with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
|
|
98
96
|
fp.write('A')
|
|
99
97
|
# The cache will be deleted here so the file will have to be downloaded
|
|
100
|
-
with FileCache() as fc:
|
|
98
|
+
with FileCache(None) as fc:
|
|
101
99
|
with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
|
|
102
100
|
print(fp.read())
|
|
103
101
|
```
|
|
104
102
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
103
|
+
The `FCPath` class is a reimplementation of the Python `Path` class to support remote
|
|
104
|
+
acess using an associated `FileCache`. Like `Path`, an `FCPath` instance can contain any
|
|
105
|
+
part of a URI, but only an absolute URI can be used when actually accessing the file
|
|
106
|
+
specified by the `FCPath`. In addition, an `FCPath` can encapsulate various arguments such
|
|
107
|
+
as `anonymous` and `time_out` so that they do not need to be specified to each access
|
|
108
|
+
method. Thus, using this class can simplify the use of a `FileCache` by allowing the user
|
|
109
|
+
to operate on paths using the simpler syntax provided by `Path`, and to not specify
|
|
110
|
+
various other parameters at each method call site. If an `FCPath` instance is created
|
|
111
|
+
without an explicitly-associated `FileCache`, then the default `FileCache()` is used,
|
|
112
|
+
which specifies a shared cache named `"global"` that will persist after the program exits.
|
|
111
113
|
|
|
112
114
|
Compare this example to the one above:
|
|
113
115
|
|
|
114
116
|
```python
|
|
115
|
-
from filecache import FileCache
|
|
116
|
-
with
|
|
117
|
+
from filecache import FileCache, FCPath
|
|
118
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
119
|
+
with FileCache(None) as fc: # Use as context manager
|
|
117
120
|
# Use GS by specifying the bucket name and one directory level
|
|
118
|
-
|
|
121
|
+
p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
|
|
119
122
|
# Use S3 by specifying the bucket name and two directory levels
|
|
120
|
-
|
|
123
|
+
# Alternative creation method
|
|
124
|
+
p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
|
|
125
|
+
anonymous=True)
|
|
121
126
|
# Access GS using a directory + filename (since only one directory level
|
|
122
|
-
# was specified by the
|
|
127
|
+
# was specified by the FCPath)
|
|
128
|
+
# The additional directory and filename are specified as an argument to open()
|
|
123
129
|
# Also use open() as a context manager
|
|
124
|
-
with
|
|
130
|
+
with p1.open('subdir2a/binary1.bin', 'rb') as fp:
|
|
125
131
|
bin1 = fp.read()
|
|
126
132
|
# Access S3 using a filename only (since two directory levels were already
|
|
127
|
-
# specified by the
|
|
128
|
-
|
|
133
|
+
# specified by the FCPath)
|
|
134
|
+
# The additional filename is specified by using the / operator to create a new
|
|
135
|
+
# FCPath instance; anonymous=True is inherited
|
|
136
|
+
with (p2 / 'binary1.bin').open(mode='rb') as fp:
|
|
129
137
|
bin2 = fp.read()
|
|
130
138
|
assert bin1 == bin2
|
|
131
139
|
# Cache automatically deleted here
|
|
@@ -154,28 +162,26 @@ Then the program could be written as:
|
|
|
154
162
|
```python
|
|
155
163
|
from filecache import FileCache
|
|
156
164
|
import os
|
|
157
|
-
with FileCache() as fc:
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
165
|
+
with FileCache(None) as fc:
|
|
166
|
+
p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
|
|
167
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
168
|
+
contents = voldesc_path.read_text()
|
|
161
169
|
# Cache automatically deleted here
|
|
162
170
|
```
|
|
163
171
|
|
|
164
172
|
If the program was going to be run multiple times in a row, or multiple copies were going
|
|
165
|
-
to be run simultaneously,
|
|
166
|
-
|
|
167
|
-
|
|
173
|
+
to be run simultaneously, using a shared cache would allow all of the processes to share
|
|
174
|
+
the same copy, thus requiring only a single download no matter how many times the program
|
|
175
|
+
was run. A shared cache is indicated by giving the cache a name (or no argument, which
|
|
176
|
+
defaults to ``"global"``); also `FCPath` defaults to using the global cache if no
|
|
177
|
+
`FileCache` is specified. This results in the simplest form of the program:
|
|
168
178
|
|
|
169
179
|
```python
|
|
170
|
-
from filecache import
|
|
180
|
+
from filecache import FCPath
|
|
171
181
|
import os
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
contents = fp.read()
|
|
176
|
-
# Cache not deleted here; must be deleted manually using fc.clean_up(final=True)
|
|
177
|
-
# If not deleted manually, the shared cache will persist until the temporary
|
|
178
|
-
# directory is purged by the operating system (which may be never)
|
|
182
|
+
p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
|
|
183
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
184
|
+
contents = voldesc_path.read_text()
|
|
179
185
|
```
|
|
180
186
|
|
|
181
187
|
Finally, there are four classes that allow direct access to the four possible storage
|
|
@@ -183,9 +189,9 @@ locations without invoking any caching behavior: :class:`FileCacheSourceLocal`,
|
|
|
183
189
|
:class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`:
|
|
184
190
|
|
|
185
191
|
```python
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
192
|
+
from filecache import FileCacheSourceGS
|
|
193
|
+
src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
194
|
+
src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
|
|
189
195
|
```
|
|
190
196
|
|
|
191
197
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -24,6 +24,7 @@ extensions = ['myst_parser', 'sphinx.ext.autodoc', 'sphinx.ext.napoleon',
|
|
|
24
24
|
templates_path = ['_templates']
|
|
25
25
|
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
|
|
26
26
|
|
|
27
|
+
autodoc_typehints = 'description'
|
|
27
28
|
|
|
28
29
|
# -- Options for HTML output -------------------------------------------------
|
|
29
30
|
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
``filecache`` Module
|
|
2
|
+
====================
|
|
3
|
+
|
|
4
|
+
.. automodule:: filecache.file_cache
|
|
5
|
+
:member-order: bysource
|
|
6
|
+
:members:
|
|
7
|
+
:undoc-members:
|
|
8
|
+
:special-members:
|
|
9
|
+
:show-inheritance:
|
|
10
|
+
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
|
|
11
|
+
|
|
12
|
+
.. automodule:: filecache.file_cache_path
|
|
13
|
+
:member-order: bysource
|
|
14
|
+
:members:
|
|
15
|
+
:undoc-members:
|
|
16
|
+
:special-members:
|
|
17
|
+
:show-inheritance:
|
|
18
|
+
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
|
|
19
|
+
|
|
20
|
+
.. automodule:: filecache.file_cache_source
|
|
21
|
+
:member-order: bysource
|
|
22
|
+
:members:
|
|
23
|
+
:undoc-members:
|
|
24
|
+
:special-members:
|
|
25
|
+
:show-inheritance:
|
|
26
|
+
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
##########################################################################################
|
|
2
|
+
# filecache/__init__.py
|
|
3
|
+
##########################################################################################
|
|
4
|
+
|
|
5
|
+
"""The purpose of the `filecache` module is to abstract away the location where files
|
|
6
|
+
used or generated by a program are stored. Files can be on the local file system, in
|
|
7
|
+
Google Cloud Storage, on Amazon Web Services S3, or on a webserver. When files to be read
|
|
8
|
+
are on the local file system, they are simply accessed in-place. Otherwise, they are
|
|
9
|
+
downloaded from the remote source to a local temporary directory. When files to be written
|
|
10
|
+
are on the local file system, they are simply written in-place. Otherwise, they are
|
|
11
|
+
written to a local temporary directory and then uploaded to the remote location (it is not
|
|
12
|
+
possible to upload to a webserver). When a cache is no longer needed, it is deleted from
|
|
13
|
+
the local disk.
|
|
14
|
+
|
|
15
|
+
The top-level file organization is provided by the :class:`FileCache` class. A
|
|
16
|
+
:class:`FileCache` instance is used to specify a particular **sharing policy** and
|
|
17
|
+
**lifetime**. For example, a cache could be private to the current process and group a set
|
|
18
|
+
of files that all have the same basic purpose. Once these files have been (downloaded and)
|
|
19
|
+
read, they are deleted as a group. Another cache could be shared among all processes on
|
|
20
|
+
the current machine and group a set of files that are needed by multiple processes, thus
|
|
21
|
+
allowing them to be downloaded from a remote source only one time, saving time and
|
|
22
|
+
bandwidth.
|
|
23
|
+
|
|
24
|
+
A :class:`FileCache` can be instantiated either directly or as a context manager. When
|
|
25
|
+
instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
|
|
26
|
+
directly to delete the cache when finished (a non-shared cache will be automatically
|
|
27
|
+
deleted on program exit). When instantiated as a context manager, a non-shared cache is
|
|
28
|
+
deleted on exit from the context. See the class documentation for full details.
|
|
29
|
+
|
|
30
|
+
Usage examples::
|
|
31
|
+
|
|
32
|
+
from filecache import FileCache
|
|
33
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
34
|
+
with FileCache(None) as fc: # Use as context manager
|
|
35
|
+
# Also use open() as a context manager
|
|
36
|
+
with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
|
|
37
|
+
anonymous=True) as fp:
|
|
38
|
+
bin1 = fp.read()
|
|
39
|
+
with fc.open('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
|
|
40
|
+
anonymous=True) as fp:
|
|
41
|
+
bin2 = fp.read()
|
|
42
|
+
assert bin1 == bin2
|
|
43
|
+
# Cache automatically deleted here
|
|
44
|
+
|
|
45
|
+
fc = FileCache(None) # Use without context manager
|
|
46
|
+
# Also retrieve file without using open context manager
|
|
47
|
+
path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
48
|
+
anonymous=True)
|
|
49
|
+
with open(path1, 'rb') as fp:
|
|
50
|
+
bin1 = fp.read()
|
|
51
|
+
path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
52
|
+
anonymous=True)
|
|
53
|
+
with open(path2, 'rb') as fp:
|
|
54
|
+
bin2 = fp.read()
|
|
55
|
+
fc.delete_cache() # Cache manually deleted here
|
|
56
|
+
assert bin1 == bin2
|
|
57
|
+
|
|
58
|
+
# Write a file to a bucket and read it back
|
|
59
|
+
with FileCache(None) as fc:
|
|
60
|
+
with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
|
|
61
|
+
fp.write('A')
|
|
62
|
+
# The cache will be deleted here so the file will have to be downloaded
|
|
63
|
+
with FileCache(None) as fc:
|
|
64
|
+
with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
|
|
65
|
+
print(fp.read())
|
|
66
|
+
|
|
67
|
+
The :class:`FCPath` class is a reimplementation of the Python `Path` class to support
|
|
68
|
+
remote acess using an associated :class:`FileCache`. Like `Path`, an :class:`FCPath`
|
|
69
|
+
instance can contain any part of a URI, but only an absolute URI can be used when actually
|
|
70
|
+
accessing the file specified by the :class:`FCPath`. In addition, an :class:`FCPath` can
|
|
71
|
+
encapsulate various arguments such as `anonymous` and `time_out` so that they do not need
|
|
72
|
+
to be specified to each access method. Thus, using this class can simplify the use of a
|
|
73
|
+
:class:`FileCache` by allowing the user to operate on paths using the simpler syntax
|
|
74
|
+
provided by `Path`, and to not specify various other parameters at each method call site.
|
|
75
|
+
If an :class:`FCPath` instance is created without an explicitly-associated
|
|
76
|
+
:class:`FileCache`, then the default `FileCache()` is used, which specifies a shared cache
|
|
77
|
+
named `"global"` that will persist after the program exits.
|
|
78
|
+
|
|
79
|
+
Compare this example to the one above::
|
|
80
|
+
|
|
81
|
+
from filecache import FileCache, FCPath
|
|
82
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
83
|
+
with FileCache(None) as fc: # Use as context manager
|
|
84
|
+
# Use GS by specifying the bucket name and one directory level
|
|
85
|
+
p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
|
|
86
|
+
# Use S3 by specifying the bucket name and two directory levels
|
|
87
|
+
# Alternative creation method
|
|
88
|
+
p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
|
|
89
|
+
anonymous=True)
|
|
90
|
+
# Access GS using a directory + filename (since only one directory level
|
|
91
|
+
# was specified by the FCPath)
|
|
92
|
+
# The additional directory and filename are specified as an argument to open()
|
|
93
|
+
# Also use open() as a context manager
|
|
94
|
+
with p1.open('subdir2a/binary1.bin', 'rb') as fp:
|
|
95
|
+
bin1 = fp.read()
|
|
96
|
+
# Access S3 using a filename only (since two directory levels were already
|
|
97
|
+
# specified by the FCPath)
|
|
98
|
+
# The additional filename is specified by using the / operator to create a new
|
|
99
|
+
# FCPath instance; anonymous=True is inherited
|
|
100
|
+
with (p2 / 'binary1.bin').open(mode='rb') as fp:
|
|
101
|
+
bin2 = fp.read()
|
|
102
|
+
assert bin1 == bin2
|
|
103
|
+
# Cache automatically deleted here
|
|
104
|
+
|
|
105
|
+
A benefit of the abstraction is that different environments can access the same files in
|
|
106
|
+
different ways without needing to change the program code. For example, consider a program
|
|
107
|
+
that needs to access the file ``COISS_2xxx/COISS_2001/voldesc.cat`` from the NASA PDS
|
|
108
|
+
archives. This file might be stored on the local disk in the user's home directory in a
|
|
109
|
+
subdirectory called ``pds3-holdings``. Or if the user does not have a local copy, it is
|
|
110
|
+
accessible from a webserver at
|
|
111
|
+
``https://pds-rings.seti.org/holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``.
|
|
112
|
+
Finally, it could be accessible from Google Cloud Storage from the ``rms-node-holdings``
|
|
113
|
+
bucket at
|
|
114
|
+
``gs://rms-node-holdings/pds3-holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``. Before
|
|
115
|
+
running the program, an environment variable could be set to one of these values::
|
|
116
|
+
|
|
117
|
+
$ export PDS3_HOLDINGS_SRC="~/pds3-holdings"
|
|
118
|
+
$ export PDS3_HOLDINGS_SRC="https://pds-rings.seti.org/holdings"
|
|
119
|
+
$ export PDS3_HOLDINGS_SRC="gs://rms-node-holdings/pds3-holdings"
|
|
120
|
+
|
|
121
|
+
Then the program could be written as::
|
|
122
|
+
|
|
123
|
+
from filecache import FileCache
|
|
124
|
+
import os
|
|
125
|
+
with FileCache(None) as fc:
|
|
126
|
+
p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
|
|
127
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
128
|
+
contents = voldesc_path.read_text()
|
|
129
|
+
# Cache automatically deleted here
|
|
130
|
+
|
|
131
|
+
If the program was going to be run multiple times in a row, or multiple copies were going
|
|
132
|
+
to be run simultaneously, using a shared cache would allow all of the processes to share
|
|
133
|
+
the same copy, thus requiring only a single download no matter how many times the program
|
|
134
|
+
was run. A shared cache is indicated by giving the cache a name (or no argument, which
|
|
135
|
+
defaults to ``"global"``); also :class:`FCPath` defaults to using the global cache if no
|
|
136
|
+
:class:`FileCache` is specified. This results in the simplest form of the program:
|
|
137
|
+
|
|
138
|
+
from filecache import FCPath
|
|
139
|
+
import os
|
|
140
|
+
p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
|
|
141
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
142
|
+
contents = voldesc_path.read_text()
|
|
143
|
+
|
|
144
|
+
Finally, there are four classes that allow direct access to the four possible storage
|
|
145
|
+
locations without invoking any caching behavior: :class:`FileCacheSourceFile`,
|
|
146
|
+
:class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`::
|
|
147
|
+
|
|
148
|
+
from filecache import FileCacheSourceGS
|
|
149
|
+
src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
150
|
+
src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
|
|
151
|
+
"""
|
|
152
|
+
|
|
153
|
+
import typing
|
|
154
|
+
|
|
155
|
+
if not typing.TYPE_CHECKING: # pragma: no cover
|
|
156
|
+
try:
|
|
157
|
+
from ._version import __version__
|
|
158
|
+
except ImportError: # pragma: no cover - only present when building a package
|
|
159
|
+
__version__ = 'Version unspecified'
|
|
160
|
+
|
|
161
|
+
from .file_cache import (get_global_logger, # noqa: ignore E401
|
|
162
|
+
register_filecachesource,
|
|
163
|
+
set_easy_logger,
|
|
164
|
+
set_global_logger,
|
|
165
|
+
FileCache)
|
|
166
|
+
from .file_cache_path import FCPath # noqa: ignore E401
|
|
167
|
+
from .file_cache_source import (FileCacheSource, # noqa: ignore E401
|
|
168
|
+
FileCacheSourceFile,
|
|
169
|
+
FileCacheSourceHTTP,
|
|
170
|
+
FileCacheSourceGS,
|
|
171
|
+
FileCacheSourceS3)
|
|
172
|
+
|
|
173
|
+
__all__ = ['get_global_logger',
|
|
174
|
+
'register_filecachesource',
|
|
175
|
+
'set_easy_logger',
|
|
176
|
+
'set_global_logger',
|
|
177
|
+
'FileCache',
|
|
178
|
+
'FCPath',
|
|
179
|
+
'FileCacheSource',
|
|
180
|
+
'FileCacheSourceFile',
|
|
181
|
+
'FileCacheSourceHTTP',
|
|
182
|
+
'FileCacheSourceGS',
|
|
183
|
+
'FileCacheSourceS3']
|