rms-filecache 1.2.4__tar.gz → 2.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.coveragerc +1 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.flake8 +3 -1
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.github/workflows/run-tests.yml +14 -2
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/PKG-INFO +61 -55
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/README.md +60 -54
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/docs/conf.py +1 -0
- rms_filecache-2.0.1/docs/module.rst +26 -0
- rms_filecache-2.0.1/filecache/__init__.py +183 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/filecache/_version.py +2 -2
- rms_filecache-2.0.1/filecache/file_cache.py +1445 -0
- rms_filecache-2.0.1/filecache/file_cache_path.py +1267 -0
- rms_filecache-2.0.1/filecache/file_cache_source.py +752 -0
- rms_filecache-2.0.1/filecache/file_cache_types.py +21 -0
- rms_filecache-2.0.1/filecache/my_glob.py +639 -0
- rms_filecache-2.0.1/mypy.ini +3 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/requirements.txt +5 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/PKG-INFO +61 -55
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/SOURCES.txt +11 -1
- rms_filecache-2.0.1/tests/__init__.py +0 -0
- rms_filecache-1.2.4/tests/test_filecache.py → rms_filecache-2.0.1/tests/test_file_cache.py +541 -416
- rms_filecache-2.0.1/tests/test_file_cache_path.py +716 -0
- rms_filecache-2.0.1/tests/test_file_cache_source.py +73 -0
- rms_filecache-2.0.1/tests/test_url_to_path.py +306 -0
- rms_filecache-1.2.4/docs/module.rst +0 -10
- rms_filecache-1.2.4/filecache/__init__.py +0 -2027
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.github/pull_request_template.md +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.github/workflows/publish_to_pypi.yml +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.github/workflows/publish_to_test_pypi.yml +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.gitignore +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.readthedocs.yaml +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/LICENSE +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/docs/Makefile +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/docs/index.rst +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/docs/make.bat +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/pyproject.toml +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/setup.cfg +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_programs/locking.py +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_programs/multi_download.py +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/s3_bucket_policy_write.txt +0 -0
- {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/s3_lifecycle_policy.txt +0 -0
|
@@ -12,7 +12,7 @@ on:
|
|
|
12
12
|
|
|
13
13
|
jobs:
|
|
14
14
|
flake8:
|
|
15
|
-
name:
|
|
15
|
+
name: Lint filecache
|
|
16
16
|
runs-on: ubuntu-latest
|
|
17
17
|
steps:
|
|
18
18
|
- name: Checkout
|
|
@@ -20,11 +20,23 @@ jobs:
|
|
|
20
20
|
with:
|
|
21
21
|
ref: ${{ github.event.pull_request.head.sha }}
|
|
22
22
|
|
|
23
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
24
|
+
uses: actions/setup-python@v5
|
|
25
|
+
with:
|
|
26
|
+
python-version: ${{ matrix.python-version }}
|
|
27
|
+
|
|
28
|
+
- name: Install dependencies
|
|
29
|
+
run: |
|
|
30
|
+
python -m pip install -r requirements.txt
|
|
31
|
+
|
|
23
32
|
- name: Flake8
|
|
24
33
|
run: |
|
|
25
|
-
pip install flake8
|
|
26
34
|
flake8 filecache tests
|
|
27
35
|
|
|
36
|
+
- name: Mypy
|
|
37
|
+
run: |
|
|
38
|
+
mypy filecache
|
|
39
|
+
|
|
28
40
|
test:
|
|
29
41
|
name: Test filecache
|
|
30
42
|
runs-on: ${{ matrix.os }}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: rms-filecache
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.1
|
|
4
4
|
Summary: File cache for files retrieved from the cloud
|
|
5
5
|
Maintainer-email: "Robert S. French" <rfrench@seti.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -79,28 +79,26 @@ pip install rms-filecache
|
|
|
79
79
|
|
|
80
80
|
# Getting Started
|
|
81
81
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
allowing them to be downloaded from a remote source only one time, saving time and
|
|
90
|
-
bandwidth.
|
|
82
|
+
The top-level file organization is provided by the `FileCache` class. A `FileCache`
|
|
83
|
+
instance is used to specify a particular **sharing policy** and **lifetime**. For example,
|
|
84
|
+
a cache could be private to the current process and group a set of files that all have the
|
|
85
|
+
same basic purpose. Once these files have been (downloaded and) read, they are deleted as
|
|
86
|
+
a group. Another cache could be shared among all processes on the current machine and
|
|
87
|
+
group a set of files that are needed by multiple processes, thus allowing them to be
|
|
88
|
+
downloaded from a remote source only one time, saving time and bandwidth.
|
|
91
89
|
|
|
92
90
|
A `FileCache` can be instantiated either directly or as a context manager. When
|
|
93
|
-
instantiated directly, the programmer is responsible for calling
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
full details.
|
|
91
|
+
instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
|
|
92
|
+
directly to delete the cache when finished (a non-shared cache will be automatically
|
|
93
|
+
deleted on program exit). When instantiated as a context manager, a non-shared cache is
|
|
94
|
+
deleted on exit from the context. See the class documentation for full details.
|
|
98
95
|
|
|
99
96
|
Usage examples:
|
|
100
97
|
|
|
101
98
|
```python
|
|
102
99
|
from filecache import FileCache
|
|
103
|
-
with
|
|
100
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
101
|
+
with FileCache(None) as fc: # Use as context manager
|
|
104
102
|
# Also use open() as a context manager
|
|
105
103
|
with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
|
|
106
104
|
anonymous=True) as fp:
|
|
@@ -111,7 +109,7 @@ with FileCache() as fc: # Use as context manager
|
|
|
111
109
|
assert bin1 == bin2
|
|
112
110
|
# Cache automatically deleted here
|
|
113
111
|
|
|
114
|
-
fc = FileCache() # Use without context manager
|
|
112
|
+
fc = FileCache(None) # Use without context manager
|
|
115
113
|
# Also retrieve file without using open context manager
|
|
116
114
|
path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
117
115
|
anonymous=True)
|
|
@@ -121,43 +119,53 @@ path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
|
121
119
|
anonymous=True)
|
|
122
120
|
with open(path2, 'rb') as fp:
|
|
123
121
|
bin2 = fp.read()
|
|
124
|
-
fc.
|
|
122
|
+
fc.delete_cache() # Cache manually deleted here
|
|
125
123
|
assert bin1 == bin2
|
|
126
124
|
|
|
127
125
|
# Write a file to a bucket and read it back
|
|
128
|
-
with FileCache() as fc:
|
|
126
|
+
with FileCache(None) as fc:
|
|
129
127
|
with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
|
|
130
128
|
fp.write('A')
|
|
131
129
|
# The cache will be deleted here so the file will have to be downloaded
|
|
132
|
-
with FileCache() as fc:
|
|
130
|
+
with FileCache(None) as fc:
|
|
133
131
|
with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
|
|
134
132
|
print(fp.read())
|
|
135
133
|
```
|
|
136
134
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
135
|
+
The `FCPath` class is a reimplementation of the Python `Path` class to support remote
|
|
136
|
+
acess using an associated `FileCache`. Like `Path`, an `FCPath` instance can contain any
|
|
137
|
+
part of a URI, but only an absolute URI can be used when actually accessing the file
|
|
138
|
+
specified by the `FCPath`. In addition, an `FCPath` can encapsulate various arguments such
|
|
139
|
+
as `anonymous` and `time_out` so that they do not need to be specified to each access
|
|
140
|
+
method. Thus, using this class can simplify the use of a `FileCache` by allowing the user
|
|
141
|
+
to operate on paths using the simpler syntax provided by `Path`, and to not specify
|
|
142
|
+
various other parameters at each method call site. If an `FCPath` instance is created
|
|
143
|
+
without an explicitly-associated `FileCache`, then the default `FileCache()` is used,
|
|
144
|
+
which specifies a shared cache named `"global"` that will persist after the program exits.
|
|
143
145
|
|
|
144
146
|
Compare this example to the one above:
|
|
145
147
|
|
|
146
148
|
```python
|
|
147
|
-
from filecache import FileCache
|
|
148
|
-
with
|
|
149
|
+
from filecache import FileCache, FCPath
|
|
150
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
151
|
+
with FileCache(None) as fc: # Use as context manager
|
|
149
152
|
# Use GS by specifying the bucket name and one directory level
|
|
150
|
-
|
|
153
|
+
p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
|
|
151
154
|
# Use S3 by specifying the bucket name and two directory levels
|
|
152
|
-
|
|
155
|
+
# Alternative creation method
|
|
156
|
+
p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
|
|
157
|
+
anonymous=True)
|
|
153
158
|
# Access GS using a directory + filename (since only one directory level
|
|
154
|
-
# was specified by the
|
|
159
|
+
# was specified by the FCPath)
|
|
160
|
+
# The additional directory and filename are specified as an argument to open()
|
|
155
161
|
# Also use open() as a context manager
|
|
156
|
-
with
|
|
162
|
+
with p1.open('subdir2a/binary1.bin', 'rb') as fp:
|
|
157
163
|
bin1 = fp.read()
|
|
158
164
|
# Access S3 using a filename only (since two directory levels were already
|
|
159
|
-
# specified by the
|
|
160
|
-
|
|
165
|
+
# specified by the FCPath)
|
|
166
|
+
# The additional filename is specified by using the / operator to create a new
|
|
167
|
+
# FCPath instance; anonymous=True is inherited
|
|
168
|
+
with (p2 / 'binary1.bin').open(mode='rb') as fp:
|
|
161
169
|
bin2 = fp.read()
|
|
162
170
|
assert bin1 == bin2
|
|
163
171
|
# Cache automatically deleted here
|
|
@@ -173,7 +181,7 @@ accessible from a webserver at
|
|
|
173
181
|
Finally, it could be accessible from Google Cloud Storage from the ``rms-node-holdings``
|
|
174
182
|
bucket at
|
|
175
183
|
``gs://rms-node-holdings/pds3-holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``. Before
|
|
176
|
-
running the program, an environment variable could be set to one of these values
|
|
184
|
+
running the program, an environment variable could be set to one of these values:
|
|
177
185
|
|
|
178
186
|
```sh
|
|
179
187
|
$ export PDS3_HOLDINGS_SRC="~/pds3-holdings"
|
|
@@ -186,38 +194,36 @@ Then the program could be written as:
|
|
|
186
194
|
```python
|
|
187
195
|
from filecache import FileCache
|
|
188
196
|
import os
|
|
189
|
-
with FileCache() as fc:
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
197
|
+
with FileCache(None) as fc:
|
|
198
|
+
p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
|
|
199
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
200
|
+
contents = voldesc_path.read_text()
|
|
193
201
|
# Cache automatically deleted here
|
|
194
202
|
```
|
|
195
203
|
|
|
196
204
|
If the program was going to be run multiple times in a row, or multiple copies were going
|
|
197
|
-
to be run simultaneously,
|
|
198
|
-
|
|
199
|
-
|
|
205
|
+
to be run simultaneously, using a shared cache would allow all of the processes to share
|
|
206
|
+
the same copy, thus requiring only a single download no matter how many times the program
|
|
207
|
+
was run. A shared cache is indicated by giving the cache a name (or no argument, which
|
|
208
|
+
defaults to ``"global"``); also `FCPath` defaults to using the global cache if no
|
|
209
|
+
`FileCache` is specified. This results in the simplest form of the program:
|
|
200
210
|
|
|
201
211
|
```python
|
|
202
|
-
from filecache import
|
|
212
|
+
from filecache import FCPath
|
|
203
213
|
import os
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
contents = fp.read()
|
|
208
|
-
# Cache not deleted here; must be deleted manually using fc.clean_up(final=True)
|
|
209
|
-
# If not deleted manually, the shared cache will persist until the temporary
|
|
210
|
-
# directory is purged by the operating system (which may be never)
|
|
214
|
+
p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
|
|
215
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
216
|
+
contents = voldesc_path.read_text()
|
|
211
217
|
```
|
|
212
218
|
|
|
213
219
|
Finally, there are four classes that allow direct access to the four possible storage
|
|
214
|
-
locations without invoking any caching behavior:
|
|
215
|
-
|
|
220
|
+
locations without invoking any caching behavior: `FileCacheSourceLocal`,
|
|
221
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
|
|
216
222
|
|
|
217
223
|
```python
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
224
|
+
from filecache import FileCacheSourceGS
|
|
225
|
+
src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
226
|
+
src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
|
|
221
227
|
```
|
|
222
228
|
|
|
223
229
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -47,28 +47,26 @@ pip install rms-filecache
|
|
|
47
47
|
|
|
48
48
|
# Getting Started
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
allowing them to be downloaded from a remote source only one time, saving time and
|
|
58
|
-
bandwidth.
|
|
50
|
+
The top-level file organization is provided by the `FileCache` class. A `FileCache`
|
|
51
|
+
instance is used to specify a particular **sharing policy** and **lifetime**. For example,
|
|
52
|
+
a cache could be private to the current process and group a set of files that all have the
|
|
53
|
+
same basic purpose. Once these files have been (downloaded and) read, they are deleted as
|
|
54
|
+
a group. Another cache could be shared among all processes on the current machine and
|
|
55
|
+
group a set of files that are needed by multiple processes, thus allowing them to be
|
|
56
|
+
downloaded from a remote source only one time, saving time and bandwidth.
|
|
59
57
|
|
|
60
58
|
A `FileCache` can be instantiated either directly or as a context manager. When
|
|
61
|
-
instantiated directly, the programmer is responsible for calling
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
full details.
|
|
59
|
+
instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
|
|
60
|
+
directly to delete the cache when finished (a non-shared cache will be automatically
|
|
61
|
+
deleted on program exit). When instantiated as a context manager, a non-shared cache is
|
|
62
|
+
deleted on exit from the context. See the class documentation for full details.
|
|
66
63
|
|
|
67
64
|
Usage examples:
|
|
68
65
|
|
|
69
66
|
```python
|
|
70
67
|
from filecache import FileCache
|
|
71
|
-
with
|
|
68
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
69
|
+
with FileCache(None) as fc: # Use as context manager
|
|
72
70
|
# Also use open() as a context manager
|
|
73
71
|
with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
|
|
74
72
|
anonymous=True) as fp:
|
|
@@ -79,7 +77,7 @@ with FileCache() as fc: # Use as context manager
|
|
|
79
77
|
assert bin1 == bin2
|
|
80
78
|
# Cache automatically deleted here
|
|
81
79
|
|
|
82
|
-
fc = FileCache() # Use without context manager
|
|
80
|
+
fc = FileCache(None) # Use without context manager
|
|
83
81
|
# Also retrieve file without using open context manager
|
|
84
82
|
path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
85
83
|
anonymous=True)
|
|
@@ -89,43 +87,53 @@ path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
|
|
|
89
87
|
anonymous=True)
|
|
90
88
|
with open(path2, 'rb') as fp:
|
|
91
89
|
bin2 = fp.read()
|
|
92
|
-
fc.
|
|
90
|
+
fc.delete_cache() # Cache manually deleted here
|
|
93
91
|
assert bin1 == bin2
|
|
94
92
|
|
|
95
93
|
# Write a file to a bucket and read it back
|
|
96
|
-
with FileCache() as fc:
|
|
94
|
+
with FileCache(None) as fc:
|
|
97
95
|
with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
|
|
98
96
|
fp.write('A')
|
|
99
97
|
# The cache will be deleted here so the file will have to be downloaded
|
|
100
|
-
with FileCache() as fc:
|
|
98
|
+
with FileCache(None) as fc:
|
|
101
99
|
with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
|
|
102
100
|
print(fp.read())
|
|
103
101
|
```
|
|
104
102
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
103
|
+
The `FCPath` class is a reimplementation of the Python `Path` class to support remote
|
|
104
|
+
acess using an associated `FileCache`. Like `Path`, an `FCPath` instance can contain any
|
|
105
|
+
part of a URI, but only an absolute URI can be used when actually accessing the file
|
|
106
|
+
specified by the `FCPath`. In addition, an `FCPath` can encapsulate various arguments such
|
|
107
|
+
as `anonymous` and `time_out` so that they do not need to be specified to each access
|
|
108
|
+
method. Thus, using this class can simplify the use of a `FileCache` by allowing the user
|
|
109
|
+
to operate on paths using the simpler syntax provided by `Path`, and to not specify
|
|
110
|
+
various other parameters at each method call site. If an `FCPath` instance is created
|
|
111
|
+
without an explicitly-associated `FileCache`, then the default `FileCache()` is used,
|
|
112
|
+
which specifies a shared cache named `"global"` that will persist after the program exits.
|
|
111
113
|
|
|
112
114
|
Compare this example to the one above:
|
|
113
115
|
|
|
114
116
|
```python
|
|
115
|
-
from filecache import FileCache
|
|
116
|
-
with
|
|
117
|
+
from filecache import FileCache, FCPath
|
|
118
|
+
# Create a cache with a unique name that will be deleted on exit
|
|
119
|
+
with FileCache(None) as fc: # Use as context manager
|
|
117
120
|
# Use GS by specifying the bucket name and one directory level
|
|
118
|
-
|
|
121
|
+
p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
|
|
119
122
|
# Use S3 by specifying the bucket name and two directory levels
|
|
120
|
-
|
|
123
|
+
# Alternative creation method
|
|
124
|
+
p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
|
|
125
|
+
anonymous=True)
|
|
121
126
|
# Access GS using a directory + filename (since only one directory level
|
|
122
|
-
# was specified by the
|
|
127
|
+
# was specified by the FCPath)
|
|
128
|
+
# The additional directory and filename are specified as an argument to open()
|
|
123
129
|
# Also use open() as a context manager
|
|
124
|
-
with
|
|
130
|
+
with p1.open('subdir2a/binary1.bin', 'rb') as fp:
|
|
125
131
|
bin1 = fp.read()
|
|
126
132
|
# Access S3 using a filename only (since two directory levels were already
|
|
127
|
-
# specified by the
|
|
128
|
-
|
|
133
|
+
# specified by the FCPath)
|
|
134
|
+
# The additional filename is specified by using the / operator to create a new
|
|
135
|
+
# FCPath instance; anonymous=True is inherited
|
|
136
|
+
with (p2 / 'binary1.bin').open(mode='rb') as fp:
|
|
129
137
|
bin2 = fp.read()
|
|
130
138
|
assert bin1 == bin2
|
|
131
139
|
# Cache automatically deleted here
|
|
@@ -141,7 +149,7 @@ accessible from a webserver at
|
|
|
141
149
|
Finally, it could be accessible from Google Cloud Storage from the ``rms-node-holdings``
|
|
142
150
|
bucket at
|
|
143
151
|
``gs://rms-node-holdings/pds3-holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``. Before
|
|
144
|
-
running the program, an environment variable could be set to one of these values
|
|
152
|
+
running the program, an environment variable could be set to one of these values:
|
|
145
153
|
|
|
146
154
|
```sh
|
|
147
155
|
$ export PDS3_HOLDINGS_SRC="~/pds3-holdings"
|
|
@@ -154,38 +162,36 @@ Then the program could be written as:
|
|
|
154
162
|
```python
|
|
155
163
|
from filecache import FileCache
|
|
156
164
|
import os
|
|
157
|
-
with FileCache() as fc:
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
165
|
+
with FileCache(None) as fc:
|
|
166
|
+
p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
|
|
167
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
168
|
+
contents = voldesc_path.read_text()
|
|
161
169
|
# Cache automatically deleted here
|
|
162
170
|
```
|
|
163
171
|
|
|
164
172
|
If the program was going to be run multiple times in a row, or multiple copies were going
|
|
165
|
-
to be run simultaneously,
|
|
166
|
-
|
|
167
|
-
|
|
173
|
+
to be run simultaneously, using a shared cache would allow all of the processes to share
|
|
174
|
+
the same copy, thus requiring only a single download no matter how many times the program
|
|
175
|
+
was run. A shared cache is indicated by giving the cache a name (or no argument, which
|
|
176
|
+
defaults to ``"global"``); also `FCPath` defaults to using the global cache if no
|
|
177
|
+
`FileCache` is specified. This results in the simplest form of the program:
|
|
168
178
|
|
|
169
179
|
```python
|
|
170
|
-
from filecache import
|
|
180
|
+
from filecache import FCPath
|
|
171
181
|
import os
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
contents = fp.read()
|
|
176
|
-
# Cache not deleted here; must be deleted manually using fc.clean_up(final=True)
|
|
177
|
-
# If not deleted manually, the shared cache will persist until the temporary
|
|
178
|
-
# directory is purged by the operating system (which may be never)
|
|
182
|
+
p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
|
|
183
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
184
|
+
contents = voldesc_path.read_text()
|
|
179
185
|
```
|
|
180
186
|
|
|
181
187
|
Finally, there are four classes that allow direct access to the four possible storage
|
|
182
|
-
locations without invoking any caching behavior:
|
|
183
|
-
|
|
188
|
+
locations without invoking any caching behavior: `FileCacheSourceLocal`,
|
|
189
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
|
|
184
190
|
|
|
185
191
|
```python
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
192
|
+
from filecache import FileCacheSourceGS
|
|
193
|
+
src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
194
|
+
src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
|
|
189
195
|
```
|
|
190
196
|
|
|
191
197
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -24,6 +24,7 @@ extensions = ['myst_parser', 'sphinx.ext.autodoc', 'sphinx.ext.napoleon',
|
|
|
24
24
|
templates_path = ['_templates']
|
|
25
25
|
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
|
|
26
26
|
|
|
27
|
+
autodoc_typehints = 'description'
|
|
27
28
|
|
|
28
29
|
# -- Options for HTML output -------------------------------------------------
|
|
29
30
|
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
``filecache`` Module
|
|
2
|
+
====================
|
|
3
|
+
|
|
4
|
+
.. automodule:: filecache.file_cache
|
|
5
|
+
:member-order: bysource
|
|
6
|
+
:members:
|
|
7
|
+
:undoc-members:
|
|
8
|
+
:special-members:
|
|
9
|
+
:show-inheritance:
|
|
10
|
+
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
|
|
11
|
+
|
|
12
|
+
.. automodule:: filecache.file_cache_path
|
|
13
|
+
:member-order: bysource
|
|
14
|
+
:members:
|
|
15
|
+
:undoc-members:
|
|
16
|
+
:special-members:
|
|
17
|
+
:show-inheritance:
|
|
18
|
+
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
|
|
19
|
+
|
|
20
|
+
.. automodule:: filecache.file_cache_source
|
|
21
|
+
:member-order: bysource
|
|
22
|
+
:members:
|
|
23
|
+
:undoc-members:
|
|
24
|
+
:special-members:
|
|
25
|
+
:show-inheritance:
|
|
26
|
+
:exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
|