rms-filecache 1.2.4__tar.gz → 2.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.coveragerc +1 -0
  2. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.flake8 +3 -1
  3. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.github/workflows/run-tests.yml +14 -2
  4. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/PKG-INFO +61 -55
  5. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/README.md +60 -54
  6. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/docs/conf.py +1 -0
  7. rms_filecache-2.0.1/docs/module.rst +26 -0
  8. rms_filecache-2.0.1/filecache/__init__.py +183 -0
  9. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/filecache/_version.py +2 -2
  10. rms_filecache-2.0.1/filecache/file_cache.py +1445 -0
  11. rms_filecache-2.0.1/filecache/file_cache_path.py +1267 -0
  12. rms_filecache-2.0.1/filecache/file_cache_source.py +752 -0
  13. rms_filecache-2.0.1/filecache/file_cache_types.py +21 -0
  14. rms_filecache-2.0.1/filecache/my_glob.py +639 -0
  15. rms_filecache-2.0.1/mypy.ini +3 -0
  16. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/requirements.txt +5 -0
  17. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/PKG-INFO +61 -55
  18. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/SOURCES.txt +11 -1
  19. rms_filecache-2.0.1/tests/__init__.py +0 -0
  20. rms_filecache-1.2.4/tests/test_filecache.py → rms_filecache-2.0.1/tests/test_file_cache.py +541 -416
  21. rms_filecache-2.0.1/tests/test_file_cache_path.py +716 -0
  22. rms_filecache-2.0.1/tests/test_file_cache_source.py +73 -0
  23. rms_filecache-2.0.1/tests/test_url_to_path.py +306 -0
  24. rms_filecache-1.2.4/docs/module.rst +0 -10
  25. rms_filecache-1.2.4/filecache/__init__.py +0 -2027
  26. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.github/pull_request_template.md +0 -0
  27. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.github/workflows/publish_to_pypi.yml +0 -0
  28. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.github/workflows/publish_to_test_pypi.yml +0 -0
  29. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.gitignore +0 -0
  30. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/.readthedocs.yaml +0 -0
  31. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/LICENSE +0 -0
  32. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/docs/Makefile +0 -0
  33. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/docs/index.rst +0 -0
  34. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/docs/make.bat +0 -0
  35. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/pyproject.toml +0 -0
  36. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/dependency_links.txt +0 -0
  37. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/requires.txt +0 -0
  38. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/rms_filecache.egg-info/top_level.txt +0 -0
  39. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/setup.cfg +0 -0
  40. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_files/expected/lorem1.txt +0 -0
  41. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_files/expected/subdir1/lorem1.txt +0 -0
  42. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
  43. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
  44. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_programs/locking.py +0 -0
  45. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/test_programs/multi_download.py +0 -0
  46. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/gs_lifecycle_config.json +0 -0
  47. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/gs_lifecycle_config_info.txt +0 -0
  48. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/s3_bucket_policy_read.txt +0 -0
  49. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/s3_bucket_policy_write.txt +0 -0
  50. {rms_filecache-1.2.4 → rms_filecache-2.0.1}/tests/s3_lifecycle_policy.txt +0 -0
@@ -3,3 +3,4 @@ branch = true
3
3
  omit =
4
4
  tests/*
5
5
  */_version.py
6
+ filecache/my_glob.py
@@ -2,4 +2,6 @@
2
2
  max-line-length: 90
3
3
  per-file-ignores =
4
4
  # E501 line too long
5
- test_filecache.py: E501
5
+ test_file_cache.py: E501
6
+ exclude =
7
+ filecache/my_glob.py
@@ -12,7 +12,7 @@ on:
12
12
 
13
13
  jobs:
14
14
  flake8:
15
- name: Flake8 filecache
15
+ name: Lint filecache
16
16
  runs-on: ubuntu-latest
17
17
  steps:
18
18
  - name: Checkout
@@ -20,11 +20,23 @@ jobs:
20
20
  with:
21
21
  ref: ${{ github.event.pull_request.head.sha }}
22
22
 
23
+ - name: Set up Python ${{ matrix.python-version }}
24
+ uses: actions/setup-python@v5
25
+ with:
26
+ python-version: ${{ matrix.python-version }}
27
+
28
+ - name: Install dependencies
29
+ run: |
30
+ python -m pip install -r requirements.txt
31
+
23
32
  - name: Flake8
24
33
  run: |
25
- pip install flake8
26
34
  flake8 filecache tests
27
35
 
36
+ - name: Mypy
37
+ run: |
38
+ mypy filecache
39
+
28
40
  test:
29
41
  name: Test filecache
30
42
  runs-on: ${{ matrix.os }}
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: rms-filecache
3
- Version: 1.2.4
3
+ Version: 2.0.1
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -79,28 +79,26 @@ pip install rms-filecache
79
79
 
80
80
  # Getting Started
81
81
 
82
-
83
- The top-level file organization is provided by the `FileCache` class. A
84
- `FileCache` instance is used to specify a particular **sharing policy** and
85
- **lifetime**. For example, a cache could be private to the current process and group a set
86
- of files that all have the same basic purpose. Once these files have been (downloaded and)
87
- read, they are deleted as a group. Another cache could be shared among all processes on
88
- the current machine and group a set of files that are needed by multiple processes, thus
89
- allowing them to be downloaded from a remote source only one time, saving time and
90
- bandwidth.
82
+ The top-level file organization is provided by the `FileCache` class. A `FileCache`
83
+ instance is used to specify a particular **sharing policy** and **lifetime**. For example,
84
+ a cache could be private to the current process and group a set of files that all have the
85
+ same basic purpose. Once these files have been (downloaded and) read, they are deleted as
86
+ a group. Another cache could be shared among all processes on the current machine and
87
+ group a set of files that are needed by multiple processes, thus allowing them to be
88
+ downloaded from a remote source only one time, saving time and bandwidth.
91
89
 
92
90
  A `FileCache` can be instantiated either directly or as a context manager. When
93
- instantiated directly, the programmer is responsible for calling
94
- `FileCache.clean_up` directly to delete the cache when finished. In addition, a
95
- non-shared cache will be deleted on program exit. When instantiated as a context manager,
96
- a non-shared cache is deleted on exit from the context. See the class documentation for
97
- full details.
91
+ instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
92
+ directly to delete the cache when finished (a non-shared cache will be automatically
93
+ deleted on program exit). When instantiated as a context manager, a non-shared cache is
94
+ deleted on exit from the context. See the class documentation for full details.
98
95
 
99
96
  Usage examples:
100
97
 
101
98
  ```python
102
99
  from filecache import FileCache
103
- with FileCache() as fc: # Use as context manager
100
+ # Create a cache with a unique name that will be deleted on exit
101
+ with FileCache(None) as fc: # Use as context manager
104
102
  # Also use open() as a context manager
105
103
  with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
106
104
  anonymous=True) as fp:
@@ -111,7 +109,7 @@ with FileCache() as fc: # Use as context manager
111
109
  assert bin1 == bin2
112
110
  # Cache automatically deleted here
113
111
 
114
- fc = FileCache() # Use without context manager
112
+ fc = FileCache(None) # Use without context manager
115
113
  # Also retrieve file without using open context manager
116
114
  path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
117
115
  anonymous=True)
@@ -121,43 +119,53 @@ path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
121
119
  anonymous=True)
122
120
  with open(path2, 'rb') as fp:
123
121
  bin2 = fp.read()
124
- fc.clean_up() # Cache manually deleted here
122
+ fc.delete_cache() # Cache manually deleted here
125
123
  assert bin1 == bin2
126
124
 
127
125
  # Write a file to a bucket and read it back
128
- with FileCache() as fc:
126
+ with FileCache(None) as fc:
129
127
  with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
130
128
  fp.write('A')
131
129
  # The cache will be deleted here so the file will have to be downloaded
132
- with FileCache() as fc:
130
+ with FileCache(None) as fc:
133
131
  with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
134
132
  print(fp.read())
135
133
  ```
136
134
 
137
- A `FileCachePrefix` instance can be used to encapsulate the storage prefix string,
138
- as well as any subdirectories, plus various arguments such as `anonymous` and `time_out`
139
- that can be specified to each `exists`, `retrieve`, or `upload` method. Thus using one
140
- of these instances can simplify the use of a `FileCache` by allowing the user to
141
- only specify the relative part of the path to be operated on, and to not specify various
142
- other parameters at each method call site.
135
+ The `FCPath` class is a reimplementation of the Python `Path` class to support remote
136
+ acess using an associated `FileCache`. Like `Path`, an `FCPath` instance can contain any
137
+ part of a URI, but only an absolute URI can be used when actually accessing the file
138
+ specified by the `FCPath`. In addition, an `FCPath` can encapsulate various arguments such
139
+ as `anonymous` and `time_out` so that they do not need to be specified to each access
140
+ method. Thus, using this class can simplify the use of a `FileCache` by allowing the user
141
+ to operate on paths using the simpler syntax provided by `Path`, and to not specify
142
+ various other parameters at each method call site. If an `FCPath` instance is created
143
+ without an explicitly-associated `FileCache`, then the default `FileCache()` is used,
144
+ which specifies a shared cache named `"global"` that will persist after the program exits.
143
145
 
144
146
  Compare this example to the one above:
145
147
 
146
148
  ```python
147
- from filecache import FileCache
148
- with FileCache() as fc: # Use as context manager
149
+ from filecache import FileCache, FCPath
150
+ # Create a cache with a unique name that will be deleted on exit
151
+ with FileCache(None) as fc: # Use as context manager
149
152
  # Use GS by specifying the bucket name and one directory level
150
- pfx1 = fc.new_prefix('gs://rms-filecache-tests/subdir1', anonymous=True)
153
+ p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
151
154
  # Use S3 by specifying the bucket name and two directory levels
152
- pfx2 = fc.new_prefix('s3://rms-filecache-tests/subdir1/subdir2a', anonymous=True)
155
+ # Alternative creation method
156
+ p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
157
+ anonymous=True)
153
158
  # Access GS using a directory + filename (since only one directory level
154
- # was specified by the prefix)
159
+ # was specified by the FCPath)
160
+ # The additional directory and filename are specified as an argument to open()
155
161
  # Also use open() as a context manager
156
- with pfx1.open('subdir2a/binary1.bin', 'rb') as fp:
162
+ with p1.open('subdir2a/binary1.bin', 'rb') as fp:
157
163
  bin1 = fp.read()
158
164
  # Access S3 using a filename only (since two directory levels were already
159
- # specified by the prefix))
160
- with pfx2.open('binary1.bin', 'rb') as fp:
165
+ # specified by the FCPath)
166
+ # The additional filename is specified by using the / operator to create a new
167
+ # FCPath instance; anonymous=True is inherited
168
+ with (p2 / 'binary1.bin').open(mode='rb') as fp:
161
169
  bin2 = fp.read()
162
170
  assert bin1 == bin2
163
171
  # Cache automatically deleted here
@@ -173,7 +181,7 @@ accessible from a webserver at
173
181
  Finally, it could be accessible from Google Cloud Storage from the ``rms-node-holdings``
174
182
  bucket at
175
183
  ``gs://rms-node-holdings/pds3-holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``. Before
176
- running the program, an environment variable could be set to one of these values::
184
+ running the program, an environment variable could be set to one of these values:
177
185
 
178
186
  ```sh
179
187
  $ export PDS3_HOLDINGS_SRC="~/pds3-holdings"
@@ -186,38 +194,36 @@ Then the program could be written as:
186
194
  ```python
187
195
  from filecache import FileCache
188
196
  import os
189
- with FileCache() as fc:
190
- pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_SRC'))
191
- with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
192
- contents = fp.read()
197
+ with FileCache(None) as fc:
198
+ p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
199
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
200
+ contents = voldesc_path.read_text()
193
201
  # Cache automatically deleted here
194
202
  ```
195
203
 
196
204
  If the program was going to be run multiple times in a row, or multiple copies were going
197
- to be run simultaneously, marking the cache as shared would allow all of the processes to
198
- share the same copy, thus requiring only a single download no matter how many times the
199
- program was run:
205
+ to be run simultaneously, using a shared cache would allow all of the processes to share
206
+ the same copy, thus requiring only a single download no matter how many times the program
207
+ was run. A shared cache is indicated by giving the cache a name (or no argument, which
208
+ defaults to ``"global"``); also `FCPath` defaults to using the global cache if no
209
+ `FileCache` is specified. This results in the simplest form of the program:
200
210
 
201
211
  ```python
202
- from filecache import FileCache
212
+ from filecache import FCPath
203
213
  import os
204
- with FileCache(shared=True) as fc:
205
- pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_DIR'))
206
- with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
207
- contents = fp.read()
208
- # Cache not deleted here; must be deleted manually using fc.clean_up(final=True)
209
- # If not deleted manually, the shared cache will persist until the temporary
210
- # directory is purged by the operating system (which may be never)
214
+ p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
215
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
216
+ contents = voldesc_path.read_text()
211
217
  ```
212
218
 
213
219
  Finally, there are four classes that allow direct access to the four possible storage
214
- locations without invoking any caching behavior: :class:`FileCacheSourceLocal`,
215
- :class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`:
220
+ locations without invoking any caching behavior: `FileCacheSourceLocal`,
221
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
216
222
 
217
223
  ```python
218
- from filecache import FileCacheSourceGS
219
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
220
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
224
+ from filecache import FileCacheSourceGS
225
+ src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
226
+ src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
221
227
  ```
222
228
 
223
229
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -47,28 +47,26 @@ pip install rms-filecache
47
47
 
48
48
  # Getting Started
49
49
 
50
-
51
- The top-level file organization is provided by the `FileCache` class. A
52
- `FileCache` instance is used to specify a particular **sharing policy** and
53
- **lifetime**. For example, a cache could be private to the current process and group a set
54
- of files that all have the same basic purpose. Once these files have been (downloaded and)
55
- read, they are deleted as a group. Another cache could be shared among all processes on
56
- the current machine and group a set of files that are needed by multiple processes, thus
57
- allowing them to be downloaded from a remote source only one time, saving time and
58
- bandwidth.
50
+ The top-level file organization is provided by the `FileCache` class. A `FileCache`
51
+ instance is used to specify a particular **sharing policy** and **lifetime**. For example,
52
+ a cache could be private to the current process and group a set of files that all have the
53
+ same basic purpose. Once these files have been (downloaded and) read, they are deleted as
54
+ a group. Another cache could be shared among all processes on the current machine and
55
+ group a set of files that are needed by multiple processes, thus allowing them to be
56
+ downloaded from a remote source only one time, saving time and bandwidth.
59
57
 
60
58
  A `FileCache` can be instantiated either directly or as a context manager. When
61
- instantiated directly, the programmer is responsible for calling
62
- `FileCache.clean_up` directly to delete the cache when finished. In addition, a
63
- non-shared cache will be deleted on program exit. When instantiated as a context manager,
64
- a non-shared cache is deleted on exit from the context. See the class documentation for
65
- full details.
59
+ instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
60
+ directly to delete the cache when finished (a non-shared cache will be automatically
61
+ deleted on program exit). When instantiated as a context manager, a non-shared cache is
62
+ deleted on exit from the context. See the class documentation for full details.
66
63
 
67
64
  Usage examples:
68
65
 
69
66
  ```python
70
67
  from filecache import FileCache
71
- with FileCache() as fc: # Use as context manager
68
+ # Create a cache with a unique name that will be deleted on exit
69
+ with FileCache(None) as fc: # Use as context manager
72
70
  # Also use open() as a context manager
73
71
  with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
74
72
  anonymous=True) as fp:
@@ -79,7 +77,7 @@ with FileCache() as fc: # Use as context manager
79
77
  assert bin1 == bin2
80
78
  # Cache automatically deleted here
81
79
 
82
- fc = FileCache() # Use without context manager
80
+ fc = FileCache(None) # Use without context manager
83
81
  # Also retrieve file without using open context manager
84
82
  path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
85
83
  anonymous=True)
@@ -89,43 +87,53 @@ path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
89
87
  anonymous=True)
90
88
  with open(path2, 'rb') as fp:
91
89
  bin2 = fp.read()
92
- fc.clean_up() # Cache manually deleted here
90
+ fc.delete_cache() # Cache manually deleted here
93
91
  assert bin1 == bin2
94
92
 
95
93
  # Write a file to a bucket and read it back
96
- with FileCache() as fc:
94
+ with FileCache(None) as fc:
97
95
  with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
98
96
  fp.write('A')
99
97
  # The cache will be deleted here so the file will have to be downloaded
100
- with FileCache() as fc:
98
+ with FileCache(None) as fc:
101
99
  with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
102
100
  print(fp.read())
103
101
  ```
104
102
 
105
- A `FileCachePrefix` instance can be used to encapsulate the storage prefix string,
106
- as well as any subdirectories, plus various arguments such as `anonymous` and `time_out`
107
- that can be specified to each `exists`, `retrieve`, or `upload` method. Thus using one
108
- of these instances can simplify the use of a `FileCache` by allowing the user to
109
- only specify the relative part of the path to be operated on, and to not specify various
110
- other parameters at each method call site.
103
+ The `FCPath` class is a reimplementation of the Python `Path` class to support remote
104
+ acess using an associated `FileCache`. Like `Path`, an `FCPath` instance can contain any
105
+ part of a URI, but only an absolute URI can be used when actually accessing the file
106
+ specified by the `FCPath`. In addition, an `FCPath` can encapsulate various arguments such
107
+ as `anonymous` and `time_out` so that they do not need to be specified to each access
108
+ method. Thus, using this class can simplify the use of a `FileCache` by allowing the user
109
+ to operate on paths using the simpler syntax provided by `Path`, and to not specify
110
+ various other parameters at each method call site. If an `FCPath` instance is created
111
+ without an explicitly-associated `FileCache`, then the default `FileCache()` is used,
112
+ which specifies a shared cache named `"global"` that will persist after the program exits.
111
113
 
112
114
  Compare this example to the one above:
113
115
 
114
116
  ```python
115
- from filecache import FileCache
116
- with FileCache() as fc: # Use as context manager
117
+ from filecache import FileCache, FCPath
118
+ # Create a cache with a unique name that will be deleted on exit
119
+ with FileCache(None) as fc: # Use as context manager
117
120
  # Use GS by specifying the bucket name and one directory level
118
- pfx1 = fc.new_prefix('gs://rms-filecache-tests/subdir1', anonymous=True)
121
+ p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
119
122
  # Use S3 by specifying the bucket name and two directory levels
120
- pfx2 = fc.new_prefix('s3://rms-filecache-tests/subdir1/subdir2a', anonymous=True)
123
+ # Alternative creation method
124
+ p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
125
+ anonymous=True)
121
126
  # Access GS using a directory + filename (since only one directory level
122
- # was specified by the prefix)
127
+ # was specified by the FCPath)
128
+ # The additional directory and filename are specified as an argument to open()
123
129
  # Also use open() as a context manager
124
- with pfx1.open('subdir2a/binary1.bin', 'rb') as fp:
130
+ with p1.open('subdir2a/binary1.bin', 'rb') as fp:
125
131
  bin1 = fp.read()
126
132
  # Access S3 using a filename only (since two directory levels were already
127
- # specified by the prefix))
128
- with pfx2.open('binary1.bin', 'rb') as fp:
133
+ # specified by the FCPath)
134
+ # The additional filename is specified by using the / operator to create a new
135
+ # FCPath instance; anonymous=True is inherited
136
+ with (p2 / 'binary1.bin').open(mode='rb') as fp:
129
137
  bin2 = fp.read()
130
138
  assert bin1 == bin2
131
139
  # Cache automatically deleted here
@@ -141,7 +149,7 @@ accessible from a webserver at
141
149
  Finally, it could be accessible from Google Cloud Storage from the ``rms-node-holdings``
142
150
  bucket at
143
151
  ``gs://rms-node-holdings/pds3-holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``. Before
144
- running the program, an environment variable could be set to one of these values::
152
+ running the program, an environment variable could be set to one of these values:
145
153
 
146
154
  ```sh
147
155
  $ export PDS3_HOLDINGS_SRC="~/pds3-holdings"
@@ -154,38 +162,36 @@ Then the program could be written as:
154
162
  ```python
155
163
  from filecache import FileCache
156
164
  import os
157
- with FileCache() as fc:
158
- pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_SRC'))
159
- with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
160
- contents = fp.read()
165
+ with FileCache(None) as fc:
166
+ p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
167
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
168
+ contents = voldesc_path.read_text()
161
169
  # Cache automatically deleted here
162
170
  ```
163
171
 
164
172
  If the program was going to be run multiple times in a row, or multiple copies were going
165
- to be run simultaneously, marking the cache as shared would allow all of the processes to
166
- share the same copy, thus requiring only a single download no matter how many times the
167
- program was run:
173
+ to be run simultaneously, using a shared cache would allow all of the processes to share
174
+ the same copy, thus requiring only a single download no matter how many times the program
175
+ was run. A shared cache is indicated by giving the cache a name (or no argument, which
176
+ defaults to ``"global"``); also `FCPath` defaults to using the global cache if no
177
+ `FileCache` is specified. This results in the simplest form of the program:
168
178
 
169
179
  ```python
170
- from filecache import FileCache
180
+ from filecache import FCPath
171
181
  import os
172
- with FileCache(shared=True) as fc:
173
- pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_DIR'))
174
- with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
175
- contents = fp.read()
176
- # Cache not deleted here; must be deleted manually using fc.clean_up(final=True)
177
- # If not deleted manually, the shared cache will persist until the temporary
178
- # directory is purged by the operating system (which may be never)
182
+ p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
183
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
184
+ contents = voldesc_path.read_text()
179
185
  ```
180
186
 
181
187
  Finally, there are four classes that allow direct access to the four possible storage
182
- locations without invoking any caching behavior: :class:`FileCacheSourceLocal`,
183
- :class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`:
188
+ locations without invoking any caching behavior: `FileCacheSourceLocal`,
189
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
184
190
 
185
191
  ```python
186
- from filecache import FileCacheSourceGS
187
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
188
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
192
+ from filecache import FileCacheSourceGS
193
+ src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
194
+ src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
189
195
  ```
190
196
 
191
197
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -24,6 +24,7 @@ extensions = ['myst_parser', 'sphinx.ext.autodoc', 'sphinx.ext.napoleon',
24
24
  templates_path = ['_templates']
25
25
  exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
26
26
 
27
+ autodoc_typehints = 'description'
27
28
 
28
29
  # -- Options for HTML output -------------------------------------------------
29
30
  # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
@@ -0,0 +1,26 @@
1
+ ``filecache`` Module
2
+ ====================
3
+
4
+ .. automodule:: filecache.file_cache
5
+ :member-order: bysource
6
+ :members:
7
+ :undoc-members:
8
+ :special-members:
9
+ :show-inheritance:
10
+ :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
11
+
12
+ .. automodule:: filecache.file_cache_path
13
+ :member-order: bysource
14
+ :members:
15
+ :undoc-members:
16
+ :special-members:
17
+ :show-inheritance:
18
+ :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
19
+
20
+ .. automodule:: filecache.file_cache_source
21
+ :member-order: bysource
22
+ :members:
23
+ :undoc-members:
24
+ :special-members:
25
+ :show-inheritance:
26
+ :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__