rms-filecache 1.2.0__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.coveragerc +1 -0
  2. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.flake8 +3 -1
  3. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.github/workflows/run-tests.yml +14 -2
  4. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/PKG-INFO +58 -52
  5. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/README.md +57 -51
  6. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/docs/conf.py +1 -0
  7. rms_filecache-2.0.0/docs/module.rst +26 -0
  8. rms_filecache-2.0.0/filecache/__init__.py +183 -0
  9. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/filecache/_version.py +2 -2
  10. rms_filecache-2.0.0/filecache/file_cache.py +1445 -0
  11. rms_filecache-2.0.0/filecache/file_cache_path.py +1258 -0
  12. rms_filecache-2.0.0/filecache/file_cache_source.py +752 -0
  13. rms_filecache-2.0.0/filecache/file_cache_types.py +21 -0
  14. rms_filecache-2.0.0/filecache/my_glob.py +639 -0
  15. rms_filecache-2.0.0/mypy.ini +3 -0
  16. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/requirements.txt +5 -0
  17. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/PKG-INFO +58 -52
  18. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/SOURCES.txt +11 -1
  19. rms_filecache-2.0.0/tests/__init__.py +0 -0
  20. rms_filecache-1.2.0/tests/test_filecache.py → rms_filecache-2.0.0/tests/test_file_cache.py +551 -414
  21. rms_filecache-2.0.0/tests/test_file_cache_path.py +703 -0
  22. rms_filecache-2.0.0/tests/test_file_cache_source.py +73 -0
  23. rms_filecache-2.0.0/tests/test_url_to_path.py +306 -0
  24. rms_filecache-1.2.0/docs/module.rst +0 -10
  25. rms_filecache-1.2.0/filecache/__init__.py +0 -2003
  26. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.github/pull_request_template.md +0 -0
  27. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.github/workflows/publish_to_pypi.yml +0 -0
  28. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.github/workflows/publish_to_test_pypi.yml +0 -0
  29. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.gitignore +0 -0
  30. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/.readthedocs.yaml +0 -0
  31. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/LICENSE +0 -0
  32. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/docs/Makefile +0 -0
  33. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/docs/index.rst +0 -0
  34. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/docs/make.bat +0 -0
  35. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/pyproject.toml +0 -0
  36. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
  37. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/requires.txt +0 -0
  38. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/rms_filecache.egg-info/top_level.txt +0 -0
  39. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/setup.cfg +0 -0
  40. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_files/expected/lorem1.txt +0 -0
  41. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_files/expected/subdir1/lorem1.txt +0 -0
  42. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
  43. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
  44. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_programs/locking.py +0 -0
  45. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/test_programs/multi_download.py +0 -0
  46. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/gs_lifecycle_config.json +0 -0
  47. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/gs_lifecycle_config_info.txt +0 -0
  48. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/s3_bucket_policy_read.txt +0 -0
  49. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/s3_bucket_policy_write.txt +0 -0
  50. {rms_filecache-1.2.0 → rms_filecache-2.0.0}/tests/s3_lifecycle_policy.txt +0 -0
@@ -3,3 +3,4 @@ branch = true
3
3
  omit =
4
4
  tests/*
5
5
  */_version.py
6
+ filecache/my_glob.py
@@ -2,4 +2,6 @@
2
2
  max-line-length: 90
3
3
  per-file-ignores =
4
4
  # E501 line too long
5
- test_filecache.py: E501
5
+ test_file_cache.py: E501
6
+ exclude =
7
+ filecache/my_glob.py
@@ -12,7 +12,7 @@ on:
12
12
 
13
13
  jobs:
14
14
  flake8:
15
- name: Flake8 filecache
15
+ name: Lint filecache
16
16
  runs-on: ubuntu-latest
17
17
  steps:
18
18
  - name: Checkout
@@ -20,11 +20,23 @@ jobs:
20
20
  with:
21
21
  ref: ${{ github.event.pull_request.head.sha }}
22
22
 
23
+ - name: Set up Python ${{ matrix.python-version }}
24
+ uses: actions/setup-python@v5
25
+ with:
26
+ python-version: ${{ matrix.python-version }}
27
+
28
+ - name: Install dependencies
29
+ run: |
30
+ python -m pip install -r requirements.txt
31
+
23
32
  - name: Flake8
24
33
  run: |
25
- pip install flake8
26
34
  flake8 filecache tests
27
35
 
36
+ - name: Mypy
37
+ run: |
38
+ mypy filecache
39
+
28
40
  test:
29
41
  name: Test filecache
30
42
  runs-on: ${{ matrix.os }}
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: rms-filecache
3
- Version: 1.2.0
3
+ Version: 2.0.0
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -79,28 +79,26 @@ pip install rms-filecache
79
79
 
80
80
  # Getting Started
81
81
 
82
-
83
- The top-level file organization is provided by the `FileCache` class. A
84
- `FileCache` instance is used to specify a particular **sharing policy** and
85
- **lifetime**. For example, a cache could be private to the current process and group a set
86
- of files that all have the same basic purpose. Once these files have been (downloaded and)
87
- read, they are deleted as a group. Another cache could be shared among all processes on
88
- the current machine and group a set of files that are needed by multiple processes, thus
89
- allowing them to be downloaded from a remote source only one time, saving time and
90
- bandwidth.
82
+ The top-level file organization is provided by the `FileCache` class. A `FileCache`
83
+ instance is used to specify a particular **sharing policy** and **lifetime**. For example,
84
+ a cache could be private to the current process and group a set of files that all have the
85
+ same basic purpose. Once these files have been (downloaded and) read, they are deleted as
86
+ a group. Another cache could be shared among all processes on the current machine and
87
+ group a set of files that are needed by multiple processes, thus allowing them to be
88
+ downloaded from a remote source only one time, saving time and bandwidth.
91
89
 
92
90
  A `FileCache` can be instantiated either directly or as a context manager. When
93
- instantiated directly, the programmer is responsible for calling
94
- `FileCache.clean_up` directly to delete the cache when finished. In addition, a
95
- non-shared cache will be deleted on program exit. When instantiated as a context manager,
96
- a non-shared cache is deleted on exit from the context. See the class documentation for
97
- full details.
91
+ instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
92
+ directly to delete the cache when finished (a non-shared cache will be automatically
93
+ deleted on program exit). When instantiated as a context manager, a non-shared cache is
94
+ deleted on exit from the context. See the class documentation for full details.
98
95
 
99
96
  Usage examples:
100
97
 
101
98
  ```python
102
99
  from filecache import FileCache
103
- with FileCache() as fc: # Use as context manager
100
+ # Create a cache with a unique name that will be deleted on exit
101
+ with FileCache(None) as fc: # Use as context manager
104
102
  # Also use open() as a context manager
105
103
  with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
106
104
  anonymous=True) as fp:
@@ -111,7 +109,7 @@ with FileCache() as fc: # Use as context manager
111
109
  assert bin1 == bin2
112
110
  # Cache automatically deleted here
113
111
 
114
- fc = FileCache() # Use without context manager
112
+ fc = FileCache(None) # Use without context manager
115
113
  # Also retrieve file without using open context manager
116
114
  path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
117
115
  anonymous=True)
@@ -121,43 +119,53 @@ path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
121
119
  anonymous=True)
122
120
  with open(path2, 'rb') as fp:
123
121
  bin2 = fp.read()
124
- fc.clean_up() # Cache manually deleted here
122
+ fc.delete_cache() # Cache manually deleted here
125
123
  assert bin1 == bin2
126
124
 
127
125
  # Write a file to a bucket and read it back
128
- with FileCache() as fc:
126
+ with FileCache(None) as fc:
129
127
  with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
130
128
  fp.write('A')
131
129
  # The cache will be deleted here so the file will have to be downloaded
132
- with FileCache() as fc:
130
+ with FileCache(None) as fc:
133
131
  with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
134
132
  print(fp.read())
135
133
  ```
136
134
 
137
- A `FileCachePrefix` instance can be used to encapsulate the storage prefix string,
138
- as well as any subdirectories, plus various arguments such as `anonymous` and `time_out`
139
- that can be specified to each `exists`, `retrieve`, or `upload` method. Thus using one
140
- of these instances can simplify the use of a `FileCache` by allowing the user to
141
- only specify the relative part of the path to be operated on, and to not specify various
142
- other parameters at each method call site.
135
+ The `FCPath` class is a reimplementation of the Python `Path` class to support remote
136
+ acess using an associated `FileCache`. Like `Path`, an `FCPath` instance can contain any
137
+ part of a URI, but only an absolute URI can be used when actually accessing the file
138
+ specified by the `FCPath`. In addition, an `FCPath` can encapsulate various arguments such
139
+ as `anonymous` and `time_out` so that they do not need to be specified to each access
140
+ method. Thus, using this class can simplify the use of a `FileCache` by allowing the user
141
+ to operate on paths using the simpler syntax provided by `Path`, and to not specify
142
+ various other parameters at each method call site. If an `FCPath` instance is created
143
+ without an explicitly-associated `FileCache`, then the default `FileCache()` is used,
144
+ which specifies a shared cache named `"global"` that will persist after the program exits.
143
145
 
144
146
  Compare this example to the one above:
145
147
 
146
148
  ```python
147
- from filecache import FileCache
148
- with FileCache() as fc: # Use as context manager
149
+ from filecache import FileCache, FCPath
150
+ # Create a cache with a unique name that will be deleted on exit
151
+ with FileCache(None) as fc: # Use as context manager
149
152
  # Use GS by specifying the bucket name and one directory level
150
- pfx1 = fc.new_prefix('gs://rms-filecache-tests/subdir1', anonymous=True)
153
+ p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
151
154
  # Use S3 by specifying the bucket name and two directory levels
152
- pfx2 = fc.new_prefix('s3://rms-filecache-tests/subdir1/subdir2a', anonymous=True)
155
+ # Alternative creation method
156
+ p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
157
+ anonymous=True)
153
158
  # Access GS using a directory + filename (since only one directory level
154
- # was specified by the prefix)
159
+ # was specified by the FCPath)
160
+ # The additional directory and filename are specified as an argument to open()
155
161
  # Also use open() as a context manager
156
- with pfx1.open('subdir2a/binary1.bin', 'rb') as fp:
162
+ with p1.open('subdir2a/binary1.bin', 'rb') as fp:
157
163
  bin1 = fp.read()
158
164
  # Access S3 using a filename only (since two directory levels were already
159
- # specified by the prefix))
160
- with pfx2.open('binary1.bin', 'rb') as fp:
165
+ # specified by the FCPath)
166
+ # The additional filename is specified by using the / operator to create a new
167
+ # FCPath instance; anonymous=True is inherited
168
+ with (p2 / 'binary1.bin').open(mode='rb') as fp:
161
169
  bin2 = fp.read()
162
170
  assert bin1 == bin2
163
171
  # Cache automatically deleted here
@@ -186,28 +194,26 @@ Then the program could be written as:
186
194
  ```python
187
195
  from filecache import FileCache
188
196
  import os
189
- with FileCache() as fc:
190
- pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_SRC'))
191
- with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
192
- contents = fp.read()
197
+ with FileCache(None) as fc:
198
+ p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
199
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
200
+ contents = voldesc_path.read_text()
193
201
  # Cache automatically deleted here
194
202
  ```
195
203
 
196
204
  If the program was going to be run multiple times in a row, or multiple copies were going
197
- to be run simultaneously, marking the cache as shared would allow all of the processes to
198
- share the same copy, thus requiring only a single download no matter how many times the
199
- program was run:
205
+ to be run simultaneously, using a shared cache would allow all of the processes to share
206
+ the same copy, thus requiring only a single download no matter how many times the program
207
+ was run. A shared cache is indicated by giving the cache a name (or no argument, which
208
+ defaults to ``"global"``); also `FCPath` defaults to using the global cache if no
209
+ `FileCache` is specified. This results in the simplest form of the program:
200
210
 
201
211
  ```python
202
- from filecache import FileCache
212
+ from filecache import FCPath
203
213
  import os
204
- with FileCache(shared=True) as fc:
205
- pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_DIR'))
206
- with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
207
- contents = fp.read()
208
- # Cache not deleted here; must be deleted manually using fc.clean_up(final=True)
209
- # If not deleted manually, the shared cache will persist until the temporary
210
- # directory is purged by the operating system (which may be never)
214
+ p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
215
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
216
+ contents = voldesc_path.read_text()
211
217
  ```
212
218
 
213
219
  Finally, there are four classes that allow direct access to the four possible storage
@@ -215,9 +221,9 @@ locations without invoking any caching behavior: :class:`FileCacheSourceLocal`,
215
221
  :class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`:
216
222
 
217
223
  ```python
218
- from filecache import FileCacheSourceGS
219
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
220
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
224
+ from filecache import FileCacheSourceGS
225
+ src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
226
+ src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
221
227
  ```
222
228
 
223
229
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -47,28 +47,26 @@ pip install rms-filecache
47
47
 
48
48
  # Getting Started
49
49
 
50
-
51
- The top-level file organization is provided by the `FileCache` class. A
52
- `FileCache` instance is used to specify a particular **sharing policy** and
53
- **lifetime**. For example, a cache could be private to the current process and group a set
54
- of files that all have the same basic purpose. Once these files have been (downloaded and)
55
- read, they are deleted as a group. Another cache could be shared among all processes on
56
- the current machine and group a set of files that are needed by multiple processes, thus
57
- allowing them to be downloaded from a remote source only one time, saving time and
58
- bandwidth.
50
+ The top-level file organization is provided by the `FileCache` class. A `FileCache`
51
+ instance is used to specify a particular **sharing policy** and **lifetime**. For example,
52
+ a cache could be private to the current process and group a set of files that all have the
53
+ same basic purpose. Once these files have been (downloaded and) read, they are deleted as
54
+ a group. Another cache could be shared among all processes on the current machine and
55
+ group a set of files that are needed by multiple processes, thus allowing them to be
56
+ downloaded from a remote source only one time, saving time and bandwidth.
59
57
 
60
58
  A `FileCache` can be instantiated either directly or as a context manager. When
61
- instantiated directly, the programmer is responsible for calling
62
- `FileCache.clean_up` directly to delete the cache when finished. In addition, a
63
- non-shared cache will be deleted on program exit. When instantiated as a context manager,
64
- a non-shared cache is deleted on exit from the context. See the class documentation for
65
- full details.
59
+ instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
60
+ directly to delete the cache when finished (a non-shared cache will be automatically
61
+ deleted on program exit). When instantiated as a context manager, a non-shared cache is
62
+ deleted on exit from the context. See the class documentation for full details.
66
63
 
67
64
  Usage examples:
68
65
 
69
66
  ```python
70
67
  from filecache import FileCache
71
- with FileCache() as fc: # Use as context manager
68
+ # Create a cache with a unique name that will be deleted on exit
69
+ with FileCache(None) as fc: # Use as context manager
72
70
  # Also use open() as a context manager
73
71
  with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
74
72
  anonymous=True) as fp:
@@ -79,7 +77,7 @@ with FileCache() as fc: # Use as context manager
79
77
  assert bin1 == bin2
80
78
  # Cache automatically deleted here
81
79
 
82
- fc = FileCache() # Use without context manager
80
+ fc = FileCache(None) # Use without context manager
83
81
  # Also retrieve file without using open context manager
84
82
  path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
85
83
  anonymous=True)
@@ -89,43 +87,53 @@ path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
89
87
  anonymous=True)
90
88
  with open(path2, 'rb') as fp:
91
89
  bin2 = fp.read()
92
- fc.clean_up() # Cache manually deleted here
90
+ fc.delete_cache() # Cache manually deleted here
93
91
  assert bin1 == bin2
94
92
 
95
93
  # Write a file to a bucket and read it back
96
- with FileCache() as fc:
94
+ with FileCache(None) as fc:
97
95
  with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
98
96
  fp.write('A')
99
97
  # The cache will be deleted here so the file will have to be downloaded
100
- with FileCache() as fc:
98
+ with FileCache(None) as fc:
101
99
  with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
102
100
  print(fp.read())
103
101
  ```
104
102
 
105
- A `FileCachePrefix` instance can be used to encapsulate the storage prefix string,
106
- as well as any subdirectories, plus various arguments such as `anonymous` and `time_out`
107
- that can be specified to each `exists`, `retrieve`, or `upload` method. Thus using one
108
- of these instances can simplify the use of a `FileCache` by allowing the user to
109
- only specify the relative part of the path to be operated on, and to not specify various
110
- other parameters at each method call site.
103
+ The `FCPath` class is a reimplementation of the Python `Path` class to support remote
104
+ acess using an associated `FileCache`. Like `Path`, an `FCPath` instance can contain any
105
+ part of a URI, but only an absolute URI can be used when actually accessing the file
106
+ specified by the `FCPath`. In addition, an `FCPath` can encapsulate various arguments such
107
+ as `anonymous` and `time_out` so that they do not need to be specified to each access
108
+ method. Thus, using this class can simplify the use of a `FileCache` by allowing the user
109
+ to operate on paths using the simpler syntax provided by `Path`, and to not specify
110
+ various other parameters at each method call site. If an `FCPath` instance is created
111
+ without an explicitly-associated `FileCache`, then the default `FileCache()` is used,
112
+ which specifies a shared cache named `"global"` that will persist after the program exits.
111
113
 
112
114
  Compare this example to the one above:
113
115
 
114
116
  ```python
115
- from filecache import FileCache
116
- with FileCache() as fc: # Use as context manager
117
+ from filecache import FileCache, FCPath
118
+ # Create a cache with a unique name that will be deleted on exit
119
+ with FileCache(None) as fc: # Use as context manager
117
120
  # Use GS by specifying the bucket name and one directory level
118
- pfx1 = fc.new_prefix('gs://rms-filecache-tests/subdir1', anonymous=True)
121
+ p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
119
122
  # Use S3 by specifying the bucket name and two directory levels
120
- pfx2 = fc.new_prefix('s3://rms-filecache-tests/subdir1/subdir2a', anonymous=True)
123
+ # Alternative creation method
124
+ p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
125
+ anonymous=True)
121
126
  # Access GS using a directory + filename (since only one directory level
122
- # was specified by the prefix)
127
+ # was specified by the FCPath)
128
+ # The additional directory and filename are specified as an argument to open()
123
129
  # Also use open() as a context manager
124
- with pfx1.open('subdir2a/binary1.bin', 'rb') as fp:
130
+ with p1.open('subdir2a/binary1.bin', 'rb') as fp:
125
131
  bin1 = fp.read()
126
132
  # Access S3 using a filename only (since two directory levels were already
127
- # specified by the prefix))
128
- with pfx2.open('binary1.bin', 'rb') as fp:
133
+ # specified by the FCPath)
134
+ # The additional filename is specified by using the / operator to create a new
135
+ # FCPath instance; anonymous=True is inherited
136
+ with (p2 / 'binary1.bin').open(mode='rb') as fp:
129
137
  bin2 = fp.read()
130
138
  assert bin1 == bin2
131
139
  # Cache automatically deleted here
@@ -154,28 +162,26 @@ Then the program could be written as:
154
162
  ```python
155
163
  from filecache import FileCache
156
164
  import os
157
- with FileCache() as fc:
158
- pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_SRC'))
159
- with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
160
- contents = fp.read()
165
+ with FileCache(None) as fc:
166
+ p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
167
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
168
+ contents = voldesc_path.read_text()
161
169
  # Cache automatically deleted here
162
170
  ```
163
171
 
164
172
  If the program was going to be run multiple times in a row, or multiple copies were going
165
- to be run simultaneously, marking the cache as shared would allow all of the processes to
166
- share the same copy, thus requiring only a single download no matter how many times the
167
- program was run:
173
+ to be run simultaneously, using a shared cache would allow all of the processes to share
174
+ the same copy, thus requiring only a single download no matter how many times the program
175
+ was run. A shared cache is indicated by giving the cache a name (or no argument, which
176
+ defaults to ``"global"``); also `FCPath` defaults to using the global cache if no
177
+ `FileCache` is specified. This results in the simplest form of the program:
168
178
 
169
179
  ```python
170
- from filecache import FileCache
180
+ from filecache import FCPath
171
181
  import os
172
- with FileCache(shared=True) as fc:
173
- pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_DIR'))
174
- with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
175
- contents = fp.read()
176
- # Cache not deleted here; must be deleted manually using fc.clean_up(final=True)
177
- # If not deleted manually, the shared cache will persist until the temporary
178
- # directory is purged by the operating system (which may be never)
182
+ p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
183
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
184
+ contents = voldesc_path.read_text()
179
185
  ```
180
186
 
181
187
  Finally, there are four classes that allow direct access to the four possible storage
@@ -183,9 +189,9 @@ locations without invoking any caching behavior: :class:`FileCacheSourceLocal`,
183
189
  :class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`:
184
190
 
185
191
  ```python
186
- from filecache import FileCacheSourceGS
187
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
188
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
192
+ from filecache import FileCacheSourceGS
193
+ src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
194
+ src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
189
195
  ```
190
196
 
191
197
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -24,6 +24,7 @@ extensions = ['myst_parser', 'sphinx.ext.autodoc', 'sphinx.ext.napoleon',
24
24
  templates_path = ['_templates']
25
25
  exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
26
26
 
27
+ autodoc_typehints = 'description'
27
28
 
28
29
  # -- Options for HTML output -------------------------------------------------
29
30
  # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
@@ -0,0 +1,26 @@
1
+ ``filecache`` Module
2
+ ====================
3
+
4
+ .. automodule:: filecache.file_cache
5
+ :member-order: bysource
6
+ :members:
7
+ :undoc-members:
8
+ :special-members:
9
+ :show-inheritance:
10
+ :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
11
+
12
+ .. automodule:: filecache.file_cache_path
13
+ :member-order: bysource
14
+ :members:
15
+ :undoc-members:
16
+ :special-members:
17
+ :show-inheritance:
18
+ :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
19
+
20
+ .. automodule:: filecache.file_cache_source
21
+ :member-order: bysource
22
+ :members:
23
+ :undoc-members:
24
+ :special-members:
25
+ :show-inheritance:
26
+ :exclude-members: __dict__, __hash__, __module__, __weakref__, __enter__, __exit__, __annotations__
@@ -0,0 +1,183 @@
1
+ ##########################################################################################
2
+ # filecache/__init__.py
3
+ ##########################################################################################
4
+
5
+ """The purpose of the `filecache` module is to abstract away the location where files
6
+ used or generated by a program are stored. Files can be on the local file system, in
7
+ Google Cloud Storage, on Amazon Web Services S3, or on a webserver. When files to be read
8
+ are on the local file system, they are simply accessed in-place. Otherwise, they are
9
+ downloaded from the remote source to a local temporary directory. When files to be written
10
+ are on the local file system, they are simply written in-place. Otherwise, they are
11
+ written to a local temporary directory and then uploaded to the remote location (it is not
12
+ possible to upload to a webserver). When a cache is no longer needed, it is deleted from
13
+ the local disk.
14
+
15
+ The top-level file organization is provided by the :class:`FileCache` class. A
16
+ :class:`FileCache` instance is used to specify a particular **sharing policy** and
17
+ **lifetime**. For example, a cache could be private to the current process and group a set
18
+ of files that all have the same basic purpose. Once these files have been (downloaded and)
19
+ read, they are deleted as a group. Another cache could be shared among all processes on
20
+ the current machine and group a set of files that are needed by multiple processes, thus
21
+ allowing them to be downloaded from a remote source only one time, saving time and
22
+ bandwidth.
23
+
24
+ A :class:`FileCache` can be instantiated either directly or as a context manager. When
25
+ instantiated directly, the programmer is responsible for calling `FileCache.delete_cache`
26
+ directly to delete the cache when finished (a non-shared cache will be automatically
27
+ deleted on program exit). When instantiated as a context manager, a non-shared cache is
28
+ deleted on exit from the context. See the class documentation for full details.
29
+
30
+ Usage examples::
31
+
32
+ from filecache import FileCache
33
+ # Create a cache with a unique name that will be deleted on exit
34
+ with FileCache(None) as fc: # Use as context manager
35
+ # Also use open() as a context manager
36
+ with fc.open('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
37
+ anonymous=True) as fp:
38
+ bin1 = fp.read()
39
+ with fc.open('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin', 'rb',
40
+ anonymous=True) as fp:
41
+ bin2 = fp.read()
42
+ assert bin1 == bin2
43
+ # Cache automatically deleted here
44
+
45
+ fc = FileCache(None) # Use without context manager
46
+ # Also retrieve file without using open context manager
47
+ path1 = fc.retrieve('gs://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
48
+ anonymous=True)
49
+ with open(path1, 'rb') as fp:
50
+ bin1 = fp.read()
51
+ path2 = fc.retrieve('s3://rms-filecache-tests/subdir1/subdir2a/binary1.bin',
52
+ anonymous=True)
53
+ with open(path2, 'rb') as fp:
54
+ bin2 = fp.read()
55
+ fc.delete_cache() # Cache manually deleted here
56
+ assert bin1 == bin2
57
+
58
+ # Write a file to a bucket and read it back
59
+ with FileCache(None) as fc:
60
+ with fc.open('gs://my-writable-bucket/output.txt', 'w') as fp:
61
+ fp.write('A')
62
+ # The cache will be deleted here so the file will have to be downloaded
63
+ with FileCache(None) as fc:
64
+ with fc.open('gs://my-writable-bucket/output.txt', 'r') as fp:
65
+ print(fp.read())
66
+
67
+ The :class:`FCPath` class is a reimplementation of the Python `Path` class to support
68
+ remote acess using an associated :class:`FileCache`. Like `Path`, an :class:`FCPath`
69
+ instance can contain any part of a URI, but only an absolute URI can be used when actually
70
+ accessing the file specified by the :class:`FCPath`. In addition, an :class:`FCPath` can
71
+ encapsulate various arguments such as `anonymous` and `time_out` so that they do not need
72
+ to be specified to each access method. Thus, using this class can simplify the use of a
73
+ :class:`FileCache` by allowing the user to operate on paths using the simpler syntax
74
+ provided by `Path`, and to not specify various other parameters at each method call site.
75
+ If an :class:`FCPath` instance is created without an explicitly-associated
76
+ :class:`FileCache`, then the default `FileCache()` is used, which specifies a shared cache
77
+ named `"global"` that will persist after the program exits.
78
+
79
+ Compare this example to the one above::
80
+
81
+ from filecache import FileCache, FCPath
82
+ # Create a cache with a unique name that will be deleted on exit
83
+ with FileCache(None) as fc: # Use as context manager
84
+ # Use GS by specifying the bucket name and one directory level
85
+ p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
86
+ # Use S3 by specifying the bucket name and two directory levels
87
+ # Alternative creation method
88
+ p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
89
+ anonymous=True)
90
+ # Access GS using a directory + filename (since only one directory level
91
+ # was specified by the FCPath)
92
+ # The additional directory and filename are specified as an argument to open()
93
+ # Also use open() as a context manager
94
+ with p1.open('subdir2a/binary1.bin', 'rb') as fp:
95
+ bin1 = fp.read()
96
+ # Access S3 using a filename only (since two directory levels were already
97
+ # specified by the FCPath)
98
+ # The additional filename is specified by using the / operator to create a new
99
+ # FCPath instance; anonymous=True is inherited
100
+ with (p2 / 'binary1.bin').open(mode='rb') as fp:
101
+ bin2 = fp.read()
102
+ assert bin1 == bin2
103
+ # Cache automatically deleted here
104
+
105
+ A benefit of the abstraction is that different environments can access the same files in
106
+ different ways without needing to change the program code. For example, consider a program
107
+ that needs to access the file ``COISS_2xxx/COISS_2001/voldesc.cat`` from the NASA PDS
108
+ archives. This file might be stored on the local disk in the user's home directory in a
109
+ subdirectory called ``pds3-holdings``. Or if the user does not have a local copy, it is
110
+ accessible from a webserver at
111
+ ``https://pds-rings.seti.org/holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``.
112
+ Finally, it could be accessible from Google Cloud Storage from the ``rms-node-holdings``
113
+ bucket at
114
+ ``gs://rms-node-holdings/pds3-holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``. Before
115
+ running the program, an environment variable could be set to one of these values::
116
+
117
+ $ export PDS3_HOLDINGS_SRC="~/pds3-holdings"
118
+ $ export PDS3_HOLDINGS_SRC="https://pds-rings.seti.org/holdings"
119
+ $ export PDS3_HOLDINGS_SRC="gs://rms-node-holdings/pds3-holdings"
120
+
121
+ Then the program could be written as::
122
+
123
+ from filecache import FileCache
124
+ import os
125
+ with FileCache(None) as fc:
126
+ p = fc.new_path(os.getenv('PDS3_HOLDINGS_SRC'))
127
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
128
+ contents = voldesc_path.read_text()
129
+ # Cache automatically deleted here
130
+
131
+ If the program was going to be run multiple times in a row, or multiple copies were going
132
+ to be run simultaneously, using a shared cache would allow all of the processes to share
133
+ the same copy, thus requiring only a single download no matter how many times the program
134
+ was run. A shared cache is indicated by giving the cache a name (or no argument, which
135
+ defaults to ``"global"``); also :class:`FCPath` defaults to using the global cache if no
136
+ :class:`FileCache` is specified. This results in the simplest form of the program:
137
+
138
+ from filecache import FCPath
139
+ import os
140
+ p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
141
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
142
+ contents = voldesc_path.read_text()
143
+
144
+ Finally, there are four classes that allow direct access to the four possible storage
145
+ locations without invoking any caching behavior: :class:`FileCacheSourceFile`,
146
+ :class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`::
147
+
148
+ from filecache import FileCacheSourceGS
149
+ src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
150
+ src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
151
+ """
152
+
153
+ import typing
154
+
155
+ if not typing.TYPE_CHECKING: # pragma: no cover
156
+ try:
157
+ from ._version import __version__
158
+ except ImportError: # pragma: no cover - only present when building a package
159
+ __version__ = 'Version unspecified'
160
+
161
+ from .file_cache import (get_global_logger, # noqa: ignore E401
162
+ register_filecachesource,
163
+ set_easy_logger,
164
+ set_global_logger,
165
+ FileCache)
166
+ from .file_cache_path import FCPath # noqa: ignore E401
167
+ from .file_cache_source import (FileCacheSource, # noqa: ignore E401
168
+ FileCacheSourceFile,
169
+ FileCacheSourceHTTP,
170
+ FileCacheSourceGS,
171
+ FileCacheSourceS3)
172
+
173
+ __all__ = ['get_global_logger',
174
+ 'register_filecachesource',
175
+ 'set_easy_logger',
176
+ 'set_global_logger',
177
+ 'FileCache',
178
+ 'FCPath',
179
+ 'FileCacheSource',
180
+ 'FileCacheSourceFile',
181
+ 'FileCacheSourceHTTP',
182
+ 'FileCacheSourceGS',
183
+ 'FileCacheSourceS3']
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '1.2.0'
16
- __version_tuple__ = version_tuple = (1, 2, 0)
15
+ __version__ = version = '2.0.0'
16
+ __version_tuple__ = version_tuple = (2, 0, 0)