rms-filecache 2.3.0__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.gitignore +2 -0
  2. {rms_filecache-2.3.0/rms_filecache.egg-info → rms_filecache-3.0.0}/PKG-INFO +27 -25
  3. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/README.md +24 -23
  4. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/__init__.py +29 -23
  5. rms_filecache-3.0.0/filecache/_version.py +34 -0
  6. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/file_cache.py +920 -122
  7. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/file_cache_path.py +345 -59
  8. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/file_cache_source.py +835 -39
  9. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/file_cache_types.py +8 -0
  10. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/requirements.txt +1 -0
  11. {rms_filecache-2.3.0 → rms_filecache-3.0.0/rms_filecache.egg-info}/PKG-INFO +27 -25
  12. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/rms_filecache.egg-info/SOURCES.txt +1 -1
  13. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/test_file_cache.py +594 -61
  14. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/test_file_cache_path.py +315 -72
  15. rms_filecache-3.0.0/tests/test_file_cache_source.py +406 -0
  16. rms_filecache-2.3.0/tests/test_url_to_path.py → rms_filecache-3.0.0/tests/test_url_mapping.py +265 -24
  17. rms_filecache-2.3.0/filecache/_version.py +0 -16
  18. rms_filecache-2.3.0/tests/test_file_cache_source.py +0 -81
  19. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.coveragerc +0 -0
  20. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.flake8 +0 -0
  21. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.github/pull_request_template.md +0 -0
  22. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.github/workflows/publish_to_pypi.yml +0 -0
  23. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.github/workflows/publish_to_test_pypi.yml +0 -0
  24. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.github/workflows/run-tests.yml +0 -0
  25. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/.readthedocs.yaml +0 -0
  26. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/LICENSE +0 -0
  27. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/Makefile +0 -0
  28. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/conf.py +0 -0
  29. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/index.rst +0 -0
  30. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/make.bat +0 -0
  31. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/docs/module.rst +0 -0
  32. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/filecache/py.typed +0 -0
  33. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/mypy.ini +0 -0
  34. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/pyproject.toml +0 -0
  35. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
  36. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/rms_filecache.egg-info/requires.txt +0 -0
  37. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/rms_filecache.egg-info/top_level.txt +0 -0
  38. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/setup.cfg +0 -0
  39. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_files/expected/lorem1.txt +0 -0
  40. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_files/expected/subdir1/lorem1.txt +0 -0
  41. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
  42. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
  43. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_programs/locking.py +0 -0
  44. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/test_programs/multi_download.py +0 -0
  45. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/__init__.py +0 -0
  46. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/gs_lifecycle_config.json +0 -0
  47. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/gs_lifecycle_config_info.txt +0 -0
  48. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/s3_bucket_policy_read.txt +0 -0
  49. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/s3_bucket_policy_write.txt +0 -0
  50. {rms_filecache-2.3.0 → rms_filecache-3.0.0}/tests/s3_lifecycle_policy.txt +0 -0
@@ -159,3 +159,5 @@ cython_debug/
159
159
  # and can be added to the global gitignore or merged into this file. For a more nuclear
160
160
  # option (not recommended) you can uncomment the following to ignore the entire idea folder.
161
161
  #.idea/
162
+
163
+ .vscode/
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: rms-filecache
3
- Version: 2.3.0
3
+ Version: 3.0.0
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -31,6 +31,7 @@ Requires-Dist: filelock
31
31
  Requires-Dist: google-cloud-storage
32
32
  Requires-Dist: requests
33
33
  Requires-Dist: typing_extensions
34
+ Dynamic: license-file
34
35
 
35
36
  [![GitHub release; latest by date](https://img.shields.io/github/v/release/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
36
37
  [![GitHub Release Date](https://img.shields.io/github/release-date/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
@@ -59,14 +60,16 @@ Requires-Dist: typing_extensions
59
60
  # Introduction
60
61
 
61
62
  `filecache` is a Python module that abstracts away the location where files used or
62
- generated by a program are stored. Files can be on the local file system, in Google Cloud
63
- Storage, on Amazon Web Services S3, or on a webserver. When files to be read are on the
64
- local file system, they are simply accessed in-place. Otherwise, they are downloaded from
65
- the remote source to a local temporary directory. When files to be written are on the
66
- local file system, they are simply written in-place. Otherwise, they are written to a
67
- local temporary directory and then uploaded to the remote location (it is not possible to
68
- upload to a webserver). When a cache is no longer needed, it is deleted from the local
69
- disk.
63
+ generated by a program are stored. Files can be on the local file system (`/` or
64
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
65
+ webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
66
+ a remote using the local filesystem as a file repository (`fake://`). When files to be read
67
+ are on the local file system, they are simply accessed in-place. Otherwise, they are
68
+ downloaded from the remote source to a local temporary directory. When files to be written
69
+ are on the local file system, they are simply written in-place. Otherwise, they are
70
+ written to a local temporary directory and then uploaded to the remote location (it is not
71
+ possible to upload to a webserver). When a cache is no longer needed, it is deleted from
72
+ the local disk.
70
73
 
71
74
  `filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
72
75
 
@@ -153,21 +156,16 @@ from filecache import FileCache, FCPath
153
156
  with FileCache(None) as fc: # Use as context manager
154
157
  # Use GS by specifying the bucket name and one directory level
155
158
  p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
156
- # Use S3 by specifying the bucket name and two directory levels
157
- # Alternative creation method
158
- p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
159
- anonymous=True)
160
159
  # Access GS using a directory + filename (since only one directory level
161
160
  # was specified by the FCPath)
162
- # The additional directory and filename are specified as an argument to open()
161
+ # The additional filename is specified by using the / operator to create a new
162
+ # FCPath instance; anonymous=True is inherited
163
163
  # Also use open() as a context manager
164
- with p1.open('subdir2a/binary1.bin', 'rb') as fp:
164
+ with (p1 / 'subdir2a/binary1.bin').open('rb') as fp:
165
165
  bin1 = fp.read()
166
166
  # Access S3 using a filename only (since two directory levels were already
167
167
  # specified by the FCPath)
168
- # The additional filename is specified by using the / operator to create a new
169
- # FCPath instance; anonymous=True is inherited
170
- with (p2 / 'binary1.bin').open(mode='rb') as fp:
168
+ with (p2 / 'binary1.bin').open('rb') as fp:
171
169
  bin2 = fp.read()
172
170
  assert bin1 == bin2
173
171
  # Cache automatically deleted here
@@ -218,14 +216,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
218
216
  contents = voldesc_path.read_text()
219
217
  ```
220
218
 
221
- Finally, there are four classes that allow direct access to the four possible storage
222
- locations without invoking any caching behavior: `FileCacheSourceLocal`,
223
- `FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
219
+ Finally, there are five classes that allow direct access to the five possible storage
220
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
221
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
222
+ The first four provide access to real storage locations, while `FileCacheSourceFake`
223
+ simulates a remote source for testing purposes:
224
224
 
225
225
  ```python
226
- from filecache import FileCacheSourceGS
227
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
228
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
226
+ from filecache import FileCacheSourceGS, FileCacheSourceFake
227
+ src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
228
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
229
+ src2 = FileCacheSourceFake('fake://fake-bucket')
230
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
229
231
  ```
230
232
 
231
233
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -25,14 +25,16 @@
25
25
  # Introduction
26
26
 
27
27
  `filecache` is a Python module that abstracts away the location where files used or
28
- generated by a program are stored. Files can be on the local file system, in Google Cloud
29
- Storage, on Amazon Web Services S3, or on a webserver. When files to be read are on the
30
- local file system, they are simply accessed in-place. Otherwise, they are downloaded from
31
- the remote source to a local temporary directory. When files to be written are on the
32
- local file system, they are simply written in-place. Otherwise, they are written to a
33
- local temporary directory and then uploaded to the remote location (it is not possible to
34
- upload to a webserver). When a cache is no longer needed, it is deleted from the local
35
- disk.
28
+ generated by a program are stored. Files can be on the local file system (`/` or
29
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
30
+ webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
31
+ a remote using the local filesystem as a file repository (`fake://`). When files to be read
32
+ are on the local file system, they are simply accessed in-place. Otherwise, they are
33
+ downloaded from the remote source to a local temporary directory. When files to be written
34
+ are on the local file system, they are simply written in-place. Otherwise, they are
35
+ written to a local temporary directory and then uploaded to the remote location (it is not
36
+ possible to upload to a webserver). When a cache is no longer needed, it is deleted from
37
+ the local disk.
36
38
 
37
39
  `filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
38
40
 
@@ -119,21 +121,16 @@ from filecache import FileCache, FCPath
119
121
  with FileCache(None) as fc: # Use as context manager
120
122
  # Use GS by specifying the bucket name and one directory level
121
123
  p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
122
- # Use S3 by specifying the bucket name and two directory levels
123
- # Alternative creation method
124
- p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
125
- anonymous=True)
126
124
  # Access GS using a directory + filename (since only one directory level
127
125
  # was specified by the FCPath)
128
- # The additional directory and filename are specified as an argument to open()
126
+ # The additional filename is specified by using the / operator to create a new
127
+ # FCPath instance; anonymous=True is inherited
129
128
  # Also use open() as a context manager
130
- with p1.open('subdir2a/binary1.bin', 'rb') as fp:
129
+ with (p1 / 'subdir2a/binary1.bin').open('rb') as fp:
131
130
  bin1 = fp.read()
132
131
  # Access S3 using a filename only (since two directory levels were already
133
132
  # specified by the FCPath)
134
- # The additional filename is specified by using the / operator to create a new
135
- # FCPath instance; anonymous=True is inherited
136
- with (p2 / 'binary1.bin').open(mode='rb') as fp:
133
+ with (p2 / 'binary1.bin').open('rb') as fp:
137
134
  bin2 = fp.read()
138
135
  assert bin1 == bin2
139
136
  # Cache automatically deleted here
@@ -184,14 +181,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
184
181
  contents = voldesc_path.read_text()
185
182
  ```
186
183
 
187
- Finally, there are four classes that allow direct access to the four possible storage
188
- locations without invoking any caching behavior: `FileCacheSourceLocal`,
189
- `FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
184
+ Finally, there are five classes that allow direct access to the five possible storage
185
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
186
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
187
+ The first four provide access to real storage locations, while `FileCacheSourceFake`
188
+ simulates a remote source for testing purposes:
190
189
 
191
190
  ```python
192
- from filecache import FileCacheSourceGS
193
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
194
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
191
+ from filecache import FileCacheSourceGS, FileCacheSourceFake
192
+ src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
193
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
194
+ src2 = FileCacheSourceFake('fake://fake-bucket')
195
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
195
196
  ```
196
197
 
197
198
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -3,14 +3,16 @@
3
3
  ##########################################################################################
4
4
 
5
5
  """The purpose of the `filecache` module is to abstract away the location where files
6
- used or generated by a program are stored. Files can be on the local file system, in
7
- Google Cloud Storage, on Amazon Web Services S3, or on a webserver. When files to be read
8
- are on the local file system, they are simply accessed in-place. Otherwise, they are
9
- downloaded from the remote source to a local temporary directory. When files to be written
10
- are on the local file system, they are simply written in-place. Otherwise, they are
11
- written to a local temporary directory and then uploaded to the remote location (it is not
12
- possible to upload to a webserver). When a cache is no longer needed, it is deleted from
13
- the local disk.
6
+ used or generated by a program are stored. Files can be on the local file system (`/` or
7
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on
8
+ a webserver (`http://` or `https://`). There is also a fake remote that can be used to
9
+ simulate a remote using the local filesystem as a file repository (`fake://`). When files
10
+ to be read are on the local file system, they are simply accessed in-place. Otherwise,
11
+ they are downloaded from the remote source to a local temporary directory. When files to
12
+ be written are on the local file system, they are simply written in-place. Otherwise, they
13
+ are written to a local temporary directory and then uploaded to the remote location (it is
14
+ not possible to upload to a webserver). When a cache is no longer needed, it is deleted
15
+ from the local disk.
14
16
 
15
17
  The top-level file organization is provided by the :class:`FileCache` class. A
16
18
  :class:`FileCache` instance is used to specify a particular **sharing policy** and
@@ -135,19 +137,21 @@ was run. A shared cache is indicated by giving the cache a name (or no argument,
135
137
  defaults to ``"global"``); also :class:`FCPath` defaults to using the global cache if no
136
138
  :class:`FileCache` is specified. This results in the simplest form of the program:
137
139
 
138
- from filecache import FCPath
139
- import os
140
- p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
141
- voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
142
- contents = voldesc_path.read_text()
143
-
144
- Finally, there are four classes that allow direct access to the four possible storage
145
- locations without invoking any caching behavior: :class:`FileCacheSourceFile`,
146
- :class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`::
147
-
148
- from filecache import FileCacheSourceGS
149
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
150
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
140
+ from filecache import FCPath import os p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
141
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat' contents =
142
+ voldesc_path.read_text()
143
+
144
+ Finally, there are five classes that allow direct access to the five possible storage
145
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
146
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and
147
+ `FileCacheSourceFake`. The first four provide access to real storage locations, while
148
+ `FileCacheSourceFake` simulates a remote source for testing purposes:
149
+
150
+ from filecache import FileCacheSourceGS, FileCacheSourceFake src1 =
151
+ FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
152
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin') src2 =
153
+ FileCacheSourceFake('fake://fake-bucket')
154
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
151
155
  """
152
156
 
153
157
  import typing
@@ -168,7 +172,8 @@ from .file_cache_source import (FileCacheSource, # noqa: ignore E401
168
172
  FileCacheSourceFile,
169
173
  FileCacheSourceHTTP,
170
174
  FileCacheSourceGS,
171
- FileCacheSourceS3)
175
+ FileCacheSourceS3,
176
+ FileCacheSourceFake)
172
177
 
173
178
  __all__ = ['get_global_logger',
174
179
  'register_filecachesource',
@@ -180,4 +185,5 @@ __all__ = ['get_global_logger',
180
185
  'FileCacheSourceFile',
181
186
  'FileCacheSourceHTTP',
182
187
  'FileCacheSourceGS',
183
- 'FileCacheSourceS3']
188
+ 'FileCacheSourceS3',
189
+ 'FileCacheSourceFake']
@@ -0,0 +1,34 @@
1
+ # file generated by setuptools-scm
2
+ # don't change, don't track in version control
3
+
4
+ __all__ = [
5
+ "__version__",
6
+ "__version_tuple__",
7
+ "version",
8
+ "version_tuple",
9
+ "__commit_id__",
10
+ "commit_id",
11
+ ]
12
+
13
+ TYPE_CHECKING = False
14
+ if TYPE_CHECKING:
15
+ from typing import Tuple
16
+ from typing import Union
17
+
18
+ VERSION_TUPLE = Tuple[Union[int, str], ...]
19
+ COMMIT_ID = Union[str, None]
20
+ else:
21
+ VERSION_TUPLE = object
22
+ COMMIT_ID = object
23
+
24
+ version: str
25
+ __version__: str
26
+ __version_tuple__: VERSION_TUPLE
27
+ version_tuple: VERSION_TUPLE
28
+ commit_id: COMMIT_ID
29
+ __commit_id__: COMMIT_ID
30
+
31
+ __version__ = version = '3.0.0'
32
+ __version_tuple__ = version_tuple = (3, 0, 0)
33
+
34
+ __commit_id__ = commit_id = 'g8dae7f090'