rms-filecache 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
filecache/__init__.py ADDED
@@ -0,0 +1,698 @@
1
+ """The purpose of the `filecache` module is to abstract away the location where files
2
+ used or generated by a program are stored. Files can be on the local file system, in
3
+ Google Cloud Storage, on Amazon Web Services S3, or on a webserver. When files to be read
4
+ are on the local file system, they are simply accessed in-place. Otherwise, they are
5
+ downloaded from the remote source to a local temporary directory. When files to be written
6
+ are on the local file system, they are simply written in-place. Otherwise, they are
7
+ written to a local temporary directory and then uploaded to the remote location (it is not
8
+ possible to upload to a webserver). When a cache is no longer needed, it is deleted from
9
+ the local disk.
10
+
11
+ The top-level file organization is provided by the :class:`FileCache` class. A
12
+ :class:`FileCache` instance is used to specify a particular **sharing policy** and
13
+ **lifetime**. For example, a cache could be private to the current process and group a set
14
+ of files that all have the same basic purpose. Once these files have been (downloaded and)
15
+ read, they are deleted as a group. Another cache could be shared among all processes on
16
+ the current machine and group a set of files that are needed by multiple processes, thus
17
+ allowing them to be downloaded from a remote source only one time, saving time and
18
+ bandwidth.
19
+
20
+ A :class:`FileCache` contains one or more :class:`FileCachePrefix` instances that each
21
+ define access to a **local or remote source/destination** for files. For example, one
22
+ instance could be used to access the local filesystem, while another could be used to
23
+ access a particular AWS S3 bucket.
24
+
25
+ A :class:`FileCache` can be instantiated either directly or as a context manager. When
26
+ instantiated directly, the programmer is responsible for calling
27
+ :meth:`FileCache.clean_up` directly to delete the cache when finished. In addition, a
28
+ non-shared cache will be deleted on program exit. When instantiated as a context manager,
29
+ a non-shared cache is deleted on exit from the context. See the particular class
30
+ documentation for full details.
31
+
32
+ Usage examples::
33
+
34
+ from filecache import FileCache
35
+ with FileCache() as fc:
36
+ pfx1 = fc.new_prefix('gs://rms-filecache-tests/subdir1')
37
+ pfx2 = fc.new_prefix('s3://rms-filecache-tests/subdir1/subdir2a')
38
+ with pfx1.open('subdir2a/binary1.bin', 'rb') as fp:
39
+ bin1 = fp.read()
40
+ with pfx2.open('binary1.bin', 'rb') as fp:
41
+ bin2 = fp.read()
42
+ assert bin1 == bin2
43
+
44
+ fc = FileCache()
45
+ pfx1 = fc.new_prefix('gs://rms-filecache-tests/subdir1')
46
+ pfx2 = fc.new_prefix('s3://rms-filecache-tests/subdir1/subdir2a')
47
+ path1 = pfx1.retrieve('subdir2a/binary1.bin')
48
+ with open(path1, 'rb') as fp:
49
+ bin1 = fp.read()
50
+ path2 = pfx2.retrieve('binary1.bin')
51
+ with open(path2, 'rb') as fp:
52
+ bin2 = fp.read()
53
+ fc.clean_up()
54
+ assert bin1 == bin2
55
+
56
+ with FileCache() as fc:
57
+ pfx = fc.new_prefix('gs://my-writable-bucket')
58
+ with pfx.open('output.txt', 'w') as fp:
59
+ fp.write('A')
60
+ with FileCache() as fc:
61
+ pfx = fc.new_prefix('gs://my-writable-bucket')
62
+ with pfx.open('output.txt', 'r') as fp:
63
+ print(fp.read())
64
+
65
+ A benefit of the abstraction is that different environments can access the same files in
66
+ different ways without needing to change the program code. For example, consider a program
67
+ that needs to access the file ``COISS_2xxx/COISS_2001/voldesc.cat`` from the NASA PDS
68
+ archives. This file might be stored on the local disk in the user's home directory in a
69
+ subdirectory called ``pds3-holdings``. Or if the user does not have a local copy, it is
70
+ accessible from a webserver at
71
+ ``https://pds-rings.seti.org/holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``.
72
+ Finally, it could be accessible from Google Cloud Storage from the ``rms-node-holdings``
73
+ bucket at
74
+ ``gs://rms-node-holdings/pds3-holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat``. Before
75
+ running the program, an environment variable could be set to one of these values::
76
+
77
+ $ export PDS3_HOLDINGS_DIR="~/pds3-holdings"
78
+ $ export PDS3_HOLDINGS_DIR="https://pds-rings.seti.org/holdings"
79
+ $ export PDS3_HOLDINGS_DIR="gs://rms-node-holdings/pds3-holdings"
80
+
81
+ Then the program could be written as::
82
+
83
+ from filecache import FileCache
84
+ import os
85
+ with FileCache() as fc:
86
+ pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_DIR'))
87
+ with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
88
+ contents = fp.read()
89
+
90
+ If the program was going to be run multiple times in a row, or multiple copies were going
91
+ to be run simultaneously, marking the cache as shared would allow all of the processes to
92
+ share the same copy, thus requiring only a single download no matter how many times the
93
+ program was run::
94
+
95
+ from filecache import FileCache
96
+ import os
97
+ with FileCache(shared=True) as fc:
98
+ pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_DIR'))
99
+ with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
100
+ contents = fp.read()
101
+ """
102
+
103
+ import atexit
104
+ import contextlib
105
+ import os
106
+ from pathlib import Path
107
+ import requests
108
+ import tempfile
109
+ import uuid
110
+
111
+ import filelock
112
+
113
+ import boto3
114
+ import botocore
115
+
116
+ from google.cloud import storage as gs_storage
117
+ import google.api_core.exceptions
118
+
119
+ try:
120
+ from ._version import __version__
121
+ except ImportError: # pragma: no cover
122
+ __version__ = 'Version unspecified'
123
+
124
+
125
+ _GLOBAL_LOGGER = False
126
+
127
+
128
+ def set_global_logger(logger):
129
+ """Set the global logger for all FileCache instances created in the future."""
130
+ global _GLOBAL_LOGGER
131
+ logger = logger if logger else False # Turn None into False
132
+ _GLOBAL_LOGGER = logger
133
+
134
+
135
+ def get_global_logger():
136
+ """Return the current global logger."""
137
+ return _GLOBAL_LOGGER
138
+
139
+
140
+ class FileCache:
141
+ _FILE_CACHE_PREFIX = '.file_cache_'
142
+
143
+ def __init__(self, temp_dir=None, shared=False, cache_owner=False, mp_safe=None,
144
+ atexit_cleanup=True, logger=None):
145
+ r"""Initialization for the FileCache class.
146
+
147
+ Note:
148
+ A shared cache will not be automatically deleted unless the `cache_owner`
149
+ option is specified or :meth:`clean_up` is called with the `final` option. If
150
+ neither of these is done, the shared cache will persist after program exit,
151
+ potentially forever, if the operating system does not purge the temporary
152
+ directory (this is, of course, also the advantage of a shared cache). Thus you
153
+ should be careful with shared caches to prevent unduly filling up your disk.
154
+
155
+ Parameters:
156
+ temp_dir (str or Path, optional): The directory in which to cache files.
157
+ In None, the system temporary directory is used, which involves checking
158
+ the environment variables ``TMPDIR``, ``TEMP``, and ``TMP``, and if none
159
+ of those are set then using ``C:\TEMP``, ``C:\TMP``, ``\TEMP``, or
160
+ ``\TMP`` on Windows and ``/tmp``, ``/var/tmp``, or ``/usr/tmp`` on other
161
+ platforms. The file cache will be stored in a sub-directory within this
162
+ temporary directory. To prevent security issues, the temporary directory
163
+ must already exist and be writeable.
164
+ shared (bool or str, optional): If False, the file cache will be
165
+ stored in a uniquely-named subdirectory of `temp_dir` with the prefix
166
+ ``.file_cache_``. If True, the file cache will be stored in a subdirectory
167
+ of `temp_dir` called ``.file_cache___global__``. If a string is specified,
168
+ the file cache will be stored in a subdirectory of `temp_dir` called
169
+ ``.file_cache_<shared>``. Shared caches will be available to all programs
170
+ creating a :class:`FileCache` with the same name, and may persist after
171
+ program exit and be available to programs running in the future.
172
+ cache_owner (bool, optional): This option is only relevant if `shared` is not
173
+ False. If `cache_owner` is True, this :class:`FileCache` instance is
174
+ considered the owner of the shared cache directory, and the cache will be
175
+ deleted on exit from a context manager, manual call to :meth:`clean_up`,
176
+ or on program exit just as if this were a non-shared cache. This option
177
+ should only be set to True if the thread/process creating this cache has
178
+ control over all other threads/processes that might be accessing it to
179
+ guarantee that the cache is not deleted until all potential users have
180
+ finished executing.
181
+ mp_safe (bool or None, optional): If False, never create new prefixes with
182
+ multiprocessor-safety locking. If True, always create new prefixes with
183
+ multiprocessor-safety locking. If None, safety locking is used if shared
184
+ is not False, as it is assumed that multiple programs will be using the
185
+ shared cache simultaneously.
186
+ atexit_cleanup (bool, optional): If True, at program exit automatically call
187
+ the clean_up method. Note that using this option will keep this class
188
+ instance around for the entire duration of the program, potentially
189
+ wasting memory. To be more memory conscious, but also be solely
190
+ responsible for calling clean_up, set this parameter to False.
191
+ logger (logger, optional): If False, do not do any logging. If None, use the
192
+ global logger set with :func:`set_global_logger`. Otherwise use the
193
+ specified logger. All messages are logged at the DEBUG level.
194
+
195
+ Notes:
196
+ :class:`FileCache` can be used as a context, such as::
197
+
198
+ with FileCache() as fc:
199
+ ...
200
+
201
+ In this case, the cache directory is created on entry to the context and
202
+ deleted on exit. However, if the cache is marked as shared, the directory will
203
+ not be deleted on exit unless the ``cache_owner=True`` option is used.
204
+ """
205
+
206
+ # We try very hard here to make sure that no possible passed-in argument for
207
+ # temp_dir or shared could result in a directory name that is anything other
208
+ # than a new cache directory. In particular, since we may be deleting this
209
+ # directory later, we want to make sure it's impossible for a bad actor to inject
210
+ # a directory or filename that could result in the deletion of system or user
211
+ # files. One key aspect of this is we do not allow the user to specify the
212
+ # specific subdirectory name without the unique prefix, and we do not allow
213
+ # the shared directory name to have additional directory components like "..".
214
+
215
+ if temp_dir is None:
216
+ temp_dir = tempfile.gettempdir()
217
+ temp_dir = Path(temp_dir).expanduser().resolve()
218
+
219
+ self._is_shared = True
220
+
221
+ if shared is False:
222
+ sub_dir = Path(f'{self._FILE_CACHE_PREFIX}{uuid.uuid4()}')
223
+ self._is_shared = False
224
+ elif shared is True:
225
+ sub_dir = Path(f'{self._FILE_CACHE_PREFIX}__global__')
226
+ elif isinstance(shared, str):
227
+ if '/' in shared or '\\' in shared:
228
+ raise ValueError('shared argument has directory elements')
229
+ sub_dir = Path(f'{self._FILE_CACHE_PREFIX}{shared}')
230
+ else:
231
+ raise TypeError('shared argument is of improper type')
232
+
233
+ if str(sub_dir.parent) != '.': # pragma: no cover - shouldn't be possible
234
+ raise ValueError('shared argument has directory elements')
235
+
236
+ self._logger = _GLOBAL_LOGGER if logger is None else logger
237
+
238
+ self._cache_dir = temp_dir / sub_dir
239
+ if self._logger:
240
+ if self._is_shared:
241
+ self._logger.debug(f'Creating shared cache {self._cache_dir}')
242
+ else:
243
+ self._logger.debug(f'Creating cache {self._cache_dir}')
244
+ self._cache_dir.mkdir(exist_ok=self._is_shared)
245
+
246
+ self._is_cache_owner = cache_owner
247
+ self._is_mp_safe = mp_safe if mp_safe is not None else self._is_shared
248
+
249
+ if atexit_cleanup:
250
+ atexit.register(self.clean_up)
251
+
252
+ @property
253
+ def cache_dir(self):
254
+ """Return the top-level directory of the cache as a Path object."""
255
+ return self._cache_dir
256
+
257
+ @property
258
+ def is_shared(self):
259
+ """Return a bool indicating whether or not this cache is shared."""
260
+ return self._is_shared
261
+
262
+ @property
263
+ def is_cache_owner(self):
264
+ """Return a bool indicating whether or not this FileCache owns a shared cache."""
265
+ return self._is_cache_owner
266
+
267
+ @property
268
+ def is_mp_safe(self):
269
+ """Return a bool indicating whether or not this FileCache is multi-processor safe.
270
+ """
271
+ return self._is_mp_safe
272
+
273
+ def new_prefix(self, prefix, anonymous=False, lock_timeout=60, **kwargs):
274
+ """Create a new FileCachePrefix with the given prefix.
275
+
276
+ Parameters:
277
+ prefix (Path or str): The prefix for the storage location. If the prefix
278
+ starts with ``gs://bucket-name`` it is from Google Storage. If the prefix
279
+ starts with ``s3://bucket-name`` it is from Amazon S3. If the prefix
280
+ starts with ``http://`` or ``https://`` it is from a website download.
281
+ Anything else is considered to be in the local filesystem and can be a str
282
+ or Path object.
283
+ anonymous (bool, optional): If True, access cloud resources (GS and S3)
284
+ without specifying credentials. Otherwise, credentials must be initialized
285
+ in the program's environment.
286
+ lock_timeout(int, optional): How long to wait, in seconds, if another process
287
+ is marked as retrieving the file before raising an exception. 0 means
288
+ to not wait at all. A negative value means to never time out.
289
+ """
290
+
291
+ return FileCachePrefix(prefix, self, anonymous=anonymous,
292
+ lock_timeout=lock_timeout, logger=self._logger,
293
+ **kwargs)
294
+
295
+ def clean_up(self, final=False):
296
+ """Delete all files stored in the cache including the cache directory.
297
+
298
+ Parameters:
299
+ final (bool, optional): If False and this :class:`FileCache` is not marked as
300
+ the `cache_owner` of the cache, a shared cache is left alone. If False and
301
+ this :class:`FileCache` is marked as the `cache_owner` of the cache, or if
302
+ True, a shared cache is deleted. Beware that this could affect other
303
+ processes using the same cache!
304
+ """
305
+
306
+ if self._logger:
307
+ self._logger.debug(f'Cleaning up cache {self._cache_dir}')
308
+
309
+ # Verify this is really a cache directory before walking it and deleting
310
+ # every file. We are just being paranoid to make sure this never does a
311
+ # "rm -rf" on a real directory like "/".
312
+ if not Path(self._cache_dir).name.startswith(self._FILE_CACHE_PREFIX):
313
+ raise ValueError(
314
+ f'Cache directory does not start with {self._FILE_CACHE_PREFIX}')
315
+
316
+ if not self._is_shared or (self._is_shared and (final or self.is_cache_owner)):
317
+ # Delete all of the files and subdirectories we left behind, including the
318
+ # file cache directory itself.
319
+ # We would like to use Path.walk() but that was only added in Python 3.12.
320
+ # We allow remove and rmdir to fail with FileNotFoundError because we could
321
+ # have two programs cleaning up a shared cache at the same time fighting
322
+ # each other, or someone could have asked for the local path to a file and
323
+ # then never written anything there.
324
+ for root, dirs, files in os.walk(self._cache_dir, topdown=False):
325
+ for name in files:
326
+ if self._logger:
327
+ self._logger.debug(f' Removing {name}')
328
+ try:
329
+ os.remove(os.path.join(root, name))
330
+ except FileNotFoundError: # pragma: no cover
331
+ pass
332
+ for name in dirs:
333
+ if self._logger:
334
+ self._logger.debug(f' Removing {name}')
335
+ try:
336
+ os.rmdir(os.path.join(root, name))
337
+ except FileNotFoundError: # pragma: no cover
338
+ pass
339
+
340
+ if self._logger:
341
+ self._logger.debug(f' Removing {self._cache_dir}')
342
+ try:
343
+ os.rmdir(self._cache_dir)
344
+ except FileNotFoundError: # pragma: no cover
345
+ pass
346
+
347
+ def __enter__(self):
348
+ return self
349
+
350
+ def __exit__(self, exc_type, exc_value, traceback):
351
+ self.clean_up()
352
+
353
+
354
+ class FileCachePrefix:
355
+ """Class for retrieving and uploading files in a FileCache."""
356
+
357
+ def __init__(self, prefix, filecache, anonymous=False, lock_timeout=60, logger=None):
358
+ """Initialization for the FileCachePrefix class.
359
+
360
+ Parameters:
361
+ prefix (str or Path): The prefix for the storage location. If the prefix
362
+ starts with ``gs://bucket-name`` it is from Google Storage. If the prefix
363
+ starts with ``s3://bucket-name`` it is from Amazon S3. If the prefix
364
+ starts with ``http://`` or ``https://`` it is from a website download.
365
+ Anything else is considered to be in the local filesystem and can be a str
366
+ or Path object.
367
+ file_cache (FileCache): The :class:`FileCache` in which to store files
368
+ retrieved from this prefix.
369
+ anonymous (bool, optional): If True, access cloud resources (GS and S3)
370
+ without specifying credentials.
371
+ lock_timeout(int, optional): How long to wait, in seconds, if another process
372
+ is marked as retrieving the file before raising an exception. 0 means to
373
+ not wait at all. A negative value means to never time out.
374
+ logger (logger, optional): If False, do not do any logging. If None, use the
375
+ global logger set with :func:`set_global_logger`. Otherwise use the
376
+ specified logger.
377
+
378
+ Notes:
379
+ If the specified :class:`FileCache` is marked as being multiprocessor-safe,
380
+ then file locking will be used to protect against multiple instances (threads
381
+ or processes) of :class:`FileCachePrefix` downloading the same file into the
382
+ same cache. Note that this will likely only work properly on a local
383
+ filesystem.
384
+
385
+ Within a given :class:`FileCache`, :class:`FileCachePrefix` instances that
386
+ reference the same local/remote source will be stored in the same location on
387
+ the local disk. Files downloaded into one instance will thus be visible in the
388
+ other instance.
389
+ """
390
+
391
+ self._filecache = filecache
392
+ self._lock_timeout = lock_timeout
393
+ self._logger = _GLOBAL_LOGGER if logger is None else logger
394
+ self._upload_counter = 0
395
+ self._download_counter = 0
396
+
397
+ if not isinstance(prefix, (str, Path)):
398
+ raise TypeError('prefix is not a str or Path')
399
+
400
+ prefix = str(prefix).rstrip('/') + '/'
401
+
402
+ if self._logger:
403
+ self._logger.debug(f'Initializing prefix {prefix}')
404
+
405
+ if prefix.startswith('gs://'):
406
+ self._init_gs_prefix(prefix, anonymous)
407
+ elif prefix.startswith('s3://'):
408
+ self._init_s3_prefix(prefix, anonymous)
409
+ elif prefix.startswith(('http://', 'https://')):
410
+ self._init_web_prefix(prefix)
411
+ else:
412
+ self._init_local_prefix(prefix)
413
+
414
+ def _init_gs_prefix(self, prefix, anonymous):
415
+ self._prefix_type = 'gs'
416
+ self._full_prefix = prefix
417
+ self._gs_client = (gs_storage.Client().create_anonymous_client()
418
+ if anonymous else gs_storage.Client())
419
+ self._gs_bucket_name, _, self._prefix = prefix.lstrip('gs://').partition('/')
420
+ self._gs_bucket = self._gs_client.bucket(self._gs_bucket_name)
421
+ self._cache_root = self._filecache._cache_dir / prefix.replace('gs://', 'gs_')
422
+
423
+ def _init_s3_prefix(self, prefix, anonymous):
424
+ self._prefix_type = 's3'
425
+ self._full_prefix = prefix
426
+ self._s3_client = (boto3.client('s3',
427
+ config=botocore.client.Config(
428
+ signature_version=botocore.UNSIGNED))
429
+ if anonymous else boto3.client('s3'))
430
+ self._s3_bucket_name, _, self._prefix = prefix.lstrip('s3://').partition('/')
431
+ self._cache_root = self._filecache._cache_dir / prefix.replace('s3://', 's3_')
432
+
433
+ def _init_web_prefix(self, prefix):
434
+ self._prefix_type = 'web'
435
+ self._full_prefix = prefix
436
+ self._prefix = prefix
437
+ self._cache_root = (self._filecache._cache_dir /
438
+ prefix.replace('http://', 'http_')
439
+ .replace('https://', 'http_'))
440
+
441
+ def _init_local_prefix(self, prefix):
442
+ self._prefix_type = 'local'
443
+ self._full_prefix = prefix
444
+ # This can raise FileNotFoundError if the prefix path doesn't exist
445
+ prefix = str(Path(prefix).expanduser().resolve(strict=True)).replace('\\', '/')
446
+ self._prefix = prefix
447
+ self._cache_root = Path(prefix)
448
+
449
+ def get_local_path(self, filename):
450
+ """Return the local path for the given filename using this prefix.
451
+
452
+ Parameters:
453
+ filename (str): The name of the file to use relative to this prefix.
454
+
455
+ Returns:
456
+ Path: The Path of the filename in the temporary directory. The file does not
457
+ have to exist because this path could be used for writing a file to upload.
458
+ Because of this, the complete parent directory structure will be created by
459
+ this function as necessary.
460
+ """
461
+
462
+ filename = filename.replace('\\', '/').lstrip('/')
463
+ local_path = self._cache_root / filename
464
+ local_path.parent.mkdir(parents=True, exist_ok=True)
465
+
466
+ if self._logger:
467
+ self._logger.debug(f'Returning local path for {filename} as {local_path}')
468
+
469
+ return local_path
470
+
471
+ def _lock_path(self, path):
472
+ path = Path(path)
473
+ return path.parent / f'.__lock__{path.name}'
474
+
475
+ def retrieve(self, filename):
476
+ """Retrieve a file from the storage location and store it in the file cache.
477
+
478
+ Parameters:
479
+ filename (str): The name of the file to retrieve relative to the prefix.
480
+
481
+ Returns:
482
+ Path: The Path of the filename in the temporary directory.
483
+
484
+ Raises:
485
+ FileNotFoundError: If the file does not exist.
486
+ TimeoutError: If we could not acquire the lock to allow downloading of the
487
+ file within the given timeout.
488
+ """
489
+
490
+ filename = filename.replace('\\', '/').lstrip('/')
491
+
492
+ if self._prefix_type == 'local':
493
+ local_path = Path(self._cache_root) / filename
494
+ if self._logger:
495
+ self._logger.debug(f'Accessing local file {filename}')
496
+ if '/..' in filename:
497
+ raise ValueError(f'Invalid filename {filename}')
498
+ if not local_path.exists():
499
+ raise FileNotFoundError(f'File does not exist: {local_path}')
500
+ # Don't tell the file cache about local files
501
+ return local_path
502
+
503
+ local_path = self._cache_root / filename
504
+
505
+ if self._filecache.is_mp_safe:
506
+ lock_path = self._lock_path(local_path)
507
+ lock = filelock.FileLock(lock_path, timeout=self._lock_timeout)
508
+ try:
509
+ lock.acquire()
510
+ except filelock._error.Timeout:
511
+ raise TimeoutError(f'Could not acquire lock on {lock_path}')
512
+ try:
513
+ return self._unprotected_retrieve(filename, local_path)
514
+ finally:
515
+ # Technically there is a potential race condition here, because after
516
+ # we release the lock, someone else could lock this file, and then we
517
+ # could delete it (because on Linux locks are only advisory). Then the
518
+ # next process to come along to try to lock this file would also succeed
519
+ # because it would really be a different lock file! However, we have
520
+ # to do it in this order because otherwise it won't work on Windows,
521
+ # where locks are not just advisory.
522
+ lock.release()
523
+ lock_path.unlink(missing_ok=True)
524
+
525
+ return self._unprotected_retrieve(filename, local_path)
526
+
527
+ def _unprotected_retrieve(self, filename, local_path):
528
+ if local_path.exists():
529
+ if self._logger:
530
+ self._logger.debug(
531
+ f'Accessing existing {self._cache_root/filename} at {local_path}')
532
+ return local_path
533
+
534
+ local_path.parent.mkdir(parents=True, exist_ok=True)
535
+
536
+ if self._logger:
537
+ self._logger.debug(
538
+ f'Downloading {self._cache_root/filename} to {local_path}')
539
+
540
+ if self._prefix_type == 'web':
541
+ ret = self._retrieve_from_web(filename, local_path)
542
+ elif self._prefix_type == 'gs':
543
+ ret = self._retrieve_from_gs(filename, local_path)
544
+ elif self._prefix_type == 's3':
545
+ ret = self._retrieve_from_s3(filename, local_path)
546
+ else:
547
+ raise AssertionError( # pragma: no cover
548
+ f'Internal error unknown prefix type {self._prefix_type}')
549
+
550
+ self._download_counter += 1
551
+
552
+ return ret
553
+
554
+ def _retrieve_from_web(self, filename, local_path):
555
+ url = f'{self._prefix}{filename}'
556
+ try:
557
+ response = requests.get(url, stream=True)
558
+ response.raise_for_status()
559
+ except requests.exceptions.RequestException as e:
560
+ raise FileNotFoundError(f'Failed to download file from: {url}') from e
561
+
562
+ temp_local_path = local_path.with_suffix(local_path.suffix + '.dltemp')
563
+ try:
564
+ with open(temp_local_path, 'wb') as f:
565
+ for chunk in response.iter_content(chunk_size=1024*1024):
566
+ f.write(chunk)
567
+ temp_local_path.rename(local_path)
568
+ except Exception: # pragma: no cover
569
+ temp_local_path.unlink(missing_ok=True)
570
+ raise
571
+
572
+ return local_path
573
+
574
+ def _retrieve_from_gs(self, filename, local_path):
575
+ blob_name = f'{self._prefix}{filename}'
576
+ blob = self._gs_bucket.blob(blob_name)
577
+
578
+ temp_local_path = local_path.with_suffix(local_path.suffix + '.dltemp')
579
+ try:
580
+ blob.download_to_filename(str(temp_local_path))
581
+ temp_local_path.rename(local_path)
582
+ except (google.api_core.exceptions.BadRequest, # bad bucket name
583
+ google.resumable_media.common.InvalidResponse, # bad bucket name
584
+ google.cloud.exceptions.NotFound): # bad filename
585
+ # The google API library will still create the file before noticing
586
+ # that it can't be downloaded, so we have to remove it here
587
+ temp_local_path.unlink(missing_ok=True)
588
+ raise FileNotFoundError(
589
+ f'Failed to download file from: gs://{self._gs_bucket_name}/'
590
+ f'{blob_name}')
591
+ except Exception: # pragma: no cover
592
+ temp_local_path.unlink(missing_ok=True)
593
+ raise
594
+
595
+ return local_path
596
+
597
+ def _retrieve_from_s3(self, filename, local_path):
598
+ s3_key = f'{self._prefix}{filename}'
599
+
600
+ temp_local_path = local_path.with_suffix(local_path.suffix + '.dltemp')
601
+ try:
602
+ self._s3_client.download_file(self._s3_bucket_name, s3_key,
603
+ str(temp_local_path))
604
+ temp_local_path.rename(local_path)
605
+ except botocore.exceptions.ClientError:
606
+ temp_local_path.unlink(missing_ok=True)
607
+ raise FileNotFoundError(
608
+ f'Failed to download file from: s3://{self._s3_bucket_name}/'
609
+ f'{s3_key}')
610
+ except Exception: # pragma: no cover
611
+ temp_local_path.unlink(missing_ok=True)
612
+ raise
613
+
614
+ return local_path
615
+
616
+ def upload(self, filename):
617
+ """Send a file from the file cache to the storage location.
618
+
619
+ Parameters:
620
+ filename (str): The name of the file to upload relative to the prefix.
621
+
622
+ Raises:
623
+ FileNotFoundError: If the file does not exist.
624
+ """
625
+
626
+ filename = filename.replace('\\', '/').lstrip('/')
627
+ local_path = self._cache_root / filename
628
+
629
+ if not local_path.exists():
630
+ raise FileNotFoundError(f'File {local_path} does not exist')
631
+
632
+ if self._prefix_type == 'local':
633
+ # We don't do anything for local paths since the file is already in the
634
+ # correct location.
635
+ return
636
+
637
+ if self._logger:
638
+ self._logger.debug(
639
+ f'Uploading {local_path} to {self._full_prefix}{filename}')
640
+
641
+ self._upload_counter += 1
642
+
643
+ if self._prefix_type == 'gs':
644
+ return self._upload_to_gs(filename, local_path)
645
+
646
+ if self._prefix_type == 's3':
647
+ return self._upload_to_s3(filename, local_path)
648
+
649
+ raise AssertionError(
650
+ f'Internal error unknown prefix type {self._prefix_type}') # pragma: no cover
651
+
652
+ def _upload_to_gs(self, filename, local_path):
653
+ blob_name = f'{self._prefix}{filename}'
654
+ blob = self._gs_bucket.blob(blob_name)
655
+
656
+ blob.upload_from_filename(str(local_path))
657
+
658
+ def _upload_to_s3(self, filename, local_path):
659
+ s3_key = f'{self._prefix}{filename}'
660
+
661
+ self._s3_client.upload_file(str(local_path), self._s3_bucket_name, s3_key)
662
+
663
+ @contextlib.contextmanager
664
+ def open(self, filename, mode='r', *args, **kwargs):
665
+ """Retrieve/open or open/upload a file as a context manager.
666
+
667
+ If `mode` is a read mode (like ``'r'`` or ``'rb'``) then the file will be first
668
+ retrieved by calling :meth:`retrieve` and then opened. If the `mode` is a write
669
+ mode (like ``'w'`` or ``'wb'``) then the file will be first opened for write, and
670
+ when this context manager is exited the file will be uploaded.
671
+
672
+ Parameters:
673
+ filename (str or Path): The filename to open.
674
+
675
+ Returns:
676
+ file-like object: The same object as would be returned by the normal `open()`
677
+ function.
678
+ """
679
+
680
+ if mode[0] == 'r':
681
+ local_path = self.retrieve(filename)
682
+ with open(local_path, mode, *args, **kwargs) as fp:
683
+ yield fp
684
+ else: # 'w', 'x', 'a'
685
+ local_path = self.get_local_path(filename)
686
+ with open(local_path, mode, *args, **kwargs) as fp:
687
+ yield fp
688
+ self.upload(filename)
689
+
690
+ @property
691
+ def download_counter(self):
692
+ """Return the number of actual file downloads that have taken place."""
693
+ return self._download_counter
694
+
695
+ @property
696
+ def upload_counter(self):
697
+ """Return the number of actual file uploads that have taken place."""
698
+ return self._upload_counter
filecache/_version.py ADDED
@@ -0,0 +1,16 @@
1
+ # file generated by setuptools_scm
2
+ # don't change, don't track in version control
3
+ TYPE_CHECKING = False
4
+ if TYPE_CHECKING:
5
+ from typing import Tuple, Union
6
+ VERSION_TUPLE = Tuple[Union[int, str], ...]
7
+ else:
8
+ VERSION_TUPLE = object
9
+
10
+ version: str
11
+ __version__: str
12
+ __version_tuple__: VERSION_TUPLE
13
+ version_tuple: VERSION_TUPLE
14
+
15
+ __version__ = version = '1.0.0'
16
+ __version_tuple__ = version_tuple = (1, 0, 0)
@@ -0,0 +1,201 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
@@ -0,0 +1,199 @@
1
+ Metadata-Version: 2.1
2
+ Name: rms-filecache
3
+ Version: 1.0.0
4
+ Summary: File cache for files retrieved from the cloud
5
+ Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
+ License: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/SETI/rms-filecache
8
+ Project-URL: Documentation, https://rms-filecache.readthedocs.io/en/latest
9
+ Project-URL: Repository, https://github.com/SETI/rms-filecache
10
+ Project-URL: Source, https://github.com/SETI/rms-filecache
11
+ Project-URL: Issues, https://github.com/SETI/rms-filecache/issues
12
+ Keywords: Google Cloud Storage,Google Storage,Amazon Web Services,S3,file cache
13
+ Classifier: Development Status :: 5 - Production/Stable
14
+ Classifier: Natural Language :: English
15
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
+ Classifier: Topic :: Utilities
17
+ Classifier: License :: OSI Approved :: Apache Software License
18
+ Classifier: Programming Language :: Python :: 3.9
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Operating System :: MacOS :: MacOS X
23
+ Classifier: Operating System :: POSIX :: Linux
24
+ Classifier: Operating System :: Microsoft :: Windows
25
+ Requires-Python: >=3.9
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+
29
+ [![GitHub release; latest by date](https://img.shields.io/github/v/release/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
30
+ [![GitHub Release Date](https://img.shields.io/github/release-date/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
31
+ [![Test Status](https://img.shields.io/github/actions/workflow/status/SETI/rms-filecache/run-tests.yml?branch=main)](https://github.com/SETI/rms-filecache/actions)
32
+ [![Documentation Status](https://readthedocs.org/projects/rms-filecache/badge/?version=latest)](https://rms-filecache.readthedocs.io/en/latest/?badge=latest)
33
+ [![Code coverage](https://img.shields.io/codecov/c/github/SETI/rms-filecache/main?logo=codecov)](https://codecov.io/gh/SETI/rms-filecache)
34
+ <br />
35
+ [![PyPI - Version](https://img.shields.io/pypi/v/rms-filecache)](https://pypi.org/project/rms-filecache)
36
+ [![PyPI - Format](https://img.shields.io/pypi/format/rms-filecache)](https://pypi.org/project/rms-filecache)
37
+ [![PyPI - Downloads](https://img.shields.io/pypi/dm/rms-filecache)](https://pypi.org/project/rms-filecache)
38
+ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/rms-filecache)](https://pypi.org/project/rms-filecache)
39
+ <br />
40
+ [![GitHub commits since latest release](https://img.shields.io/github/commits-since/SETI/rms-filecache/latest)](https://github.com/SETI/rms-filecache/commits/main/)
41
+ [![GitHub commit activity](https://img.shields.io/github/commit-activity/m/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/commits/main/)
42
+ [![GitHub last commit](https://img.shields.io/github/last-commit/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/commits/main/)
43
+ <br />
44
+ [![Number of GitHub open issues](https://img.shields.io/github/issues-raw/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/issues)
45
+ [![Number of GitHub closed issues](https://img.shields.io/github/issues-closed-raw/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/issues)
46
+ [![Number of GitHub open pull requests](https://img.shields.io/github/issues-pr-raw/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/pulls)
47
+ [![Number of GitHub closed pull requests](https://img.shields.io/github/issues-pr-closed-raw/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/pulls)
48
+ <br />
49
+ ![GitHub License](https://img.shields.io/github/license/SETI/rms-filecache)
50
+ [![Number of GitHub stars](https://img.shields.io/github/stars/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/stargazers)
51
+ ![GitHub forks](https://img.shields.io/github/forks/SETI/rms-filecache)
52
+
53
+ # Introduction
54
+
55
+ `filecache` is a Python module that abstracts away the location where files used or
56
+ generated by a program are stored. Files can be on the local file system, in Google Cloud
57
+ Storage, on Amazon Web Services S3, or on a webserver. When files to be read are on the
58
+ local file system, they are simply accessed in-place. Otherwise, they are downloaded from
59
+ the remote source to a local temporary directory. When files to be written are on the
60
+ local file system, they are simply written in-place. Otherwise, they are written to a
61
+ local temporary directory and then uploaded to the remote location (it is not possible to
62
+ upload to a webserver). When a cache is no longer needed, it is deleted from the local
63
+ disk.
64
+
65
+ `filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
66
+
67
+ # Installation
68
+
69
+ The `filecache` module is available via the `rms-filecache` package on PyPI and can be
70
+ installed with:
71
+
72
+ ```sh
73
+ pip install rms-filecache
74
+ ```
75
+
76
+ # Getting Started
77
+
78
+ The top-level file organization is provided by the `FileCache` class. A `FileCache`
79
+ instance is used to specify a particular **sharing policy** and **lifetime**. For example,
80
+ a cache could be private to the current process and group a set of files that all have the
81
+ same basic purpose. Once these files have been (downloaded and) read, they are deleted as
82
+ a group. Another cache could be shared among all processes on the current machine and
83
+ group a set of files that are needed by multiple processes, thus allowing them to be
84
+ downloaded from a remote source only one time, saving time and bandwidth.
85
+
86
+ A `FileCache` contains one or more `FileCachePrefix` instances that each define access to
87
+ a **local or remote source/destination** for files. For example, one instance could be
88
+ used to access the local filesystem, while another could be used to access a particular
89
+ AWS S3 bucket.
90
+
91
+ Usage examples:
92
+
93
+ ```python
94
+ from filecache import FileCache
95
+ with FileCache() as fc: # Context manager
96
+ # Use GS by specifying the bucket name and one directory level
97
+ pfx1 = fc.new_prefix('gs://rms-filecache-tests/subdir1')
98
+ # Use S3 by specifying the bucket namd and two directory levels
99
+ pfx2 = fc.new_prefix('s3://rms-filecache-tests/subdir1/subdir2a')
100
+ # Access GS using a directory + filename (since only one directory level
101
+ # was specified by the prefix)
102
+ with pfx1.open('subdir2a/binary1.bin', 'rb') as fp:
103
+ bin1 = fp.read()
104
+ # Access S3 using a filename only (since two directory levels were already
105
+ # specified by the prefix))
106
+ with pfx2.open('binary1.bin', 'rb') as fp:
107
+ bin2 = fp.read()
108
+ assert bin1 == bin2
109
+ # Cache automatically deleted here
110
+
111
+ # Same as above example but not using context managers for FileCache
112
+ fc = FileCache()
113
+ pfx1 = fc.new_prefix('gs://rms-filecache-tests/subdir1')
114
+ pfx2 = fc.new_prefix('s3://rms-filecache-tests/subdir1/subdir2a')
115
+ path1 = pfx1.retrieve('subdir2a/binary1.bin')
116
+ with open(path1, 'rb') as fp:
117
+ bin1 = fp.read()
118
+ path2 = pfx2.retrieve('binary1.bin')
119
+ with open(path2, 'rb') as fp:
120
+ bin2 = fp.read()
121
+ fc.clean_up() # Cache manually deleted here
122
+ assert bin1 == bin2
123
+
124
+ # Write a file to a bucket and read it back
125
+ with FileCache() as fc:
126
+ pfx = fc.new_prefix('gs://my-writable-bucket')
127
+ with pfx.open('output.txt', 'w') as fp:
128
+ fp.write('A')
129
+ # The cache will be deleted here so the file will have to be downloaded
130
+ with FileCache() as fc:
131
+ pfx = fc.new_prefix('gs://my-writable-bucket')
132
+ with pfx.open('output.txt', 'r') as fp:
133
+ print(fp.read())
134
+ ```
135
+
136
+ A benefit of the abstraction is that different environments can access the same files in
137
+ different ways without needing to change the program code. For example, consider a program
138
+ that needs to access the file `COISS_2xxx/COISS_2001/voldesc.cat` from the NASA PDS
139
+ archives. This file might be stored on the local disk in the user's home directory in a
140
+ subdirectory called `pds3-holdings`. Or if the user does not have a local copy, it is
141
+ accessible from a webserver at
142
+ `https://pds-rings.seti.org/holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat`.
143
+ Finally, it could be accessible from Google Cloud Storage from the `rms-node-holdings`
144
+ bucket at
145
+ `gs://rms-node-holdings/pds3-holdings/volumes/COISS_2xxx/COISS_2001/voldesc.cat`. Before
146
+ running the program, an environment variable could be set to one of these values:
147
+
148
+ ```sh
149
+ $ export PDS3_HOLDINGS_DIR="~/pds3-holdings"
150
+ $ export PDS3_HOLDINGS_DIR="https://pds-rings.seti.org/holdings"
151
+ $ export PDS3_HOLDINGS_DIR="gs://rms-node-holdings/pds3-holdings"
152
+ ```
153
+
154
+ Then the program could be written as::
155
+
156
+ ```python
157
+ from filecache import FileCache
158
+ import os
159
+ with FileCache() as fc:
160
+ pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_DIR'))
161
+ with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
162
+ contents = fp.read()
163
+ # Cache automatically deleted here
164
+ ```
165
+
166
+ If the program was going to be run multiple times in a row, or multiple copies were going
167
+ to be run simultaneously, marking the cache as shared would allow all of the processes to
168
+ share the same copy, thus requiring only a single download no matter how many times the
169
+ program was run:
170
+
171
+ ```python
172
+ from filecache import FileCache
173
+ import os
174
+ with FileCache(shared=True) as fc:
175
+ pfx = fc.new_prefix(os.getenv('PDS3_HOLDINGS_DIR'))
176
+ with pfx.open('volumes/COISS_2xxx/COISS_2001/voldesc.cat', 'r') as fp:
177
+ contents = fp.read()
178
+ # Cache not deleted here; must be deleted manually using fc.clean_up(final=True)
179
+ # If not deleted manually, the shared cache will persist until the temporary
180
+ # directory is purged by the operating system (which may be never)
181
+ ```
182
+
183
+ Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
184
+
185
+ # Contributing
186
+
187
+ Information on contributing to this package can be found in the
188
+ [Contributing Guide](https://github.com/SETI/rms-filecache/blob/main/CONTRIBUTING.md).
189
+
190
+ # Links
191
+
192
+ - [Documentation](https://rms-filecache.readthedocs.io)
193
+ - [Repository](https://github.com/SETI/rms-filecache)
194
+ - [Issue tracker](https://github.com/SETI/rms-filecache/issues)
195
+ - [PyPi](https://pypi.org/project/rms-filecache)
196
+
197
+ # Licensing
198
+
199
+ This code is licensed under the [Apache License v2.0](https://github.com/SETI/rms-filecache/blob/main/LICENSE).
@@ -0,0 +1,7 @@
1
+ filecache/__init__.py,sha256=Jy2pjzwyffjmEHtrrmuU6m-txjyPPN8mYmbrn07JZ8Q,31567
2
+ filecache/_version.py,sha256=DGJ4pj32xs3_DRJhSzQwCiRNnAQrMgo09USYpyMZsKc,411
3
+ rms_filecache-1.0.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
4
+ rms_filecache-1.0.0.dist-info/METADATA,sha256=OqbBy9f8iQErTlLKXWAArKu27PPPfFOwusPxUJAVBJY,10124
5
+ rms_filecache-1.0.0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
6
+ rms_filecache-1.0.0.dist-info/top_level.txt,sha256=E5EL4H3-Xj61UrzhbmBZg3Yq9bbCKMOTXxc7FjuQ0Os,10
7
+ rms_filecache-1.0.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (75.1.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ filecache