rms-filecache 1.2.0__tar.gz → 1.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/PKG-INFO +1 -1
  2. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/filecache/__init__.py +37 -13
  3. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/filecache/_version.py +2 -2
  4. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/PKG-INFO +1 -1
  5. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/test_filecache.py +12 -0
  6. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.coveragerc +0 -0
  7. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.flake8 +0 -0
  8. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.github/pull_request_template.md +0 -0
  9. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.github/workflows/publish_to_pypi.yml +0 -0
  10. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.github/workflows/publish_to_test_pypi.yml +0 -0
  11. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.github/workflows/run-tests.yml +0 -0
  12. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.gitignore +0 -0
  13. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.readthedocs.yaml +0 -0
  14. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/LICENSE +0 -0
  15. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/README.md +0 -0
  16. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/Makefile +0 -0
  17. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/conf.py +0 -0
  18. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/index.rst +0 -0
  19. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/make.bat +0 -0
  20. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/module.rst +0 -0
  21. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/pyproject.toml +0 -0
  22. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/requirements.txt +0 -0
  23. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/SOURCES.txt +0 -0
  24. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/dependency_links.txt +0 -0
  25. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/requires.txt +0 -0
  26. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/top_level.txt +0 -0
  27. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/setup.cfg +0 -0
  28. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/lorem1.txt +0 -0
  29. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/subdir1/lorem1.txt +0 -0
  30. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
  31. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
  32. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_programs/locking.py +0 -0
  33. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_programs/multi_download.py +0 -0
  34. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/gs_lifecycle_config.json +0 -0
  35. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/gs_lifecycle_config_info.txt +0 -0
  36. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/s3_bucket_policy_read.txt +0 -0
  37. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/s3_bucket_policy_write.txt +0 -0
  38. {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/s3_lifecycle_policy.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: rms-filecache
3
- Version: 1.2.0
3
+ Version: 1.2.4
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -410,7 +410,7 @@ class FileCache:
410
410
  sub_path = full_path[idx+1:]
411
411
  else:
412
412
  # Local
413
- sub_path = full_path
413
+ sub_path = str(Path(full_path).expanduser().resolve())
414
414
  if not src_str.startswith(('gs://', 's3://')):
415
415
  # No such thing as needing credentials for a local file or HTTP
416
416
  # so don't overconstrain the source cache
@@ -474,7 +474,7 @@ class FileCache:
474
474
 
475
475
  return ret
476
476
 
477
- def get_local_path(self, full_path, anonymous=False):
477
+ def get_local_path(self, full_path, anonymous=False, create_parents=True):
478
478
  """Return the local path for the given full_path.
479
479
 
480
480
  Parameters:
@@ -483,19 +483,25 @@ class FileCache:
483
483
  without specifying credentials. Otherwise, credentials must be initialized
484
484
  in the program's environment. This parameter can be overridden by the
485
485
  :meth:`__init__` `all_anonymous` argument.
486
+ create_parents (bool, optional): If True, create all parent directories. This
487
+ is useful when getting the local path of a file that will be uploaded.
486
488
 
487
489
  Returns:
488
490
  Path: The Path of the filename in the temporary directory, or the `full_path`
489
491
  if the file source is local. The file does not have to exist because this path
490
492
  could be used for writing a file to upload. To facilitate this, a side effect
491
- of this call is that the complete parent directory structure will be created
492
- by this function as necessary.
493
+ of this call (if `create_parents` is True) is that the complete parent
494
+ directory structure will be created by this function as necessary.
493
495
  """
494
496
 
495
497
  source, sub_path, local_path = self._get_source_and_paths(full_path, anonymous)
496
- local_path.parent.mkdir(parents=True, exist_ok=True)
498
+ if create_parents:
499
+ local_path.parent.mkdir(parents=True, exist_ok=True)
497
500
 
498
- self._log_debug(f'Returning local path for {full_path} as {local_path}')
501
+ if source._src_type == 'local':
502
+ self._log_debug(f'Returning local path for {full_path} (local file)')
503
+ else:
504
+ self._log_debug(f'Returning local path for {full_path} as {local_path}')
499
505
 
500
506
  return local_path
501
507
 
@@ -927,7 +933,10 @@ class FileCache:
927
933
 
928
934
  source, sub_path, local_path = self._get_source_and_paths(full_path, anonymous)
929
935
 
930
- self._log_debug(f'Uploading {local_path} to {source._src_prefix_}{sub_path}')
936
+ if source._src_type == 'local':
937
+ self._log_debug(f'Uploading {local_path} (local file)')
938
+ else:
939
+ self._log_debug(f'Uploading {local_path} to {source._src_prefix_}{sub_path}')
931
940
 
932
941
  try:
933
942
  ret = source.upload(sub_path, local_path)
@@ -1149,6 +1158,8 @@ class FileCache:
1149
1158
  os.rmdir(self._cache_dir)
1150
1159
  except FileNotFoundError: # pragma: no cover - race condition only
1151
1160
  pass
1161
+ else:
1162
+ self._log_debug(f' Ignoring shared cache {self._cache_dir}')
1152
1163
 
1153
1164
  def __enter__(self):
1154
1165
  """Enter the context manager for creating a FileCache."""
@@ -1802,7 +1813,7 @@ class FileCachePrefix:
1802
1813
  if not isinstance(prefix, (str, Path)):
1803
1814
  raise TypeError('prefix is not a str or Path')
1804
1815
 
1805
- self._prefix_ = str(prefix).rstrip('/') + '/'
1816
+ self._prefix_ = str(prefix).replace('\\', '/').rstrip('/') + '/'
1806
1817
 
1807
1818
  self._filecache._log_debug(f'Initializing prefix {self._prefix_}')
1808
1819
 
@@ -1825,22 +1836,25 @@ class FileCachePrefix:
1825
1836
  return self._filecache.exists(f'{self._prefix_}{sub_path}',
1826
1837
  anonymous=self._anonymous)
1827
1838
 
1828
- def get_local_path(self, sub_path):
1839
+ def get_local_path(self, sub_path, create_parents=True):
1829
1840
  """Return the local path for the given sub_path relative to the prefix.
1830
1841
 
1831
1842
  Parameters:
1832
1843
  sub_path (str): The path of the file relative to the prefix.
1844
+ create_parents (bool, optional): If True, create all parent directories. This
1845
+ is useful when getting the local path of a file that will be uploaded.
1833
1846
 
1834
1847
  Returns:
1835
- Path: The Path of the filename in the temporary directory, or the `sub_path`
1848
+ Path: The Path of the filename in the temporary directory, or the `full_path`
1836
1849
  if the file source is local. The file does not have to exist because this path
1837
1850
  could be used for writing a file to upload. To facilitate this, a side effect
1838
- of this call is that the complete parent directory structure will be created
1839
- by this function as necessary.
1851
+ of this call (if `create_parents` is True) is that the complete parent
1852
+ directory structure will be created by this function as necessary.
1840
1853
  """
1841
1854
 
1842
1855
  return self._filecache.get_local_path(f'{self._prefix_}{sub_path}',
1843
- anonymous=self._anonymous)
1856
+ anonymous=self._anonymous,
1857
+ create_parents=create_parents)
1844
1858
 
1845
1859
  def retrieve(self, sub_path, nthreads=None, exception_on_fail=True):
1846
1860
  """Retrieve a file(s) from the given sub_path and store it in the file cache.
@@ -2001,3 +2015,13 @@ class FileCachePrefix:
2001
2015
  def upload_counter(self):
2002
2016
  """The number of actual file uploads that have taken place."""
2003
2017
  return self._upload_counter
2018
+
2019
+ @property
2020
+ def prefix(self):
2021
+ """The URI prefix including a trailing slash."""
2022
+ return self._prefix_
2023
+
2024
+ @property
2025
+ def is_local(self):
2026
+ """A bool indicating whether or not the prefix refers to the local filesystem."""
2027
+ return not self._prefix_.startswith(('http://', 'https://', 'gs://', 's3://'))
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '1.2.0'
16
- __version_tuple__ = version_tuple = (1, 2, 0)
15
+ __version__ = version = '1.2.4'
16
+ __version_tuple__ = version_tuple = (1, 2, 4)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: rms-filecache
3
- Version: 1.2.0
3
+ Version: 1.2.4
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -400,6 +400,7 @@ def test_local_retr_pfx_good(shared):
400
400
  for pass_no in range(5): # Make sure the expected dir doesn't get modified
401
401
  with FileCache(shared=shared) as fc:
402
402
  lf = fc.new_prefix(EXPECTED_DIR)
403
+ assert lf.is_local
403
404
  for filename in EXPECTED_FILENAMES:
404
405
  path = lf.retrieve(filename)
405
406
  assert str(path).replace('\\', '/') == \
@@ -439,6 +440,7 @@ def test_cloud_retr_good(shared, prefix):
439
440
  def test_cloud_retr_pfx_good(shared, prefix):
440
441
  with FileCache(shared=shared) as fc:
441
442
  pfx = fc.new_prefix(prefix, anonymous=True)
443
+ assert not pfx.is_local
442
444
  for filename in LIMITED_FILENAMES:
443
445
  path = pfx.retrieve(filename)
444
446
  assert str(path).replace('\\', '/').endswith(filename)
@@ -997,7 +999,11 @@ def test_local_upl_good():
997
999
  with tempfile.TemporaryDirectory() as temp_dir:
998
1000
  temp_dir = Path(temp_dir).expanduser().resolve()
999
1001
  with FileCache() as fc:
1002
+ local_path = fc.get_local_path(temp_dir / 'dir1/test_file.txt',
1003
+ create_parents=False)
1004
+ assert not local_path.parent.is_dir()
1000
1005
  local_path = fc.get_local_path(temp_dir / 'dir1/test_file.txt')
1006
+ assert local_path.parent.is_dir()
1001
1007
  assert local_path == (temp_dir / 'dir1/test_file.txt')
1002
1008
  _copy_file(EXPECTED_DIR / EXPECTED_FILENAMES[0], local_path)
1003
1009
  assert fc.upload(f'{temp_dir}/dir1/test_file.txt') == local_path
@@ -1011,7 +1017,11 @@ def test_local_upl_pfx_good():
1011
1017
  temp_dir = Path(temp_dir).expanduser().resolve()
1012
1018
  with FileCache() as fc:
1013
1019
  pfx = fc.new_prefix(temp_dir)
1020
+ local_path = pfx.get_local_path('dir1/test_file.txt',
1021
+ create_parents=False)
1022
+ assert not local_path.parent.is_dir()
1014
1023
  local_path = pfx.get_local_path('dir1/test_file.txt')
1024
+ assert local_path.parent.is_dir()
1015
1025
  assert local_path == (temp_dir / 'dir1/test_file.txt')
1016
1026
  _copy_file(EXPECTED_DIR / EXPECTED_FILENAMES[0], local_path)
1017
1027
  assert pfx.upload('dir1/test_file.txt') == local_path
@@ -1395,6 +1405,8 @@ def test_complex_retr_multi_pfx_3(shared, prefix):
1395
1405
  with FileCache(shared=shared, cache_owner=True, all_anonymous=True) as fc:
1396
1406
  # Retrieve some cloud files with a bad name included
1397
1407
  pfx = fc.new_prefix(prefix)
1408
+ clean_prefix = str(prefix).replace('\\', '/').rstrip('/')
1409
+ assert pfx.prefix == f'{clean_prefix}/'
1398
1410
  full_paths = ['nonexistent.txt'] + list(EXPECTED_FILENAMES)
1399
1411
  with pytest.raises(FileNotFoundError):
1400
1412
  local_paths = pfx.retrieve(full_paths, exception_on_fail=True)
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes