rms-filecache 1.2.0__tar.gz → 1.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/PKG-INFO +1 -1
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/filecache/__init__.py +37 -13
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/filecache/_version.py +2 -2
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/PKG-INFO +1 -1
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/test_filecache.py +12 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.coveragerc +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.flake8 +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.github/pull_request_template.md +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.github/workflows/publish_to_pypi.yml +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.github/workflows/publish_to_test_pypi.yml +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.github/workflows/run-tests.yml +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.gitignore +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/.readthedocs.yaml +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/LICENSE +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/README.md +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/Makefile +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/conf.py +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/index.rst +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/make.bat +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/docs/module.rst +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/pyproject.toml +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/requirements.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/SOURCES.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/setup.cfg +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_programs/locking.py +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_programs/multi_download.py +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/s3_bucket_policy_write.txt +0 -0
- {rms_filecache-1.2.0 → rms_filecache-1.2.4}/tests/s3_lifecycle_policy.txt +0 -0
|
@@ -410,7 +410,7 @@ class FileCache:
|
|
|
410
410
|
sub_path = full_path[idx+1:]
|
|
411
411
|
else:
|
|
412
412
|
# Local
|
|
413
|
-
sub_path = full_path
|
|
413
|
+
sub_path = str(Path(full_path).expanduser().resolve())
|
|
414
414
|
if not src_str.startswith(('gs://', 's3://')):
|
|
415
415
|
# No such thing as needing credentials for a local file or HTTP
|
|
416
416
|
# so don't overconstrain the source cache
|
|
@@ -474,7 +474,7 @@ class FileCache:
|
|
|
474
474
|
|
|
475
475
|
return ret
|
|
476
476
|
|
|
477
|
-
def get_local_path(self, full_path, anonymous=False):
|
|
477
|
+
def get_local_path(self, full_path, anonymous=False, create_parents=True):
|
|
478
478
|
"""Return the local path for the given full_path.
|
|
479
479
|
|
|
480
480
|
Parameters:
|
|
@@ -483,19 +483,25 @@ class FileCache:
|
|
|
483
483
|
without specifying credentials. Otherwise, credentials must be initialized
|
|
484
484
|
in the program's environment. This parameter can be overridden by the
|
|
485
485
|
:meth:`__init__` `all_anonymous` argument.
|
|
486
|
+
create_parents (bool, optional): If True, create all parent directories. This
|
|
487
|
+
is useful when getting the local path of a file that will be uploaded.
|
|
486
488
|
|
|
487
489
|
Returns:
|
|
488
490
|
Path: The Path of the filename in the temporary directory, or the `full_path`
|
|
489
491
|
if the file source is local. The file does not have to exist because this path
|
|
490
492
|
could be used for writing a file to upload. To facilitate this, a side effect
|
|
491
|
-
of this call is that the complete parent
|
|
492
|
-
by this function as necessary.
|
|
493
|
+
of this call (if `create_parents` is True) is that the complete parent
|
|
494
|
+
directory structure will be created by this function as necessary.
|
|
493
495
|
"""
|
|
494
496
|
|
|
495
497
|
source, sub_path, local_path = self._get_source_and_paths(full_path, anonymous)
|
|
496
|
-
|
|
498
|
+
if create_parents:
|
|
499
|
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
|
497
500
|
|
|
498
|
-
|
|
501
|
+
if source._src_type == 'local':
|
|
502
|
+
self._log_debug(f'Returning local path for {full_path} (local file)')
|
|
503
|
+
else:
|
|
504
|
+
self._log_debug(f'Returning local path for {full_path} as {local_path}')
|
|
499
505
|
|
|
500
506
|
return local_path
|
|
501
507
|
|
|
@@ -927,7 +933,10 @@ class FileCache:
|
|
|
927
933
|
|
|
928
934
|
source, sub_path, local_path = self._get_source_and_paths(full_path, anonymous)
|
|
929
935
|
|
|
930
|
-
|
|
936
|
+
if source._src_type == 'local':
|
|
937
|
+
self._log_debug(f'Uploading {local_path} (local file)')
|
|
938
|
+
else:
|
|
939
|
+
self._log_debug(f'Uploading {local_path} to {source._src_prefix_}{sub_path}')
|
|
931
940
|
|
|
932
941
|
try:
|
|
933
942
|
ret = source.upload(sub_path, local_path)
|
|
@@ -1149,6 +1158,8 @@ class FileCache:
|
|
|
1149
1158
|
os.rmdir(self._cache_dir)
|
|
1150
1159
|
except FileNotFoundError: # pragma: no cover - race condition only
|
|
1151
1160
|
pass
|
|
1161
|
+
else:
|
|
1162
|
+
self._log_debug(f' Ignoring shared cache {self._cache_dir}')
|
|
1152
1163
|
|
|
1153
1164
|
def __enter__(self):
|
|
1154
1165
|
"""Enter the context manager for creating a FileCache."""
|
|
@@ -1802,7 +1813,7 @@ class FileCachePrefix:
|
|
|
1802
1813
|
if not isinstance(prefix, (str, Path)):
|
|
1803
1814
|
raise TypeError('prefix is not a str or Path')
|
|
1804
1815
|
|
|
1805
|
-
self._prefix_ = str(prefix).rstrip('/') + '/'
|
|
1816
|
+
self._prefix_ = str(prefix).replace('\\', '/').rstrip('/') + '/'
|
|
1806
1817
|
|
|
1807
1818
|
self._filecache._log_debug(f'Initializing prefix {self._prefix_}')
|
|
1808
1819
|
|
|
@@ -1825,22 +1836,25 @@ class FileCachePrefix:
|
|
|
1825
1836
|
return self._filecache.exists(f'{self._prefix_}{sub_path}',
|
|
1826
1837
|
anonymous=self._anonymous)
|
|
1827
1838
|
|
|
1828
|
-
def get_local_path(self, sub_path):
|
|
1839
|
+
def get_local_path(self, sub_path, create_parents=True):
|
|
1829
1840
|
"""Return the local path for the given sub_path relative to the prefix.
|
|
1830
1841
|
|
|
1831
1842
|
Parameters:
|
|
1832
1843
|
sub_path (str): The path of the file relative to the prefix.
|
|
1844
|
+
create_parents (bool, optional): If True, create all parent directories. This
|
|
1845
|
+
is useful when getting the local path of a file that will be uploaded.
|
|
1833
1846
|
|
|
1834
1847
|
Returns:
|
|
1835
|
-
Path: The Path of the filename in the temporary directory, or the `
|
|
1848
|
+
Path: The Path of the filename in the temporary directory, or the `full_path`
|
|
1836
1849
|
if the file source is local. The file does not have to exist because this path
|
|
1837
1850
|
could be used for writing a file to upload. To facilitate this, a side effect
|
|
1838
|
-
of this call is that the complete parent
|
|
1839
|
-
by this function as necessary.
|
|
1851
|
+
of this call (if `create_parents` is True) is that the complete parent
|
|
1852
|
+
directory structure will be created by this function as necessary.
|
|
1840
1853
|
"""
|
|
1841
1854
|
|
|
1842
1855
|
return self._filecache.get_local_path(f'{self._prefix_}{sub_path}',
|
|
1843
|
-
anonymous=self._anonymous
|
|
1856
|
+
anonymous=self._anonymous,
|
|
1857
|
+
create_parents=create_parents)
|
|
1844
1858
|
|
|
1845
1859
|
def retrieve(self, sub_path, nthreads=None, exception_on_fail=True):
|
|
1846
1860
|
"""Retrieve a file(s) from the given sub_path and store it in the file cache.
|
|
@@ -2001,3 +2015,13 @@ class FileCachePrefix:
|
|
|
2001
2015
|
def upload_counter(self):
|
|
2002
2016
|
"""The number of actual file uploads that have taken place."""
|
|
2003
2017
|
return self._upload_counter
|
|
2018
|
+
|
|
2019
|
+
@property
|
|
2020
|
+
def prefix(self):
|
|
2021
|
+
"""The URI prefix including a trailing slash."""
|
|
2022
|
+
return self._prefix_
|
|
2023
|
+
|
|
2024
|
+
@property
|
|
2025
|
+
def is_local(self):
|
|
2026
|
+
"""A bool indicating whether or not the prefix refers to the local filesystem."""
|
|
2027
|
+
return not self._prefix_.startswith(('http://', 'https://', 'gs://', 's3://'))
|
|
@@ -400,6 +400,7 @@ def test_local_retr_pfx_good(shared):
|
|
|
400
400
|
for pass_no in range(5): # Make sure the expected dir doesn't get modified
|
|
401
401
|
with FileCache(shared=shared) as fc:
|
|
402
402
|
lf = fc.new_prefix(EXPECTED_DIR)
|
|
403
|
+
assert lf.is_local
|
|
403
404
|
for filename in EXPECTED_FILENAMES:
|
|
404
405
|
path = lf.retrieve(filename)
|
|
405
406
|
assert str(path).replace('\\', '/') == \
|
|
@@ -439,6 +440,7 @@ def test_cloud_retr_good(shared, prefix):
|
|
|
439
440
|
def test_cloud_retr_pfx_good(shared, prefix):
|
|
440
441
|
with FileCache(shared=shared) as fc:
|
|
441
442
|
pfx = fc.new_prefix(prefix, anonymous=True)
|
|
443
|
+
assert not pfx.is_local
|
|
442
444
|
for filename in LIMITED_FILENAMES:
|
|
443
445
|
path = pfx.retrieve(filename)
|
|
444
446
|
assert str(path).replace('\\', '/').endswith(filename)
|
|
@@ -997,7 +999,11 @@ def test_local_upl_good():
|
|
|
997
999
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
998
1000
|
temp_dir = Path(temp_dir).expanduser().resolve()
|
|
999
1001
|
with FileCache() as fc:
|
|
1002
|
+
local_path = fc.get_local_path(temp_dir / 'dir1/test_file.txt',
|
|
1003
|
+
create_parents=False)
|
|
1004
|
+
assert not local_path.parent.is_dir()
|
|
1000
1005
|
local_path = fc.get_local_path(temp_dir / 'dir1/test_file.txt')
|
|
1006
|
+
assert local_path.parent.is_dir()
|
|
1001
1007
|
assert local_path == (temp_dir / 'dir1/test_file.txt')
|
|
1002
1008
|
_copy_file(EXPECTED_DIR / EXPECTED_FILENAMES[0], local_path)
|
|
1003
1009
|
assert fc.upload(f'{temp_dir}/dir1/test_file.txt') == local_path
|
|
@@ -1011,7 +1017,11 @@ def test_local_upl_pfx_good():
|
|
|
1011
1017
|
temp_dir = Path(temp_dir).expanduser().resolve()
|
|
1012
1018
|
with FileCache() as fc:
|
|
1013
1019
|
pfx = fc.new_prefix(temp_dir)
|
|
1020
|
+
local_path = pfx.get_local_path('dir1/test_file.txt',
|
|
1021
|
+
create_parents=False)
|
|
1022
|
+
assert not local_path.parent.is_dir()
|
|
1014
1023
|
local_path = pfx.get_local_path('dir1/test_file.txt')
|
|
1024
|
+
assert local_path.parent.is_dir()
|
|
1015
1025
|
assert local_path == (temp_dir / 'dir1/test_file.txt')
|
|
1016
1026
|
_copy_file(EXPECTED_DIR / EXPECTED_FILENAMES[0], local_path)
|
|
1017
1027
|
assert pfx.upload('dir1/test_file.txt') == local_path
|
|
@@ -1395,6 +1405,8 @@ def test_complex_retr_multi_pfx_3(shared, prefix):
|
|
|
1395
1405
|
with FileCache(shared=shared, cache_owner=True, all_anonymous=True) as fc:
|
|
1396
1406
|
# Retrieve some cloud files with a bad name included
|
|
1397
1407
|
pfx = fc.new_prefix(prefix)
|
|
1408
|
+
clean_prefix = str(prefix).replace('\\', '/').rstrip('/')
|
|
1409
|
+
assert pfx.prefix == f'{clean_prefix}/'
|
|
1398
1410
|
full_paths = ['nonexistent.txt'] + list(EXPECTED_FILENAMES)
|
|
1399
1411
|
with pytest.raises(FileNotFoundError):
|
|
1400
1412
|
local_paths = pfx.retrieve(full_paths, exception_on_fail=True)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/subdir1/subdir2a/binary1.bin
RENAMED
|
File without changes
|
{rms_filecache-1.2.0 → rms_filecache-1.2.4}/test_files/expected/subdir1/subdir2b/binary1.bin
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|