rms-filecache 2.2.0__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-2.2.0/rms_filecache.egg-info → rms_filecache-2.3.0}/PKG-INFO +1 -1
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/filecache/_version.py +2 -2
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/filecache/file_cache_path.py +97 -132
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/pyproject.toml +1 -1
- {rms_filecache-2.2.0 → rms_filecache-2.3.0/rms_filecache.egg-info}/PKG-INFO +1 -1
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/test_file_cache.py +0 -9
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/test_file_cache_path.py +72 -32
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/.coveragerc +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/.flake8 +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/.github/pull_request_template.md +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/.github/workflows/publish_to_pypi.yml +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/.github/workflows/publish_to_test_pypi.yml +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/.github/workflows/run-tests.yml +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/.gitignore +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/.readthedocs.yaml +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/LICENSE +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/README.md +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/docs/Makefile +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/docs/conf.py +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/docs/index.rst +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/docs/make.bat +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/docs/module.rst +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/filecache/__init__.py +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/filecache/file_cache.py +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/filecache/file_cache_source.py +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/filecache/file_cache_types.py +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/filecache/py.typed +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/mypy.ini +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/requirements.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/rms_filecache.egg-info/SOURCES.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/setup.cfg +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/test_programs/locking.py +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/test_programs/multi_download.py +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/__init__.py +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/s3_bucket_policy_write.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/s3_lifecycle_policy.txt +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/test_file_cache_source.py +0 -0
- {rms_filecache-2.2.0 → rms_filecache-2.3.0}/tests/test_url_to_path.py +0 -0
|
@@ -23,7 +23,6 @@ if TYPE_CHECKING: # pragma: no cover
|
|
|
23
23
|
from .file_cache import FileCache # Circular import
|
|
24
24
|
|
|
25
25
|
from .file_cache_types import (StrOrPathOrSeqType,
|
|
26
|
-
StrOrSeqType,
|
|
27
26
|
UrlToPathFuncOrSeqType)
|
|
28
27
|
|
|
29
28
|
|
|
@@ -561,6 +560,14 @@ class FCPath:
|
|
|
561
560
|
|
|
562
561
|
return FCPath._is_absolute(self._path)
|
|
563
562
|
|
|
563
|
+
def as_absolute(self) -> FCPath:
|
|
564
|
+
"""Return the absolute version of this possibly-relative path."""
|
|
565
|
+
|
|
566
|
+
if FCPath._is_absolute(self._path):
|
|
567
|
+
return self
|
|
568
|
+
return FCPath(self.as_pathlib().expanduser().absolute().resolve(),
|
|
569
|
+
copy_from=self)
|
|
570
|
+
|
|
564
571
|
def match(self,
|
|
565
572
|
path_pattern: str | Path | FCPath) -> bool:
|
|
566
573
|
"""Return True if this path matches the given pattern.
|
|
@@ -614,6 +621,24 @@ class FCPath:
|
|
|
614
621
|
return _DEFAULT_FILECACHE
|
|
615
622
|
return self._filecache
|
|
616
623
|
|
|
624
|
+
def _make_paths_absolute(self,
|
|
625
|
+
sub_path: Optional[StrOrPathOrSeqType]) -> str | list[str]:
|
|
626
|
+
if isinstance(sub_path, (list, tuple)):
|
|
627
|
+
new_sub_paths: list[str] = []
|
|
628
|
+
for p in sub_path:
|
|
629
|
+
new_sub_path = FCPath._join(self._path, p)
|
|
630
|
+
if not FCPath._is_absolute(new_sub_path):
|
|
631
|
+
new_sub_path = (FCPath(Path(new_sub_path)
|
|
632
|
+
.expanduser().absolute().resolve())
|
|
633
|
+
.as_posix())
|
|
634
|
+
new_sub_paths.append(new_sub_path)
|
|
635
|
+
return new_sub_paths
|
|
636
|
+
new_sub_path = FCPath._join(self._path, sub_path)
|
|
637
|
+
if not FCPath._is_absolute(new_sub_path):
|
|
638
|
+
return (FCPath(Path(new_sub_path).expanduser().absolute().resolve())
|
|
639
|
+
.as_posix())
|
|
640
|
+
return new_sub_path
|
|
641
|
+
|
|
617
642
|
def get_local_path(self,
|
|
618
643
|
sub_path: Optional[StrOrPathOrSeqType] = None,
|
|
619
644
|
*,
|
|
@@ -625,6 +650,9 @@ class FCPath:
|
|
|
625
650
|
Parameters:
|
|
626
651
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
627
652
|
path is used. If `sub_path` is a list or tuple, all paths are processed.
|
|
653
|
+
If the resulting derived path is not absolute, it is assumed to be a
|
|
654
|
+
relative local path and is converted to an absolute path by expanding
|
|
655
|
+
usernames and resolving links.
|
|
628
656
|
create_parents: If True, create all parent directories. This is useful when
|
|
629
657
|
getting the local path of a file that will be uploaded.
|
|
630
658
|
url_to_path: The function (or list of functions) that is used to translate
|
|
@@ -658,26 +686,10 @@ class FCPath:
|
|
|
658
686
|
because a Path could be used for writing a file to upload. To facilitate this,
|
|
659
687
|
a side effect of this call (if `create_parents` is True) is that the complete
|
|
660
688
|
parent directory structure will be created for each returned Path.
|
|
661
|
-
|
|
662
|
-
Raises:
|
|
663
|
-
ValueError: If the derived path is not absolute.
|
|
664
689
|
"""
|
|
665
690
|
|
|
666
|
-
|
|
667
|
-
new_sub_paths = [FCPath._join(self._path, p) for p in sub_path]
|
|
668
|
-
if not all([FCPath._is_absolute(x) for x in new_sub_paths]):
|
|
669
|
-
raise ValueError(
|
|
670
|
-
f'Derived paths must be absolute, got {new_sub_paths}')
|
|
671
|
-
return self._filecache_to_use.get_local_path(cast(StrOrPathOrSeqType,
|
|
672
|
-
new_sub_paths),
|
|
673
|
-
anonymous=self._anonymous,
|
|
674
|
-
create_parents=create_parents,
|
|
675
|
-
url_to_path=url_to_path)
|
|
691
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
676
692
|
|
|
677
|
-
new_sub_path = FCPath._join(self._path, sub_path)
|
|
678
|
-
if not FCPath._is_absolute(str(new_sub_path)):
|
|
679
|
-
raise ValueError(
|
|
680
|
-
f'Derived path must be absolute, got {new_sub_path}')
|
|
681
693
|
return self._filecache_to_use.get_local_path(cast(StrOrPathOrSeqType,
|
|
682
694
|
new_sub_path),
|
|
683
695
|
anonymous=self._anonymous,
|
|
@@ -695,7 +707,9 @@ class FCPath:
|
|
|
695
707
|
|
|
696
708
|
Parameters:
|
|
697
709
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
698
|
-
path is used.
|
|
710
|
+
path is used. If the resulting derived path is not absolute, it is assumed
|
|
711
|
+
to be a relative local path and is converted to an absolute path by
|
|
712
|
+
expanding usernames and resolving links.
|
|
699
713
|
bypass_cache: If False, check for the file first in the local cache, and if
|
|
700
714
|
not found there then on the remote server. If True, only check on the
|
|
701
715
|
remote server.
|
|
@@ -732,37 +746,21 @@ class FCPath:
|
|
|
732
746
|
still not be downloadable due to permissions. False if the file does not
|
|
733
747
|
exist. This includes bad bucket or webserver names, lack of permission to
|
|
734
748
|
examine a bucket's contents, etc.
|
|
735
|
-
|
|
736
|
-
Raises:
|
|
737
|
-
ValueError: If the derived path is not absolute.
|
|
738
749
|
"""
|
|
739
750
|
|
|
740
751
|
nthreads = self._validate_nthreads(nthreads)
|
|
741
752
|
|
|
742
|
-
|
|
743
|
-
new_sub_paths = [FCPath._join(self._path, p) for p in sub_path]
|
|
744
|
-
if not all([FCPath._is_absolute(x) for x in new_sub_paths]):
|
|
745
|
-
raise ValueError(
|
|
746
|
-
f'Derived paths must be absolute, got {new_sub_paths}')
|
|
747
|
-
return self._filecache_to_use.exists(cast(StrOrPathOrSeqType,
|
|
748
|
-
new_sub_paths),
|
|
749
|
-
bypass_cache=bypass_cache,
|
|
750
|
-
nthreads=nthreads,
|
|
751
|
-
anonymous=self._anonymous,
|
|
752
|
-
url_to_path=url_to_path)
|
|
753
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
753
754
|
|
|
754
|
-
new_sub_path = FCPath._join(self._path, sub_path)
|
|
755
|
-
if not FCPath._is_absolute(str(new_sub_path)):
|
|
756
|
-
raise ValueError(
|
|
757
|
-
f'Derived path must be absolute, got {new_sub_path}')
|
|
758
755
|
return self._filecache_to_use.exists(cast(StrOrPathOrSeqType,
|
|
759
756
|
new_sub_path),
|
|
760
757
|
bypass_cache=bypass_cache,
|
|
758
|
+
nthreads=nthreads,
|
|
761
759
|
anonymous=self._anonymous,
|
|
762
760
|
url_to_path=url_to_path)
|
|
763
761
|
|
|
764
762
|
def retrieve(self,
|
|
765
|
-
sub_path: Optional[
|
|
763
|
+
sub_path: Optional[StrOrPathOrSeqType] = None,
|
|
766
764
|
*,
|
|
767
765
|
lock_timeout: Optional[int] = None,
|
|
768
766
|
nthreads: Optional[int] = None,
|
|
@@ -775,7 +773,10 @@ class FCPath:
|
|
|
775
773
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
776
774
|
path is used. If `sub_path` is a list or tuple, the complete list of files
|
|
777
775
|
is retrieved. Depending on the storage location, this may be more
|
|
778
|
-
efficient because files can be downloaded in parallel.
|
|
776
|
+
efficient because files can be downloaded in parallel. If the resulting
|
|
777
|
+
derived path is not absolute, it is assumed to be a relative local path
|
|
778
|
+
and is converted to an absolute path by expanding usernames and resolving
|
|
779
|
+
links.
|
|
779
780
|
nthreads: The maximum number of threads to use when doing multiple-file
|
|
780
781
|
retrieval or upload. If None, use the default value given when this
|
|
781
782
|
:class:`FCPath` was created.
|
|
@@ -829,7 +830,6 @@ class FCPath:
|
|
|
829
830
|
within the given timeout or, for a multi-file download, if we timed out
|
|
830
831
|
waiting for other processes to download locked files, and
|
|
831
832
|
exception_on_fail is True.
|
|
832
|
-
ValueError: If the derived path is not absolute.
|
|
833
833
|
|
|
834
834
|
Notes:
|
|
835
835
|
File download is normally an atomic operation; a program will never see a
|
|
@@ -845,30 +845,16 @@ class FCPath:
|
|
|
845
845
|
if lock_timeout is None:
|
|
846
846
|
lock_timeout = self._lock_timeout
|
|
847
847
|
|
|
848
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
849
|
+
|
|
848
850
|
try:
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
anonymous=self._anonymous,
|
|
857
|
-
lock_timeout=lock_timeout,
|
|
858
|
-
nthreads=nthreads,
|
|
859
|
-
exception_on_fail=exception_on_fail,
|
|
860
|
-
url_to_path=url_to_path)
|
|
861
|
-
else:
|
|
862
|
-
new_sub_path2 = FCPath._join(self._path, sub_path)
|
|
863
|
-
if not FCPath._is_absolute(str(new_sub_path2)):
|
|
864
|
-
raise ValueError(
|
|
865
|
-
f'Derived path must be absolute, got {new_sub_path2}')
|
|
866
|
-
ret = self._filecache_to_use.retrieve(cast(StrOrPathOrSeqType,
|
|
867
|
-
new_sub_path2),
|
|
868
|
-
anonymous=self._anonymous,
|
|
869
|
-
lock_timeout=lock_timeout,
|
|
870
|
-
exception_on_fail=exception_on_fail,
|
|
871
|
-
url_to_path=url_to_path)
|
|
851
|
+
ret = self._filecache_to_use.retrieve(cast(StrOrPathOrSeqType,
|
|
852
|
+
new_sub_path),
|
|
853
|
+
anonymous=self._anonymous,
|
|
854
|
+
lock_timeout=lock_timeout,
|
|
855
|
+
nthreads=nthreads,
|
|
856
|
+
exception_on_fail=exception_on_fail,
|
|
857
|
+
url_to_path=url_to_path)
|
|
872
858
|
finally:
|
|
873
859
|
self._download_counter += (self._filecache_to_use.download_counter -
|
|
874
860
|
old_download_counter)
|
|
@@ -876,7 +862,7 @@ class FCPath:
|
|
|
876
862
|
return ret
|
|
877
863
|
|
|
878
864
|
def upload(self,
|
|
879
|
-
sub_path: Optional[
|
|
865
|
+
sub_path: Optional[StrOrPathOrSeqType] = None,
|
|
880
866
|
*,
|
|
881
867
|
nthreads: Optional[int] = None,
|
|
882
868
|
exception_on_fail: bool = True,
|
|
@@ -888,7 +874,9 @@ class FCPath:
|
|
|
888
874
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
889
875
|
path is used. If `sub_path` is a list or tuple, the complete list of files
|
|
890
876
|
is uploaded. This may be more efficient because files can be uploaded in
|
|
891
|
-
parallel.
|
|
877
|
+
parallel. If the resulting derived path is not absolute, it is assumed to
|
|
878
|
+
be a relative local path and is converted to an absolute path by expanding
|
|
879
|
+
usernames and resolving links.
|
|
892
880
|
nthreads: The maximum number of threads to use when doing multiple-file
|
|
893
881
|
retrieval or upload. If None, use the default value given when this
|
|
894
882
|
:class:`FileCache` was created.
|
|
@@ -931,35 +919,21 @@ class FCPath:
|
|
|
931
919
|
Raises:
|
|
932
920
|
FileNotFoundError: If a file to upload does not exist or the upload failed,
|
|
933
921
|
and exception_on_fail is True.
|
|
934
|
-
ValueError: If the derived path is not absolute.
|
|
935
922
|
"""
|
|
936
923
|
|
|
937
924
|
old_upload_counter = self._filecache_to_use.upload_counter
|
|
938
925
|
|
|
939
926
|
nthreads = self._validate_nthreads(nthreads)
|
|
940
927
|
|
|
928
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
929
|
+
|
|
941
930
|
try:
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
new_sub_paths),
|
|
949
|
-
anonymous=self._anonymous,
|
|
950
|
-
nthreads=nthreads,
|
|
951
|
-
exception_on_fail=exception_on_fail,
|
|
952
|
-
url_to_path=url_to_path)
|
|
953
|
-
else:
|
|
954
|
-
new_sub_path = FCPath._join(self._path, sub_path)
|
|
955
|
-
if not FCPath._is_absolute(str(new_sub_path)):
|
|
956
|
-
raise ValueError(
|
|
957
|
-
f'Derived path must be absolute, got {new_sub_path}')
|
|
958
|
-
ret = self._filecache_to_use.upload(cast(StrOrPathOrSeqType,
|
|
959
|
-
new_sub_path),
|
|
960
|
-
anonymous=self._anonymous,
|
|
961
|
-
exception_on_fail=exception_on_fail,
|
|
962
|
-
url_to_path=url_to_path)
|
|
931
|
+
ret = self._filecache_to_use.upload(cast(StrOrPathOrSeqType,
|
|
932
|
+
new_sub_path),
|
|
933
|
+
anonymous=self._anonymous,
|
|
934
|
+
nthreads=nthreads,
|
|
935
|
+
exception_on_fail=exception_on_fail,
|
|
936
|
+
url_to_path=url_to_path)
|
|
963
937
|
finally:
|
|
964
938
|
self._upload_counter += (self._filecache_to_use.upload_counter -
|
|
965
939
|
old_upload_counter)
|
|
@@ -1030,7 +1004,7 @@ class FCPath:
|
|
|
1030
1004
|
self.upload(sub_path, url_to_path=url_to_path)
|
|
1031
1005
|
|
|
1032
1006
|
def unlink(self,
|
|
1033
|
-
sub_path: Optional[
|
|
1007
|
+
sub_path: Optional[StrOrPathOrSeqType] = None,
|
|
1034
1008
|
*,
|
|
1035
1009
|
missing_ok: bool = False,
|
|
1036
1010
|
nthreads: Optional[int] = None,
|
|
@@ -1043,7 +1017,10 @@ class FCPath:
|
|
|
1043
1017
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
1044
1018
|
path is used. If `sub_path` is a list or tuple, the complete list of files
|
|
1045
1019
|
is retrieved. Depending on the storage location, this may be more
|
|
1046
|
-
efficient because files can be downloaded in parallel.
|
|
1020
|
+
efficient because files can be downloaded in parallel. If the resulting
|
|
1021
|
+
derived path is not absolute, it is assumed to be a relative local path
|
|
1022
|
+
and is converted to an absolute path by expanding usernames and resolving
|
|
1023
|
+
links.
|
|
1047
1024
|
missing_ok: True to ignore attempting to unlink a file that doesn't exist.
|
|
1048
1025
|
nthreads: The maximum number of threads to use when doing multiple-file
|
|
1049
1026
|
retrieval or upload. If None, use the default value given when this
|
|
@@ -1081,30 +1058,17 @@ class FCPath:
|
|
|
1081
1058
|
See `pathlib.Path.unlink` for full documentation.
|
|
1082
1059
|
"""
|
|
1083
1060
|
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
url_to_path=url_to_path)
|
|
1096
|
-
else:
|
|
1097
|
-
new_sub_path2 = FCPath._join(self._path, sub_path)
|
|
1098
|
-
if not FCPath._is_absolute(str(new_sub_path2)):
|
|
1099
|
-
raise ValueError(
|
|
1100
|
-
f'Derived path must be absolute, got {new_sub_path2}')
|
|
1101
|
-
return self._filecache_to_use.unlink(cast(StrOrPathOrSeqType,
|
|
1102
|
-
new_sub_path2),
|
|
1103
|
-
missing_ok=missing_ok,
|
|
1104
|
-
anonymous=self._anonymous,
|
|
1105
|
-
nthreads=nthreads,
|
|
1106
|
-
exception_on_fail=exception_on_fail,
|
|
1107
|
-
url_to_path=url_to_path)
|
|
1061
|
+
nthreads = self._validate_nthreads(nthreads)
|
|
1062
|
+
|
|
1063
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
1064
|
+
|
|
1065
|
+
return self._filecache_to_use.unlink(cast(StrOrPathOrSeqType,
|
|
1066
|
+
new_sub_path),
|
|
1067
|
+
missing_ok=missing_ok,
|
|
1068
|
+
anonymous=self._anonymous,
|
|
1069
|
+
nthreads=nthreads,
|
|
1070
|
+
exception_on_fail=exception_on_fail,
|
|
1071
|
+
url_to_path=url_to_path)
|
|
1108
1072
|
|
|
1109
1073
|
@property
|
|
1110
1074
|
def download_counter(self) -> int:
|
|
@@ -1290,32 +1254,31 @@ class FCPath:
|
|
|
1290
1254
|
|
|
1291
1255
|
if not isinstance(target, FCPath):
|
|
1292
1256
|
target = FCPath(target)
|
|
1257
|
+
target = target.as_absolute()
|
|
1293
1258
|
|
|
1294
|
-
|
|
1295
|
-
raise ValueError('Source and target files must be absolute '
|
|
1296
|
-
f'{self.path!r} and f{target.path!r}')
|
|
1259
|
+
src = self.as_absolute()
|
|
1297
1260
|
|
|
1298
|
-
if
|
|
1261
|
+
if src.is_local() != target.is_local():
|
|
1299
1262
|
raise ValueError('Unable to rename files between local and remote locations: '
|
|
1300
|
-
f'{
|
|
1263
|
+
f'{src.path!r} and f{target.path!r}')
|
|
1301
1264
|
|
|
1302
|
-
drive1, root1, subpath1 = FCPath._split_parts(
|
|
1265
|
+
drive1, root1, subpath1 = FCPath._split_parts(src.path)
|
|
1303
1266
|
drive2, root2, subpath2 = FCPath._split_parts(target.path)
|
|
1304
1267
|
|
|
1305
1268
|
if drive1 != drive2 or root1 != root2:
|
|
1306
|
-
raise ValueError('Unable to rename files across
|
|
1307
|
-
f'{
|
|
1269
|
+
raise ValueError('Unable to rename files across locations: '
|
|
1270
|
+
f'{src.path!r} and f{target.path!r}')
|
|
1308
1271
|
|
|
1309
|
-
if
|
|
1272
|
+
if src.is_local():
|
|
1310
1273
|
# Local to local - just do an OS rename and be done with it
|
|
1311
1274
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1312
|
-
|
|
1275
|
+
src.as_pathlib().rename(target.as_pathlib())
|
|
1313
1276
|
return target
|
|
1314
1277
|
|
|
1315
1278
|
# Since you generally can't rename on a remote cloud location, first
|
|
1316
1279
|
# download the file, then rename it locally, then upload it to the new name,
|
|
1317
1280
|
# then delete the old name
|
|
1318
|
-
local_path = cast(Path,
|
|
1281
|
+
local_path = cast(Path, src.retrieve())
|
|
1319
1282
|
target_local_path = cast(Path, target.get_local_path())
|
|
1320
1283
|
local_path.rename(target_local_path) # Rename in the cache
|
|
1321
1284
|
try:
|
|
@@ -1323,7 +1286,7 @@ class FCPath:
|
|
|
1323
1286
|
except Exception:
|
|
1324
1287
|
target_local_path.unlink(missing_ok=True)
|
|
1325
1288
|
raise
|
|
1326
|
-
|
|
1289
|
+
src.unlink(missing_ok=True) # Delete the old version
|
|
1327
1290
|
|
|
1328
1291
|
return target
|
|
1329
1292
|
|
|
@@ -1617,7 +1580,8 @@ class FCPath:
|
|
|
1617
1580
|
"""
|
|
1618
1581
|
|
|
1619
1582
|
if self.is_local():
|
|
1620
|
-
return FCPath(self.as_pathlib().resolve(strict=strict),
|
|
1583
|
+
return FCPath(self.as_pathlib().absolute().resolve(strict=strict),
|
|
1584
|
+
copy_from=self)
|
|
1621
1585
|
|
|
1622
1586
|
return self
|
|
1623
1587
|
|
|
@@ -1991,12 +1955,13 @@ class _StringGlobber:
|
|
|
1991
1955
|
|
|
1992
1956
|
def select_recursive(path: str,
|
|
1993
1957
|
exists: bool = False) -> Generator[FCPath]:
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
1958
|
+
path_str = str(path)
|
|
1959
|
+
if path_str and path_str[-1] != '/':
|
|
1960
|
+
path_str = f'{path_str}/'
|
|
1961
|
+
match_pos = len(path_str)
|
|
1962
|
+
if match is None or match(path_str, match_pos):
|
|
1963
|
+
yield from select_next(path_str, exists)
|
|
1964
|
+
stack = [path_str]
|
|
2000
1965
|
while stack:
|
|
2001
1966
|
yield from select_recursive_step(stack, match_pos)
|
|
2002
1967
|
|
|
@@ -2361,16 +2361,7 @@ def test_rename_bad():
|
|
|
2361
2361
|
spath = FCPath(f'{S3_TEST_BUCKET_ROOT}/{uuid.uuid4()}')
|
|
2362
2362
|
gpath1 = gpath / 'test_file1.txt'
|
|
2363
2363
|
spath1 = spath / 'test_file1.txt'
|
|
2364
|
-
apath1 = '/tmp/test_file1.txt'
|
|
2365
|
-
lpath1 = 'test_file1.txt'
|
|
2366
2364
|
fpath1 = FCPath('/tmp/test_file1.txt')
|
|
2367
|
-
flpath1 = FCPath('test_file1.txt')
|
|
2368
|
-
with pytest.raises(ValueError):
|
|
2369
|
-
flpath1.rename(apath1)
|
|
2370
|
-
with pytest.raises(ValueError):
|
|
2371
|
-
fpath1.rename(flpath1)
|
|
2372
|
-
with pytest.raises(ValueError):
|
|
2373
|
-
fpath1.rename(lpath1)
|
|
2374
2365
|
with pytest.raises(ValueError):
|
|
2375
2366
|
gpath1.rename(spath1)
|
|
2376
2367
|
with pytest.raises(ValueError):
|
|
@@ -6,6 +6,7 @@ import os
|
|
|
6
6
|
from pathlib import Path, PurePath
|
|
7
7
|
import platform
|
|
8
8
|
import sys
|
|
9
|
+
import tempfile
|
|
9
10
|
import uuid
|
|
10
11
|
|
|
11
12
|
import pytest
|
|
@@ -676,38 +677,6 @@ def test_default_filecache():
|
|
|
676
677
|
assert p2.get_local_path() == p3.get_local_path()
|
|
677
678
|
|
|
678
679
|
|
|
679
|
-
def test_operations_relative_paths():
|
|
680
|
-
p = FCPath('b/c')
|
|
681
|
-
with pytest.raises(ValueError):
|
|
682
|
-
p.get_local_path()
|
|
683
|
-
with pytest.raises(ValueError):
|
|
684
|
-
p.get_local_path('d')
|
|
685
|
-
with pytest.raises(ValueError):
|
|
686
|
-
p.get_local_path(['d', 'e'])
|
|
687
|
-
with pytest.raises(ValueError):
|
|
688
|
-
p.retrieve()
|
|
689
|
-
with pytest.raises(ValueError):
|
|
690
|
-
p.retrieve('d')
|
|
691
|
-
with pytest.raises(ValueError):
|
|
692
|
-
p.retrieve(['d', 'e'])
|
|
693
|
-
with pytest.raises(ValueError):
|
|
694
|
-
p.exists()
|
|
695
|
-
with pytest.raises(ValueError):
|
|
696
|
-
p.exists('d')
|
|
697
|
-
with pytest.raises(ValueError):
|
|
698
|
-
p.exists(['d', 'e'])
|
|
699
|
-
with pytest.raises(ValueError):
|
|
700
|
-
p.upload()
|
|
701
|
-
with pytest.raises(ValueError):
|
|
702
|
-
p.upload('d')
|
|
703
|
-
with pytest.raises(ValueError):
|
|
704
|
-
p.upload(['d', 'e'])
|
|
705
|
-
with pytest.raises(ValueError):
|
|
706
|
-
p.unlink('d')
|
|
707
|
-
with pytest.raises(ValueError):
|
|
708
|
-
p.unlink(['d', 'e'])
|
|
709
|
-
|
|
710
|
-
|
|
711
680
|
def test_relative():
|
|
712
681
|
assert (FCPath(f'{GS_TEST_BUCKET_ROOT}/a/b/c.txt')
|
|
713
682
|
.relative_to(f'{GS_TEST_BUCKET_ROOT}/a/b')) == FCPath('c.txt')
|
|
@@ -1010,3 +979,74 @@ def test_bad_threads():
|
|
|
1010
979
|
FCPath('http://bucket/a/b/c', nthreads='a')
|
|
1011
980
|
with pytest.raises(ValueError):
|
|
1012
981
|
FCPath('http://bucket/a/b/c', nthreads=-1)
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
def test_relative_paths():
|
|
985
|
+
f_cur_dir = FCPath.cwd()
|
|
986
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
|
987
|
+
temp_dir = Path(temp_dir).expanduser().resolve()
|
|
988
|
+
try:
|
|
989
|
+
os.chdir(str(temp_dir))
|
|
990
|
+
with FileCache(cache_name=None) as fc:
|
|
991
|
+
rp1 = 'file1.txt'
|
|
992
|
+
rp2 = 'file2.txt'
|
|
993
|
+
ap1 = temp_dir / rp1
|
|
994
|
+
ap2 = temp_dir / rp2
|
|
995
|
+
frp1 = fc.new_path(rp1)
|
|
996
|
+
frp2 = fc.new_path(rp2)
|
|
997
|
+
fap1 = fc.new_path(ap1)
|
|
998
|
+
fap2 = fc.new_path(ap2)
|
|
999
|
+
assert fap1.get_local_path() == ap1
|
|
1000
|
+
assert fap2.get_local_path() == ap2
|
|
1001
|
+
assert frp1.get_local_path() == ap1
|
|
1002
|
+
assert frp2.get_local_path() == ap2
|
|
1003
|
+
assert FCPath('').get_local_path([rp1, rp2]) == [ap1, ap2]
|
|
1004
|
+
|
|
1005
|
+
frp2.touch()
|
|
1006
|
+
|
|
1007
|
+
assert not fap1.exists()
|
|
1008
|
+
assert fap2.exists()
|
|
1009
|
+
assert not frp1.exists()
|
|
1010
|
+
assert frp2.exists()
|
|
1011
|
+
assert FCPath('').exists([rp1, rp2]) == [False, True]
|
|
1012
|
+
|
|
1013
|
+
with pytest.raises(FileNotFoundError):
|
|
1014
|
+
frp1.retrieve()
|
|
1015
|
+
assert frp2.retrieve() == ap2
|
|
1016
|
+
ret = FCPath('').retrieve([rp1, rp2], exception_on_fail=False)
|
|
1017
|
+
assert isinstance(ret[0], FileNotFoundError)
|
|
1018
|
+
assert ret[1] == ap2
|
|
1019
|
+
|
|
1020
|
+
with pytest.raises(FileNotFoundError):
|
|
1021
|
+
frp1.upload()
|
|
1022
|
+
assert frp2.upload() == ap2
|
|
1023
|
+
ret = FCPath('').upload([rp1, rp2], exception_on_fail=False)
|
|
1024
|
+
assert isinstance(ret[0], FileNotFoundError)
|
|
1025
|
+
assert ret[1] == ap2
|
|
1026
|
+
|
|
1027
|
+
assert ap2.exists()
|
|
1028
|
+
frp2.unlink()
|
|
1029
|
+
assert not ap2.exists()
|
|
1030
|
+
|
|
1031
|
+
frp1.touch()
|
|
1032
|
+
frp2.touch()
|
|
1033
|
+
assert ap1.exists()
|
|
1034
|
+
assert ap2.exists()
|
|
1035
|
+
FCPath('').unlink([rp1, rp2])
|
|
1036
|
+
assert not ap1.exists()
|
|
1037
|
+
assert not ap2.exists()
|
|
1038
|
+
|
|
1039
|
+
frp1.touch()
|
|
1040
|
+
frp1.rename(frp2)
|
|
1041
|
+
assert not ap1.exists()
|
|
1042
|
+
assert ap2.exists()
|
|
1043
|
+
ap2.unlink()
|
|
1044
|
+
|
|
1045
|
+
frp1.touch()
|
|
1046
|
+
frp1.rename(rp2)
|
|
1047
|
+
assert not ap1.exists()
|
|
1048
|
+
assert ap2.exists()
|
|
1049
|
+
ap2.unlink()
|
|
1050
|
+
|
|
1051
|
+
finally:
|
|
1052
|
+
os.chdir(f_cur_dir.as_posix())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rms_filecache-2.2.0 → rms_filecache-2.3.0}/test_files/expected/subdir1/subdir2a/binary1.bin
RENAMED
|
File without changes
|
{rms_filecache-2.2.0 → rms_filecache-2.3.0}/test_files/expected/subdir1/subdir2b/binary1.bin
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|