rms-filecache 2.2.1__tar.gz → 2.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.gitignore +2 -0
- {rms_filecache-2.2.1/rms_filecache.egg-info → rms_filecache-2.3.1}/PKG-INFO +23 -16
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/README.md +20 -14
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/__init__.py +29 -23
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/_version.py +9 -4
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/file_cache.py +2 -1
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/file_cache_path.py +90 -126
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/file_cache_source.py +215 -3
- {rms_filecache-2.2.1 → rms_filecache-2.3.1/rms_filecache.egg-info}/PKG-INFO +23 -16
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/test_file_cache.py +0 -9
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/test_file_cache_path.py +120 -32
- rms_filecache-2.3.1/tests/test_file_cache_source.py +347 -0
- rms_filecache-2.2.1/tests/test_file_cache_source.py +0 -81
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.coveragerc +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.flake8 +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.github/pull_request_template.md +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.github/workflows/publish_to_pypi.yml +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.github/workflows/publish_to_test_pypi.yml +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.github/workflows/run-tests.yml +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.readthedocs.yaml +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/LICENSE +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/Makefile +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/conf.py +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/index.rst +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/make.bat +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/module.rst +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/file_cache_types.py +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/py.typed +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/mypy.ini +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/pyproject.toml +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/requirements.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/rms_filecache.egg-info/SOURCES.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/setup.cfg +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_programs/locking.py +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_programs/multi_download.py +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/__init__.py +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/s3_bucket_policy_write.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/s3_lifecycle_policy.txt +0 -0
- {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/test_url_to_path.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: rms-filecache
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.1
|
|
4
4
|
Summary: File cache for files retrieved from the cloud
|
|
5
5
|
Maintainer-email: "Robert S. French" <rfrench@seti.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -31,6 +31,7 @@ Requires-Dist: filelock
|
|
|
31
31
|
Requires-Dist: google-cloud-storage
|
|
32
32
|
Requires-Dist: requests
|
|
33
33
|
Requires-Dist: typing_extensions
|
|
34
|
+
Dynamic: license-file
|
|
34
35
|
|
|
35
36
|
[](https://github.com/SETI/rms-filecache/releases)
|
|
36
37
|
[](https://github.com/SETI/rms-filecache/releases)
|
|
@@ -59,14 +60,16 @@ Requires-Dist: typing_extensions
|
|
|
59
60
|
# Introduction
|
|
60
61
|
|
|
61
62
|
`filecache` is a Python module that abstracts away the location where files used or
|
|
62
|
-
generated by a program are stored. Files can be on the local file system
|
|
63
|
-
Storage, on Amazon Web Services S3, or on a
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
local file system, they are simply
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
63
|
+
generated by a program are stored. Files can be on the local file system (`/` or
|
|
64
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
|
|
65
|
+
webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
|
|
66
|
+
a remote using the local filesystem as a file repository (`fake://`). When files to be read
|
|
67
|
+
are on the local file system, they are simply accessed in-place. Otherwise, they are
|
|
68
|
+
downloaded from the remote source to a local temporary directory. When files to be written
|
|
69
|
+
are on the local file system, they are simply written in-place. Otherwise, they are
|
|
70
|
+
written to a local temporary directory and then uploaded to the remote location (it is not
|
|
71
|
+
possible to upload to a webserver). When a cache is no longer needed, it is deleted from
|
|
72
|
+
the local disk.
|
|
70
73
|
|
|
71
74
|
`filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
|
|
72
75
|
|
|
@@ -218,14 +221,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
|
218
221
|
contents = voldesc_path.read_text()
|
|
219
222
|
```
|
|
220
223
|
|
|
221
|
-
Finally, there are
|
|
222
|
-
locations without invoking any caching behavior: `
|
|
223
|
-
`FileCacheSourceHTTP`, `FileCacheSourceGS`, and `
|
|
224
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
225
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
226
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
|
|
227
|
+
The first four provide access to real storage locations, while `FileCacheSourceFake`
|
|
228
|
+
simulates a remote source for testing purposes:
|
|
224
229
|
|
|
225
230
|
```python
|
|
226
|
-
from filecache import FileCacheSourceGS
|
|
227
|
-
|
|
228
|
-
|
|
231
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake
|
|
232
|
+
src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
233
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
|
|
234
|
+
src2 = FileCacheSourceFake('fake://fake-bucket')
|
|
235
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
229
236
|
```
|
|
230
237
|
|
|
231
238
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -25,14 +25,16 @@
|
|
|
25
25
|
# Introduction
|
|
26
26
|
|
|
27
27
|
`filecache` is a Python module that abstracts away the location where files used or
|
|
28
|
-
generated by a program are stored. Files can be on the local file system
|
|
29
|
-
Storage, on Amazon Web Services S3, or on a
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
local file system, they are simply
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
generated by a program are stored. Files can be on the local file system (`/` or
|
|
29
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
|
|
30
|
+
webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
|
|
31
|
+
a remote using the local filesystem as a file repository (`fake://`). When files to be read
|
|
32
|
+
are on the local file system, they are simply accessed in-place. Otherwise, they are
|
|
33
|
+
downloaded from the remote source to a local temporary directory. When files to be written
|
|
34
|
+
are on the local file system, they are simply written in-place. Otherwise, they are
|
|
35
|
+
written to a local temporary directory and then uploaded to the remote location (it is not
|
|
36
|
+
possible to upload to a webserver). When a cache is no longer needed, it is deleted from
|
|
37
|
+
the local disk.
|
|
36
38
|
|
|
37
39
|
`filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
|
|
38
40
|
|
|
@@ -184,14 +186,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
|
|
|
184
186
|
contents = voldesc_path.read_text()
|
|
185
187
|
```
|
|
186
188
|
|
|
187
|
-
Finally, there are
|
|
188
|
-
locations without invoking any caching behavior: `
|
|
189
|
-
`FileCacheSourceHTTP`, `FileCacheSourceGS`, and `
|
|
189
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
190
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
191
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
|
|
192
|
+
The first four provide access to real storage locations, while `FileCacheSourceFake`
|
|
193
|
+
simulates a remote source for testing purposes:
|
|
190
194
|
|
|
191
195
|
```python
|
|
192
|
-
from filecache import FileCacheSourceGS
|
|
193
|
-
|
|
194
|
-
|
|
196
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake
|
|
197
|
+
src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
198
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
|
|
199
|
+
src2 = FileCacheSourceFake('fake://fake-bucket')
|
|
200
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
195
201
|
```
|
|
196
202
|
|
|
197
203
|
Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
|
|
@@ -3,14 +3,16 @@
|
|
|
3
3
|
##########################################################################################
|
|
4
4
|
|
|
5
5
|
"""The purpose of the `filecache` module is to abstract away the location where files
|
|
6
|
-
used or generated by a program are stored. Files can be on the local file system
|
|
7
|
-
Google Cloud Storage, on Amazon Web Services S3, or on
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
are on the local file system, they are simply
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
6
|
+
used or generated by a program are stored. Files can be on the local file system (`/` or
|
|
7
|
+
`file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on
|
|
8
|
+
a webserver (`http://` or `https://`). There is also a fake remote that can be used to
|
|
9
|
+
simulate a remote using the local filesystem as a file repository (`fake://`). When files
|
|
10
|
+
to be read are on the local file system, they are simply accessed in-place. Otherwise,
|
|
11
|
+
they are downloaded from the remote source to a local temporary directory. When files to
|
|
12
|
+
be written are on the local file system, they are simply written in-place. Otherwise, they
|
|
13
|
+
are written to a local temporary directory and then uploaded to the remote location (it is
|
|
14
|
+
not possible to upload to a webserver). When a cache is no longer needed, it is deleted
|
|
15
|
+
from the local disk.
|
|
14
16
|
|
|
15
17
|
The top-level file organization is provided by the :class:`FileCache` class. A
|
|
16
18
|
:class:`FileCache` instance is used to specify a particular **sharing policy** and
|
|
@@ -135,19 +137,21 @@ was run. A shared cache is indicated by giving the cache a name (or no argument,
|
|
|
135
137
|
defaults to ``"global"``); also :class:`FCPath` defaults to using the global cache if no
|
|
136
138
|
:class:`FileCache` is specified. This results in the simplest form of the program:
|
|
137
139
|
|
|
138
|
-
from filecache import FCPath
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
from filecache import FileCacheSourceGS
|
|
149
|
-
|
|
150
|
-
|
|
140
|
+
from filecache import FCPath import os p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
|
|
141
|
+
voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat' contents =
|
|
142
|
+
voldesc_path.read_text()
|
|
143
|
+
|
|
144
|
+
Finally, there are five classes that allow direct access to the five possible storage
|
|
145
|
+
locations without invoking any caching behavior: `FileCacheSourceFile`,
|
|
146
|
+
`FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and
|
|
147
|
+
`FileCacheSourceFake`. The first four provide access to real storage locations, while
|
|
148
|
+
`FileCacheSourceFake` simulates a remote source for testing purposes:
|
|
149
|
+
|
|
150
|
+
from filecache import FileCacheSourceGS, FileCacheSourceFake src1 =
|
|
151
|
+
FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
|
|
152
|
+
src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin') src2 =
|
|
153
|
+
FileCacheSourceFake('fake://fake-bucket')
|
|
154
|
+
src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
|
|
151
155
|
"""
|
|
152
156
|
|
|
153
157
|
import typing
|
|
@@ -168,7 +172,8 @@ from .file_cache_source import (FileCacheSource, # noqa: ignore E401
|
|
|
168
172
|
FileCacheSourceFile,
|
|
169
173
|
FileCacheSourceHTTP,
|
|
170
174
|
FileCacheSourceGS,
|
|
171
|
-
FileCacheSourceS3
|
|
175
|
+
FileCacheSourceS3,
|
|
176
|
+
FileCacheSourceFake)
|
|
172
177
|
|
|
173
178
|
__all__ = ['get_global_logger',
|
|
174
179
|
'register_filecachesource',
|
|
@@ -180,4 +185,5 @@ __all__ = ['get_global_logger',
|
|
|
180
185
|
'FileCacheSourceFile',
|
|
181
186
|
'FileCacheSourceHTTP',
|
|
182
187
|
'FileCacheSourceGS',
|
|
183
|
-
'FileCacheSourceS3'
|
|
188
|
+
'FileCacheSourceS3',
|
|
189
|
+
'FileCacheSourceFake']
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
# file generated by
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
2
|
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
|
|
5
|
+
|
|
3
6
|
TYPE_CHECKING = False
|
|
4
7
|
if TYPE_CHECKING:
|
|
5
|
-
from typing import Tuple
|
|
8
|
+
from typing import Tuple
|
|
9
|
+
from typing import Union
|
|
10
|
+
|
|
6
11
|
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
7
12
|
else:
|
|
8
13
|
VERSION_TUPLE = object
|
|
@@ -12,5 +17,5 @@ __version__: str
|
|
|
12
17
|
__version_tuple__: VERSION_TUPLE
|
|
13
18
|
version_tuple: VERSION_TUPLE
|
|
14
19
|
|
|
15
|
-
__version__ = version = '2.
|
|
16
|
-
__version_tuple__ = version_tuple = (2,
|
|
20
|
+
__version__ = version = '2.3.1'
|
|
21
|
+
__version_tuple__ = version_tuple = (2, 3, 1)
|
|
@@ -33,7 +33,7 @@ from .file_cache_source import (FileCacheSource,
|
|
|
33
33
|
FileCacheSourceHTTP,
|
|
34
34
|
FileCacheSourceGS,
|
|
35
35
|
FileCacheSourceS3,
|
|
36
|
-
)
|
|
36
|
+
FileCacheSourceFake)
|
|
37
37
|
from .file_cache_path import FCPath
|
|
38
38
|
from .file_cache_types import (StrOrPathOrSeqType,
|
|
39
39
|
UrlToPathFuncOrSeqType)
|
|
@@ -59,6 +59,7 @@ register_filecachesource(FileCacheSourceFile)
|
|
|
59
59
|
register_filecachesource(FileCacheSourceHTTP)
|
|
60
60
|
register_filecachesource(FileCacheSourceGS)
|
|
61
61
|
register_filecachesource(FileCacheSourceS3)
|
|
62
|
+
register_filecachesource(FileCacheSourceFake)
|
|
62
63
|
|
|
63
64
|
|
|
64
65
|
# Default logger for all FileCache instances that didn't specify one explicitly
|
|
@@ -23,7 +23,6 @@ if TYPE_CHECKING: # pragma: no cover
|
|
|
23
23
|
from .file_cache import FileCache # Circular import
|
|
24
24
|
|
|
25
25
|
from .file_cache_types import (StrOrPathOrSeqType,
|
|
26
|
-
StrOrSeqType,
|
|
27
26
|
UrlToPathFuncOrSeqType)
|
|
28
27
|
|
|
29
28
|
|
|
@@ -561,6 +560,14 @@ class FCPath:
|
|
|
561
560
|
|
|
562
561
|
return FCPath._is_absolute(self._path)
|
|
563
562
|
|
|
563
|
+
def as_absolute(self) -> FCPath:
|
|
564
|
+
"""Return the absolute version of this possibly-relative path."""
|
|
565
|
+
|
|
566
|
+
if FCPath._is_absolute(self._path):
|
|
567
|
+
return self
|
|
568
|
+
return FCPath(self.as_pathlib().expanduser().absolute().resolve(),
|
|
569
|
+
copy_from=self)
|
|
570
|
+
|
|
564
571
|
def match(self,
|
|
565
572
|
path_pattern: str | Path | FCPath) -> bool:
|
|
566
573
|
"""Return True if this path matches the given pattern.
|
|
@@ -614,6 +621,24 @@ class FCPath:
|
|
|
614
621
|
return _DEFAULT_FILECACHE
|
|
615
622
|
return self._filecache
|
|
616
623
|
|
|
624
|
+
def _make_paths_absolute(self,
|
|
625
|
+
sub_path: Optional[StrOrPathOrSeqType]) -> str | list[str]:
|
|
626
|
+
if isinstance(sub_path, (list, tuple)):
|
|
627
|
+
new_sub_paths: list[str] = []
|
|
628
|
+
for p in sub_path:
|
|
629
|
+
new_sub_path = FCPath._join(self._path, p)
|
|
630
|
+
if not FCPath._is_absolute(new_sub_path):
|
|
631
|
+
new_sub_path = (FCPath(Path(new_sub_path)
|
|
632
|
+
.expanduser().absolute().resolve())
|
|
633
|
+
.as_posix())
|
|
634
|
+
new_sub_paths.append(new_sub_path)
|
|
635
|
+
return new_sub_paths
|
|
636
|
+
new_sub_path = FCPath._join(self._path, sub_path)
|
|
637
|
+
if not FCPath._is_absolute(new_sub_path):
|
|
638
|
+
return (FCPath(Path(new_sub_path).expanduser().absolute().resolve())
|
|
639
|
+
.as_posix())
|
|
640
|
+
return new_sub_path
|
|
641
|
+
|
|
617
642
|
def get_local_path(self,
|
|
618
643
|
sub_path: Optional[StrOrPathOrSeqType] = None,
|
|
619
644
|
*,
|
|
@@ -625,6 +650,9 @@ class FCPath:
|
|
|
625
650
|
Parameters:
|
|
626
651
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
627
652
|
path is used. If `sub_path` is a list or tuple, all paths are processed.
|
|
653
|
+
If the resulting derived path is not absolute, it is assumed to be a
|
|
654
|
+
relative local path and is converted to an absolute path by expanding
|
|
655
|
+
usernames and resolving links.
|
|
628
656
|
create_parents: If True, create all parent directories. This is useful when
|
|
629
657
|
getting the local path of a file that will be uploaded.
|
|
630
658
|
url_to_path: The function (or list of functions) that is used to translate
|
|
@@ -658,26 +686,10 @@ class FCPath:
|
|
|
658
686
|
because a Path could be used for writing a file to upload. To facilitate this,
|
|
659
687
|
a side effect of this call (if `create_parents` is True) is that the complete
|
|
660
688
|
parent directory structure will be created for each returned Path.
|
|
661
|
-
|
|
662
|
-
Raises:
|
|
663
|
-
ValueError: If the derived path is not absolute.
|
|
664
689
|
"""
|
|
665
690
|
|
|
666
|
-
|
|
667
|
-
new_sub_paths = [FCPath._join(self._path, p) for p in sub_path]
|
|
668
|
-
if not all([FCPath._is_absolute(x) for x in new_sub_paths]):
|
|
669
|
-
raise ValueError(
|
|
670
|
-
f'Derived paths must be absolute, got {new_sub_paths}')
|
|
671
|
-
return self._filecache_to_use.get_local_path(cast(StrOrPathOrSeqType,
|
|
672
|
-
new_sub_paths),
|
|
673
|
-
anonymous=self._anonymous,
|
|
674
|
-
create_parents=create_parents,
|
|
675
|
-
url_to_path=url_to_path)
|
|
691
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
676
692
|
|
|
677
|
-
new_sub_path = FCPath._join(self._path, sub_path)
|
|
678
|
-
if not FCPath._is_absolute(str(new_sub_path)):
|
|
679
|
-
raise ValueError(
|
|
680
|
-
f'Derived path must be absolute, got {new_sub_path}')
|
|
681
693
|
return self._filecache_to_use.get_local_path(cast(StrOrPathOrSeqType,
|
|
682
694
|
new_sub_path),
|
|
683
695
|
anonymous=self._anonymous,
|
|
@@ -695,7 +707,9 @@ class FCPath:
|
|
|
695
707
|
|
|
696
708
|
Parameters:
|
|
697
709
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
698
|
-
path is used.
|
|
710
|
+
path is used. If the resulting derived path is not absolute, it is assumed
|
|
711
|
+
to be a relative local path and is converted to an absolute path by
|
|
712
|
+
expanding usernames and resolving links.
|
|
699
713
|
bypass_cache: If False, check for the file first in the local cache, and if
|
|
700
714
|
not found there then on the remote server. If True, only check on the
|
|
701
715
|
remote server.
|
|
@@ -732,37 +746,21 @@ class FCPath:
|
|
|
732
746
|
still not be downloadable due to permissions. False if the file does not
|
|
733
747
|
exist. This includes bad bucket or webserver names, lack of permission to
|
|
734
748
|
examine a bucket's contents, etc.
|
|
735
|
-
|
|
736
|
-
Raises:
|
|
737
|
-
ValueError: If the derived path is not absolute.
|
|
738
749
|
"""
|
|
739
750
|
|
|
740
751
|
nthreads = self._validate_nthreads(nthreads)
|
|
741
752
|
|
|
742
|
-
|
|
743
|
-
new_sub_paths = [FCPath._join(self._path, p) for p in sub_path]
|
|
744
|
-
if not all([FCPath._is_absolute(x) for x in new_sub_paths]):
|
|
745
|
-
raise ValueError(
|
|
746
|
-
f'Derived paths must be absolute, got {new_sub_paths}')
|
|
747
|
-
return self._filecache_to_use.exists(cast(StrOrPathOrSeqType,
|
|
748
|
-
new_sub_paths),
|
|
749
|
-
bypass_cache=bypass_cache,
|
|
750
|
-
nthreads=nthreads,
|
|
751
|
-
anonymous=self._anonymous,
|
|
752
|
-
url_to_path=url_to_path)
|
|
753
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
753
754
|
|
|
754
|
-
new_sub_path = FCPath._join(self._path, sub_path)
|
|
755
|
-
if not FCPath._is_absolute(str(new_sub_path)):
|
|
756
|
-
raise ValueError(
|
|
757
|
-
f'Derived path must be absolute, got {new_sub_path}')
|
|
758
755
|
return self._filecache_to_use.exists(cast(StrOrPathOrSeqType,
|
|
759
756
|
new_sub_path),
|
|
760
757
|
bypass_cache=bypass_cache,
|
|
758
|
+
nthreads=nthreads,
|
|
761
759
|
anonymous=self._anonymous,
|
|
762
760
|
url_to_path=url_to_path)
|
|
763
761
|
|
|
764
762
|
def retrieve(self,
|
|
765
|
-
sub_path: Optional[
|
|
763
|
+
sub_path: Optional[StrOrPathOrSeqType] = None,
|
|
766
764
|
*,
|
|
767
765
|
lock_timeout: Optional[int] = None,
|
|
768
766
|
nthreads: Optional[int] = None,
|
|
@@ -775,7 +773,10 @@ class FCPath:
|
|
|
775
773
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
776
774
|
path is used. If `sub_path` is a list or tuple, the complete list of files
|
|
777
775
|
is retrieved. Depending on the storage location, this may be more
|
|
778
|
-
efficient because files can be downloaded in parallel.
|
|
776
|
+
efficient because files can be downloaded in parallel. If the resulting
|
|
777
|
+
derived path is not absolute, it is assumed to be a relative local path
|
|
778
|
+
and is converted to an absolute path by expanding usernames and resolving
|
|
779
|
+
links.
|
|
779
780
|
nthreads: The maximum number of threads to use when doing multiple-file
|
|
780
781
|
retrieval or upload. If None, use the default value given when this
|
|
781
782
|
:class:`FCPath` was created.
|
|
@@ -829,7 +830,6 @@ class FCPath:
|
|
|
829
830
|
within the given timeout or, for a multi-file download, if we timed out
|
|
830
831
|
waiting for other processes to download locked files, and
|
|
831
832
|
exception_on_fail is True.
|
|
832
|
-
ValueError: If the derived path is not absolute.
|
|
833
833
|
|
|
834
834
|
Notes:
|
|
835
835
|
File download is normally an atomic operation; a program will never see a
|
|
@@ -845,30 +845,16 @@ class FCPath:
|
|
|
845
845
|
if lock_timeout is None:
|
|
846
846
|
lock_timeout = self._lock_timeout
|
|
847
847
|
|
|
848
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
849
|
+
|
|
848
850
|
try:
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
anonymous=self._anonymous,
|
|
857
|
-
lock_timeout=lock_timeout,
|
|
858
|
-
nthreads=nthreads,
|
|
859
|
-
exception_on_fail=exception_on_fail,
|
|
860
|
-
url_to_path=url_to_path)
|
|
861
|
-
else:
|
|
862
|
-
new_sub_path2 = FCPath._join(self._path, sub_path)
|
|
863
|
-
if not FCPath._is_absolute(str(new_sub_path2)):
|
|
864
|
-
raise ValueError(
|
|
865
|
-
f'Derived path must be absolute, got {new_sub_path2}')
|
|
866
|
-
ret = self._filecache_to_use.retrieve(cast(StrOrPathOrSeqType,
|
|
867
|
-
new_sub_path2),
|
|
868
|
-
anonymous=self._anonymous,
|
|
869
|
-
lock_timeout=lock_timeout,
|
|
870
|
-
exception_on_fail=exception_on_fail,
|
|
871
|
-
url_to_path=url_to_path)
|
|
851
|
+
ret = self._filecache_to_use.retrieve(cast(StrOrPathOrSeqType,
|
|
852
|
+
new_sub_path),
|
|
853
|
+
anonymous=self._anonymous,
|
|
854
|
+
lock_timeout=lock_timeout,
|
|
855
|
+
nthreads=nthreads,
|
|
856
|
+
exception_on_fail=exception_on_fail,
|
|
857
|
+
url_to_path=url_to_path)
|
|
872
858
|
finally:
|
|
873
859
|
self._download_counter += (self._filecache_to_use.download_counter -
|
|
874
860
|
old_download_counter)
|
|
@@ -876,7 +862,7 @@ class FCPath:
|
|
|
876
862
|
return ret
|
|
877
863
|
|
|
878
864
|
def upload(self,
|
|
879
|
-
sub_path: Optional[
|
|
865
|
+
sub_path: Optional[StrOrPathOrSeqType] = None,
|
|
880
866
|
*,
|
|
881
867
|
nthreads: Optional[int] = None,
|
|
882
868
|
exception_on_fail: bool = True,
|
|
@@ -888,7 +874,9 @@ class FCPath:
|
|
|
888
874
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
889
875
|
path is used. If `sub_path` is a list or tuple, the complete list of files
|
|
890
876
|
is uploaded. This may be more efficient because files can be uploaded in
|
|
891
|
-
parallel.
|
|
877
|
+
parallel. If the resulting derived path is not absolute, it is assumed to
|
|
878
|
+
be a relative local path and is converted to an absolute path by expanding
|
|
879
|
+
usernames and resolving links.
|
|
892
880
|
nthreads: The maximum number of threads to use when doing multiple-file
|
|
893
881
|
retrieval or upload. If None, use the default value given when this
|
|
894
882
|
:class:`FileCache` was created.
|
|
@@ -931,35 +919,21 @@ class FCPath:
|
|
|
931
919
|
Raises:
|
|
932
920
|
FileNotFoundError: If a file to upload does not exist or the upload failed,
|
|
933
921
|
and exception_on_fail is True.
|
|
934
|
-
ValueError: If the derived path is not absolute.
|
|
935
922
|
"""
|
|
936
923
|
|
|
937
924
|
old_upload_counter = self._filecache_to_use.upload_counter
|
|
938
925
|
|
|
939
926
|
nthreads = self._validate_nthreads(nthreads)
|
|
940
927
|
|
|
928
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
929
|
+
|
|
941
930
|
try:
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
new_sub_paths),
|
|
949
|
-
anonymous=self._anonymous,
|
|
950
|
-
nthreads=nthreads,
|
|
951
|
-
exception_on_fail=exception_on_fail,
|
|
952
|
-
url_to_path=url_to_path)
|
|
953
|
-
else:
|
|
954
|
-
new_sub_path = FCPath._join(self._path, sub_path)
|
|
955
|
-
if not FCPath._is_absolute(str(new_sub_path)):
|
|
956
|
-
raise ValueError(
|
|
957
|
-
f'Derived path must be absolute, got {new_sub_path}')
|
|
958
|
-
ret = self._filecache_to_use.upload(cast(StrOrPathOrSeqType,
|
|
959
|
-
new_sub_path),
|
|
960
|
-
anonymous=self._anonymous,
|
|
961
|
-
exception_on_fail=exception_on_fail,
|
|
962
|
-
url_to_path=url_to_path)
|
|
931
|
+
ret = self._filecache_to_use.upload(cast(StrOrPathOrSeqType,
|
|
932
|
+
new_sub_path),
|
|
933
|
+
anonymous=self._anonymous,
|
|
934
|
+
nthreads=nthreads,
|
|
935
|
+
exception_on_fail=exception_on_fail,
|
|
936
|
+
url_to_path=url_to_path)
|
|
963
937
|
finally:
|
|
964
938
|
self._upload_counter += (self._filecache_to_use.upload_counter -
|
|
965
939
|
old_upload_counter)
|
|
@@ -1030,7 +1004,7 @@ class FCPath:
|
|
|
1030
1004
|
self.upload(sub_path, url_to_path=url_to_path)
|
|
1031
1005
|
|
|
1032
1006
|
def unlink(self,
|
|
1033
|
-
sub_path: Optional[
|
|
1007
|
+
sub_path: Optional[StrOrPathOrSeqType] = None,
|
|
1034
1008
|
*,
|
|
1035
1009
|
missing_ok: bool = False,
|
|
1036
1010
|
nthreads: Optional[int] = None,
|
|
@@ -1043,7 +1017,10 @@ class FCPath:
|
|
|
1043
1017
|
sub_path: The path of the file relative to this path. If not specified, this
|
|
1044
1018
|
path is used. If `sub_path` is a list or tuple, the complete list of files
|
|
1045
1019
|
is retrieved. Depending on the storage location, this may be more
|
|
1046
|
-
efficient because files can be downloaded in parallel.
|
|
1020
|
+
efficient because files can be downloaded in parallel. If the resulting
|
|
1021
|
+
derived path is not absolute, it is assumed to be a relative local path
|
|
1022
|
+
and is converted to an absolute path by expanding usernames and resolving
|
|
1023
|
+
links.
|
|
1047
1024
|
missing_ok: True to ignore attempting to unlink a file that doesn't exist.
|
|
1048
1025
|
nthreads: The maximum number of threads to use when doing multiple-file
|
|
1049
1026
|
retrieval or upload. If None, use the default value given when this
|
|
@@ -1081,30 +1058,17 @@ class FCPath:
|
|
|
1081
1058
|
See `pathlib.Path.unlink` for full documentation.
|
|
1082
1059
|
"""
|
|
1083
1060
|
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
url_to_path=url_to_path)
|
|
1096
|
-
else:
|
|
1097
|
-
new_sub_path2 = FCPath._join(self._path, sub_path)
|
|
1098
|
-
if not FCPath._is_absolute(str(new_sub_path2)):
|
|
1099
|
-
raise ValueError(
|
|
1100
|
-
f'Derived path must be absolute, got {new_sub_path2}')
|
|
1101
|
-
return self._filecache_to_use.unlink(cast(StrOrPathOrSeqType,
|
|
1102
|
-
new_sub_path2),
|
|
1103
|
-
missing_ok=missing_ok,
|
|
1104
|
-
anonymous=self._anonymous,
|
|
1105
|
-
nthreads=nthreads,
|
|
1106
|
-
exception_on_fail=exception_on_fail,
|
|
1107
|
-
url_to_path=url_to_path)
|
|
1061
|
+
nthreads = self._validate_nthreads(nthreads)
|
|
1062
|
+
|
|
1063
|
+
new_sub_path = self._make_paths_absolute(sub_path)
|
|
1064
|
+
|
|
1065
|
+
return self._filecache_to_use.unlink(cast(StrOrPathOrSeqType,
|
|
1066
|
+
new_sub_path),
|
|
1067
|
+
missing_ok=missing_ok,
|
|
1068
|
+
anonymous=self._anonymous,
|
|
1069
|
+
nthreads=nthreads,
|
|
1070
|
+
exception_on_fail=exception_on_fail,
|
|
1071
|
+
url_to_path=url_to_path)
|
|
1108
1072
|
|
|
1109
1073
|
@property
|
|
1110
1074
|
def download_counter(self) -> int:
|
|
@@ -1290,32 +1254,31 @@ class FCPath:
|
|
|
1290
1254
|
|
|
1291
1255
|
if not isinstance(target, FCPath):
|
|
1292
1256
|
target = FCPath(target)
|
|
1257
|
+
target = target.as_absolute()
|
|
1293
1258
|
|
|
1294
|
-
|
|
1295
|
-
raise ValueError('Source and target files must be absolute '
|
|
1296
|
-
f'{self.path!r} and f{target.path!r}')
|
|
1259
|
+
src = self.as_absolute()
|
|
1297
1260
|
|
|
1298
|
-
if
|
|
1261
|
+
if src.is_local() != target.is_local():
|
|
1299
1262
|
raise ValueError('Unable to rename files between local and remote locations: '
|
|
1300
|
-
f'{
|
|
1263
|
+
f'{src.path!r} and f{target.path!r}')
|
|
1301
1264
|
|
|
1302
|
-
drive1, root1, subpath1 = FCPath._split_parts(
|
|
1265
|
+
drive1, root1, subpath1 = FCPath._split_parts(src.path)
|
|
1303
1266
|
drive2, root2, subpath2 = FCPath._split_parts(target.path)
|
|
1304
1267
|
|
|
1305
1268
|
if drive1 != drive2 or root1 != root2:
|
|
1306
|
-
raise ValueError('Unable to rename files across
|
|
1307
|
-
f'{
|
|
1269
|
+
raise ValueError('Unable to rename files across locations: '
|
|
1270
|
+
f'{src.path!r} and f{target.path!r}')
|
|
1308
1271
|
|
|
1309
|
-
if
|
|
1272
|
+
if src.is_local():
|
|
1310
1273
|
# Local to local - just do an OS rename and be done with it
|
|
1311
1274
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1312
|
-
|
|
1275
|
+
src.as_pathlib().rename(target.as_pathlib())
|
|
1313
1276
|
return target
|
|
1314
1277
|
|
|
1315
1278
|
# Since you generally can't rename on a remote cloud location, first
|
|
1316
1279
|
# download the file, then rename it locally, then upload it to the new name,
|
|
1317
1280
|
# then delete the old name
|
|
1318
|
-
local_path = cast(Path,
|
|
1281
|
+
local_path = cast(Path, src.retrieve())
|
|
1319
1282
|
target_local_path = cast(Path, target.get_local_path())
|
|
1320
1283
|
local_path.rename(target_local_path) # Rename in the cache
|
|
1321
1284
|
try:
|
|
@@ -1323,7 +1286,7 @@ class FCPath:
|
|
|
1323
1286
|
except Exception:
|
|
1324
1287
|
target_local_path.unlink(missing_ok=True)
|
|
1325
1288
|
raise
|
|
1326
|
-
|
|
1289
|
+
src.unlink(missing_ok=True) # Delete the old version
|
|
1327
1290
|
|
|
1328
1291
|
return target
|
|
1329
1292
|
|
|
@@ -1617,7 +1580,8 @@ class FCPath:
|
|
|
1617
1580
|
"""
|
|
1618
1581
|
|
|
1619
1582
|
if self.is_local():
|
|
1620
|
-
return FCPath(self.as_pathlib().resolve(strict=strict),
|
|
1583
|
+
return FCPath(self.as_pathlib().absolute().resolve(strict=strict),
|
|
1584
|
+
copy_from=self)
|
|
1621
1585
|
|
|
1622
1586
|
return self
|
|
1623
1587
|
|