rms-filecache 2.2.1__tar.gz → 2.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.gitignore +2 -0
  2. {rms_filecache-2.2.1/rms_filecache.egg-info → rms_filecache-2.3.1}/PKG-INFO +23 -16
  3. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/README.md +20 -14
  4. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/__init__.py +29 -23
  5. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/_version.py +9 -4
  6. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/file_cache.py +2 -1
  7. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/file_cache_path.py +90 -126
  8. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/file_cache_source.py +215 -3
  9. {rms_filecache-2.2.1 → rms_filecache-2.3.1/rms_filecache.egg-info}/PKG-INFO +23 -16
  10. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/test_file_cache.py +0 -9
  11. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/test_file_cache_path.py +120 -32
  12. rms_filecache-2.3.1/tests/test_file_cache_source.py +347 -0
  13. rms_filecache-2.2.1/tests/test_file_cache_source.py +0 -81
  14. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.coveragerc +0 -0
  15. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.flake8 +0 -0
  16. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.github/pull_request_template.md +0 -0
  17. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.github/workflows/publish_to_pypi.yml +0 -0
  18. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.github/workflows/publish_to_test_pypi.yml +0 -0
  19. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.github/workflows/run-tests.yml +0 -0
  20. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/.readthedocs.yaml +0 -0
  21. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/LICENSE +0 -0
  22. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/Makefile +0 -0
  23. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/conf.py +0 -0
  24. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/index.rst +0 -0
  25. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/make.bat +0 -0
  26. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/docs/module.rst +0 -0
  27. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/file_cache_types.py +0 -0
  28. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/filecache/py.typed +0 -0
  29. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/mypy.ini +0 -0
  30. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/pyproject.toml +0 -0
  31. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/requirements.txt +0 -0
  32. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/rms_filecache.egg-info/SOURCES.txt +0 -0
  33. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/rms_filecache.egg-info/dependency_links.txt +0 -0
  34. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/rms_filecache.egg-info/requires.txt +0 -0
  35. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/rms_filecache.egg-info/top_level.txt +0 -0
  36. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/setup.cfg +0 -0
  37. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_files/expected/lorem1.txt +0 -0
  38. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_files/expected/subdir1/lorem1.txt +0 -0
  39. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
  40. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
  41. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_programs/locking.py +0 -0
  42. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/test_programs/multi_download.py +0 -0
  43. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/__init__.py +0 -0
  44. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/gs_lifecycle_config.json +0 -0
  45. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/gs_lifecycle_config_info.txt +0 -0
  46. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/s3_bucket_policy_read.txt +0 -0
  47. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/s3_bucket_policy_write.txt +0 -0
  48. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/s3_lifecycle_policy.txt +0 -0
  49. {rms_filecache-2.2.1 → rms_filecache-2.3.1}/tests/test_url_to_path.py +0 -0
@@ -159,3 +159,5 @@ cython_debug/
159
159
  # and can be added to the global gitignore or merged into this file. For a more nuclear
160
160
  # option (not recommended) you can uncomment the following to ignore the entire idea folder.
161
161
  #.idea/
162
+
163
+ .vscode/
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: rms-filecache
3
- Version: 2.2.1
3
+ Version: 2.3.1
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -31,6 +31,7 @@ Requires-Dist: filelock
31
31
  Requires-Dist: google-cloud-storage
32
32
  Requires-Dist: requests
33
33
  Requires-Dist: typing_extensions
34
+ Dynamic: license-file
34
35
 
35
36
  [![GitHub release; latest by date](https://img.shields.io/github/v/release/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
36
37
  [![GitHub Release Date](https://img.shields.io/github/release-date/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
@@ -59,14 +60,16 @@ Requires-Dist: typing_extensions
59
60
  # Introduction
60
61
 
61
62
  `filecache` is a Python module that abstracts away the location where files used or
62
- generated by a program are stored. Files can be on the local file system, in Google Cloud
63
- Storage, on Amazon Web Services S3, or on a webserver. When files to be read are on the
64
- local file system, they are simply accessed in-place. Otherwise, they are downloaded from
65
- the remote source to a local temporary directory. When files to be written are on the
66
- local file system, they are simply written in-place. Otherwise, they are written to a
67
- local temporary directory and then uploaded to the remote location (it is not possible to
68
- upload to a webserver). When a cache is no longer needed, it is deleted from the local
69
- disk.
63
+ generated by a program are stored. Files can be on the local file system (`/` or
64
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
65
+ webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
66
+ a remote using the local filesystem as a file repository (`fake://`). When files to be read
67
+ are on the local file system, they are simply accessed in-place. Otherwise, they are
68
+ downloaded from the remote source to a local temporary directory. When files to be written
69
+ are on the local file system, they are simply written in-place. Otherwise, they are
70
+ written to a local temporary directory and then uploaded to the remote location (it is not
71
+ possible to upload to a webserver). When a cache is no longer needed, it is deleted from
72
+ the local disk.
70
73
 
71
74
  `filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
72
75
 
@@ -218,14 +221,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
218
221
  contents = voldesc_path.read_text()
219
222
  ```
220
223
 
221
- Finally, there are four classes that allow direct access to the four possible storage
222
- locations without invoking any caching behavior: `FileCacheSourceLocal`,
223
- `FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
224
+ Finally, there are five classes that allow direct access to the five possible storage
225
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
226
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
227
+ The first four provide access to real storage locations, while `FileCacheSourceFake`
228
+ simulates a remote source for testing purposes:
224
229
 
225
230
  ```python
226
- from filecache import FileCacheSourceGS
227
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
228
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
231
+ from filecache import FileCacheSourceGS, FileCacheSourceFake
232
+ src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
233
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
234
+ src2 = FileCacheSourceFake('fake://fake-bucket')
235
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
229
236
  ```
230
237
 
231
238
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -25,14 +25,16 @@
25
25
  # Introduction
26
26
 
27
27
  `filecache` is a Python module that abstracts away the location where files used or
28
- generated by a program are stored. Files can be on the local file system, in Google Cloud
29
- Storage, on Amazon Web Services S3, or on a webserver. When files to be read are on the
30
- local file system, they are simply accessed in-place. Otherwise, they are downloaded from
31
- the remote source to a local temporary directory. When files to be written are on the
32
- local file system, they are simply written in-place. Otherwise, they are written to a
33
- local temporary directory and then uploaded to the remote location (it is not possible to
34
- upload to a webserver). When a cache is no longer needed, it is deleted from the local
35
- disk.
28
+ generated by a program are stored. Files can be on the local file system (`/` or
29
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
30
+ webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
31
+ a remote using the local filesystem as a file repository (`fake://`). When files to be read
32
+ are on the local file system, they are simply accessed in-place. Otherwise, they are
33
+ downloaded from the remote source to a local temporary directory. When files to be written
34
+ are on the local file system, they are simply written in-place. Otherwise, they are
35
+ written to a local temporary directory and then uploaded to the remote location (it is not
36
+ possible to upload to a webserver). When a cache is no longer needed, it is deleted from
37
+ the local disk.
36
38
 
37
39
  `filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
38
40
 
@@ -184,14 +186,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
184
186
  contents = voldesc_path.read_text()
185
187
  ```
186
188
 
187
- Finally, there are four classes that allow direct access to the four possible storage
188
- locations without invoking any caching behavior: `FileCacheSourceLocal`,
189
- `FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
189
+ Finally, there are five classes that allow direct access to the five possible storage
190
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
191
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
192
+ The first four provide access to real storage locations, while `FileCacheSourceFake`
193
+ simulates a remote source for testing purposes:
190
194
 
191
195
  ```python
192
- from filecache import FileCacheSourceGS
193
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
194
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
196
+ from filecache import FileCacheSourceGS, FileCacheSourceFake
197
+ src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
198
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
199
+ src2 = FileCacheSourceFake('fake://fake-bucket')
200
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
195
201
  ```
196
202
 
197
203
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -3,14 +3,16 @@
3
3
  ##########################################################################################
4
4
 
5
5
  """The purpose of the `filecache` module is to abstract away the location where files
6
- used or generated by a program are stored. Files can be on the local file system, in
7
- Google Cloud Storage, on Amazon Web Services S3, or on a webserver. When files to be read
8
- are on the local file system, they are simply accessed in-place. Otherwise, they are
9
- downloaded from the remote source to a local temporary directory. When files to be written
10
- are on the local file system, they are simply written in-place. Otherwise, they are
11
- written to a local temporary directory and then uploaded to the remote location (it is not
12
- possible to upload to a webserver). When a cache is no longer needed, it is deleted from
13
- the local disk.
6
+ used or generated by a program are stored. Files can be on the local file system (`/` or
7
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on
8
+ a webserver (`http://` or `https://`). There is also a fake remote that can be used to
9
+ simulate a remote using the local filesystem as a file repository (`fake://`). When files
10
+ to be read are on the local file system, they are simply accessed in-place. Otherwise,
11
+ they are downloaded from the remote source to a local temporary directory. When files to
12
+ be written are on the local file system, they are simply written in-place. Otherwise, they
13
+ are written to a local temporary directory and then uploaded to the remote location (it is
14
+ not possible to upload to a webserver). When a cache is no longer needed, it is deleted
15
+ from the local disk.
14
16
 
15
17
  The top-level file organization is provided by the :class:`FileCache` class. A
16
18
  :class:`FileCache` instance is used to specify a particular **sharing policy** and
@@ -135,19 +137,21 @@ was run. A shared cache is indicated by giving the cache a name (or no argument,
135
137
  defaults to ``"global"``); also :class:`FCPath` defaults to using the global cache if no
136
138
  :class:`FileCache` is specified. This results in the simplest form of the program:
137
139
 
138
- from filecache import FCPath
139
- import os
140
- p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
141
- voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
142
- contents = voldesc_path.read_text()
143
-
144
- Finally, there are four classes that allow direct access to the four possible storage
145
- locations without invoking any caching behavior: :class:`FileCacheSourceFile`,
146
- :class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`::
147
-
148
- from filecache import FileCacheSourceGS
149
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
150
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
140
+ from filecache import FCPath import os p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
141
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat' contents =
142
+ voldesc_path.read_text()
143
+
144
+ Finally, there are five classes that allow direct access to the five possible storage
145
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
146
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and
147
+ `FileCacheSourceFake`. The first four provide access to real storage locations, while
148
+ `FileCacheSourceFake` simulates a remote source for testing purposes:
149
+
150
+ from filecache import FileCacheSourceGS, FileCacheSourceFake src1 =
151
+ FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
152
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin') src2 =
153
+ FileCacheSourceFake('fake://fake-bucket')
154
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
151
155
  """
152
156
 
153
157
  import typing
@@ -168,7 +172,8 @@ from .file_cache_source import (FileCacheSource, # noqa: ignore E401
168
172
  FileCacheSourceFile,
169
173
  FileCacheSourceHTTP,
170
174
  FileCacheSourceGS,
171
- FileCacheSourceS3)
175
+ FileCacheSourceS3,
176
+ FileCacheSourceFake)
172
177
 
173
178
  __all__ = ['get_global_logger',
174
179
  'register_filecachesource',
@@ -180,4 +185,5 @@ __all__ = ['get_global_logger',
180
185
  'FileCacheSourceFile',
181
186
  'FileCacheSourceHTTP',
182
187
  'FileCacheSourceGS',
183
- 'FileCacheSourceS3']
188
+ 'FileCacheSourceS3',
189
+ 'FileCacheSourceFake']
@@ -1,8 +1,13 @@
1
- # file generated by setuptools_scm
1
+ # file generated by setuptools-scm
2
2
  # don't change, don't track in version control
3
+
4
+ __all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
5
+
3
6
  TYPE_CHECKING = False
4
7
  if TYPE_CHECKING:
5
- from typing import Tuple, Union
8
+ from typing import Tuple
9
+ from typing import Union
10
+
6
11
  VERSION_TUPLE = Tuple[Union[int, str], ...]
7
12
  else:
8
13
  VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
12
17
  __version_tuple__: VERSION_TUPLE
13
18
  version_tuple: VERSION_TUPLE
14
19
 
15
- __version__ = version = '2.2.1'
16
- __version_tuple__ = version_tuple = (2, 2, 1)
20
+ __version__ = version = '2.3.1'
21
+ __version_tuple__ = version_tuple = (2, 3, 1)
@@ -33,7 +33,7 @@ from .file_cache_source import (FileCacheSource,
33
33
  FileCacheSourceHTTP,
34
34
  FileCacheSourceGS,
35
35
  FileCacheSourceS3,
36
- )
36
+ FileCacheSourceFake)
37
37
  from .file_cache_path import FCPath
38
38
  from .file_cache_types import (StrOrPathOrSeqType,
39
39
  UrlToPathFuncOrSeqType)
@@ -59,6 +59,7 @@ register_filecachesource(FileCacheSourceFile)
59
59
  register_filecachesource(FileCacheSourceHTTP)
60
60
  register_filecachesource(FileCacheSourceGS)
61
61
  register_filecachesource(FileCacheSourceS3)
62
+ register_filecachesource(FileCacheSourceFake)
62
63
 
63
64
 
64
65
  # Default logger for all FileCache instances that didn't specify one explicitly
@@ -23,7 +23,6 @@ if TYPE_CHECKING: # pragma: no cover
23
23
  from .file_cache import FileCache # Circular import
24
24
 
25
25
  from .file_cache_types import (StrOrPathOrSeqType,
26
- StrOrSeqType,
27
26
  UrlToPathFuncOrSeqType)
28
27
 
29
28
 
@@ -561,6 +560,14 @@ class FCPath:
561
560
 
562
561
  return FCPath._is_absolute(self._path)
563
562
 
563
+ def as_absolute(self) -> FCPath:
564
+ """Return the absolute version of this possibly-relative path."""
565
+
566
+ if FCPath._is_absolute(self._path):
567
+ return self
568
+ return FCPath(self.as_pathlib().expanduser().absolute().resolve(),
569
+ copy_from=self)
570
+
564
571
  def match(self,
565
572
  path_pattern: str | Path | FCPath) -> bool:
566
573
  """Return True if this path matches the given pattern.
@@ -614,6 +621,24 @@ class FCPath:
614
621
  return _DEFAULT_FILECACHE
615
622
  return self._filecache
616
623
 
624
+ def _make_paths_absolute(self,
625
+ sub_path: Optional[StrOrPathOrSeqType]) -> str | list[str]:
626
+ if isinstance(sub_path, (list, tuple)):
627
+ new_sub_paths: list[str] = []
628
+ for p in sub_path:
629
+ new_sub_path = FCPath._join(self._path, p)
630
+ if not FCPath._is_absolute(new_sub_path):
631
+ new_sub_path = (FCPath(Path(new_sub_path)
632
+ .expanduser().absolute().resolve())
633
+ .as_posix())
634
+ new_sub_paths.append(new_sub_path)
635
+ return new_sub_paths
636
+ new_sub_path = FCPath._join(self._path, sub_path)
637
+ if not FCPath._is_absolute(new_sub_path):
638
+ return (FCPath(Path(new_sub_path).expanduser().absolute().resolve())
639
+ .as_posix())
640
+ return new_sub_path
641
+
617
642
  def get_local_path(self,
618
643
  sub_path: Optional[StrOrPathOrSeqType] = None,
619
644
  *,
@@ -625,6 +650,9 @@ class FCPath:
625
650
  Parameters:
626
651
  sub_path: The path of the file relative to this path. If not specified, this
627
652
  path is used. If `sub_path` is a list or tuple, all paths are processed.
653
+ If the resulting derived path is not absolute, it is assumed to be a
654
+ relative local path and is converted to an absolute path by expanding
655
+ usernames and resolving links.
628
656
  create_parents: If True, create all parent directories. This is useful when
629
657
  getting the local path of a file that will be uploaded.
630
658
  url_to_path: The function (or list of functions) that is used to translate
@@ -658,26 +686,10 @@ class FCPath:
658
686
  because a Path could be used for writing a file to upload. To facilitate this,
659
687
  a side effect of this call (if `create_parents` is True) is that the complete
660
688
  parent directory structure will be created for each returned Path.
661
-
662
- Raises:
663
- ValueError: If the derived path is not absolute.
664
689
  """
665
690
 
666
- if isinstance(sub_path, (list, tuple)):
667
- new_sub_paths = [FCPath._join(self._path, p) for p in sub_path]
668
- if not all([FCPath._is_absolute(x) for x in new_sub_paths]):
669
- raise ValueError(
670
- f'Derived paths must be absolute, got {new_sub_paths}')
671
- return self._filecache_to_use.get_local_path(cast(StrOrPathOrSeqType,
672
- new_sub_paths),
673
- anonymous=self._anonymous,
674
- create_parents=create_parents,
675
- url_to_path=url_to_path)
691
+ new_sub_path = self._make_paths_absolute(sub_path)
676
692
 
677
- new_sub_path = FCPath._join(self._path, sub_path)
678
- if not FCPath._is_absolute(str(new_sub_path)):
679
- raise ValueError(
680
- f'Derived path must be absolute, got {new_sub_path}')
681
693
  return self._filecache_to_use.get_local_path(cast(StrOrPathOrSeqType,
682
694
  new_sub_path),
683
695
  anonymous=self._anonymous,
@@ -695,7 +707,9 @@ class FCPath:
695
707
 
696
708
  Parameters:
697
709
  sub_path: The path of the file relative to this path. If not specified, this
698
- path is used.
710
+ path is used. If the resulting derived path is not absolute, it is assumed
711
+ to be a relative local path and is converted to an absolute path by
712
+ expanding usernames and resolving links.
699
713
  bypass_cache: If False, check for the file first in the local cache, and if
700
714
  not found there then on the remote server. If True, only check on the
701
715
  remote server.
@@ -732,37 +746,21 @@ class FCPath:
732
746
  still not be downloadable due to permissions. False if the file does not
733
747
  exist. This includes bad bucket or webserver names, lack of permission to
734
748
  examine a bucket's contents, etc.
735
-
736
- Raises:
737
- ValueError: If the derived path is not absolute.
738
749
  """
739
750
 
740
751
  nthreads = self._validate_nthreads(nthreads)
741
752
 
742
- if isinstance(sub_path, (list, tuple)):
743
- new_sub_paths = [FCPath._join(self._path, p) for p in sub_path]
744
- if not all([FCPath._is_absolute(x) for x in new_sub_paths]):
745
- raise ValueError(
746
- f'Derived paths must be absolute, got {new_sub_paths}')
747
- return self._filecache_to_use.exists(cast(StrOrPathOrSeqType,
748
- new_sub_paths),
749
- bypass_cache=bypass_cache,
750
- nthreads=nthreads,
751
- anonymous=self._anonymous,
752
- url_to_path=url_to_path)
753
+ new_sub_path = self._make_paths_absolute(sub_path)
753
754
 
754
- new_sub_path = FCPath._join(self._path, sub_path)
755
- if not FCPath._is_absolute(str(new_sub_path)):
756
- raise ValueError(
757
- f'Derived path must be absolute, got {new_sub_path}')
758
755
  return self._filecache_to_use.exists(cast(StrOrPathOrSeqType,
759
756
  new_sub_path),
760
757
  bypass_cache=bypass_cache,
758
+ nthreads=nthreads,
761
759
  anonymous=self._anonymous,
762
760
  url_to_path=url_to_path)
763
761
 
764
762
  def retrieve(self,
765
- sub_path: Optional[StrOrSeqType] = None,
763
+ sub_path: Optional[StrOrPathOrSeqType] = None,
766
764
  *,
767
765
  lock_timeout: Optional[int] = None,
768
766
  nthreads: Optional[int] = None,
@@ -775,7 +773,10 @@ class FCPath:
775
773
  sub_path: The path of the file relative to this path. If not specified, this
776
774
  path is used. If `sub_path` is a list or tuple, the complete list of files
777
775
  is retrieved. Depending on the storage location, this may be more
778
- efficient because files can be downloaded in parallel.
776
+ efficient because files can be downloaded in parallel. If the resulting
777
+ derived path is not absolute, it is assumed to be a relative local path
778
+ and is converted to an absolute path by expanding usernames and resolving
779
+ links.
779
780
  nthreads: The maximum number of threads to use when doing multiple-file
780
781
  retrieval or upload. If None, use the default value given when this
781
782
  :class:`FCPath` was created.
@@ -829,7 +830,6 @@ class FCPath:
829
830
  within the given timeout or, for a multi-file download, if we timed out
830
831
  waiting for other processes to download locked files, and
831
832
  exception_on_fail is True.
832
- ValueError: If the derived path is not absolute.
833
833
 
834
834
  Notes:
835
835
  File download is normally an atomic operation; a program will never see a
@@ -845,30 +845,16 @@ class FCPath:
845
845
  if lock_timeout is None:
846
846
  lock_timeout = self._lock_timeout
847
847
 
848
+ new_sub_path = self._make_paths_absolute(sub_path)
849
+
848
850
  try:
849
- if isinstance(sub_path, (list, tuple)):
850
- new_sub_paths = [FCPath._join(self._path, p) for p in sub_path]
851
- if not all([FCPath._is_absolute(x) for x in new_sub_paths]):
852
- raise ValueError(
853
- f'Derived paths must be absolute, got {new_sub_paths}')
854
- ret = self._filecache_to_use.retrieve(cast(StrOrPathOrSeqType,
855
- new_sub_paths),
856
- anonymous=self._anonymous,
857
- lock_timeout=lock_timeout,
858
- nthreads=nthreads,
859
- exception_on_fail=exception_on_fail,
860
- url_to_path=url_to_path)
861
- else:
862
- new_sub_path2 = FCPath._join(self._path, sub_path)
863
- if not FCPath._is_absolute(str(new_sub_path2)):
864
- raise ValueError(
865
- f'Derived path must be absolute, got {new_sub_path2}')
866
- ret = self._filecache_to_use.retrieve(cast(StrOrPathOrSeqType,
867
- new_sub_path2),
868
- anonymous=self._anonymous,
869
- lock_timeout=lock_timeout,
870
- exception_on_fail=exception_on_fail,
871
- url_to_path=url_to_path)
851
+ ret = self._filecache_to_use.retrieve(cast(StrOrPathOrSeqType,
852
+ new_sub_path),
853
+ anonymous=self._anonymous,
854
+ lock_timeout=lock_timeout,
855
+ nthreads=nthreads,
856
+ exception_on_fail=exception_on_fail,
857
+ url_to_path=url_to_path)
872
858
  finally:
873
859
  self._download_counter += (self._filecache_to_use.download_counter -
874
860
  old_download_counter)
@@ -876,7 +862,7 @@ class FCPath:
876
862
  return ret
877
863
 
878
864
  def upload(self,
879
- sub_path: Optional[StrOrSeqType] = None,
865
+ sub_path: Optional[StrOrPathOrSeqType] = None,
880
866
  *,
881
867
  nthreads: Optional[int] = None,
882
868
  exception_on_fail: bool = True,
@@ -888,7 +874,9 @@ class FCPath:
888
874
  sub_path: The path of the file relative to this path. If not specified, this
889
875
  path is used. If `sub_path` is a list or tuple, the complete list of files
890
876
  is uploaded. This may be more efficient because files can be uploaded in
891
- parallel.
877
+ parallel. If the resulting derived path is not absolute, it is assumed to
878
+ be a relative local path and is converted to an absolute path by expanding
879
+ usernames and resolving links.
892
880
  nthreads: The maximum number of threads to use when doing multiple-file
893
881
  retrieval or upload. If None, use the default value given when this
894
882
  :class:`FileCache` was created.
@@ -931,35 +919,21 @@ class FCPath:
931
919
  Raises:
932
920
  FileNotFoundError: If a file to upload does not exist or the upload failed,
933
921
  and exception_on_fail is True.
934
- ValueError: If the derived path is not absolute.
935
922
  """
936
923
 
937
924
  old_upload_counter = self._filecache_to_use.upload_counter
938
925
 
939
926
  nthreads = self._validate_nthreads(nthreads)
940
927
 
928
+ new_sub_path = self._make_paths_absolute(sub_path)
929
+
941
930
  try:
942
- if isinstance(sub_path, (list, tuple)):
943
- new_sub_paths = [FCPath._join(self._path, p) for p in sub_path]
944
- if not all([FCPath._is_absolute(x) for x in new_sub_paths]):
945
- raise ValueError(
946
- f'Derived paths must be absolute, got {new_sub_paths}')
947
- ret = self._filecache_to_use.upload(cast(StrOrPathOrSeqType,
948
- new_sub_paths),
949
- anonymous=self._anonymous,
950
- nthreads=nthreads,
951
- exception_on_fail=exception_on_fail,
952
- url_to_path=url_to_path)
953
- else:
954
- new_sub_path = FCPath._join(self._path, sub_path)
955
- if not FCPath._is_absolute(str(new_sub_path)):
956
- raise ValueError(
957
- f'Derived path must be absolute, got {new_sub_path}')
958
- ret = self._filecache_to_use.upload(cast(StrOrPathOrSeqType,
959
- new_sub_path),
960
- anonymous=self._anonymous,
961
- exception_on_fail=exception_on_fail,
962
- url_to_path=url_to_path)
931
+ ret = self._filecache_to_use.upload(cast(StrOrPathOrSeqType,
932
+ new_sub_path),
933
+ anonymous=self._anonymous,
934
+ nthreads=nthreads,
935
+ exception_on_fail=exception_on_fail,
936
+ url_to_path=url_to_path)
963
937
  finally:
964
938
  self._upload_counter += (self._filecache_to_use.upload_counter -
965
939
  old_upload_counter)
@@ -1030,7 +1004,7 @@ class FCPath:
1030
1004
  self.upload(sub_path, url_to_path=url_to_path)
1031
1005
 
1032
1006
  def unlink(self,
1033
- sub_path: Optional[StrOrSeqType] = None,
1007
+ sub_path: Optional[StrOrPathOrSeqType] = None,
1034
1008
  *,
1035
1009
  missing_ok: bool = False,
1036
1010
  nthreads: Optional[int] = None,
@@ -1043,7 +1017,10 @@ class FCPath:
1043
1017
  sub_path: The path of the file relative to this path. If not specified, this
1044
1018
  path is used. If `sub_path` is a list or tuple, the complete list of files
1045
1019
  is retrieved. Depending on the storage location, this may be more
1046
- efficient because files can be downloaded in parallel.
1020
+ efficient because files can be downloaded in parallel. If the resulting
1021
+ derived path is not absolute, it is assumed to be a relative local path
1022
+ and is converted to an absolute path by expanding usernames and resolving
1023
+ links.
1047
1024
  missing_ok: True to ignore attempting to unlink a file that doesn't exist.
1048
1025
  nthreads: The maximum number of threads to use when doing multiple-file
1049
1026
  retrieval or upload. If None, use the default value given when this
@@ -1081,30 +1058,17 @@ class FCPath:
1081
1058
  See `pathlib.Path.unlink` for full documentation.
1082
1059
  """
1083
1060
 
1084
- if isinstance(sub_path, (list, tuple)):
1085
- new_sub_paths = [FCPath._join(self._path, p) for p in sub_path]
1086
- if not all([FCPath._is_absolute(x) for x in new_sub_paths]):
1087
- raise ValueError(
1088
- f'Derived paths must be absolute, got {new_sub_paths}')
1089
- return self._filecache_to_use.unlink(cast(StrOrPathOrSeqType,
1090
- new_sub_paths),
1091
- missing_ok=missing_ok,
1092
- anonymous=self._anonymous,
1093
- nthreads=nthreads,
1094
- exception_on_fail=exception_on_fail,
1095
- url_to_path=url_to_path)
1096
- else:
1097
- new_sub_path2 = FCPath._join(self._path, sub_path)
1098
- if not FCPath._is_absolute(str(new_sub_path2)):
1099
- raise ValueError(
1100
- f'Derived path must be absolute, got {new_sub_path2}')
1101
- return self._filecache_to_use.unlink(cast(StrOrPathOrSeqType,
1102
- new_sub_path2),
1103
- missing_ok=missing_ok,
1104
- anonymous=self._anonymous,
1105
- nthreads=nthreads,
1106
- exception_on_fail=exception_on_fail,
1107
- url_to_path=url_to_path)
1061
+ nthreads = self._validate_nthreads(nthreads)
1062
+
1063
+ new_sub_path = self._make_paths_absolute(sub_path)
1064
+
1065
+ return self._filecache_to_use.unlink(cast(StrOrPathOrSeqType,
1066
+ new_sub_path),
1067
+ missing_ok=missing_ok,
1068
+ anonymous=self._anonymous,
1069
+ nthreads=nthreads,
1070
+ exception_on_fail=exception_on_fail,
1071
+ url_to_path=url_to_path)
1108
1072
 
1109
1073
  @property
1110
1074
  def download_counter(self) -> int:
@@ -1290,32 +1254,31 @@ class FCPath:
1290
1254
 
1291
1255
  if not isinstance(target, FCPath):
1292
1256
  target = FCPath(target)
1257
+ target = target.as_absolute()
1293
1258
 
1294
- if not self.is_absolute() or not target.is_absolute():
1295
- raise ValueError('Source and target files must be absolute '
1296
- f'{self.path!r} and f{target.path!r}')
1259
+ src = self.as_absolute()
1297
1260
 
1298
- if self.is_local() != target.is_local():
1261
+ if src.is_local() != target.is_local():
1299
1262
  raise ValueError('Unable to rename files between local and remote locations: '
1300
- f'{self.path!r} and f{target.path!r}')
1263
+ f'{src.path!r} and f{target.path!r}')
1301
1264
 
1302
- drive1, root1, subpath1 = FCPath._split_parts(self.path)
1265
+ drive1, root1, subpath1 = FCPath._split_parts(src.path)
1303
1266
  drive2, root2, subpath2 = FCPath._split_parts(target.path)
1304
1267
 
1305
1268
  if drive1 != drive2 or root1 != root2:
1306
- raise ValueError('Unable to rename files across location: '
1307
- f'{self.path!r} and f{target.path!r}')
1269
+ raise ValueError('Unable to rename files across locations: '
1270
+ f'{src.path!r} and f{target.path!r}')
1308
1271
 
1309
- if self.is_local():
1272
+ if src.is_local():
1310
1273
  # Local to local - just do an OS rename and be done with it
1311
1274
  target.parent.mkdir(parents=True, exist_ok=True)
1312
- self.as_pathlib().rename(target.as_pathlib())
1275
+ src.as_pathlib().rename(target.as_pathlib())
1313
1276
  return target
1314
1277
 
1315
1278
  # Since you generally can't rename on a remote cloud location, first
1316
1279
  # download the file, then rename it locally, then upload it to the new name,
1317
1280
  # then delete the old name
1318
- local_path = cast(Path, self.retrieve())
1281
+ local_path = cast(Path, src.retrieve())
1319
1282
  target_local_path = cast(Path, target.get_local_path())
1320
1283
  local_path.rename(target_local_path) # Rename in the cache
1321
1284
  try:
@@ -1323,7 +1286,7 @@ class FCPath:
1323
1286
  except Exception:
1324
1287
  target_local_path.unlink(missing_ok=True)
1325
1288
  raise
1326
- self.unlink(missing_ok=True) # Delete the old version
1289
+ src.unlink(missing_ok=True) # Delete the old version
1327
1290
 
1328
1291
  return target
1329
1292
 
@@ -1617,7 +1580,8 @@ class FCPath:
1617
1580
  """
1618
1581
 
1619
1582
  if self.is_local():
1620
- return FCPath(self.as_pathlib().resolve(strict=strict), copy_from=self)
1583
+ return FCPath(self.as_pathlib().absolute().resolve(strict=strict),
1584
+ copy_from=self)
1621
1585
 
1622
1586
  return self
1623
1587