rms-filecache 2.3.0__tar.gz → 2.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.gitignore +2 -0
  2. {rms_filecache-2.3.0/rms_filecache.egg-info → rms_filecache-2.3.1}/PKG-INFO +23 -16
  3. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/README.md +20 -14
  4. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/__init__.py +29 -23
  5. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/_version.py +9 -4
  6. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/file_cache.py +2 -1
  7. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/file_cache_source.py +215 -3
  8. {rms_filecache-2.3.0 → rms_filecache-2.3.1/rms_filecache.egg-info}/PKG-INFO +23 -16
  9. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/test_file_cache_path.py +49 -1
  10. rms_filecache-2.3.1/tests/test_file_cache_source.py +347 -0
  11. rms_filecache-2.3.0/tests/test_file_cache_source.py +0 -81
  12. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.coveragerc +0 -0
  13. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.flake8 +0 -0
  14. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.github/pull_request_template.md +0 -0
  15. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.github/workflows/publish_to_pypi.yml +0 -0
  16. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.github/workflows/publish_to_test_pypi.yml +0 -0
  17. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.github/workflows/run-tests.yml +0 -0
  18. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/.readthedocs.yaml +0 -0
  19. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/LICENSE +0 -0
  20. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/Makefile +0 -0
  21. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/conf.py +0 -0
  22. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/index.rst +0 -0
  23. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/make.bat +0 -0
  24. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/docs/module.rst +0 -0
  25. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/file_cache_path.py +0 -0
  26. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/file_cache_types.py +0 -0
  27. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/filecache/py.typed +0 -0
  28. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/mypy.ini +0 -0
  29. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/pyproject.toml +0 -0
  30. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/requirements.txt +0 -0
  31. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/rms_filecache.egg-info/SOURCES.txt +0 -0
  32. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/rms_filecache.egg-info/dependency_links.txt +0 -0
  33. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/rms_filecache.egg-info/requires.txt +0 -0
  34. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/rms_filecache.egg-info/top_level.txt +0 -0
  35. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/setup.cfg +0 -0
  36. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/lorem1.txt +0 -0
  37. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/subdir1/lorem1.txt +0 -0
  38. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
  39. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
  40. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_programs/locking.py +0 -0
  41. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/test_programs/multi_download.py +0 -0
  42. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/__init__.py +0 -0
  43. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/gs_lifecycle_config.json +0 -0
  44. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/gs_lifecycle_config_info.txt +0 -0
  45. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/s3_bucket_policy_read.txt +0 -0
  46. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/s3_bucket_policy_write.txt +0 -0
  47. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/s3_lifecycle_policy.txt +0 -0
  48. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/test_file_cache.py +0 -0
  49. {rms_filecache-2.3.0 → rms_filecache-2.3.1}/tests/test_url_to_path.py +0 -0
@@ -159,3 +159,5 @@ cython_debug/
159
159
  # and can be added to the global gitignore or merged into this file. For a more nuclear
160
160
  # option (not recommended) you can uncomment the following to ignore the entire idea folder.
161
161
  #.idea/
162
+
163
+ .vscode/
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: rms-filecache
3
- Version: 2.3.0
3
+ Version: 2.3.1
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -31,6 +31,7 @@ Requires-Dist: filelock
31
31
  Requires-Dist: google-cloud-storage
32
32
  Requires-Dist: requests
33
33
  Requires-Dist: typing_extensions
34
+ Dynamic: license-file
34
35
 
35
36
  [![GitHub release; latest by date](https://img.shields.io/github/v/release/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
36
37
  [![GitHub Release Date](https://img.shields.io/github/release-date/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
@@ -59,14 +60,16 @@ Requires-Dist: typing_extensions
59
60
  # Introduction
60
61
 
61
62
  `filecache` is a Python module that abstracts away the location where files used or
62
- generated by a program are stored. Files can be on the local file system, in Google Cloud
63
- Storage, on Amazon Web Services S3, or on a webserver. When files to be read are on the
64
- local file system, they are simply accessed in-place. Otherwise, they are downloaded from
65
- the remote source to a local temporary directory. When files to be written are on the
66
- local file system, they are simply written in-place. Otherwise, they are written to a
67
- local temporary directory and then uploaded to the remote location (it is not possible to
68
- upload to a webserver). When a cache is no longer needed, it is deleted from the local
69
- disk.
63
+ generated by a program are stored. Files can be on the local file system (`/` or
64
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
65
+ webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
66
+ a remote using the local filesystem as a file repository (`fake://`). When files to be read
67
+ are on the local file system, they are simply accessed in-place. Otherwise, they are
68
+ downloaded from the remote source to a local temporary directory. When files to be written
69
+ are on the local file system, they are simply written in-place. Otherwise, they are
70
+ written to a local temporary directory and then uploaded to the remote location (it is not
71
+ possible to upload to a webserver). When a cache is no longer needed, it is deleted from
72
+ the local disk.
70
73
 
71
74
  `filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
72
75
 
@@ -218,14 +221,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
218
221
  contents = voldesc_path.read_text()
219
222
  ```
220
223
 
221
- Finally, there are four classes that allow direct access to the four possible storage
222
- locations without invoking any caching behavior: `FileCacheSourceLocal`,
223
- `FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
224
+ Finally, there are five classes that allow direct access to the five possible storage
225
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
226
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
227
+ The first four provide access to real storage locations, while `FileCacheSourceFake`
228
+ simulates a remote source for testing purposes:
224
229
 
225
230
  ```python
226
- from filecache import FileCacheSourceGS
227
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
228
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
231
+ from filecache import FileCacheSourceGS, FileCacheSourceFake
232
+ src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
233
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
234
+ src2 = FileCacheSourceFake('fake://fake-bucket')
235
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
229
236
  ```
230
237
 
231
238
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -25,14 +25,16 @@
25
25
  # Introduction
26
26
 
27
27
  `filecache` is a Python module that abstracts away the location where files used or
28
- generated by a program are stored. Files can be on the local file system, in Google Cloud
29
- Storage, on Amazon Web Services S3, or on a webserver. When files to be read are on the
30
- local file system, they are simply accessed in-place. Otherwise, they are downloaded from
31
- the remote source to a local temporary directory. When files to be written are on the
32
- local file system, they are simply written in-place. Otherwise, they are written to a
33
- local temporary directory and then uploaded to the remote location (it is not possible to
34
- upload to a webserver). When a cache is no longer needed, it is deleted from the local
35
- disk.
28
+ generated by a program are stored. Files can be on the local file system (`/` or
29
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
30
+ webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
31
+ a remote using the local filesystem as a file repository (`fake://`). When files to be read
32
+ are on the local file system, they are simply accessed in-place. Otherwise, they are
33
+ downloaded from the remote source to a local temporary directory. When files to be written
34
+ are on the local file system, they are simply written in-place. Otherwise, they are
35
+ written to a local temporary directory and then uploaded to the remote location (it is not
36
+ possible to upload to a webserver). When a cache is no longer needed, it is deleted from
37
+ the local disk.
36
38
 
37
39
  `filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
38
40
 
@@ -184,14 +186,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
184
186
  contents = voldesc_path.read_text()
185
187
  ```
186
188
 
187
- Finally, there are four classes that allow direct access to the four possible storage
188
- locations without invoking any caching behavior: `FileCacheSourceLocal`,
189
- `FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
189
+ Finally, there are five classes that allow direct access to the five possible storage
190
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
191
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
192
+ The first four provide access to real storage locations, while `FileCacheSourceFake`
193
+ simulates a remote source for testing purposes:
190
194
 
191
195
  ```python
192
- from filecache import FileCacheSourceGS
193
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
194
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
196
+ from filecache import FileCacheSourceGS, FileCacheSourceFake
197
+ src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
198
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
199
+ src2 = FileCacheSourceFake('fake://fake-bucket')
200
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
195
201
  ```
196
202
 
197
203
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -3,14 +3,16 @@
3
3
  ##########################################################################################
4
4
 
5
5
  """The purpose of the `filecache` module is to abstract away the location where files
6
- used or generated by a program are stored. Files can be on the local file system, in
7
- Google Cloud Storage, on Amazon Web Services S3, or on a webserver. When files to be read
8
- are on the local file system, they are simply accessed in-place. Otherwise, they are
9
- downloaded from the remote source to a local temporary directory. When files to be written
10
- are on the local file system, they are simply written in-place. Otherwise, they are
11
- written to a local temporary directory and then uploaded to the remote location (it is not
12
- possible to upload to a webserver). When a cache is no longer needed, it is deleted from
13
- the local disk.
6
+ used or generated by a program are stored. Files can be on the local file system (`/` or
7
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on
8
+ a webserver (`http://` or `https://`). There is also a fake remote that can be used to
9
+ simulate a remote using the local filesystem as a file repository (`fake://`). When files
10
+ to be read are on the local file system, they are simply accessed in-place. Otherwise,
11
+ they are downloaded from the remote source to a local temporary directory. When files to
12
+ be written are on the local file system, they are simply written in-place. Otherwise, they
13
+ are written to a local temporary directory and then uploaded to the remote location (it is
14
+ not possible to upload to a webserver). When a cache is no longer needed, it is deleted
15
+ from the local disk.
14
16
 
15
17
  The top-level file organization is provided by the :class:`FileCache` class. A
16
18
  :class:`FileCache` instance is used to specify a particular **sharing policy** and
@@ -135,19 +137,21 @@ was run. A shared cache is indicated by giving the cache a name (or no argument,
135
137
  defaults to ``"global"``); also :class:`FCPath` defaults to using the global cache if no
136
138
  :class:`FileCache` is specified. This results in the simplest form of the program:
137
139
 
138
- from filecache import FCPath
139
- import os
140
- p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
141
- voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
142
- contents = voldesc_path.read_text()
143
-
144
- Finally, there are four classes that allow direct access to the four possible storage
145
- locations without invoking any caching behavior: :class:`FileCacheSourceFile`,
146
- :class:`FileCacheSourceHTTP`, :class:`FileCacheSourceGS`, and :class:`FileSourceCacheS3`::
147
-
148
- from filecache import FileCacheSourceGS
149
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
150
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
140
+ from filecache import FCPath import os p = FCPath(os.getenv('PDS3_HOLDINGS_DIR'))
141
+ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat' contents =
142
+ voldesc_path.read_text()
143
+
144
+ Finally, there are five classes that allow direct access to the five possible storage
145
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
146
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and
147
+ `FileCacheSourceFake`. The first four provide access to real storage locations, while
148
+ `FileCacheSourceFake` simulates a remote source for testing purposes:
149
+
150
+ from filecache import FileCacheSourceGS, FileCacheSourceFake src1 =
151
+ FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
152
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin') src2 =
153
+ FileCacheSourceFake('fake://fake-bucket')
154
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
151
155
  """
152
156
 
153
157
  import typing
@@ -168,7 +172,8 @@ from .file_cache_source import (FileCacheSource, # noqa: ignore E401
168
172
  FileCacheSourceFile,
169
173
  FileCacheSourceHTTP,
170
174
  FileCacheSourceGS,
171
- FileCacheSourceS3)
175
+ FileCacheSourceS3,
176
+ FileCacheSourceFake)
172
177
 
173
178
  __all__ = ['get_global_logger',
174
179
  'register_filecachesource',
@@ -180,4 +185,5 @@ __all__ = ['get_global_logger',
180
185
  'FileCacheSourceFile',
181
186
  'FileCacheSourceHTTP',
182
187
  'FileCacheSourceGS',
183
- 'FileCacheSourceS3']
188
+ 'FileCacheSourceS3',
189
+ 'FileCacheSourceFake']
@@ -1,8 +1,13 @@
1
- # file generated by setuptools_scm
1
+ # file generated by setuptools-scm
2
2
  # don't change, don't track in version control
3
+
4
+ __all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
5
+
3
6
  TYPE_CHECKING = False
4
7
  if TYPE_CHECKING:
5
- from typing import Tuple, Union
8
+ from typing import Tuple
9
+ from typing import Union
10
+
6
11
  VERSION_TUPLE = Tuple[Union[int, str], ...]
7
12
  else:
8
13
  VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
12
17
  __version_tuple__: VERSION_TUPLE
13
18
  version_tuple: VERSION_TUPLE
14
19
 
15
- __version__ = version = '2.3.0'
16
- __version_tuple__ = version_tuple = (2, 3, 0)
20
+ __version__ = version = '2.3.1'
21
+ __version_tuple__ = version_tuple = (2, 3, 1)
@@ -33,7 +33,7 @@ from .file_cache_source import (FileCacheSource,
33
33
  FileCacheSourceHTTP,
34
34
  FileCacheSourceGS,
35
35
  FileCacheSourceS3,
36
- )
36
+ FileCacheSourceFake)
37
37
  from .file_cache_path import FCPath
38
38
  from .file_cache_types import (StrOrPathOrSeqType,
39
39
  UrlToPathFuncOrSeqType)
@@ -59,6 +59,7 @@ register_filecachesource(FileCacheSourceFile)
59
59
  register_filecachesource(FileCacheSourceHTTP)
60
60
  register_filecachesource(FileCacheSourceGS)
61
61
  register_filecachesource(FileCacheSourceS3)
62
+ register_filecachesource(FileCacheSourceFake)
62
63
 
63
64
 
64
65
  # Default logger for all FileCache instances that didn't specify one explicitly
@@ -10,6 +10,8 @@ from concurrent import futures
10
10
  from concurrent.futures import ThreadPoolExecutor
11
11
  from pathlib import Path
12
12
  import requests
13
+ import shutil
14
+ import tempfile
13
15
  from typing import Iterator
14
16
  import uuid
15
17
 
@@ -606,7 +608,7 @@ class FileCacheSourceHTTP(FileCacheSource):
606
608
  except requests.exceptions.RequestException as e:
607
609
  raise FileNotFoundError(f'Failed to download file: {url}') from e
608
610
 
609
- temp_local_path = local_path.with_suffix(local_path.suffix + str(uuid.uuid4()))
611
+ temp_local_path = local_path.with_suffix(f'.{local_path.suffix}_{uuid.uuid4()}')
610
612
  try:
611
613
  with open(temp_local_path, 'wb') as f:
612
614
  for chunk in response.iter_content(chunk_size=1024*1024):
@@ -758,7 +760,7 @@ class FileCacheSourceGS(FileCacheSource):
758
760
 
759
761
  blob = self._bucket.blob(sub_path)
760
762
 
761
- temp_local_path = local_path.with_suffix(f'{local_path.suffix}.{uuid.uuid4()}')
763
+ temp_local_path = local_path.with_suffix(f'.{local_path.suffix}_{uuid.uuid4()}')
762
764
  try:
763
765
  blob.download_to_filename(str(temp_local_path))
764
766
  temp_local_path.rename(local_path)
@@ -963,7 +965,7 @@ class FileCacheSourceS3(FileCacheSource):
963
965
 
964
966
  local_path.parent.mkdir(parents=True, exist_ok=True)
965
967
 
966
- temp_local_path = local_path.with_suffix(f'{local_path.suffix}.{uuid.uuid4()}')
968
+ temp_local_path = local_path.with_suffix(f'.{local_path.suffix}_{uuid.uuid4()}')
967
969
  try:
968
970
  self._client.download_file(self._bucket_name, sub_path,
969
971
  str(temp_local_path))
@@ -1068,3 +1070,213 @@ class FileCacheSourceS3(FileCacheSource):
1068
1070
  self._client.delete_object(Bucket=self._bucket_name, Key=sub_path)
1069
1071
 
1070
1072
  return sub_path
1073
+
1074
+
1075
+ class FileCacheSourceFake(FileCacheSource):
1076
+ """Class that simulates a remote file source using a local directory structure.
1077
+
1078
+ This class is useful for testing file operations without requiring actual remote
1079
+ connections. Files are stored in a local directory that simulates the remote storage,
1080
+ including the need for uploads and downloads. By default, the storage directory is
1081
+ ``<TEMPDIR>/.filecache_fake_remote`` and persists across program runs.
1082
+ """
1083
+
1084
+ _DEFAULT_STORAGE_DIR = Path(tempfile.gettempdir()) / '.filecache_fake_remote'
1085
+
1086
+ @classmethod
1087
+ def get_default_storage_dir(cls) -> Path:
1088
+ """Get the current default storage directory for fake remote files.
1089
+
1090
+ Returns:
1091
+ The current default storage directory Path.
1092
+ """
1093
+
1094
+ return cls._DEFAULT_STORAGE_DIR
1095
+
1096
+ @classmethod
1097
+ def set_default_storage_dir(cls, directory: str | Path) -> None:
1098
+ """Set the default storage directory for fake remote files.
1099
+
1100
+ Parameters:
1101
+ directory: The directory to use as the default storage location. The directory
1102
+ is expanded and resolved to an absolute path.
1103
+ """
1104
+
1105
+ cls._DEFAULT_STORAGE_DIR = Path(directory).expanduser().resolve()
1106
+
1107
+ @classmethod
1108
+ def delete_default_storage_dir(cls) -> None:
1109
+ """Delete the current default storage directory and all its contents.
1110
+
1111
+ This is useful for cleanup after testing.
1112
+ """
1113
+
1114
+ if cls._DEFAULT_STORAGE_DIR.exists():
1115
+ shutil.rmtree(cls._DEFAULT_STORAGE_DIR)
1116
+
1117
+ def __init__(self,
1118
+ scheme: str,
1119
+ remote: str,
1120
+ *,
1121
+ anonymous: bool = False,
1122
+ storage_dir: str | Path | None = None):
1123
+ """Initialize the FileCacheSourceFake class.
1124
+
1125
+ Parameters:
1126
+ scheme: The scheme of the source. Must be "fake".
1127
+ remote: The simulated remote/bucket name.
1128
+ anonymous: Not used for this class.
1129
+ storage_dir: Base directory in which to store the fake remote files. If None,
1130
+ uses the class default storage directory.
1131
+ """
1132
+
1133
+ if scheme != 'fake':
1134
+ raise ValueError('Scheme must be "fake"')
1135
+
1136
+ if remote == '':
1137
+ raise ValueError('remote parameter must have a value')
1138
+
1139
+ super().__init__(scheme, remote, anonymous=anonymous)
1140
+
1141
+ self._storage_base = (Path(storage_dir).expanduser().resolve()
1142
+ if storage_dir is not None
1143
+ else self._DEFAULT_STORAGE_DIR)
1144
+ self._storage_dir = self._storage_base / remote
1145
+ self._storage_dir.mkdir(parents=True, exist_ok=True)
1146
+ self._cache_subdir = self._src_prefix.replace('fake://', 'fake_')
1147
+
1148
+ @classmethod
1149
+ def schemes(cls) -> tuple[str, ...]:
1150
+ """The URL schemes supported by this class."""
1151
+
1152
+ return ('fake',)
1153
+
1154
+ @classmethod
1155
+ def uses_anonymous(cls) -> bool:
1156
+ """Whether this class has the concept of anonymous accesses."""
1157
+
1158
+ return False
1159
+
1160
+ def exists(self, sub_path: str) -> bool:
1161
+ """Check if a file exists in the fake remote storage.
1162
+
1163
+ Parameters:
1164
+ sub_path: The path of the file relative to the storage directory.
1165
+
1166
+ Returns:
1167
+ True if the file exists, False otherwise.
1168
+ """
1169
+
1170
+ return (self._storage_dir / sub_path).is_file()
1171
+
1172
+ def retrieve(self,
1173
+ sub_path: str,
1174
+ local_path: str | Path) -> Path:
1175
+ """Retrieve a file from the fake remote storage.
1176
+
1177
+ Parameters:
1178
+ sub_path: The path of the file relative to the storage directory.
1179
+ local_path: The path where the file should be copied to.
1180
+
1181
+ Returns:
1182
+ The Path where the file was stored (same as local_path).
1183
+
1184
+ Raises:
1185
+ FileNotFoundError: If the remote file does not exist.
1186
+ """
1187
+
1188
+ source_path = self._storage_dir / sub_path
1189
+ local_path = Path(local_path)
1190
+
1191
+ if not source_path.is_file():
1192
+ raise FileNotFoundError(f'File does not exist: {self._src_prefix_}{sub_path}')
1193
+
1194
+ local_path.parent.mkdir(parents=True, exist_ok=True)
1195
+
1196
+ temp_local_path = local_path.with_suffix(f'.{local_path.suffix}_{uuid.uuid4()}')
1197
+ try:
1198
+ shutil.copy2(source_path, temp_local_path)
1199
+ temp_local_path.rename(local_path)
1200
+ except Exception:
1201
+ temp_local_path.unlink(missing_ok=True)
1202
+ raise
1203
+
1204
+ return local_path
1205
+
1206
+ def upload(self,
1207
+ sub_path: str,
1208
+ local_path: str | Path) -> Path:
1209
+ """Upload a file to the fake remote storage.
1210
+
1211
+ Parameters:
1212
+ sub_path: The destination path relative to the storage directory.
1213
+ local_path: The path of the local file to upload.
1214
+
1215
+ Returns:
1216
+ The Path of the local file that was uploaded.
1217
+
1218
+ Raises:
1219
+ FileNotFoundError: If the local file does not exist.
1220
+ """
1221
+
1222
+ local_path = Path(local_path)
1223
+ if not local_path.is_file():
1224
+ raise FileNotFoundError(f'File does not exist: {local_path}')
1225
+
1226
+ dest_path = self._storage_dir / sub_path
1227
+ dest_path.parent.mkdir(parents=True, exist_ok=True)
1228
+
1229
+ temp_dest_path = dest_path.with_suffix(f'.{dest_path.suffix}_{uuid.uuid4()}')
1230
+ try:
1231
+ shutil.copy2(local_path, temp_dest_path)
1232
+ temp_dest_path.rename(dest_path)
1233
+ except Exception:
1234
+ temp_dest_path.unlink(missing_ok=True)
1235
+ raise
1236
+
1237
+ return local_path
1238
+
1239
+ def iterdir_type(self, sub_path: str) -> Iterator[tuple[str, bool]]:
1240
+ """Iterate over the contents of a directory in the fake remote storage.
1241
+
1242
+ Parameters:
1243
+ sub_path: The path of the directory relative to the storage directory.
1244
+
1245
+ Yields:
1246
+ Tuples of (path, is_dir) for each item in the directory.
1247
+ """
1248
+
1249
+ dir_path = self._storage_dir / sub_path if sub_path else self._storage_dir
1250
+
1251
+ if not dir_path.is_dir():
1252
+ return
1253
+
1254
+ for item in dir_path.iterdir():
1255
+ relative_path = str(item.relative_to(self._storage_dir)).replace('\\', '/')
1256
+ yield f'{self._src_prefix_}{relative_path}', item.is_dir()
1257
+
1258
+ def unlink(self,
1259
+ sub_path: str,
1260
+ *,
1261
+ missing_ok: bool = False) -> str:
1262
+ """Remove a file from the fake remote storage.
1263
+
1264
+ Parameters:
1265
+ sub_path: The path of the file relative to the storage directory.
1266
+ missing_ok: If True, don't raise an error if the file doesn't exist.
1267
+
1268
+ Returns:
1269
+ The sub_path that was removed.
1270
+
1271
+ Raises:
1272
+ FileNotFoundError: If the file doesn't exist and missing_ok is False.
1273
+ """
1274
+
1275
+ file_path = self._storage_dir / sub_path
1276
+ try:
1277
+ file_path.unlink()
1278
+ except FileNotFoundError:
1279
+ if not missing_ok:
1280
+ raise
1281
+
1282
+ return sub_path
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: rms-filecache
3
- Version: 2.3.0
3
+ Version: 2.3.1
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -31,6 +31,7 @@ Requires-Dist: filelock
31
31
  Requires-Dist: google-cloud-storage
32
32
  Requires-Dist: requests
33
33
  Requires-Dist: typing_extensions
34
+ Dynamic: license-file
34
35
 
35
36
  [![GitHub release; latest by date](https://img.shields.io/github/v/release/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
36
37
  [![GitHub Release Date](https://img.shields.io/github/release-date/SETI/rms-filecache)](https://github.com/SETI/rms-filecache/releases)
@@ -59,14 +60,16 @@ Requires-Dist: typing_extensions
59
60
  # Introduction
60
61
 
61
62
  `filecache` is a Python module that abstracts away the location where files used or
62
- generated by a program are stored. Files can be on the local file system, in Google Cloud
63
- Storage, on Amazon Web Services S3, or on a webserver. When files to be read are on the
64
- local file system, they are simply accessed in-place. Otherwise, they are downloaded from
65
- the remote source to a local temporary directory. When files to be written are on the
66
- local file system, they are simply written in-place. Otherwise, they are written to a
67
- local temporary directory and then uploaded to the remote location (it is not possible to
68
- upload to a webserver). When a cache is no longer needed, it is deleted from the local
69
- disk.
63
+ generated by a program are stored. Files can be on the local file system (`/` or
64
+ `file://`), in Google Cloud Storage (`gs://`), on Amazon Web Services S3 (`s3://`), or on a
65
+ webserver (`http://` or `https://`). There is also a fake remote that can be used to simulate
66
+ a remote using the local filesystem as a file repository (`fake://`). When files to be read
67
+ are on the local file system, they are simply accessed in-place. Otherwise, they are
68
+ downloaded from the remote source to a local temporary directory. When files to be written
69
+ are on the local file system, they are simply written in-place. Otherwise, they are
70
+ written to a local temporary directory and then uploaded to the remote location (it is not
71
+ possible to upload to a webserver). When a cache is no longer needed, it is deleted from
72
+ the local disk.
70
73
 
71
74
  `filecache` is a product of the [PDS Ring-Moon Systems Node](https://pds-rings.seti.org).
72
75
 
@@ -218,14 +221,18 @@ voldesc_path = p / 'volumes/COISS_2xxx/COISS_2001/voldesc.cat'
218
221
  contents = voldesc_path.read_text()
219
222
  ```
220
223
 
221
- Finally, there are four classes that allow direct access to the four possible storage
222
- locations without invoking any caching behavior: `FileCacheSourceLocal`,
223
- `FileCacheSourceHTTP`, `FileCacheSourceGS`, and `FileSourceCacheS3`:
224
+ Finally, there are five classes that allow direct access to the five possible storage
225
+ locations without invoking any caching behavior: `FileCacheSourceFile`,
226
+ `FileCacheSourceHTTP`, `FileCacheSourceGS`, `FileSourceCacheS3`, and `FileCacheSourceFake`.
227
+ The first four provide access to real storage locations, while `FileCacheSourceFake`
228
+ simulates a remote source for testing purposes:
224
229
 
225
230
  ```python
226
- from filecache import FileCacheSourceGS
227
- src = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
228
- src.retrieve('subdir1/subdir2a/binary1.bin', 'local_file.bin')
231
+ from filecache import FileCacheSourceGS, FileCacheSourceFake
232
+ src1 = FileCacheSourceGS('gs://rms-filecache-tests', anonymous=True)
233
+ src1.retrieve('subdir1/subdir2a/binary1.bin', 'local_file1.bin')
234
+ src2 = FileCacheSourceFake('fake://fake-bucket')
235
+ src2.retrieve('subdir1/subdir2a/binary1.bin', 'local_file2.bin')
229
236
  ```
230
237
 
231
238
  Details of each class are available in the [module documentation](https://rms-filecache.readthedocs.io/en/latest/module.html).
@@ -11,7 +11,7 @@ import uuid
11
11
 
12
12
  import pytest
13
13
 
14
- from filecache import FileCache, FCPath
14
+ from filecache import FileCache, FileCacheSourceFake, FCPath
15
15
 
16
16
  from tests.test_file_cache import (GS_TEST_BUCKET_ROOT,
17
17
  S3_TEST_BUCKET_ROOT,
@@ -667,6 +667,45 @@ def test_read_write():
667
667
  p = FCPath('a')
668
668
  p.write_text(5)
669
669
 
670
+ FileCacheSourceFake.delete_default_storage_dir()
671
+ try:
672
+ with FileCache(cache_name=None) as fc:
673
+ fake_prefix = 'fake://test-bucket'
674
+ pfx = fc.new_path(fake_prefix)
675
+ p1 = pfx / 'test_file.bin'
676
+ p2 = pfx / 'test_file.txt'
677
+
678
+ p1.write_bytes(b'A')
679
+ p2.write_text('ABC\n')
680
+ assert p1.read_bytes() == b'A'
681
+ assert p2.read_text() == 'ABC\n'
682
+
683
+ # Should behave like a remote source with uploads
684
+ assert fc.download_counter == 0
685
+ assert fc.upload_counter == 2
686
+ assert p1.download_counter == 0
687
+ assert p1.upload_counter == 1
688
+ assert p2.download_counter == 0
689
+ assert p2.upload_counter == 1
690
+
691
+ # Test in new cache to verify persistence
692
+ with FileCache(cache_name=None) as fc:
693
+ pfx = fc.new_path(fake_prefix)
694
+ p1 = pfx / 'test_file.bin'
695
+ p2 = pfx / 'test_file.txt'
696
+
697
+ # Reading should require downloads from "remote" storage
698
+ assert p1.read_bytes() == b'A'
699
+ assert p2.read_text() == 'ABC\n'
700
+ assert fc.download_counter == 2
701
+ assert fc.upload_counter == 0
702
+ assert p1.download_counter == 1
703
+ assert p1.upload_counter == 0
704
+ assert p2.download_counter == 1
705
+ assert p2.upload_counter == 0
706
+ finally:
707
+ FileCacheSourceFake.delete_default_storage_dir()
708
+
670
709
 
671
710
  def test_default_filecache():
672
711
  with FileCache(delete_on_exit=True) as fc:
@@ -676,6 +715,15 @@ def test_default_filecache():
676
715
  p3 = FCPath(GS_WRITABLE_TEST_BUCKET_ROOT) / EXPECTED_FILENAMES[0]
677
716
  assert p2.get_local_path() == p3.get_local_path()
678
717
 
718
+ fake_path = 'fake://test-bucket/test.txt'
719
+ p1 = FCPath(fake_path)
720
+ p2 = fc.new_path(fake_path)
721
+ p1.write_text('test content') # Write using default FileCache
722
+ assert p2.read_text() == 'test content' # Read using explicit FileCache
723
+ assert p1.get_local_path() == p2.get_local_path()
724
+ p3 = FCPath(fake_path) # Create new path with default FileCache
725
+ assert p2.get_local_path() == p3.get_local_path()
726
+
679
727
 
680
728
  def test_relative():
681
729
  assert (FCPath(f'{GS_TEST_BUCKET_ROOT}/a/b/c.txt')
@@ -0,0 +1,347 @@
1
+ ################################################################################
2
+ # tests/test_file_cache_source.py
3
+ ################################################################################
4
+
5
+ import pathlib
6
+ from pathlib import Path
7
+ import pytest
8
+ import shutil
9
+ import os
10
+
11
+ from filecache import (FileCacheSource,
12
+ FileCacheSourceFile,
13
+ FileCacheSourceHTTP,
14
+ FileCacheSourceGS,
15
+ FileCacheSourceS3,
16
+ FileCacheSourceFake)
17
+
18
+ from .test_file_cache import EXPECTED_DIR, EXPECTED_FILENAMES
19
+
20
+
21
+ def test_source_bad():
22
+ with pytest.raises(ValueError):
23
+ FileCacheSourceFile('fred', 'hi')
24
+
25
+ with pytest.raises(ValueError):
26
+ FileCacheSourceHTTP('fred', 'hi')
27
+ with pytest.raises(ValueError):
28
+ FileCacheSourceHTTP('http', 'hi/hi')
29
+ with pytest.raises(ValueError):
30
+ FileCacheSourceHTTP('https', '')
31
+
32
+ with pytest.raises(ValueError):
33
+ FileCacheSourceGS('fred', 'hi')
34
+ with pytest.raises(ValueError):
35
+ FileCacheSourceGS('gs', 'hi/hi')
36
+ with pytest.raises(ValueError):
37
+ FileCacheSourceGS('gs', '')
38
+
39
+ with pytest.raises(ValueError):
40
+ FileCacheSourceS3('fred', 'hi')
41
+ with pytest.raises(ValueError):
42
+ FileCacheSourceS3('s3', 'hi/hi')
43
+ with pytest.raises(ValueError):
44
+ FileCacheSourceS3('s3', '')
45
+
46
+ with pytest.raises(ValueError):
47
+ FileCacheSourceFake('not-fake', 'test-bucket')
48
+ with pytest.raises(ValueError):
49
+ FileCacheSourceFake('fake', 'hi/hi')
50
+ with pytest.raises(ValueError):
51
+ FileCacheSourceFake('fake', '')
52
+
53
+
54
+ def test_filesource_bad():
55
+ sl = FileCacheSourceFile('file', '')
56
+ with pytest.raises(FileNotFoundError):
57
+ sl.upload('non-existent.txt', 'non-existent.txt')
58
+ assert not sl.exists('non-existent.txt')
59
+
60
+
61
+ def test_filesource_good():
62
+ sl = FileCacheSourceFile('file', '')
63
+ assert sl.exists(EXPECTED_DIR / EXPECTED_FILENAMES[1])
64
+
65
+
66
+ def test_source_notimp():
67
+ with pytest.raises(TypeError):
68
+ FileCacheSource('', '').exists('')
69
+ with pytest.raises(NotImplementedError):
70
+ FileCacheSourceHTTP('http', 'fred').upload('', '')
71
+ with pytest.raises(NotImplementedError):
72
+ FileCacheSourceHTTP('http', 'fred').iterdir_type('')
73
+ with pytest.raises(NotImplementedError):
74
+ FileCacheSourceHTTP('http', 'fred').unlink('')
75
+
76
+
77
+ def test_source_nthreads_bad():
78
+ with pytest.raises(ValueError):
79
+ FileCacheSourceFile('file', '').retrieve_multi(['/test'], ['/test'], nthreads=-1)
80
+ with pytest.raises(ValueError):
81
+ FileCacheSourceFile('file', '').retrieve_multi(['/test'], ['/test'], nthreads=4.5)
82
+ with pytest.raises(ValueError):
83
+ FileCacheSourceFile('file', '').upload_multi(['/test'], ['/test'], nthreads=-1)
84
+ with pytest.raises(ValueError):
85
+ FileCacheSourceFile('file', '').upload_multi(['/test'], ['/test'], nthreads=4.5)
86
+ with pytest.raises(ValueError):
87
+ FileCacheSourceFile('file', '').exists_multi(['/test'], nthreads=-1)
88
+ with pytest.raises(ValueError):
89
+ FileCacheSourceFile('file', '').exists_multi(['/test'], nthreads=4.5)
90
+ with pytest.raises(ValueError):
91
+ FileCacheSourceFile('file', '').unlink_multi(['/test'], nthreads=-1)
92
+ with pytest.raises(ValueError):
93
+ FileCacheSourceFile('file', '').unlink_multi(['/test'], nthreads=4.5)
94
+
95
+
96
+ def test_fake_source_init():
97
+ """Test FileCacheSourceFake initialization."""
98
+ FileCacheSourceFake.delete_default_storage_dir()
99
+ try:
100
+ # Test valid initialization
101
+ fake = FileCacheSourceFake('fake', 'test-bucket')
102
+ assert fake.schemes() == ('fake',)
103
+ assert not fake.uses_anonymous()
104
+
105
+ # Test invalid scheme
106
+ with pytest.raises(ValueError, match='Scheme must be "fake"'):
107
+ FileCacheSourceFake('not-fake', 'test-bucket')
108
+
109
+ # Test empty remote
110
+ with pytest.raises(ValueError, match='remote parameter must have a value'):
111
+ FileCacheSourceFake('fake', '')
112
+ finally:
113
+ FileCacheSourceFake.delete_default_storage_dir()
114
+
115
+
116
+ def test_fake_source_default_storage_dir(tmp_path: Path):
117
+ """Test the default storage directory management."""
118
+ FileCacheSourceFake.delete_default_storage_dir()
119
+ try:
120
+ # Test getting default directory
121
+ default_dir = FileCacheSourceFake.get_default_storage_dir()
122
+ assert '.filecache_fake_remote' in str(default_dir)
123
+
124
+ # Test setting default directory
125
+ FileCacheSourceFake.set_default_storage_dir(tmp_path)
126
+ assert FileCacheSourceFake.get_default_storage_dir() == tmp_path
127
+
128
+ # Test creating a source uses the default directory
129
+ fake = FileCacheSourceFake('fake', 'test-bucket')
130
+ assert fake._storage_base == tmp_path
131
+
132
+ # Test deleting default directory
133
+ path = tmp_path / 'test.txt'
134
+ path.write_text('test')
135
+ assert path.exists()
136
+ FileCacheSourceFake.delete_default_storage_dir()
137
+ assert not path.exists()
138
+ finally:
139
+ FileCacheSourceFake.delete_default_storage_dir()
140
+
141
+
142
+ def test_fake_source_file_operations(tmp_path: Path):
143
+ """Test basic file operations with FileCacheSourceFake."""
144
+ FileCacheSourceFake.delete_default_storage_dir()
145
+ try:
146
+ fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
147
+
148
+ # Test file that doesn't exist
149
+ assert not fake.exists('test.txt')
150
+
151
+ # Test upload
152
+ source_file = tmp_path / 'source.txt'
153
+ source_file.write_text('test content')
154
+
155
+ fake.upload('test.txt', source_file)
156
+ assert fake.exists('test.txt')
157
+
158
+ # Test retrieve
159
+ dest_file = tmp_path / 'dest.txt'
160
+ fake.retrieve('test.txt', dest_file)
161
+ assert dest_file.read_text() == 'test content'
162
+
163
+ # Test unlink
164
+ fake.unlink('test.txt')
165
+ assert not fake.exists('test.txt')
166
+
167
+ # Test unlink missing file
168
+ with pytest.raises(FileNotFoundError):
169
+ fake.unlink('missing.txt')
170
+
171
+ # Test unlink missing file with missing_ok=True
172
+ fake.unlink('missing.txt', missing_ok=True)
173
+ finally:
174
+ FileCacheSourceFake.delete_default_storage_dir()
175
+
176
+
177
+ def test_fake_source_directory_operations(tmp_path: Path):
178
+ """Test directory operations with FileCacheSourceFake."""
179
+ FileCacheSourceFake.delete_default_storage_dir()
180
+ try:
181
+ fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
182
+
183
+ # Create some test files and directories
184
+ (tmp_path / 'test-bucket/dir1').mkdir(parents=True)
185
+ (tmp_path / 'test-bucket/dir1/file1.txt').write_text('content1')
186
+ (tmp_path / 'test-bucket/dir2').mkdir(parents=True)
187
+ (tmp_path / 'test-bucket/dir2/file2.txt').write_text('content2')
188
+ (tmp_path / 'test-bucket/file3.txt').write_text('content3')
189
+
190
+ # Test iterdir_type
191
+ entries = list(fake.iterdir_type(''))
192
+ assert len(entries) == 3
193
+
194
+ # Sort entries for consistent testing
195
+ entries.sort()
196
+
197
+ # Test full paths and types
198
+ assert entries[0] == ('fake://test-bucket/dir1', True)
199
+ assert entries[1] == ('fake://test-bucket/dir2', True)
200
+ assert entries[2] == ('fake://test-bucket/file3.txt', False)
201
+
202
+ # Test subdirectory listing
203
+ entries = list(fake.iterdir_type('dir1'))
204
+ assert len(entries) == 1
205
+ assert entries[0] == ('fake://test-bucket/dir1/file1.txt', False)
206
+ finally:
207
+ FileCacheSourceFake.delete_default_storage_dir()
208
+
209
+
210
+ def test_fake_source_error_cases(tmp_path: Path):
211
+ """Test error cases with FileCacheSourceFake."""
212
+ FileCacheSourceFake.delete_default_storage_dir()
213
+ try:
214
+ fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
215
+
216
+ # Test retrieve non-existent file
217
+ with pytest.raises(FileNotFoundError):
218
+ fake.retrieve('missing.txt', tmp_path / 'dest.txt')
219
+
220
+ # Test upload non-existent file
221
+ with pytest.raises(FileNotFoundError):
222
+ fake.upload('dest.txt', tmp_path / 'missing.txt')
223
+
224
+ # Test iterdir_type on non-existent directory
225
+ assert list(fake.iterdir_type('missing-dir')) == []
226
+
227
+ # Create a test file
228
+ source_path = tmp_path / 'test-bucket/test.txt'
229
+ source_path.parent.mkdir(parents=True, exist_ok=True)
230
+ source_path.write_text('test content')
231
+
232
+ # Create a destination directory that exists but is not writable
233
+ dest_dir = tmp_path / 'dest'
234
+ dest_dir.mkdir()
235
+ dest_path = dest_dir / 'test.txt'
236
+
237
+ # Mock shutil.copy2 to raise an error after creating the temp file
238
+ original_copy2 = shutil.copy2
239
+
240
+ def mock_copy2(src, dst):
241
+ # Create the temp file
242
+ with open(dst, 'w') as f:
243
+ f.write('partial content')
244
+ # Then raise an error
245
+ raise OSError('Mock copy error')
246
+
247
+ shutil.copy2 = mock_copy2
248
+ try:
249
+ with pytest.raises(OSError, match="Mock copy error"):
250
+ fake.retrieve('test.txt', dest_path)
251
+ finally:
252
+ shutil.copy2 = original_copy2
253
+ finally:
254
+ FileCacheSourceFake.delete_default_storage_dir()
255
+
256
+
257
+ def test_fake_source_multi_operations(tmp_path: Path):
258
+ """Test multi-file operations with FileCacheSourceFake."""
259
+ FileCacheSourceFake.delete_default_storage_dir()
260
+ try:
261
+ fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
262
+
263
+ # Create test files
264
+ files = []
265
+ for i in range(3):
266
+ path = tmp_path / f'source{i}.txt'
267
+ path.write_text(f'content{i}')
268
+ files.append(path)
269
+
270
+ # Test exists_multi
271
+ results = fake.exists_multi(['test1.txt', 'test2.txt', 'test3.txt'])
272
+ assert results == [False, False, False]
273
+
274
+ # Test upload_multi
275
+ sub_paths = [f'test{i}.txt' for i in range(3)]
276
+ results = fake.upload_multi(sub_paths[:2], files[:2])
277
+ assert all(isinstance(r, Path) for r in results)
278
+
279
+ # Test exists_multi after upload
280
+ results = fake.exists_multi(sub_paths)
281
+ assert results == [True, True, False]
282
+
283
+ # Test retrieve_multi
284
+ dest_files = [tmp_path / f'dest{i}.txt' for i in range(3)]
285
+ results = fake.retrieve_multi(sub_paths, dest_files)
286
+ assert isinstance(results[0], Path)
287
+ assert isinstance(results[1], Path)
288
+ assert isinstance(results[2], BaseException)
289
+ assert dest_files[0].exists()
290
+ assert dest_files[1].exists()
291
+ assert not dest_files[2].exists()
292
+
293
+ # Test unlink_multi
294
+ results = fake.unlink_multi(sub_paths)
295
+ assert isinstance(results[0], str)
296
+ assert isinstance(results[1], str)
297
+ assert isinstance(results[2], BaseException)
298
+ assert not any(fake.exists(p) for p in sub_paths)
299
+ finally:
300
+ FileCacheSourceFake.delete_default_storage_dir()
301
+
302
+
303
+ def test_fake_source_atomic_operations(tmp_path: Path):
304
+ """Test that uploads and downloads are atomic operations."""
305
+ FileCacheSourceFake.delete_default_storage_dir()
306
+ try:
307
+ fake = FileCacheSourceFake('fake', 'test-bucket', storage_dir=tmp_path)
308
+
309
+ # Test atomic upload
310
+ source_file = tmp_path / 'source.txt'
311
+ source_file.write_text('test content')
312
+
313
+ # Verify upload creates temp file and then renames
314
+ def mock_rename(src, dst):
315
+ # Verify temp file exists and destination doesn't
316
+ assert src.exists()
317
+ assert not dst.exists()
318
+ # Just verify it's a temp file with a UUID-like pattern
319
+ assert '.txt_' in str(src)
320
+ os.rename(str(src), str(dst)) # Use os.rename to avoid recursion
321
+
322
+ original_rename = pathlib.Path.rename
323
+ pathlib.Path.rename = mock_rename
324
+ try:
325
+ fake.upload('test.txt', source_file)
326
+ finally:
327
+ pathlib.Path.rename = original_rename
328
+
329
+ # Test atomic download
330
+ dest_file = tmp_path / 'dest.txt'
331
+
332
+ # Verify download creates temp file and then renames
333
+ def mock_rename2(src, dst):
334
+ # Verify temp file exists and destination doesn't
335
+ assert src.exists()
336
+ assert not dst.exists()
337
+ # Just verify it's a temp file with a UUID-like pattern
338
+ assert '.txt_' in str(src)
339
+ os.rename(str(src), str(dst)) # Use os.rename to avoid recursion
340
+
341
+ pathlib.Path.rename = mock_rename2
342
+ try:
343
+ fake.retrieve('test.txt', dest_file)
344
+ finally:
345
+ pathlib.Path.rename = original_rename
346
+ finally:
347
+ FileCacheSourceFake.delete_default_storage_dir()
@@ -1,81 +0,0 @@
1
- ################################################################################
2
- # tests/test_file_cache_source.py
3
- ################################################################################
4
-
5
- import pytest
6
-
7
- from filecache import (FileCacheSource,
8
- FileCacheSourceFile,
9
- FileCacheSourceHTTP,
10
- FileCacheSourceGS,
11
- FileCacheSourceS3)
12
-
13
- from .test_file_cache import EXPECTED_DIR, EXPECTED_FILENAMES
14
-
15
-
16
- def test_source_bad():
17
- with pytest.raises(ValueError):
18
- FileCacheSourceFile('fred', 'hi')
19
-
20
- with pytest.raises(ValueError):
21
- FileCacheSourceHTTP('fred', 'hi')
22
- with pytest.raises(ValueError):
23
- FileCacheSourceHTTP('http', 'hi/hi')
24
- with pytest.raises(ValueError):
25
- FileCacheSourceHTTP('https', '')
26
-
27
- with pytest.raises(ValueError):
28
- FileCacheSourceGS('fred', 'hi')
29
- with pytest.raises(ValueError):
30
- FileCacheSourceGS('gs', 'hi/hi')
31
- with pytest.raises(ValueError):
32
- FileCacheSourceGS('gs', '')
33
-
34
- with pytest.raises(ValueError):
35
- FileCacheSourceS3('fred', 'hi')
36
- with pytest.raises(ValueError):
37
- FileCacheSourceS3('s3', 'hi/hi')
38
- with pytest.raises(ValueError):
39
- FileCacheSourceS3('s3', '')
40
-
41
-
42
- def test_filesource_bad():
43
- sl = FileCacheSourceFile('file', '')
44
- with pytest.raises(FileNotFoundError):
45
- sl.upload('non-existent.txt', 'non-existent.txt')
46
- assert not sl.exists('non-existent.txt')
47
-
48
-
49
- def test_filesource_good():
50
- sl = FileCacheSourceFile('file', '')
51
- assert sl.exists(EXPECTED_DIR / EXPECTED_FILENAMES[1])
52
-
53
-
54
- def test_source_notimp():
55
- with pytest.raises(TypeError):
56
- FileCacheSource('', '').exists('')
57
- with pytest.raises(NotImplementedError):
58
- FileCacheSourceHTTP('http', 'fred').upload('', '')
59
- with pytest.raises(NotImplementedError):
60
- FileCacheSourceHTTP('http', 'fred').iterdir_type('')
61
- with pytest.raises(NotImplementedError):
62
- FileCacheSourceHTTP('http', 'fred').unlink('')
63
-
64
-
65
- def test_source_nthreads_bad():
66
- with pytest.raises(ValueError):
67
- FileCacheSourceFile('file', '').retrieve_multi(['/test'], ['/test'], nthreads=-1)
68
- with pytest.raises(ValueError):
69
- FileCacheSourceFile('file', '').retrieve_multi(['/test'], ['/test'], nthreads=4.5)
70
- with pytest.raises(ValueError):
71
- FileCacheSourceFile('file', '').upload_multi(['/test'], ['/test'], nthreads=-1)
72
- with pytest.raises(ValueError):
73
- FileCacheSourceFile('file', '').upload_multi(['/test'], ['/test'], nthreads=4.5)
74
- with pytest.raises(ValueError):
75
- FileCacheSourceFile('file', '').exists_multi(['/test'], nthreads=-1)
76
- with pytest.raises(ValueError):
77
- FileCacheSourceFile('file', '').exists_multi(['/test'], nthreads=4.5)
78
- with pytest.raises(ValueError):
79
- FileCacheSourceFile('file', '').unlink_multi(['/test'], nthreads=-1)
80
- with pytest.raises(ValueError):
81
- FileCacheSourceFile('file', '').unlink_multi(['/test'], nthreads=4.5)
File without changes
File without changes
File without changes
File without changes
File without changes