rms-filecache 2.3.1__tar.gz → 3.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.github/workflows/run-tests.yml +1 -1
- {rms_filecache-2.3.1/rms_filecache.egg-info → rms_filecache-3.1.0}/PKG-INFO +6 -12
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/README.md +4 -9
- rms_filecache-3.1.0/docs/examples.rst +592 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/index.rst +2 -0
- rms_filecache-3.1.0/docs/introduction.rst +556 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/module.rst +2 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/_version.py +16 -3
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/file_cache.py +931 -135
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/file_cache_path.py +409 -96
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/file_cache_source.py +766 -60
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/file_cache_types.py +8 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/pyproject.toml +1 -2
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/requirements.txt +1 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0/rms_filecache.egg-info}/PKG-INFO +6 -12
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/rms_filecache.egg-info/SOURCES.txt +3 -1
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/test_file_cache.py +639 -66
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/test_file_cache_path.py +292 -82
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/test_file_cache_source.py +71 -12
- rms_filecache-2.3.1/tests/test_url_to_path.py → rms_filecache-3.1.0/tests/test_url_mapping.py +265 -24
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.coveragerc +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.flake8 +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.github/pull_request_template.md +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.github/workflows/publish_to_pypi.yml +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.github/workflows/publish_to_test_pypi.yml +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.gitignore +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.readthedocs.yaml +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/LICENSE +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/Makefile +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/conf.py +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/make.bat +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/__init__.py +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/py.typed +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/mypy.ini +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/rms_filecache.egg-info/requires.txt +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/rms_filecache.egg-info/top_level.txt +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/setup.cfg +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_files/expected/lorem1.txt +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_files/expected/subdir1/lorem1.txt +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_programs/locking.py +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_programs/multi_download.py +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/__init__.py +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/gs_lifecycle_config.json +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/gs_lifecycle_config_info.txt +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/s3_bucket_policy_read.txt +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/s3_bucket_policy_write.txt +0 -0
- {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/s3_lifecycle_policy.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rms-filecache
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.1.0
|
|
4
4
|
Summary: File cache for files retrieved from the cloud
|
|
5
5
|
Maintainer-email: "Robert S. French" <rfrench@seti.org>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -15,7 +15,6 @@ Classifier: Natural Language :: English
|
|
|
15
15
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
16
16
|
Classifier: Topic :: Utilities
|
|
17
17
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.10
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.11
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
@@ -23,7 +22,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
23
22
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
24
23
|
Classifier: Operating System :: POSIX :: Linux
|
|
25
24
|
Classifier: Operating System :: Microsoft :: Windows
|
|
26
|
-
Requires-Python: >=3.
|
|
25
|
+
Requires-Python: >=3.10
|
|
27
26
|
Description-Content-Type: text/markdown
|
|
28
27
|
License-File: LICENSE
|
|
29
28
|
Requires-Dist: boto3
|
|
@@ -156,21 +155,16 @@ from filecache import FileCache, FCPath
|
|
|
156
155
|
with FileCache(None) as fc: # Use as context manager
|
|
157
156
|
# Use GS by specifying the bucket name and one directory level
|
|
158
157
|
p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
|
|
159
|
-
# Use S3 by specifying the bucket name and two directory levels
|
|
160
|
-
# Alternative creation method
|
|
161
|
-
p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
|
|
162
|
-
anonymous=True)
|
|
163
158
|
# Access GS using a directory + filename (since only one directory level
|
|
164
159
|
# was specified by the FCPath)
|
|
165
|
-
# The additional
|
|
160
|
+
# The additional filename is specified by using the / operator to create a new
|
|
161
|
+
# FCPath instance; anonymous=True is inherited
|
|
166
162
|
# Also use open() as a context manager
|
|
167
|
-
with p1
|
|
163
|
+
with (p1 / 'subdir2a/binary1.bin').open('rb') as fp:
|
|
168
164
|
bin1 = fp.read()
|
|
169
165
|
# Access S3 using a filename only (since two directory levels were already
|
|
170
166
|
# specified by the FCPath)
|
|
171
|
-
|
|
172
|
-
# FCPath instance; anonymous=True is inherited
|
|
173
|
-
with (p2 / 'binary1.bin').open(mode='rb') as fp:
|
|
167
|
+
with (p2 / 'binary1.bin').open('rb') as fp:
|
|
174
168
|
bin2 = fp.read()
|
|
175
169
|
assert bin1 == bin2
|
|
176
170
|
# Cache automatically deleted here
|
|
@@ -121,21 +121,16 @@ from filecache import FileCache, FCPath
|
|
|
121
121
|
with FileCache(None) as fc: # Use as context manager
|
|
122
122
|
# Use GS by specifying the bucket name and one directory level
|
|
123
123
|
p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
|
|
124
|
-
# Use S3 by specifying the bucket name and two directory levels
|
|
125
|
-
# Alternative creation method
|
|
126
|
-
p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
|
|
127
|
-
anonymous=True)
|
|
128
124
|
# Access GS using a directory + filename (since only one directory level
|
|
129
125
|
# was specified by the FCPath)
|
|
130
|
-
# The additional
|
|
126
|
+
# The additional filename is specified by using the / operator to create a new
|
|
127
|
+
# FCPath instance; anonymous=True is inherited
|
|
131
128
|
# Also use open() as a context manager
|
|
132
|
-
with p1
|
|
129
|
+
with (p1 / 'subdir2a/binary1.bin').open('rb') as fp:
|
|
133
130
|
bin1 = fp.read()
|
|
134
131
|
# Access S3 using a filename only (since two directory levels were already
|
|
135
132
|
# specified by the FCPath)
|
|
136
|
-
|
|
137
|
-
# FCPath instance; anonymous=True is inherited
|
|
138
|
-
with (p2 / 'binary1.bin').open(mode='rb') as fp:
|
|
133
|
+
with (p2 / 'binary1.bin').open('rb') as fp:
|
|
139
134
|
bin2 = fp.read()
|
|
140
135
|
assert bin1 == bin2
|
|
141
136
|
# Cache automatically deleted here
|
|
@@ -0,0 +1,592 @@
|
|
|
1
|
+
Examples
|
|
2
|
+
========
|
|
3
|
+
|
|
4
|
+
FileCache Examples
|
|
5
|
+
******************
|
|
6
|
+
|
|
7
|
+
Here are some examples of how to use :class:`FileCache` in practice.
|
|
8
|
+
|
|
9
|
+
Example 1: Ephemeral Cache - Automatically Deleted on Exit
|
|
10
|
+
----------------------------------------------------------
|
|
11
|
+
|
|
12
|
+
This creates a unique, temporary cache that is automatically deleted when the
|
|
13
|
+
context manager exits. Useful for one-time operations where you don't want
|
|
14
|
+
to leave files behind.
|
|
15
|
+
|
|
16
|
+
.. code-block:: python
|
|
17
|
+
|
|
18
|
+
from filecache import FileCache
|
|
19
|
+
|
|
20
|
+
with FileCache(cache_name=None) as fc:
|
|
21
|
+
# Retrieve a file from Google Cloud Storage
|
|
22
|
+
local_path = fc.retrieve('gs://my-bucket/data/file.txt')
|
|
23
|
+
with open(local_path, 'r') as f:
|
|
24
|
+
content = f.read()
|
|
25
|
+
print(f'Read {len(content)} bytes')
|
|
26
|
+
# Cache is automatically deleted here
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
Example 2: Shared Persistent Cache (Default 'global')
|
|
30
|
+
-----------------------------------------------------
|
|
31
|
+
|
|
32
|
+
This uses the default shared cache named 'global' that persists after
|
|
33
|
+
program exit. Multiple processes can share the same downloaded files.
|
|
34
|
+
|
|
35
|
+
.. code-block:: python
|
|
36
|
+
|
|
37
|
+
from filecache import FileCache
|
|
38
|
+
|
|
39
|
+
fc = FileCache() # Uses default cache_name='global'
|
|
40
|
+
# First process downloads the file
|
|
41
|
+
path1 = fc.retrieve('https://example.com/data.txt')
|
|
42
|
+
# Second process (or same process later) reuses the cached file
|
|
43
|
+
path2 = fc.retrieve('https://example.com/data.txt')
|
|
44
|
+
assert path1 == path2 # Same cached file location
|
|
45
|
+
# Cache persists after program exits
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
Example 3: Named Shared Cache with Custom Location
|
|
49
|
+
--------------------------------------------------
|
|
50
|
+
|
|
51
|
+
Creates a named cache in a specific directory. Useful for organizing
|
|
52
|
+
different types of cached data separately.
|
|
53
|
+
|
|
54
|
+
.. code-block:: python
|
|
55
|
+
|
|
56
|
+
from filecache import FileCache
|
|
57
|
+
from pathlib import Path
|
|
58
|
+
|
|
59
|
+
# Create a cache named 'myproject' in a custom location
|
|
60
|
+
fc = FileCache(cache_name='myproject', cache_root=Path.home() / 'my_caches')
|
|
61
|
+
local_path = fc.retrieve('s3://my-bucket/project_data/file.dat')
|
|
62
|
+
print(f'Cached at: {local_path}')
|
|
63
|
+
# Cache persists and can be shared by other processes using the same name
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
Example 4: Time-Sensitive Cache with Metadata Caching
|
|
67
|
+
-----------------------------------------------------
|
|
68
|
+
|
|
69
|
+
Preserves file modification times and caches metadata for efficiency.
|
|
70
|
+
Useful when you need to track when files were last modified.
|
|
71
|
+
|
|
72
|
+
.. code-block:: python
|
|
73
|
+
|
|
74
|
+
from filecache import FileCache
|
|
75
|
+
|
|
76
|
+
fc = FileCache(time_sensitive=True, cache_metadata=True)
|
|
77
|
+
# First call retrieves modification time from server
|
|
78
|
+
mtime1 = fc.modification_time('gs://my-bucket/data.txt')
|
|
79
|
+
# Second call uses cached value
|
|
80
|
+
mtime2 = fc.modification_time('gs://my-bucket/data.txt')
|
|
81
|
+
assert mtime1 == mtime2 # From cache, no network call
|
|
82
|
+
|
|
83
|
+
# Retrieval downloads and sets modification time
|
|
84
|
+
path1 = fc.retrieve('gs://my-bucket/data.txt')
|
|
85
|
+
# Local file's modification time should match the server's
|
|
86
|
+
assert path1.stat().st_mtime == mtime1
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
Example 5: Parallel File Operations
|
|
90
|
+
-----------------------------------
|
|
91
|
+
|
|
92
|
+
Downloads multiple files simultaneously using multiple threads.
|
|
93
|
+
This is significantly faster when retrieving many files from the same source.
|
|
94
|
+
Download errors are returned as Exceptions in the return list.
|
|
95
|
+
|
|
96
|
+
.. code-block:: python
|
|
97
|
+
|
|
98
|
+
from filecache import FileCache
|
|
99
|
+
|
|
100
|
+
fc = FileCache(nthreads=4) # Use 4 threads for parallel operations
|
|
101
|
+
urls = [
|
|
102
|
+
'gs://my-bucket/file1.txt',
|
|
103
|
+
'gs://my-bucket/file2.txt',
|
|
104
|
+
'gs://my-bucket/file3.txt',
|
|
105
|
+
'gs://my-bucket/file4.txt'
|
|
106
|
+
]
|
|
107
|
+
# All files downloaded in parallel
|
|
108
|
+
paths = fc.retrieve(urls, exception_on_fail=False)
|
|
109
|
+
for path in paths:
|
|
110
|
+
if isinstance(path, Exception):
|
|
111
|
+
print(f'Download failed: {path}')
|
|
112
|
+
continue
|
|
113
|
+
print(f'Downloaded: {path}')
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
Example 6: Upload Operations
|
|
117
|
+
-----------------------------
|
|
118
|
+
|
|
119
|
+
Writes files to remote storage. Files are written locally first, then uploaded.
|
|
120
|
+
|
|
121
|
+
.. code-block:: python
|
|
122
|
+
|
|
123
|
+
from filecache import FileCache
|
|
124
|
+
|
|
125
|
+
with FileCache(cache_name=None) as fc:
|
|
126
|
+
# Write a file to cloud storage
|
|
127
|
+
with fc.open('gs://my-bucket/output.txt', 'w') as f:
|
|
128
|
+
f.write('Hello, World!')
|
|
129
|
+
# File is automatically uploaded when the context manager for the file handle exits
|
|
130
|
+
# Ephemeral cache is automatically deleted here
|
|
131
|
+
|
|
132
|
+
# Verify it was uploaded by reading it back using a difference cache
|
|
133
|
+
with FileCache(cache_name=None) as fc:
|
|
134
|
+
with fc.open('gs://my-bucket/output.txt', 'r') as f:
|
|
135
|
+
content = f.read()
|
|
136
|
+
print(f'File contents: {content}')
|
|
137
|
+
# Ephemeral cache is automatically deleted here
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
.. _example_url_to_url:
|
|
141
|
+
|
|
142
|
+
Example 7: URL to URL Translation
|
|
143
|
+
---------------------------------
|
|
144
|
+
|
|
145
|
+
Translates URLs from one source to another, allowing code to work with
|
|
146
|
+
different data layouts without modification.
|
|
147
|
+
|
|
148
|
+
For example, assume the data is laid out on a webserver as:
|
|
149
|
+
|
|
150
|
+
.. code-block:: text
|
|
151
|
+
|
|
152
|
+
/data/file11.txt
|
|
153
|
+
/data/file12.txt
|
|
154
|
+
/data/file21.txt
|
|
155
|
+
/data/file22.txt
|
|
156
|
+
|
|
157
|
+
and the data is available in a cloud storage bucket as:
|
|
158
|
+
|
|
159
|
+
.. code-block:: text
|
|
160
|
+
|
|
161
|
+
gs://my-bucket/data/dir1/file11.txt
|
|
162
|
+
gs://my-bucket/data/dir1/file12.txt
|
|
163
|
+
gs://my-bucket/data/dir2/file21.txt
|
|
164
|
+
gs://my-bucket/data/dir2/file22.txt
|
|
165
|
+
|
|
166
|
+
The code was written using the layout of the webserver. Later, the Google Cloud Storage
|
|
167
|
+
bucket was made available. Rather than rewriting the code (and thus making it incompatible
|
|
168
|
+
with the original webserver layout), a mapping function can be used to translate the URLs:
|
|
169
|
+
|
|
170
|
+
.. code-block:: python
|
|
171
|
+
|
|
172
|
+
def url_to_url(scheme, remote, path):
|
|
173
|
+
if scheme == "https" and remote == "data.com" and path.startswith("data/"):
|
|
174
|
+
dir_num_match = re.match(r"data/file(\d+)\.txt", path)
|
|
175
|
+
if dir_num_match:
|
|
176
|
+
dir_num = dir_num_match.group(1)
|
|
177
|
+
return f"gs://my-bucket/data/dir{dir_num[0]}/file{dir_num}.txt"
|
|
178
|
+
return None
|
|
179
|
+
|
|
180
|
+
This code will now work both with the original webserver layout and the new Google Cloud Storage
|
|
181
|
+
layout:
|
|
182
|
+
|
|
183
|
+
.. code-block:: python
|
|
184
|
+
|
|
185
|
+
fc = FileCache()
|
|
186
|
+
# This will download from the webserver
|
|
187
|
+
fc.retrieve("https://data.com/data/file11.txt")
|
|
188
|
+
|
|
189
|
+
fc = FileCache(url_to_url=url_to_url)
|
|
190
|
+
# This will download from the Google Cloud Storage bucket
|
|
191
|
+
fc.retrieve("https://data.com/data/file11.txt")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
.. _example_url_to_path:
|
|
195
|
+
|
|
196
|
+
Example 8: URL to Path Translation
|
|
197
|
+
----------------------------------
|
|
198
|
+
|
|
199
|
+
Translates URLs into local paths, allowing code to store files in the local cache in a
|
|
200
|
+
different hierarchy than the remote source. Assume the data is available in a cloud
|
|
201
|
+
storage bucket as:
|
|
202
|
+
|
|
203
|
+
.. code-block:: text
|
|
204
|
+
|
|
205
|
+
gs://my-bucket/data/dir1/file11.txt
|
|
206
|
+
gs://my-bucket/data/dir1/file12.txt
|
|
207
|
+
gs://my-bucket/data/dir2/file21.txt
|
|
208
|
+
gs://my-bucket/data/dir2/file22.txt
|
|
209
|
+
|
|
210
|
+
To store the above Google Cloud Storage data in a flat hierarchy like this:
|
|
211
|
+
|
|
212
|
+
.. code-block:: text
|
|
213
|
+
|
|
214
|
+
/data/file11.txt
|
|
215
|
+
/data/file12.txt
|
|
216
|
+
/data/file21.txt
|
|
217
|
+
/data/file22.txt
|
|
218
|
+
|
|
219
|
+
.. code-block:: python
|
|
220
|
+
|
|
221
|
+
def url_to_path(scheme, remote, path, cache_dir, cache_subdir):
|
|
222
|
+
if scheme == "gs" and remote == "my-bucket" and path.startswith("data/dir"):
|
|
223
|
+
path_split = path.split("/", 2)
|
|
224
|
+
if len(path_split) > 2:
|
|
225
|
+
new_path = path_split[2]
|
|
226
|
+
ret = f"{cache_dir}/{cache_subdir}/data/{new_path}"
|
|
227
|
+
print(ret)
|
|
228
|
+
return ret
|
|
229
|
+
return None
|
|
230
|
+
|
|
231
|
+
fc = FileCache()
|
|
232
|
+
# This file will be stored in <cache_root>/_filecache_global/gs_my-bucket/data/dir1/file11.txt
|
|
233
|
+
fc.retrieve("gs://my-bucket/data/dir1/file11.txt")
|
|
234
|
+
|
|
235
|
+
fc = FileCache(url_to_path=url_to_path)
|
|
236
|
+
# This file will be stored in <cache_root>/_filecache_global/gs_my-bucket/data/file11.txt
|
|
237
|
+
fc.retrieve("gs://my-bucket/data/dir1/file11.txt")
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
Example 9: Local File Access
|
|
241
|
+
------------------------------
|
|
242
|
+
|
|
243
|
+
:class:`FileCache` can also work with local files, providing a unified interface
|
|
244
|
+
regardless of file location.
|
|
245
|
+
|
|
246
|
+
.. code-block:: python
|
|
247
|
+
|
|
248
|
+
from filecache import FileCache
|
|
249
|
+
from pathlib import Path
|
|
250
|
+
|
|
251
|
+
fc = FileCache()
|
|
252
|
+
# Access a local file (no download needed, accessed in-place)
|
|
253
|
+
local_file = Path.home() / 'myfile.txt'
|
|
254
|
+
local_file.write_text('Local content')
|
|
255
|
+
|
|
256
|
+
# FileCache handles local files transparently
|
|
257
|
+
path = fc.retrieve(str(local_file))
|
|
258
|
+
with open(path, 'r') as f:
|
|
259
|
+
content = f.read()
|
|
260
|
+
print(f'Read local file: {content}')
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
Example 10: Manual Cache Management (Non-Context Manager)
|
|
264
|
+
---------------------------------------------------------
|
|
265
|
+
|
|
266
|
+
When using a shared cache, you must manually manage cache deletion.
|
|
267
|
+
Useful when you need more control over cache lifetime.
|
|
268
|
+
|
|
269
|
+
.. code-block:: python
|
|
270
|
+
|
|
271
|
+
from filecache import FileCache
|
|
272
|
+
|
|
273
|
+
# Create a permanent cache (would not auto-delete on program exit)
|
|
274
|
+
fc = FileCache(cache_name='mycache')
|
|
275
|
+
|
|
276
|
+
# Use the cache
|
|
277
|
+
path1 = fc.retrieve('gs://my-bucket/file1.txt')
|
|
278
|
+
path2 = fc.retrieve('gs://my-bucket/file2.txt')
|
|
279
|
+
|
|
280
|
+
# Manually delete cache when done
|
|
281
|
+
# Be careful that another process is not using the cache at the same time!
|
|
282
|
+
fc.delete_cache()
|
|
283
|
+
print('Cache manually deleted')
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
FCPath Examples
|
|
287
|
+
***************
|
|
288
|
+
|
|
289
|
+
Here are examples showcasing the unique features of :class:`FCPath`, which provides a
|
|
290
|
+
Path-like interface for working with remote files.
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
Example 11: Using FCPath for Simpler Syntax
|
|
294
|
+
-------------------------------------------
|
|
295
|
+
|
|
296
|
+
Here are examples of creating :class:`FCPath` instances and using them to access files.
|
|
297
|
+
|
|
298
|
+
.. code-block:: python
|
|
299
|
+
|
|
300
|
+
from filecache import FileCache, FCPath
|
|
301
|
+
|
|
302
|
+
with FileCache(cache_name=None) as fc:
|
|
303
|
+
# Create an FCPath that encapsulates the cache settings
|
|
304
|
+
base_path = fc.new_path('gs://my-bucket/data')
|
|
305
|
+
|
|
306
|
+
# Use Path-like operations
|
|
307
|
+
file1 = base_path / 'subdir' / 'file1.txt'
|
|
308
|
+
file2 = base_path / 'subdir' / 'file2.txt'
|
|
309
|
+
|
|
310
|
+
# Read files using simple Path methods
|
|
311
|
+
content1 = file1.read_text()
|
|
312
|
+
content2 = file2.read_text()
|
|
313
|
+
print(f'Read {len(content1)} and {len(content2)} bytes')
|
|
314
|
+
|
|
315
|
+
The same code but using the :class:`FCPath` constructor directly:
|
|
316
|
+
|
|
317
|
+
.. code-block:: python
|
|
318
|
+
|
|
319
|
+
from filecache import FileCache, FCPath
|
|
320
|
+
|
|
321
|
+
fc = FileCache(cache_name=None)
|
|
322
|
+
base_path = FCPath('gs://my-bucket/data', filecache=fc)
|
|
323
|
+
|
|
324
|
+
file1 = base_path / 'subdir' / 'file1.txt'
|
|
325
|
+
file2 = base_path / 'subdir' / 'file2.txt'
|
|
326
|
+
|
|
327
|
+
content1 = file1.read_text()
|
|
328
|
+
content2 = file2.read_text()
|
|
329
|
+
print(f'Read {len(content1)} and {len(content2)} bytes')
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
Example 12: Path Joining with the / Operator
|
|
333
|
+
--------------------------------------------
|
|
334
|
+
|
|
335
|
+
:class:`FCPath` supports the ``/`` operator for joining paths, just like `pathlib.Path`.
|
|
336
|
+
The new :class:`FCPath` inherits all settings (FileCache, time_sensitive, etc.) from the
|
|
337
|
+
left-hand side.
|
|
338
|
+
|
|
339
|
+
.. code-block:: python
|
|
340
|
+
|
|
341
|
+
from filecache import FileCache, FCPath
|
|
342
|
+
|
|
343
|
+
with FileCache(cache_name=None, time_sensitive=True) as fc:
|
|
344
|
+
# Create a base FCPath with specific settings
|
|
345
|
+
base = fc.new_path('gs://my-bucket/data')
|
|
346
|
+
|
|
347
|
+
# Join paths using the / operator
|
|
348
|
+
file1 = base / 'subdir' / 'file1.txt'
|
|
349
|
+
file2 = base / 'subdir' / 'file2.txt'
|
|
350
|
+
|
|
351
|
+
# All resulting FCPath objects inherit the FileCache and time_sensitive setting
|
|
352
|
+
content1 = file1.read_text() # Uses same cache and time_sensitive=True
|
|
353
|
+
content2 = file2.read_text() # Uses same cache and time_sensitive=True
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
Example 13: Creating FCPath from Different Types
|
|
357
|
+
-------------------------------------------------
|
|
358
|
+
|
|
359
|
+
:class:`FCPath` can be created from strings, `Path` objects, or other `FCPath` objects.
|
|
360
|
+
When created from an existing :class:`FCPath`, it inherits all settings.
|
|
361
|
+
|
|
362
|
+
.. code-block:: python
|
|
363
|
+
|
|
364
|
+
from filecache import FileCache, FCPath
|
|
365
|
+
from pathlib import Path
|
|
366
|
+
|
|
367
|
+
with FileCache(cache_name='myproject', time_sensitive=True) as fc:
|
|
368
|
+
# Create from string
|
|
369
|
+
path1 = FCPath('gs://my-bucket/data/file.txt', filecache=fc)
|
|
370
|
+
|
|
371
|
+
# Create from Path object
|
|
372
|
+
local_path = Path('/local/path/file.txt')
|
|
373
|
+
path2 = FCPath(local_path, filecache=fc)
|
|
374
|
+
|
|
375
|
+
# Create from another FCPath (inherits all settings)
|
|
376
|
+
path3 = FCPath(path1) # Inherits filecache, time_sensitive, etc. from path1
|
|
377
|
+
path4 = path1 / 'subdir' / 'file2.txt' # Also inherits via / operator
|
|
378
|
+
|
|
379
|
+
# All can be used the same way
|
|
380
|
+
content1 = path1.read_text()
|
|
381
|
+
content2 = path2.read_text()
|
|
382
|
+
content3 = path3.read_text()
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
Example 14: Inheriting FileCache Settings
|
|
386
|
+
------------------------------------------
|
|
387
|
+
|
|
388
|
+
When creating new :class:`FCPath` objects from existing ones, all FileCache-related
|
|
389
|
+
settings are automatically inherited, making it easy to work with paths that share
|
|
390
|
+
the same configuration.
|
|
391
|
+
|
|
392
|
+
.. code-block:: python
|
|
393
|
+
|
|
394
|
+
from filecache import FileCache, FCPath
|
|
395
|
+
|
|
396
|
+
# Create a FileCache with specific settings
|
|
397
|
+
fc = FileCache(cache_name='project', time_sensitive=True, nthreads=4)
|
|
398
|
+
|
|
399
|
+
# Create base FCPath with a different lock timeout
|
|
400
|
+
base = fc.new_path('gs://my-bucket/data', lock_timeout=30)
|
|
401
|
+
|
|
402
|
+
# All child paths inherit: filecache, lock_timeout=30
|
|
403
|
+
file1 = base / 'dir1' / 'file1.txt'
|
|
404
|
+
file2 = base / 'dir2' / 'file2.txt'
|
|
405
|
+
file3 = FCPath(base) / 'dir3' / 'file3.txt'
|
|
406
|
+
|
|
407
|
+
# All operations use the same settings
|
|
408
|
+
paths = [file1, file2, file3]
|
|
409
|
+
contents = [f.read_text() for f in paths] # All use time_sensitive=True, lock_timeout=30
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
Example 15: Using glob() for Pattern Matching
|
|
413
|
+
---------------------------------------------
|
|
414
|
+
|
|
415
|
+
:class:`FCPath` supports `glob()` for pattern matching on remote directories, a feature
|
|
416
|
+
not available directly in :class:`FileCache`. This works on both local and remote paths.
|
|
417
|
+
|
|
418
|
+
.. code-block:: python
|
|
419
|
+
|
|
420
|
+
from filecache import FileCache, FCPath
|
|
421
|
+
|
|
422
|
+
with FileCache(cache_name=None) as fc:
|
|
423
|
+
# Create base path to a directory
|
|
424
|
+
base_dir = fc.new_path('gs://my-bucket/data')
|
|
425
|
+
|
|
426
|
+
# Find all .txt files in the directory
|
|
427
|
+
txt_files = list(base_dir.glob('*.txt'))
|
|
428
|
+
for txt_file in txt_files:
|
|
429
|
+
print(f'Found: {txt_file.path}')
|
|
430
|
+
content = txt_file.read_text()
|
|
431
|
+
|
|
432
|
+
# Find files in subdirectories
|
|
433
|
+
all_txt = list(base_dir.glob('**/*.txt')) # Recursive
|
|
434
|
+
for file in all_txt:
|
|
435
|
+
print(f'Recursive match: {file.path}')
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
Example 16: Using rglob() for Recursive Pattern Matching
|
|
439
|
+
--------------------------------------------------------
|
|
440
|
+
|
|
441
|
+
:meth:`FCPath.rglob()` is a convenience method that automatically adds ``**/`` to the pattern,
|
|
442
|
+
making recursive searches easier.
|
|
443
|
+
|
|
444
|
+
.. code-block:: python
|
|
445
|
+
|
|
446
|
+
from filecache import FileCache, FCPath
|
|
447
|
+
|
|
448
|
+
with FileCache() as fc:
|
|
449
|
+
base = fc.new_path('gs://my-bucket/project')
|
|
450
|
+
|
|
451
|
+
# Find all Python files recursively
|
|
452
|
+
# rglob('*.py') is equivalent to glob('**/*.py')
|
|
453
|
+
python_files = list(base.rglob('*.py'))
|
|
454
|
+
|
|
455
|
+
for py_file in python_files:
|
|
456
|
+
print(f'Python file: {py_file.path}')
|
|
457
|
+
# Process each file
|
|
458
|
+
content = py_file.read_text()
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
Example 17: Directory Traversal with iterdir()
|
|
462
|
+
----------------------------------------------
|
|
463
|
+
|
|
464
|
+
:class:`FCPath` provides :meth:`FCPath.iterdir()` for iterating over directory contents, returning
|
|
465
|
+
:class:`FCPath` objects that inherit settings from the parent.
|
|
466
|
+
|
|
467
|
+
.. code-block:: python
|
|
468
|
+
|
|
469
|
+
from filecache import FileCache, FCPath
|
|
470
|
+
|
|
471
|
+
with FileCache(cache_name=None) as fc:
|
|
472
|
+
base_dir = fc.new_path('gs://my-bucket/data')
|
|
473
|
+
|
|
474
|
+
# Iterate over directory contents
|
|
475
|
+
for item in base_dir.iterdir():
|
|
476
|
+
if item.is_dir():
|
|
477
|
+
print(f'Directory: {item.path}')
|
|
478
|
+
# Recursively process subdirectories
|
|
479
|
+
for subitem in item.iterdir():
|
|
480
|
+
print(f' Subitem: {subitem.path}')
|
|
481
|
+
else:
|
|
482
|
+
print(f'File: {item.path}')
|
|
483
|
+
# Read file content
|
|
484
|
+
content = item.read_text()
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
Example 18: Directory Walking with walk()
|
|
488
|
+
-----------------------------------------
|
|
489
|
+
|
|
490
|
+
The `walk()` method provides a convenient way to traverse directory trees, similar to
|
|
491
|
+
`os.walk()` but returning :class:`FCPath` objects.
|
|
492
|
+
|
|
493
|
+
.. code-block:: python
|
|
494
|
+
|
|
495
|
+
from filecache import FileCache, FCPath
|
|
496
|
+
|
|
497
|
+
with FileCache() as fc:
|
|
498
|
+
root = fc.new_path('gs://my-bucket/project')
|
|
499
|
+
|
|
500
|
+
# Walk the directory tree
|
|
501
|
+
for dirpath, dirnames, filenames in root.walk():
|
|
502
|
+
print(f'Directory: {dirpath.path}')
|
|
503
|
+
print(f' Subdirectories: {dirnames}')
|
|
504
|
+
print(f' Files: {filenames}')
|
|
505
|
+
|
|
506
|
+
# Process files in this directory
|
|
507
|
+
for filename in filenames:
|
|
508
|
+
file_path = dirpath / filename
|
|
509
|
+
content = file_path.read_text()
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
Example 19: Combining Path Operations
|
|
513
|
+
-------------------------------------
|
|
514
|
+
|
|
515
|
+
:class:`FCPath` supports chaining of path operations, making complex path manipulations
|
|
516
|
+
easy and readable.
|
|
517
|
+
|
|
518
|
+
.. code-block:: python
|
|
519
|
+
|
|
520
|
+
from filecache import FileCache, FCPath
|
|
521
|
+
|
|
522
|
+
with FileCache(cache_name='analysis') as fc:
|
|
523
|
+
# Start with a base URL
|
|
524
|
+
base = fc.new_path('gs://data-bucket')
|
|
525
|
+
|
|
526
|
+
# Build complex paths using chaining
|
|
527
|
+
data_file = base / 'year' / '2024' / 'month' / '01' / 'data.csv'
|
|
528
|
+
|
|
529
|
+
# Read both files
|
|
530
|
+
data = data_file.read_text()
|
|
531
|
+
config = (base / 'config' / 'settings.json').read_text()
|
|
532
|
+
|
|
533
|
+
# Create output path in same structure
|
|
534
|
+
output_dir = base / 'output' / '2024' / '01'
|
|
535
|
+
|
|
536
|
+
# Write to output
|
|
537
|
+
(output_dir / 'results.txt').write_text('Analysis results')
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
Example 20: Working with Metadata via iterdir_metadata()
|
|
541
|
+
--------------------------------------------------------
|
|
542
|
+
|
|
543
|
+
The `iterdir_metadata()` method provides directory contents along with metadata
|
|
544
|
+
(is_dir, mtime, size), useful for filtering or sorting files.
|
|
545
|
+
|
|
546
|
+
.. code-block:: python
|
|
547
|
+
|
|
548
|
+
from filecache import FileCache, FCPath
|
|
549
|
+
|
|
550
|
+
with FileCache() as fc:
|
|
551
|
+
base_dir = fc.new_path('gs://my-bucket/data')
|
|
552
|
+
|
|
553
|
+
# Get directory contents with metadata
|
|
554
|
+
files_by_size = []
|
|
555
|
+
for item, metadata in base_dir.iterdir_metadata():
|
|
556
|
+
if metadata and not metadata['is_dir']:
|
|
557
|
+
files_by_size.append((item, metadata['size']))
|
|
558
|
+
|
|
559
|
+
# Sort by size (largest first)
|
|
560
|
+
files_by_size.sort(key=lambda x: x[1] or 0, reverse=True)
|
|
561
|
+
|
|
562
|
+
# Process largest files first
|
|
563
|
+
for item, size in files_by_size:
|
|
564
|
+
print(f'Processing {item.path} ({size} bytes)')
|
|
565
|
+
content = item.read_text()
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
Example 21: Creating FCPath Without Explicit FileCache
|
|
569
|
+
-------------------------------------------------------
|
|
570
|
+
|
|
571
|
+
When an :class:`FCPath` is created without specifying a `filecache`, it uses the default
|
|
572
|
+
global :class:`FileCache` when an operation is performed. This allows for simpler
|
|
573
|
+
syntax when the default cache is sufficient.
|
|
574
|
+
|
|
575
|
+
.. code-block:: python
|
|
576
|
+
|
|
577
|
+
from filecache import FCPath
|
|
578
|
+
|
|
579
|
+
# Create FCPath without explicit FileCache
|
|
580
|
+
# Will use default global cache when operations are performed
|
|
581
|
+
base = FCPath('gs://my-bucket/data')
|
|
582
|
+
|
|
583
|
+
# Operations automatically use the default FileCache
|
|
584
|
+
file1 = base / 'file1.txt'
|
|
585
|
+
file2 = base / 'file2.txt'
|
|
586
|
+
|
|
587
|
+
# Read files (uses default global cache)
|
|
588
|
+
content1 = file1.read_text()
|
|
589
|
+
content2 = file2.read_text()
|
|
590
|
+
|
|
591
|
+
# All paths share the same default cache
|
|
592
|
+
assert file1.filecache == file2.filecache
|