rms-filecache 2.3.1__tar.gz → 3.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.github/workflows/run-tests.yml +1 -1
  2. {rms_filecache-2.3.1/rms_filecache.egg-info → rms_filecache-3.1.0}/PKG-INFO +6 -12
  3. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/README.md +4 -9
  4. rms_filecache-3.1.0/docs/examples.rst +592 -0
  5. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/index.rst +2 -0
  6. rms_filecache-3.1.0/docs/introduction.rst +556 -0
  7. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/module.rst +2 -0
  8. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/_version.py +16 -3
  9. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/file_cache.py +931 -135
  10. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/file_cache_path.py +409 -96
  11. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/file_cache_source.py +766 -60
  12. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/file_cache_types.py +8 -0
  13. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/pyproject.toml +1 -2
  14. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/requirements.txt +1 -0
  15. {rms_filecache-2.3.1 → rms_filecache-3.1.0/rms_filecache.egg-info}/PKG-INFO +6 -12
  16. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/rms_filecache.egg-info/SOURCES.txt +3 -1
  17. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/test_file_cache.py +639 -66
  18. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/test_file_cache_path.py +292 -82
  19. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/test_file_cache_source.py +71 -12
  20. rms_filecache-2.3.1/tests/test_url_to_path.py → rms_filecache-3.1.0/tests/test_url_mapping.py +265 -24
  21. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.coveragerc +0 -0
  22. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.flake8 +0 -0
  23. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.github/pull_request_template.md +0 -0
  24. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.github/workflows/publish_to_pypi.yml +0 -0
  25. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.github/workflows/publish_to_test_pypi.yml +0 -0
  26. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.gitignore +0 -0
  27. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/.readthedocs.yaml +0 -0
  28. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/LICENSE +0 -0
  29. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/Makefile +0 -0
  30. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/conf.py +0 -0
  31. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/docs/make.bat +0 -0
  32. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/__init__.py +0 -0
  33. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/filecache/py.typed +0 -0
  34. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/mypy.ini +0 -0
  35. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/rms_filecache.egg-info/dependency_links.txt +0 -0
  36. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/rms_filecache.egg-info/requires.txt +0 -0
  37. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/rms_filecache.egg-info/top_level.txt +0 -0
  38. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/setup.cfg +0 -0
  39. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_files/expected/lorem1.txt +0 -0
  40. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_files/expected/subdir1/lorem1.txt +0 -0
  41. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_files/expected/subdir1/subdir2a/binary1.bin +0 -0
  42. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_files/expected/subdir1/subdir2b/binary1.bin +0 -0
  43. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_programs/locking.py +0 -0
  44. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/test_programs/multi_download.py +0 -0
  45. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/__init__.py +0 -0
  46. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/gs_lifecycle_config.json +0 -0
  47. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/gs_lifecycle_config_info.txt +0 -0
  48. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/s3_bucket_policy_read.txt +0 -0
  49. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/s3_bucket_policy_write.txt +0 -0
  50. {rms_filecache-2.3.1 → rms_filecache-3.1.0}/tests/s3_lifecycle_policy.txt +0 -0
@@ -43,7 +43,7 @@ jobs:
43
43
  strategy:
44
44
  matrix:
45
45
  os: [ubuntu-latest, macos-latest, windows-latest]
46
- python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
46
+ python-version: ['3.10', '3.11', '3.12', '3.13']
47
47
  fail-fast: false
48
48
  permissions:
49
49
  contents: read
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rms-filecache
3
- Version: 2.3.1
3
+ Version: 3.1.0
4
4
  Summary: File cache for files retrieved from the cloud
5
5
  Maintainer-email: "Robert S. French" <rfrench@seti.org>
6
6
  License: Apache-2.0
@@ -15,7 +15,6 @@ Classifier: Natural Language :: English
15
15
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
16
16
  Classifier: Topic :: Utilities
17
17
  Classifier: License :: OSI Approved :: Apache Software License
18
- Classifier: Programming Language :: Python :: 3.9
19
18
  Classifier: Programming Language :: Python :: 3.10
20
19
  Classifier: Programming Language :: Python :: 3.11
21
20
  Classifier: Programming Language :: Python :: 3.12
@@ -23,7 +22,7 @@ Classifier: Programming Language :: Python :: 3.13
23
22
  Classifier: Operating System :: MacOS :: MacOS X
24
23
  Classifier: Operating System :: POSIX :: Linux
25
24
  Classifier: Operating System :: Microsoft :: Windows
26
- Requires-Python: >=3.9
25
+ Requires-Python: >=3.10
27
26
  Description-Content-Type: text/markdown
28
27
  License-File: LICENSE
29
28
  Requires-Dist: boto3
@@ -156,21 +155,16 @@ from filecache import FileCache, FCPath
156
155
  with FileCache(None) as fc: # Use as context manager
157
156
  # Use GS by specifying the bucket name and one directory level
158
157
  p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
159
- # Use S3 by specifying the bucket name and two directory levels
160
- # Alternative creation method
161
- p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
162
- anonymous=True)
163
158
  # Access GS using a directory + filename (since only one directory level
164
159
  # was specified by the FCPath)
165
- # The additional directory and filename are specified as an argument to open()
160
+ # The additional filename is specified by using the / operator to create a new
161
+ # FCPath instance; anonymous=True is inherited
166
162
  # Also use open() as a context manager
167
- with p1.open('subdir2a/binary1.bin', 'rb') as fp:
163
+ with (p1 / 'subdir2a/binary1.bin').open('rb') as fp:
168
164
  bin1 = fp.read()
169
165
  # Access S3 using a filename only (since two directory levels were already
170
166
  # specified by the FCPath)
171
- # The additional filename is specified by using the / operator to create a new
172
- # FCPath instance; anonymous=True is inherited
173
- with (p2 / 'binary1.bin').open(mode='rb') as fp:
167
+ with (p2 / 'binary1.bin').open('rb') as fp:
174
168
  bin2 = fp.read()
175
169
  assert bin1 == bin2
176
170
  # Cache automatically deleted here
@@ -121,21 +121,16 @@ from filecache import FileCache, FCPath
121
121
  with FileCache(None) as fc: # Use as context manager
122
122
  # Use GS by specifying the bucket name and one directory level
123
123
  p1 = fc.new_path('gs://rms-filecache-tests/subdir1', anonymous=True)
124
- # Use S3 by specifying the bucket name and two directory levels
125
- # Alternative creation method
126
- p2 = FCPath('s3://rms-filecache-tests/subdir1/subdir2a', filecache=fc,
127
- anonymous=True)
128
124
  # Access GS using a directory + filename (since only one directory level
129
125
  # was specified by the FCPath)
130
- # The additional directory and filename are specified as an argument to open()
126
+ # The additional filename is specified by using the / operator to create a new
127
+ # FCPath instance; anonymous=True is inherited
131
128
  # Also use open() as a context manager
132
- with p1.open('subdir2a/binary1.bin', 'rb') as fp:
129
+ with (p1 / 'subdir2a/binary1.bin').open('rb') as fp:
133
130
  bin1 = fp.read()
134
131
  # Access S3 using a filename only (since two directory levels were already
135
132
  # specified by the FCPath)
136
- # The additional filename is specified by using the / operator to create a new
137
- # FCPath instance; anonymous=True is inherited
138
- with (p2 / 'binary1.bin').open(mode='rb') as fp:
133
+ with (p2 / 'binary1.bin').open('rb') as fp:
139
134
  bin2 = fp.read()
140
135
  assert bin1 == bin2
141
136
  # Cache automatically deleted here
@@ -0,0 +1,592 @@
1
+ Examples
2
+ ========
3
+
4
+ FileCache Examples
5
+ ******************
6
+
7
+ Here are some examples of how to use :class:`FileCache` in practice.
8
+
9
+ Example 1: Ephemeral Cache - Automatically Deleted on Exit
10
+ ----------------------------------------------------------
11
+
12
+ This creates a unique, temporary cache that is automatically deleted when the
13
+ context manager exits. Useful for one-time operations where you don't want
14
+ to leave files behind.
15
+
16
+ .. code-block:: python
17
+
18
+ from filecache import FileCache
19
+
20
+ with FileCache(cache_name=None) as fc:
21
+ # Retrieve a file from Google Cloud Storage
22
+ local_path = fc.retrieve('gs://my-bucket/data/file.txt')
23
+ with open(local_path, 'r') as f:
24
+ content = f.read()
25
+ print(f'Read {len(content)} bytes')
26
+ # Cache is automatically deleted here
27
+
28
+
29
+ Example 2: Shared Persistent Cache (Default 'global')
30
+ -----------------------------------------------------
31
+
32
+ This uses the default shared cache named 'global' that persists after
33
+ program exit. Multiple processes can share the same downloaded files.
34
+
35
+ .. code-block:: python
36
+
37
+ from filecache import FileCache
38
+
39
+ fc = FileCache() # Uses default cache_name='global'
40
+ # First process downloads the file
41
+ path1 = fc.retrieve('https://example.com/data.txt')
42
+ # Second process (or same process later) reuses the cached file
43
+ path2 = fc.retrieve('https://example.com/data.txt')
44
+ assert path1 == path2 # Same cached file location
45
+ # Cache persists after program exits
46
+
47
+
48
+ Example 3: Named Shared Cache with Custom Location
49
+ --------------------------------------------------
50
+
51
+ Creates a named cache in a specific directory. Useful for organizing
52
+ different types of cached data separately.
53
+
54
+ .. code-block:: python
55
+
56
+ from filecache import FileCache
57
+ from pathlib import Path
58
+
59
+ # Create a cache named 'myproject' in a custom location
60
+ fc = FileCache(cache_name='myproject', cache_root=Path.home() / 'my_caches')
61
+ local_path = fc.retrieve('s3://my-bucket/project_data/file.dat')
62
+ print(f'Cached at: {local_path}')
63
+ # Cache persists and can be shared by other processes using the same name
64
+
65
+
66
+ Example 4: Time-Sensitive Cache with Metadata Caching
67
+ -----------------------------------------------------
68
+
69
+ Preserves file modification times and caches metadata for efficiency.
70
+ Useful when you need to track when files were last modified.
71
+
72
+ .. code-block:: python
73
+
74
+ from filecache import FileCache
75
+
76
+ fc = FileCache(time_sensitive=True, cache_metadata=True)
77
+ # First call retrieves modification time from server
78
+ mtime1 = fc.modification_time('gs://my-bucket/data.txt')
79
+ # Second call uses cached value
80
+ mtime2 = fc.modification_time('gs://my-bucket/data.txt')
81
+ assert mtime1 == mtime2 # From cache, no network call
82
+
83
+ # Retrieval downloads and sets modification time
84
+ path1 = fc.retrieve('gs://my-bucket/data.txt')
85
+ # Local file's modification time should match the server's
86
+ assert path1.stat().st_mtime == mtime1
87
+
88
+
89
+ Example 5: Parallel File Operations
90
+ -----------------------------------
91
+
92
+ Downloads multiple files simultaneously using multiple threads.
93
+ This is significantly faster when retrieving many files from the same source.
94
+ Download errors are returned as Exceptions in the return list.
95
+
96
+ .. code-block:: python
97
+
98
+ from filecache import FileCache
99
+
100
+ fc = FileCache(nthreads=4) # Use 4 threads for parallel operations
101
+ urls = [
102
+ 'gs://my-bucket/file1.txt',
103
+ 'gs://my-bucket/file2.txt',
104
+ 'gs://my-bucket/file3.txt',
105
+ 'gs://my-bucket/file4.txt'
106
+ ]
107
+ # All files downloaded in parallel
108
+ paths = fc.retrieve(urls, exception_on_fail=False)
109
+ for path in paths:
110
+ if isinstance(path, Exception):
111
+ print(f'Download failed: {path}')
112
+ continue
113
+ print(f'Downloaded: {path}')
114
+
115
+
116
+ Example 6: Upload Operations
117
+ -----------------------------
118
+
119
+ Writes files to remote storage. Files are written locally first, then uploaded.
120
+
121
+ .. code-block:: python
122
+
123
+ from filecache import FileCache
124
+
125
+ with FileCache(cache_name=None) as fc:
126
+ # Write a file to cloud storage
127
+ with fc.open('gs://my-bucket/output.txt', 'w') as f:
128
+ f.write('Hello, World!')
129
+ # File is automatically uploaded when the context manager for the file handle exits
130
+ # Ephemeral cache is automatically deleted here
131
+
132
+ # Verify it was uploaded by reading it back using a difference cache
133
+ with FileCache(cache_name=None) as fc:
134
+ with fc.open('gs://my-bucket/output.txt', 'r') as f:
135
+ content = f.read()
136
+ print(f'File contents: {content}')
137
+ # Ephemeral cache is automatically deleted here
138
+
139
+
140
+ .. _example_url_to_url:
141
+
142
+ Example 7: URL to URL Translation
143
+ ---------------------------------
144
+
145
+ Translates URLs from one source to another, allowing code to work with
146
+ different data layouts without modification.
147
+
148
+ For example, assume the data is laid out on a webserver as:
149
+
150
+ .. code-block:: text
151
+
152
+ /data/file11.txt
153
+ /data/file12.txt
154
+ /data/file21.txt
155
+ /data/file22.txt
156
+
157
+ and the data is available in a cloud storage bucket as:
158
+
159
+ .. code-block:: text
160
+
161
+ gs://my-bucket/data/dir1/file11.txt
162
+ gs://my-bucket/data/dir1/file12.txt
163
+ gs://my-bucket/data/dir2/file21.txt
164
+ gs://my-bucket/data/dir2/file22.txt
165
+
166
+ The code was written using the layout of the webserver. Later, the Google Cloud Storage
167
+ bucket was made available. Rather than rewriting the code (and thus making it incompatible
168
+ with the original webserver layout), a mapping function can be used to translate the URLs:
169
+
170
+ .. code-block:: python
171
+
172
+ def url_to_url(scheme, remote, path):
173
+ if scheme == "https" and remote == "data.com" and path.startswith("data/"):
174
+ dir_num_match = re.match(r"data/file(\d+)\.txt", path)
175
+ if dir_num_match:
176
+ dir_num = dir_num_match.group(1)
177
+ return f"gs://my-bucket/data/dir{dir_num[0]}/file{dir_num}.txt"
178
+ return None
179
+
180
+ This code will now work both with the original webserver layout and the new Google Cloud Storage
181
+ layout:
182
+
183
+ .. code-block:: python
184
+
185
+ fc = FileCache()
186
+ # This will download from the webserver
187
+ fc.retrieve("https://data.com/data/file11.txt")
188
+
189
+ fc = FileCache(url_to_url=url_to_url)
190
+ # This will download from the Google Cloud Storage bucket
191
+ fc.retrieve("https://data.com/data/file11.txt")
192
+
193
+
194
+ .. _example_url_to_path:
195
+
196
+ Example 8: URL to Path Translation
197
+ ----------------------------------
198
+
199
+ Translates URLs into local paths, allowing code to store files in the local cache in a
200
+ different hierarchy than the remote source. Assume the data is available in a cloud
201
+ storage bucket as:
202
+
203
+ .. code-block:: text
204
+
205
+ gs://my-bucket/data/dir1/file11.txt
206
+ gs://my-bucket/data/dir1/file12.txt
207
+ gs://my-bucket/data/dir2/file21.txt
208
+ gs://my-bucket/data/dir2/file22.txt
209
+
210
+ To store the above Google Cloud Storage data in a flat hierarchy like this:
211
+
212
+ .. code-block:: text
213
+
214
+ /data/file11.txt
215
+ /data/file12.txt
216
+ /data/file21.txt
217
+ /data/file22.txt
218
+
219
+ .. code-block:: python
220
+
221
+ def url_to_path(scheme, remote, path, cache_dir, cache_subdir):
222
+ if scheme == "gs" and remote == "my-bucket" and path.startswith("data/dir"):
223
+ path_split = path.split("/", 2)
224
+ if len(path_split) > 2:
225
+ new_path = path_split[2]
226
+ ret = f"{cache_dir}/{cache_subdir}/data/{new_path}"
227
+ print(ret)
228
+ return ret
229
+ return None
230
+
231
+ fc = FileCache()
232
+ # This file will be stored in <cache_root>/_filecache_global/gs_my-bucket/data/dir1/file11.txt
233
+ fc.retrieve("gs://my-bucket/data/dir1/file11.txt")
234
+
235
+ fc = FileCache(url_to_path=url_to_path)
236
+ # This file will be stored in <cache_root>/_filecache_global/gs_my-bucket/data/file11.txt
237
+ fc.retrieve("gs://my-bucket/data/dir1/file11.txt")
238
+
239
+
240
+ Example 9: Local File Access
241
+ ------------------------------
242
+
243
+ :class:`FileCache` can also work with local files, providing a unified interface
244
+ regardless of file location.
245
+
246
+ .. code-block:: python
247
+
248
+ from filecache import FileCache
249
+ from pathlib import Path
250
+
251
+ fc = FileCache()
252
+ # Access a local file (no download needed, accessed in-place)
253
+ local_file = Path.home() / 'myfile.txt'
254
+ local_file.write_text('Local content')
255
+
256
+ # FileCache handles local files transparently
257
+ path = fc.retrieve(str(local_file))
258
+ with open(path, 'r') as f:
259
+ content = f.read()
260
+ print(f'Read local file: {content}')
261
+
262
+
263
+ Example 10: Manual Cache Management (Non-Context Manager)
264
+ ---------------------------------------------------------
265
+
266
+ When using a shared cache, you must manually manage cache deletion.
267
+ Useful when you need more control over cache lifetime.
268
+
269
+ .. code-block:: python
270
+
271
+ from filecache import FileCache
272
+
273
+ # Create a permanent cache (would not auto-delete on program exit)
274
+ fc = FileCache(cache_name='mycache')
275
+
276
+ # Use the cache
277
+ path1 = fc.retrieve('gs://my-bucket/file1.txt')
278
+ path2 = fc.retrieve('gs://my-bucket/file2.txt')
279
+
280
+ # Manually delete cache when done
281
+ # Be careful that another process is not using the cache at the same time!
282
+ fc.delete_cache()
283
+ print('Cache manually deleted')
284
+
285
+
286
+ FCPath Examples
287
+ ***************
288
+
289
+ Here are examples showcasing the unique features of :class:`FCPath`, which provides a
290
+ Path-like interface for working with remote files.
291
+
292
+
293
+ Example 11: Using FCPath for Simpler Syntax
294
+ -------------------------------------------
295
+
296
+ Here are examples of creating :class:`FCPath` instances and using them to access files.
297
+
298
+ .. code-block:: python
299
+
300
+ from filecache import FileCache, FCPath
301
+
302
+ with FileCache(cache_name=None) as fc:
303
+ # Create an FCPath that encapsulates the cache settings
304
+ base_path = fc.new_path('gs://my-bucket/data')
305
+
306
+ # Use Path-like operations
307
+ file1 = base_path / 'subdir' / 'file1.txt'
308
+ file2 = base_path / 'subdir' / 'file2.txt'
309
+
310
+ # Read files using simple Path methods
311
+ content1 = file1.read_text()
312
+ content2 = file2.read_text()
313
+ print(f'Read {len(content1)} and {len(content2)} bytes')
314
+
315
+ The same code but using the :class:`FCPath` constructor directly:
316
+
317
+ .. code-block:: python
318
+
319
+ from filecache import FileCache, FCPath
320
+
321
+ fc = FileCache(cache_name=None)
322
+ base_path = FCPath('gs://my-bucket/data', filecache=fc)
323
+
324
+ file1 = base_path / 'subdir' / 'file1.txt'
325
+ file2 = base_path / 'subdir' / 'file2.txt'
326
+
327
+ content1 = file1.read_text()
328
+ content2 = file2.read_text()
329
+ print(f'Read {len(content1)} and {len(content2)} bytes')
330
+
331
+
332
+ Example 12: Path Joining with the / Operator
333
+ --------------------------------------------
334
+
335
+ :class:`FCPath` supports the ``/`` operator for joining paths, just like `pathlib.Path`.
336
+ The new :class:`FCPath` inherits all settings (FileCache, time_sensitive, etc.) from the
337
+ left-hand side.
338
+
339
+ .. code-block:: python
340
+
341
+ from filecache import FileCache, FCPath
342
+
343
+ with FileCache(cache_name=None, time_sensitive=True) as fc:
344
+ # Create a base FCPath with specific settings
345
+ base = fc.new_path('gs://my-bucket/data')
346
+
347
+ # Join paths using the / operator
348
+ file1 = base / 'subdir' / 'file1.txt'
349
+ file2 = base / 'subdir' / 'file2.txt'
350
+
351
+ # All resulting FCPath objects inherit the FileCache and time_sensitive setting
352
+ content1 = file1.read_text() # Uses same cache and time_sensitive=True
353
+ content2 = file2.read_text() # Uses same cache and time_sensitive=True
354
+
355
+
356
+ Example 13: Creating FCPath from Different Types
357
+ -------------------------------------------------
358
+
359
+ :class:`FCPath` can be created from strings, `Path` objects, or other `FCPath` objects.
360
+ When created from an existing :class:`FCPath`, it inherits all settings.
361
+
362
+ .. code-block:: python
363
+
364
+ from filecache import FileCache, FCPath
365
+ from pathlib import Path
366
+
367
+ with FileCache(cache_name='myproject', time_sensitive=True) as fc:
368
+ # Create from string
369
+ path1 = FCPath('gs://my-bucket/data/file.txt', filecache=fc)
370
+
371
+ # Create from Path object
372
+ local_path = Path('/local/path/file.txt')
373
+ path2 = FCPath(local_path, filecache=fc)
374
+
375
+ # Create from another FCPath (inherits all settings)
376
+ path3 = FCPath(path1) # Inherits filecache, time_sensitive, etc. from path1
377
+ path4 = path1 / 'subdir' / 'file2.txt' # Also inherits via / operator
378
+
379
+ # All can be used the same way
380
+ content1 = path1.read_text()
381
+ content2 = path2.read_text()
382
+ content3 = path3.read_text()
383
+
384
+
385
+ Example 14: Inheriting FileCache Settings
386
+ ------------------------------------------
387
+
388
+ When creating new :class:`FCPath` objects from existing ones, all FileCache-related
389
+ settings are automatically inherited, making it easy to work with paths that share
390
+ the same configuration.
391
+
392
+ .. code-block:: python
393
+
394
+ from filecache import FileCache, FCPath
395
+
396
+ # Create a FileCache with specific settings
397
+ fc = FileCache(cache_name='project', time_sensitive=True, nthreads=4)
398
+
399
+ # Create base FCPath with a different lock timeout
400
+ base = fc.new_path('gs://my-bucket/data', lock_timeout=30)
401
+
402
+ # All child paths inherit: filecache, lock_timeout=30
403
+ file1 = base / 'dir1' / 'file1.txt'
404
+ file2 = base / 'dir2' / 'file2.txt'
405
+ file3 = FCPath(base) / 'dir3' / 'file3.txt'
406
+
407
+ # All operations use the same settings
408
+ paths = [file1, file2, file3]
409
+ contents = [f.read_text() for f in paths] # All use time_sensitive=True, lock_timeout=30
410
+
411
+
412
+ Example 15: Using glob() for Pattern Matching
413
+ ---------------------------------------------
414
+
415
+ :class:`FCPath` supports `glob()` for pattern matching on remote directories, a feature
416
+ not available directly in :class:`FileCache`. This works on both local and remote paths.
417
+
418
+ .. code-block:: python
419
+
420
+ from filecache import FileCache, FCPath
421
+
422
+ with FileCache(cache_name=None) as fc:
423
+ # Create base path to a directory
424
+ base_dir = fc.new_path('gs://my-bucket/data')
425
+
426
+ # Find all .txt files in the directory
427
+ txt_files = list(base_dir.glob('*.txt'))
428
+ for txt_file in txt_files:
429
+ print(f'Found: {txt_file.path}')
430
+ content = txt_file.read_text()
431
+
432
+ # Find files in subdirectories
433
+ all_txt = list(base_dir.glob('**/*.txt')) # Recursive
434
+ for file in all_txt:
435
+ print(f'Recursive match: {file.path}')
436
+
437
+
438
+ Example 16: Using rglob() for Recursive Pattern Matching
439
+ --------------------------------------------------------
440
+
441
+ :meth:`FCPath.rglob()` is a convenience method that automatically adds ``**/`` to the pattern,
442
+ making recursive searches easier.
443
+
444
+ .. code-block:: python
445
+
446
+ from filecache import FileCache, FCPath
447
+
448
+ with FileCache() as fc:
449
+ base = fc.new_path('gs://my-bucket/project')
450
+
451
+ # Find all Python files recursively
452
+ # rglob('*.py') is equivalent to glob('**/*.py')
453
+ python_files = list(base.rglob('*.py'))
454
+
455
+ for py_file in python_files:
456
+ print(f'Python file: {py_file.path}')
457
+ # Process each file
458
+ content = py_file.read_text()
459
+
460
+
461
+ Example 17: Directory Traversal with iterdir()
462
+ ----------------------------------------------
463
+
464
+ :class:`FCPath` provides :meth:`FCPath.iterdir()` for iterating over directory contents, returning
465
+ :class:`FCPath` objects that inherit settings from the parent.
466
+
467
+ .. code-block:: python
468
+
469
+ from filecache import FileCache, FCPath
470
+
471
+ with FileCache(cache_name=None) as fc:
472
+ base_dir = fc.new_path('gs://my-bucket/data')
473
+
474
+ # Iterate over directory contents
475
+ for item in base_dir.iterdir():
476
+ if item.is_dir():
477
+ print(f'Directory: {item.path}')
478
+ # Recursively process subdirectories
479
+ for subitem in item.iterdir():
480
+ print(f' Subitem: {subitem.path}')
481
+ else:
482
+ print(f'File: {item.path}')
483
+ # Read file content
484
+ content = item.read_text()
485
+
486
+
487
+ Example 18: Directory Walking with walk()
488
+ -----------------------------------------
489
+
490
+ The `walk()` method provides a convenient way to traverse directory trees, similar to
491
+ `os.walk()` but returning :class:`FCPath` objects.
492
+
493
+ .. code-block:: python
494
+
495
+ from filecache import FileCache, FCPath
496
+
497
+ with FileCache() as fc:
498
+ root = fc.new_path('gs://my-bucket/project')
499
+
500
+ # Walk the directory tree
501
+ for dirpath, dirnames, filenames in root.walk():
502
+ print(f'Directory: {dirpath.path}')
503
+ print(f' Subdirectories: {dirnames}')
504
+ print(f' Files: {filenames}')
505
+
506
+ # Process files in this directory
507
+ for filename in filenames:
508
+ file_path = dirpath / filename
509
+ content = file_path.read_text()
510
+
511
+
512
+ Example 19: Combining Path Operations
513
+ -------------------------------------
514
+
515
+ :class:`FCPath` supports chaining of path operations, making complex path manipulations
516
+ easy and readable.
517
+
518
+ .. code-block:: python
519
+
520
+ from filecache import FileCache, FCPath
521
+
522
+ with FileCache(cache_name='analysis') as fc:
523
+ # Start with a base URL
524
+ base = fc.new_path('gs://data-bucket')
525
+
526
+ # Build complex paths using chaining
527
+ data_file = base / 'year' / '2024' / 'month' / '01' / 'data.csv'
528
+
529
+ # Read both files
530
+ data = data_file.read_text()
531
+ config = (base / 'config' / 'settings.json').read_text()
532
+
533
+ # Create output path in same structure
534
+ output_dir = base / 'output' / '2024' / '01'
535
+
536
+ # Write to output
537
+ (output_dir / 'results.txt').write_text('Analysis results')
538
+
539
+
540
+ Example 20: Working with Metadata via iterdir_metadata()
541
+ --------------------------------------------------------
542
+
543
+ The `iterdir_metadata()` method provides directory contents along with metadata
544
+ (is_dir, mtime, size), useful for filtering or sorting files.
545
+
546
+ .. code-block:: python
547
+
548
+ from filecache import FileCache, FCPath
549
+
550
+ with FileCache() as fc:
551
+ base_dir = fc.new_path('gs://my-bucket/data')
552
+
553
+ # Get directory contents with metadata
554
+ files_by_size = []
555
+ for item, metadata in base_dir.iterdir_metadata():
556
+ if metadata and not metadata['is_dir']:
557
+ files_by_size.append((item, metadata['size']))
558
+
559
+ # Sort by size (largest first)
560
+ files_by_size.sort(key=lambda x: x[1] or 0, reverse=True)
561
+
562
+ # Process largest files first
563
+ for item, size in files_by_size:
564
+ print(f'Processing {item.path} ({size} bytes)')
565
+ content = item.read_text()
566
+
567
+
568
+ Example 21: Creating FCPath Without Explicit FileCache
569
+ -------------------------------------------------------
570
+
571
+ When an :class:`FCPath` is created without specifying a `filecache`, it uses the default
572
+ global :class:`FileCache` when an operation is performed. This allows for simpler
573
+ syntax when the default cache is sufficient.
574
+
575
+ .. code-block:: python
576
+
577
+ from filecache import FCPath
578
+
579
+ # Create FCPath without explicit FileCache
580
+ # Will use default global cache when operations are performed
581
+ base = FCPath('gs://my-bucket/data')
582
+
583
+ # Operations automatically use the default FileCache
584
+ file1 = base / 'file1.txt'
585
+ file2 = base / 'file2.txt'
586
+
587
+ # Read files (uses default global cache)
588
+ content1 = file1.read_text()
589
+ content2 = file2.read_text()
590
+
591
+ # All paths share the same default cache
592
+ assert file1.filecache == file2.filecache
@@ -14,6 +14,8 @@ Welcome to ``filecache``'s documentation!
14
14
  :maxdepth: 2
15
15
  :caption: Contents:
16
16
 
17
+ introduction
18
+ examples
17
19
  module
18
20
 
19
21