xtr-storage 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xtr_storage/.agents/skills/xtr-storage/SKILL.md +207 -0
- xtr_storage/__init__.py +158 -0
- xtr_storage/adapter/__init__.py +29 -0
- xtr_storage/adapter/_fsspec_bridge.py +212 -0
- xtr_storage/adapter/_object_store.py +108 -0
- xtr_storage/adapter/fsspec_adapter.py +528 -0
- xtr_storage/adapter/gcs_adapter.py +325 -0
- xtr_storage/adapter/generic_fsspec_adapter.py +60 -0
- xtr_storage/adapter/in_memory_adapter.py +174 -0
- xtr_storage/adapter/local_adapter.py +259 -0
- xtr_storage/adapter/path_prefixed_adapter.py +354 -0
- xtr_storage/adapter/portable_visibility_converter.py +98 -0
- xtr_storage/adapter/read_only_adapter.py +231 -0
- xtr_storage/adapter/s3_adapter.py +441 -0
- xtr_storage/adapter/storage_adapter_interface.py +217 -0
- xtr_storage/adapter/visibility_converter_interface.py +88 -0
- xtr_storage/bundle/__init__.py +28 -0
- xtr_storage/bundle/adapter_configs.py +227 -0
- xtr_storage/bundle/storage_bundle.py +328 -0
- xtr_storage/bundle/storage_config.py +103 -0
- xtr_storage/bundle/storage_definition.py +85 -0
- xtr_storage/checksum_provider_interface.py +42 -0
- xtr_storage/config.py +260 -0
- xtr_storage/directory_attributes.py +80 -0
- xtr_storage/directory_listing.py +149 -0
- xtr_storage/exception/__init__.py +75 -0
- xtr_storage/exception/checksum_algorithm_not_supported_error.py +29 -0
- xtr_storage/exception/corrupted_path_detected_error.py +27 -0
- xtr_storage/exception/feature_not_supported_error.py +39 -0
- xtr_storage/exception/invalid_argument_error.py +29 -0
- xtr_storage/exception/invalid_stream_error.py +30 -0
- xtr_storage/exception/invalid_visibility_error.py +30 -0
- xtr_storage/exception/missing_backend_error.py +34 -0
- xtr_storage/exception/path_traversal_detected_error.py +27 -0
- xtr_storage/exception/storage_error.py +19 -0
- xtr_storage/exception/storage_operation_failed_error.py +32 -0
- xtr_storage/exception/symbolic_link_encountered_error.py +26 -0
- xtr_storage/exception/unable_to_check_directory_existence_error.py +24 -0
- xtr_storage/exception/unable_to_check_existence_error.py +32 -0
- xtr_storage/exception/unable_to_check_file_existence_error.py +24 -0
- xtr_storage/exception/unable_to_copy_file_error.py +39 -0
- xtr_storage/exception/unable_to_create_directory_error.py +32 -0
- xtr_storage/exception/unable_to_delete_directory_error.py +36 -0
- xtr_storage/exception/unable_to_delete_file_error.py +35 -0
- xtr_storage/exception/unable_to_generate_public_url_error.py +30 -0
- xtr_storage/exception/unable_to_generate_temporary_url_error.py +30 -0
- xtr_storage/exception/unable_to_list_contents_error.py +37 -0
- xtr_storage/exception/unable_to_move_file_error.py +39 -0
- xtr_storage/exception/unable_to_provide_checksum_error.py +30 -0
- xtr_storage/exception/unable_to_read_file_error.py +36 -0
- xtr_storage/exception/unable_to_resolve_mount_error.py +30 -0
- xtr_storage/exception/unable_to_retrieve_metadata_error.py +61 -0
- xtr_storage/exception/unable_to_set_visibility_error.py +36 -0
- xtr_storage/exception/unable_to_write_file_error.py +35 -0
- xtr_storage/feature.py +23 -0
- xtr_storage/file_attributes.py +88 -0
- xtr_storage/identical_path_policy.py +27 -0
- xtr_storage/link_handling.py +25 -0
- xtr_storage/mime_type/__init__.py +8 -0
- xtr_storage/mime_type/extension_mime_type_detector.py +53 -0
- xtr_storage/mime_type/mime_type_detector_interface.py +32 -0
- xtr_storage/mount_manager.py +461 -0
- xtr_storage/operation.py +31 -0
- xtr_storage/path/__init__.py +9 -0
- xtr_storage/path/path_normalizer_interface.py +33 -0
- xtr_storage/path/path_prefixer.py +66 -0
- xtr_storage/path/whitespace_path_normalizer.py +68 -0
- xtr_storage/py.typed +0 -0
- xtr_storage/recipe/__init__.py +15 -0
- xtr_storage/recipe/manifest.toml +21 -0
- xtr_storage/storage.py +480 -0
- xtr_storage/storage_attributes.py +18 -0
- xtr_storage/storage_operator_interface.py +91 -0
- xtr_storage/storage_reader_interface.py +189 -0
- xtr_storage/storage_writer_interface.py +172 -0
- xtr_storage/url_generation/__init__.py +17 -0
- xtr_storage/url_generation/chained_public_url_generator.py +66 -0
- xtr_storage/url_generation/prefix_public_url_generator.py +54 -0
- xtr_storage/url_generation/public_url_generator_interface.py +41 -0
- xtr_storage/url_generation/sharded_prefix_public_url_generator.py +68 -0
- xtr_storage/url_generation/temporary_url_generator_interface.py +45 -0
- xtr_storage/visibility.py +48 -0
- xtr_storage-3.0.0.dist-info/METADATA +475 -0
- xtr_storage-3.0.0.dist-info/RECORD +87 -0
- xtr_storage-3.0.0.dist-info/WHEEL +4 -0
- xtr_storage-3.0.0.dist-info/entry_points.txt +6 -0
- xtr_storage-3.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: xtr-storage
|
|
3
|
+
description: How to read, write, list, copy and move files with xtr-storage behind one async interface, on local disk, in memory, in Amazon S3, in Google Cloud Storage or on any fsspec backend. Use when code stores uploads, reports, exports or other files, must work against a disk in development and a bucket in production, needs public or signed urls, file visibility, checksums, streaming of large files, several named storages or a read-only or path-prefixed view; also when adding StorageBundle and named storages to an application on xtr-dependency-injection, or testing code that writes files.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# xtr-storage
|
|
7
|
+
|
|
8
|
+
Where the bytes live is a constructor argument. A `Storage` wraps one adapter and turns paths into
|
|
9
|
+
reads, writes, listings, copies and moves; swapping the adapter moves the files without touching
|
|
10
|
+
the code that stores them. Every call that reaches the backend is awaited.
|
|
11
|
+
|
|
12
|
+
## Quick reference
|
|
13
|
+
|
|
14
|
+
- Depend on `StorageOperatorInterface` (or `StorageReaderInterface` / `StorageWriterInterface`),
|
|
15
|
+
never on an adapter or on `Storage` itself.
|
|
16
|
+
- Build one: `Storage(LocalAdapter("var/storage"))`, `Storage(InMemoryAdapter())`,
|
|
17
|
+
`Storage(S3Adapter("bucket", prefix="avatars"))`.
|
|
18
|
+
- Write and read: `await storage.write("a/b.txt", b"...")`, `await storage.read("a/b.txt")`.
|
|
19
|
+
- Large files: `storage.read_stream(path)` and `await storage.write_stream(path, chunks)`.
|
|
20
|
+
- Per-call options lay over the storage's own: `await storage.write(path, data, {"visibility": "public"})`.
|
|
21
|
+
- Close what you build: `async with Storage(adapter) as storage:`, or `await storage.close()`.
|
|
22
|
+
- Several storages behind one: `MountManager({"uploads": ..., "reports": ...})`, paths `uploads://a.png`.
|
|
23
|
+
- In an application: activate `StorageBundle`, inject `StorageOperatorInterface`.
|
|
24
|
+
|
|
25
|
+
## Read and write
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
from xtr_storage import InMemoryAdapter, Storage, Visibility
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
async with Storage(InMemoryAdapter()) as storage:
|
|
32
|
+
await storage.write("notes/todo.txt", b"buy milk", {"visibility": Visibility.PUBLIC})
|
|
33
|
+
data = await storage.read("notes/todo.txt") # b"buy milk"
|
|
34
|
+
await storage.file_exists("notes/todo.txt") # True
|
|
35
|
+
await storage.mime_type("notes/todo.txt") # "text/plain"
|
|
36
|
+
await storage.file_size("notes/todo.txt") # 8
|
|
37
|
+
await storage.copy("notes/todo.txt", "notes/copy.txt")
|
|
38
|
+
await storage.move("notes/copy.txt", "archive/copy.txt")
|
|
39
|
+
names = sorted([entry.path async for entry in storage.list_contents("notes", deep=True)])
|
|
40
|
+
await storage.delete("notes/todo.txt")
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
| Call | Does |
|
|
44
|
+
| --- | --- |
|
|
45
|
+
| `read(path)`, `read_stream(path)` | The whole file, or its chunks as an async iterator |
|
|
46
|
+
| `write(path, data, config=None)`, `write_stream(path, stream, config=None)` | Creates or replaces; the stream may be async, a plain iterable, or a binary file object |
|
|
47
|
+
| `file_exists`, `directory_exists`, `has` | Existence; `has` is either |
|
|
48
|
+
| `list_contents(path="", deep=False)` | A lazy listing of `FileAttributes` / `DirectoryAttributes`; nothing is read until iterated |
|
|
49
|
+
| `file_size`, `last_modified`, `mime_type`, `visibility` | Metadata |
|
|
50
|
+
| `copy`, `move`, `delete`, `delete_directory`, `create_directory` | Changes |
|
|
51
|
+
| `set_visibility(path, visibility)` | `Visibility.PUBLIC` or `Visibility.PRIVATE`; each member is its string |
|
|
52
|
+
| `checksum(path, config=None)` | The backend's digest when it keeps one, else hashed here |
|
|
53
|
+
| `public_url(path)`, `temporary_url(path, expires_at)` | Lasting and expiring addresses |
|
|
54
|
+
|
|
55
|
+
Paths are normalized before an adapter sees them: backslashes become `/`, `.` and empty segments
|
|
56
|
+
drop. A control or zero-width character raises `CorruptedPathDetectedError`; `..` climbing past
|
|
57
|
+
the root raises `PathTraversalDetectedError` when `allow_relative_path_traversal` is `False`.
|
|
58
|
+
|
|
59
|
+
## Pick an adapter
|
|
60
|
+
|
|
61
|
+
| Adapter | Where | Extra | Visibility | Urls |
|
|
62
|
+
| --- | --- | --- | --- | --- |
|
|
63
|
+
| `LocalAdapter(root)` | A directory on this machine | — | permission bits | only through a configured generator |
|
|
64
|
+
| `InMemoryAdapter()` | A dictionary in this process | — | remembered per file | only through a configured generator |
|
|
65
|
+
| `S3Adapter(bucket, prefix=..., region=...)` | An S3 bucket | `s3` | access list | public and signed |
|
|
66
|
+
| `GcsAdapter(bucket, prefix=...)` | A Google Cloud Storage bucket | `gcs` | refused | public and signed |
|
|
67
|
+
| `GenericFsspecAdapter(filesystem, root=...)` | Any fsspec filesystem | — | refused | only through a configured generator |
|
|
68
|
+
| `ReadOnlyAdapter(adapter)` | Any of the above, writes refused | — | as wrapped | as wrapped |
|
|
69
|
+
| `PathPrefixedAdapter(adapter, prefix)` | Any of the above, under a prefix | — | as wrapped | as wrapped |
|
|
70
|
+
|
|
71
|
+
An adapter whose extra is missing raises `MissingBackendError` naming it. A capability a backend
|
|
72
|
+
lacks raises `FeatureNotSupportedError` instead of guessing. No adapter opens anything when built;
|
|
73
|
+
`LocalAdapter` creates its root on the first write.
|
|
74
|
+
|
|
75
|
+
## Several storages
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from xtr_storage import InMemoryAdapter, MountManager, Storage
|
|
79
|
+
|
|
80
|
+
async with MountManager(
|
|
81
|
+
{"uploads": Storage(InMemoryAdapter()), "reports": Storage(InMemoryAdapter())}
|
|
82
|
+
) as files:
|
|
83
|
+
await files.write("uploads://a.txt", b"hi")
|
|
84
|
+
await files.copy("uploads://a.txt", "reports://a.txt") # streamed across storages
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
There is no default mount: an unknown name, a missing `://` or an empty name raises
|
|
88
|
+
`UnableToResolveMountError`.
|
|
89
|
+
|
|
90
|
+
## Testing
|
|
91
|
+
|
|
92
|
+
The package ships no test helpers; `InMemoryAdapter` is the one a test needs.
|
|
93
|
+
|
|
94
|
+
- Hand the code under test `Storage(InMemoryAdapter())`. Two in-memory adapters share nothing, so
|
|
95
|
+
a test that writes has nothing to undo.
|
|
96
|
+
- Assert what was stored by reading it back through the same storage.
|
|
97
|
+
- In an application test, replace the service:
|
|
98
|
+
`boot_for_test(kernel, overrides={StorageOperatorInterface: Storage(InMemoryAdapter())})`.
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from xtr_storage import InMemoryAdapter, Storage
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
async def test_the_report_is_stored() -> None:
|
|
105
|
+
storage = Storage(InMemoryAdapter())
|
|
106
|
+
|
|
107
|
+
await write_report(storage)
|
|
108
|
+
|
|
109
|
+
assert await storage.read("reports/2026.txt") == b"hello"
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Use in an application
|
|
113
|
+
|
|
114
|
+
`uv run xtr-recipes recipes:sync` applies the recipe shipped with this package: it lists
|
|
115
|
+
`StorageBundle` and ignores `/var/storage/`. That is the steps below a recipe can do; the others it
|
|
116
|
+
prints for you to make.
|
|
117
|
+
|
|
118
|
+
1. **Install** — `uv add "xtr-storage[di]"`; add `s3` or `gcs` for the object stores you reach.
|
|
119
|
+
2. **Activate** — `StorageBundle: {"all": True}` in `BUNDLES` in `<app>/bundles.py`, imported from
|
|
120
|
+
`xtr_storage.bundle`.
|
|
121
|
+
3. **Brings along** — nothing; the bundle requires no other.
|
|
122
|
+
4. **Configure** — optional; with no configuration there is one `default` storage on local files
|
|
123
|
+
under `var/storage`, and nothing is opened until it is first used:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
# <app>/config/storage.py
|
|
127
|
+
from xtr_dependency_injection import configure, env
|
|
128
|
+
from xtr_storage.bundle import LocalAdapterConfig, S3AdapterConfig, StorageConfig, StorageDefinition
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@configure
|
|
132
|
+
def storage() -> StorageConfig:
|
|
133
|
+
return StorageConfig(
|
|
134
|
+
storages={
|
|
135
|
+
"default": LocalAdapterConfig(),
|
|
136
|
+
"uploads": StorageDefinition(
|
|
137
|
+
adapter=S3AdapterConfig(bucket=env("UPLOADS_BUCKET"), region=env("AWS_REGION")),
|
|
138
|
+
prefix="avatars",
|
|
139
|
+
visibility="public",
|
|
140
|
+
),
|
|
141
|
+
},
|
|
142
|
+
)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
An adapter entry is `LocalAdapterConfig`, `MemoryAdapterConfig`, `S3AdapterConfig`,
|
|
146
|
+
`GcsAdapterConfig`, `FsspecAdapterConfig`, or a `Reference` to an adapter the container
|
|
147
|
+
provides. A `StorageDefinition` adds `visibility`, `directory_visibility`,
|
|
148
|
+
`retain_visibility`, `public_url`, `read_only` and `prefix`.
|
|
149
|
+
5. **Environment** — nothing required; credentials and directories given as `env(...)` must be set
|
|
150
|
+
when the application boots, because booting checks every storage.
|
|
151
|
+
6. **Ignore** — `/var/storage/` in `.gitignore`.
|
|
152
|
+
7. **Use** — inject `StorageOperatorInterface` for the `default` storage, or qualify by name. A
|
|
153
|
+
`MountManager` over every storage is provided too.
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
from typing import Annotated
|
|
157
|
+
|
|
158
|
+
from xtr_dependency_injection import Target, as_service
|
|
159
|
+
from xtr_storage import StorageOperatorInterface
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@as_service
|
|
163
|
+
class Avatars:
|
|
164
|
+
def __init__(
|
|
165
|
+
self,
|
|
166
|
+
storage: StorageOperatorInterface, # the default storage
|
|
167
|
+
uploads: Annotated[StorageOperatorInterface, Target("uploads")],
|
|
168
|
+
) -> None: ...
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
8. **Check** — `debug:bundles` shows `storage` as `listed` and `active`.
|
|
172
|
+
9. **Remove** — drop the `BUNDLES` entry, delete `<app>/config/storage.py`, then
|
|
173
|
+
`uv remove xtr-storage`.
|
|
174
|
+
|
|
175
|
+
## Errors
|
|
176
|
+
|
|
177
|
+
Everything derives from `StorageError`; a failure of an operation the backend refused derives from
|
|
178
|
+
`StorageOperationFailedError` and names the operation. No message repeats a credential.
|
|
179
|
+
|
|
180
|
+
| Error | Raised when |
|
|
181
|
+
| --- | --- |
|
|
182
|
+
| `UnableToReadFileError`, `UnableToWriteFileError` | A read or write fails, or the file is missing |
|
|
183
|
+
| `UnableToDeleteFileError`, `UnableToDeleteDirectoryError`, `UnableToCreateDirectoryError` | A delete or a directory creation fails |
|
|
184
|
+
| `UnableToCopyFileError`, `UnableToMoveFileError` | A copy or move fails, or the paths are the same under a `fail` policy |
|
|
185
|
+
| `UnableToRetrieveMetadataError`, `UnableToSetVisibilityError` | Metadata cannot be read, or a visibility set |
|
|
186
|
+
| `UnableToListContentsError` | A listing fails while it is iterated |
|
|
187
|
+
| `UnableToProvideChecksumError`, `ChecksumAlgorithmNotSupportedError` | No checksum can be produced |
|
|
188
|
+
| `UnableToGeneratePublicUrlError`, `UnableToGenerateTemporaryUrlError` | No url can be produced |
|
|
189
|
+
| `UnableToResolveMountError` | A mount location names no storage, or is malformed |
|
|
190
|
+
| `PathTraversalDetectedError`, `CorruptedPathDetectedError` | A path climbs past the root, or carries a control character |
|
|
191
|
+
| `FeatureNotSupportedError` | The backend lacks the capability (also a `NotImplementedError`) |
|
|
192
|
+
| `MissingBackendError` | The adapter's extra is not installed (also an `ImportError`) |
|
|
193
|
+
| `InvalidVisibilityError`, `InvalidArgumentError` | A bad visibility string or option (also a `ValueError`) |
|
|
194
|
+
| `InvalidStreamError` | A write stream of a type the storage cannot read (also a `TypeError`) |
|
|
195
|
+
|
|
196
|
+
## Do not
|
|
197
|
+
|
|
198
|
+
- Do not depend on an adapter or on `Storage` in application code; depend on
|
|
199
|
+
`StorageOperatorInterface` and let the container pick the backend.
|
|
200
|
+
- Do not build paths with `os.path` or `pathlib`; pass `/`-separated strings, as every backend
|
|
201
|
+
sees them.
|
|
202
|
+
- Do not read a large file whole with `read`; stream it with `read_stream` and `write_stream`.
|
|
203
|
+
- Do not expect a backend to fake a capability: GCS and generic fsspec backends refuse visibility,
|
|
204
|
+
and a local or in-memory storage gives urls only through a configured generator.
|
|
205
|
+
- Do not leave a storage you built open; use `async with` or `close()`. The container closes the
|
|
206
|
+
ones it built.
|
|
207
|
+
- Do not put credentials in code; read them with `env(...)` in `config/storage.py`.
|
xtr_storage/__init__.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Files on local disk, in memory or in object stores, behind one async interface.
|
|
2
|
+
|
|
3
|
+
A ``Storage`` wraps one adapter and turns paths into reads, writes, listings,
|
|
4
|
+
copies and moves. The adapter decides where the bytes live — a directory on this
|
|
5
|
+
machine, a dictionary in this process, a bucket on an object store — so code that
|
|
6
|
+
stores files is written once and moved between them by its constructor.
|
|
7
|
+
|
|
8
|
+
Every call that touches a backend is awaited.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
14
|
+
|
|
15
|
+
from xtr_storage.adapter import (
|
|
16
|
+
FsspecAdapter,
|
|
17
|
+
GcsAdapter,
|
|
18
|
+
GenericFsspecAdapter,
|
|
19
|
+
InMemoryAdapter,
|
|
20
|
+
LocalAdapter,
|
|
21
|
+
PathPrefixedAdapter,
|
|
22
|
+
PortableVisibilityConverter,
|
|
23
|
+
ReadOnlyAdapter,
|
|
24
|
+
S3Adapter,
|
|
25
|
+
StorageAdapterInterface,
|
|
26
|
+
VisibilityConverterInterface,
|
|
27
|
+
)
|
|
28
|
+
from xtr_storage.checksum_provider_interface import ChecksumProviderInterface
|
|
29
|
+
from xtr_storage.config import Config
|
|
30
|
+
from xtr_storage.directory_attributes import DirectoryAttributes
|
|
31
|
+
from xtr_storage.directory_listing import DirectoryListing
|
|
32
|
+
from xtr_storage.exception import (
|
|
33
|
+
ChecksumAlgorithmNotSupportedError,
|
|
34
|
+
CorruptedPathDetectedError,
|
|
35
|
+
FeatureNotSupportedError,
|
|
36
|
+
InvalidArgumentError,
|
|
37
|
+
InvalidStreamError,
|
|
38
|
+
InvalidVisibilityError,
|
|
39
|
+
MissingBackendError,
|
|
40
|
+
PathTraversalDetectedError,
|
|
41
|
+
StorageError,
|
|
42
|
+
StorageOperationFailedError,
|
|
43
|
+
SymbolicLinkEncounteredError,
|
|
44
|
+
UnableToCheckDirectoryExistenceError,
|
|
45
|
+
UnableToCheckExistenceError,
|
|
46
|
+
UnableToCheckFileExistenceError,
|
|
47
|
+
UnableToCopyFileError,
|
|
48
|
+
UnableToCreateDirectoryError,
|
|
49
|
+
UnableToDeleteDirectoryError,
|
|
50
|
+
UnableToDeleteFileError,
|
|
51
|
+
UnableToGeneratePublicUrlError,
|
|
52
|
+
UnableToGenerateTemporaryUrlError,
|
|
53
|
+
UnableToListContentsError,
|
|
54
|
+
UnableToMoveFileError,
|
|
55
|
+
UnableToProvideChecksumError,
|
|
56
|
+
UnableToReadFileError,
|
|
57
|
+
UnableToResolveMountError,
|
|
58
|
+
UnableToRetrieveMetadataError,
|
|
59
|
+
UnableToSetVisibilityError,
|
|
60
|
+
UnableToWriteFileError,
|
|
61
|
+
)
|
|
62
|
+
from xtr_storage.feature import Feature
|
|
63
|
+
from xtr_storage.file_attributes import FileAttributes
|
|
64
|
+
from xtr_storage.identical_path_policy import IdenticalPathPolicy
|
|
65
|
+
from xtr_storage.link_handling import LinkHandling
|
|
66
|
+
from xtr_storage.mime_type import ExtensionMimeTypeDetector, MimeTypeDetectorInterface
|
|
67
|
+
from xtr_storage.mount_manager import MountManager
|
|
68
|
+
from xtr_storage.operation import Operation
|
|
69
|
+
from xtr_storage.path import PathNormalizerInterface, PathPrefixer, WhitespacePathNormalizer
|
|
70
|
+
from xtr_storage.storage import Storage
|
|
71
|
+
from xtr_storage.storage_attributes import StorageAttributes
|
|
72
|
+
from xtr_storage.storage_operator_interface import StorageOperatorInterface
|
|
73
|
+
from xtr_storage.storage_reader_interface import StorageReaderInterface
|
|
74
|
+
from xtr_storage.storage_writer_interface import StorageWriterInterface
|
|
75
|
+
from xtr_storage.url_generation import (
|
|
76
|
+
ChainedPublicUrlGenerator,
|
|
77
|
+
PrefixPublicUrlGenerator,
|
|
78
|
+
PublicUrlGeneratorInterface,
|
|
79
|
+
ShardedPrefixPublicUrlGenerator,
|
|
80
|
+
TemporaryUrlGeneratorInterface,
|
|
81
|
+
)
|
|
82
|
+
from xtr_storage.visibility import Visibility
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
__version__ = version("xtr-storage")
|
|
86
|
+
except PackageNotFoundError: # pragma: no cover
|
|
87
|
+
# Running from a source tree or a vendored copy, with no installed
|
|
88
|
+
# metadata to read. Having no version is better than refusing to import.
|
|
89
|
+
__version__ = "0+unknown"
|
|
90
|
+
|
|
91
|
+
__all__ = [
|
|
92
|
+
"ChainedPublicUrlGenerator",
|
|
93
|
+
"ChecksumAlgorithmNotSupportedError",
|
|
94
|
+
"ChecksumProviderInterface",
|
|
95
|
+
"Config",
|
|
96
|
+
"CorruptedPathDetectedError",
|
|
97
|
+
"DirectoryAttributes",
|
|
98
|
+
"DirectoryListing",
|
|
99
|
+
"ExtensionMimeTypeDetector",
|
|
100
|
+
"Feature",
|
|
101
|
+
"FeatureNotSupportedError",
|
|
102
|
+
"FileAttributes",
|
|
103
|
+
"FsspecAdapter",
|
|
104
|
+
"GcsAdapter",
|
|
105
|
+
"GenericFsspecAdapter",
|
|
106
|
+
"IdenticalPathPolicy",
|
|
107
|
+
"InMemoryAdapter",
|
|
108
|
+
"InvalidArgumentError",
|
|
109
|
+
"InvalidStreamError",
|
|
110
|
+
"InvalidVisibilityError",
|
|
111
|
+
"LinkHandling",
|
|
112
|
+
"LocalAdapter",
|
|
113
|
+
"MimeTypeDetectorInterface",
|
|
114
|
+
"MissingBackendError",
|
|
115
|
+
"MountManager",
|
|
116
|
+
"Operation",
|
|
117
|
+
"PathNormalizerInterface",
|
|
118
|
+
"PathPrefixedAdapter",
|
|
119
|
+
"PathPrefixer",
|
|
120
|
+
"PathTraversalDetectedError",
|
|
121
|
+
"PortableVisibilityConverter",
|
|
122
|
+
"PrefixPublicUrlGenerator",
|
|
123
|
+
"PublicUrlGeneratorInterface",
|
|
124
|
+
"ReadOnlyAdapter",
|
|
125
|
+
"S3Adapter",
|
|
126
|
+
"ShardedPrefixPublicUrlGenerator",
|
|
127
|
+
"Storage",
|
|
128
|
+
"StorageAdapterInterface",
|
|
129
|
+
"StorageAttributes",
|
|
130
|
+
"StorageError",
|
|
131
|
+
"StorageOperationFailedError",
|
|
132
|
+
"StorageOperatorInterface",
|
|
133
|
+
"StorageReaderInterface",
|
|
134
|
+
"StorageWriterInterface",
|
|
135
|
+
"SymbolicLinkEncounteredError",
|
|
136
|
+
"TemporaryUrlGeneratorInterface",
|
|
137
|
+
"UnableToCheckDirectoryExistenceError",
|
|
138
|
+
"UnableToCheckExistenceError",
|
|
139
|
+
"UnableToCheckFileExistenceError",
|
|
140
|
+
"UnableToCopyFileError",
|
|
141
|
+
"UnableToCreateDirectoryError",
|
|
142
|
+
"UnableToDeleteDirectoryError",
|
|
143
|
+
"UnableToDeleteFileError",
|
|
144
|
+
"UnableToGeneratePublicUrlError",
|
|
145
|
+
"UnableToGenerateTemporaryUrlError",
|
|
146
|
+
"UnableToListContentsError",
|
|
147
|
+
"UnableToMoveFileError",
|
|
148
|
+
"UnableToProvideChecksumError",
|
|
149
|
+
"UnableToReadFileError",
|
|
150
|
+
"UnableToResolveMountError",
|
|
151
|
+
"UnableToRetrieveMetadataError",
|
|
152
|
+
"UnableToSetVisibilityError",
|
|
153
|
+
"UnableToWriteFileError",
|
|
154
|
+
"Visibility",
|
|
155
|
+
"VisibilityConverterInterface",
|
|
156
|
+
"WhitespacePathNormalizer",
|
|
157
|
+
"__version__",
|
|
158
|
+
]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Where the bytes actually live: one adapter per kind of backend."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .fsspec_adapter import FsspecAdapter
|
|
6
|
+
from .gcs_adapter import GcsAdapter
|
|
7
|
+
from .generic_fsspec_adapter import GenericFsspecAdapter
|
|
8
|
+
from .in_memory_adapter import InMemoryAdapter
|
|
9
|
+
from .local_adapter import LocalAdapter
|
|
10
|
+
from .path_prefixed_adapter import PathPrefixedAdapter
|
|
11
|
+
from .portable_visibility_converter import PortableVisibilityConverter
|
|
12
|
+
from .read_only_adapter import ReadOnlyAdapter
|
|
13
|
+
from .s3_adapter import S3Adapter
|
|
14
|
+
from .storage_adapter_interface import StorageAdapterInterface
|
|
15
|
+
from .visibility_converter_interface import VisibilityConverterInterface
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"FsspecAdapter",
|
|
19
|
+
"GcsAdapter",
|
|
20
|
+
"GenericFsspecAdapter",
|
|
21
|
+
"InMemoryAdapter",
|
|
22
|
+
"LocalAdapter",
|
|
23
|
+
"PathPrefixedAdapter",
|
|
24
|
+
"PortableVisibilityConverter",
|
|
25
|
+
"ReadOnlyAdapter",
|
|
26
|
+
"S3Adapter",
|
|
27
|
+
"StorageAdapterInterface",
|
|
28
|
+
"VisibilityConverterInterface",
|
|
29
|
+
]
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# pyright: reportMissingTypeStubs=false, reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportAttributeAccessIssue=false, reportAny=false, reportExplicitAny=false
|
|
2
|
+
# fsspec ships no type information, so every attribute reached on the wrapped
|
|
3
|
+
# filesystem is untyped to the checker; the directives above are confined to
|
|
4
|
+
# this one module, and every value is copied into a typed shape at the boundary.
|
|
5
|
+
"""The one place a filesystem is spoken to, turned into typed coroutines.
|
|
6
|
+
|
|
7
|
+
Every backend this package builds on is an fsspec filesystem, and fsspec speaks
|
|
8
|
+
two dialects: a synchronous one whose blocking calls must be pushed off the
|
|
9
|
+
event loop, and an asynchronous one whose real work hides behind ``_``-prefixed
|
|
10
|
+
coroutines that need a session opened first. This bridge is the seam that hides
|
|
11
|
+
that split: an adapter awaits :class:`FsspecBridge` methods and never learns
|
|
12
|
+
which dialect answered, and the untyped surface of fsspec is fenced to this file
|
|
13
|
+
so the rest of the package stays fully typed.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
from typing import TYPE_CHECKING, Any, final
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from collections.abc import Awaitable, Callable
|
|
23
|
+
|
|
24
|
+
from fsspec import AbstractFileSystem
|
|
25
|
+
|
|
26
|
+
__all__ = ["FsspecBridge"]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@final
|
|
30
|
+
class FsspecBridge:
|
|
31
|
+
"""A typed async front to one fsspec filesystem, sync or asynchronous.
|
|
32
|
+
|
|
33
|
+
The bridge owns nothing the filesystem does not already own except the
|
|
34
|
+
session it opens on demand. It creates no filesystem — an adapter hands it
|
|
35
|
+
one — and it does no I/O until a method is awaited, so constructing it is
|
|
36
|
+
free. On an asynchronous filesystem the first awaited method opens the
|
|
37
|
+
session once, under a lock, and ``close`` hands that session back to a
|
|
38
|
+
per-backend closer; on a synchronous one every call is pushed to a worker
|
|
39
|
+
thread and there is nothing to close.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
__slots__ = (
|
|
43
|
+
"_closed",
|
|
44
|
+
"_closer",
|
|
45
|
+
"_fs",
|
|
46
|
+
"_session_created",
|
|
47
|
+
"_session_lock",
|
|
48
|
+
"_session_ready",
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
filesystem: AbstractFileSystem,
|
|
54
|
+
*,
|
|
55
|
+
closer: Callable[[AbstractFileSystem], Awaitable[None]] | None = None,
|
|
56
|
+
) -> None:
|
|
57
|
+
"""Wrap a filesystem, remembering how to close any session it opens.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
filesystem: The fsspec filesystem to speak to; kept as given, its
|
|
61
|
+
dialect read from ``async_impl`` and ``asynchronous`` on each
|
|
62
|
+
call so a caller may flip ``asynchronous`` before first use.
|
|
63
|
+
closer: What to run once, on ``close``, when a session was opened —
|
|
64
|
+
the S3 and object-store adapters pass the coroutine that shuts
|
|
65
|
+
their client down. ``None`` means there is nothing to close.
|
|
66
|
+
"""
|
|
67
|
+
self._fs = filesystem
|
|
68
|
+
self._closer = closer
|
|
69
|
+
self._session_lock = asyncio.Lock()
|
|
70
|
+
self._session_ready = False
|
|
71
|
+
self._session_created = False
|
|
72
|
+
self._closed = False
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def _is_async(self) -> bool:
|
|
76
|
+
"""Whether calls must go through the ``_``-prefixed coroutines."""
|
|
77
|
+
return bool(self._fs.async_impl and self._fs.asynchronous) # ty: ignore[unresolved-attribute] -- fsspec ships no type information; asynchronous is set on the async filesystem
|
|
78
|
+
|
|
79
|
+
async def _ensure_session(self) -> None:
|
|
80
|
+
"""Open the filesystem's session once, the first async call to need it.
|
|
81
|
+
|
|
82
|
+
A filesystem without ``set_session`` (a plain asynchronous one) needs no
|
|
83
|
+
session; one with it (s3fs, gcsfs) gets exactly one call however many
|
|
84
|
+
coroutines race here, because the lock and the flag together let only
|
|
85
|
+
the first through.
|
|
86
|
+
"""
|
|
87
|
+
if self._session_ready:
|
|
88
|
+
return
|
|
89
|
+
async with self._session_lock:
|
|
90
|
+
if self._session_ready:
|
|
91
|
+
return
|
|
92
|
+
set_session = getattr(self._fs, "set_session", None)
|
|
93
|
+
if set_session is not None:
|
|
94
|
+
_ = await set_session()
|
|
95
|
+
self._session_created = True
|
|
96
|
+
self._session_ready = True
|
|
97
|
+
|
|
98
|
+
async def _dispatch(self, name: str, *args: object, **kwargs: object) -> Any: # noqa: ANN401
|
|
99
|
+
"""Run one filesystem method the way its dialect requires.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
name: The synchronous method name; the asynchronous dialect is
|
|
103
|
+
reached at ``_`` + ``name``.
|
|
104
|
+
*args: Positional arguments passed through unchanged.
|
|
105
|
+
**kwargs: Keyword arguments passed through unchanged.
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
Whatever the filesystem returned, untyped — fsspec ships no types, so
|
|
109
|
+
this one seam is ``Any`` and every caller copies it into a typed
|
|
110
|
+
shape at the boundary before it escapes the module.
|
|
111
|
+
"""
|
|
112
|
+
if self._is_async:
|
|
113
|
+
await self._ensure_session()
|
|
114
|
+
return await getattr(self._fs, "_" + name)(*args, **kwargs)
|
|
115
|
+
return await asyncio.to_thread(getattr(self._fs, name), *args, **kwargs)
|
|
116
|
+
|
|
117
|
+
async def info(self, path: str) -> dict[str, object]:
|
|
118
|
+
"""Return what the filesystem knows about one path, copied out."""
|
|
119
|
+
result = await self._dispatch("info", path)
|
|
120
|
+
return dict(result)
|
|
121
|
+
|
|
122
|
+
async def ls(self, path: str, *, detail: bool = True) -> list[dict[str, object]]:
|
|
123
|
+
"""List one directory in detail, each entry copied out."""
|
|
124
|
+
result = await self._dispatch("ls", path, detail=detail)
|
|
125
|
+
return [dict(entry) for entry in result]
|
|
126
|
+
|
|
127
|
+
async def find(self, path: str, *, withdirs: bool) -> dict[str, dict[str, object]]:
|
|
128
|
+
"""Walk a tree, returning every path mapped to its detail, copied out."""
|
|
129
|
+
result = await self._dispatch("find", path, withdirs=withdirs, detail=True)
|
|
130
|
+
return {key: dict(value) for key, value in result.items()}
|
|
131
|
+
|
|
132
|
+
async def cat_file(self, path: str, start: int | None = None, end: int | None = None) -> bytes:
|
|
133
|
+
"""Read a file, or the byte range ``[start, end)`` of it."""
|
|
134
|
+
result = await self._dispatch("cat_file", path, start=start, end=end)
|
|
135
|
+
return bytes(result)
|
|
136
|
+
|
|
137
|
+
async def pipe_file(self, path: str, data: bytes, **kwargs: object) -> None:
|
|
138
|
+
"""Write ``data`` to ``path`` in one call."""
|
|
139
|
+
_ = await self._dispatch("pipe_file", path, data, **kwargs)
|
|
140
|
+
|
|
141
|
+
async def put_file(self, local_path: str, remote_path: str, **kwargs: object) -> None:
|
|
142
|
+
"""Upload a local file to ``remote_path``."""
|
|
143
|
+
_ = await self._dispatch("put_file", local_path, remote_path, **kwargs)
|
|
144
|
+
|
|
145
|
+
async def rm_file(self, path: str) -> None:
|
|
146
|
+
"""Remove one file."""
|
|
147
|
+
_ = await self._dispatch("rm_file", path)
|
|
148
|
+
|
|
149
|
+
async def rm(self, path: str, *, recursive: bool = True) -> None:
|
|
150
|
+
"""Remove a path, its whole tree by default."""
|
|
151
|
+
_ = await self._dispatch("rm", path, recursive=recursive)
|
|
152
|
+
|
|
153
|
+
async def mkdir(self, path: str, *, create_parents: bool = True) -> None:
|
|
154
|
+
"""Make a directory, its parents too by default."""
|
|
155
|
+
_ = await self._dispatch("mkdir", path, create_parents=create_parents)
|
|
156
|
+
|
|
157
|
+
async def cp_file(self, src: str, dst: str, **kwargs: object) -> None:
|
|
158
|
+
"""Copy one file within the filesystem."""
|
|
159
|
+
_ = await self._dispatch("cp_file", src, dst, **kwargs)
|
|
160
|
+
|
|
161
|
+
async def mv(self, src: str, dst: str) -> None:
|
|
162
|
+
"""Move one file, falling back to copy-then-delete where ``_mv`` is absent.
|
|
163
|
+
|
|
164
|
+
The synchronous dialect has ``mv``; the asynchronous base does not
|
|
165
|
+
expose ``_mv``, so there the move is a copy followed by a delete —
|
|
166
|
+
exactly what an object store does anyway.
|
|
167
|
+
"""
|
|
168
|
+
if self._is_async:
|
|
169
|
+
await self._ensure_session()
|
|
170
|
+
mv_method = getattr(self._fs, "_mv", None)
|
|
171
|
+
if mv_method is not None:
|
|
172
|
+
_ = await mv_method(src, dst)
|
|
173
|
+
return
|
|
174
|
+
await self.cp_file(src, dst)
|
|
175
|
+
await self.rm_file(src)
|
|
176
|
+
return
|
|
177
|
+
_ = await asyncio.to_thread(self._fs.mv, src, dst)
|
|
178
|
+
|
|
179
|
+
async def sign(self, path: str, expiration: int) -> str:
|
|
180
|
+
"""Return a signed URL for ``path``, valid ``expiration`` seconds.
|
|
181
|
+
|
|
182
|
+
The asynchronous dialect signs through ``_url`` when a backend provides
|
|
183
|
+
it (s3fs, gcsfs); otherwise, and for the synchronous dialect, the
|
|
184
|
+
blocking ``sign`` is pushed to a worker thread.
|
|
185
|
+
"""
|
|
186
|
+
if self._is_async:
|
|
187
|
+
await self._ensure_session()
|
|
188
|
+
url_method = getattr(self._fs, "_url", None)
|
|
189
|
+
if url_method is not None:
|
|
190
|
+
return str(await url_method(path, expires=expiration))
|
|
191
|
+
return str(await asyncio.to_thread(self._fs.sign, path, expiration))
|
|
192
|
+
|
|
193
|
+
async def exists(self, path: str) -> bool:
|
|
194
|
+
"""Whether the path exists."""
|
|
195
|
+
return bool(await self._dispatch("exists", path))
|
|
196
|
+
|
|
197
|
+
async def call(self, name: str, *args: object, **kwargs: object) -> object:
|
|
198
|
+
"""Run a backend-specific method by name, dispatched like the rest.
|
|
199
|
+
|
|
200
|
+
The escape hatch for what only one backend does — an ACL read, a chmod —
|
|
201
|
+
so those adapters need no second seam. The return is untyped; the caller
|
|
202
|
+
parses it.
|
|
203
|
+
"""
|
|
204
|
+
return await self._dispatch(name, *args, **kwargs)
|
|
205
|
+
|
|
206
|
+
async def close(self) -> None:
|
|
207
|
+
"""Close the session, once, if one was opened; safe to call again."""
|
|
208
|
+
if self._closed:
|
|
209
|
+
return
|
|
210
|
+
self._closed = True
|
|
211
|
+
if self._closer is not None and self._session_created:
|
|
212
|
+
await self._closer(self._fs)
|