xtr-storage 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. xtr_storage/.agents/skills/xtr-storage/SKILL.md +207 -0
  2. xtr_storage/__init__.py +158 -0
  3. xtr_storage/adapter/__init__.py +29 -0
  4. xtr_storage/adapter/_fsspec_bridge.py +212 -0
  5. xtr_storage/adapter/_object_store.py +108 -0
  6. xtr_storage/adapter/fsspec_adapter.py +528 -0
  7. xtr_storage/adapter/gcs_adapter.py +325 -0
  8. xtr_storage/adapter/generic_fsspec_adapter.py +60 -0
  9. xtr_storage/adapter/in_memory_adapter.py +174 -0
  10. xtr_storage/adapter/local_adapter.py +259 -0
  11. xtr_storage/adapter/path_prefixed_adapter.py +354 -0
  12. xtr_storage/adapter/portable_visibility_converter.py +98 -0
  13. xtr_storage/adapter/read_only_adapter.py +231 -0
  14. xtr_storage/adapter/s3_adapter.py +441 -0
  15. xtr_storage/adapter/storage_adapter_interface.py +217 -0
  16. xtr_storage/adapter/visibility_converter_interface.py +88 -0
  17. xtr_storage/bundle/__init__.py +28 -0
  18. xtr_storage/bundle/adapter_configs.py +227 -0
  19. xtr_storage/bundle/storage_bundle.py +328 -0
  20. xtr_storage/bundle/storage_config.py +103 -0
  21. xtr_storage/bundle/storage_definition.py +85 -0
  22. xtr_storage/checksum_provider_interface.py +42 -0
  23. xtr_storage/config.py +260 -0
  24. xtr_storage/directory_attributes.py +80 -0
  25. xtr_storage/directory_listing.py +149 -0
  26. xtr_storage/exception/__init__.py +75 -0
  27. xtr_storage/exception/checksum_algorithm_not_supported_error.py +29 -0
  28. xtr_storage/exception/corrupted_path_detected_error.py +27 -0
  29. xtr_storage/exception/feature_not_supported_error.py +39 -0
  30. xtr_storage/exception/invalid_argument_error.py +29 -0
  31. xtr_storage/exception/invalid_stream_error.py +30 -0
  32. xtr_storage/exception/invalid_visibility_error.py +30 -0
  33. xtr_storage/exception/missing_backend_error.py +34 -0
  34. xtr_storage/exception/path_traversal_detected_error.py +27 -0
  35. xtr_storage/exception/storage_error.py +19 -0
  36. xtr_storage/exception/storage_operation_failed_error.py +32 -0
  37. xtr_storage/exception/symbolic_link_encountered_error.py +26 -0
  38. xtr_storage/exception/unable_to_check_directory_existence_error.py +24 -0
  39. xtr_storage/exception/unable_to_check_existence_error.py +32 -0
  40. xtr_storage/exception/unable_to_check_file_existence_error.py +24 -0
  41. xtr_storage/exception/unable_to_copy_file_error.py +39 -0
  42. xtr_storage/exception/unable_to_create_directory_error.py +32 -0
  43. xtr_storage/exception/unable_to_delete_directory_error.py +36 -0
  44. xtr_storage/exception/unable_to_delete_file_error.py +35 -0
  45. xtr_storage/exception/unable_to_generate_public_url_error.py +30 -0
  46. xtr_storage/exception/unable_to_generate_temporary_url_error.py +30 -0
  47. xtr_storage/exception/unable_to_list_contents_error.py +37 -0
  48. xtr_storage/exception/unable_to_move_file_error.py +39 -0
  49. xtr_storage/exception/unable_to_provide_checksum_error.py +30 -0
  50. xtr_storage/exception/unable_to_read_file_error.py +36 -0
  51. xtr_storage/exception/unable_to_resolve_mount_error.py +30 -0
  52. xtr_storage/exception/unable_to_retrieve_metadata_error.py +61 -0
  53. xtr_storage/exception/unable_to_set_visibility_error.py +36 -0
  54. xtr_storage/exception/unable_to_write_file_error.py +35 -0
  55. xtr_storage/feature.py +23 -0
  56. xtr_storage/file_attributes.py +88 -0
  57. xtr_storage/identical_path_policy.py +27 -0
  58. xtr_storage/link_handling.py +25 -0
  59. xtr_storage/mime_type/__init__.py +8 -0
  60. xtr_storage/mime_type/extension_mime_type_detector.py +53 -0
  61. xtr_storage/mime_type/mime_type_detector_interface.py +32 -0
  62. xtr_storage/mount_manager.py +461 -0
  63. xtr_storage/operation.py +31 -0
  64. xtr_storage/path/__init__.py +9 -0
  65. xtr_storage/path/path_normalizer_interface.py +33 -0
  66. xtr_storage/path/path_prefixer.py +66 -0
  67. xtr_storage/path/whitespace_path_normalizer.py +68 -0
  68. xtr_storage/py.typed +0 -0
  69. xtr_storage/recipe/__init__.py +15 -0
  70. xtr_storage/recipe/manifest.toml +21 -0
  71. xtr_storage/storage.py +480 -0
  72. xtr_storage/storage_attributes.py +18 -0
  73. xtr_storage/storage_operator_interface.py +91 -0
  74. xtr_storage/storage_reader_interface.py +189 -0
  75. xtr_storage/storage_writer_interface.py +172 -0
  76. xtr_storage/url_generation/__init__.py +17 -0
  77. xtr_storage/url_generation/chained_public_url_generator.py +66 -0
  78. xtr_storage/url_generation/prefix_public_url_generator.py +54 -0
  79. xtr_storage/url_generation/public_url_generator_interface.py +41 -0
  80. xtr_storage/url_generation/sharded_prefix_public_url_generator.py +68 -0
  81. xtr_storage/url_generation/temporary_url_generator_interface.py +45 -0
  82. xtr_storage/visibility.py +48 -0
  83. xtr_storage-3.0.0.dist-info/METADATA +475 -0
  84. xtr_storage-3.0.0.dist-info/RECORD +87 -0
  85. xtr_storage-3.0.0.dist-info/WHEEL +4 -0
  86. xtr_storage-3.0.0.dist-info/entry_points.txt +6 -0
  87. xtr_storage-3.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,207 @@
1
+ ---
2
+ name: xtr-storage
3
+ description: How to read, write, list, copy and move files with xtr-storage behind one async interface, on local disk, in memory, in Amazon S3, in Google Cloud Storage or on any fsspec backend. Use when code stores uploads, reports, exports or other files, must work against a disk in development and a bucket in production, needs public or signed urls, file visibility, checksums, streaming of large files, several named storages or a read-only or path-prefixed view; also when adding StorageBundle and named storages to an application on xtr-dependency-injection, or testing code that writes files.
4
+ ---
5
+
6
+ # xtr-storage
7
+
8
+ Where the bytes live is a constructor argument. A `Storage` wraps one adapter and turns paths into
9
+ reads, writes, listings, copies and moves; swapping the adapter moves the files without touching
10
+ the code that stores them. Every call that reaches the backend is awaited.
11
+
12
+ ## Quick reference
13
+
14
+ - Depend on `StorageOperatorInterface` (or `StorageReaderInterface` / `StorageWriterInterface`),
15
+ never on an adapter or on `Storage` itself.
16
+ - Build one: `Storage(LocalAdapter("var/storage"))`, `Storage(InMemoryAdapter())`,
17
+ `Storage(S3Adapter("bucket", prefix="avatars"))`.
18
+ - Write and read: `await storage.write("a/b.txt", b"...")`, `await storage.read("a/b.txt")`.
19
+ - Large files: `storage.read_stream(path)` and `await storage.write_stream(path, chunks)`.
20
+ - Per-call options lay over the storage's own: `await storage.write(path, data, {"visibility": "public"})`.
21
+ - Close what you build: `async with Storage(adapter) as storage:`, or `await storage.close()`.
22
+ - Several storages behind one: `MountManager({"uploads": ..., "reports": ...})`, paths `uploads://a.png`.
23
+ - In an application: activate `StorageBundle`, inject `StorageOperatorInterface`.
24
+
25
+ ## Read and write
26
+
27
+ ```python
28
+ from xtr_storage import InMemoryAdapter, Storage, Visibility
29
+
30
+
31
+ async with Storage(InMemoryAdapter()) as storage:
32
+ await storage.write("notes/todo.txt", b"buy milk", {"visibility": Visibility.PUBLIC})
33
+ data = await storage.read("notes/todo.txt") # b"buy milk"
34
+ await storage.file_exists("notes/todo.txt") # True
35
+ await storage.mime_type("notes/todo.txt") # "text/plain"
36
+ await storage.file_size("notes/todo.txt") # 8
37
+ await storage.copy("notes/todo.txt", "notes/copy.txt")
38
+ await storage.move("notes/copy.txt", "archive/copy.txt")
39
+ names = sorted([entry.path async for entry in storage.list_contents("notes", deep=True)])
40
+ await storage.delete("notes/todo.txt")
41
+ ```
42
+
43
+ | Call | Does |
44
+ | --- | --- |
45
+ | `read(path)`, `read_stream(path)` | The whole file, or its chunks as an async iterator |
46
+ | `write(path, data, config=None)`, `write_stream(path, stream, config=None)` | Creates or replaces; the stream may be async, a plain iterable, or a binary file object |
47
+ | `file_exists`, `directory_exists`, `has` | Existence; `has` is either |
48
+ | `list_contents(path="", deep=False)` | A lazy listing of `FileAttributes` / `DirectoryAttributes`; nothing is read until iterated |
49
+ | `file_size`, `last_modified`, `mime_type`, `visibility` | Metadata |
50
+ | `copy`, `move`, `delete`, `delete_directory`, `create_directory` | Changes |
51
+ | `set_visibility(path, visibility)` | `Visibility.PUBLIC` or `Visibility.PRIVATE`; each member is its string |
52
+ | `checksum(path, config=None)` | The backend's digest when it keeps one, else hashed here |
53
+ | `public_url(path)`, `temporary_url(path, expires_at)` | Lasting and expiring addresses |
54
+
55
+ Paths are normalized before an adapter sees them: backslashes become `/`, `.` and empty segments
56
+ drop. A control or zero-width character raises `CorruptedPathDetectedError`; `..` climbing past
57
+ the root raises `PathTraversalDetectedError` when `allow_relative_path_traversal` is `False`.
58
+
59
+ ## Pick an adapter
60
+
61
+ | Adapter | Where | Extra | Visibility | Urls |
62
+ | --- | --- | --- | --- | --- |
63
+ | `LocalAdapter(root)` | A directory on this machine | — | permission bits | only through a configured generator |
64
+ | `InMemoryAdapter()` | A dictionary in this process | — | remembered per file | only through a configured generator |
65
+ | `S3Adapter(bucket, prefix=..., region=...)` | An S3 bucket | `s3` | access list | public and signed |
66
+ | `GcsAdapter(bucket, prefix=...)` | A Google Cloud Storage bucket | `gcs` | refused | public and signed |
67
+ | `GenericFsspecAdapter(filesystem, root=...)` | Any fsspec filesystem | — | refused | only through a configured generator |
68
+ | `ReadOnlyAdapter(adapter)` | Any of the above, writes refused | — | as wrapped | as wrapped |
69
+ | `PathPrefixedAdapter(adapter, prefix)` | Any of the above, under a prefix | — | as wrapped | as wrapped |
70
+
71
+ An adapter whose extra is missing raises `MissingBackendError` naming it. A capability a backend
72
+ lacks raises `FeatureNotSupportedError` instead of guessing. No adapter opens anything when built;
73
+ `LocalAdapter` creates its root on the first write.
74
+
75
+ ## Several storages
76
+
77
+ ```python
78
+ from xtr_storage import InMemoryAdapter, MountManager, Storage
79
+
80
+ async with MountManager(
81
+ {"uploads": Storage(InMemoryAdapter()), "reports": Storage(InMemoryAdapter())}
82
+ ) as files:
83
+ await files.write("uploads://a.txt", b"hi")
84
+ await files.copy("uploads://a.txt", "reports://a.txt") # streamed across storages
85
+ ```
86
+
87
+ There is no default mount: an unknown name, a missing `://` or an empty name raises
88
+ `UnableToResolveMountError`.
89
+
90
+ ## Testing
91
+
92
+ The package ships no test helpers; `InMemoryAdapter` is the one a test needs.
93
+
94
+ - Hand the code under test `Storage(InMemoryAdapter())`. Two in-memory adapters share nothing, so
95
+ a test that writes has nothing to undo.
96
+ - Assert what was stored by reading it back through the same storage.
97
+ - In an application test, replace the service:
98
+ `boot_for_test(kernel, overrides={StorageOperatorInterface: Storage(InMemoryAdapter())})`.
99
+
100
+ ```python
101
+ from xtr_storage import InMemoryAdapter, Storage
102
+
103
+
104
+ async def test_the_report_is_stored() -> None:
105
+ storage = Storage(InMemoryAdapter())
106
+
107
+ await write_report(storage)
108
+
109
+ assert await storage.read("reports/2026.txt") == b"hello"
110
+ ```
111
+
112
+ ## Use in an application
113
+
114
+ `uv run xtr-recipes recipes:sync` applies the recipe shipped with this package: it lists
115
+ `StorageBundle` and ignores `/var/storage/`. That is the steps below a recipe can do; the others it
116
+ prints for you to make.
117
+
118
+ 1. **Install** — `uv add "xtr-storage[di]"`; add `s3` or `gcs` for the object stores you reach.
119
+ 2. **Activate** — `StorageBundle: {"all": True}` in `BUNDLES` in `<app>/bundles.py`, imported from
120
+ `xtr_storage.bundle`.
121
+ 3. **Brings along** — nothing; the bundle requires no other.
122
+ 4. **Configure** — optional; with no configuration there is one `default` storage on local files
123
+ under `var/storage`, and nothing is opened until it is first used:
124
+
125
+ ```python
126
+ # <app>/config/storage.py
127
+ from xtr_dependency_injection import configure, env
128
+ from xtr_storage.bundle import LocalAdapterConfig, S3AdapterConfig, StorageConfig, StorageDefinition
129
+
130
+
131
+ @configure
132
+ def storage() -> StorageConfig:
133
+ return StorageConfig(
134
+ storages={
135
+ "default": LocalAdapterConfig(),
136
+ "uploads": StorageDefinition(
137
+ adapter=S3AdapterConfig(bucket=env("UPLOADS_BUCKET"), region=env("AWS_REGION")),
138
+ prefix="avatars",
139
+ visibility="public",
140
+ ),
141
+ },
142
+ )
143
+ ```
144
+
145
+ An adapter entry is `LocalAdapterConfig`, `MemoryAdapterConfig`, `S3AdapterConfig`,
146
+ `GcsAdapterConfig`, `FsspecAdapterConfig`, or a `Reference` to an adapter the container
147
+ provides. A `StorageDefinition` adds `visibility`, `directory_visibility`,
148
+ `retain_visibility`, `public_url`, `read_only` and `prefix`.
149
+ 5. **Environment** — nothing required; credentials and directories given as `env(...)` must be set
150
+ when the application boots, because booting checks every storage.
151
+ 6. **Ignore** — `/var/storage/` in `.gitignore`.
152
+ 7. **Use** — inject `StorageOperatorInterface` for the `default` storage, or qualify by name. A
153
+ `MountManager` over every storage is provided too.
154
+
155
+ ```python
156
+ from typing import Annotated
157
+
158
+ from xtr_dependency_injection import Target, as_service
159
+ from xtr_storage import StorageOperatorInterface
160
+
161
+
162
+ @as_service
163
+ class Avatars:
164
+ def __init__(
165
+ self,
166
+ storage: StorageOperatorInterface, # the default storage
167
+ uploads: Annotated[StorageOperatorInterface, Target("uploads")],
168
+ ) -> None: ...
169
+ ```
170
+
171
+ 8. **Check** — `debug:bundles` shows `storage` as `listed` and `active`.
172
+ 9. **Remove** — drop the `BUNDLES` entry, delete `<app>/config/storage.py`, then
173
+ `uv remove xtr-storage`.
174
+
175
+ ## Errors
176
+
177
+ Everything derives from `StorageError`; a failure of an operation the backend refused derives from
178
+ `StorageOperationFailedError` and names the operation. No message repeats a credential.
179
+
180
+ | Error | Raised when |
181
+ | --- | --- |
182
+ | `UnableToReadFileError`, `UnableToWriteFileError` | A read or write fails, or the file is missing |
183
+ | `UnableToDeleteFileError`, `UnableToDeleteDirectoryError`, `UnableToCreateDirectoryError` | A delete or a directory creation fails |
184
+ | `UnableToCopyFileError`, `UnableToMoveFileError` | A copy or move fails, or the paths are the same under a `fail` policy |
185
+ | `UnableToRetrieveMetadataError`, `UnableToSetVisibilityError` | Metadata cannot be read, or a visibility set |
186
+ | `UnableToListContentsError` | A listing fails while it is iterated |
187
+ | `UnableToProvideChecksumError`, `ChecksumAlgorithmNotSupportedError` | No checksum can be produced |
188
+ | `UnableToGeneratePublicUrlError`, `UnableToGenerateTemporaryUrlError` | No url can be produced |
189
+ | `UnableToResolveMountError` | A mount location names no storage, or is malformed |
190
+ | `PathTraversalDetectedError`, `CorruptedPathDetectedError` | A path climbs past the root, or carries a control character |
191
+ | `FeatureNotSupportedError` | The backend lacks the capability (also a `NotImplementedError`) |
192
+ | `MissingBackendError` | The adapter's extra is not installed (also an `ImportError`) |
193
+ | `InvalidVisibilityError`, `InvalidArgumentError` | A bad visibility string or option (also a `ValueError`) |
194
+ | `InvalidStreamError` | A write stream of a type the storage cannot read (also a `TypeError`) |
195
+
196
+ ## Do not
197
+
198
+ - Do not depend on an adapter or on `Storage` in application code; depend on
199
+ `StorageOperatorInterface` and let the container pick the backend.
200
+ - Do not build paths with `os.path` or `pathlib`; pass `/`-separated strings, as every backend
201
+ sees them.
202
+ - Do not read a large file whole with `read`; stream it with `read_stream` and `write_stream`.
203
+ - Do not expect a backend to fake a capability: GCS and generic fsspec backends refuse visibility,
204
+ and a local or in-memory storage gives urls only through a configured generator.
205
+ - Do not leave a storage you built open; use `async with` or `close()`. The container closes the
206
+ ones it built.
207
+ - Do not put credentials in code; read them with `env(...)` in `config/storage.py`.
@@ -0,0 +1,158 @@
1
+ """Files on local disk, in memory or in object stores, behind one async interface.
2
+
3
+ A ``Storage`` wraps one adapter and turns paths into reads, writes, listings,
4
+ copies and moves. The adapter decides where the bytes live — a directory on this
5
+ machine, a dictionary in this process, a bucket on an object store — so code that
6
+ stores files is written once and moved between them by its constructor.
7
+
8
+ Every call that touches a backend is awaited.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from importlib.metadata import PackageNotFoundError, version
14
+
15
+ from xtr_storage.adapter import (
16
+ FsspecAdapter,
17
+ GcsAdapter,
18
+ GenericFsspecAdapter,
19
+ InMemoryAdapter,
20
+ LocalAdapter,
21
+ PathPrefixedAdapter,
22
+ PortableVisibilityConverter,
23
+ ReadOnlyAdapter,
24
+ S3Adapter,
25
+ StorageAdapterInterface,
26
+ VisibilityConverterInterface,
27
+ )
28
+ from xtr_storage.checksum_provider_interface import ChecksumProviderInterface
29
+ from xtr_storage.config import Config
30
+ from xtr_storage.directory_attributes import DirectoryAttributes
31
+ from xtr_storage.directory_listing import DirectoryListing
32
+ from xtr_storage.exception import (
33
+ ChecksumAlgorithmNotSupportedError,
34
+ CorruptedPathDetectedError,
35
+ FeatureNotSupportedError,
36
+ InvalidArgumentError,
37
+ InvalidStreamError,
38
+ InvalidVisibilityError,
39
+ MissingBackendError,
40
+ PathTraversalDetectedError,
41
+ StorageError,
42
+ StorageOperationFailedError,
43
+ SymbolicLinkEncounteredError,
44
+ UnableToCheckDirectoryExistenceError,
45
+ UnableToCheckExistenceError,
46
+ UnableToCheckFileExistenceError,
47
+ UnableToCopyFileError,
48
+ UnableToCreateDirectoryError,
49
+ UnableToDeleteDirectoryError,
50
+ UnableToDeleteFileError,
51
+ UnableToGeneratePublicUrlError,
52
+ UnableToGenerateTemporaryUrlError,
53
+ UnableToListContentsError,
54
+ UnableToMoveFileError,
55
+ UnableToProvideChecksumError,
56
+ UnableToReadFileError,
57
+ UnableToResolveMountError,
58
+ UnableToRetrieveMetadataError,
59
+ UnableToSetVisibilityError,
60
+ UnableToWriteFileError,
61
+ )
62
+ from xtr_storage.feature import Feature
63
+ from xtr_storage.file_attributes import FileAttributes
64
+ from xtr_storage.identical_path_policy import IdenticalPathPolicy
65
+ from xtr_storage.link_handling import LinkHandling
66
+ from xtr_storage.mime_type import ExtensionMimeTypeDetector, MimeTypeDetectorInterface
67
+ from xtr_storage.mount_manager import MountManager
68
+ from xtr_storage.operation import Operation
69
+ from xtr_storage.path import PathNormalizerInterface, PathPrefixer, WhitespacePathNormalizer
70
+ from xtr_storage.storage import Storage
71
+ from xtr_storage.storage_attributes import StorageAttributes
72
+ from xtr_storage.storage_operator_interface import StorageOperatorInterface
73
+ from xtr_storage.storage_reader_interface import StorageReaderInterface
74
+ from xtr_storage.storage_writer_interface import StorageWriterInterface
75
+ from xtr_storage.url_generation import (
76
+ ChainedPublicUrlGenerator,
77
+ PrefixPublicUrlGenerator,
78
+ PublicUrlGeneratorInterface,
79
+ ShardedPrefixPublicUrlGenerator,
80
+ TemporaryUrlGeneratorInterface,
81
+ )
82
+ from xtr_storage.visibility import Visibility
83
+
84
+ try:
85
+ __version__ = version("xtr-storage")
86
+ except PackageNotFoundError: # pragma: no cover
87
+ # Running from a source tree or a vendored copy, with no installed
88
+ # metadata to read. Having no version is better than refusing to import.
89
+ __version__ = "0+unknown"
90
+
91
+ __all__ = [
92
+ "ChainedPublicUrlGenerator",
93
+ "ChecksumAlgorithmNotSupportedError",
94
+ "ChecksumProviderInterface",
95
+ "Config",
96
+ "CorruptedPathDetectedError",
97
+ "DirectoryAttributes",
98
+ "DirectoryListing",
99
+ "ExtensionMimeTypeDetector",
100
+ "Feature",
101
+ "FeatureNotSupportedError",
102
+ "FileAttributes",
103
+ "FsspecAdapter",
104
+ "GcsAdapter",
105
+ "GenericFsspecAdapter",
106
+ "IdenticalPathPolicy",
107
+ "InMemoryAdapter",
108
+ "InvalidArgumentError",
109
+ "InvalidStreamError",
110
+ "InvalidVisibilityError",
111
+ "LinkHandling",
112
+ "LocalAdapter",
113
+ "MimeTypeDetectorInterface",
114
+ "MissingBackendError",
115
+ "MountManager",
116
+ "Operation",
117
+ "PathNormalizerInterface",
118
+ "PathPrefixedAdapter",
119
+ "PathPrefixer",
120
+ "PathTraversalDetectedError",
121
+ "PortableVisibilityConverter",
122
+ "PrefixPublicUrlGenerator",
123
+ "PublicUrlGeneratorInterface",
124
+ "ReadOnlyAdapter",
125
+ "S3Adapter",
126
+ "ShardedPrefixPublicUrlGenerator",
127
+ "Storage",
128
+ "StorageAdapterInterface",
129
+ "StorageAttributes",
130
+ "StorageError",
131
+ "StorageOperationFailedError",
132
+ "StorageOperatorInterface",
133
+ "StorageReaderInterface",
134
+ "StorageWriterInterface",
135
+ "SymbolicLinkEncounteredError",
136
+ "TemporaryUrlGeneratorInterface",
137
+ "UnableToCheckDirectoryExistenceError",
138
+ "UnableToCheckExistenceError",
139
+ "UnableToCheckFileExistenceError",
140
+ "UnableToCopyFileError",
141
+ "UnableToCreateDirectoryError",
142
+ "UnableToDeleteDirectoryError",
143
+ "UnableToDeleteFileError",
144
+ "UnableToGeneratePublicUrlError",
145
+ "UnableToGenerateTemporaryUrlError",
146
+ "UnableToListContentsError",
147
+ "UnableToMoveFileError",
148
+ "UnableToProvideChecksumError",
149
+ "UnableToReadFileError",
150
+ "UnableToResolveMountError",
151
+ "UnableToRetrieveMetadataError",
152
+ "UnableToSetVisibilityError",
153
+ "UnableToWriteFileError",
154
+ "Visibility",
155
+ "VisibilityConverterInterface",
156
+ "WhitespacePathNormalizer",
157
+ "__version__",
158
+ ]
@@ -0,0 +1,29 @@
1
+ """Where the bytes actually live: one adapter per kind of backend."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .fsspec_adapter import FsspecAdapter
6
+ from .gcs_adapter import GcsAdapter
7
+ from .generic_fsspec_adapter import GenericFsspecAdapter
8
+ from .in_memory_adapter import InMemoryAdapter
9
+ from .local_adapter import LocalAdapter
10
+ from .path_prefixed_adapter import PathPrefixedAdapter
11
+ from .portable_visibility_converter import PortableVisibilityConverter
12
+ from .read_only_adapter import ReadOnlyAdapter
13
+ from .s3_adapter import S3Adapter
14
+ from .storage_adapter_interface import StorageAdapterInterface
15
+ from .visibility_converter_interface import VisibilityConverterInterface
16
+
17
+ __all__ = [
18
+ "FsspecAdapter",
19
+ "GcsAdapter",
20
+ "GenericFsspecAdapter",
21
+ "InMemoryAdapter",
22
+ "LocalAdapter",
23
+ "PathPrefixedAdapter",
24
+ "PortableVisibilityConverter",
25
+ "ReadOnlyAdapter",
26
+ "S3Adapter",
27
+ "StorageAdapterInterface",
28
+ "VisibilityConverterInterface",
29
+ ]
@@ -0,0 +1,212 @@
1
+ # pyright: reportMissingTypeStubs=false, reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false, reportAttributeAccessIssue=false, reportAny=false, reportExplicitAny=false
2
+ # fsspec ships no type information, so every attribute reached on the wrapped
3
+ # filesystem is untyped to the checker; the directives above are confined to
4
+ # this one module, and every value is copied into a typed shape at the boundary.
5
+ """The one place a filesystem is spoken to, turned into typed coroutines.
6
+
7
+ Every backend this package builds on is an fsspec filesystem, and fsspec speaks
8
+ two dialects: a synchronous one whose blocking calls must be pushed off the
9
+ event loop, and an asynchronous one whose real work hides behind ``_``-prefixed
10
+ coroutines that need a session opened first. This bridge is the seam that hides
11
+ that split: an adapter awaits :class:`FsspecBridge` methods and never learns
12
+ which dialect answered, and the untyped surface of fsspec is fenced to this file
13
+ so the rest of the package stays fully typed.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import asyncio
19
+ from typing import TYPE_CHECKING, Any, final
20
+
21
+ if TYPE_CHECKING:
22
+ from collections.abc import Awaitable, Callable
23
+
24
+ from fsspec import AbstractFileSystem
25
+
26
+ __all__ = ["FsspecBridge"]
27
+
28
+
29
+ @final
30
+ class FsspecBridge:
31
+ """A typed async front to one fsspec filesystem, sync or asynchronous.
32
+
33
+ The bridge owns nothing the filesystem does not already own except the
34
+ session it opens on demand. It creates no filesystem — an adapter hands it
35
+ one — and it does no I/O until a method is awaited, so constructing it is
36
+ free. On an asynchronous filesystem the first awaited method opens the
37
+ session once, under a lock, and ``close`` hands that session back to a
38
+ per-backend closer; on a synchronous one every call is pushed to a worker
39
+ thread and there is nothing to close.
40
+ """
41
+
42
+ __slots__ = (
43
+ "_closed",
44
+ "_closer",
45
+ "_fs",
46
+ "_session_created",
47
+ "_session_lock",
48
+ "_session_ready",
49
+ )
50
+
51
+ def __init__(
52
+ self,
53
+ filesystem: AbstractFileSystem,
54
+ *,
55
+ closer: Callable[[AbstractFileSystem], Awaitable[None]] | None = None,
56
+ ) -> None:
57
+ """Wrap a filesystem, remembering how to close any session it opens.
58
+
59
+ Args:
60
+ filesystem: The fsspec filesystem to speak to; kept as given, its
61
+ dialect read from ``async_impl`` and ``asynchronous`` on each
62
+ call so a caller may flip ``asynchronous`` before first use.
63
+ closer: What to run once, on ``close``, when a session was opened —
64
+ the S3 and object-store adapters pass the coroutine that shuts
65
+ their client down. ``None`` means there is nothing to close.
66
+ """
67
+ self._fs = filesystem
68
+ self._closer = closer
69
+ self._session_lock = asyncio.Lock()
70
+ self._session_ready = False
71
+ self._session_created = False
72
+ self._closed = False
73
+
74
+ @property
75
+ def _is_async(self) -> bool:
76
+ """Whether calls must go through the ``_``-prefixed coroutines."""
77
+ return bool(self._fs.async_impl and self._fs.asynchronous) # ty: ignore[unresolved-attribute] -- fsspec ships no type information; asynchronous is set on the async filesystem
78
+
79
+ async def _ensure_session(self) -> None:
80
+ """Open the filesystem's session once, the first async call to need it.
81
+
82
+ A filesystem without ``set_session`` (a plain asynchronous one) needs no
83
+ session; one with it (s3fs, gcsfs) gets exactly one call however many
84
+ coroutines race here, because the lock and the flag together let only
85
+ the first through.
86
+ """
87
+ if self._session_ready:
88
+ return
89
+ async with self._session_lock:
90
+ if self._session_ready:
91
+ return
92
+ set_session = getattr(self._fs, "set_session", None)
93
+ if set_session is not None:
94
+ _ = await set_session()
95
+ self._session_created = True
96
+ self._session_ready = True
97
+
98
+ async def _dispatch(self, name: str, *args: object, **kwargs: object) -> Any: # noqa: ANN401
99
+ """Run one filesystem method the way its dialect requires.
100
+
101
+ Args:
102
+ name: The synchronous method name; the asynchronous dialect is
103
+ reached at ``_`` + ``name``.
104
+ *args: Positional arguments passed through unchanged.
105
+ **kwargs: Keyword arguments passed through unchanged.
106
+
107
+ Returns:
108
+ Whatever the filesystem returned, untyped — fsspec ships no types, so
109
+ this one seam is ``Any`` and every caller copies it into a typed
110
+ shape at the boundary before it escapes the module.
111
+ """
112
+ if self._is_async:
113
+ await self._ensure_session()
114
+ return await getattr(self._fs, "_" + name)(*args, **kwargs)
115
+ return await asyncio.to_thread(getattr(self._fs, name), *args, **kwargs)
116
+
117
+ async def info(self, path: str) -> dict[str, object]:
118
+ """Return what the filesystem knows about one path, copied out."""
119
+ result = await self._dispatch("info", path)
120
+ return dict(result)
121
+
122
+ async def ls(self, path: str, *, detail: bool = True) -> list[dict[str, object]]:
123
+ """List one directory in detail, each entry copied out."""
124
+ result = await self._dispatch("ls", path, detail=detail)
125
+ return [dict(entry) for entry in result]
126
+
127
+ async def find(self, path: str, *, withdirs: bool) -> dict[str, dict[str, object]]:
128
+ """Walk a tree, returning every path mapped to its detail, copied out."""
129
+ result = await self._dispatch("find", path, withdirs=withdirs, detail=True)
130
+ return {key: dict(value) for key, value in result.items()}
131
+
132
+ async def cat_file(self, path: str, start: int | None = None, end: int | None = None) -> bytes:
133
+ """Read a file, or the byte range ``[start, end)`` of it."""
134
+ result = await self._dispatch("cat_file", path, start=start, end=end)
135
+ return bytes(result)
136
+
137
+ async def pipe_file(self, path: str, data: bytes, **kwargs: object) -> None:
138
+ """Write ``data`` to ``path`` in one call."""
139
+ _ = await self._dispatch("pipe_file", path, data, **kwargs)
140
+
141
+ async def put_file(self, local_path: str, remote_path: str, **kwargs: object) -> None:
142
+ """Upload a local file to ``remote_path``."""
143
+ _ = await self._dispatch("put_file", local_path, remote_path, **kwargs)
144
+
145
+ async def rm_file(self, path: str) -> None:
146
+ """Remove one file."""
147
+ _ = await self._dispatch("rm_file", path)
148
+
149
+ async def rm(self, path: str, *, recursive: bool = True) -> None:
150
+ """Remove a path, its whole tree by default."""
151
+ _ = await self._dispatch("rm", path, recursive=recursive)
152
+
153
+ async def mkdir(self, path: str, *, create_parents: bool = True) -> None:
154
+ """Make a directory, its parents too by default."""
155
+ _ = await self._dispatch("mkdir", path, create_parents=create_parents)
156
+
157
+ async def cp_file(self, src: str, dst: str, **kwargs: object) -> None:
158
+ """Copy one file within the filesystem."""
159
+ _ = await self._dispatch("cp_file", src, dst, **kwargs)
160
+
161
+ async def mv(self, src: str, dst: str) -> None:
162
+ """Move one file, falling back to copy-then-delete where ``_mv`` is absent.
163
+
164
+ The synchronous dialect has ``mv``; the asynchronous base does not
165
+ expose ``_mv``, so there the move is a copy followed by a delete —
166
+ exactly what an object store does anyway.
167
+ """
168
+ if self._is_async:
169
+ await self._ensure_session()
170
+ mv_method = getattr(self._fs, "_mv", None)
171
+ if mv_method is not None:
172
+ _ = await mv_method(src, dst)
173
+ return
174
+ await self.cp_file(src, dst)
175
+ await self.rm_file(src)
176
+ return
177
+ _ = await asyncio.to_thread(self._fs.mv, src, dst)
178
+
179
+ async def sign(self, path: str, expiration: int) -> str:
180
+ """Return a signed URL for ``path``, valid ``expiration`` seconds.
181
+
182
+ The asynchronous dialect signs through ``_url`` when a backend provides
183
+ it (s3fs, gcsfs); otherwise, and for the synchronous dialect, the
184
+ blocking ``sign`` is pushed to a worker thread.
185
+ """
186
+ if self._is_async:
187
+ await self._ensure_session()
188
+ url_method = getattr(self._fs, "_url", None)
189
+ if url_method is not None:
190
+ return str(await url_method(path, expires=expiration))
191
+ return str(await asyncio.to_thread(self._fs.sign, path, expiration))
192
+
193
+ async def exists(self, path: str) -> bool:
194
+ """Whether the path exists."""
195
+ return bool(await self._dispatch("exists", path))
196
+
197
+ async def call(self, name: str, *args: object, **kwargs: object) -> object:
198
+ """Run a backend-specific method by name, dispatched like the rest.
199
+
200
+ The escape hatch for what only one backend does — an ACL read, a chmod —
201
+ so those adapters need no second seam. The return is untyped; the caller
202
+ parses it.
203
+ """
204
+ return await self._dispatch(name, *args, **kwargs)
205
+
206
+ async def close(self) -> None:
207
+ """Close the session, once, if one was opened; safe to call again."""
208
+ if self._closed:
209
+ return
210
+ self._closed = True
211
+ if self._closer is not None and self._session_created:
212
+ await self._closer(self._fs)