swarmauri_storage_s3 0.11.0.dev5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swarmauri_storage_s3/__init__.py +8 -0
- swarmauri_storage_s3/s3_storage_adapter.py +248 -0
- swarmauri_storage_s3-0.11.0.dev5.dist-info/METADATA +84 -0
- swarmauri_storage_s3-0.11.0.dev5.dist-info/RECORD +7 -0
- swarmauri_storage_s3-0.11.0.dev5.dist-info/WHEEL +4 -0
- swarmauri_storage_s3-0.11.0.dev5.dist-info/entry_points.txt +6 -0
- swarmauri_storage_s3-0.11.0.dev5.dist-info/licenses/LICENSE +3 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
try:
|
|
2
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
3
|
+
except ImportError: # pragma: no cover
|
|
4
|
+
from importlib_metadata import PackageNotFoundError, version
|
|
5
|
+
|
|
6
|
+
from .s3_storage_adapter import S3StorageAdapter
|
|
7
|
+
|
|
8
|
+
__all__ = ["PackageNotFoundError", "S3StorageAdapter", "version"]
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
"""Generic S3-compatible storage adapter using direct S3 client semantics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import io
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import BinaryIO, Literal
|
|
10
|
+
|
|
11
|
+
import boto3
|
|
12
|
+
from botocore.config import Config
|
|
13
|
+
from botocore.exceptions import ClientError
|
|
14
|
+
from pydantic import SecretStr
|
|
15
|
+
|
|
16
|
+
from swarmauri_base.ComponentBase import ComponentBase
|
|
17
|
+
from swarmauri_base.storage import StorageAdapterBase
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@ComponentBase.register_type(StorageAdapterBase, "S3StorageAdapter")
|
|
21
|
+
class S3StorageAdapter(StorageAdapterBase):
|
|
22
|
+
"""Store and retrieve artifacts through an S3-compatible API endpoint."""
|
|
23
|
+
|
|
24
|
+
type: Literal["S3StorageAdapter"] = "S3StorageAdapter"
|
|
25
|
+
|
|
26
|
+
def __init__(
|
|
27
|
+
self,
|
|
28
|
+
bucket: str,
|
|
29
|
+
*,
|
|
30
|
+
endpoint_url: str | None = None,
|
|
31
|
+
region_name: str | None = None,
|
|
32
|
+
access_key: SecretStr | str | None = None,
|
|
33
|
+
secret_key: SecretStr | str | None = None,
|
|
34
|
+
session_token: SecretStr | str | None = None,
|
|
35
|
+
use_ssl: bool = True,
|
|
36
|
+
verify: bool | str | None = None,
|
|
37
|
+
addressing_style: str | None = None,
|
|
38
|
+
prefix: str = "",
|
|
39
|
+
client_kwargs: dict | None = None,
|
|
40
|
+
config_kwargs: dict | None = None,
|
|
41
|
+
**kwargs,
|
|
42
|
+
) -> None:
|
|
43
|
+
super().__init__(**kwargs)
|
|
44
|
+
self._bucket = bucket
|
|
45
|
+
self._prefix = self.normalize_prefix(prefix)
|
|
46
|
+
|
|
47
|
+
config_options = dict(config_kwargs or {})
|
|
48
|
+
if addressing_style:
|
|
49
|
+
config_options.setdefault("s3", {})["addressing_style"] = (
|
|
50
|
+
addressing_style
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
options = dict(client_kwargs or {})
|
|
54
|
+
options.update(
|
|
55
|
+
{
|
|
56
|
+
"service_name": "s3",
|
|
57
|
+
"endpoint_url": endpoint_url,
|
|
58
|
+
"region_name": region_name,
|
|
59
|
+
"aws_access_key_id": self._secret_value(access_key),
|
|
60
|
+
"aws_secret_access_key": self._secret_value(secret_key),
|
|
61
|
+
"aws_session_token": self._secret_value(session_token),
|
|
62
|
+
"use_ssl": use_ssl,
|
|
63
|
+
"verify": verify,
|
|
64
|
+
}
|
|
65
|
+
)
|
|
66
|
+
options = {
|
|
67
|
+
key: value for key, value in options.items() if value is not None
|
|
68
|
+
}
|
|
69
|
+
if config_options:
|
|
70
|
+
options["config"] = Config(**config_options)
|
|
71
|
+
|
|
72
|
+
self._client = boto3.client(**options)
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def _secret_value(value: SecretStr | str | None) -> str | None:
|
|
76
|
+
if isinstance(value, SecretStr):
|
|
77
|
+
return value.get_secret_value() or None
|
|
78
|
+
return value or None
|
|
79
|
+
|
|
80
|
+
def _full_key(self, key: str) -> str:
|
|
81
|
+
return self.compose_key(self._prefix, key, allow_empty=True)
|
|
82
|
+
|
|
83
|
+
def _relative_key(self, key: str) -> str:
|
|
84
|
+
if self._prefix and key.startswith(f"{self._prefix.rstrip('/')}/"):
|
|
85
|
+
return key[len(self._prefix.rstrip("/")) + 1 :]
|
|
86
|
+
return key
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def root_uri(self) -> str:
|
|
90
|
+
"""Return the base ``s3://`` URI for this adapter."""
|
|
91
|
+
base = f"s3://{self._bucket}"
|
|
92
|
+
uri = f"{base}/{self._prefix.rstrip('/')}" if self._prefix else base
|
|
93
|
+
return uri.rstrip("/") + "/"
|
|
94
|
+
|
|
95
|
+
def upload(self, key: str, data: BinaryIO) -> str:
|
|
96
|
+
"""Upload *data* under *key* and return its ``s3://`` URI."""
|
|
97
|
+
normalized_key = self.normalize_key(key)
|
|
98
|
+
self._client.upload_fileobj(data, self._bucket, self._full_key(key))
|
|
99
|
+
return f"{self.root_uri}{normalized_key}"
|
|
100
|
+
|
|
101
|
+
def download(self, key: str) -> BinaryIO:
|
|
102
|
+
"""Download *key* into a binary in-memory stream."""
|
|
103
|
+
buffer = io.BytesIO()
|
|
104
|
+
try:
|
|
105
|
+
self._client.download_fileobj(
|
|
106
|
+
self._bucket, self._full_key(key), buffer
|
|
107
|
+
)
|
|
108
|
+
except ClientError as exc:
|
|
109
|
+
if self._is_missing(exc):
|
|
110
|
+
raise FileNotFoundError(key) from exc
|
|
111
|
+
raise
|
|
112
|
+
buffer.seek(0)
|
|
113
|
+
return buffer
|
|
114
|
+
|
|
115
|
+
def upload_dir(self, src: str | os.PathLike, *, prefix: str = "") -> None:
|
|
116
|
+
"""Recursively upload files from *src* under ``prefix``."""
|
|
117
|
+
base = Path(src)
|
|
118
|
+
for path in base.rglob("*"):
|
|
119
|
+
if path.is_file():
|
|
120
|
+
rel = path.relative_to(base).as_posix()
|
|
121
|
+
key = self.compose_key(prefix, rel)
|
|
122
|
+
with path.open("rb") as handle:
|
|
123
|
+
self.upload(key, handle)
|
|
124
|
+
|
|
125
|
+
def iter_prefix(self, prefix: str):
|
|
126
|
+
"""Yield stored keys below ``prefix`` relative to the adapter root."""
|
|
127
|
+
continuation_token = None
|
|
128
|
+
list_prefix = self._full_key(self.normalize_prefix(prefix))
|
|
129
|
+
while True:
|
|
130
|
+
request = {
|
|
131
|
+
"Bucket": self._bucket,
|
|
132
|
+
"Prefix": list_prefix,
|
|
133
|
+
}
|
|
134
|
+
if continuation_token:
|
|
135
|
+
request["ContinuationToken"] = continuation_token
|
|
136
|
+
response = self._client.list_objects_v2(**request)
|
|
137
|
+
for item in response.get("Contents", []):
|
|
138
|
+
rel = self._relative_key(item["Key"])
|
|
139
|
+
if rel:
|
|
140
|
+
yield rel
|
|
141
|
+
if not response.get("IsTruncated"):
|
|
142
|
+
break
|
|
143
|
+
continuation_token = response.get("NextContinuationToken")
|
|
144
|
+
|
|
145
|
+
def download_dir(self, prefix: str, dest_dir: str | os.PathLike) -> None:
|
|
146
|
+
"""Download all stored artifacts under ``prefix`` into ``dest_dir``."""
|
|
147
|
+
dest = Path(dest_dir)
|
|
148
|
+
normalized_prefix = self.normalize_prefix(prefix)
|
|
149
|
+
for rel_key in self.iter_prefix(prefix):
|
|
150
|
+
target_rel = rel_key
|
|
151
|
+
if normalized_prefix and rel_key.startswith(
|
|
152
|
+
f"{normalized_prefix}/"
|
|
153
|
+
):
|
|
154
|
+
target_rel = rel_key[len(normalized_prefix) + 1 :]
|
|
155
|
+
if not target_rel:
|
|
156
|
+
continue
|
|
157
|
+
target = self.download_target_for_key(dest, target_rel)
|
|
158
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
159
|
+
data = self.download(rel_key)
|
|
160
|
+
with target.open("wb") as handle:
|
|
161
|
+
shutil.copyfileobj(data, handle)
|
|
162
|
+
|
|
163
|
+
async def ensure_bucket(self) -> None:
|
|
164
|
+
"""Create the configured bucket if it does not already exist."""
|
|
165
|
+
try:
|
|
166
|
+
self._client.head_bucket(Bucket=self._bucket)
|
|
167
|
+
except ClientError as exc:
|
|
168
|
+
if self._is_missing(exc):
|
|
169
|
+
self._client.create_bucket(Bucket=self._bucket)
|
|
170
|
+
return
|
|
171
|
+
raise
|
|
172
|
+
|
|
173
|
+
async def put_bytes(
|
|
174
|
+
self, object_key: str, data: bytes, content_type: str
|
|
175
|
+
) -> None:
|
|
176
|
+
"""Store raw bytes under *object_key* with an explicit content type."""
|
|
177
|
+
self._client.put_object(
|
|
178
|
+
Bucket=self._bucket,
|
|
179
|
+
Key=self._full_key(object_key),
|
|
180
|
+
Body=data,
|
|
181
|
+
ContentType=content_type,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
async def get_bytes(self, object_key: str) -> bytes:
|
|
185
|
+
"""Retrieve the stored object as bytes."""
|
|
186
|
+
try:
|
|
187
|
+
response = self._client.get_object(
|
|
188
|
+
Bucket=self._bucket,
|
|
189
|
+
Key=self._full_key(object_key),
|
|
190
|
+
)
|
|
191
|
+
except ClientError as exc:
|
|
192
|
+
if self._is_missing(exc):
|
|
193
|
+
raise FileNotFoundError(object_key) from exc
|
|
194
|
+
raise
|
|
195
|
+
return response["Body"].read()
|
|
196
|
+
|
|
197
|
+
async def get_range(
|
|
198
|
+
self, object_key: str, start: int, length: int
|
|
199
|
+
) -> bytes:
|
|
200
|
+
"""Retrieve a byte range using S3 range requests."""
|
|
201
|
+
try:
|
|
202
|
+
head = self._client.head_object(
|
|
203
|
+
Bucket=self._bucket,
|
|
204
|
+
Key=self._full_key(object_key),
|
|
205
|
+
)
|
|
206
|
+
except ClientError as exc:
|
|
207
|
+
if self._is_missing(exc):
|
|
208
|
+
raise FileNotFoundError(object_key) from exc
|
|
209
|
+
raise
|
|
210
|
+
parsed_start, parsed_end = self._parse_range(
|
|
211
|
+
start, length, head["ContentLength"]
|
|
212
|
+
)
|
|
213
|
+
response = self._client.get_object(
|
|
214
|
+
Bucket=self._bucket,
|
|
215
|
+
Key=self._full_key(object_key),
|
|
216
|
+
Range=f"bytes={parsed_start}-{parsed_end - 1}",
|
|
217
|
+
)
|
|
218
|
+
return response["Body"].read()
|
|
219
|
+
|
|
220
|
+
async def remove_object(self, object_key: str) -> None:
|
|
221
|
+
"""Delete ``object_key`` when it exists."""
|
|
222
|
+
self._client.delete_object(
|
|
223
|
+
Bucket=self._bucket, Key=self._full_key(object_key)
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
@staticmethod
|
|
227
|
+
def _is_missing(exc: ClientError) -> bool:
|
|
228
|
+
code = exc.response.get("Error", {}).get("Code", "")
|
|
229
|
+
return code in {"404", "NoSuchKey", "NoSuchBucket", "NotFound"}
|
|
230
|
+
|
|
231
|
+
@classmethod
|
|
232
|
+
def from_uri(cls, uri: str) -> "S3StorageAdapter":
|
|
233
|
+
"""Instantiate the adapter from an ``s3://bucket/prefix`` URI."""
|
|
234
|
+
from urllib.parse import urlparse
|
|
235
|
+
|
|
236
|
+
parsed = urlparse(uri)
|
|
237
|
+
if parsed.scheme != "s3":
|
|
238
|
+
raise ValueError("URI must start with s3://")
|
|
239
|
+
|
|
240
|
+
return cls(
|
|
241
|
+
bucket=parsed.netloc,
|
|
242
|
+
prefix=parsed.path.lstrip("/"),
|
|
243
|
+
endpoint_url=os.getenv("AWS_ENDPOINT_URL"),
|
|
244
|
+
region_name=os.getenv("AWS_REGION"),
|
|
245
|
+
access_key=os.getenv("AWS_ACCESS_KEY_ID"),
|
|
246
|
+
secret_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
|
|
247
|
+
session_token=os.getenv("AWS_SESSION_TOKEN"),
|
|
248
|
+
)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: swarmauri_storage_s3
|
|
3
|
+
Version: 0.11.0.dev5
|
|
4
|
+
Summary: Generic S3-compatible storage adapter for SwarmauriSDK
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: swarmauri,sdk,standards,storage,s3,aws-s3,object-storage,s3-compatible
|
|
8
|
+
Author: Jacob Stewart
|
|
9
|
+
Author-email: jacob@swarmauri.com
|
|
10
|
+
Requires-Python: >=3.10,<3.13
|
|
11
|
+
Classifier: Development Status :: 1 - Planning
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Requires-Dist: boto3
|
|
21
|
+
Requires-Dist: swarmauri_base
|
|
22
|
+
Requires-Dist: swarmauri_core
|
|
23
|
+
Requires-Dist: swarmauri_standard
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+

|
|
27
|
+
|
|
28
|
+
<p align="center">
|
|
29
|
+
<a href="https://pepy.tech/project/swarmauri_storage_s3/">
|
|
30
|
+
<img src="https://static.pepy.tech/badge/swarmauri_storage_s3/month" alt="PyPI - Downloads"/></a>
|
|
31
|
+
<a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3/">
|
|
32
|
+
<img alt="Hits" src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3.svg"/></a>
|
|
33
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3/">
|
|
34
|
+
<img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue" alt="Supported Python Versions"/></a>
|
|
35
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3/">
|
|
36
|
+
<img src="https://img.shields.io/pypi/l/swarmauri_storage_s3" alt="License"/></a>
|
|
37
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3/">
|
|
38
|
+
<img src="https://img.shields.io/pypi/v/swarmauri_storage_s3?label=swarmauri_storage_s3&color=green" alt="Release Version"/></a>
|
|
39
|
+
<a href="https://discord.gg/N4UpBuQv8T">
|
|
40
|
+
<img src="https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white" alt="Discord"/></a>
|
|
41
|
+
</p>
|
|
42
|
+
|
|
43
|
+
# Swarmauri Storage S3
|
|
44
|
+
|
|
45
|
+
Generic S3-compatible storage adapter for SwarmauriSDK.
|
|
46
|
+
|
|
47
|
+
## Features
|
|
48
|
+
|
|
49
|
+
- Stores and retrieves objects through direct S3 client operations.
|
|
50
|
+
- Supports AWS S3 and S3-conformant endpoints such as MinIO, Cloudflare R2, Wasabi, Ceph, DigitalOcean Spaces, Garage, and BucketWarden when they expose the required S3 API.
|
|
51
|
+
- Accepts endpoint, region, credentials, TLS verification, path-style addressing, and client configuration options.
|
|
52
|
+
- Provides discoverable `swarmauri.storage_adapters` and `peagen.plugins.storage_adapters` entry points.
|
|
53
|
+
|
|
54
|
+
## Installation
|
|
55
|
+
|
|
56
|
+
Install this package with `uv` or `pip`.
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
uv add swarmauri_storage_s3
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install swarmauri_storage_s3
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Usage
|
|
67
|
+
|
|
68
|
+
Use this adapter when the workflow should talk to an S3-compatible object API directly.
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from swarmauri_storage_s3 import S3StorageAdapter
|
|
72
|
+
|
|
73
|
+
adapter = S3StorageAdapter.from_uri("s3://model-artifacts/runs")
|
|
74
|
+
uri = adapter.put_blob("latest/result.json", b'{"status":"ok"}')
|
|
75
|
+
payload = adapter.get_blob("latest/result.json")
|
|
76
|
+
|
|
77
|
+
print(uri)
|
|
78
|
+
print(payload)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Configure provider credentials and endpoint settings in the constructor or through environment variables owned by the surrounding Swarmauri or Tigrbl workflow.
|
|
82
|
+
|
|
83
|
+
License: Apache-2.0. See `LICENSE`.
|
|
84
|
+
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
swarmauri_storage_s3/__init__.py,sha256=l8RZELCDVsQNZ4VEWiFFAsYsrWqIgs3GBvQ2ryCF0v8,292
|
|
2
|
+
swarmauri_storage_s3/s3_storage_adapter.py,sha256=D7sUmvUJ7LkW8NA419vFWca-wOYSbwqzgOtX2iReEas,9099
|
|
3
|
+
swarmauri_storage_s3-0.11.0.dev5.dist-info/METADATA,sha256=zi81HDqOll-F8vrxbQVrUu0yArtWhs1fsQgdhv7a2Cc,3551
|
|
4
|
+
swarmauri_storage_s3-0.11.0.dev5.dist-info/WHEEL,sha256=eY7nduwzv-ldUxpzbRlxwvC693Hg6PX8bWDjEHjZ_dk,88
|
|
5
|
+
swarmauri_storage_s3-0.11.0.dev5.dist-info/entry_points.txt,sha256=Zyq3hZkjcq97o7xI8QvbrSaQhs5za0V-L8C-C-ZF0MI,199
|
|
6
|
+
swarmauri_storage_s3-0.11.0.dev5.dist-info/licenses/LICENSE,sha256=vpKDaVkCf4hDLNM1gy6oi-cH-OQ6YYAhFQ04CR7GwsQ,85
|
|
7
|
+
swarmauri_storage_s3-0.11.0.dev5.dist-info/RECORD,,
|