swarmauri_storage_s3 0.11.0.dev5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ https://www.apache.org/licenses/LICENSE-2.0
@@ -0,0 +1,84 @@
1
+ Metadata-Version: 2.4
2
+ Name: swarmauri_storage_s3
3
+ Version: 0.11.0.dev5
4
+ Summary: Generic S3-compatible storage adapter for SwarmauriSDK
5
+ License-Expression: Apache-2.0
6
+ License-File: LICENSE
7
+ Keywords: swarmauri,sdk,standards,storage,s3,aws-s3,object-storage,s3-compatible
8
+ Author: Jacob Stewart
9
+ Author-email: jacob@swarmauri.com
10
+ Requires-Python: >=3.10,<3.13
11
+ Classifier: Development Status :: 1 - Planning
12
+ Classifier: License :: OSI Approved :: Apache Software License
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Requires-Dist: boto3
21
+ Requires-Dist: swarmauri_base
22
+ Requires-Dist: swarmauri_core
23
+ Requires-Dist: swarmauri_standard
24
+ Description-Content-Type: text/markdown
25
+
26
+ ![Swarmauri Logo](https://raw.githubusercontent.com/swarmauri/swarmauri-sdk/3d4d1cfa949399d7019ae9d8f296afba773dfb7f/assets/swarmauri.brand.theme.svg)
27
+
28
+ <p align="center">
29
+ <a href="https://pepy.tech/project/swarmauri_storage_s3/">
30
+ <img src="https://static.pepy.tech/badge/swarmauri_storage_s3/month" alt="PyPI - Downloads"/></a>
31
+ <a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3/">
32
+ <img alt="Hits" src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3.svg"/></a>
33
+ <a href="https://pypi.org/project/swarmauri_storage_s3/">
34
+ <img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue" alt="Supported Python Versions"/></a>
35
+ <a href="https://pypi.org/project/swarmauri_storage_s3/">
36
+ <img src="https://img.shields.io/pypi/l/swarmauri_storage_s3" alt="License"/></a>
37
+ <a href="https://pypi.org/project/swarmauri_storage_s3/">
38
+ <img src="https://img.shields.io/pypi/v/swarmauri_storage_s3?label=swarmauri_storage_s3&color=green" alt="Release Version"/></a>
39
+ <a href="https://discord.gg/N4UpBuQv8T">
40
+ <img src="https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white" alt="Discord"/></a>
41
+ </p>
42
+
43
+ # Swarmauri Storage S3
44
+
45
+ Generic S3-compatible storage adapter for SwarmauriSDK.
46
+
47
+ ## Features
48
+
49
+ - Stores and retrieves objects through direct S3 client operations.
50
+ - Supports AWS S3 and S3-conformant endpoints such as MinIO, Cloudflare R2, Wasabi, Ceph, DigitalOcean Spaces, Garage, and BucketWarden when they expose the required S3 API.
51
+ - Accepts endpoint, region, credentials, TLS verification, path-style addressing, and client configuration options.
52
+ - Provides discoverable `swarmauri.storage_adapters` and `peagen.plugins.storage_adapters` entry points.
53
+
54
+ ## Installation
55
+
56
+ Install this package with `uv` or `pip`.
57
+
58
+ ```bash
59
+ uv add swarmauri_storage_s3
60
+ ```
61
+
62
+ ```bash
63
+ pip install swarmauri_storage_s3
64
+ ```
65
+
66
+ ## Usage
67
+
68
+ Use this adapter when the workflow should talk to an S3-compatible object API directly.
69
+
70
+ ```python
71
+ from swarmauri_storage_s3 import S3StorageAdapter
72
+
73
+ adapter = S3StorageAdapter.from_uri("s3://model-artifacts/runs")
74
+ uri = adapter.put_blob("latest/result.json", b'{"status":"ok"}')
75
+ payload = adapter.get_blob("latest/result.json")
76
+
77
+ print(uri)
78
+ print(payload)
79
+ ```
80
+
81
+ Configure provider credentials and endpoint settings in the constructor or through environment variables owned by the surrounding Swarmauri or Tigrbl workflow.
82
+
83
+ License: Apache-2.0. See `LICENSE`.
84
+
@@ -0,0 +1,58 @@
1
+ ![Swarmauri Logo](https://raw.githubusercontent.com/swarmauri/swarmauri-sdk/3d4d1cfa949399d7019ae9d8f296afba773dfb7f/assets/swarmauri.brand.theme.svg)
2
+
3
+ <p align="center">
4
+ <a href="https://pepy.tech/project/swarmauri_storage_s3/">
5
+ <img src="https://static.pepy.tech/badge/swarmauri_storage_s3/month" alt="PyPI - Downloads"/></a>
6
+ <a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3/">
7
+ <img alt="Hits" src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3.svg"/></a>
8
+ <a href="https://pypi.org/project/swarmauri_storage_s3/">
9
+ <img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue" alt="Supported Python Versions"/></a>
10
+ <a href="https://pypi.org/project/swarmauri_storage_s3/">
11
+ <img src="https://img.shields.io/pypi/l/swarmauri_storage_s3" alt="License"/></a>
12
+ <a href="https://pypi.org/project/swarmauri_storage_s3/">
13
+ <img src="https://img.shields.io/pypi/v/swarmauri_storage_s3?label=swarmauri_storage_s3&color=green" alt="Release Version"/></a>
14
+ <a href="https://discord.gg/N4UpBuQv8T">
15
+ <img src="https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white" alt="Discord"/></a>
16
+ </p>
17
+
18
+ # Swarmauri Storage S3
19
+
20
+ Generic S3-compatible storage adapter for SwarmauriSDK.
21
+
22
+ ## Features
23
+
24
+ - Stores and retrieves objects through direct S3 client operations.
25
+ - Supports AWS S3 and S3-conformant endpoints such as MinIO, Cloudflare R2, Wasabi, Ceph, DigitalOcean Spaces, Garage, and BucketWarden when they expose the required S3 API.
26
+ - Accepts endpoint, region, credentials, TLS verification, path-style addressing, and client configuration options.
27
+ - Provides discoverable `swarmauri.storage_adapters` and `peagen.plugins.storage_adapters` entry points.
28
+
29
+ ## Installation
30
+
31
+ Install this package with `uv` or `pip`.
32
+
33
+ ```bash
34
+ uv add swarmauri_storage_s3
35
+ ```
36
+
37
+ ```bash
38
+ pip install swarmauri_storage_s3
39
+ ```
40
+
41
+ ## Usage
42
+
43
+ Use this adapter when the workflow should talk to an S3-compatible object API directly.
44
+
45
+ ```python
46
+ from swarmauri_storage_s3 import S3StorageAdapter
47
+
48
+ adapter = S3StorageAdapter.from_uri("s3://model-artifacts/runs")
49
+ uri = adapter.put_blob("latest/result.json", b'{"status":"ok"}')
50
+ payload = adapter.get_blob("latest/result.json")
51
+
52
+ print(uri)
53
+ print(payload)
54
+ ```
55
+
56
+ Configure provider credentials and endpoint settings in the constructor or through environment variables owned by the surrounding Swarmauri or Tigrbl workflow.
57
+
58
+ License: Apache-2.0. See `LICENSE`.
@@ -0,0 +1,87 @@
1
+ [project]
2
+ name = "swarmauri_storage_s3"
3
+ version = "0.11.0.dev5"
4
+ authors = [{ name = "Jacob Stewart", email = "jacob@swarmauri.com" }]
5
+ description = "Generic S3-compatible storage adapter for SwarmauriSDK"
6
+ readme = "README.md"
7
+ license = "Apache-2.0"
8
+ repository = "http://github.com/swarmauri/swarmauri-sdk"
9
+ requires-python = ">=3.10,<3.13"
10
+ classifiers = [
11
+ "Development Status :: 1 - Planning",
12
+ "License :: OSI Approved :: Apache Software License",
13
+ "Operating System :: OS Independent",
14
+ "Programming Language :: Python",
15
+ "Programming Language :: Python :: 3",
16
+ "Programming Language :: Python :: 3 :: Only",
17
+ "Programming Language :: Python :: 3.10",
18
+ "Programming Language :: Python :: 3.11",
19
+ "Programming Language :: Python :: 3.12",
20
+ ]
21
+ dependencies = [
22
+ "boto3",
23
+ "swarmauri_core",
24
+ "swarmauri_base",
25
+ "swarmauri_standard",
26
+ ]
27
+ keywords = [
28
+ "swarmauri",
29
+ "sdk",
30
+ "standards",
31
+ "storage",
32
+ "s3",
33
+ "aws-s3",
34
+ "object-storage",
35
+ "s3-compatible",
36
+ ]
37
+
38
+ [tool.uv.sources]
39
+ swarmauri_core = { workspace = true }
40
+ swarmauri_base = { workspace = true }
41
+ swarmauri_standard = { workspace = true }
42
+
43
+ [project.entry-points."swarmauri.storage_adapters"]
44
+ S3StorageAdapter = "swarmauri_storage_s3.s3_storage_adapter:S3StorageAdapter"
45
+
46
+ [project.entry-points."peagen.plugins.storage_adapters"]
47
+ s3 = "swarmauri_storage_s3.s3_storage_adapter:S3StorageAdapter"
48
+
49
+ [tool.pytest.ini_options]
50
+ addopts = "--basetemp=.pytest-tmp"
51
+ norecursedirs = [".uv-cache", "combined", "scripts"]
52
+ markers = [
53
+ "test: standard test",
54
+ "unit: Unit tests",
55
+ "i9n: Integration tests",
56
+ "r8n: Regression tests",
57
+ "timeout: mark test to timeout after X seconds",
58
+ "xpass: Expected passes",
59
+ "xfail: Expected failures",
60
+ "acceptance: Acceptance tests",
61
+ "perf: Performance tests that measure execution time and resource usage",
62
+ "example: Example usage tests",
63
+ ]
64
+ timeout = 300
65
+ log_cli = true
66
+ log_cli_level = "INFO"
67
+ log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
68
+ log_cli_date_format = "%Y-%m-%d %H:%M:%S"
69
+ asyncio_default_fixture_loop_scope = "function"
70
+
71
+ [build-system]
72
+ requires = ["poetry-core>=1.0.0"]
73
+ build-backend = "poetry.core.masonry.api"
74
+
75
+ [dependency-groups]
76
+ dev = [
77
+ "pytest>=8.0",
78
+ "pytest-asyncio>=0.24.0",
79
+ "pytest-xdist>=3.6.1",
80
+ "pytest-json-report>=1.5.0",
81
+ "python-dotenv",
82
+ "httpx>=0.27",
83
+ "flake8>=7.0",
84
+ "pytest-timeout>=2.3.1",
85
+ "ruff>=0.9.9",
86
+ "pytest-benchmark>=4.0.0",
87
+ ]
@@ -0,0 +1,8 @@
1
+ try:
2
+ from importlib.metadata import PackageNotFoundError, version
3
+ except ImportError: # pragma: no cover
4
+ from importlib_metadata import PackageNotFoundError, version
5
+
6
+ from .s3_storage_adapter import S3StorageAdapter
7
+
8
+ __all__ = ["PackageNotFoundError", "S3StorageAdapter", "version"]
@@ -0,0 +1,248 @@
1
+ """Generic S3-compatible storage adapter using direct S3 client semantics."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import io
6
+ import os
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import BinaryIO, Literal
10
+
11
+ import boto3
12
+ from botocore.config import Config
13
+ from botocore.exceptions import ClientError
14
+ from pydantic import SecretStr
15
+
16
+ from swarmauri_base.ComponentBase import ComponentBase
17
+ from swarmauri_base.storage import StorageAdapterBase
18
+
19
+
20
+ @ComponentBase.register_type(StorageAdapterBase, "S3StorageAdapter")
21
+ class S3StorageAdapter(StorageAdapterBase):
22
+ """Store and retrieve artifacts through an S3-compatible API endpoint."""
23
+
24
+ type: Literal["S3StorageAdapter"] = "S3StorageAdapter"
25
+
26
+ def __init__(
27
+ self,
28
+ bucket: str,
29
+ *,
30
+ endpoint_url: str | None = None,
31
+ region_name: str | None = None,
32
+ access_key: SecretStr | str | None = None,
33
+ secret_key: SecretStr | str | None = None,
34
+ session_token: SecretStr | str | None = None,
35
+ use_ssl: bool = True,
36
+ verify: bool | str | None = None,
37
+ addressing_style: str | None = None,
38
+ prefix: str = "",
39
+ client_kwargs: dict | None = None,
40
+ config_kwargs: dict | None = None,
41
+ **kwargs,
42
+ ) -> None:
43
+ super().__init__(**kwargs)
44
+ self._bucket = bucket
45
+ self._prefix = self.normalize_prefix(prefix)
46
+
47
+ config_options = dict(config_kwargs or {})
48
+ if addressing_style:
49
+ config_options.setdefault("s3", {})["addressing_style"] = (
50
+ addressing_style
51
+ )
52
+
53
+ options = dict(client_kwargs or {})
54
+ options.update(
55
+ {
56
+ "service_name": "s3",
57
+ "endpoint_url": endpoint_url,
58
+ "region_name": region_name,
59
+ "aws_access_key_id": self._secret_value(access_key),
60
+ "aws_secret_access_key": self._secret_value(secret_key),
61
+ "aws_session_token": self._secret_value(session_token),
62
+ "use_ssl": use_ssl,
63
+ "verify": verify,
64
+ }
65
+ )
66
+ options = {
67
+ key: value for key, value in options.items() if value is not None
68
+ }
69
+ if config_options:
70
+ options["config"] = Config(**config_options)
71
+
72
+ self._client = boto3.client(**options)
73
+
74
+ @staticmethod
75
+ def _secret_value(value: SecretStr | str | None) -> str | None:
76
+ if isinstance(value, SecretStr):
77
+ return value.get_secret_value() or None
78
+ return value or None
79
+
80
+ def _full_key(self, key: str) -> str:
81
+ return self.compose_key(self._prefix, key, allow_empty=True)
82
+
83
+ def _relative_key(self, key: str) -> str:
84
+ if self._prefix and key.startswith(f"{self._prefix.rstrip('/')}/"):
85
+ return key[len(self._prefix.rstrip("/")) + 1 :]
86
+ return key
87
+
88
+ @property
89
+ def root_uri(self) -> str:
90
+ """Return the base ``s3://`` URI for this adapter."""
91
+ base = f"s3://{self._bucket}"
92
+ uri = f"{base}/{self._prefix.rstrip('/')}" if self._prefix else base
93
+ return uri.rstrip("/") + "/"
94
+
95
+ def upload(self, key: str, data: BinaryIO) -> str:
96
+ """Upload *data* under *key* and return its ``s3://`` URI."""
97
+ normalized_key = self.normalize_key(key)
98
+ self._client.upload_fileobj(data, self._bucket, self._full_key(key))
99
+ return f"{self.root_uri}{normalized_key}"
100
+
101
+ def download(self, key: str) -> BinaryIO:
102
+ """Download *key* into a binary in-memory stream."""
103
+ buffer = io.BytesIO()
104
+ try:
105
+ self._client.download_fileobj(
106
+ self._bucket, self._full_key(key), buffer
107
+ )
108
+ except ClientError as exc:
109
+ if self._is_missing(exc):
110
+ raise FileNotFoundError(key) from exc
111
+ raise
112
+ buffer.seek(0)
113
+ return buffer
114
+
115
+ def upload_dir(self, src: str | os.PathLike, *, prefix: str = "") -> None:
116
+ """Recursively upload files from *src* under ``prefix``."""
117
+ base = Path(src)
118
+ for path in base.rglob("*"):
119
+ if path.is_file():
120
+ rel = path.relative_to(base).as_posix()
121
+ key = self.compose_key(prefix, rel)
122
+ with path.open("rb") as handle:
123
+ self.upload(key, handle)
124
+
125
+ def iter_prefix(self, prefix: str):
126
+ """Yield stored keys below ``prefix`` relative to the adapter root."""
127
+ continuation_token = None
128
+ list_prefix = self._full_key(self.normalize_prefix(prefix))
129
+ while True:
130
+ request = {
131
+ "Bucket": self._bucket,
132
+ "Prefix": list_prefix,
133
+ }
134
+ if continuation_token:
135
+ request["ContinuationToken"] = continuation_token
136
+ response = self._client.list_objects_v2(**request)
137
+ for item in response.get("Contents", []):
138
+ rel = self._relative_key(item["Key"])
139
+ if rel:
140
+ yield rel
141
+ if not response.get("IsTruncated"):
142
+ break
143
+ continuation_token = response.get("NextContinuationToken")
144
+
145
+ def download_dir(self, prefix: str, dest_dir: str | os.PathLike) -> None:
146
+ """Download all stored artifacts under ``prefix`` into ``dest_dir``."""
147
+ dest = Path(dest_dir)
148
+ normalized_prefix = self.normalize_prefix(prefix)
149
+ for rel_key in self.iter_prefix(prefix):
150
+ target_rel = rel_key
151
+ if normalized_prefix and rel_key.startswith(
152
+ f"{normalized_prefix}/"
153
+ ):
154
+ target_rel = rel_key[len(normalized_prefix) + 1 :]
155
+ if not target_rel:
156
+ continue
157
+ target = self.download_target_for_key(dest, target_rel)
158
+ target.parent.mkdir(parents=True, exist_ok=True)
159
+ data = self.download(rel_key)
160
+ with target.open("wb") as handle:
161
+ shutil.copyfileobj(data, handle)
162
+
163
+ async def ensure_bucket(self) -> None:
164
+ """Create the configured bucket if it does not already exist."""
165
+ try:
166
+ self._client.head_bucket(Bucket=self._bucket)
167
+ except ClientError as exc:
168
+ if self._is_missing(exc):
169
+ self._client.create_bucket(Bucket=self._bucket)
170
+ return
171
+ raise
172
+
173
+ async def put_bytes(
174
+ self, object_key: str, data: bytes, content_type: str
175
+ ) -> None:
176
+ """Store raw bytes under *object_key* with an explicit content type."""
177
+ self._client.put_object(
178
+ Bucket=self._bucket,
179
+ Key=self._full_key(object_key),
180
+ Body=data,
181
+ ContentType=content_type,
182
+ )
183
+
184
+ async def get_bytes(self, object_key: str) -> bytes:
185
+ """Retrieve the stored object as bytes."""
186
+ try:
187
+ response = self._client.get_object(
188
+ Bucket=self._bucket,
189
+ Key=self._full_key(object_key),
190
+ )
191
+ except ClientError as exc:
192
+ if self._is_missing(exc):
193
+ raise FileNotFoundError(object_key) from exc
194
+ raise
195
+ return response["Body"].read()
196
+
197
+ async def get_range(
198
+ self, object_key: str, start: int, length: int
199
+ ) -> bytes:
200
+ """Retrieve a byte range using S3 range requests."""
201
+ try:
202
+ head = self._client.head_object(
203
+ Bucket=self._bucket,
204
+ Key=self._full_key(object_key),
205
+ )
206
+ except ClientError as exc:
207
+ if self._is_missing(exc):
208
+ raise FileNotFoundError(object_key) from exc
209
+ raise
210
+ parsed_start, parsed_end = self._parse_range(
211
+ start, length, head["ContentLength"]
212
+ )
213
+ response = self._client.get_object(
214
+ Bucket=self._bucket,
215
+ Key=self._full_key(object_key),
216
+ Range=f"bytes={parsed_start}-{parsed_end - 1}",
217
+ )
218
+ return response["Body"].read()
219
+
220
+ async def remove_object(self, object_key: str) -> None:
221
+ """Delete ``object_key`` when it exists."""
222
+ self._client.delete_object(
223
+ Bucket=self._bucket, Key=self._full_key(object_key)
224
+ )
225
+
226
+ @staticmethod
227
+ def _is_missing(exc: ClientError) -> bool:
228
+ code = exc.response.get("Error", {}).get("Code", "")
229
+ return code in {"404", "NoSuchKey", "NoSuchBucket", "NotFound"}
230
+
231
+ @classmethod
232
+ def from_uri(cls, uri: str) -> "S3StorageAdapter":
233
+ """Instantiate the adapter from an ``s3://bucket/prefix`` URI."""
234
+ from urllib.parse import urlparse
235
+
236
+ parsed = urlparse(uri)
237
+ if parsed.scheme != "s3":
238
+ raise ValueError("URI must start with s3://")
239
+
240
+ return cls(
241
+ bucket=parsed.netloc,
242
+ prefix=parsed.path.lstrip("/"),
243
+ endpoint_url=os.getenv("AWS_ENDPOINT_URL"),
244
+ region_name=os.getenv("AWS_REGION"),
245
+ access_key=os.getenv("AWS_ACCESS_KEY_ID"),
246
+ secret_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
247
+ session_token=os.getenv("AWS_SESSION_TOKEN"),
248
+ )