swarmauri_storage_s3fs 0.11.0.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swarmauri_storage_s3fs/__init__.py +8 -0
- swarmauri_storage_s3fs/s3fs_storage_adapter.py +176 -0
- swarmauri_storage_s3fs-0.11.0.dev3.dist-info/METADATA +84 -0
- swarmauri_storage_s3fs-0.11.0.dev3.dist-info/RECORD +7 -0
- swarmauri_storage_s3fs-0.11.0.dev3.dist-info/WHEEL +4 -0
- swarmauri_storage_s3fs-0.11.0.dev3.dist-info/entry_points.txt +6 -0
- swarmauri_storage_s3fs-0.11.0.dev3.dist-info/licenses/LICENSE +3 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
try:
|
|
2
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
3
|
+
except ImportError: # pragma: no cover
|
|
4
|
+
from importlib_metadata import PackageNotFoundError, version
|
|
5
|
+
|
|
6
|
+
from .s3fs_storage_adapter import S3FSStorageAdapter
|
|
7
|
+
|
|
8
|
+
__all__ = ["PackageNotFoundError", "S3FSStorageAdapter", "version"]
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""S3 storage adapter backed strictly by the :mod:`s3fs` filesystem API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import BinaryIO, Literal
|
|
9
|
+
|
|
10
|
+
import s3fs
|
|
11
|
+
from pydantic import SecretStr
|
|
12
|
+
|
|
13
|
+
from swarmauri_base.ComponentBase import ComponentBase
|
|
14
|
+
from swarmauri_base.storage import StorageAdapterBase
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@ComponentBase.register_type(StorageAdapterBase, "S3FSStorageAdapter")
|
|
18
|
+
class S3FSStorageAdapter(StorageAdapterBase):
|
|
19
|
+
"""Store and retrieve artifacts from S3 through ``s3fs.S3FileSystem``."""
|
|
20
|
+
|
|
21
|
+
type: Literal["S3FSStorageAdapter"] = "S3FSStorageAdapter"
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
bucket: str,
|
|
26
|
+
*,
|
|
27
|
+
key: SecretStr | str | None = None,
|
|
28
|
+
secret: SecretStr | str | None = None,
|
|
29
|
+
token: SecretStr | str | None = None,
|
|
30
|
+
endpoint_url: str | None = None,
|
|
31
|
+
region_name: str | None = None,
|
|
32
|
+
use_ssl: bool = True,
|
|
33
|
+
anon: bool = False,
|
|
34
|
+
prefix: str = "",
|
|
35
|
+
client_kwargs: dict | None = None,
|
|
36
|
+
config_kwargs: dict | None = None,
|
|
37
|
+
s3_additional_kwargs: dict | None = None,
|
|
38
|
+
**kwargs,
|
|
39
|
+
) -> None:
|
|
40
|
+
super().__init__(**kwargs)
|
|
41
|
+
self._bucket = bucket
|
|
42
|
+
self._prefix = self.normalize_prefix(prefix)
|
|
43
|
+
|
|
44
|
+
fs_client_kwargs = dict(client_kwargs or {})
|
|
45
|
+
if endpoint_url:
|
|
46
|
+
fs_client_kwargs["endpoint_url"] = endpoint_url
|
|
47
|
+
if region_name:
|
|
48
|
+
fs_client_kwargs["region_name"] = region_name
|
|
49
|
+
|
|
50
|
+
self._fs = s3fs.S3FileSystem(
|
|
51
|
+
key=self._secret_value(key),
|
|
52
|
+
secret=self._secret_value(secret),
|
|
53
|
+
token=self._secret_value(token),
|
|
54
|
+
endpoint_url=endpoint_url,
|
|
55
|
+
use_ssl=use_ssl,
|
|
56
|
+
anon=anon,
|
|
57
|
+
client_kwargs=fs_client_kwargs or None,
|
|
58
|
+
config_kwargs=config_kwargs,
|
|
59
|
+
s3_additional_kwargs=s3_additional_kwargs,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
@staticmethod
|
|
63
|
+
def _secret_value(value: SecretStr | str | None) -> str | None:
|
|
64
|
+
if isinstance(value, SecretStr):
|
|
65
|
+
return value.get_secret_value() or None
|
|
66
|
+
return value or None
|
|
67
|
+
|
|
68
|
+
def _full_key(self, key: str) -> str:
|
|
69
|
+
key = self.compose_key(self._prefix, key, allow_empty=True)
|
|
70
|
+
return f"{self._bucket}/{key}" if key else self._bucket
|
|
71
|
+
|
|
72
|
+
def _relative_key(self, path: str) -> str:
|
|
73
|
+
rel = path.removeprefix(f"{self._bucket}/")
|
|
74
|
+
if self._prefix and rel.startswith(f"{self._prefix.rstrip('/')}/"):
|
|
75
|
+
rel = rel[len(self._prefix.rstrip("/")) + 1 :]
|
|
76
|
+
return rel
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def root_uri(self) -> str:
|
|
80
|
+
"""Return the base ``s3://`` URI for this adapter."""
|
|
81
|
+
base = f"s3://{self._bucket}"
|
|
82
|
+
uri = f"{base}/{self._prefix.rstrip('/')}" if self._prefix else base
|
|
83
|
+
return uri.rstrip("/") + "/"
|
|
84
|
+
|
|
85
|
+
def upload(self, key: str, data: BinaryIO) -> str:
|
|
86
|
+
"""Upload *data* under *key* and return its ``s3://`` URI."""
|
|
87
|
+
normalized_key = self.normalize_key(key)
|
|
88
|
+
with self._fs.open(self._full_key(key), "wb") as handle:
|
|
89
|
+
shutil.copyfileobj(data, handle)
|
|
90
|
+
return f"{self.root_uri}{normalized_key}"
|
|
91
|
+
|
|
92
|
+
def download(self, key: str) -> BinaryIO:
|
|
93
|
+
"""Open *key* for binary reads."""
|
|
94
|
+
try:
|
|
95
|
+
return self._fs.open(self._full_key(key), "rb")
|
|
96
|
+
except FileNotFoundError:
|
|
97
|
+
raise
|
|
98
|
+
except OSError as exc:
|
|
99
|
+
raise FileNotFoundError(key) from exc
|
|
100
|
+
|
|
101
|
+
def upload_dir(self, src: str | os.PathLike, *, prefix: str = "") -> None:
|
|
102
|
+
"""Recursively upload files from *src* under ``prefix``."""
|
|
103
|
+
base = Path(src)
|
|
104
|
+
for path in base.rglob("*"):
|
|
105
|
+
if path.is_file():
|
|
106
|
+
rel = path.relative_to(base).as_posix()
|
|
107
|
+
key = self.compose_key(prefix, rel)
|
|
108
|
+
with path.open("rb") as handle:
|
|
109
|
+
self.upload(key, handle)
|
|
110
|
+
|
|
111
|
+
def iter_prefix(self, prefix: str):
|
|
112
|
+
"""Yield stored keys below ``prefix`` relative to the adapter root."""
|
|
113
|
+
for path in self._fs.find(self._full_key(prefix)):
|
|
114
|
+
rel = self._relative_key(path)
|
|
115
|
+
if rel:
|
|
116
|
+
yield rel
|
|
117
|
+
|
|
118
|
+
def download_dir(self, prefix: str, dest_dir: str | os.PathLike) -> None:
|
|
119
|
+
"""Download all stored artifacts under ``prefix`` into ``dest_dir``."""
|
|
120
|
+
dest = Path(dest_dir)
|
|
121
|
+
normalized_prefix = self.normalize_prefix(prefix)
|
|
122
|
+
for rel_key in self.iter_prefix(prefix):
|
|
123
|
+
target_rel = rel_key
|
|
124
|
+
if normalized_prefix and rel_key.startswith(
|
|
125
|
+
f"{normalized_prefix}/"
|
|
126
|
+
):
|
|
127
|
+
target_rel = rel_key[len(normalized_prefix) + 1 :]
|
|
128
|
+
if not target_rel:
|
|
129
|
+
continue
|
|
130
|
+
target = self.download_target_for_key(dest, target_rel)
|
|
131
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
132
|
+
data = self.download(rel_key)
|
|
133
|
+
with target.open("wb") as handle:
|
|
134
|
+
shutil.copyfileobj(data, handle)
|
|
135
|
+
|
|
136
|
+
async def ensure_bucket(self) -> None:
|
|
137
|
+
"""Create the configured bucket if it does not already exist."""
|
|
138
|
+
if not self._fs.exists(self._bucket):
|
|
139
|
+
self._fs.mkdir(self._bucket)
|
|
140
|
+
|
|
141
|
+
async def get_range(
|
|
142
|
+
self, object_key: str, start: int, length: int
|
|
143
|
+
) -> bytes:
|
|
144
|
+
"""Retrieve a byte range using the ``s3fs`` file object."""
|
|
145
|
+
info = self._fs.info(self._full_key(object_key))
|
|
146
|
+
parsed_start, parsed_end = self._parse_range(
|
|
147
|
+
start, length, info["size"]
|
|
148
|
+
)
|
|
149
|
+
with self._fs.open(self._full_key(object_key), "rb") as handle:
|
|
150
|
+
handle.seek(parsed_start)
|
|
151
|
+
return handle.read(parsed_end - parsed_start)
|
|
152
|
+
|
|
153
|
+
async def remove_object(self, object_key: str) -> None:
|
|
154
|
+
"""Delete ``object_key`` when it exists."""
|
|
155
|
+
path = self._full_key(object_key)
|
|
156
|
+
if self._fs.exists(path):
|
|
157
|
+
self._fs.rm(path)
|
|
158
|
+
|
|
159
|
+
@classmethod
|
|
160
|
+
def from_uri(cls, uri: str) -> "S3FSStorageAdapter":
|
|
161
|
+
"""Instantiate the adapter from an ``s3://bucket/prefix`` URI."""
|
|
162
|
+
from urllib.parse import urlparse
|
|
163
|
+
|
|
164
|
+
parsed = urlparse(uri)
|
|
165
|
+
if parsed.scheme != "s3":
|
|
166
|
+
raise ValueError("URI must start with s3://")
|
|
167
|
+
|
|
168
|
+
return cls(
|
|
169
|
+
bucket=parsed.netloc,
|
|
170
|
+
prefix=parsed.path.lstrip("/"),
|
|
171
|
+
key=os.getenv("AWS_ACCESS_KEY_ID"),
|
|
172
|
+
secret=os.getenv("AWS_SECRET_ACCESS_KEY"),
|
|
173
|
+
token=os.getenv("AWS_SESSION_TOKEN"),
|
|
174
|
+
endpoint_url=os.getenv("AWS_ENDPOINT_URL"),
|
|
175
|
+
region_name=os.getenv("AWS_REGION"),
|
|
176
|
+
)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: swarmauri_storage_s3fs
|
|
3
|
+
Version: 0.11.0.dev3
|
|
4
|
+
Summary: s3fs-backed S3 storage adapter for SwarmauriSDK
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: swarmauri,sdk,standards,storage,s3,s3fs,fsspec,object-storage
|
|
8
|
+
Author: Jacob Stewart
|
|
9
|
+
Author-email: jacob@swarmauri.com
|
|
10
|
+
Requires-Python: >=3.10,<3.13
|
|
11
|
+
Classifier: Development Status :: 1 - Planning
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Requires-Dist: s3fs
|
|
21
|
+
Requires-Dist: swarmauri_base
|
|
22
|
+
Requires-Dist: swarmauri_core
|
|
23
|
+
Requires-Dist: swarmauri_standard
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+

|
|
27
|
+
|
|
28
|
+
<p align="center">
|
|
29
|
+
<a href="https://pepy.tech/project/swarmauri_storage_s3fs/">
|
|
30
|
+
<img src="https://static.pepy.tech/badge/swarmauri_storage_s3fs/month" alt="PyPI - Downloads"/></a>
|
|
31
|
+
<a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3fs/">
|
|
32
|
+
<img alt="Hits" src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3fs.svg"/></a>
|
|
33
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3fs/">
|
|
34
|
+
<img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue" alt="Supported Python Versions"/></a>
|
|
35
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3fs/">
|
|
36
|
+
<img src="https://img.shields.io/pypi/l/swarmauri_storage_s3fs" alt="License"/></a>
|
|
37
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3fs/">
|
|
38
|
+
<img src="https://img.shields.io/pypi/v/swarmauri_storage_s3fs?label=swarmauri_storage_s3fs&color=green" alt="Release Version"/></a>
|
|
39
|
+
<a href="https://discord.gg/N4UpBuQv8T">
|
|
40
|
+
<img src="https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white" alt="Discord"/></a>
|
|
41
|
+
</p>
|
|
42
|
+
|
|
43
|
+
# Swarmauri Storage S3FS
|
|
44
|
+
|
|
45
|
+
Filesystem-style S3 storage adapter for SwarmauriSDK backed strictly by `s3fs`.
|
|
46
|
+
|
|
47
|
+
## Features
|
|
48
|
+
|
|
49
|
+
- Stores and retrieves objects from `s3://` locations through `s3fs.S3FileSystem`.
|
|
50
|
+
- Supports S3-compatible endpoints through `endpoint_url` and `client_kwargs`.
|
|
51
|
+
- Provides discoverable `swarmauri.storage_adapters` and `peagen.plugins.storage_adapters` entry points.
|
|
52
|
+
- Handles object upload, download, range reads, prefix iteration, directory push/pull, bucket creation, and object removal.
|
|
53
|
+
|
|
54
|
+
## Installation
|
|
55
|
+
|
|
56
|
+
Install this package with `uv` or `pip`.
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
uv add swarmauri_storage_s3fs
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install swarmauri_storage_s3fs
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Usage
|
|
67
|
+
|
|
68
|
+
Use this adapter when the workflow should use the `s3fs` filesystem API.
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from swarmauri_storage_s3fs import S3FSStorageAdapter
|
|
72
|
+
|
|
73
|
+
adapter = S3FSStorageAdapter.from_uri("s3://model-artifacts/runs")
|
|
74
|
+
uri = adapter.put_blob("latest/result.json", b'{"status":"ok"}')
|
|
75
|
+
payload = adapter.get_blob("latest/result.json")
|
|
76
|
+
|
|
77
|
+
print(uri)
|
|
78
|
+
print(payload)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Configure credentials through `s3fs`, environment variables, or explicit constructor arguments owned by the surrounding Swarmauri or Tigrbl workflow.
|
|
82
|
+
|
|
83
|
+
License: Apache-2.0. See `LICENSE`.
|
|
84
|
+
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
swarmauri_storage_s3fs/__init__.py,sha256=osULRLUAJcbplliYT2mI0MMCsBZhO2MmFZq1TdPieZk,298
|
|
2
|
+
swarmauri_storage_s3fs/s3fs_storage_adapter.py,sha256=RFWgldOKu6hxc2Ezs_NEM7QBglJP7-hjFiMWo81UoNo,6578
|
|
3
|
+
swarmauri_storage_s3fs-0.11.0.dev3.dist-info/METADATA,sha256=evSoUCEf0uz4bou9EDAowXuzyTug8hqDbrVvbr1eNuQ,3495
|
|
4
|
+
swarmauri_storage_s3fs-0.11.0.dev3.dist-info/WHEEL,sha256=eY7nduwzv-ldUxpzbRlxwvC693Hg6PX8bWDjEHjZ_dk,88
|
|
5
|
+
swarmauri_storage_s3fs-0.11.0.dev3.dist-info/entry_points.txt,sha256=oNefM3GYps3ObLe40gy_3Xs1BXinDw76mgNP3t528Dc,215
|
|
6
|
+
swarmauri_storage_s3fs-0.11.0.dev3.dist-info/licenses/LICENSE,sha256=vpKDaVkCf4hDLNM1gy6oi-cH-OQ6YYAhFQ04CR7GwsQ,85
|
|
7
|
+
swarmauri_storage_s3fs-0.11.0.dev3.dist-info/RECORD,,
|