swarmauri_storage_s3fs 0.11.0.dev3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,8 @@
1
+ try:
2
+ from importlib.metadata import PackageNotFoundError, version
3
+ except ImportError: # pragma: no cover
4
+ from importlib_metadata import PackageNotFoundError, version
5
+
6
+ from .s3fs_storage_adapter import S3FSStorageAdapter
7
+
8
+ __all__ = ["PackageNotFoundError", "S3FSStorageAdapter", "version"]
@@ -0,0 +1,176 @@
1
+ """S3 storage adapter backed strictly by the :mod:`s3fs` filesystem API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import shutil
7
+ from pathlib import Path
8
+ from typing import BinaryIO, Literal
9
+
10
+ import s3fs
11
+ from pydantic import SecretStr
12
+
13
+ from swarmauri_base.ComponentBase import ComponentBase
14
+ from swarmauri_base.storage import StorageAdapterBase
15
+
16
+
17
+ @ComponentBase.register_type(StorageAdapterBase, "S3FSStorageAdapter")
18
+ class S3FSStorageAdapter(StorageAdapterBase):
19
+ """Store and retrieve artifacts from S3 through ``s3fs.S3FileSystem``."""
20
+
21
+ type: Literal["S3FSStorageAdapter"] = "S3FSStorageAdapter"
22
+
23
+ def __init__(
24
+ self,
25
+ bucket: str,
26
+ *,
27
+ key: SecretStr | str | None = None,
28
+ secret: SecretStr | str | None = None,
29
+ token: SecretStr | str | None = None,
30
+ endpoint_url: str | None = None,
31
+ region_name: str | None = None,
32
+ use_ssl: bool = True,
33
+ anon: bool = False,
34
+ prefix: str = "",
35
+ client_kwargs: dict | None = None,
36
+ config_kwargs: dict | None = None,
37
+ s3_additional_kwargs: dict | None = None,
38
+ **kwargs,
39
+ ) -> None:
40
+ super().__init__(**kwargs)
41
+ self._bucket = bucket
42
+ self._prefix = self.normalize_prefix(prefix)
43
+
44
+ fs_client_kwargs = dict(client_kwargs or {})
45
+ if endpoint_url:
46
+ fs_client_kwargs["endpoint_url"] = endpoint_url
47
+ if region_name:
48
+ fs_client_kwargs["region_name"] = region_name
49
+
50
+ self._fs = s3fs.S3FileSystem(
51
+ key=self._secret_value(key),
52
+ secret=self._secret_value(secret),
53
+ token=self._secret_value(token),
54
+ endpoint_url=endpoint_url,
55
+ use_ssl=use_ssl,
56
+ anon=anon,
57
+ client_kwargs=fs_client_kwargs or None,
58
+ config_kwargs=config_kwargs,
59
+ s3_additional_kwargs=s3_additional_kwargs,
60
+ )
61
+
62
+ @staticmethod
63
+ def _secret_value(value: SecretStr | str | None) -> str | None:
64
+ if isinstance(value, SecretStr):
65
+ return value.get_secret_value() or None
66
+ return value or None
67
+
68
+ def _full_key(self, key: str) -> str:
69
+ key = self.compose_key(self._prefix, key, allow_empty=True)
70
+ return f"{self._bucket}/{key}" if key else self._bucket
71
+
72
+ def _relative_key(self, path: str) -> str:
73
+ rel = path.removeprefix(f"{self._bucket}/")
74
+ if self._prefix and rel.startswith(f"{self._prefix.rstrip('/')}/"):
75
+ rel = rel[len(self._prefix.rstrip("/")) + 1 :]
76
+ return rel
77
+
78
+ @property
79
+ def root_uri(self) -> str:
80
+ """Return the base ``s3://`` URI for this adapter."""
81
+ base = f"s3://{self._bucket}"
82
+ uri = f"{base}/{self._prefix.rstrip('/')}" if self._prefix else base
83
+ return uri.rstrip("/") + "/"
84
+
85
+ def upload(self, key: str, data: BinaryIO) -> str:
86
+ """Upload *data* under *key* and return its ``s3://`` URI."""
87
+ normalized_key = self.normalize_key(key)
88
+ with self._fs.open(self._full_key(key), "wb") as handle:
89
+ shutil.copyfileobj(data, handle)
90
+ return f"{self.root_uri}{normalized_key}"
91
+
92
+ def download(self, key: str) -> BinaryIO:
93
+ """Open *key* for binary reads."""
94
+ try:
95
+ return self._fs.open(self._full_key(key), "rb")
96
+ except FileNotFoundError:
97
+ raise
98
+ except OSError as exc:
99
+ raise FileNotFoundError(key) from exc
100
+
101
+ def upload_dir(self, src: str | os.PathLike, *, prefix: str = "") -> None:
102
+ """Recursively upload files from *src* under ``prefix``."""
103
+ base = Path(src)
104
+ for path in base.rglob("*"):
105
+ if path.is_file():
106
+ rel = path.relative_to(base).as_posix()
107
+ key = self.compose_key(prefix, rel)
108
+ with path.open("rb") as handle:
109
+ self.upload(key, handle)
110
+
111
+ def iter_prefix(self, prefix: str):
112
+ """Yield stored keys below ``prefix`` relative to the adapter root."""
113
+ for path in self._fs.find(self._full_key(prefix)):
114
+ rel = self._relative_key(path)
115
+ if rel:
116
+ yield rel
117
+
118
+ def download_dir(self, prefix: str, dest_dir: str | os.PathLike) -> None:
119
+ """Download all stored artifacts under ``prefix`` into ``dest_dir``."""
120
+ dest = Path(dest_dir)
121
+ normalized_prefix = self.normalize_prefix(prefix)
122
+ for rel_key in self.iter_prefix(prefix):
123
+ target_rel = rel_key
124
+ if normalized_prefix and rel_key.startswith(
125
+ f"{normalized_prefix}/"
126
+ ):
127
+ target_rel = rel_key[len(normalized_prefix) + 1 :]
128
+ if not target_rel:
129
+ continue
130
+ target = self.download_target_for_key(dest, target_rel)
131
+ target.parent.mkdir(parents=True, exist_ok=True)
132
+ data = self.download(rel_key)
133
+ with target.open("wb") as handle:
134
+ shutil.copyfileobj(data, handle)
135
+
136
+ async def ensure_bucket(self) -> None:
137
+ """Create the configured bucket if it does not already exist."""
138
+ if not self._fs.exists(self._bucket):
139
+ self._fs.mkdir(self._bucket)
140
+
141
+ async def get_range(
142
+ self, object_key: str, start: int, length: int
143
+ ) -> bytes:
144
+ """Retrieve a byte range using the ``s3fs`` file object."""
145
+ info = self._fs.info(self._full_key(object_key))
146
+ parsed_start, parsed_end = self._parse_range(
147
+ start, length, info["size"]
148
+ )
149
+ with self._fs.open(self._full_key(object_key), "rb") as handle:
150
+ handle.seek(parsed_start)
151
+ return handle.read(parsed_end - parsed_start)
152
+
153
+ async def remove_object(self, object_key: str) -> None:
154
+ """Delete ``object_key`` when it exists."""
155
+ path = self._full_key(object_key)
156
+ if self._fs.exists(path):
157
+ self._fs.rm(path)
158
+
159
+ @classmethod
160
+ def from_uri(cls, uri: str) -> "S3FSStorageAdapter":
161
+ """Instantiate the adapter from an ``s3://bucket/prefix`` URI."""
162
+ from urllib.parse import urlparse
163
+
164
+ parsed = urlparse(uri)
165
+ if parsed.scheme != "s3":
166
+ raise ValueError("URI must start with s3://")
167
+
168
+ return cls(
169
+ bucket=parsed.netloc,
170
+ prefix=parsed.path.lstrip("/"),
171
+ key=os.getenv("AWS_ACCESS_KEY_ID"),
172
+ secret=os.getenv("AWS_SECRET_ACCESS_KEY"),
173
+ token=os.getenv("AWS_SESSION_TOKEN"),
174
+ endpoint_url=os.getenv("AWS_ENDPOINT_URL"),
175
+ region_name=os.getenv("AWS_REGION"),
176
+ )
@@ -0,0 +1,84 @@
1
+ Metadata-Version: 2.4
2
+ Name: swarmauri_storage_s3fs
3
+ Version: 0.11.0.dev3
4
+ Summary: s3fs-backed S3 storage adapter for SwarmauriSDK
5
+ License-Expression: Apache-2.0
6
+ License-File: LICENSE
7
+ Keywords: swarmauri,sdk,standards,storage,s3,s3fs,fsspec,object-storage
8
+ Author: Jacob Stewart
9
+ Author-email: jacob@swarmauri.com
10
+ Requires-Python: >=3.10,<3.13
11
+ Classifier: Development Status :: 1 - Planning
12
+ Classifier: License :: OSI Approved :: Apache Software License
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Requires-Dist: s3fs
21
+ Requires-Dist: swarmauri_base
22
+ Requires-Dist: swarmauri_core
23
+ Requires-Dist: swarmauri_standard
24
+ Description-Content-Type: text/markdown
25
+
26
+ ![Swarmauri Logo](https://raw.githubusercontent.com/swarmauri/swarmauri-sdk/3d4d1cfa949399d7019ae9d8f296afba773dfb7f/assets/swarmauri.brand.theme.svg)
27
+
28
+ <p align="center">
29
+ <a href="https://pepy.tech/project/swarmauri_storage_s3fs/">
30
+ <img src="https://static.pepy.tech/badge/swarmauri_storage_s3fs/month" alt="PyPI - Downloads"/></a>
31
+ <a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3fs/">
32
+ <img alt="Hits" src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3fs.svg"/></a>
33
+ <a href="https://pypi.org/project/swarmauri_storage_s3fs/">
34
+ <img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue" alt="Supported Python Versions"/></a>
35
+ <a href="https://pypi.org/project/swarmauri_storage_s3fs/">
36
+ <img src="https://img.shields.io/pypi/l/swarmauri_storage_s3fs" alt="License"/></a>
37
+ <a href="https://pypi.org/project/swarmauri_storage_s3fs/">
38
+ <img src="https://img.shields.io/pypi/v/swarmauri_storage_s3fs?label=swarmauri_storage_s3fs&color=green" alt="Release Version"/></a>
39
+ <a href="https://discord.gg/N4UpBuQv8T">
40
+ <img src="https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white" alt="Discord"/></a>
41
+ </p>
42
+
43
+ # Swarmauri Storage S3FS
44
+
45
+ Filesystem-style S3 storage adapter for SwarmauriSDK backed strictly by `s3fs`.
46
+
47
+ ## Features
48
+
49
+ - Stores and retrieves objects from `s3://` locations through `s3fs.S3FileSystem`.
50
+ - Supports S3-compatible endpoints through `endpoint_url` and `client_kwargs`.
51
+ - Provides discoverable `swarmauri.storage_adapters` and `peagen.plugins.storage_adapters` entry points.
52
+ - Handles object upload, download, range reads, prefix iteration, directory push/pull, bucket creation, and object removal.
53
+
54
+ ## Installation
55
+
56
+ Install this package with `uv` or `pip`.
57
+
58
+ ```bash
59
+ uv add swarmauri_storage_s3fs
60
+ ```
61
+
62
+ ```bash
63
+ pip install swarmauri_storage_s3fs
64
+ ```
65
+
66
+ ## Usage
67
+
68
+ Use this adapter when the workflow should use the `s3fs` filesystem API.
69
+
70
+ ```python
71
+ from swarmauri_storage_s3fs import S3FSStorageAdapter
72
+
73
+ adapter = S3FSStorageAdapter.from_uri("s3://model-artifacts/runs")
74
+ uri = adapter.put_blob("latest/result.json", b'{"status":"ok"}')
75
+ payload = adapter.get_blob("latest/result.json")
76
+
77
+ print(uri)
78
+ print(payload)
79
+ ```
80
+
81
+ Configure credentials through `s3fs`, environment variables, or explicit constructor arguments owned by the surrounding Swarmauri or Tigrbl workflow.
82
+
83
+ License: Apache-2.0. See `LICENSE`.
84
+
@@ -0,0 +1,7 @@
1
+ swarmauri_storage_s3fs/__init__.py,sha256=osULRLUAJcbplliYT2mI0MMCsBZhO2MmFZq1TdPieZk,298
2
+ swarmauri_storage_s3fs/s3fs_storage_adapter.py,sha256=RFWgldOKu6hxc2Ezs_NEM7QBglJP7-hjFiMWo81UoNo,6578
3
+ swarmauri_storage_s3fs-0.11.0.dev3.dist-info/METADATA,sha256=evSoUCEf0uz4bou9EDAowXuzyTug8hqDbrVvbr1eNuQ,3495
4
+ swarmauri_storage_s3fs-0.11.0.dev3.dist-info/WHEEL,sha256=eY7nduwzv-ldUxpzbRlxwvC693Hg6PX8bWDjEHjZ_dk,88
5
+ swarmauri_storage_s3fs-0.11.0.dev3.dist-info/entry_points.txt,sha256=oNefM3GYps3ObLe40gy_3Xs1BXinDw76mgNP3t528Dc,215
6
+ swarmauri_storage_s3fs-0.11.0.dev3.dist-info/licenses/LICENSE,sha256=vpKDaVkCf4hDLNM1gy6oi-cH-OQ6YYAhFQ04CR7GwsQ,85
7
+ swarmauri_storage_s3fs-0.11.0.dev3.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: poetry-core 2.4.1
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,6 @@
1
+ [peagen.plugins.storage_adapters]
2
+ s3fs=swarmauri_storage_s3fs.s3fs_storage_adapter:S3FSStorageAdapter
3
+
4
+ [swarmauri.storage_adapters]
5
+ S3FSStorageAdapter=swarmauri_storage_s3fs.s3fs_storage_adapter:S3FSStorageAdapter
6
+
@@ -0,0 +1,3 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ https://www.apache.org/licenses/LICENSE-2.0