xyberos-storage 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xyberos_storage-0.1.0/PKG-INFO +71 -0
- xyberos_storage-0.1.0/README.md +53 -0
- xyberos_storage-0.1.0/pyproject.toml +29 -0
- xyberos_storage-0.1.0/setup.cfg +4 -0
- xyberos_storage-0.1.0/tests/test_adapters.py +137 -0
- xyberos_storage-0.1.0/tests/test_plugin.py +52 -0
- xyberos_storage-0.1.0/xyberos_storage/__init__.py +7 -0
- xyberos_storage-0.1.0/xyberos_storage/adapters.py +196 -0
- xyberos_storage-0.1.0/xyberos_storage/contract.py +24 -0
- xyberos_storage-0.1.0/xyberos_storage/http.py +91 -0
- xyberos_storage-0.1.0/xyberos_storage/plugin.py +114 -0
- xyberos_storage-0.1.0/xyberos_storage.egg-info/PKG-INFO +71 -0
- xyberos_storage-0.1.0/xyberos_storage.egg-info/SOURCES.txt +15 -0
- xyberos_storage-0.1.0/xyberos_storage.egg-info/dependency_links.txt +1 -0
- xyberos_storage-0.1.0/xyberos_storage.egg-info/entry_points.txt +2 -0
- xyberos_storage-0.1.0/xyberos_storage.egg-info/requires.txt +13 -0
- xyberos_storage-0.1.0/xyberos_storage.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xyberos-storage
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Object storage plugin (RFC-0019, M9): S3, Azure Blob, GCS, OneDrive behind one ObjectStore contract
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Keywords: xyberos,plugin,storage,s3,azure,gcs,onedrive
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: xyberos>=1.0
|
|
10
|
+
Provides-Extra: s3
|
|
11
|
+
Requires-Dist: boto3; extra == "s3"
|
|
12
|
+
Provides-Extra: azure
|
|
13
|
+
Requires-Dist: azure-storage-blob; extra == "azure"
|
|
14
|
+
Provides-Extra: gcs
|
|
15
|
+
Requires-Dist: google-cloud-storage; extra == "gcs"
|
|
16
|
+
Provides-Extra: test
|
|
17
|
+
Requires-Dist: pytest; extra == "test"
|
|
18
|
+
|
|
19
|
+
# xyberos-storage
|
|
20
|
+
|
|
21
|
+
**Object storage plugin — RFC-0019, M9.** S3, Azure Blob, GCS, and OneDrive
|
|
22
|
+
behind one `ObjectStore` contract (`list` / `upload` / `download`).
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install -e ./storage
|
|
28
|
+
pip install xyberos-storage[s3] # optional: boto3
|
|
29
|
+
pip install xyberos-storage[azure] # optional: azure-storage-blob
|
|
30
|
+
pip install xyberos-storage[gcs] # optional: google-cloud-storage
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Usage
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from xyberos import create_app
|
|
37
|
+
from xyberos_storage import StoragePlugin
|
|
38
|
+
|
|
39
|
+
app = create_app()
|
|
40
|
+
app.load_plugin(StoragePlugin(provider="s3", bucket="my-bucket")) # or STORAGE_PROVIDER
|
|
41
|
+
|
|
42
|
+
app.tools.execute("storage_list", None, prefix="data/")
|
|
43
|
+
app.tools.execute("storage_upload", None, key="data/x.txt", file_path="x.txt")
|
|
44
|
+
app.tools.execute("storage_download", None, key="data/x.txt", output_path="out.txt")
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
| Provider | Key / config |
|
|
48
|
+
| -------- | ------------ |
|
|
49
|
+
| `s3` | `STORAGE_BUCKET` + AWS creds (lazy boto3) |
|
|
50
|
+
| `azure` | `AZURE_STORAGE_CONNECTION_STRING` + `STORAGE_CONTAINER` (lazy SDK) |
|
|
51
|
+
| `gcs` | `STORAGE_BUCKET` + GCP creds (lazy SDK) |
|
|
52
|
+
| `onedrive` | `ONEDRIVE_ACCESS_TOKEN` (Graph REST) |
|
|
53
|
+
|
|
54
|
+
## Tools
|
|
55
|
+
|
|
56
|
+
- `storage_list(prefix="")`
|
|
57
|
+
- `storage_upload(key, file_path)`
|
|
58
|
+
- `storage_download(key, output_path)`
|
|
59
|
+
|
|
60
|
+
## Tests
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install pytest
|
|
64
|
+
pytest tests/
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Fake clients / injectable transports — no cloud, no network.
|
|
68
|
+
|
|
69
|
+
## Ship location
|
|
70
|
+
|
|
71
|
+
Plugin (`xyberos.plugins` entry point) — enterprise storage (M9).
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# xyberos-storage
|
|
2
|
+
|
|
3
|
+
**Object storage plugin — RFC-0019, M9.** S3, Azure Blob, GCS, and OneDrive
|
|
4
|
+
behind one `ObjectStore` contract (`list` / `upload` / `download`).
|
|
5
|
+
|
|
6
|
+
## Install
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install -e ./storage
|
|
10
|
+
pip install xyberos-storage[s3] # optional: boto3
|
|
11
|
+
pip install xyberos-storage[azure] # optional: azure-storage-blob
|
|
12
|
+
pip install xyberos-storage[gcs] # optional: google-cloud-storage
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from xyberos import create_app
|
|
19
|
+
from xyberos_storage import StoragePlugin
|
|
20
|
+
|
|
21
|
+
app = create_app()
|
|
22
|
+
app.load_plugin(StoragePlugin(provider="s3", bucket="my-bucket")) # or STORAGE_PROVIDER
|
|
23
|
+
|
|
24
|
+
app.tools.execute("storage_list", None, prefix="data/")
|
|
25
|
+
app.tools.execute("storage_upload", None, key="data/x.txt", file_path="x.txt")
|
|
26
|
+
app.tools.execute("storage_download", None, key="data/x.txt", output_path="out.txt")
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
| Provider | Key / config |
|
|
30
|
+
| -------- | ------------ |
|
|
31
|
+
| `s3` | `STORAGE_BUCKET` + AWS creds (lazy boto3) |
|
|
32
|
+
| `azure` | `AZURE_STORAGE_CONNECTION_STRING` + `STORAGE_CONTAINER` (lazy SDK) |
|
|
33
|
+
| `gcs` | `STORAGE_BUCKET` + GCP creds (lazy SDK) |
|
|
34
|
+
| `onedrive` | `ONEDRIVE_ACCESS_TOKEN` (Graph REST) |
|
|
35
|
+
|
|
36
|
+
## Tools
|
|
37
|
+
|
|
38
|
+
- `storage_list(prefix="")`
|
|
39
|
+
- `storage_upload(key, file_path)`
|
|
40
|
+
- `storage_download(key, output_path)`
|
|
41
|
+
|
|
42
|
+
## Tests
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install pytest
|
|
46
|
+
pytest tests/
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Fake clients / injectable transports — no cloud, no network.
|
|
50
|
+
|
|
51
|
+
## Ship location
|
|
52
|
+
|
|
53
|
+
Plugin (`xyberos.plugins` entry point) — enterprise storage (M9).
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "xyberos-storage"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Object storage plugin (RFC-0019, M9): S3, Azure Blob, GCS, OneDrive behind one ObjectStore contract"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = {text = "Apache-2.0"}
|
|
12
|
+
dependencies = ["xyberos>=1.0"]
|
|
13
|
+
keywords = ["xyberos", "plugin", "storage", "s3", "azure", "gcs", "onedrive"]
|
|
14
|
+
|
|
15
|
+
[project.optional-dependencies]
|
|
16
|
+
s3 = ["boto3"]
|
|
17
|
+
azure = ["azure-storage-blob"]
|
|
18
|
+
gcs = ["google-cloud-storage"]
|
|
19
|
+
test = ["pytest"]
|
|
20
|
+
|
|
21
|
+
[project.entry-points."xyberos.plugins"]
|
|
22
|
+
storage = "xyberos_storage.plugin:plugin"
|
|
23
|
+
|
|
24
|
+
[tool.setuptools]
|
|
25
|
+
packages = ["xyberos_storage"]
|
|
26
|
+
|
|
27
|
+
[tool.pytest.ini_options]
|
|
28
|
+
testpaths = ["tests"]
|
|
29
|
+
pythonpath = ["."]
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Tests for the object store adapters (fake clients / transports, no network)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
from xyberos.exceptions.provider import ProviderError
|
|
7
|
+
|
|
8
|
+
from xyberos_storage import AzureBlobStore, GcsStore, OneDriveStore, S3Store
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class _FakeS3Client:
|
|
12
|
+
def __init__(self):
|
|
13
|
+
self.objects: dict[str, bytes] = {}
|
|
14
|
+
|
|
15
|
+
def get_paginator(self, _name):
|
|
16
|
+
return self
|
|
17
|
+
|
|
18
|
+
def paginate(self, **kwargs):
|
|
19
|
+
prefix = kwargs.get("Prefix", "")
|
|
20
|
+
return [
|
|
21
|
+
{"Contents": [{"Key": key} for key in self.objects if key.startswith(prefix)]}
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
def put_object(self, **kwargs):
|
|
25
|
+
self.objects[kwargs["Key"]] = kwargs["Body"]
|
|
26
|
+
|
|
27
|
+
def get_object(self, **kwargs):
|
|
28
|
+
return {"Body": __import__("io").BytesIO(self.objects[kwargs["Key"]])}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_s3_roundtrip():
|
|
32
|
+
client = _FakeS3Client()
|
|
33
|
+
store = S3Store("bucket", client=client)
|
|
34
|
+
store.upload("dir/a.txt", b"hello")
|
|
35
|
+
assert store.list("dir/") == ["dir/a.txt"]
|
|
36
|
+
assert store.download("dir/a.txt") == b"hello"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _FakeStream:
|
|
40
|
+
def __init__(self, data):
|
|
41
|
+
self._data = data
|
|
42
|
+
|
|
43
|
+
def readall(self):
|
|
44
|
+
return self._data
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _FakeBlob:
|
|
48
|
+
def __init__(self, name):
|
|
49
|
+
self.name = name
|
|
50
|
+
self.data = b""
|
|
51
|
+
|
|
52
|
+
def download_blob(self):
|
|
53
|
+
return _FakeStream(self.data)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class _FakeContainer:
|
|
57
|
+
def __init__(self):
|
|
58
|
+
self.blobs: dict[str, _FakeBlob] = {}
|
|
59
|
+
|
|
60
|
+
def list_blobs(self, name_starts_with=""):
|
|
61
|
+
return [b for name, b in self.blobs.items() if name.startswith(name_starts_with)]
|
|
62
|
+
|
|
63
|
+
def upload_blob(self, name, data, **kwargs):
|
|
64
|
+
blob = _FakeBlob(name)
|
|
65
|
+
blob.data = data
|
|
66
|
+
self.blobs[name] = blob
|
|
67
|
+
|
|
68
|
+
def get_blob_client(self, name):
|
|
69
|
+
return self.blobs[name]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class _FakeAzure:
|
|
73
|
+
def get_container_client(self, _name):
|
|
74
|
+
return self._container
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_azure_roundtrip():
|
|
78
|
+
container = _FakeContainer()
|
|
79
|
+
service = _FakeAzure()
|
|
80
|
+
service._container = container
|
|
81
|
+
store = AzureBlobStore(container="c", client=service)
|
|
82
|
+
store.upload("a.txt", b"hi")
|
|
83
|
+
assert store.list() == ["a.txt"]
|
|
84
|
+
assert store.download("a.txt") == b"hi"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_gcs_roundtrip():
|
|
88
|
+
blobs: dict[str, bytes] = {}
|
|
89
|
+
|
|
90
|
+
class _FakeBlobGcs:
|
|
91
|
+
def __init__(self, name):
|
|
92
|
+
self.name = name
|
|
93
|
+
self._data = b""
|
|
94
|
+
|
|
95
|
+
def upload_from_string(self, data):
|
|
96
|
+
blobs[self.name] = data
|
|
97
|
+
|
|
98
|
+
def download_as_bytes(self):
|
|
99
|
+
return blobs[self.name]
|
|
100
|
+
|
|
101
|
+
class _FakeBucket:
|
|
102
|
+
def list_blobs(self, prefix=""):
|
|
103
|
+
return [_FakeBlobGcs(name) for name in blobs if name.startswith(prefix)]
|
|
104
|
+
|
|
105
|
+
def blob(self, name):
|
|
106
|
+
if name not in blobs:
|
|
107
|
+
blobs[name] = b""
|
|
108
|
+
return _FakeBlobGcs(name)
|
|
109
|
+
|
|
110
|
+
class _FakeGcsClient:
|
|
111
|
+
def bucket(self, _name):
|
|
112
|
+
return _FakeBucket()
|
|
113
|
+
|
|
114
|
+
store = GcsStore("bucket", client=_FakeGcsClient())
|
|
115
|
+
store.upload("k.txt", b"data")
|
|
116
|
+
assert store.list() == ["k.txt"]
|
|
117
|
+
assert store.download("k.txt") == b"data"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_onedrive_roundtrip():
|
|
121
|
+
def request(method, url, **kwargs):
|
|
122
|
+
if url.endswith(":/children"):
|
|
123
|
+
return 200, {"value": [{"name": "a.txt"}]}
|
|
124
|
+
if method == "PUT":
|
|
125
|
+
return 201, {"name": "a.txt"}
|
|
126
|
+
return 200, b"content"
|
|
127
|
+
|
|
128
|
+
store = OneDriveStore(access_token="tok", request=request)
|
|
129
|
+
assert store.list() == ["a.txt"]
|
|
130
|
+
assert store.upload("a.txt", b"data") == "a.txt"
|
|
131
|
+
assert store.download("a.txt") == b"content"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_onedrive_requires_token():
|
|
135
|
+
store = OneDriveStore(access_token=None, request=lambda *a, **k: (200, {}))
|
|
136
|
+
with pytest.raises(ProviderError, match="ONEDRIVE_ACCESS_TOKEN"):
|
|
137
|
+
store.list()
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Tests for loading the storage plugin into a Xyberos app."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from xyberos import create_app
|
|
6
|
+
|
|
7
|
+
from xyberos_storage import StoragePlugin
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class _FakeS3Client:
|
|
11
|
+
def __init__(self):
|
|
12
|
+
self.objects: dict[str, bytes] = {}
|
|
13
|
+
|
|
14
|
+
def get_paginator(self, _name):
|
|
15
|
+
return self
|
|
16
|
+
|
|
17
|
+
def paginate(self, **kwargs):
|
|
18
|
+
prefix = kwargs.get("Prefix", "")
|
|
19
|
+
return [{"Contents": [{"Key": k} for k in self.objects if k.startswith(prefix)]}]
|
|
20
|
+
|
|
21
|
+
def put_object(self, **kwargs):
|
|
22
|
+
self.objects[kwargs["Key"]] = kwargs["Body"]
|
|
23
|
+
|
|
24
|
+
def get_object(self, **kwargs):
|
|
25
|
+
return {"Body": __import__("io").BytesIO(self.objects[kwargs["Key"]])}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_plugin_registers_and_executes(tmp_path):
|
|
29
|
+
from xyberos_storage import S3Store
|
|
30
|
+
|
|
31
|
+
client = _FakeS3Client()
|
|
32
|
+
app = create_app()
|
|
33
|
+
# Build the plugin but swap in a store with the fake client.
|
|
34
|
+
plugin = StoragePlugin(provider="s3", bucket="bucket")
|
|
35
|
+
store = S3Store("bucket", client=client)
|
|
36
|
+
plugin._provider = "s3"
|
|
37
|
+
# Replace the plugin's object_store with the fake-backed one for the test.
|
|
38
|
+
plugin.object_store = lambda: store # type: ignore[method-assign]
|
|
39
|
+
app.load_plugin(plugin)
|
|
40
|
+
|
|
41
|
+
assert "storage_list" in app.tools.names
|
|
42
|
+
|
|
43
|
+
src = tmp_path / "src.txt"
|
|
44
|
+
src.write_bytes(b"hello")
|
|
45
|
+
app.tools.execute("storage_upload", None, key="dir/a.txt", file_path=str(src))
|
|
46
|
+
assert app.tools.execute("storage_list", None, prefix="dir/") == ["dir/a.txt"]
|
|
47
|
+
|
|
48
|
+
out = tmp_path / "out.txt"
|
|
49
|
+
assert app.tools.execute("storage_download", None, key="dir/a.txt", output_path=str(out)) == str(out)
|
|
50
|
+
assert out.read_bytes() == b"hello"
|
|
51
|
+
|
|
52
|
+
app.unload_plugin("storage")
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Object storage plugin (RFC-0019, M9)."""
|
|
2
|
+
|
|
3
|
+
from .adapters import AzureBlobStore, GcsStore, OneDriveStore, S3Store
|
|
4
|
+
from .contract import ObjectStore
|
|
5
|
+
from .plugin import StoragePlugin
|
|
6
|
+
|
|
7
|
+
__all__ = ["AzureBlobStore", "GcsStore", "ObjectStore", "OneDriveStore", "S3Store", "StoragePlugin"]
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Object store adapters: S3, Azure Blob, GCS, OneDrive.
|
|
2
|
+
|
|
3
|
+
S3 / Azure / GCS lazy-import their SDK and accept an injectable client for
|
|
4
|
+
tests; OneDrive uses the Microsoft Graph REST API with an injectable transport.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import importlib
|
|
10
|
+
import os
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from xyberos.exceptions.provider import ProviderError
|
|
14
|
+
|
|
15
|
+
from .http import RequestTransport, default_request
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class S3Store:
|
|
19
|
+
"""AWS S3 (lazy ``boto3``)."""
|
|
20
|
+
|
|
21
|
+
name = "s3"
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
bucket: str,
|
|
26
|
+
*,
|
|
27
|
+
client: Any | None = None,
|
|
28
|
+
region: str | None = None,
|
|
29
|
+
) -> None:
|
|
30
|
+
self._bucket = bucket
|
|
31
|
+
self._client = client
|
|
32
|
+
self._region = region or os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION")
|
|
33
|
+
|
|
34
|
+
def list(self, prefix: str = "") -> list[str]:
|
|
35
|
+
client = self._get_client()
|
|
36
|
+
keys: list[str] = []
|
|
37
|
+
paginator = client.get_paginator("list_objects_v2")
|
|
38
|
+
for page in paginator.paginate(Bucket=self._bucket, Prefix=prefix):
|
|
39
|
+
for obj in page.get("Contents", []):
|
|
40
|
+
keys.append(obj["Key"])
|
|
41
|
+
return keys
|
|
42
|
+
|
|
43
|
+
def upload(self, key: str, data: bytes) -> str:
|
|
44
|
+
self._get_client().put_object(Bucket=self._bucket, Key=key, Body=data)
|
|
45
|
+
return key
|
|
46
|
+
|
|
47
|
+
def download(self, key: str) -> bytes:
|
|
48
|
+
response = self._get_client().get_object(Bucket=self._bucket, Key=key)
|
|
49
|
+
return response["Body"].read()
|
|
50
|
+
|
|
51
|
+
def _get_client(self) -> Any:
|
|
52
|
+
if self._client is not None:
|
|
53
|
+
return self._client
|
|
54
|
+
try:
|
|
55
|
+
boto3 = importlib.import_module("boto3")
|
|
56
|
+
except ImportError as exc:
|
|
57
|
+
raise ProviderError(
|
|
58
|
+
"the 'boto3' package is required for S3; install with 'pip install xyberos-storage[s3]'"
|
|
59
|
+
) from exc
|
|
60
|
+
kwargs = {"service_name": "s3"}
|
|
61
|
+
if self._region:
|
|
62
|
+
kwargs["region_name"] = self._region
|
|
63
|
+
self._client = boto3.client(**kwargs)
|
|
64
|
+
return self._client
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class AzureBlobStore:
|
|
68
|
+
"""Azure Blob Storage (lazy ``azure-storage-blob``)."""
|
|
69
|
+
|
|
70
|
+
name = "azure"
|
|
71
|
+
|
|
72
|
+
def __init__(self, connection_string: str | None = None, *, container: str, client: Any | None = None) -> None:
|
|
73
|
+
self._connection_string = connection_string or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
|
|
74
|
+
self._container = container
|
|
75
|
+
self._client = client
|
|
76
|
+
|
|
77
|
+
def _container_client(self) -> Any:
|
|
78
|
+
if self._client is not None:
|
|
79
|
+
return self._client.get_container_client(self._container)
|
|
80
|
+
try:
|
|
81
|
+
azure_blob = importlib.import_module("azure.storage.blob")
|
|
82
|
+
except ImportError as exc:
|
|
83
|
+
raise ProviderError(
|
|
84
|
+
"the 'azure-storage-blob' package is required; install with "
|
|
85
|
+
"'pip install xyberos-storage[azure]'"
|
|
86
|
+
) from exc
|
|
87
|
+
if not self._connection_string:
|
|
88
|
+
raise ProviderError("Azure Blob requires AZURE_STORAGE_CONNECTION_STRING")
|
|
89
|
+
return azure_blob.BlobServiceClient.from_connection_string(self._connection_string).get_container_client(self._container)
|
|
90
|
+
|
|
91
|
+
def list(self, prefix: str = "") -> list[str]:
|
|
92
|
+
container = self._container_client()
|
|
93
|
+
return [blob.name for blob in container.list_blobs(name_starts_with=prefix)]
|
|
94
|
+
|
|
95
|
+
def upload(self, key: str, data: bytes) -> str:
|
|
96
|
+
self._container_client().upload_blob(key, data, overwrite=True)
|
|
97
|
+
return key
|
|
98
|
+
|
|
99
|
+
def download(self, key: str) -> bytes:
|
|
100
|
+
blob = self._container_client().get_blob_client(key)
|
|
101
|
+
return blob.download_blob().readall()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class GcsStore:
|
|
105
|
+
"""Google Cloud Storage (lazy ``google-cloud-storage``)."""
|
|
106
|
+
|
|
107
|
+
name = "gcs"
|
|
108
|
+
|
|
109
|
+
def __init__(self, bucket: str, *, client: Any | None = None) -> None:
|
|
110
|
+
self._bucket = bucket
|
|
111
|
+
self._client = client
|
|
112
|
+
|
|
113
|
+
def _get_bucket(self) -> Any:
|
|
114
|
+
if self._client is not None:
|
|
115
|
+
return self._client.bucket(self._bucket)
|
|
116
|
+
try:
|
|
117
|
+
storage = importlib.import_module("google.cloud.storage")
|
|
118
|
+
except ImportError as exc:
|
|
119
|
+
raise ProviderError(
|
|
120
|
+
"the 'google-cloud-storage' package is required; install with "
|
|
121
|
+
"'pip install xyberos-storage[gcs]'"
|
|
122
|
+
) from exc
|
|
123
|
+
return storage.Client().bucket(self._bucket)
|
|
124
|
+
|
|
125
|
+
def list(self, prefix: str = "") -> list[str]:
|
|
126
|
+
return [blob.name for blob in self._get_bucket().list_blobs(prefix=prefix)]
|
|
127
|
+
|
|
128
|
+
def upload(self, key: str, data: bytes) -> str:
|
|
129
|
+
blob = self._get_bucket().blob(key)
|
|
130
|
+
blob.upload_from_string(data)
|
|
131
|
+
return key
|
|
132
|
+
|
|
133
|
+
def download(self, key: str) -> bytes:
|
|
134
|
+
blob = self._get_bucket().blob(key)
|
|
135
|
+
return blob.download_as_bytes()
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class OneDriveStore:
|
|
139
|
+
"""Microsoft OneDrive via the Graph REST API (injectable transport)."""
|
|
140
|
+
|
|
141
|
+
name = "onedrive"
|
|
142
|
+
base_url = "https://graph.microsoft.com/v1.0"
|
|
143
|
+
|
|
144
|
+
def __init__(
|
|
145
|
+
self,
|
|
146
|
+
access_token: str | None = None,
|
|
147
|
+
*,
|
|
148
|
+
request: RequestTransport | None = None,
|
|
149
|
+
timeout: float = 30.0,
|
|
150
|
+
) -> None:
|
|
151
|
+
self._access_token = access_token if access_token is not None else os.getenv("ONEDRIVE_ACCESS_TOKEN")
|
|
152
|
+
self._request = request or default_request
|
|
153
|
+
self._timeout = timeout
|
|
154
|
+
|
|
155
|
+
def list(self, prefix: str = "") -> list[str]:
|
|
156
|
+
status, body = self._request(
|
|
157
|
+
"GET",
|
|
158
|
+
f"{self.base_url}/me/drive/root:{prefix}:/children",
|
|
159
|
+
headers=self._headers(),
|
|
160
|
+
timeout=self._timeout,
|
|
161
|
+
)
|
|
162
|
+
self._check(status, body)
|
|
163
|
+
return [item.get("name") for item in body.get("value", [])]
|
|
164
|
+
|
|
165
|
+
def upload(self, key: str, data: bytes) -> str:
|
|
166
|
+
status, body = self._request(
|
|
167
|
+
"PUT",
|
|
168
|
+
f"{self.base_url}/me/drive/root:/{key}:/content",
|
|
169
|
+
raw_body=data,
|
|
170
|
+
headers=self._headers(),
|
|
171
|
+
timeout=self._timeout,
|
|
172
|
+
)
|
|
173
|
+
self._check(status, body)
|
|
174
|
+
return key
|
|
175
|
+
|
|
176
|
+
def download(self, key: str) -> bytes:
|
|
177
|
+
status, body = self._request(
|
|
178
|
+
"GET",
|
|
179
|
+
f"{self.base_url}/me/drive/root:/{key}:/content",
|
|
180
|
+
headers=self._headers(),
|
|
181
|
+
timeout=self._timeout,
|
|
182
|
+
)
|
|
183
|
+
self._check(status, body)
|
|
184
|
+
return body.encode("utf-8") if isinstance(body, str) else body
|
|
185
|
+
|
|
186
|
+
def _headers(self) -> dict[str, str]:
|
|
187
|
+
if not self._access_token:
|
|
188
|
+
raise ProviderError("OneDrive requires an access token (set ONEDRIVE_ACCESS_TOKEN)")
|
|
189
|
+
return {"Authorization": f"Bearer {self._access_token}"}
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
def _check(status: int, body: Any) -> None:
|
|
193
|
+
if 200 <= status < 300:
|
|
194
|
+
return
|
|
195
|
+
message = body if isinstance(body, str) else str(body)
|
|
196
|
+
raise ProviderError(f"OneDrive API returned HTTP {status}: {message[:200]}")
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""The ObjectStore contract (RFC-0019, M9)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Protocol, runtime_checkable
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@runtime_checkable
|
|
9
|
+
class ObjectStore(Protocol):
|
|
10
|
+
"""Anything that lists, uploads and downloads objects by key."""
|
|
11
|
+
|
|
12
|
+
name: str
|
|
13
|
+
|
|
14
|
+
def list(self, prefix: str = "") -> list[str]:
|
|
15
|
+
"""Return the object keys under ``prefix``."""
|
|
16
|
+
...
|
|
17
|
+
|
|
18
|
+
def upload(self, key: str, data: bytes) -> str:
|
|
19
|
+
"""Store ``data`` under ``key``; return the key."""
|
|
20
|
+
...
|
|
21
|
+
|
|
22
|
+
def download(self, key: str) -> bytes:
|
|
23
|
+
"""Return the bytes stored under ``key``."""
|
|
24
|
+
...
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""A tiny stdlib HTTP helper (no third-party deps).
|
|
2
|
+
|
|
3
|
+
``default_request`` performs one HTTP request with ``urllib`` and returns
|
|
4
|
+
``(status, body)`` where ``body`` is parsed JSON when the response is JSON,
|
|
5
|
+
otherwise raw text. Injectable so tests run without a network.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import urllib.error
|
|
12
|
+
import urllib.parse
|
|
13
|
+
import urllib.request
|
|
14
|
+
from typing import Any, Callable
|
|
15
|
+
|
|
16
|
+
#: (method, url, *, json_body, raw_body, headers, query, timeout) -> (status, body)
|
|
17
|
+
RequestTransport = Callable[..., tuple[int, Any]]
|
|
18
|
+
RawRequestTransport = Callable[..., tuple[int, bytes]]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def default_request(
|
|
22
|
+
method: str,
|
|
23
|
+
url: str,
|
|
24
|
+
*,
|
|
25
|
+
json_body: Any = None,
|
|
26
|
+
raw_body: bytes | None = None,
|
|
27
|
+
headers: dict[str, str] | None = None,
|
|
28
|
+
query: dict[str, Any] | None = None,
|
|
29
|
+
timeout: float = 30.0,
|
|
30
|
+
) -> tuple[int, Any]:
|
|
31
|
+
"""Send one request and return ``(status, parsed_json_or_text)``."""
|
|
32
|
+
request = _build(method, url, json_body=json_body, raw_body=raw_body, headers=headers, query=query)
|
|
33
|
+
try:
|
|
34
|
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
35
|
+
raw = response.read()
|
|
36
|
+
content_type = response.headers.get("Content-Type", "")
|
|
37
|
+
except urllib.error.HTTPError as exc:
|
|
38
|
+
return exc.code, exc.read().decode("utf-8", errors="replace")
|
|
39
|
+
|
|
40
|
+
text = raw.decode("utf-8", errors="replace")
|
|
41
|
+
if "application/json" in content_type or text.lstrip().startswith("{"):
|
|
42
|
+
try:
|
|
43
|
+
return 200, json.loads(text)
|
|
44
|
+
except json.JSONDecodeError:
|
|
45
|
+
return 200, text
|
|
46
|
+
return 200, text
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def default_raw_request(
|
|
50
|
+
method: str,
|
|
51
|
+
url: str,
|
|
52
|
+
*,
|
|
53
|
+
json_body: Any = None,
|
|
54
|
+
raw_body: bytes | None = None,
|
|
55
|
+
headers: dict[str, str] | None = None,
|
|
56
|
+
query: dict[str, Any] | None = None,
|
|
57
|
+
timeout: float = 30.0,
|
|
58
|
+
) -> tuple[int, bytes]:
|
|
59
|
+
"""Send one request and return ``(status, raw_bytes)``."""
|
|
60
|
+
request = _build(method, url, json_body=json_body, raw_body=raw_body, headers=headers, query=query)
|
|
61
|
+
try:
|
|
62
|
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
63
|
+
return int(getattr(response, "status", 200)), response.read()
|
|
64
|
+
except urllib.error.HTTPError as exc:
|
|
65
|
+
return exc.code, exc.read()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _build(
|
|
69
|
+
method: str,
|
|
70
|
+
url: str,
|
|
71
|
+
*,
|
|
72
|
+
json_body: Any,
|
|
73
|
+
raw_body: bytes | None,
|
|
74
|
+
headers: dict[str, str] | None,
|
|
75
|
+
query: dict[str, Any] | None,
|
|
76
|
+
) -> urllib.request.Request:
|
|
77
|
+
final_url = url
|
|
78
|
+
if query:
|
|
79
|
+
separator = "&" if "?" in url else "?"
|
|
80
|
+
final_url = url + separator + urllib.parse.urlencode(query)
|
|
81
|
+
|
|
82
|
+
data: bytes | None = None
|
|
83
|
+
request_headers = dict(headers or {})
|
|
84
|
+
if raw_body is not None:
|
|
85
|
+
data = raw_body
|
|
86
|
+
elif json_body is not None:
|
|
87
|
+
data = json.dumps(json_body).encode("utf-8")
|
|
88
|
+
request_headers.setdefault("Content-Type", "application/json")
|
|
89
|
+
return urllib.request.Request(
|
|
90
|
+
final_url, data=data, headers=request_headers, method=method
|
|
91
|
+
)
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Storage plugin entry point (RFC-0019, M9)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, cast
|
|
8
|
+
|
|
9
|
+
from xyberos.contracts import Plugin, Tool
|
|
10
|
+
from xyberos.tools import FunctionTool
|
|
11
|
+
|
|
12
|
+
from .adapters import AzureBlobStore, GcsStore, OneDriveStore, S3Store
|
|
13
|
+
from .contract import ObjectStore
|
|
14
|
+
from .http import RequestTransport
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _pop_tool(registry: Any, name: str) -> None:
|
|
18
|
+
unregister = getattr(registry, "unregister", None)
|
|
19
|
+
if callable(unregister):
|
|
20
|
+
unregister(name)
|
|
21
|
+
return
|
|
22
|
+
store = getattr(registry, "_tools", None)
|
|
23
|
+
if isinstance(store, dict):
|
|
24
|
+
cast(dict[str, Any], store).pop(name, None)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class StoragePlugin(Plugin):
|
|
28
|
+
"""Registers object-storage tools backed by a configured provider."""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
provider: str | None = None,
|
|
33
|
+
*,
|
|
34
|
+
env_prefix: str = "STORAGE",
|
|
35
|
+
bucket: str | None = None,
|
|
36
|
+
container: str | None = None,
|
|
37
|
+
access_token: str | None = None,
|
|
38
|
+
request: RequestTransport | None = None,
|
|
39
|
+
) -> None:
|
|
40
|
+
self._provider = (provider or os.getenv(f"{env_prefix}_PROVIDER") or "s3").lower()
|
|
41
|
+
self._bucket = bucket
|
|
42
|
+
self._container = container or os.getenv(f"{env_prefix}_CONTAINER", "default")
|
|
43
|
+
self._access_token = access_token
|
|
44
|
+
self._request = request
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def name(self) -> str:
|
|
48
|
+
return "storage"
|
|
49
|
+
|
|
50
|
+
def object_store(self) -> ObjectStore:
|
|
51
|
+
name = self._provider
|
|
52
|
+
if name == "s3":
|
|
53
|
+
bucket = self._bucket or os.getenv("STORAGE_BUCKET") or os.getenv("S3_BUCKET")
|
|
54
|
+
if not bucket:
|
|
55
|
+
raise ValueError("s3 provider requires a bucket (STORAGE_BUCKET)")
|
|
56
|
+
return S3Store(bucket)
|
|
57
|
+
if name == "azure":
|
|
58
|
+
return AzureBlobStore(container=self._container)
|
|
59
|
+
if name == "gcs":
|
|
60
|
+
bucket = self._bucket or os.getenv("STORAGE_BUCKET") or os.getenv("GCS_BUCKET")
|
|
61
|
+
if not bucket:
|
|
62
|
+
raise ValueError("gcs provider requires a bucket (STORAGE_BUCKET)")
|
|
63
|
+
return GcsStore(bucket)
|
|
64
|
+
if name == "onedrive":
|
|
65
|
+
return OneDriveStore(self._access_token, request=self._request)
|
|
66
|
+
raise ValueError(f"unknown storage provider '{name}' (s3 | azure | gcs | onedrive)")
|
|
67
|
+
|
|
68
|
+
def tools(self) -> list[Tool]:
|
|
69
|
+
store = self.object_store()
|
|
70
|
+
|
|
71
|
+
def _list(prefix: str = "") -> list[str]:
|
|
72
|
+
return store.list(prefix)
|
|
73
|
+
|
|
74
|
+
def _upload(key: str, file_path: str) -> str:
|
|
75
|
+
data = Path(file_path).read_bytes()
|
|
76
|
+
return store.upload(key, data)
|
|
77
|
+
|
|
78
|
+
def _download(key: str, output_path: str) -> str:
|
|
79
|
+
data = store.download(key)
|
|
80
|
+
path = Path(output_path)
|
|
81
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
82
|
+
path.write_bytes(data)
|
|
83
|
+
return str(path)
|
|
84
|
+
|
|
85
|
+
return [
|
|
86
|
+
FunctionTool("storage_list", _list, description=f"List objects via {self._provider}."),
|
|
87
|
+
FunctionTool("storage_upload", _upload, description=f"Upload a file to {self._provider}."),
|
|
88
|
+
FunctionTool("storage_download", _download, description=f"Download an object from {self._provider}."),
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
def register(self, kernel: object) -> None:
|
|
92
|
+
try:
|
|
93
|
+
tools = self.tools()
|
|
94
|
+
except ValueError as exc:
|
|
95
|
+
logger = getattr(kernel, "logger", None)
|
|
96
|
+
if logger is not None and callable(getattr(logger, "warning", None)):
|
|
97
|
+
logger.warning("storage plugin not configured: %s", exc)
|
|
98
|
+
return
|
|
99
|
+
registry = kernel.resolve("tools")
|
|
100
|
+
for tool in tools:
|
|
101
|
+
registry.register(tool)
|
|
102
|
+
|
|
103
|
+
def unregister(self, kernel: object) -> None:
|
|
104
|
+
try:
|
|
105
|
+
tools = self.tools()
|
|
106
|
+
except ValueError:
|
|
107
|
+
return # never configured -> nothing was registered
|
|
108
|
+
registry = kernel.resolve("tools")
|
|
109
|
+
for tool in tools:
|
|
110
|
+
_pop_tool(registry, tool.name)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
#: Auto-discovered by ``app.load_entry_points()``.
|
|
114
|
+
plugin = StoragePlugin()
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xyberos-storage
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Object storage plugin (RFC-0019, M9): S3, Azure Blob, GCS, OneDrive behind one ObjectStore contract
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Keywords: xyberos,plugin,storage,s3,azure,gcs,onedrive
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: xyberos>=1.0
|
|
10
|
+
Provides-Extra: s3
|
|
11
|
+
Requires-Dist: boto3; extra == "s3"
|
|
12
|
+
Provides-Extra: azure
|
|
13
|
+
Requires-Dist: azure-storage-blob; extra == "azure"
|
|
14
|
+
Provides-Extra: gcs
|
|
15
|
+
Requires-Dist: google-cloud-storage; extra == "gcs"
|
|
16
|
+
Provides-Extra: test
|
|
17
|
+
Requires-Dist: pytest; extra == "test"
|
|
18
|
+
|
|
19
|
+
# xyberos-storage
|
|
20
|
+
|
|
21
|
+
**Object storage plugin — RFC-0019, M9.** S3, Azure Blob, GCS, and OneDrive
|
|
22
|
+
behind one `ObjectStore` contract (`list` / `upload` / `download`).
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install -e ./storage
|
|
28
|
+
pip install xyberos-storage[s3] # optional: boto3
|
|
29
|
+
pip install xyberos-storage[azure] # optional: azure-storage-blob
|
|
30
|
+
pip install xyberos-storage[gcs] # optional: google-cloud-storage
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Usage
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from xyberos import create_app
|
|
37
|
+
from xyberos_storage import StoragePlugin
|
|
38
|
+
|
|
39
|
+
app = create_app()
|
|
40
|
+
app.load_plugin(StoragePlugin(provider="s3", bucket="my-bucket")) # or STORAGE_PROVIDER
|
|
41
|
+
|
|
42
|
+
app.tools.execute("storage_list", None, prefix="data/")
|
|
43
|
+
app.tools.execute("storage_upload", None, key="data/x.txt", file_path="x.txt")
|
|
44
|
+
app.tools.execute("storage_download", None, key="data/x.txt", output_path="out.txt")
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
| Provider | Key / config |
|
|
48
|
+
| -------- | ------------ |
|
|
49
|
+
| `s3` | `STORAGE_BUCKET` + AWS creds (lazy boto3) |
|
|
50
|
+
| `azure` | `AZURE_STORAGE_CONNECTION_STRING` + `STORAGE_CONTAINER` (lazy SDK) |
|
|
51
|
+
| `gcs` | `STORAGE_BUCKET` + GCP creds (lazy SDK) |
|
|
52
|
+
| `onedrive` | `ONEDRIVE_ACCESS_TOKEN` (Graph REST) |
|
|
53
|
+
|
|
54
|
+
## Tools
|
|
55
|
+
|
|
56
|
+
- `storage_list(prefix="")`
|
|
57
|
+
- `storage_upload(key, file_path)`
|
|
58
|
+
- `storage_download(key, output_path)`
|
|
59
|
+
|
|
60
|
+
## Tests
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install pytest
|
|
64
|
+
pytest tests/
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Fake clients / injectable transports — no cloud, no network.
|
|
68
|
+
|
|
69
|
+
## Ship location
|
|
70
|
+
|
|
71
|
+
Plugin (`xyberos.plugins` entry point) — enterprise storage (M9).
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
tests/test_adapters.py
|
|
4
|
+
tests/test_plugin.py
|
|
5
|
+
xyberos_storage/__init__.py
|
|
6
|
+
xyberos_storage/adapters.py
|
|
7
|
+
xyberos_storage/contract.py
|
|
8
|
+
xyberos_storage/http.py
|
|
9
|
+
xyberos_storage/plugin.py
|
|
10
|
+
xyberos_storage.egg-info/PKG-INFO
|
|
11
|
+
xyberos_storage.egg-info/SOURCES.txt
|
|
12
|
+
xyberos_storage.egg-info/dependency_links.txt
|
|
13
|
+
xyberos_storage.egg-info/entry_points.txt
|
|
14
|
+
xyberos_storage.egg-info/requires.txt
|
|
15
|
+
xyberos_storage.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
xyberos_storage
|