microsoft-agents-storage-blob 0.0.0__tar.gz

microsoft_agents_storage_blob-0.0.0/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: microsoft-agents-storage-blob
+Version: 0.0.0
+Summary: A blob storage library for Microsoft Agents
+Author: Microsoft Corporation
+Project-URL: Homepage, https://github.com/microsoft/Agents
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Requires-Dist: microsoft-agents-hosting-core==0.0.0
+Requires-Dist: azure-core
+Requires-Dist: azure-storage-blob
+Dynamic: requires-dist

microsoft_agents_storage_blob-0.0.0/microsoft/agents/storage/blob/__init__.py

@@ -0,0 +1,4 @@
+from .blob_storage import BlobStorage
+from .blob_storage_config import BlobStorageConfig
+
+__all__ = ["BlobStorage", "BlobStorageConfig"]

microsoft_agents_storage_blob-0.0.0/microsoft/agents/storage/blob/blob_storage.py

@@ -0,0 +1,100 @@
+import json
+from typing import TypeVar, Union
+from io import BytesIO
+
+from azure.storage.blob.aio import (
+    ContainerClient,
+    BlobServiceClient,
+)
+
+from microsoft.agents.hosting.core.storage import StoreItem
+from microsoft.agents.hosting.core.storage.storage import AsyncStorageBase
+from microsoft.agents.hosting.core.storage._type_aliases import JSON
+from microsoft.agents.hosting.core.storage.error_handling import (
+    ignore_error,
+    is_status_code_error,
+)
+
+from .blob_storage_config import BlobStorageConfig
+
+StoreItemT = TypeVar("StoreItemT", bound=StoreItem)
+
+
+class BlobStorage(AsyncStorageBase):
+
+    def __init__(self, config: BlobStorageConfig):
+
+        if not config.container_name:
+            raise ValueError("BlobStorage: Container name is required.")
+
+        self.config = config
+
+        blob_service_client: BlobServiceClient = self._create_client()
+        self._container_client: ContainerClient = (
+            blob_service_client.get_container_client(config.container_name)
+        )
+        self._initialized: bool = False
+
+    def _create_client(self) -> BlobServiceClient:
+        if self.config.url:  # connect with URL and credentials
+            if not self.config.credential:
+                raise ValueError(
+                    "BlobStorage: Credential is required when using a custom service URL."
+                )
+            return BlobServiceClient(
+                account_url=self.config.url, credential=self.config.credential
+            )
+
+        else:  # connect with connection string
+            return BlobServiceClient.from_connection_string(
+                self.config.connection_string
+            )
+
+    async def initialize(self) -> None:
+        """Initializes the storage container"""
+        if not self._initialized:
+            # This should only happen once - assuming this is a singleton.
+            await ignore_error(
+                self._container_client.create_container(), is_status_code_error(409)
+            )
+            self._initialized = True
+
+    async def _read_item(
+        self, key: str, *, target_cls: StoreItemT = None, **kwargs
+    ) -> tuple[Union[str, None], Union[StoreItemT, None]]:
+        item = await ignore_error(
+            self._container_client.download_blob(blob=key),
+            is_status_code_error(404),
+        )
+        if not item:
+            return None, None
+
+        item_rep: str = await item.readall()
+        item_JSON: JSON = json.loads(item_rep)
+        try:
+            return key, target_cls.from_json_to_store_item(item_JSON)
+        except AttributeError as error:
+            raise TypeError(
+                f"BlobStorage.read_item(): could not deserialize blob item into {target_cls} class. Error: {error}"
+            )
+
+    async def _write_item(self, key: str, item: StoreItem) -> None:
+        item_JSON: JSON = item.store_item_to_json()
+        if item_JSON is None:
+            raise ValueError(
+                "BlobStorage.write(): StoreItem serialization cannot return None"
+            )
+        item_rep_bytes = json.dumps(item_JSON).encode("utf-8")
+
+        # getting the length is important for performance with large blobs
+        await self._container_client.upload_blob(
+            name=key,
+            data=BytesIO(item_rep_bytes),
+            overwrite=True,
+            length=len(item_rep_bytes),
+        )
+
+    async def _delete_item(self, key: str) -> None:
+        await ignore_error(
+            self._container_client.delete_blob(blob=key), is_status_code_error(404)
+        )

microsoft_agents_storage_blob-0.0.0/microsoft/agents/storage/blob/blob_storage_config.py

@@ -0,0 +1,28 @@
+from typing import Union
+
+from azure.core.credentials import TokenCredential
+
+
+class BlobStorageConfig:
+    """Configuration settings for BlobStorage."""
+
+    def __init__(
+        self,
+        container_name: str,
+        connection_string: str = "",
+        url: str = "",
+        credential: Union[TokenCredential, None] = None,
+    ):
+        """Configuration settings for BlobStorage.
+
+        container_name: The name of the blob container.
+        connection_string: The connection string to the storage account.
+        url: The URL of the blob service. If provided, credential must also be provided.
+        credential: The TokenCredential to use for authentication when using a custom URL.
+
+        credential-based authentication is prioritized over connection string authentication.
+        """
+        self.container_name: str = container_name
+        self.connection_string: str = connection_string
+        self.url: str = url
+        self.credential: Union[TokenCredential, None] = credential

microsoft_agents_storage_blob-0.0.0/microsoft_agents_storage_blob.egg-info/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: microsoft-agents-storage-blob
+Version: 0.0.0
+Summary: A blob storage library for Microsoft Agents
+Author: Microsoft Corporation
+Project-URL: Homepage, https://github.com/microsoft/Agents
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Requires-Dist: microsoft-agents-hosting-core==0.0.0
+Requires-Dist: azure-core
+Requires-Dist: azure-storage-blob
+Dynamic: requires-dist

microsoft_agents_storage_blob-0.0.0/microsoft_agents_storage_blob.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+pyproject.toml
+setup.py
+microsoft/agents/storage/blob/__init__.py
+microsoft/agents/storage/blob/blob_storage.py
+microsoft/agents/storage/blob/blob_storage_config.py
+microsoft_agents_storage_blob.egg-info/PKG-INFO
+microsoft_agents_storage_blob.egg-info/SOURCES.txt
+microsoft_agents_storage_blob.egg-info/dependency_links.txt
+microsoft_agents_storage_blob.egg-info/requires.txt
+microsoft_agents_storage_blob.egg-info/top_level.txt
+tests/test_blob_storage.py

microsoft_agents_storage_blob-0.0.0/microsoft_agents_storage_blob.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

microsoft_agents_storage_blob-0.0.0/microsoft_agents_storage_blob.egg-info/requires.txt

@@ -0,0 +1,3 @@
+microsoft-agents-hosting-core==0.0.0
+azure-core
+azure-storage-blob

microsoft_agents_storage_blob-0.0.0/microsoft_agents_storage_blob.egg-info/top_level.txt

@@ -0,0 +1 @@
+microsoft

microsoft_agents_storage_blob-0.0.0/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "microsoft-agents-storage-blob"
+dynamic = ["version", "dependencies"]
+description = "A blob storage library for Microsoft Agents"
+authors = [{name = "Microsoft Corporation"}]
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/microsoft/Agents"

microsoft_agents_storage_blob-0.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

microsoft_agents_storage_blob-0.0.0/setup.py

@@ -0,0 +1,13 @@
+from os import environ
+from setuptools import setup
+
+package_version = environ.get("PackageVersion", "0.0.0")
+
+setup(
+    version=package_version,
+    install_requires=[
+        f"microsoft-agents-hosting-core=={package_version}",
+        "azure-core",
+        "azure-storage-blob",
+    ],
+)

microsoft_agents_storage_blob-0.0.0/tests/test_blob_storage.py

@@ -0,0 +1,196 @@
+import json
+import gc
+import os
+from io import BytesIO
+
+import pytest
+import pytest_asyncio
+
+from microsoft.agents.storage.blob import BlobStorage, BlobStorageConfig
+from azure.storage.blob.aio import BlobServiceClient
+from azure.core.exceptions import ResourceNotFoundError
+
+from microsoft.agents.hosting.core.storage.storage_test_utils import (
+    CRUDStorageTests,
+    StorageBaseline,
+    MockStoreItem,
+    MockStoreItemB,
+)
+
+EMULATOR_RUNNING = False
+
+
+async def blob_storage_instance(existing=False):
+
+    # Default Azure Storage Emulator connection string
+    connection_string = (
+        "AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq"
+        + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint="
+        + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;"
+        + "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
+    )
+
+    blob_service_client = BlobServiceClient.from_connection_string(connection_string)
+
+    container_name = "asdkunittest"
+
+    if not existing:
+
+        # reset state of test container
+        try:
+            container_client = blob_service_client.get_container_client(container_name)
+            await container_client.delete_container()
+        except ResourceNotFoundError:
+            pass
+
+        container_client = await blob_service_client.create_container(container_name)
+    else:
+        container_client = blob_service_client.get_container_client(container_name)
+
+    blob_storage_config = BlobStorageConfig(
+        container_name=container_name,
+        connection_string=connection_string,
+    )
+
+    storage = BlobStorage(blob_storage_config)
+    return storage, container_client
+
+
+@pytest_asyncio.fixture
+async def blob_storage():
+
+    # setup
+    storage, container_client = await blob_storage_instance()
+
+    yield storage
+
+    # teardown
+    await container_client.delete_container()
+
+
+@pytest.mark.skipif(not EMULATOR_RUNNING, reason="Needs the emulator to run.")
+class TestBlobStorage(CRUDStorageTests):
+
+    async def storage(self, initial_data=None, existing=False):
+        if not initial_data:
+            initial_data = {}
+        storage, container_client = await blob_storage_instance(existing=existing)
+
+        for key, value in initial_data.items():
+            value_rep = json.dumps(value.store_item_to_json())
+            await container_client.upload_blob(name=key, data=value_rep, overwrite=True)
+
+        return storage
+
+    @pytest.mark.asyncio
+    async def test_initialize(self, blob_storage):
+        await blob_storage.initialize()
+        await blob_storage.initialize()
+        await blob_storage.write(
+            {"key": MockStoreItem({"id": "item", "value": "data"})}
+        )
+        await blob_storage.initialize()
+        assert (await blob_storage.read(["key"], target_cls=MockStoreItem)) == {
+            "key": MockStoreItem({"id": "item", "value": "data"})
+        }
+
+    @pytest.mark.asyncio
+    async def test_external_change_is_visible(self):
+        blob_storage, container_client = await blob_storage_instance()
+        assert (await blob_storage.read(["key"], target_cls=MockStoreItem)) == {}
+        assert (await blob_storage.read(["key2"], target_cls=MockStoreItem)) == {}
+        await container_client.upload_blob(
+            name="key", data=json.dumps({"id": "item", "value": "data"}), overwrite=True
+        )
+        await container_client.upload_blob(
+            name="key2",
+            data=json.dumps({"id": "another_item", "value": "new_val"}),
+            overwrite=True,
+        )
+        assert (await blob_storage.read(["key"], target_cls=MockStoreItem))[
+            "key"
+        ] == MockStoreItem({"id": "item", "value": "data"})
+        assert (await blob_storage.read(["key2"], target_cls=MockStoreItem))[
+            "key2"
+        ] == MockStoreItem({"id": "another_item", "value": "new_val"})
+
+    @pytest.mark.asyncio
+    async def test_blob_storage_flow_existing_container_and_persistence(self):
+
+        connection_string = (
+            "AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq"
+            + "2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint="
+            + "http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;"
+            + "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
+        )
+        blob_service_client = BlobServiceClient.from_connection_string(
+            connection_string
+        )
+        container_name = "asdkunittestpopulated"
+        container_client = blob_service_client.get_container_client(container_name)
+
+        # reset state of test container
+        try:
+            await container_client.delete_container()
+        except ResourceNotFoundError:
+            pass
+        await container_client.create_container()
+
+        initial_data = {
+            "item1": MockStoreItem({"id": "item1", "value": "data1"}),
+            "__some_key": MockStoreItem({"id": "item2", "value": "data2"}),
+            "!another_key": MockStoreItem({"id": "item3", "value": "data3"}),
+            "1230": MockStoreItemB({"id": "item8", "value": "data"}, False),
+            "key-with-dash": MockStoreItem({"id": "item4", "value": "data"}),
+            "key.with.dot": MockStoreItem({"id": "item5", "value": "data"}),
+            "key/with/slash": MockStoreItem({"id": "item6", "value": "data"}),
+            "another key": MockStoreItemB({"id": "item7", "value": "data"}, True),
+        }
+
+        baseline_storage = StorageBaseline(initial_data)
+
+        for key, value in initial_data.items():
+            value_rep = json.dumps(value.store_item_to_json()).encode("utf-8")
+            await container_client.upload_blob(
+                name=key, data=BytesIO(value_rep), overwrite=True
+            )
+
+        blob_storage_config = BlobStorageConfig(
+            container_name=container_name, connection_string=connection_string
+        )
+
+        storage = BlobStorage(blob_storage_config)
+
+        assert await baseline_storage.equals(storage)
+        assert (
+            await storage.read(["1230", "another key"], target_cls=MockStoreItemB)
+        ) == baseline_storage.read(["1230", "another key"])
+
+        changes = {
+            "item1": MockStoreItem({"id": "item1", "value": "data1_changed"}),
+            "__some_key": MockStoreItem({"id": "item2", "value": "data2_changed"}),
+            "new_item": MockStoreItem({"id": "new_item", "value": "new_data"}),
+        }
+
+        baseline_storage.write(changes)
+        await storage.write(changes)
+
+        baseline_storage.delete(["!another_key", "item1"])
+        await storage.delete(["!another_key", "item1"])
+        assert await baseline_storage.equals(storage)
+
+        del storage
+        gc.collect()
+
+        blob_client = container_client.get_blob_client("item1")
+        with pytest.raises(ResourceNotFoundError):
+            await (await blob_client.download_blob()).readall()
+
+        blob_client = container_client.get_blob_client("1230")
+        item = await (await blob_client.download_blob()).readall()
+        assert (
+            MockStoreItemB.from_json_to_store_item(json.loads(item))
+            == initial_data["1230"]
+        )
+
+        await container_client.delete_container()