semantic-kernel 1.8.3__zip

semantic_kernel-1.8.3/semantic_kernel/connectors/memory/qdrant/qdrant_store.py

@@ -0,0 +1,135 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+import sys
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, TypeVar
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+from pydantic import ValidationError
+from qdrant_client.async_qdrant_client import AsyncQdrantClient
+
+from semantic_kernel.connectors.memory.qdrant.qdrant_collection import QdrantCollection
+from semantic_kernel.data.vector_store import VectorStore
+from semantic_kernel.data.vector_store_model_definition import VectorStoreRecordDefinition
+from semantic_kernel.exceptions import MemoryConnectorInitializationError
+from semantic_kernel.utils.experimental_decorator import experimental_class
+from semantic_kernel.utils.telemetry.user_agent import APP_INFO, prepend_semantic_kernel_to_user_agent
+
+if TYPE_CHECKING:
+    from semantic_kernel.data.vector_store_record_collection import VectorStoreRecordCollection
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+TModel = TypeVar("TModel")
+TKey = TypeVar("TKey", str, int)
+
+
+@experimental_class
+class QdrantStore(VectorStore):
+    """A QdrantStore is a memory store that uses Qdrant as the backend."""
+
+    qdrant_client: AsyncQdrantClient
+
+    def __init__(
+        self,
+        url: str | None = None,
+        api_key: str | None = None,
+        host: str | None = None,
+        port: int | None = None,
+        grpc_port: int | None = None,
+        path: str | None = None,
+        location: str | None = None,
+        prefer_grpc: bool | None = None,
+        client: AsyncQdrantClient | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """Initializes a new instance of the QdrantVectorRecordStore.
+
+        When using qdrant client, make sure to supply url and api_key.
+        When using qdrant server, make sure to supply url or host and optionally port.
+        When using qdrant local, either supply path to use a persisted qdrant instance
+            or set location to ":memory:" to use an in-memory qdrant instance.
+        When nothing is supplied, it defaults to an in-memory qdrant instance.
+        You can also supply a async qdrant client directly.
+
+        Args:
+            url (str): The URL of the Qdrant server (default: {None}).
+            api_key (str): The API key for the Qdrant server (default: {None}).
+            host (str): The host of the Qdrant server (default: {None}).
+            port (int): The port of the Qdrant server (default: {None}).
+            grpc_port (int): The gRPC port of the Qdrant server (default: {None}).
+            path (str): The path of the Qdrant server (default: {None}).
+            location (str): The location of the Qdrant server (default: {None}).
+            prefer_grpc (bool): If true, gRPC will be preferred (default: {None}).
+            client (AsyncQdrantClient): The Qdrant client to use (default: {None}).
+            env_file_path (str): Use the environment settings file as a fallback to environment variables.
+            env_file_encoding (str): The encoding of the environment settings file.
+            **kwargs: Additional keyword arguments passed to the client constructor.
+
+        """
+        if client:
+            super().__init__(qdrant_client=client, **kwargs)
+            return
+
+        from semantic_kernel.connectors.memory.qdrant.qdrant_settings import QdrantSettings
+
+        try:
+            settings = QdrantSettings.create(
+                url=url,
+                api_key=api_key,
+                host=host,
+                port=port,
+                grpc_port=grpc_port,
+                path=path,
+                location=location,
+                prefer_grpc=prefer_grpc,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise MemoryConnectorInitializationError("Failed to create Qdrant settings.", ex) from ex
+        if APP_INFO:
+            kwargs.setdefault("metadata", {})
+            kwargs["metadata"] = prepend_semantic_kernel_to_user_agent(kwargs["metadata"])
+        try:
+            client = AsyncQdrantClient(**settings.model_dump(exclude_none=True), **kwargs)
+        except ValueError as ex:
+            raise MemoryConnectorInitializationError("Failed to create Qdrant client.", ex) from ex
+        super().__init__(qdrant_client=client)
+
+    def get_collection(
+        self,
+        collection_name: str,
+        data_model_type: type[TModel],
+        data_model_definition: VectorStoreRecordDefinition | None = None,
+        **kwargs: Any,
+    ) -> "VectorStoreRecordCollection":
+        """Get a QdrantCollection tied to a collection.
+
+        Args:
+            collection_name (str): The name of the collection.
+            data_model_type (type[TModel]): The type of the data model.
+            data_model_definition (VectorStoreRecordDefinition | None): The model fields, optional.
+            **kwargs: Additional keyword arguments, passed to the collection constructor.
+        """
+        if collection_name not in self.vector_record_collections:
+            self.vector_record_collections[collection_name] = QdrantCollection[data_model_type](
+                data_model_type=data_model_type,
+                data_model_definition=data_model_definition,
+                collection_name=collection_name,
+                client=self.qdrant_client,
+                **kwargs,
+            )
+        return self.vector_record_collections[collection_name]
+
+    @override
+    async def list_collection_names(self, **kwargs: Any) -> Sequence[str]:
+        collections = await self.qdrant_client.get_collections()
+        return [collection.name for collection in collections.collections]

semantic_kernel-1.8.3/semantic_kernel/connectors/memory/qdrant/utils.py

@@ -0,0 +1,15 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+import contextlib
+
+from qdrant_client.async_qdrant_client import AsyncQdrantClient
+
+
+class AsyncQdrantClientWrapper(AsyncQdrantClient):
+    """Wrapper to make sure the connection is closed when the object is deleted."""
+
+    def __del__(self) -> None:
+        """Async close connection, done when the object is deleted, used when SK creates a client."""
+        with contextlib.suppress(Exception):
+            asyncio.get_running_loop().create_task(self.close())

semantic_kernel-1.8.3/semantic_kernel/connectors/memory/redis/README.md

@@ -0,0 +1,31 @@
+# semantic_kernel.connectors.memory.redis
+
+This connector uses Redis to implement Semantic Memory. It requires the [RediSearch](https://redis.io/docs/interact/search-and-query/) module to be enabled on Redis to implement vector similarity search.
+
+See the [.net README](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Connectors/Connectors.Memory.Redis/README.md) for more information.
+
+## Quick start
+
+1. Run with Docker:
+
+```bash
+docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest
+```
+
+2. To use Redis as a semantic memory store:
+
+```python
+    import semantic_kernel as sk
+    import semantic_kernel.connectors.ai.open_ai as sk_oai
+    from semantic_kernel.connectors.memory.redis import RedisMemoryStore
+
+    kernel = sk.Kernel()
+
+    api_key, org_id = sk.openai_settings_from_dot_env()
+    kernel.add_service(sk_oai.OpenAITextCompletion(service_id="dv", ai_model_id="text-davinci-003", api_key=api_key, org_id=org_id))
+    embedding_generator = sk_oai.OpenAITextEmbedding(service_id="ada", ai_model_id="text-embedding-ada-002", api_key=api_key, org_id=org_id)
+    kernel.add_service(embedding_generator)
+
+    redis_connection_string = sk.redis_settings_from_dot_env()
+    kernel.use_memory(storage=RedisMemoryStore(connection_string=redis_connection_string), embeddings_generator=embedding_generator)
+```

semantic_kernel-1.8.3/semantic_kernel/connectors/memory/redis/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from semantic_kernel.connectors.memory.redis.const import RedisCollectionTypes
+from semantic_kernel.connectors.memory.redis.redis_collection import RedisHashsetCollection, RedisJsonCollection
+from semantic_kernel.connectors.memory.redis.redis_memory_store import (
+    RedisMemoryStore,
+)
+from semantic_kernel.connectors.memory.redis.redis_settings import RedisSettings
+from semantic_kernel.connectors.memory.redis.redis_store import RedisStore
+
+__all__ = [
+    "RedisCollectionTypes",
+    "RedisHashsetCollection",
+    "RedisJsonCollection",
+    "RedisMemoryStore",
+    "RedisSettings",
+    "RedisStore",
+]

semantic_kernel-1.8.3/semantic_kernel/connectors/memory/redis/const.py

@@ -0,0 +1,39 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+
+from enum import Enum
+
+from redis.commands.search.indexDefinition import IndexType
+
+from semantic_kernel.data.const import DistanceFunction
+
+
+class RedisCollectionTypes(str, Enum):
+    JSON = "json"
+    HASHSET = "hashset"
+
+
+INDEX_TYPE_MAP = {
+    RedisCollectionTypes.JSON: IndexType.JSON,
+    RedisCollectionTypes.HASHSET: IndexType.HASH,
+}
+
+DISTANCE_FUNCTION_MAP = {
+    DistanceFunction.COSINE: "COSINE",
+    DistanceFunction.DOT_PROD: "IP",
+    DistanceFunction.EUCLIDEAN: "L2",
+    "default": "COSINE",
+}
+
+TYPE_MAPPER_VECTOR = {
+    "float": "FLOAT32",
+    "int": "FLOAT16",
+    "binary": "FLOAT16",
+    "ndarray": "FLOAT32",
+    "default": "FLOAT32",
+}
+
+__all__ = [
+    "DISTANCE_FUNCTION_MAP",
+    "TYPE_MAPPER_VECTOR",
+]

semantic_kernel-1.8.3/semantic_kernel/connectors/memory/redis/redis_collection.py

@@ -0,0 +1,333 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+import json
+import logging
+import sys
+from collections.abc import Sequence
+from copy import copy
+from typing import Any, ClassVar, TypeVar
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+import numpy as np
+from pydantic import ValidationError
+from redis.asyncio.client import Redis
+from redis.commands.search.indexDefinition import IndexDefinition
+
+from semantic_kernel.connectors.memory.redis.const import INDEX_TYPE_MAP, RedisCollectionTypes
+from semantic_kernel.connectors.memory.redis.utils import RedisWrapper, data_model_definition_to_redis_fields
+from semantic_kernel.data.vector_store_model_definition import VectorStoreRecordDefinition
+from semantic_kernel.data.vector_store_record_collection import VectorStoreRecordCollection
+from semantic_kernel.data.vector_store_record_fields import (
+    VectorStoreRecordKeyField,
+    VectorStoreRecordVectorField,
+)
+from semantic_kernel.exceptions.memory_connector_exceptions import (
+    MemoryConnectorException,
+    MemoryConnectorInitializationError,
+)
+from semantic_kernel.utils.experimental_decorator import experimental_class
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+TModel = TypeVar("TModel")
+
+
+@experimental_class
+class RedisCollection(VectorStoreRecordCollection[str, TModel]):
+    """A vector store record collection implementation using Redis."""
+
+    redis_database: Redis
+    prefix_collection_name_to_key_names: bool
+    collection_type: RedisCollectionTypes
+    supported_key_types: ClassVar[list[str] | None] = ["str"]
+    supported_vector_types: ClassVar[list[str] | None] = ["float"]
+
+    def __init__(
+        self,
+        data_model_type: type[TModel],
+        data_model_definition: VectorStoreRecordDefinition | None = None,
+        collection_name: str | None = None,
+        redis_database: Redis | None = None,
+        prefix_collection_name_to_key_names: bool = True,
+        collection_type: RedisCollectionTypes = RedisCollectionTypes.HASHSET,
+        connection_string: str | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+    ) -> None:
+        """RedisMemoryStore is an abstracted interface to interact with a Redis node connection.
+
+        See documentation about connections: https://redis-py.readthedocs.io/en/stable/connections.html
+        See documentation about vector attributes: https://redis.io/docs/stack/search/reference/vectors.
+
+        """
+        if redis_database:
+            super().__init__(
+                data_model_type=data_model_type,
+                data_model_definition=data_model_definition,
+                collection_name=collection_name,
+                redis_database=redis_database,
+                prefix_collection_name_to_key_names=prefix_collection_name_to_key_names,
+                collection_type=collection_type,
+            )
+            return
+        try:
+            from semantic_kernel.connectors.memory.redis.redis_settings import RedisSettings
+
+            redis_settings = RedisSettings.create(
+                connection_string=connection_string,
+                env_file_path=env_file_path,
+                env_file_encoding=env_file_encoding,
+            )
+        except ValidationError as ex:
+            raise MemoryConnectorInitializationError("Failed to create Redis settings.", ex) from ex
+        super().__init__(
+            data_model_type=data_model_type,
+            data_model_definition=data_model_definition,
+            collection_name=collection_name,
+            redis_database=RedisWrapper.from_url(redis_settings.connection_string.get_secret_value()),
+            prefix_collection_name_to_key_names=prefix_collection_name_to_key_names,
+            collection_type=collection_type,
+        )
+
+    def _get_redis_key(self, key: str) -> str:
+        if self.prefix_collection_name_to_key_names:
+            return f"{self.collection_name}:{key}"
+        return key
+
+    def _unget_redis_key(self, key: str) -> str:
+        if self.prefix_collection_name_to_key_names and ":" in key:
+            return key[len(self.collection_name) + 1 :]
+        return key
+
+    @override
+    async def create_collection(self, **kwargs) -> None:
+        """Create a new index in Redis.
+
+        Args:
+            **kwargs: Additional keyword arguments.
+                fields (list[Fields]): The fields to create the index with, when not supplied,
+                    these are created from the data_model_definition.
+                index_definition (IndexDefinition): The search index to create, if this is supplied
+                    this is used instead of a index created based on the definition.
+                other kwargs are passed to the create_index method.
+        """
+        if (index_definition := kwargs.pop("index_definition", None)) and (fields := kwargs.pop("fields", None)):
+            if isinstance(index_definition, IndexDefinition):
+                await self.redis_database.ft(self.collection_name).create_index(
+                    fields, definition=index_definition, **kwargs
+                )
+                return
+            raise MemoryConnectorException("Invalid index type supplied.")
+        fields = data_model_definition_to_redis_fields(self.data_model_definition, self.collection_type)
+        index_definition = IndexDefinition(
+            prefix=f"{self.collection_name}:", index_type=INDEX_TYPE_MAP[self.collection_type]
+        )
+        await self.redis_database.ft(self.collection_name).create_index(fields, definition=index_definition, **kwargs)
+
+    @override
+    async def does_collection_exist(self, **kwargs) -> bool:
+        try:
+            await self.redis_database.ft(self.collection_name).info()
+            return True
+        except Exception:
+            return False
+
+    @override
+    async def delete_collection(self, **kwargs) -> None:
+        exists = await self.does_collection_exist()
+        if exists:
+            await self.redis_database.ft(self.collection_name).dropindex(**kwargs)
+        else:
+            logger.debug("Collection does not exist, skipping deletion.")
+
+
+@experimental_class
+class RedisHashsetCollection(RedisCollection):
+    """A vector store record collection implementation using Redis Hashsets."""
+
+    def __init__(
+        self,
+        data_model_type: type[TModel],
+        data_model_definition: VectorStoreRecordDefinition | None = None,
+        collection_name: str | None = None,
+        redis_database: Redis | None = None,
+        prefix_collection_name_to_key_names: bool = False,
+        connection_string: str | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """RedisMemoryStore is an abstracted interface to interact with a Redis node connection.
+
+        See documentation about connections: https://redis-py.readthedocs.io/en/stable/connections.html
+        See documentation about vector attributes: https://redis.io/docs/stack/search/reference/vectors.
+
+        """
+        super().__init__(
+            data_model_type=data_model_type,
+            data_model_definition=data_model_definition,
+            collection_name=collection_name,
+            redis_database=redis_database,
+            prefix_collection_name_to_key_names=prefix_collection_name_to_key_names,
+            collection_type=RedisCollectionTypes.HASHSET,
+            connection_string=connection_string,
+            env_file_path=env_file_path,
+            env_file_encoding=env_file_encoding,
+            **kwargs,
+        )
+
+    @override
+    async def _inner_upsert(self, records: Sequence[Any], **kwargs: Any) -> Sequence[str]:
+        return await asyncio.gather(*[self._single_upsert(record) for record in records])
+
+    async def _single_upsert(self, upsert_record: Any) -> str:
+        await self.redis_database.hset(**upsert_record)
+        return self._unget_redis_key(upsert_record["name"])
+
+    @override
+    async def _inner_get(self, keys: Sequence[str], **kwargs) -> Sequence[dict[bytes, bytes]] | None:
+        results = await asyncio.gather(*[self.redis_database.hgetall(self._get_redis_key(key)) for key in keys])
+        return [result for result in results if result]
+
+    @override
+    async def _inner_delete(self, keys: Sequence[str], **kwargs: Any) -> None:
+        await self.redis_database.delete(*[self._get_redis_key(key) for key in keys])
+
+    @override
+    def _serialize_dicts_to_store_models(
+        self,
+        records: Sequence[dict[str, Any]],
+        **kwargs: Any,
+    ) -> Sequence[dict[str, Any]]:
+        """Serialize the dict to a Redis store model."""
+        results = []
+        for record in records:
+            result = {"mapping": {}}
+            metadata = {}
+            for name, field in self.data_model_definition.fields.items():
+                if isinstance(field, VectorStoreRecordVectorField):
+                    if not isinstance(record[name], np.ndarray):
+                        record[name] = np.array(record[name])
+                    result["mapping"][name] = record[name].tobytes()
+                    continue
+                if isinstance(field, VectorStoreRecordKeyField):
+                    result["name"] = self._get_redis_key(record[name])
+                    continue
+                metadata[name] = record[field.name]
+            result["mapping"]["metadata"] = json.dumps(metadata)
+            results.append(result)
+        return results
+
+    @override
+    def _deserialize_store_models_to_dicts(
+        self,
+        records: Sequence[dict[bytes, bytes]],
+        keys: Sequence[str],
+        **kwargs: Any,
+    ) -> Sequence[dict[str, Any]]:
+        results = []
+        for key, record in zip(keys, records):
+            if record:
+                flattened = json.loads(record[b"metadata"])
+                for name, field in self.data_model_definition.fields.items():
+                    if isinstance(field, VectorStoreRecordKeyField):
+                        flattened[name] = self._unget_redis_key(key)
+                    if isinstance(field, VectorStoreRecordVectorField):
+                        flattened[name] = np.frombuffer(record[name.encode()]).tolist()
+                results.append(flattened)
+        return results
+
+
+@experimental_class
+class RedisJsonCollection(RedisCollection):
+    """A vector store record collection implementation using Redis Json."""
+
+    def __init__(
+        self,
+        data_model_type: type[TModel],
+        data_model_definition: VectorStoreRecordDefinition | None = None,
+        collection_name: str | None = None,
+        redis_database: Redis | None = None,
+        prefix_collection_name_to_key_names: bool = False,
+        connection_string: str | None = None,
+        env_file_path: str | None = None,
+        env_file_encoding: str | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """RedisMemoryStore is an abstracted interface to interact with a Redis node connection.
+
+        See documentation about connections: https://redis-py.readthedocs.io/en/stable/connections.html
+        See documentation about vector attributes: https://redis.io/docs/stack/search/reference/vectors.
+
+        """
+        super().__init__(
+            data_model_type=data_model_type,
+            data_model_definition=data_model_definition,
+            collection_name=collection_name,
+            redis_database=redis_database,
+            prefix_collection_name_to_key_names=prefix_collection_name_to_key_names,
+            collection_type=RedisCollectionTypes.JSON,
+            connection_string=connection_string,
+            env_file_path=env_file_path,
+            env_file_encoding=env_file_encoding,
+            **kwargs,
+        )
+
+    @override
+    async def _inner_upsert(self, records: Sequence[Any], **kwargs: Any) -> Sequence[str]:
+        return await asyncio.gather(*[self._single_upsert(record) for record in records])
+
+    async def _single_upsert(self, upsert_record: Any) -> str:
+        await self.redis_database.json().set(upsert_record["name"], "$", upsert_record["value"])
+        return self._unget_redis_key(upsert_record["name"])
+
+    @override
+    async def _inner_get(self, keys: Sequence[str], **kwargs) -> Sequence[dict[bytes, bytes]] | None:
+        kwargs_copy = copy(kwargs)
+        kwargs_copy.pop("include_vectors", None)
+        results = await self.redis_database.json().mget([self._get_redis_key(key) for key in keys], "$", **kwargs_copy)
+        return [result[0] for result in results if result]
+
+    @override
+    async def _inner_delete(self, keys: Sequence[str], **kwargs: Any) -> None:
+        await asyncio.gather(*[self.redis_database.json().delete(key, **kwargs) for key in keys])
+
+    @override
+    def _serialize_dicts_to_store_models(
+        self,
+        records: Sequence[dict[str, Any]],
+        **kwargs: Any,
+    ) -> Sequence[dict[str, Any]]:
+        """Serialize the dict to a Redis store model."""
+        results = []
+        for record in records:
+            result = {"value": {}}
+            for name, field in self.data_model_definition.fields.items():
+                if isinstance(field, VectorStoreRecordKeyField):
+                    result["name"] = self._get_redis_key(record[name])
+                    continue
+                if isinstance(field, VectorStoreRecordVectorField):
+                    if isinstance(record[name], np.ndarray):
+                        record[name] = record[name].tolist()
+                    result["value"][name] = record[name]
+                result["value"][name] = record[name]
+            results.append(result)
+        return results
+
+    @override
+    def _deserialize_store_models_to_dicts(
+        self,
+        records: Sequence[dict[str, Any]],
+        keys: Sequence[str],
+        **kwargs: Any,
+    ) -> Sequence[dict[str, Any]]:
+        results = []
+        for key, record in zip(keys, records):
+            record[self.data_model_definition.key_field_name] = self._unget_redis_key(key)
+            results.append(record)
+        return results