semantic-kernel 1.8.3__zip

semantic_kernel-1.8.3/semantic_kernel/memory/memory_record.py

@@ -0,0 +1,150 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from datetime import datetime
+
+from numpy import ndarray
+
+from semantic_kernel.utils.experimental_decorator import experimental_class
+
+
+@experimental_class
+class MemoryRecord:
+    """The in-built memory record."""
+
+    _key: str
+    _timestamp: datetime | None
+    _is_reference: bool
+    _external_source_name: str | None
+    _id: str
+    _description: str | None
+    _text: str | None
+    _additional_metadata: str | None
+    _embedding: ndarray
+
+    def __init__(
+        self,
+        is_reference: bool,
+        external_source_name: str | None,
+        id: str,
+        description: str | None,
+        text: str | None,
+        additional_metadata: str | None,
+        embedding: ndarray | None,
+        key: str | None = None,
+        timestamp: datetime | None = None,
+    ) -> None:
+        """Initialize a new instance of MemoryRecord.
+
+        Args:
+            is_reference (bool): Whether the record is a reference record.
+            external_source_name (Optional[str]): The name of the external source.
+            id (str): A unique for the record.
+            description (Optional[str]): The description of the record.
+            text (Optional[str]): The text of the record.
+            additional_metadata (Optional[str]): Custom metadata for the record.
+            embedding (ndarray): The embedding of the record.
+            key (Optional[str]): The key of the record.
+            timestamp (Optional[datetime]): The timestamp of the record.
+        """
+        self._key = key
+        self._timestamp = timestamp
+        self._is_reference = is_reference
+        self._external_source_name = external_source_name
+        self._id = id
+        self._description = description
+        self._text = text
+        self._additional_metadata = additional_metadata
+        self._embedding = embedding
+
+    @staticmethod
+    def reference_record(
+        external_id: str,
+        source_name: str,
+        description: str | None,
+        additional_metadata: str | None,
+        embedding: ndarray,
+    ) -> "MemoryRecord":
+        """Create a reference record.
+
+        Args:
+            external_id (str): The external id of the record.
+            source_name (str): The name of the external source.
+            description (Optional[str]): The description of the record.
+            additional_metadata (Optional[str]): Custom metadata for the record.
+            embedding (ndarray): The embedding of the record.
+
+        Returns:
+            MemoryRecord: The reference record.
+        """
+        return MemoryRecord(
+            is_reference=True,
+            external_source_name=source_name,
+            id=external_id,
+            description=description,
+            text=None,
+            additional_metadata=additional_metadata,
+            embedding=embedding,
+        )
+
+    @staticmethod
+    def local_record(
+        id: str,
+        text: str,
+        description: str | None,
+        additional_metadata: str | None,
+        embedding: ndarray,
+        timestamp: datetime | None = None,
+    ) -> "MemoryRecord":
+        """Create a local record.
+
+        Args:
+            id (str): A unique for the record.
+            text (str): The text of the record.
+            description (Optional[str]): The description of the record.
+            additional_metadata (Optional[str]): Custom metadata for the record.
+            embedding (ndarray): The embedding of the record.
+            timestamp (Optional[datetime]): The timestamp of the record.
+
+        Returns:
+            MemoryRecord: The local record.
+        """
+        return MemoryRecord(
+            is_reference=False,
+            external_source_name=None,
+            id=id,
+            description=description,
+            text=text,
+            additional_metadata=additional_metadata,
+            timestamp=timestamp,
+            embedding=embedding,
+        )
+
+    @property
+    def id(self):
+        """Get the unique identifier for the memory record."""
+        return self._id
+
+    @property
+    def embedding(self) -> ndarray:
+        """Get the embedding of the memory record."""
+        return self._embedding
+
+    @property
+    def text(self):
+        """Get the text of the memory record."""
+        return self._text
+
+    @property
+    def additional_metadata(self):
+        """Get the additional metadata of the memory record."""
+        return self._additional_metadata
+
+    @property
+    def description(self):
+        """Get the description of the memory record."""
+        return self._description
+
+    @property
+    def timestamp(self):
+        """Get the timestamp of the memory record."""
+        return self._timestamp

semantic_kernel-1.8.3/semantic_kernel/memory/memory_store_base.py

@@ -0,0 +1,198 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from abc import ABC, abstractmethod
+
+from numpy import ndarray
+
+from semantic_kernel.memory.memory_record import MemoryRecord
+from semantic_kernel.utils.experimental_decorator import experimental_class
+
+
+@experimental_class
+class MemoryStoreBase(ABC):
+    """Base class for memory store."""
+
+    async def __aenter__(self):
+        """Enter the context manager."""
+        return self
+
+    async def __aexit__(self, *args):
+        """Exit the context manager."""
+        await self.close()
+
+    async def close(self):
+        """Close the connection."""
+        pass
+
+    @abstractmethod
+    async def create_collection(self, collection_name: str) -> None:
+        """Creates a new collection in the data store.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+        """
+        pass
+
+    @abstractmethod
+    async def get_collections(
+        self,
+    ) -> list[str]:
+        """Gets all collection names in the data store.
+
+        Returns:
+            List[str]: A group of collection names.
+        """
+        pass
+
+    @abstractmethod
+    async def delete_collection(self, collection_name: str) -> None:
+        """Deletes a collection from the data store.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+        """
+        pass
+
+    @abstractmethod
+    async def does_collection_exist(self, collection_name: str) -> bool:
+        """Determines if a collection exists in the data store.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+
+        Returns:
+            bool: True if given collection exists, False if not.
+        """
+        pass
+
+    @abstractmethod
+    async def upsert(self, collection_name: str, record: MemoryRecord) -> str:
+        """Upserts a memory record into the data store.
+
+        Does not guarantee that the collection exists.
+        If the record already exists, it will be updated.
+        If the record does not exist, it will be created.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+            record (MemoryRecord): The memory record to upsert.
+
+        Returns:
+            str: The unique identifier for the memory record.
+        """
+        pass
+
+    @abstractmethod
+    async def upsert_batch(self, collection_name: str, records: list[MemoryRecord]) -> list[str]:
+        """Upserts a group of memory records into the data store.
+
+        Does not guarantee that the collection exists.
+        If the record already exists, it will be updated.
+        If the record does not exist, it will be created.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+            records (MemoryRecord): The memory records to upsert.
+
+        Returns:
+            List[str]: The unique identifiers for the memory records.
+        """
+        pass
+
+    @abstractmethod
+    async def get(self, collection_name: str, key: str, with_embedding: bool) -> MemoryRecord:
+        """Gets a memory record from the data store. Does not guarantee that the collection exists.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+            key (str): The unique id associated with the memory record to get.
+            with_embedding (bool): If true, the embedding will be returned in the memory record.
+
+        Returns:
+            MemoryRecord: The memory record if found
+        """
+        pass
+
+    @abstractmethod
+    async def get_batch(
+        self,
+        collection_name: str,
+        keys: list[str],
+        with_embeddings: bool,
+    ) -> list[MemoryRecord]:
+        """Gets a batch of memory records from the data store. Does not guarantee that the collection exists.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+            keys (List[str]): The unique ids associated with the memory records to get.
+            with_embeddings (bool): If true, the embedding will be returned in the memory records.
+
+        Returns:
+            List[MemoryRecord]: The memory records associated with the unique keys provided.
+        """
+        pass
+
+    @abstractmethod
+    async def remove(self, collection_name: str, key: str) -> None:
+        """Removes a memory record from the data store. Does not guarantee that the collection exists.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+            key (str): The unique id associated with the memory record to remove.
+        """
+        pass
+
+    @abstractmethod
+    async def remove_batch(self, collection_name: str, keys: list[str]) -> None:
+        """Removes a batch of memory records from the data store. Does not guarantee that the collection exists.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+            keys (List[str]): The unique ids associated with the memory records to remove.
+        """
+        pass
+
+    @abstractmethod
+    async def get_nearest_matches(
+        self,
+        collection_name: str,
+        embedding: ndarray,
+        limit: int,
+        min_relevance_score: float,
+        with_embeddings: bool,
+    ) -> list[tuple[MemoryRecord, float]]:
+        """Gets the nearest matches to an embedding of type float. Does not guarantee that the collection exists.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+            embedding (ndarray): The embedding to compare the collection's embeddings with.
+            limit (int): The maximum number of similarity results to return.
+            min_relevance_score (float): The minimum relevance threshold for returned results.
+            with_embeddings (bool): If true, the embeddings will be returned in the memory records.
+
+        Returns:
+            List[Tuple[MemoryRecord, float]]: A list of tuples where item1 is a MemoryRecord and item2
+                is its similarity score as a float.
+        """
+        pass
+
+    @abstractmethod
+    async def get_nearest_match(
+        self,
+        collection_name: str,
+        embedding: ndarray,
+        min_relevance_score: float,
+        with_embedding: bool,
+    ) -> tuple[MemoryRecord, float]:
+        """Gets the nearest match to an embedding of type float. Does not guarantee that the collection exists.
+
+        Args:
+            collection_name (str): The name associated with a collection of embeddings.
+            embedding (ndarray): The embedding to compare the collection's embeddings with.
+            min_relevance_score (float): The minimum relevance threshold for returned result.
+            with_embedding (bool): If true, the embeddings will be returned in the memory record.
+
+        Returns:
+            Tuple[MemoryRecord, float]: A tuple consisting of the MemoryRecord and the similarity score as a float.
+        """
+        pass

semantic_kernel-1.8.3/semantic_kernel/memory/null_memory.py

@@ -0,0 +1,54 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from semantic_kernel.memory.memory_query_result import MemoryQueryResult
+from semantic_kernel.memory.semantic_text_memory_base import SemanticTextMemoryBase
+from semantic_kernel.utils.experimental_decorator import experimental_class
+
+
+@experimental_class
+class NullMemory(SemanticTextMemoryBase):
+    """Class for null memory."""
+
+    async def save_information(
+        self,
+        collection: str,
+        text: str,
+        id: str,
+        description: str | None = None,
+        additional_metadata: str | None = None,
+    ) -> None:
+        """Nullifies behavior of SemanticTextMemoryBase save_information."""
+        return
+
+    async def save_reference(
+        self,
+        collection: str,
+        text: str,
+        external_id: str,
+        external_source_name: str,
+        description: str | None = None,
+        additional_metadata: str | None = None,
+    ) -> None:
+        """Nullifies behavior of SemanticTextMemoryBase save_reference."""
+        return
+
+    async def get(self, collection: str, query: str) -> MemoryQueryResult | None:
+        """Nullifies behavior of SemanticTextMemoryBase get."""
+        return None
+
+    async def search(
+        self,
+        collection: str,
+        query: str,
+        limit: int = 1,
+        min_relevance_score: float = 0.7,
+    ) -> list[MemoryQueryResult]:
+        """Nullifies behavior of SemanticTextMemoryBase search."""
+        return []
+
+    async def get_collections(self) -> list[str]:
+        """Nullifies behavior of SemanticTextMemoryBase get_collections."""
+        return []
+
+
+NullMemory.instance = NullMemory()  # type: ignore

semantic_kernel-1.8.3/semantic_kernel/memory/semantic_text_memory.py

@@ -0,0 +1,158 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from typing import Any
+
+from pydantic import PrivateAttr
+
+from semantic_kernel.connectors.ai.embeddings.embedding_generator_base import EmbeddingGeneratorBase
+from semantic_kernel.memory.memory_query_result import MemoryQueryResult
+from semantic_kernel.memory.memory_record import MemoryRecord
+from semantic_kernel.memory.memory_store_base import MemoryStoreBase
+from semantic_kernel.memory.semantic_text_memory_base import SemanticTextMemoryBase
+from semantic_kernel.utils.experimental_decorator import experimental_class
+
+
+@experimental_class
+class SemanticTextMemory(SemanticTextMemoryBase):
+    """Class for semantic text memory."""
+
+    _storage: MemoryStoreBase = PrivateAttr()
+    _embeddings_generator: EmbeddingGeneratorBase = PrivateAttr()
+
+    def __init__(self, storage: MemoryStoreBase, embeddings_generator: EmbeddingGeneratorBase) -> None:
+        """Initialize a new instance of SemanticTextMemory.
+
+        Args:
+            storage (MemoryStoreBase): The MemoryStoreBase to use for storage.
+            embeddings_generator (EmbeddingGeneratorBase): The EmbeddingGeneratorBase
+                to use for generating embeddings.
+        """
+        super().__init__()
+        self._storage = storage
+        self._embeddings_generator = embeddings_generator
+
+    async def save_information(
+        self,
+        collection: str,
+        text: str,
+        id: str,
+        description: str | None = None,
+        additional_metadata: str | None = None,
+        embeddings_kwargs: dict[str, Any] | None = {},
+    ) -> None:
+        """Save information to the memory (calls the memory store's upsert method).
+
+        Args:
+            collection (str): The collection to save the information to.
+            text (str): The text to save.
+            id (str): The id of the information.
+            description (Optional[str]): The description of the information.
+            additional_metadata (Optional[str]): Additional metadata of the information.
+            embeddings_kwargs (Optional[Dict[str, Any]]): The embeddings kwargs of the information.
+        """
+        if not await self._storage.does_collection_exist(collection_name=collection):
+            await self._storage.create_collection(collection_name=collection)
+
+        embedding = (await self._embeddings_generator.generate_embeddings([text], **embeddings_kwargs))[0]
+        data = MemoryRecord.local_record(
+            id=id,
+            text=text,
+            description=description,
+            additional_metadata=additional_metadata,
+            embedding=embedding,
+        )
+
+        await self._storage.upsert(collection_name=collection, record=data)
+
+    async def save_reference(
+        self,
+        collection: str,
+        text: str,
+        external_id: str,
+        external_source_name: str,
+        description: str | None = None,
+        additional_metadata: str | None = None,
+        embeddings_kwargs: dict[str, Any] | None = {},
+    ) -> None:
+        """Save a reference to the memory (calls the memory store's upsert method).
+
+        Args:
+            collection (str): The collection to save the reference to.
+            text (str): The text to save.
+            external_id (str): The external id of the reference.
+            external_source_name (str): The external source name of the reference.
+            description (Optional[str]): The description of the reference.
+            additional_metadata (Optional[str]): Additional metadata of the reference.
+            embeddings_kwargs (Optional[Dict[str, Any]]): The embeddings kwargs of the reference.
+        """
+        if not await self._storage.does_collection_exist(collection_name=collection):
+            await self._storage.create_collection(collection_name=collection)
+
+        embedding = (await self._embeddings_generator.generate_embeddings([text], **embeddings_kwargs))[0]
+        data = MemoryRecord.reference_record(
+            external_id=external_id,
+            source_name=external_source_name,
+            description=description,
+            additional_metadata=additional_metadata,
+            embedding=embedding,
+        )
+
+        await self._storage.upsert(collection_name=collection, record=data)
+
+    async def get(
+        self,
+        collection: str,
+        key: str,
+    ) -> MemoryQueryResult | None:
+        """Get information from the memory (calls the memory store's get method).
+
+        Args:
+            collection (str): The collection to get the information from.
+            key (str): The key of the information.
+
+        Returns:
+            Optional[MemoryQueryResult]: The MemoryQueryResult if found, None otherwise.
+        """
+        record = await self._storage.get(collection_name=collection, key=key)
+        return MemoryQueryResult.from_memory_record(record, 1.0) if record else None
+
+    async def search(
+        self,
+        collection: str,
+        query: str,
+        limit: int = 1,
+        min_relevance_score: float = 0.0,
+        with_embeddings: bool = False,
+        embeddings_kwargs: dict[str, Any] | None = {},
+    ) -> list[MemoryQueryResult]:
+        """Search the memory (calls the memory store's get_nearest_matches method).
+
+        Args:
+            collection (str): The collection to search in.
+            query (str): The query to search for.
+            limit (int): The maximum number of results to return. (default: {1})
+            min_relevance_score (float): The minimum relevance score to return. (default: {0.0})
+            with_embeddings (bool): Whether to return the embeddings of the results. (default: {False})
+            embeddings_kwargs (Optional[Dict[str, Any]]): The embeddings kwargs of the information.
+
+        Returns:
+            List[MemoryQueryResult]: The list of MemoryQueryResult found.
+        """
+        query_embedding = (await self._embeddings_generator.generate_embeddings([query], **embeddings_kwargs))[0]
+        results = await self._storage.get_nearest_matches(
+            collection_name=collection,
+            embedding=query_embedding,
+            limit=limit,
+            min_relevance_score=min_relevance_score,
+            with_embeddings=with_embeddings,
+        )
+
+        return [MemoryQueryResult.from_memory_record(r[0], r[1]) for r in results]
+
+    async def get_collections(self) -> list[str]:
+        """Get the list of collections in the memory (calls the memory store's get_collections method).
+
+        Returns:
+            List[str]: The list of all the memory collection names.
+        """
+        return await self._storage.get_collections()

semantic_kernel-1.8.3/semantic_kernel/memory/semantic_text_memory_base.py

@@ -0,0 +1,111 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from semantic_kernel.kernel_pydantic import KernelBaseModel
+from semantic_kernel.utils.experimental_decorator import experimental_class
+
+if TYPE_CHECKING:
+    from semantic_kernel.memory.memory_query_result import MemoryQueryResult
+
+SemanticTextMemoryT = TypeVar("SemanticTextMemoryT", bound="SemanticTextMemoryBase")
+
+
+@experimental_class
+class SemanticTextMemoryBase(KernelBaseModel):
+    """Base class for semantic text memory."""
+
+    @abstractmethod
+    async def save_information(
+        self,
+        collection: str,
+        text: str,
+        id: str,
+        description: str | None = None,
+        additional_metadata: str | None = None,
+        embeddings_kwargs: dict[str, Any] | None = None,
+    ) -> None:
+        """Save information to the memory (calls the memory store's upsert method).
+
+        Args:
+            collection (str): The collection to save the information to.
+            text (str): The text to save.
+            id (str): The id of the information.
+            description (Optional[str]): The description of the information.
+            additional_metadata (Optional[str]): Additional metadata of the information.
+            embeddings_kwargs (Optional[Dict[str, Any]]): The embeddings kwargs of the information.
+
+        """
+        pass
+
+    @abstractmethod
+    async def save_reference(
+        self,
+        collection: str,
+        text: str,
+        external_id: str,
+        external_source_name: str,
+        description: str | None = None,
+        additional_metadata: str | None = None,
+    ) -> None:
+        """Save a reference to the memory (calls the memory store's upsert method).
+
+        Args:
+            collection (str): The collection to save the reference to.
+            text (str): The text to save.
+            external_id (str): The external id of the reference.
+            external_source_name (str): The external source name of the reference.
+            description (Optional[str]): The description of the reference.
+            additional_metadata (Optional[str]): Additional metadata of the reference.
+
+        """
+        pass
+
+    @abstractmethod
+    async def get(
+        self,
+        collection: str,
+        key: str,
+    ) -> "MemoryQueryResult | None":
+        """Get information from the memory (calls the memory store's get method).
+
+        Args:
+            collection (str): The collection to get the information from.
+            key (str): The key of the information.
+
+        Returns:
+            Optional[MemoryQueryResult]: The MemoryQueryResult if found, None otherwise.
+        """
+        pass
+
+    @abstractmethod
+    async def search(
+        self,
+        collection: str,
+        query: str,
+        limit: int = 1,
+        min_relevance_score: float = 0.7,
+    ) -> list["MemoryQueryResult"]:
+        """Search the memory (calls the memory store's get_nearest_matches method).
+
+        Args:
+            collection (str): The collection to search in.
+            query (str): The query to search for.
+            limit (int): The maximum number of results to return. (default: {1})
+            min_relevance_score (float): The minimum relevance score to return. (default: {0.0})
+            with_embeddings (bool): Whether to return the embeddings of the results. (default: {False})
+
+        Returns:
+            List[MemoryQueryResult]: The list of MemoryQueryResult found.
+        """
+        pass
+
+    @abstractmethod
+    async def get_collections(self) -> list[str]:
+        """Get the list of collections in the memory (calls the memory store's get_collections method).
+
+        Returns:
+            List[str]: The list of all the memory collection names.
+        """
+        pass