semantic-kernel 0.2.4.dev0__py3-none-any.whl → 0.2.5.dev0__py3-none-any.whl

semantic_kernel/memory/memory_store_base.py

@@ -1,10 +1,79 @@
 # Copyright (c) Microsoft. All rights reserved.
 
-from abc import ABC
+from abc import abstractmethod
+from typing import List, Tuple
 
-from semantic_kernel.ai.embeddings.embedding_index_base import EmbeddingIndexBase
-from semantic_kernel.memory.storage.data_store_base import DataStoreBase
+from numpy import ndarray
 
+from semantic_kernel.memory.memory_record import MemoryRecord
 
-class MemoryStoreBase(DataStoreBase, EmbeddingIndexBase, ABC):
-    pass
+
+class MemoryStoreBase:
+    @abstractmethod
+    async def create_collection_async(self, collection_name: SystemError) -> None:
+        pass
+
+    @abstractmethod
+    async def get_collections_async(
+        self,
+    ) -> List[str]:
+        pass
+
+    @abstractmethod
+    async def delete_collection_async(self, collection_name: str) -> None:
+        pass
+
+    @abstractmethod
+    async def does_collection_exist_async(self, collection_name: str) -> bool:
+        pass
+
+    @abstractmethod
+    async def upsert_async(self, collection_name: str, record: MemoryRecord) -> str:
+        pass
+
+    @abstractmethod
+    async def upsert_batch_async(
+        self, collection_name: str, records: List[MemoryRecord]
+    ) -> List[str]:
+        pass
+
+    @abstractmethod
+    async def get_async(
+        self, collection_name: str, key: str, with_embedding: bool
+    ) -> MemoryRecord:
+        pass
+
+    @abstractmethod
+    async def get_batch_async(
+        self, collection_name: str, keys: List[str], with_embeddings: bool
+    ) -> List[MemoryRecord]:
+        pass
+
+    @abstractmethod
+    async def remove_async(self, collection_name: str, key: str) -> None:
+        pass
+
+    @abstractmethod
+    async def remove_batch_async(self, collection_name: str, keys: List[str]) -> None:
+        pass
+
+    @abstractmethod
+    async def get_nearest_matches_async(
+        self,
+        collection_name: str,
+        embedding: ndarray,
+        limit: int,
+        min_relevance_score: float,
+        with_embeddings: bool,
+    ) -> List[Tuple[MemoryRecord, float]]:
+        pass
+
+    @abstractmethod
+    async def get_nearest_match_async(
+        self,
+        collection_name: str,
+        embedding: ndarray,
+        min_relevance_score: float,
+        with_embedding: bool,
+    ) -> Tuple[MemoryRecord, float]:
+        pass

semantic_kernel/memory/semantic_text_memory.py

@@ -2,7 +2,7 @@
 
 from typing import List, Optional
 
-from semantic_kernel.ai.embeddings.embedding_generator_base import (
+from semantic_kernel.connectors.ai.embeddings.embedding_generator_base import (
     EmbeddingGeneratorBase,
 )
 from semantic_kernel.memory.memory_query_result import MemoryQueryResult
@@ -18,6 +18,16 @@ class SemanticTextMemory(SemanticTextMemoryBase):
     def __init__(
         self, storage: MemoryStoreBase, embeddings_generator: EmbeddingGeneratorBase
     ) -> None:
+        """Initialize a new instance of SemanticTextMemory.
+
+        Arguments:
+            storage {MemoryStoreBase} -- The MemoryStoreBase to use for storage.
+            embeddings_generator {EmbeddingGeneratorBase} -- The EmbeddingGeneratorBase
+                to use for generating embeddings.
+
+        Returns:
+            None -- None.
+        """
         self._storage = storage
         self._embeddings_generator = embeddings_generator
 
@@ -28,10 +38,29 @@ class SemanticTextMemory(SemanticTextMemoryBase):
         id: str,
         description: Optional[str] = None,
     ) -> None:
+        """Save information to the memory (calls the memory store's upsert method).
+
+        Arguments:
+            collection {str} -- The collection to save the information to.
+            text {str} -- The text to save.
+            id {str} -- The id of the information.
+            description {Optional[str]} -- The description of the information.
+
+        Returns:
+            None -- None.
+        """
+        # TODO: not the best place to create collection, but will address this behavior together with .NET SK
+        if not await self._storage.does_collection_exist_async(
+            collection_name=collection
+        ):
+            await self._storage.create_collection_async(collection_name=collection)
+
         embedding = await self._embeddings_generator.generate_embeddings_async([text])
-        data = MemoryRecord.local_record(id, text, description, embedding)
+        data = MemoryRecord.local_record(
+            id=id, text=text, description=description, embedding=embedding
+        )
 
-        await self._storage.put_value_async(collection, id, data)
+        await self._storage.upsert_async(collection_name=collection, record=data)
 
     async def save_reference_async(
         self,
@@ -41,35 +70,88 @@ class SemanticTextMemory(SemanticTextMemoryBase):
         external_source_name: str,
         description: Optional[str] = None,
     ) -> None:
+        """Save a reference to the memory (calls the memory store's upsert method).
+
+        Arguments:
+            collection {str} -- The collection to save the reference to.
+            text {str} -- The text to save.
+            external_id {str} -- The external id of the reference.
+            external_source_name {str} -- The external source name of the reference.
+            description {Optional[str]} -- The description of the reference.
+
+        Returns:
+            None -- None.
+        """
+        # TODO: not the best place to create collection, but will address this behavior together with .NET SK
+        if not await self._storage.does_collection_exist_async(
+            collection_name=collection
+        ):
+            await self._storage.create_collection_async(collection_name=collection)
+
         embedding = await self._embeddings_generator.generate_embeddings_async([text])
         data = MemoryRecord.reference_record(
-            external_id, external_source_name, description, embedding
+            id=external_id,
+            source_name=external_source_name,
+            description=description,
+            embedding=embedding,
         )
 
-        await self._storage.put_value_async(collection, external_id, data)
+        await self._storage.upsert_async(collection_name=collection, record=data)
 
     async def get_async(
         self,
         collection: str,
-        query: str,
+        key: str,
     ) -> Optional[MemoryQueryResult]:
-        raise NotImplementedError()
+        """Get information from the memory (calls the memory store's get method).
+
+        Arguments:
+            collection {str} -- The collection to get the information from.
+            key {str} -- The key of the information.
+
+        Returns:
+            Optional[MemoryQueryResult] -- The MemoryQueryResult if found, None otherwise.
+        """
+        record = await self._storage.get_async(collection_name=collection, key=key)
+        return MemoryQueryResult.from_memory_record(record) if record else None
 
     async def search_async(
         self,
         collection: str,
         query: str,
         limit: int = 1,
-        min_relevance_score: float = 0.7,
+        min_relevance_score: float = 0.0,
+        with_embeddings: bool = False,
     ) -> List[MemoryQueryResult]:
+        """Search the memory (calls the memory store's get_nearest_matches method).
+
+        Arguments:
+            collection {str} -- The collection to search in.
+            query {str} -- The query to search for.
+            limit {int} -- The maximum number of results to return. (default: {1})
+            min_relevance_score {float} -- The minimum relevance score to return. (default: {0.0})
+            with_embeddings {bool} -- Whether to return the embeddings of the results. (default: {False})
+
+        Returns:
+            List[MemoryQueryResult] -- The list of MemoryQueryResult found.
+        """
         query_embedding = await self._embeddings_generator.generate_embeddings_async(
             [query]
         )
         results = await self._storage.get_nearest_matches_async(
-            collection, query_embedding, limit, min_relevance_score
+            collection_name=collection,
+            embedding=query_embedding,
+            limit=limit,
+            min_relevance_score=min_relevance_score,
+            with_embeddings=with_embeddings,
         )
 
         return [MemoryQueryResult.from_memory_record(r[0], r[1]) for r in results]
 
     async def get_collections_async(self) -> List[str]:
-        raise NotImplementedError()
+        """Get the list of collections in the memory (calls the memory store's get_collections method).
+
+        Returns:
+            List[str] -- The list of all the memory collection names.
+        """
+        return await self._storage.get_collections_async()

semantic_kernel/memory/semantic_text_memory_base.py

@@ -26,7 +26,6 @@ class SemanticTextMemoryBase(ABC):
         external_id: str,
         external_source_name: str,
         description: Optional[str] = None,
-        # TODO: ctoken?
     ) -> None:
         pass
 
@@ -34,7 +33,7 @@ class SemanticTextMemoryBase(ABC):
     async def get_async(
         self,
         collection: str,
-        query: str,  # TODO: ctoken?
+        query: str,
     ) -> Optional[MemoryQueryResult]:
         pass
 

semantic_kernel/memory/volatile_memory_store.py

@@ -1,37 +1,259 @@
 # Copyright (c) Microsoft. All rights reserved.
 
+from copy import deepcopy
 from logging import Logger
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 
 from numpy import array, linalg, ndarray
 
 from semantic_kernel.memory.memory_record import MemoryRecord
 from semantic_kernel.memory.memory_store_base import MemoryStoreBase
-from semantic_kernel.memory.storage.volatile_data_store import VolatileDataStore
 from semantic_kernel.utils.null_logger import NullLogger
 
 
-class VolatileMemoryStore(VolatileDataStore, MemoryStoreBase):
+class VolatileMemoryStore(MemoryStoreBase):
+    _store: Dict[str, Dict[str, MemoryRecord]]
+    _logger: Logger
+
     def __init__(self, logger: Optional[Logger] = None) -> None:
-        super().__init__()
+        self._store = {}
         self._logger = logger or NullLogger()
+        """Initializes a new instance of the VolatileMemoryStore class.
+
+        Arguments:
+            logger {Optional[Logger]} -- The logger to use. (default: {None})
+        """
+
+    async def create_collection_async(self, collection_name: str) -> None:
+        """Creates a new collection if it does not exist.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to create.
+
+        Returns:
+            None
+        """
+        if collection_name in self._store:
+            pass
+        else:
+            self._store[collection_name] = {}
+
+    async def get_collections_async(
+        self,
+    ) -> List[str]:
+        """Gets the list of collections.
+
+        Returns:
+            List[str] -- The list of collections.
+        """
+        return list(self._store.keys())
+
+    async def delete_collection_async(self, collection_name: str) -> None:
+        """Deletes a collection.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to delete.
+
+        Returns:
+            None
+        """
+        if collection_name in self._store:
+            del self._store[collection_name]
+
+    async def does_collection_exist_async(self, collection_name: str) -> bool:
+        """Checks if a collection exists.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to check.
+
+        Returns:
+            bool -- True if the collection exists; otherwise, False.
+        """
+        return collection_name in self._store
+
+    async def upsert_async(self, collection_name: str, record: MemoryRecord) -> str:
+        """Upserts a record.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to upsert the record into.
+            record {MemoryRecord} -- The record to upsert.
+
+        Returns:
+            str -- The unqiue database key of the record.
+        """
+        if collection_name not in self._store:
+            raise Exception(f"Collection '{collection_name}' does not exist")
+
+        record._key = record._id
+        self._store[collection_name][record._key] = record
+        return record._key
+
+    async def upsert_batch_async(
+        self, collection_name: str, records: List[MemoryRecord]
+    ) -> List[str]:
+        """Upserts a batch of records.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to upsert the records into.
+            records {List[MemoryRecord]} -- The records to upsert.
+
+        Returns:
+            List[str] -- The unqiue database keys of the records.
+        """
+        if collection_name not in self._store:
+            raise Exception(f"Collection '{collection_name}' does not exist")
+
+        for record in records:
+            record._key = record._id
+            self._store[collection_name][record._key] = record
+        return [record._key for record in records]
+
+    async def get_async(
+        self, collection_name: str, key: str, with_embedding: bool = False
+    ) -> MemoryRecord:
+        """Gets a record.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to get the record from.
+            key {str} -- The unique database key of the record.
+            with_embedding {bool} -- Whether to include the embedding in the result. (default: {False})
+
+        Returns:
+            MemoryRecord -- The record.
+        """
+        if collection_name not in self._store:
+            raise Exception(f"Collection '{collection_name}' does not exist")
+
+        if key not in self._store[collection_name]:
+            raise Exception(f"Key '{key}' not found in collection '{collection_name}'")
+
+        result = self._store[collection_name][key]
+
+        if not with_embedding:
+            # create copy of results without embeddings
+            result = deepcopy(result)
+            result._embedding = None
+        return result
+
+    async def get_batch_async(
+        self, collection_name: str, keys: List[str], with_embeddings: bool = False
+    ) -> List[MemoryRecord]:
+        """Gets a batch of records.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to get the records from.
+            keys {List[str]} -- The unique database keys of the records.
+            with_embeddings {bool} -- Whether to include the embeddings in the results. (default: {False})
+
+        Returns:
+            List[MemoryRecord] -- The records.
+        """
+        if collection_name not in self._store:
+            raise Exception(f"Collection '{collection_name}' does not exist")
+
+        results = [
+            self._store[collection_name][key]
+            for key in keys
+            if key in self._store[collection_name]
+        ]
+
+        if not with_embeddings:
+            # create copy of results without embeddings
+            for result in results:
+                result = deepcopy(result)
+                result._embedding = None
+        return results
+
+    async def remove_async(self, collection_name: str, key: str) -> None:
+        """Removes a record.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to remove the record from.
+            key {str} -- The unique database key of the record to remove.
+
+        Returns:
+            None
+        """
+        if collection_name not in self._store:
+            raise Exception(f"Collection '{collection_name}' does not exist")
+
+        if key not in self._store[collection_name]:
+            raise Exception(f"Key '{key}' not found in collection '{collection_name}'")
+
+        del self._store[collection_name][key]
+
+    async def remove_batch_async(self, collection_name: str, keys: List[str]) -> None:
+        """Removes a batch of records.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to remove the records from.
+            keys {List[str]} -- The unique database keys of the records to remove.
+
+        Returns:
+            None
+        """
+        if collection_name not in self._store:
+            raise Exception(f"Collection '{collection_name}' does not exist")
+
+        for key in keys:
+            if key in self._store[collection_name]:
+                del self._store[collection_name][key]
+
+    async def get_nearest_match_async(
+        self,
+        collection_name: str,
+        embedding: ndarray,
+        min_relevance_score: float = 0.0,
+        with_embedding: bool = False,
+    ) -> Tuple[MemoryRecord, float]:
+        """Gets the nearest match to an embedding using cosine similarity.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to get the nearest match from.
+            embedding {ndarray} -- The embedding to find the nearest match to.
+            min_relevance_score {float} -- The minimum relevance score of the match. (default: {0.0})
+            with_embedding {bool} -- Whether to include the embedding in the result. (default: {False})
+
+        Returns:
+            Tuple[MemoryRecord, float] -- The record and the relevance score.
+        """
+        return self.get_nearest_matches_async(
+            collection_name=collection_name,
+            embedding=embedding,
+            limit=1,
+            min_relevance_score=min_relevance_score,
+            with_embeddings=with_embedding,
+        )
 
     async def get_nearest_matches_async(
         self,
-        collection: str,
+        collection_name: str,
         embedding: ndarray,
-        limit: int = 1,
-        min_relevance_score: float = 0.7,
+        limit: int,
+        min_relevance_score: float = 0.0,
+        with_embeddings: bool = False,
     ) -> List[Tuple[MemoryRecord, float]]:
-        if collection not in self._store:
+        """Gets the nearest matches to an embedding using cosine similarity.
+
+        Arguments:
+            collection_name {str} -- The name of the collection to get the nearest matches from.
+            embedding {ndarray} -- The embedding to find the nearest matches to.
+            limit {int} -- The maximum number of matches to return.
+            min_relevance_score {float} -- The minimum relevance score of the matches. (default: {0.0})
+            with_embeddings {bool} -- Whether to include the embeddings in the results. (default: {False})
+
+        Returns:
+            List[Tuple[MemoryRecord, float]] -- The records and their relevance scores.
+        """
+        if collection_name not in self._store:
             return []
 
-        embedding_collection = list([x.value for x in self._store[collection].values()])
+        # Get all the records in the collection
+        memory_records = list(self._store[collection_name].values())
+
         # Convert the collection of embeddings into a numpy array (stacked)
-        embedding_array = array(
-            [x.embedding for x in embedding_collection], dtype=float
-        )
-        embedding_array = embedding_array.reshape(embedding_array.shape[0], -1)
+        embeddings = array([x._embedding for x in memory_records], dtype=float)
+        embeddings = embeddings.reshape(embeddings.shape[0], -1)
 
         # If the query embedding has shape (1, embedding_size),
         # reshape it to (embedding_size,)
@@ -42,11 +264,11 @@ class VolatileMemoryStore(VolatileDataStore, MemoryStoreBase):
 
         # Use numpy to get the cosine similarity between the query
         # embedding and all the embeddings in the collection
-        similarity_scores = self.compute_similarity_scores(embedding, embedding_array)
+        similarity_scores = self.compute_similarity_scores(embedding, embeddings)
 
         # Then, sort the results by the similarity score
         sorted_results = sorted(
-            zip(embedding_collection, similarity_scores),
+            zip(memory_records, similarity_scores),
             key=lambda x: x[1],
             reverse=True,
         )
@@ -56,22 +278,26 @@ class VolatileMemoryStore(VolatileDataStore, MemoryStoreBase):
 
         # Then, take the top N results
         top_results = filtered_results[:limit]
+
+        if not with_embeddings:
+            # create copy of results without embeddings
+            for result in top_results:
+                result = deepcopy(result)
+                result[0]._embedding = None
         return top_results
 
     def compute_similarity_scores(
         self, embedding: ndarray, embedding_array: ndarray
     ) -> ndarray:
-        """Compute the similarity scores between the
-        query embedding and all the embeddings in the collection.
-        Ignore the corresponding operation if zero vectors
-        are involved (in query embedding or the embedding collection)
-
-        :param embedding: The query embedding.
-        :param embedding_array: The collection of embeddings.
-        :return: similarity_scores: The similarity scores between the query embedding
-            and all the embeddings in the collection.
-        """
+        """Computes the cosine similarity scores between a query embedding and a group of embeddings.
 
+        Arguments:
+            embedding {ndarray} -- The query embedding.
+            embedding_array {ndarray} -- The group of embeddings.
+
+        Returns:
+            ndarray -- The cosine similarity scores.
+        """
         query_norm = linalg.norm(embedding)
         collection_norm = linalg.norm(embedding_array, axis=1)