MemoryOS 2.0.3__py3-none-any.whl

memos/memories/textual/prefer_text_memory/spliter.py

@@ -0,0 +1,132 @@
+import copy
+
+from memos.chunkers import ChunkerFactory
+from memos.configs.chunker import ChunkerConfigFactory
+from memos.configs.parser import ParserConfigFactory
+from memos.parsers.factory import ParserFactory
+from memos.types import MessageList
+
+
+class Splitter:
+    """Splitter."""
+
+    def __init__(
+        self,
+        lookback_turns: int = 1,
+        chunk_size: int = 256,
+        chunk_overlap: int = 128,
+        min_sentences_per_chunk: int = 1,
+        tokenizer: str = "gpt2",
+        parser_backend: str = "markitdown",
+        chunker_backend: str = "sentence",
+    ):
+        """Initialize the splitter."""
+        self.lookback_turns = lookback_turns
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.min_sentences_per_chunk = min_sentences_per_chunk
+        self.tokenizer = tokenizer
+        self.chunker_backend = chunker_backend
+        self.parser_backend = parser_backend
+        # Initialize parser
+        parser_config = ParserConfigFactory.model_validate(
+            {
+                "backend": self.parser_backend,
+                "config": {},
+            }
+        )
+        self.parser = ParserFactory.from_config(parser_config)
+
+        # Initialize chunker
+        chunker_config = ChunkerConfigFactory.model_validate(
+            {
+                "backend": self.chunker_backend,
+                "config": {
+                    "tokenizer_or_token_counter": self.tokenizer,
+                    "chunk_size": self.chunk_size,
+                    "chunk_overlap": self.chunk_overlap,
+                    "min_sentences_per_chunk": self.min_sentences_per_chunk,
+                },
+            }
+        )
+        self.chunker = ChunkerFactory.from_config(chunker_config)
+
+    def _split_with_lookback(self, data: MessageList) -> list[MessageList]:
+        """Split the messages or files into chunks by looking back fixed number of turns.
+        adjacent chunk with high duplicate rate,
+        default lookback turns is 1, only current turn in chunk"""
+        # Build QA pairs from chat history
+        pairs = self.build_qa_pairs(data)
+        chunks = []
+
+        # Create chunks by looking back fixed number of turns
+        for i in range(len(pairs)):
+            # Calculate the start index for lookback
+            start_idx = max(0, i + 1 - self.lookback_turns)
+            # Get the chunk of pairs (as many as available, up to lookback_turns)
+            chunk_pairs = pairs[start_idx : i + 1]
+
+            # Flatten chunk_pairs (list[list[dict]]) to MessageList (list[dict])
+            chunk_messages = []
+            for pair in chunk_pairs:
+                chunk_messages.extend(pair)
+
+            chunks.append(chunk_messages)
+        return chunks
+
+    def _split_with_overlap(self, data: MessageList) -> list[MessageList]:
+        """split the messages or files into chunks with overlap.
+        adjacent chunk with low duplicate rate"""
+        chunks = []
+        chunk = []
+        for i, item in enumerate(data):
+            chunk.append(item)
+            # 5 turns (Q + A = 10) each chunk
+            if len(chunk) >= 10:
+                chunks.append(chunk)
+                # overlap 1 turns (Q + A = 2)
+                context = copy.deepcopy(chunk[-2:]) if i + 1 < len(data) else []
+                chunk = context
+        if chunk:
+            chunks.append(chunk)
+
+        return chunks
+
+    def split_chunks(self, data: MessageList | str, **kwargs) -> list[MessageList] | list[str]:
+        """Split the messages or files into chunks.
+
+        Args:
+            data: MessageList or string to split
+
+        Returns:
+            List of MessageList chunks or list of string chunks
+        """
+        if isinstance(data, list):
+            if kwargs.get("split_type") == "lookback":
+                chunks = self._split_with_lookback(data)
+            elif kwargs.get("split_type") == "overlap":
+                chunks = self._split_with_overlap(data)
+            return chunks
+        else:
+            # Parse and chunk the string data using pre-initialized components
+            text = self.parser.parse(data)
+            chunks = self.chunker.chunk(text)
+
+            return [chunk.text for chunk in chunks]
+
+    def build_qa_pairs(self, chat_history: MessageList) -> list[MessageList]:
+        """Build QA pairs from chat history."""
+        qa_pairs = []
+        current_qa_pair = []
+
+        for message in chat_history:
+            if message["role"] == "user":
+                current_qa_pair.append(message)
+            elif message["role"] == "assistant":
+                if not current_qa_pair:
+                    continue
+                current_qa_pair.append(message)
+                qa_pairs.append(current_qa_pair.copy())
+                current_qa_pair = []  # reset
+
+        return qa_pairs

memos/memories/textual/prefer_text_memory/utils.py

@@ -0,0 +1,93 @@
+import json
+import re
+
+from memos.dependency import require_python_package
+from memos.memories.textual.item import TextualMemoryItem
+from memos.types import MessageList
+
+
+def convert_messages_to_string(messages: MessageList) -> str:
+    """Convert a list of messages to a string."""
+    message_text = ""
+    for message in messages:
+        content = message.get("content", "")
+        content = (
+            content.strip()
+            if isinstance(content, str)
+            else json.dumps(content, ensure_ascii=False).strip()
+        )
+        if message["role"] == "system":
+            continue
+        if message["role"] == "user":
+            message_text += f"User: {content}\n" if content else ""
+        elif message["role"] == "assistant":
+            tool_calls = message.get("tool_calls", [])
+            tool_calls_str = (
+                f"[tool_calls]: {json.dumps(tool_calls, ensure_ascii=False)}" if tool_calls else ""
+            )
+            line_str = (
+                f"Assistant: {content} {tool_calls_str}".strip()
+                if content or tool_calls_str
+                else ""
+            )
+            message_text += f"{line_str}\n" if line_str else ""
+        elif message["role"] == "tool":
+            tool_call_id = message.get("tool_call_id", "")
+            line_str = (
+                f"Tool: {content} [tool_call_id]: {tool_call_id}".strip()
+                if tool_call_id
+                else f"Tool: {content}".strip()
+            )
+            message_text += f"{line_str}\n" if line_str else ""
+    return message_text.strip()
+
+
+@require_python_package(
+    import_name="datasketch",
+    install_command="pip install datasketch",
+    install_link="https://github.com/ekzhu/datasketch",
+)
+def deduplicate_preferences(
+    prefs: list[TextualMemoryItem], similarity_threshold: float = 0.6, num_perm: int = 256
+) -> list[TextualMemoryItem]:
+    """
+    Deduplicate preference texts using MinHash algorithm.
+
+    Args:
+        prefs: List of preference memory items to deduplicate
+        similarity_threshold: Jaccard similarity threshold (0.0-1.0), default 0.8
+
+    Returns:
+        Deduplicated list of preference items
+    """
+    from datasketch import MinHash, MinHashLSH
+
+    if not prefs:
+        return prefs
+
+    # Use MinHashLSH for efficient similarity search
+    lsh = MinHashLSH(threshold=similarity_threshold, num_perm=num_perm)
+    unique_prefs = []
+
+    for i, pref in enumerate(prefs):
+        # Extract preference text
+        if hasattr(pref.metadata, "preference") and pref.metadata.preference:
+            text = pref.metadata.preference
+        else:
+            text = pref.memory
+
+        # Create MinHash from text tokens
+        minhash = MinHash(num_perm=num_perm)
+        # Simple tokenization: split by whitespace and clean
+        tokens = re.findall(r"\w+", text.lower())
+        for token in tokens:
+            minhash.update(token.encode("utf8"))
+
+        # Check for duplicates using LSH
+        similar_items = lsh.query(minhash)
+
+        if not similar_items:  # No similar items found
+            lsh.insert(i, minhash)
+            unique_prefs.append(pref)
+
+    return unique_prefs

memos/memories/textual/preference.py

@@ -0,0 +1,344 @@
+import json
+import os
+
+from datetime import datetime
+from typing import Any
+
+from memos.configs.memory import PreferenceTextMemoryConfig
+from memos.embedders.factory import (
+    ArkEmbedder,
+    EmbedderFactory,
+    OllamaEmbedder,
+    SenTranEmbedder,
+    UniversalAPIEmbedder,
+)
+from memos.llms.factory import AzureLLM, LLMFactory, OllamaLLM, OpenAILLM
+from memos.log import get_logger
+from memos.memories.textual.base import BaseTextMemory
+from memos.memories.textual.item import PreferenceTextualMemoryMetadata, TextualMemoryItem
+from memos.memories.textual.prefer_text_memory.factory import (
+    AdderFactory,
+    ExtractorFactory,
+    RetrieverFactory,
+)
+from memos.reranker.factory import RerankerFactory
+from memos.types import MessageList
+from memos.vec_dbs.factory import MilvusVecDB, QdrantVecDB, VecDBFactory
+from memos.vec_dbs.item import VecDBItem
+
+
+logger = get_logger(__name__)
+
+
+class PreferenceTextMemory(BaseTextMemory):
+    """Preference textual memory implementation for storing and retrieving memories."""
+
+    def __init__(self, config: PreferenceTextMemoryConfig):
+        """Initialize memory with the given configuration."""
+        self.config: PreferenceTextMemoryConfig = config
+        self.extractor_llm: OpenAILLM | OllamaLLM | AzureLLM = LLMFactory.from_config(
+            config.extractor_llm
+        )
+        self.vector_db: MilvusVecDB | QdrantVecDB = VecDBFactory.from_config(config.vector_db)
+        self.embedder: OllamaEmbedder | ArkEmbedder | SenTranEmbedder | UniversalAPIEmbedder = (
+            EmbedderFactory.from_config(config.embedder)
+        )
+        self.reranker = RerankerFactory.from_config(config.reranker)
+
+        self.extractor = ExtractorFactory.from_config(
+            config.extractor,
+            llm_provider=self.extractor_llm,
+            embedder=self.embedder,
+            vector_db=self.vector_db,
+        )
+
+        self.adder = AdderFactory.from_config(
+            config.adder,
+            llm_provider=self.extractor_llm,
+            embedder=self.embedder,
+            vector_db=self.vector_db,
+        )
+        self.retriever = RetrieverFactory.from_config(
+            config.retriever,
+            llm_provider=self.extractor_llm,
+            embedder=self.embedder,
+            reranker=self.reranker,
+            vector_db=self.vector_db,
+        )
+
+    def get_memory(
+        self, messages: list[MessageList], type: str, info: dict[str, Any]
+    ) -> list[TextualMemoryItem]:
+        """Get memory based on the messages.
+        Args:
+            messages (list[MessageList]): The messages to get memory from.
+            type (str): The type of memory to get.
+            info (dict[str, Any]): The info to get memory.
+        """
+        return self.extractor.extract(messages, type, info)
+
+    def search(
+        self, query: str, top_k: int, info=None, search_filter=None, **kwargs
+    ) -> list[TextualMemoryItem]:
+        """Search for memories based on a query.
+        Args:
+            query (str): The query to search for.
+            top_k (int): The number of top results to return.
+            info (dict): Leave a record of memory consumption.
+        Returns:
+            list[TextualMemoryItem]: List of matching memories.
+        """
+        if not isinstance(search_filter, dict):
+            search_filter = {}
+        search_filter.update({"status": "activated"})
+        logger.info(f"search_filter for preference memory: {search_filter}")
+        return self.retriever.retrieve(query, top_k, info, search_filter)
+
+    def load(self, dir: str) -> None:
+        """Load memories from the specified directory.
+        Args:
+            dir (str): The directory containing the memory files.
+        """
+        # For preference memory, we don't need to load from files
+        # as the data is stored in the vector database
+        try:
+            memory_file = os.path.join(dir, self.config.memory_filename)
+
+            if not os.path.exists(memory_file):
+                logger.warning(f"Memory file not found: {memory_file}")
+                return
+
+            with open(memory_file, encoding="utf-8") as f:
+                memories = json.load(f)
+            for collection_name, items in memories.items():
+                vec_db_items = [VecDBItem.from_dict(m) for m in items]
+                self.vector_db.add(collection_name, vec_db_items)
+                logger.info(f"Loaded {len(items)} memories from {collection_name} in {memory_file}")
+
+        except FileNotFoundError:
+            logger.error(f"Memory file not found in directory: {dir}")
+        except json.JSONDecodeError as e:
+            if e.pos == 0 and "Expecting value" in str(e):
+                logger.warning(f"Memory file is empty or contains only whitespace: {memory_file}")
+            else:
+                logger.error(f"Error decoding JSON from memory file: {e}")
+        except Exception as e:
+            logger.error(f"An error occurred while loading memories: {e}")
+
+    def dump(self, dir: str) -> None:
+        """Dump memories to the specified directory.
+        Args:
+            dir (str): The directory where the memory files will be saved.
+        """
+        # For preference memory, we don't need to dump to files
+        # as the data is stored in the vector database
+        try:
+            json_memories = {}
+            for collection_name in self.vector_db.config.collection_name:
+                items = self.vector_db.get_all(collection_name)
+                json_memories[collection_name] = [memory.to_dict() for memory in items]
+
+            os.makedirs(dir, exist_ok=True)
+            memory_file = os.path.join(dir, self.config.memory_filename)
+            with open(memory_file, "w", encoding="utf-8") as f:
+                json.dump(json_memories, f, indent=4, ensure_ascii=False)
+
+            logger.info(
+                f"Dumped {len(json_memories)} collections, {sum(len(items) for items in json_memories.values())} memories to {memory_file}"
+            )
+
+        except Exception as e:
+            logger.error(f"An error occurred while dumping memories: {e}")
+            raise
+
+    def extract(self, messages: MessageList) -> list[TextualMemoryItem]:
+        """Extract memories based on the messages.
+        Args:
+            messages (MessageList): The messages to extract memories from.
+        Returns:
+            list[TextualMemoryItem]: List of extracted memory items.
+        """
+        raise NotImplementedError
+
+    def add(self, memories: list[TextualMemoryItem | dict[str, Any]]) -> list[str]:
+        """Add memories.
+
+        Args:
+            memories: List of TextualMemoryItem objects or dictionaries to add.
+        """
+        return self.adder.add(memories)
+
+    def update(self, memory_id: str, new_memory: TextualMemoryItem | dict[str, Any]) -> None:
+        """Update a memory by memory_id."""
+        raise NotImplementedError
+
+    def get(self, memory_id: str, user_name: str | None = None) -> TextualMemoryItem:
+        """Get a memory by its ID.
+        Args:
+            memory_id (str): The ID of the memory to retrieve.
+        Returns:
+            TextualMemoryItem: The memory with the given ID.
+        """
+        raise NotImplementedError
+
+    def get_with_collection_name(
+        self, collection_name: str, memory_id: str
+    ) -> TextualMemoryItem | None:
+        """Get a memory by its ID and collection name.
+        Args:
+            memory_id (str): The ID of the memory to retrieve.
+            collection_name (str): The name of the collection to retrieve the memory from.
+        Returns:
+            TextualMemoryItem: The memory with the given ID and collection name.
+        """
+        try:
+            res = self.vector_db.get_by_id(collection_name, memory_id)
+            if res is None:
+                return None
+            return TextualMemoryItem(
+                id=res.id,
+                memory=res.memory,
+                metadata=PreferenceTextualMemoryMetadata(**res.payload),
+            )
+        except Exception as e:
+            # Convert any other exception to ValueError for consistent error handling
+            raise ValueError(
+                f"Memory with ID {memory_id} not found in collection {collection_name}: {e}"
+            ) from e
+
+    def get_by_ids(self, memory_ids: list[str]) -> list[TextualMemoryItem]:
+        """Get memories by their IDs.
+        Args:
+            memory_ids (list[str]): List of memory IDs to retrieve.
+        Returns:
+            list[TextualMemoryItem]: List of memories with the specified IDs.
+        """
+        raise NotImplementedError
+
+    def get_by_ids_with_collection_name(
+        self, collection_name: str, memory_ids: list[str]
+    ) -> list[TextualMemoryItem]:
+        """Get memories by their IDs and collection name.
+        Args:
+            collection_name (str): The name of the collection to retrieve the memory from.
+            memory_ids (list[str]): List of memory IDs to retrieve.
+        Returns:
+            list[TextualMemoryItem]: List of memories with the specified IDs and collection name.
+        """
+        try:
+            res = self.vector_db.get_by_ids(collection_name, memory_ids)
+            if not res:
+                return []
+            return [
+                TextualMemoryItem(
+                    id=memo.id,
+                    memory=memo.memory,
+                    metadata=PreferenceTextualMemoryMetadata(**memo.payload),
+                )
+                for memo in res
+            ]
+        except Exception as e:
+            # Convert any other exception to ValueError for consistent error handling
+            raise ValueError(
+                f"Memory with IDs {memory_ids} not found in collection {collection_name}: {e}"
+            ) from e
+
+    def get_all(self) -> list[TextualMemoryItem]:
+        """Get all memories.
+        Returns:
+            list[TextualMemoryItem]: List of all memories.
+        """
+        all_collections = ["explicit_preference", "implicit_preference"]
+        all_memories = {}
+        for collection_name in all_collections:
+            items = self.vector_db.get_all(collection_name)
+            all_memories[collection_name] = [
+                TextualMemoryItem(
+                    id=memo.id,
+                    memory=memo.memory,
+                    metadata=PreferenceTextualMemoryMetadata(**memo.payload),
+                )
+                for memo in items
+            ]
+        return all_memories
+
+    def get_memory_by_filter(
+        self,
+        filter: dict[str, Any] | None = None,
+        page: int | None = None,
+        page_size: int | None = None,
+    ):
+        """Get memories by filter.
+        Args:
+            filter (dict[str, Any]): Filter criteria.
+        Returns:
+            list[TextualMemoryItem]: List of memories that match the filter.
+        """
+        collection_list = self.vector_db.config.collection_name
+
+        memories = []
+        for collection_name in collection_list:
+            db_items = self.vector_db.get_by_filter(collection_name=collection_name, filter=filter)
+            db_items_memory = [
+                TextualMemoryItem(
+                    id=memo.id,
+                    memory=memo.memory,
+                    metadata=PreferenceTextualMemoryMetadata(**memo.payload),
+                )
+                for memo in db_items
+            ]
+            memories.extend(db_items_memory)
+
+        # sort
+        sorted_memories = sorted(
+            memories,
+            key=lambda item: datetime.fromisoformat(item.metadata.created_at),
+            reverse=True,
+        )
+        if page and page_size:
+            if page < 1:
+                page = 1
+            if page_size < 1:
+                page_size = 10
+            pick_memories = sorted_memories[(page - 1) * page_size : page * page_size]
+            return pick_memories, len(sorted_memories)
+
+        return sorted_memories, len(sorted_memories)
+
+    def delete(self, memory_ids: list[str]) -> None:
+        """Delete memories.
+        Args:
+            memory_ids (list[str]): List of memory IDs to delete.
+        """
+        collection_list = self.vector_db.config.collection_name
+        for collection_name in collection_list:
+            self.vector_db.delete(collection_name, memory_ids)
+
+    def delete_by_filter(self, filter: dict[str, Any]) -> None:
+        """Delete memories by filter.
+        Args:
+            filter (dict[str, Any]): Filter criteria.
+        """
+        collection_list = self.vector_db.config.collection_name
+        for collection_name in collection_list:
+            self.vector_db.delete_by_filter(collection_name=collection_name, filter=filter)
+
+    def delete_with_collection_name(self, collection_name: str, memory_ids: list[str]) -> None:
+        """Delete memories by their IDs and collection name.
+        Args:
+            collection_name (str): The name of the collection to delete the memory from.
+            memory_ids (list[str]): List of memory IDs to delete.
+        """
+        self.vector_db.delete(collection_name, memory_ids)
+
+    def delete_all(self) -> None:
+        """Delete all memories."""
+        for collection_name in self.vector_db.config.collection_name:
+            self.vector_db.delete_collection(collection_name)
+        self.vector_db.create_collection()
+
+    def drop(
+        self,
+    ) -> None:
+        """Drop all databases."""
+        raise NotImplementedError