MemoryOS 0.0.1__py3-none-any.whl → 0.1.12__py3-none-any.whl

memos/memories/textual/general.py

@@ -0,0 +1,286 @@
+import json
+import os
+
+from datetime import datetime
+from typing import Any
+
+from tenacity import retry, retry_if_exception_type, stop_after_attempt
+
+from memos.configs.memory import GeneralTextMemoryConfig
+from memos.embedders.factory import EmbedderFactory, OllamaEmbedder
+from memos.llms.factory import LLMFactory, OllamaLLM, OpenAILLM
+from memos.log import get_logger
+from memos.memories.textual.base import BaseTextMemory
+from memos.memories.textual.item import TextualMemoryItem
+from memos.types import MessageList
+from memos.vec_dbs.factory import QdrantVecDB, VecDBFactory
+from memos.vec_dbs.item import VecDBItem
+
+
+logger = get_logger(__name__)
+
+
+class GeneralTextMemory(BaseTextMemory):
+    """General textual memory implementation for storing and retrieving memories."""
+
+    def __init__(self, config: GeneralTextMemoryConfig):
+        """Initialize memory with the given configuration."""
+        self.config: GeneralTextMemoryConfig = config
+        self.extractor_llm: OpenAILLM | OllamaLLM = LLMFactory.from_config(config.extractor_llm)
+        self.vector_db: QdrantVecDB = VecDBFactory.from_config(config.vector_db)
+        self.embedder: OllamaEmbedder = EmbedderFactory.from_config(config.embedder)
+
+    @retry(
+        stop=stop_after_attempt(3),
+        retry=retry_if_exception_type(json.JSONDecodeError),
+        before_sleep=lambda retry_state: logger.warning(
+            EXTRACTION_RETRY_LOG.format(
+                error=retry_state.outcome.exception(),
+                attempt_number=retry_state.attempt_number,
+                max_attempt_number=3,
+            )
+        ),
+    )
+    def extract(self, messages: MessageList) -> list[TextualMemoryItem]:
+        """Extract memories based on the messages.
+
+        Args:
+            messages: List of message dictionaries to extract memories from.
+
+        Returns:
+            List of TextualMemoryItem objects representing the extracted memories.
+        """
+        str_messages = json.dumps(messages)
+        user_query = EXTRACTION_PROMPT_PART_1 + EXTRACTION_PROMPT_PART_2.format(
+            messages=str_messages
+        )
+        response = self.extractor_llm.generate([{"role": "user", "content": user_query}])
+        raw_extracted_memories = json.loads(response)
+        extracted_memories = [
+            TextualMemoryItem(**memory_dict) for memory_dict in raw_extracted_memories
+        ]
+
+        return extracted_memories
+
+    def add(self, memories: list[TextualMemoryItem | dict[str, Any]]) -> None:
+        """Add memories.
+
+        Args:
+            memories: List of TextualMemoryItem objects or dictionaries to add.
+        """
+        memory_items = [TextualMemoryItem(**m) if isinstance(m, dict) else m for m in memories]
+
+        # Memory encode
+        embed_memories = self.embedder.embed([m.memory for m in memory_items])
+
+        # Create vector db items
+        vec_db_items = []
+        for item, emb in zip(memory_items, embed_memories, strict=True):
+            vec_db_items.append(
+                VecDBItem(
+                    id=item.id,
+                    payload=item.model_dump(),
+                    vector=emb,
+                )
+            )
+
+        # Add to vector db
+        self.vector_db.add(vec_db_items)
+
+    def update(self, memory_id: str, new_memory: TextualMemoryItem | dict[str, Any]) -> None:
+        """Update a memory by memory_id."""
+        memory_item = (
+            TextualMemoryItem(**new_memory) if isinstance(new_memory, dict) else new_memory
+        )
+        memory_item.id = memory_id
+
+        vec_db_item = VecDBItem(
+            id=memory_item.id,
+            payload=memory_item.model_dump(),
+            vector=self._embed_one_sentence(memory_item.memory),
+        )
+
+        self.vector_db.update(memory_id, vec_db_item)
+
+    def search(self, query: str, top_k: int) -> list[TextualMemoryItem]:
+        """Search for memories based on a query.
+        Args:
+            query (str): The query to search for.
+            top_k (int): The number of top results to return.
+        Returns:
+            list[TextualMemoryItem]: List of matching memories.
+        """
+        query_vector = self._embed_one_sentence(query)
+        search_results = self.vector_db.search(query_vector, top_k)
+        search_results = sorted(  # make higher score first
+            search_results, key=lambda x: x.score, reverse=True
+        )
+        result_memories = [
+            TextualMemoryItem(**search_item.payload) for search_item in search_results
+        ]
+        return result_memories
+
+    def get(self, memory_id: str) -> TextualMemoryItem:
+        """Get a memory by its ID."""
+        result = self.vector_db.get_by_id(memory_id)
+        if result is None:
+            raise ValueError(f"Memory with ID {memory_id} not found")
+        return TextualMemoryItem(**result.payload)
+
+    def get_by_ids(self, memory_ids: list[str]) -> list[TextualMemoryItem]:
+        """Get memories by their IDs.
+        Args:
+            memory_ids (list[str]): List of memory IDs to retrieve.
+        Returns:
+            list[TextualMemoryItem]: List of memories with the specified IDs.
+        """
+        db_items = self.vector_db.get_by_ids(memory_ids)
+        memories = [TextualMemoryItem(**db_item.payload) for db_item in db_items]
+        return memories
+
+    def get_all(self) -> list[TextualMemoryItem]:
+        """Get all memories.
+        Returns:
+            list[TextualMemoryItem]: List of all memories.
+        """
+        all_items = self.vector_db.get_all()
+        all_memories = [TextualMemoryItem(**memo.payload) for memo in all_items]
+        return all_memories
+
+    def delete(self, memory_ids: list[str]) -> None:
+        """Delete a memory."""
+        self.vector_db.delete(memory_ids)
+
+    def delete_all(self) -> None:
+        """Delete all memories."""
+        self.vector_db.delete_collection(self.vector_db.config.collection_name)
+        self.vector_db.create_collection()
+
+    def load(self, dir: str) -> None:
+        try:
+            memory_file = os.path.join(dir, self.config.memory_filename)
+
+            if not os.path.exists(memory_file):
+                logger.warning(f"Memory file not found: {memory_file}")
+                return
+
+            with open(memory_file, encoding="utf-8") as f:
+                memories = json.load(f)
+
+            vec_db_items = [VecDBItem.from_dict(m) for m in memories]
+            self.vector_db.add(vec_db_items)
+            logger.info(f"Loaded {len(memories)} memories from {memory_file}")
+
+        except FileNotFoundError:
+            logger.error(f"Memory file not found in directory: {dir}")
+        except json.JSONDecodeError as e:
+            logger.error(f"Error decoding JSON from memory file: {e}")
+        except Exception as e:
+            logger.error(f"An error occurred while loading memories: {e}")
+
+    def dump(self, dir: str) -> None:
+        """Dump memories to os.path.join(dir, self.config.memory_filename)"""
+        try:
+            all_vec_db_items = self.vector_db.get_all()
+            json_memories = [memory.to_dict() for memory in all_vec_db_items]
+
+            os.makedirs(dir, exist_ok=True)
+            memory_file = os.path.join(dir, self.config.memory_filename)
+            with open(memory_file, "w", encoding="utf-8") as f:
+                json.dump(json_memories, f, indent=4, ensure_ascii=False)
+
+            logger.info(f"Dumped {len(all_vec_db_items)} memories to {memory_file}")
+
+        except Exception as e:
+            logger.error(f"An error occurred while dumping memories: {e}")
+            raise
+
+    def drop(
+        self,
+    ) -> None:
+        pass
+
+    def _embed_one_sentence(self, sentence: str) -> list[float]:
+        """Embed a single sentence."""
+        return self.embedder.embed(sentence)[0]
+
+
+EXTRACTION_PROMPT_PART_1 = f"""You are a memory extractor. Your task is to extract memories from the given messages.
+* You will receive a list of messages, each with a role (user or assistant) and content.
+* Your job is to extract memories related to the user's long-term goals, interests, and emotional states.
+* Each memory should be a dictionary with the following keys:
+    - "memory": The content of the memory (string). Rephrase the content if necessary.
+    - "metadata": A dictionary containing additional information about the memory.
+* The metadata dictionary should include:
+    - "type": The type of memory (string), e.g., "procedure", "fact", "event", "opinion", etc.
+    - "memory_time": The time the memory occurred or refers to (string). Must be in standard `YYYY-MM-DD` format. Relative expressions such as "yesterday" or "tomorrow" are not allowed.
+    - "source": The origin of the memory (string), e.g., `"conversation"`, `"retrieved"`, `"web"`, `"file"`.
+    - "confidence": A numeric score (float between 0 and 100) indicating how certain you are about the accuracy or reliability of the memory.
+    - "entities": A list of key entities (array of strings) mentioned in the memory, e.g., people, places, organizations, e.g., `["Alice", "Paris", "OpenAI"]`.
+    - "tags": A list of keywords or thematic labels (array of strings) associated with the memory for categorization or retrieval, e.g., `["travel", "health", "project-x"]`.
+    - "visibility": The accessibility scope of the memory (string), e.g., `"private"`, `"public"`, `"session"`, determining who or what contexts can access it.
+    - "updated_at": The timestamp of the last modification to the memory (string). Useful for tracking memory freshness or change history. Format: ISO 8601 or natural language.
+* Current date and time is {datetime.now().isoformat()}.
+* Only return the list of memories in JSON format.
+* Do not include any explanations
+* Do not include any extra text
+* Do not include code blocks (```json```)
+
+## Example
+
+### Input
+
+[
+    {{"role": "user", "content": "I plan to visit Paris next week."}},
+    {{"role": "assistant", "content": "Paris is a beautiful city with many attractions."}},
+    {{"role": "user", "content": "I love the Eiffel Tower."}},
+    {{"role": "assistant", "content": "The Eiffel Tower is a must-see landmark in Paris."}}
+]
+
+### Output
+
+[
+  {{
+    "memory": "The user plans to visit Paris on 05-26-2025.",
+    "metadata": {{
+      "type": "event",
+      "memory_time": "2025-05-26",
+      "source": "conversation",
+      "confidence": 90.0,
+      "entities": ["Paris"],
+      "tags": ["travel", "plans"],
+      "visibility": "private",
+      "updated_at": "2025-05-19T00:00:00"
+    }}
+  }},
+  {{
+    "memory": "The user loves the Eiffel Tower.",
+    "metadata": {{
+      "type": "opinion",
+      "memory_time": "2025-05-19",
+      "source": "conversation",
+      "confidence": 100.0,
+      "entities": ["Eiffel Tower"],
+      "tags": ["opinions", "landmarks"],
+      "visibility": "session",
+      "updated_at": "2025-05-19T00:00:00"
+    }}
+  }}
+]
+
+"""
+
+EXTRACTION_PROMPT_PART_2 = """
+## Query
+
+### Input
+
+{messages}
+
+### Output
+
+"""
+
+EXTRACTION_RETRY_LOG = """Extracting memory failed due to JSON decode error: {error},
+Attempt retry: {attempt_number} / {max_attempt_number}
+"""

memos/memories/textual/item.py

@@ -0,0 +1,167 @@
+"""Defines memory item types for textual memory."""
+
+import uuid
+
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class TextualMemoryMetadata(BaseModel):
+    """Metadata for a memory item.
+
+    This includes information such as the type of memory, when it occurred,
+    its source, and other relevant details.
+    """
+
+    user_id: str | None = Field(
+        default=None,
+        description="The ID of the user associated with the memory. Useful for multi-user systems.",
+    )
+    session_id: str | None = Field(
+        default=None,
+        description="The ID of the session during which the memory was created. Useful for tracking context in conversations.",
+    )
+    status: Literal["activated", "archived", "deleted"] | None = Field(
+        default="activated",
+        description="The status of the memory, e.g., 'activated', 'archived', 'deleted'.",
+    )
+    type: Literal["procedure", "fact", "event", "opinion", "topic"] | None = Field(default=None)
+    memory_time: str | None = Field(
+        default=None,
+        description='The time the memory occurred or refers to. Must be in standard `YYYY-MM-DD` format. Relative expressions such as "yesterday" or "tomorrow" are not allowed.',
+    )
+    source: Literal["conversation", "retrieved", "web", "file"] | None = Field(
+        default=None, description="The origin of the memory"
+    )
+    confidence: float | None = Field(
+        default=None,
+        description="A numeric score (float between 0 and 100) indicating how certain you are about the accuracy or reliability of the memory.",
+    )
+    entities: list[str] | None = Field(
+        default=None,
+        description='A list of key entities mentioned in the memory, e.g., people, places, organizations, e.g., `["Alice", "Paris", "OpenAI"]`.',
+    )
+    tags: list[str] | None = Field(
+        default=None,
+        description='A list of keywords or thematic labels associated with the memory for categorization or retrieval, e.g., `["travel", "health", "project-x"]`.',
+    )
+    visibility: Literal["private", "public", "session"] | None = Field(
+        default=None, description="e.g., 'private', 'public', 'session'"
+    )
+    updated_at: str | None = Field(
+        default_factory=lambda: datetime.now().isoformat(),
+        description="The timestamp of the last modification to the memory. Useful for tracking memory freshness or change history. Format: ISO 8601.",
+    )
+
+    model_config = ConfigDict(extra="allow")
+
+    @field_validator("memory_time")
+    @classmethod
+    def validate_memory_time(cls, v):
+        try:
+            if v:
+                datetime.strptime(v, "%Y-%m-%d")
+        except ValueError as e:
+            raise ValueError("Invalid date format. Use YYYY-MM-DD.") from e
+        return v
+
+    @field_validator("confidence")
+    @classmethod
+    def validate_confidence(cls, v):
+        if v is not None and (v < 0 or v > 100):
+            raise ValueError("Confidence must be between 0 and 100.")
+        return v
+
+    def __str__(self) -> str:
+        """Pretty string representation of the metadata."""
+        meta = self.model_dump(exclude_none=True)
+        return ", ".join(f"{k}={v}" for k, v in meta.items())
+
+
+class TreeNodeTextualMemoryMetadata(TextualMemoryMetadata):
+    """Extended metadata for structured memory, layered retrieval, and lifecycle tracking."""
+
+    memory_type: Literal["WorkingMemory", "LongTermMemory", "UserMemory"] = Field(
+        default="WorkingMemory", description="Memory lifecycle type."
+    )
+    key: str | None = Field(default=None, description="Memory key or title.")
+    sources: list[str] | None = Field(
+        default=None, description="Multiple origins of the memory (e.g., URLs, notes)."
+    )
+    embedding: list[float] | None = Field(
+        default=None,
+        description="The vector embedding of the memory content, used for semantic search or clustering.",
+    )
+    created_at: str | None = Field(
+        default_factory=lambda: datetime.now().isoformat(),
+        description="The timestamp of the first creation to the memory. Useful "
+        "for tracking memory initialization. Format: ISO 8601.",
+    )
+    usage: list[str] | None = Field(
+        default=[],
+        description="Usage history of this node",
+    )
+    background: str | None = Field(
+        default="",
+        description="background of this node",
+    )
+
+    @field_validator("sources")
+    @classmethod
+    def validate_sources(cls, v):
+        if v is not None and not isinstance(v, list):
+            raise ValueError("Sources must be a list of strings.")
+        return v
+
+    def __str__(self) -> str:
+        """Pretty string representation of the metadata."""
+        meta = self.model_dump(exclude_none=True)
+        return ", ".join([f"{k}={v}" for k, v in meta.items() if k != "embedding"])
+
+
+class SearchedTreeNodeTextualMemoryMetadata(TreeNodeTextualMemoryMetadata):
+    """Metadata for nodes returned by search, includes similarity info."""
+
+    relativity: float | None = Field(
+        default=None, description="Similarity score with respect to the query, 0 ~ 1."
+    )
+
+
+class TextualMemoryItem(BaseModel):
+    """Represents a single memory item in the textual memory.
+
+    This serves as a standardized format for memory items across different
+    textual memory implementations.
+    """
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    memory: str
+    metadata: (
+        TextualMemoryMetadata
+        | TreeNodeTextualMemoryMetadata
+        | SearchedTreeNodeTextualMemoryMetadata
+    ) = Field(default_factory=TextualMemoryMetadata)
+
+    model_config = ConfigDict(extra="forbid")
+
+    @field_validator("id")
+    @classmethod
+    def validate_id(cls, v):
+        try:
+            uuid.UUID(v)
+        except ValueError as e:
+            raise ValueError("Invalid UUID format") from e
+        return v
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "TextualMemoryItem":
+        return cls(**data)
+
+    def to_dict(self) -> dict:
+        return self.model_dump(exclude_none=True)
+
+    def __str__(self) -> str:
+        """Pretty string representation of the memory item."""
+        return f"<ID: {self.id} | Memory: {self.memory} | Metadata: {self.metadata!s}>"

memos/memories/textual/naive.py

@@ -0,0 +1,185 @@
+import json
+import os
+
+from datetime import datetime
+from typing import Any
+
+from memos.configs.memory import NaiveTextMemoryConfig
+from memos.llms.factory import LLMFactory
+from memos.log import get_logger
+from memos.memories.textual.base import BaseTextMemory
+from memos.memories.textual.item import TextualMemoryItem, TextualMemoryMetadata
+from memos.types import MessageList
+
+
+logger = get_logger(__name__)
+
+
+EXTRACTION_PROMPT_PART_1 = f"""You are a memory extractor. Your task is to extract memories from the given messages.
+* You will receive a list of messages, each with a role (user or assistant) and content.
+* Your job is to extract the memories from these messages.
+* Each memory should be a dictionary with the following keys:
+    - "memory": The content of the memory (string). Rephrase the content if necessary.
+    - "type": The type of memory (string), e.g., "procedure", "fact", "event", "opinion", etc.
+* Current date and time is {datetime.now().isoformat()}.
+* Only return the list of memories in JSON format.
+* Do not include any other text or explanation.
+
+## Example
+
+### Input
+
+[
+    {{"role": "user", "content": "I plan to visit Paris next week."}},
+    {{"role": "assistant", "content": "Paris is a beautiful city with many attractions."}},
+    {{"role": "user", "content": "I love the Eiffel Tower."}},
+    {{"role": "assistant", "content": "The Eiffel Tower is a must-see landmark in Paris."}}
+]
+
+### Output
+
+[
+    {{"memory": "User plans to visit Paris next week.", "metadata": {{"type": "event"}}}},
+    {{"memory": "User loves the Eiffel Tower.", "metadata": {{"type": "opinion"}}}},
+]
+"""
+
+EXTRACTION_PROMPT_PART_2 = """
+## Query
+
+### Input
+
+{messages}
+
+### Output
+
+"""
+
+
+class NaiveTextMemory(BaseTextMemory):
+    """Naive textual memory implementation for storing and retrieving memories."""
+
+    def __init__(self, config: NaiveTextMemoryConfig):
+        """Initialize memory with the given configuration."""
+        self.config = config
+        self.extractor_llm = LLMFactory.from_config(config.extractor_llm)
+        self.memories = []
+
+    def extract(self, messages: MessageList) -> list[TextualMemoryItem]:
+        """Extract memories based on the messages."""
+        str_messages = json.dumps(messages)
+        user_query = EXTRACTION_PROMPT_PART_1 + EXTRACTION_PROMPT_PART_2.format(
+            messages=str_messages
+        )
+        response = self.extractor_llm.generate([{"role": "user", "content": user_query}])
+        raw_extracted_memories = json.loads(response)
+
+        # Convert raw dictionaries to TextualMemoryItem objects
+        extracted_memories = []
+        for memory_dict in raw_extracted_memories:
+            # Ensure proper structure with memory and metadata
+            memory_content = memory_dict.get("memory", "")
+            metadata_dict = memory_dict.get("metadata", {})
+
+            # Create a TextualMemoryItem with properly structured metadata
+            memory_item = TextualMemoryItem(memory=memory_content, metadata=metadata_dict)
+            extracted_memories.append(memory_item)
+
+        return extracted_memories
+
+    def add(self, memories: list[TextualMemoryItem | dict[str, Any]]) -> None:
+        """Add memories."""
+        for m in memories:
+            # Convert dict to TextualMemoryItem if needed
+            memory_item = TextualMemoryItem(**m) if isinstance(m, dict) else m
+
+            # Convert to dictionary for storage
+            memory_dict = memory_item.model_dump()
+
+            if memory_dict["id"] not in [m["id"] for m in self.memories]:
+                self.memories.append(memory_dict)
+
+    def update(self, memory_id: str, new_memory: TextualMemoryItem | dict[str, Any]) -> None:
+        """Update a memory by memory_id."""
+        # Convert dict to TextualMemoryItem if needed
+        memory_item = (
+            TextualMemoryItem(**new_memory) if isinstance(new_memory, dict) else new_memory
+        )
+
+        # Ensure the memory item has the correct ID
+        memory_item.id = memory_id
+        memory_dict = memory_item.model_dump()
+
+        for i, memory in enumerate(self.memories):
+            if memory["id"] == memory_id:
+                self.memories[i] = memory_dict
+                break
+
+    def search(self, query: str, top_k: int) -> list[TextualMemoryItem]:
+        """Search for memories based on a query."""
+        sims = [
+            (memory, len(set(query.split()) & set(memory["memory"].split())))
+            for memory in self.memories
+        ]
+        sims.sort(key=lambda x: x[1], reverse=True)
+        # Convert search results to TextualMemoryItem objects
+        return [TextualMemoryItem(**memory) for memory, _ in sims[:top_k]]
+
+    def get(self, memory_id: str) -> TextualMemoryItem:
+        """Get a memory by its ID."""
+        for memory in self.memories:
+            if memory["id"] == memory_id:
+                return TextualMemoryItem(**memory)
+        # Return empty memory item if not found
+        return TextualMemoryItem(id=memory_id, memory="", metadata=TextualMemoryMetadata())
+
+    def get_all(self) -> list[TextualMemoryItem]:
+        """Get all memories."""
+        return [TextualMemoryItem(**memory) for memory in self.memories]
+
+    def get_by_ids(self, memory_ids: list[str]) -> list[TextualMemoryItem]:
+        """Get memories by their IDs.
+        Args:
+            memory_ids (list[str]): List of memory IDs to retrieve.
+        Returns:
+            list[TextualMemoryItem]: List of memories with the specified IDs.
+        """
+        return [self.get(memory_id) for memory_id in memory_ids]
+
+    def delete(self, memory_ids: list[str]) -> None:
+        """Delete memories.
+        Args:
+            memory_ids (list[str]): List of memory IDs to delete.
+        """
+        self.memories = [m for m in self.memories if m["id"] not in memory_ids]
+
+    def delete_all(self) -> None:
+        """Delete all memories."""
+        self.memories = []
+
+    def load(self, dir: str) -> None:
+        try:
+            with open(os.path.join(dir, self.config.memory_filename), encoding="utf-8") as file:
+                raw_memories = json.load(file)
+                self.add(raw_memories)
+        except FileNotFoundError:
+            logger.error(f"Directory not found: {dir}")
+        except json.JSONDecodeError:
+            logger.error(f"Error decoding JSON from file in directory: {dir}")
+        except Exception as e:
+            logger.error(f"An error occurred while loading memories: {e}")
+
+    def dump(self, dir: str) -> None:
+        try:
+            os.makedirs(dir, exist_ok=True)
+            memory_file = os.path.join(dir, self.config.memory_filename)
+            with open(memory_file, "w", encoding="utf-8") as file:
+                json.dump(self.memories, file, indent=4, ensure_ascii=False)
+        except Exception as e:
+            logger.error(f"An error occurred while dumping memories: {e}")
+            raise
+
+    def drop(
+        self,
+    ) -> None:
+        pass