MemoryOS 2.0.3__py3-none-any.whl

memos/memories/textual/base.py

@@ -0,0 +1,92 @@
+from abc import abstractmethod
+from typing import Any
+
+from memos.configs.memory import BaseTextMemoryConfig
+from memos.memories.base import BaseMemory
+from memos.memories.textual.item import TextualMemoryItem
+from memos.types import MessageList
+
+
+class BaseTextMemory(BaseMemory):
+    """Base class for all textual memory implementations."""
+
+    # Default mode configuration - can be overridden by subclasses
+    mode: str = "sync"  # Default mode: 'async' or 'sync'
+
+    @abstractmethod
+    def __init__(self, config: BaseTextMemoryConfig):
+        """Initialize memory with the given configuration."""
+
+    @abstractmethod
+    def extract(self, messages: MessageList) -> list[TextualMemoryItem]:
+        """Extract memories based on the messages.
+        Args:
+            messages (MessageList): The messages to extract memories from.
+        Returns:
+            list[TextualMemoryItem]: List of extracted memory items.
+        """
+
+    @abstractmethod
+    def add(self, memories: list[TextualMemoryItem | dict[str, Any]], **kwargs) -> list[str]:
+        """Add memories.
+
+        Args:
+            memories: List of TextualMemoryItem objects or dictionaries to add.
+        """
+
+    @abstractmethod
+    def update(self, memory_id: str, new_memory: TextualMemoryItem | dict[str, Any]) -> None:
+        """Update a memory by memory_id."""
+
+    @abstractmethod
+    def search(self, query: str, top_k: int, info=None, **kwargs) -> list[TextualMemoryItem]:
+        """Search for memories based on a query.
+        Args:
+            query (str): The query to search for.
+            top_k (int): The number of top results to return.
+            info (dict): Leave a record of memory consumption.
+        Returns:
+            list[TextualMemoryItem]: List of matching memories.
+        """
+
+    @abstractmethod
+    def get(self, memory_id: str, user_name: str | None = None) -> TextualMemoryItem:
+        """Get a memory by its ID.
+        Args:
+            memory_id (str): The ID of the memory to retrieve.
+        Returns:
+            TextualMemoryItem: The memory with the given ID.
+        """
+
+    @abstractmethod
+    def get_by_ids(self, memory_ids: list[str]) -> list[TextualMemoryItem]:
+        """Get memories by their IDs.
+        Args:
+            memory_ids (list[str]): List of memory IDs to retrieve.
+        Returns:
+            list[TextualMemoryItem]: List of memories with the specified IDs.
+        """
+
+    @abstractmethod
+    def get_all(self) -> list[TextualMemoryItem]:
+        """Get all memories.
+        Returns:
+            list[TextualMemoryItem]: List of all memories.
+        """
+
+    @abstractmethod
+    def delete(self, memory_ids: list[str]) -> None:
+        """Delete memories.
+        Args:
+            memory_ids (list[str]): List of memory IDs to delete.
+        """
+
+    @abstractmethod
+    def delete_all(self) -> None:
+        """Delete all memories."""
+
+    @abstractmethod
+    def drop(
+        self,
+    ) -> None:
+        """Drop all databases."""

memos/memories/textual/general.py

@@ -0,0 +1,236 @@
+import json
+import os
+
+from datetime import datetime
+from typing import Any
+
+from tenacity import retry, retry_if_exception_type, stop_after_attempt
+
+from memos.configs.memory import GeneralTextMemoryConfig
+from memos.embedders.factory import ArkEmbedder, EmbedderFactory, OllamaEmbedder
+from memos.llms.factory import AzureLLM, LLMFactory, OllamaLLM, OpenAILLM
+from memos.log import get_logger
+from memos.memories.textual.base import BaseTextMemory
+from memos.memories.textual.item import TextualMemoryItem
+from memos.templates.mem_reader_prompts import SIMPLE_STRUCT_MEM_READER_PROMPT
+from memos.types import MessageList
+from memos.vec_dbs.factory import QdrantVecDB, VecDBFactory
+from memos.vec_dbs.item import VecDBItem
+
+
+logger = get_logger(__name__)
+
+
+class GeneralTextMemory(BaseTextMemory):
+    """General textual memory implementation for storing and retrieving memories."""
+
+    def __init__(self, config: GeneralTextMemoryConfig):
+        """Initialize memory with the given configuration."""
+        # Set mode from class default or override if needed
+        self.mode = getattr(self.__class__, "mode", "sync")
+        self.config: GeneralTextMemoryConfig = config
+        self.extractor_llm: OpenAILLM | OllamaLLM | AzureLLM = LLMFactory.from_config(
+            config.extractor_llm
+        )
+        self.vector_db: QdrantVecDB = VecDBFactory.from_config(config.vector_db)
+        self.embedder: OllamaEmbedder | ArkEmbedder = EmbedderFactory.from_config(config.embedder)
+
+    @retry(
+        stop=stop_after_attempt(3),
+        retry=retry_if_exception_type(json.JSONDecodeError),
+        before_sleep=lambda retry_state: logger.warning(
+            f"Extracting memory failed due to JSON decode error: {retry_state.outcome.exception()}, Attempt retry: {retry_state.attempt_number} / {3}"
+        ),
+    )
+    def extract(self, messages: MessageList) -> list[TextualMemoryItem]:
+        """Extract memories based on the messages.
+
+        Args:
+            messages: List of message dictionaries to extract memories from.
+
+        Returns:
+            List of TextualMemoryItem objects representing the extracted memories.
+        """
+
+        str_messages = "\n".join(
+            [message["role"] + ":" + message["content"] for message in messages]
+        )
+
+        prompt = SIMPLE_STRUCT_MEM_READER_PROMPT.replace("${conversation}", str_messages).replace(
+            "${custom_tags_prompt}", ""
+        )
+        messages = [{"role": "user", "content": prompt}]
+        response_text = self.extractor_llm.generate(messages)
+        response_json = self.parse_json_result(response_text)
+
+        extracted_memories = [
+            TextualMemoryItem(
+                memory=memory_dict["value"],
+                metadata={
+                    "key": memory_dict["key"],
+                    "source": "conversation",
+                    "tags": memory_dict["tags"],
+                    "updated_at": datetime.now().isoformat(),
+                },
+            )
+            for memory_dict in response_json["memory list"]
+        ]
+
+        return extracted_memories
+
+    def add(self, memories: list[TextualMemoryItem | dict[str, Any]]) -> None:
+        """Add memories.
+
+        Args:
+            memories: List of TextualMemoryItem objects or dictionaries to add.
+        """
+        memory_items = [TextualMemoryItem(**m) if isinstance(m, dict) else m for m in memories]
+
+        # Memory encode
+        embed_memories = self.embedder.embed([m.memory for m in memory_items])
+
+        # Create vector db items
+        vec_db_items = []
+        for item, emb in zip(memory_items, embed_memories, strict=True):
+            vec_db_items.append(
+                VecDBItem(
+                    id=item.id,
+                    payload=item.model_dump(),
+                    vector=emb,
+                )
+            )
+
+        # Add to vector db
+        self.vector_db.add(vec_db_items)
+
+    def update(self, memory_id: str, new_memory: TextualMemoryItem | dict[str, Any]) -> None:
+        """Update a memory by memory_id."""
+        memory_item = (
+            TextualMemoryItem(**new_memory) if isinstance(new_memory, dict) else new_memory
+        )
+        memory_item.id = memory_id
+
+        vec_db_item = VecDBItem(
+            id=memory_item.id,
+            payload=memory_item.model_dump(),
+            vector=self._embed_one_sentence(memory_item.memory),
+        )
+
+        self.vector_db.update(memory_id, vec_db_item)
+
+    def search(self, query: str, top_k: int, info=None, **kwargs) -> list[TextualMemoryItem]:
+        """Search for memories based on a query.
+        Args:
+            query (str): The query to search for.
+            top_k (int): The number of top results to return.
+        Returns:
+            list[TextualMemoryItem]: List of matching memories.
+        """
+        query_vector = self._embed_one_sentence(query)
+        search_results = self.vector_db.search(query_vector, top_k)
+        search_results = sorted(  # make higher score first
+            search_results, key=lambda x: x.score, reverse=True
+        )
+        result_memories = [
+            TextualMemoryItem(**search_item.payload) for search_item in search_results
+        ]
+        return result_memories
+
+    def get(self, memory_id: str, user_name: str | None = None) -> TextualMemoryItem:
+        """Get a memory by its ID."""
+        result = self.vector_db.get_by_id(memory_id)
+        if result is None:
+            raise ValueError(f"Memory with ID {memory_id} not found")
+        return TextualMemoryItem(**result.payload)
+
+    def get_by_ids(self, memory_ids: list[str]) -> list[TextualMemoryItem]:
+        """Get memories by their IDs.
+        Args:
+            memory_ids (list[str]): List of memory IDs to retrieve.
+        Returns:
+            list[TextualMemoryItem]: List of memories with the specified IDs.
+        """
+        db_items = self.vector_db.get_by_ids(memory_ids)
+        memories = [TextualMemoryItem(**db_item.payload) for db_item in db_items]
+        return memories
+
+    def get_all(self) -> list[TextualMemoryItem]:
+        """Get all memories.
+        Returns:
+            list[TextualMemoryItem]: List of all memories.
+        """
+        all_items = self.vector_db.get_all()
+        all_memories = [TextualMemoryItem(**memo.payload) for memo in all_items]
+        return all_memories
+
+    def delete(self, memory_ids: list[str]) -> None:
+        """Delete a memory."""
+        self.vector_db.delete(memory_ids)
+
+    def delete_all(self) -> None:
+        """Delete all memories."""
+        self.vector_db.delete_collection(self.vector_db.config.collection_name)
+        self.vector_db.create_collection()
+
+    def load(self, dir: str) -> None:
+        try:
+            memory_file = os.path.join(dir, self.config.memory_filename)
+
+            if not os.path.exists(memory_file):
+                logger.warning(f"Memory file not found: {memory_file}")
+                return
+
+            with open(memory_file, encoding="utf-8") as f:
+                memories = json.load(f)
+
+            vec_db_items = [VecDBItem.from_dict(m) for m in memories]
+            self.vector_db.add(vec_db_items)
+            logger.info(f"Loaded {len(memories)} memories from {memory_file}")
+
+        except FileNotFoundError:
+            logger.error(f"Memory file not found in directory: {dir}")
+        except json.JSONDecodeError as e:
+            logger.error(f"Error decoding JSON from memory file: {e}")
+        except Exception as e:
+            logger.error(f"An error occurred while loading memories: {e}")
+
+    def dump(self, dir: str) -> None:
+        """Dump memories to os.path.join(dir, self.config.memory_filename)"""
+        try:
+            all_vec_db_items = self.vector_db.get_all()
+            json_memories = [memory.to_dict() for memory in all_vec_db_items]
+
+            os.makedirs(dir, exist_ok=True)
+            memory_file = os.path.join(dir, self.config.memory_filename)
+            with open(memory_file, "w", encoding="utf-8") as f:
+                json.dump(json_memories, f, indent=4, ensure_ascii=False)
+
+            logger.info(f"Dumped {len(all_vec_db_items)} memories to {memory_file}")
+
+        except Exception as e:
+            logger.error(f"An error occurred while dumping memories: {e}")
+            raise
+
+    def drop(
+        self,
+    ) -> None:
+        pass
+
+    def _embed_one_sentence(self, sentence: str) -> list[float]:
+        """Embed a single sentence."""
+        return self.embedder.embed([sentence])[0]
+
+    def parse_json_result(self, response_text):
+        try:
+            json_start = response_text.find("{")
+            response_text = response_text[json_start:]
+            response_text = response_text.replace("```", "").strip()
+            if response_text[-1] != "}":
+                response_text += "}"
+            response_json = json.loads(response_text)
+            return response_json
+        except json.JSONDecodeError as e:
+            logger.warning(
+                f"Failed to parse LLM response as JSON: {e}\nRaw response:\n{response_text}"
+            )
+            return {}

memos/memories/textual/item.py

@@ -0,0 +1,304 @@
+"""Defines memory item types for textual memory."""
+
+import json
+import logging
+import uuid
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+ALLOWED_ROLES = {"user", "assistant", "system"}
+
+
+class SourceMessage(BaseModel):
+    """
+    Purpose: **memory provenance / traceability**.
+
+    Capture the minimal, reproducible origin context of a memory item so it can be
+    audited, traced, rolled back, or de-duplicated later.
+
+    Fields & conventions:
+        - type: Source kind (e.g., "chat", "doc", "web", "file", "system", ...).
+            If not provided, upstream logic may infer it:
+            presence of `role` ⇒ "chat"; otherwise ⇒ "doc".
+        - role: Conversation role ("user" | "assistant" | "system" | "tool") when the
+            source is a chat turn.
+        - content: Minimal reproducible snippet from the source. If omitted,
+            upstream may fall back to `doc_path` / `url` / `message_id`.
+        - file_info: File information for file source.
+        - chat_time / message_id / doc_path: Locators for precisely pointing back
+            to the original record (timestamp, message id, document path).
+        - Extra fields: Allowed (`model_config.extra="allow"`) to carry arbitrary
+            provenance attributes (e.g., url, page, offset, span, local_confidence).
+    """
+
+    type: str | None = "chat"
+    role: Literal["user", "assistant", "system", "tool"] | None = None
+    chat_time: str | None = None
+    message_id: str | None = None
+    content: str | None = None
+    doc_path: str | None = None
+    file_info: dict | None = None
+    model_config = ConfigDict(extra="allow")
+
+
+class TextualMemoryMetadata(BaseModel):
+    """Metadata for a memory item.
+
+    This includes information such as the type of memory, when it occurred,
+    its source, and other relevant details.
+    """
+
+    user_id: str | None = Field(
+        default=None,
+        description="The ID of the user associated with the memory. Useful for multi-user systems.",
+    )
+    session_id: str | None = Field(
+        default=None,
+        description="The ID of the session during which the memory was created. Useful for tracking context in conversations.",
+    )
+    status: Literal["activated", "archived", "deleted"] | None = Field(
+        default="activated",
+        description="The status of the memory, e.g., 'activated', 'archived', 'deleted'.",
+    )
+    type: str | None = Field(default=None)
+    key: str | None = Field(default=None, description="Memory key or title.")
+    confidence: float | None = Field(
+        default=None,
+        description="A numeric score (float between 0 and 100) indicating how certain you are about the accuracy or reliability of the memory.",
+    )
+    source: Literal["conversation", "retrieved", "web", "file", "system"] | None = Field(
+        default=None, description="The origin of the memory"
+    )
+    tags: list[str] | None = Field(
+        default=None,
+        description='A list of keywords or thematic labels associated with the memory for categorization or retrieval, e.g., `["travel", "health", "project-x"]`.',
+    )
+    visibility: Literal["private", "public", "session"] | None = Field(
+        default=None, description="e.g., 'private', 'public', 'session'"
+    )
+    updated_at: str | None = Field(
+        default_factory=lambda: datetime.now().isoformat(),
+        description="The timestamp of the last modification to the memory. Useful for tracking memory freshness or change history. Format: ISO 8601.",
+    )
+    info: dict | None = Field(
+        default=None,
+        description="Arbitrary key-value pairs for additional metadata.",
+    )
+
+    model_config = ConfigDict(extra="allow")
+
+    covered_history: Any | None = Field(
+        default=None,
+        description="Record the memory id covered by the update",
+    )
+
+    def __str__(self) -> str:
+        """Pretty string representation of the metadata."""
+        meta = self.model_dump(exclude_none=True)
+        return ", ".join(f"{k}={v}" for k, v in meta.items())
+
+
+class TreeNodeTextualMemoryMetadata(TextualMemoryMetadata):
+    """Extended metadata for structured memory, layered retrieval, and lifecycle tracking."""
+
+    memory_type: Literal[
+        "WorkingMemory",
+        "LongTermMemory",
+        "UserMemory",
+        "OuterMemory",
+        "ToolSchemaMemory",
+        "ToolTrajectoryMemory",
+    ] = Field(default="WorkingMemory", description="Memory lifecycle type.")
+    sources: list[SourceMessage] | None = Field(
+        default=None, description="Multiple origins of the memory (e.g., URLs, notes)."
+    )
+    embedding: list[float] | None = Field(
+        default=None,
+        description="The vector embedding of the memory content, used for semantic search or clustering.",
+    )
+    created_at: str | None = Field(
+        default_factory=lambda: datetime.now().isoformat(),
+        description="The timestamp of the first creation to the memory. Useful "
+        "for tracking memory initialization. Format: ISO 8601.",
+    )
+    usage: list[str] = Field(
+        default_factory=list,
+        description="Usage history of this node",
+    )
+    background: str | None = Field(
+        default="",
+        description="background of this node",
+    )
+
+    file_ids: list[str] | None = Field(
+        default_factory=list,
+        description="The ids of the files associated with the memory.",
+    )
+
+    @field_validator("sources", mode="before")
+    @classmethod
+    def coerce_sources(cls, v):
+        if v is None:
+            return v
+            # Handle string representation of sources (e.g., from PostgreSQL array or malformed data)
+        if isinstance(v, str):
+            logging.info(f"[coerce_sources] v: {v} type: {type(v)}")
+            # If it's a string that looks like a list representation, try to parse it
+            # This handles cases like: "[uuid1, uuid2, uuid3]" or "[item1, item2]"
+            v_stripped = v.strip()
+            if v_stripped.startswith("[") and v_stripped.endswith("]"):
+                # Remove brackets and split by comma
+                content = v_stripped[1:-1].strip()
+                if content:
+                    # Split by comma and clean up each item
+                    items = [item.strip() for item in content.split(",")]
+                    # Convert to list of strings
+                    v = items
+                else:
+                    v = []
+            else:
+                # Single string, wrap in list
+                v = [v]
+        if not isinstance(v, list):
+            raise TypeError("sources must be a list")
+        out = []
+        for item in v:
+            if isinstance(item, SourceMessage):
+                out.append(item)
+
+            elif isinstance(item, dict):
+                d = dict(item)
+                if d.get("type") is None:
+                    d["type"] = "chat" if d.get("role") in ALLOWED_ROLES else "doc"
+                out.append(SourceMessage(**d))
+
+            elif isinstance(item, str):
+                try:
+                    parsed = json.loads(item)
+                except Exception:
+                    parsed = None
+
+                if isinstance(parsed, dict):
+                    if parsed.get("type") is None:
+                        parsed["type"] = "chat" if parsed.get("role") in ALLOWED_ROLES else "doc"
+                    out.append(SourceMessage(**parsed))
+                else:
+                    out.append(SourceMessage(type="doc", content=item))
+
+            else:
+                out.append(SourceMessage(type="doc", content=str(item)))
+        return out
+
+    def __str__(self) -> str:
+        """Pretty string representation of the metadata."""
+        meta = self.model_dump(exclude_none=True)
+        return ", ".join([f"{k}={v}" for k, v in meta.items() if k != "embedding"])
+
+
+class SearchedTreeNodeTextualMemoryMetadata(TreeNodeTextualMemoryMetadata):
+    """Metadata for nodes returned by search, includes similarity info."""
+
+    relativity: float | None = Field(
+        default=None, description="Similarity score with respect to the query, 0 ~ 1."
+    )
+
+
+class PreferenceTextualMemoryMetadata(TextualMemoryMetadata):
+    """Metadata for preference memory item."""
+
+    preference_type: Literal["explicit_preference", "implicit_preference"] = Field(
+        default="explicit_preference", description="Type of preference."
+    )
+    dialog_id: str | None = Field(default=None, description="ID of the dialog.")
+    original_text: str | None = Field(default=None, description="String of the dialog.")
+    embedding: list[float] | None = Field(default=None, description="Vector of the dialog.")
+    preference: str | None = Field(default=None, description="Preference.")
+    created_at: str | None = Field(default=None, description="Timestamp of the dialog.")
+    mem_cube_id: str | None = Field(default=None, description="ID of the MemCube.")
+    score: float | None = Field(default=None, description="Score of the retrieval result.")
+
+
+class TextualMemoryItem(BaseModel):
+    """Represents a single memory item in the textual memory.
+
+    This serves as a standardized format for memory items across different
+    textual memory implementations.
+    """
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    memory: str
+    metadata: (
+        SearchedTreeNodeTextualMemoryMetadata
+        | TreeNodeTextualMemoryMetadata
+        | TextualMemoryMetadata
+        | PreferenceTextualMemoryMetadata
+    ) = Field(default_factory=TextualMemoryMetadata)
+
+    model_config = ConfigDict(extra="forbid")
+
+    @field_validator("id")
+    @classmethod
+    def _validate_id(cls, v: str) -> str:
+        uuid.UUID(v)
+        return v
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "TextualMemoryItem":
+        return cls(**data)
+
+    def to_dict(self) -> dict:
+        return self.model_dump(exclude_none=True)
+
+    @field_validator("metadata", mode="before")
+    @classmethod
+    def _coerce_metadata(cls, v: Any):
+        if isinstance(
+            v,
+            SearchedTreeNodeTextualMemoryMetadata
+            | TreeNodeTextualMemoryMetadata
+            | TextualMemoryMetadata
+            | PreferenceTextualMemoryMetadata,
+        ):
+            return v
+        if isinstance(v, dict):
+            if "metadata" in v and isinstance(v["metadata"], dict):
+                nested_metadata = v["metadata"]
+                nested_metadata = nested_metadata.copy()
+                nested_metadata.pop("id", None)
+                nested_metadata.pop("memory", None)
+                v = nested_metadata
+            else:
+                v = v.copy()
+                v.pop("id", None)
+                v.pop("memory", None)
+
+            if v.get("relativity") is not None:
+                return SearchedTreeNodeTextualMemoryMetadata(**v)
+            if v.get("preference_type") is not None:
+                return PreferenceTextualMemoryMetadata(**v)
+            if any(k in v for k in ("sources", "memory_type", "embedding", "background", "usage")):
+                return TreeNodeTextualMemoryMetadata(**v)
+            return TextualMemoryMetadata(**v)
+        return v
+
+    def __str__(self) -> str:
+        """Pretty string representation of the memory item."""
+        return f"<ID: {self.id} | Memory: {self.memory} | Metadata: {self.metadata!s}>"
+
+
+def list_all_fields() -> list[str]:
+    """List all possible fields of the TextualMemoryItem model."""
+    top = list(TextualMemoryItem.model_fields.keys())
+    meta_models = [
+        TextualMemoryMetadata,
+        TreeNodeTextualMemoryMetadata,
+        SearchedTreeNodeTextualMemoryMetadata,
+        PreferenceTextualMemoryMetadata,
+    ]
+    meta_all = sorted(set().union(*[set(m.model_fields.keys()) for m in meta_models]))
+
+    return top + meta_all