MemoryOS 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

memos/mem_scheduler/schemas/monitor_schemas.py

@@ -0,0 +1,329 @@
+from collections import Counter
+from datetime import datetime
+from pathlib import Path
+from typing import ClassVar
+from uuid import uuid4
+
+from pydantic import BaseModel, Field, computed_field, field_validator
+
+from memos.log import get_logger
+from memos.mem_scheduler.modules.misc import AutoDroppingQueue, DictConversionMixin
+from memos.mem_scheduler.schemas.general_schemas import (
+    DEFAULT_MAX_QUERY_KEY_WORDS,
+    DEFAULT_WEIGHT_VECTOR_FOR_RANKING,
+    NOT_INITIALIZED,
+)
+from memos.mem_scheduler.utils.filter_utils import transform_name_to_key
+from memos.memories.textual.tree import TextualMemoryItem
+
+
+logger = get_logger(__name__)
+
+FILE_PATH = Path(__file__).absolute()
+BASE_DIR = FILE_PATH.parent.parent.parent.parent.parent
+
+
+# ============== Queries ==============
+class QueryMonitorItem(BaseModel, DictConversionMixin):
+    item_id: str = Field(
+        description="Unique identifier for the query item", default_factory=lambda: str(uuid4())
+    )
+    query_text: str = Field(
+        ...,
+        description="The actual user query text content",
+        min_length=1,
+    )
+    keywords: list[str] | None = Field(
+        default=None,
+        min_length=1,  # If provided, shouldn't be empty
+        description="Semantic keywords extracted from the query text",
+    )
+    max_keywords: ClassVar[int] = DEFAULT_MAX_QUERY_KEY_WORDS
+
+    timestamp: datetime = Field(
+        default_factory=datetime.now, description="Timestamp indicating when query was submitted"
+    )
+
+    @field_validator("keywords", mode="before")
+    @classmethod
+    def validate_keywords(cls, v, values):
+        if v is None:
+            return None
+
+        if not isinstance(v, list):
+            raise ValueError("Keywords must be a list")
+
+        if len(v) > cls.max_keywords:
+            logger.warning(
+                f"Keywords list truncated from {len(v)} to {cls.max_keywords} items. "
+                f"Configure max_keywords class attribute to adjust this limit."
+            )
+            return v[: cls.max_keywords]
+        return v
+
+    @classmethod
+    def with_max_keywords(cls, limit: int):
+        """Create a new class with custom keywords limit."""
+        if not isinstance(limit, int) or limit <= 0:
+            raise ValueError("Max keywords limit must be positive integer")
+
+        return type(f"{cls.__name__}_MaxKeywords{limit}", (cls,), {"max_keywords": limit})
+
+
+class QueryMonitorQueue(AutoDroppingQueue[QueryMonitorItem]):
+    """
+    A thread-safe queue for monitoring queries with timestamp and keyword tracking.
+    Each item is expected to be a dictionary containing:
+    """
+
+    def put(self, item: QueryMonitorItem, block: bool = True, timeout: float | None = None) -> None:
+        """
+        Add a query item to the queue. Ensures the item is of correct type.
+
+        Args:
+            item: A QueryMonitorItem instance
+        """
+        if not isinstance(item, QueryMonitorItem):
+            raise ValueError("Item must be an instance of QueryMonitorItem")
+        super().put(item, block, timeout)
+
+    def get_queries_by_timestamp(
+        self, start_time: datetime, end_time: datetime
+    ) -> list[QueryMonitorItem]:
+        """
+        Retrieve queries added between the specified time range.
+        """
+        with self.mutex:
+            return [item for item in self.queue if start_time <= item.timestamp <= end_time]
+
+    def get_keywords_collections(self) -> Counter:
+        """
+        Generate a Counter containing keyword frequencies across all queries.
+
+        Returns:
+            Counter object with keyword counts
+        """
+        with self.mutex:
+            all_keywords = [kw for item in self.queue for kw in item.keywords]
+            return Counter(all_keywords)
+
+    def get_queries_with_timesort(self, reverse: bool = True) -> list[str]:
+        """
+        Retrieve all queries sorted by timestamp.
+
+        Args:
+            reverse: If True, sort in descending order (newest first),
+                     otherwise sort in ascending order (oldest first)
+
+        Returns:
+            List of query items sorted by timestamp
+        """
+        with self.mutex:
+            return [
+                monitor.query_text
+                for monitor in sorted(self.queue, key=lambda x: x.timestamp, reverse=reverse)
+            ]
+
+
+# ============== Memories ==============
+class MemoryMonitorItem(BaseModel, DictConversionMixin):
+    item_id: str = Field(
+        description="Unique identifier for the memory item", default_factory=lambda: str(uuid4())
+    )
+    memory_text: str = Field(
+        ...,
+        description="The actual content of the memory",
+        min_length=1,
+    )
+    tree_memory_item: TextualMemoryItem | None = Field(
+        default=None, description="Optional textual memory item"
+    )
+    tree_memory_item_mapping_key: str = Field(
+        description="Key generated from memory_text using transform_name_to_key",
+    )
+    keywords_score: float = Field(
+        default=NOT_INITIALIZED,
+        description="The score generate by counting keywords in queries",
+        ge=NOT_INITIALIZED,  # Minimum value of 0
+    )
+    sorting_score: float = Field(
+        default=NOT_INITIALIZED,
+        description="The score generate from rerank process",
+        ge=NOT_INITIALIZED,  # Minimum value of 0
+    )
+    importance_score: float = Field(
+        default=NOT_INITIALIZED,
+        description="Numerical score representing the memory's importance",
+        ge=NOT_INITIALIZED,  # Minimum value of 0
+    )
+    recording_count: int = Field(
+        default=1,
+        description="How many times this memory has been recorded",
+        ge=1,  # Greater than or equal to 1
+    )
+
+    @field_validator("tree_memory_item_mapping_key", mode="before")
+    def generate_mapping_key(cls, v, values):  # noqa: N805
+        if v is None and "memory_text" in values:
+            return transform_name_to_key(values["memory_text"])
+        return v
+
+    def get_importance_score(self, weight_vector: list[float] | None = None) -> float:
+        """
+        Calculate the effective score for the memory item.
+
+        Returns:
+            float: The importance_score if it has been initialized (>=0),
+                   otherwise the recording_count converted to float.
+
+        Note:
+            This method provides a unified way to retrieve a comparable score
+            for memory items, regardless of whether their importance has been explicitly set.
+        """
+        if weight_vector is None:
+            logger.warning("weight_vector of get_importance_score is None.")
+            weight_vector = DEFAULT_WEIGHT_VECTOR_FOR_RANKING
+        assert sum(weight_vector) == 1
+        normalized_keywords_score = min(self.keywords_score * weight_vector[1], 5)
+        normalized_recording_count_score = min(self.recording_count * weight_vector[2], 2)
+        self.importance_score = (
+            self.sorting_score * weight_vector[0]
+            + normalized_keywords_score
+            + normalized_recording_count_score
+        )
+        return self.importance_score
+
+
+class MemoryMonitorManager(BaseModel, DictConversionMixin):
+    user_id: str = Field(..., description="Required user identifier", min_length=1)
+    mem_cube_id: str = Field(..., description="Required memory cube identifier", min_length=1)
+    memories: list[MemoryMonitorItem] = Field(
+        default_factory=list, description="Collection of memory items"
+    )
+    max_capacity: int | None = Field(
+        default=None, description="Maximum number of memories allowed (None for unlimited)", ge=1
+    )
+
+    @computed_field
+    @property
+    def memory_size(self) -> int:
+        """Automatically calculated count of memory items."""
+        return len(self.memories)
+
+    @property
+    def memories_mapping_dict(self) -> dict[str, MemoryMonitorItem]:
+        """
+        Generate a mapping dictionary for the memories in MemoryMonitorManager,
+        using tree_memory_item_mapping_key as the key and MemoryMonitorItem as the value.
+
+        Returns:
+            Dict[str, MemoryMonitorItem]: A dictionary where keys are
+            tree_memory_item_mapping_key values from MemoryMonitorItem,
+            and values are the corresponding MemoryMonitorItem objects.
+        """
+        mapping_dict = {
+            mem_item.tree_memory_item_mapping_key: mem_item for mem_item in self.memories
+        }
+
+        logger.debug(
+            f"Generated memories mapping dict for user_id={self.user_id}, "
+            f"mem_cube_id={self.mem_cube_id}, "
+            f"total_items={len(mapping_dict)}, "
+            f"source_memory_count={len(self.memories)}"
+        )
+        return mapping_dict
+
+    def get_sorted_mem_monitors(self, reverse=True) -> list[MemoryMonitorItem]:
+        """
+        Retrieve memory monitors sorted by their ranking score in descending order.
+
+        Returns:
+            list[MemoryMonitorItem]: Sorted list of memory monitor items.
+        """
+        return sorted(
+            self.memories,
+            key=lambda item: item.get_importance_score(
+                weight_vector=DEFAULT_WEIGHT_VECTOR_FOR_RANKING
+            ),
+            reverse=reverse,
+        )
+
+    def update_memories(
+        self, new_memory_monitors: list[MemoryMonitorItem], partial_retention_number: int
+    ) -> MemoryMonitorItem:
+        """
+        Update memories based on monitor_working_memories.
+        """
+
+        # Validate partial_retention_number
+        if partial_retention_number < 0:
+            raise ValueError("partial_retention_number must be non-negative")
+
+        # Step 1: Update existing memories or add new ones
+        added_count = 0
+        memories_mapping_dict = self.memories_mapping_dict
+        new_mem_set = set()
+        for memory_monitor in new_memory_monitors:
+            if memory_monitor.tree_memory_item_mapping_key in memories_mapping_dict:
+                # Update existing memory
+                item: MemoryMonitorItem = memories_mapping_dict[
+                    memory_monitor.tree_memory_item_mapping_key
+                ]
+                item.recording_count += 1
+                item.keywords_score = memory_monitor.keywords_score
+                item.sorting_score = memory_monitor.sorting_score
+            else:
+                # Add new memory
+                self.memories.append(memory_monitor)
+                added_count += 1
+
+            new_mem_set.add(memory_monitor.tree_memory_item_mapping_key)
+
+        # Step 2: Identify memories to remove
+        old_mem_monitor_list = []
+        for mem_monitor in self.memories:
+            if mem_monitor.tree_memory_item_mapping_key not in new_mem_set:
+                old_mem_monitor_list.append(mem_monitor)
+
+        # Sort memories by recording_count in descending order
+        sorted_old_mem_monitors = sorted(
+            old_mem_monitor_list,
+            key=lambda item: item.get_importance_score(
+                weight_vector=DEFAULT_WEIGHT_VECTOR_FOR_RANKING
+            ),
+            reverse=True,
+        )
+
+        # Keep the top N old memories
+        memories_to_remove = sorted_old_mem_monitors[partial_retention_number:]
+        memories_to_change_score = sorted_old_mem_monitors[:partial_retention_number]
+
+        # Step 3: Remove identified memories and change the scores of left old memories
+        for memory in memories_to_remove:
+            self.memories.remove(memory)
+
+        for memory in memories_to_change_score:
+            memory.sorting_score = 0
+            memory.recording_count = 0
+            memory.keywords_score = 0
+
+        # Step 4: Enforce max_capacity if set
+        sorted_memories = sorted(
+            self.memories,
+            key=lambda item: item.get_importance_score(
+                weight_vector=DEFAULT_WEIGHT_VECTOR_FOR_RANKING
+            ),
+            reverse=True,
+        )
+        # Keep only the top max_capacity memories
+        self.memories = sorted_memories[: self.max_capacity]
+
+        # Log the update result
+        logger.info(
+            f"Updated monitor manager for user {self.user_id}, mem_cube {self.mem_cube_id}: "
+            f"Total memories: {len(self.memories)}, "
+            f"Added/Updated: {added_count}, "
+            f"Removed: {len(memories_to_remove)} (excluding top {partial_retention_number} by recording_count)"
+        )
+
+        return self.memories

memos/mem_scheduler/utils/filter_utils.py

@@ -0,0 +1,176 @@
+import re
+
+from memos.dependency import require_python_package
+from memos.log import get_logger
+
+
+logger = get_logger(__name__)
+
+
+def transform_name_to_key(name):
+    """
+    Normalize text by removing all punctuation marks, keeping only letters, numbers, and word characters.
+
+    Args:
+        name (str): Input text to be processed
+
+    Returns:
+        str: Processed text with all punctuation removed
+    """
+    # Match all characters that are NOT:
+    # \w - word characters (letters, digits, underscore)
+    # \u4e00-\u9fff - Chinese/Japanese/Korean characters
+    # \s - whitespace
+    pattern = r"[^\w\u4e00-\u9fff\s]"
+
+    # Substitute all matched punctuation marks with empty string
+    # re.UNICODE flag ensures proper handling of Unicode characters
+    normalized = re.sub(pattern, "", name, flags=re.UNICODE)
+
+    # Optional: Collapse multiple whitespaces into single space
+    normalized = "_".join(normalized.split())
+
+    normalized = normalized.lower()
+
+    return normalized
+
+
+def is_all_english(input_string: str) -> bool:
+    """Determine if the string consists entirely of English characters (including spaces)"""
+    return all(char.isascii() or char.isspace() for char in input_string)
+
+
+def is_all_chinese(input_string: str) -> bool:
+    """Determine if the string consists entirely of Chinese characters (including Chinese punctuation and spaces)"""
+    return all(
+        ("\u4e00" <= char <= "\u9fff")  # Basic Chinese characters
+        or ("\u3400" <= char <= "\u4dbf")  # Extension A
+        or ("\u20000" <= char <= "\u2a6df")  # Extension B
+        or ("\u2a700" <= char <= "\u2b73f")  # Extension C
+        or ("\u2b740" <= char <= "\u2b81f")  # Extension D
+        or ("\u2b820" <= char <= "\u2ceaf")  # Extension E
+        or ("\u2f800" <= char <= "\u2fa1f")  # Extension F
+        or char.isspace()  # Spaces
+        for char in input_string
+    )
+
+
+@require_python_package(
+    import_name="sklearn",
+    install_command="pip install scikit-learn",
+    install_link="https://scikit-learn.org/stable/install.html",
+)
+def filter_similar_memories(
+    text_memories: list[str], similarity_threshold: float = 0.75
+) -> list[str]:
+    """
+    Filters out low-quality or duplicate memories based on text similarity.
+
+    Args:
+        text_memories: List of text memories to filter
+        similarity_threshold: Threshold for considering memories duplicates (0.0-1.0)
+                            Higher values mean stricter filtering
+
+    Returns:
+        List of filtered memories with duplicates removed
+    """
+    from sklearn.feature_extraction.text import TfidfVectorizer
+    from sklearn.metrics.pairwise import cosine_similarity
+
+    if not text_memories:
+        logger.warning("Received empty memories list - nothing to filter")
+        return []
+
+    for idx in range(len(text_memories)):
+        if not isinstance(text_memories[idx], str):
+            logger.error(
+                f"{text_memories[idx]} in memories is not a string,"
+                f" and now has been transformed to be a string."
+            )
+            text_memories[idx] = str(text_memories[idx])
+
+    try:
+        # Step 1: Vectorize texts using TF-IDF
+        vectorizer = TfidfVectorizer()
+        tfidf_matrix = vectorizer.fit_transform(text_memories)
+
+        # Step 2: Calculate pairwise similarity matrix
+        similarity_matrix = cosine_similarity(tfidf_matrix)
+
+        # Step 3: Identify duplicates
+        to_keep = set(range(len(text_memories)))  # Start with all indices
+        for i in range(len(similarity_matrix)):
+            if i not in to_keep:
+                continue  # Already marked for removal
+
+            # Find all similar items to this one (excluding self and already removed)
+            similar_indices = [
+                j
+                for j in range(i + 1, len(similarity_matrix))
+                if similarity_matrix[i][j] >= similarity_threshold and j in to_keep
+            ]
+            similar_indices = set(similar_indices)
+
+            # Remove all similar items (keeping the first one - i)
+            to_keep -= similar_indices
+
+        # Return filtered memories
+        filtered_memories = [text_memories[i] for i in sorted(to_keep)]
+        logger.debug(f"filtered_memories: {filtered_memories}")
+        return filtered_memories
+
+    except Exception as e:
+        logger.error(f"Error filtering memories: {e!s}")
+        return text_memories  # Return original list if error occurs
+
+
+def filter_too_short_memories(
+    text_memories: list[str], min_length_threshold: int = 20
+) -> list[str]:
+    """
+    Filters out text memories that fall below the minimum length requirement.
+    Handles both English (word count) and Chinese (character count) differently.
+
+    Args:
+        text_memories: List of text memories to be filtered
+        min_length_threshold: Minimum length required to keep a memory.
+                            For English: word count, for Chinese: character count.
+
+    Returns:
+        List of filtered memories meeting the length requirement
+    """
+    if not text_memories:
+        logger.debug("Empty memories list received in short memory filter")
+        return []
+
+    filtered_memories = []
+    removed_count = 0
+
+    for memory in text_memories:
+        stripped_memory = memory.strip()
+        if not stripped_memory:  # Skip empty/whitespace memories
+            removed_count += 1
+            continue
+
+        # Determine measurement method based on language
+        if is_all_english(stripped_memory):
+            length = len(stripped_memory.split())  # Word count for English
+        elif is_all_chinese(stripped_memory):
+            length = len(stripped_memory)  # Character count for Chinese
+        else:
+            logger.debug(f"Mixed-language memory, using character count: {stripped_memory[:50]}...")
+            length = len(stripped_memory)  # Default to character count
+
+        if length >= min_length_threshold:
+            filtered_memories.append(memory)
+        else:
+            removed_count += 1
+
+    if removed_count > 0:
+        logger.info(
+            f"Filtered out {removed_count} short memories "
+            f"(below {min_length_threshold} units). "
+            f"Total remaining: {len(filtered_memories)}"
+        )
+
+    return filtered_memories

memos/mem_scheduler/utils/misc_utils.py

@@ -0,0 +1,61 @@
+import json
+
+from functools import wraps
+from pathlib import Path
+
+import yaml
+
+from memos.log import get_logger
+
+
+logger = get_logger(__name__)
+
+
+def extract_json_dict(text: str):
+    text = text.strip()
+    patterns_to_remove = ["json```", "```python", "```json", "latex```", "```latex", "```"]
+    for pattern in patterns_to_remove:
+        text = text.replace(pattern, "")
+    res = json.loads(text.strip())
+    return res
+
+
+def parse_yaml(yaml_file: str | Path):
+    yaml_path = Path(yaml_file)
+    if not yaml_path.is_file():
+        raise FileNotFoundError(f"No such file: {yaml_file}")
+
+    with yaml_path.open("r", encoding="utf-8") as fr:
+        data = yaml.safe_load(fr)
+
+    return data
+
+
+def log_exceptions(logger=logger):
+    """
+    Exception-catching decorator that automatically logs errors (including stack traces)
+
+    Args:
+        logger: Optional logger object (default: module-level logger)
+
+    Example:
+        @log_exceptions()
+        def risky_function():
+            raise ValueError("Oops!")
+
+        @log_exceptions(logger=custom_logger)
+        def another_risky_function():
+            might_fail()
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                logger.error(f"Error in {func.__name__}: {e}", exc_info=True)
+
+        return wrapper
+
+    return decorator

memos/mem_user/factory.py

@@ -0,0 +1,94 @@
+from typing import Any, ClassVar
+
+from memos.configs.mem_user import UserManagerConfigFactory
+from memos.mem_user.mysql_user_manager import MySQLUserManager
+from memos.mem_user.user_manager import UserManager
+
+
+class UserManagerFactory:
+    """Factory class for creating user manager instances."""
+
+    backend_to_class: ClassVar[dict[str, Any]] = {
+        "sqlite": UserManager,
+        "mysql": MySQLUserManager,
+    }
+
+    @classmethod
+    def from_config(
+        cls, config_factory: UserManagerConfigFactory
+    ) -> UserManager | MySQLUserManager:
+        """Create a user manager instance from configuration.
+
+        Args:
+            config_factory: Configuration factory containing backend and config
+
+        Returns:
+            User manager instance
+
+        Raises:
+            ValueError: If backend is not supported
+        """
+        backend = config_factory.backend
+        if backend not in cls.backend_to_class:
+            raise ValueError(f"Invalid user manager backend: {backend}")
+
+        user_manager_class = cls.backend_to_class[backend]
+        config = config_factory.config
+
+        # Use model_dump() to convert Pydantic model to dict and unpack as kwargs
+        return user_manager_class(**config.model_dump())
+
+    @classmethod
+    def create_sqlite(cls, db_path: str | None = None, user_id: str = "root") -> UserManager:
+        """Create SQLite user manager with default configuration.
+
+        Args:
+            db_path: Path to SQLite database file
+            user_id: Default user ID for initialization
+
+        Returns:
+            SQLite user manager instance
+        """
+        config_factory = UserManagerConfigFactory(
+            backend="sqlite", config={"db_path": db_path, "user_id": user_id}
+        )
+        return cls.from_config(config_factory)
+
+    @classmethod
+    def create_mysql(
+        cls,
+        user_id: str = "root",
+        host: str = "localhost",
+        port: int = 3306,
+        username: str = "root",
+        password: str = "",
+        database: str = "memos_users",
+        charset: str = "utf8mb4",
+    ) -> MySQLUserManager:
+        """Create MySQL user manager with specified configuration.
+
+        Args:
+            user_id: Default user ID for initialization
+            host: MySQL server host
+            port: MySQL server port
+            username: MySQL username
+            password: MySQL password
+            database: MySQL database name
+            charset: MySQL charset
+
+        Returns:
+            MySQL user manager instance
+        """
+        config_factory = UserManagerConfigFactory(
+            backend="mysql",
+            config={
+                "user_id": user_id,
+                "host": host,
+                "port": port,
+                "username": username,
+                "password": password,
+                "database": database,
+                "charset": charset,
+            },
+        )
+        return cls.from_config(config_factory)