MemoryOS 1.0.1__py3-none-any.whl → 1.1.1__py3-none-any.whl

memos/mem_scheduler/orm_modules/monitor_models.py

@@ -0,0 +1,261 @@
+from typing import TypeVar
+
+from sqlalchemy import Index
+from sqlalchemy.engine import Engine
+
+from memos.log import get_logger
+from memos.mem_scheduler.schemas.monitor_schemas import (
+    MemoryMonitorItem,
+    MemoryMonitorManager,
+    QueryMonitorItem,
+    QueryMonitorQueue,
+)
+
+from .base_model import BaseDBManager, LockableORM
+
+
+logger = get_logger(__name__)
+
+# Type variables for generic type hints
+T = TypeVar("T")  # The model type (MemoryMonitorManager, QueryMonitorManager, etc.)
+ORM = TypeVar("ORM")  # The ORM model type
+
+
+class MemoryMonitorManagerORM(LockableORM):
+    """ORM model for MemoryMonitorManager persistence
+
+    This table stores serialized MemoryMonitorManager instances with
+    proper indexing for efficient user and memory cube lookups.
+    """
+
+    __tablename__ = "memory_monitor_manager"
+
+    # Database indexes for performance optimization
+    __table_args__ = (Index("idx_memory_monitor_user_memcube", "user_id", "mem_cube_id"),)
+
+
+class QueryMonitorQueueORM(LockableORM):
+    """ORM model for QueryMonitorQueue persistence
+
+    This table stores serialized QueryMonitorQueue instances with
+    proper indexing for efficient user and memory cube lookups.
+    """
+
+    __tablename__ = "query_monitor_queue"
+
+    # Database indexes for performance optimization
+    __table_args__ = (Index("idx_query_monitor_user_memcube", "user_id", "mem_cube_id"),)
+
+
+class DBManagerForMemoryMonitorManager(BaseDBManager):
+    """Database manager for MemoryMonitorManager objects
+
+    This class handles persistence, synchronization, and locking
+    for MemoryMonitorManager instances in the database.
+    """
+
+    def __init__(
+        self,
+        engine: Engine,
+        user_id: str | None = None,
+        mem_cube_id: str | None = None,
+        obj: MemoryMonitorManager | None = None,
+        lock_timeout: int = 10,
+    ):
+        """
+        Initialize the MemoryMonitorManager database manager.
+
+        Args:
+            engine: SQLAlchemy engine instance
+            user_id: Unique identifier for the user
+            mem_cube_id: Unique identifier for the memory cube
+            obj: Optional MemoryMonitorManager instance to manage
+            lock_timeout: Timeout in seconds for lock acquisition
+        """
+        super().__init__(
+            engine=engine, user_id=user_id, mem_cube_id=mem_cube_id, lock_timeout=lock_timeout
+        )
+        self.obj: MemoryMonitorManager | None = obj
+
+    @property
+    def orm_class(self) -> type[MemoryMonitorManagerORM]:
+        return MemoryMonitorManagerORM
+
+    @property
+    def obj_class(self) -> type[MemoryMonitorManager]:
+        return MemoryMonitorManager
+
+    def merge_items(
+        self,
+        orm_instance: MemoryMonitorManagerORM,
+        obj_instance: MemoryMonitorManager,
+        size_limit: int,
+    ):
+        """Merge memory monitor items from database with current object
+
+        This method combines items from the database with items in the current
+        object, prioritizing current object items and applying size limits.
+
+        Args:
+            orm_instance: ORM instance containing serialized database data
+            obj_instance: Current MemoryMonitorManager instance
+            size_limit: Maximum number of items to keep after merge
+
+        Returns:
+            Updated obj_instance with merged items
+        """
+        logger.debug(f"Starting merge_items for MemoryMonitorManager with size_limit={size_limit}")
+
+        try:
+            # Deserialize the database instance
+            db_instance: MemoryMonitorManager = MemoryMonitorManager.from_json(
+                orm_instance.serialized_data
+            )
+        except Exception as e:
+            logger.error(f"Failed to deserialize database instance: {e}", exc_info=True)
+            logger.warning("Skipping merge due to deserialization error, using current object only")
+            return obj_instance
+
+        # Merge items - prioritize existing ones in current object
+        merged_items: list[MemoryMonitorItem] = []
+        seen_ids = set()
+
+        # First, add all items from current object (higher priority)
+        for item in obj_instance.memories:
+            if item.item_id not in seen_ids:
+                merged_items.append(item)
+                seen_ids.add(item.item_id)
+
+        # Then, add items from database that aren't in current object
+        for item in db_instance.memories:
+            if item.item_id not in seen_ids:
+                merged_items.append(item)
+                seen_ids.add(item.item_id)
+
+        # Apply size limit if specified (keep most recent items)
+        if size_limit is not None and size_limit > 0:
+            try:
+                # Sort by sorting_score descending (highest priority first) and take top N
+                # Note: MemoryMonitorItem doesn't have timestamp, so we use sorting_score instead
+                merged_items = sorted(merged_items, key=lambda x: x.sorting_score, reverse=True)[
+                    :size_limit
+                ]
+                logger.debug(f"Applied size limit of {size_limit}, kept {len(merged_items)} items")
+            except AttributeError as e:
+                logger.error(f"Error sorting MemoryMonitorItem objects: {e}")
+                logger.error(
+                    "Available attributes: "
+                    + ", ".join(dir(merged_items[0]) if merged_items else [])
+                )
+                raise
+            except Exception as e:
+                logger.error(f"Unexpected error during sorting: {e}")
+                raise
+
+        # Update the object with merged items
+        obj_instance.memories = merged_items
+
+        logger.info(
+            f"Merged {len(merged_items)} memory items for {obj_instance} (size_limit: {size_limit})"
+        )
+
+        return obj_instance
+
+
+class DBManagerForQueryMonitorQueue(BaseDBManager):
+    """Database manager for QueryMonitorQueue objects
+
+    This class handles persistence, synchronization, and locking
+    for QueryMonitorQueue instances in the database.
+    """
+
+    def __init__(
+        self,
+        engine: Engine,
+        user_id: str | None = None,
+        mem_cube_id: str | None = None,
+        obj: QueryMonitorQueue | None = None,
+        lock_timeout: int = 10,
+    ):
+        """
+        Initialize the QueryMonitorQueue database manager.
+
+        Args:
+            engine: SQLAlchemy engine instance
+            user_id: Unique identifier for the user
+            mem_cube_id: Unique identifier for the memory cube
+            obj: Optional QueryMonitorQueue instance to manage
+            lock_timeout: Timeout in seconds for lock acquisition
+        """
+        super().__init__(
+            engine=engine, user_id=user_id, mem_cube_id=mem_cube_id, lock_timeout=lock_timeout
+        )
+        self.obj: QueryMonitorQueue | None = obj
+
+    @property
+    def orm_class(self) -> type[QueryMonitorQueueORM]:
+        return QueryMonitorQueueORM
+
+    @property
+    def obj_class(self) -> type[QueryMonitorQueue]:
+        return QueryMonitorQueue
+
+    def merge_items(
+        self, orm_instance: QueryMonitorQueueORM, obj_instance: QueryMonitorQueue, size_limit: int
+    ):
+        """Merge query monitor items from database with current queue
+
+        This method combines items from the database with items in the current
+        queue, prioritizing current queue items and applying size limits.
+
+        Args:
+            orm_instance: ORM instance containing serialized database data
+            obj_instance: Current QueryMonitorQueue instance
+            size_limit: Maximum number of items to keep after merge
+
+        Returns:
+            Updated obj_instance with merged items
+        """
+        try:
+            # Deserialize the database instance
+            db_instance: QueryMonitorQueue = QueryMonitorQueue.from_json(
+                orm_instance.serialized_data
+            )
+        except Exception as e:
+            logger.error(f"Failed to deserialize database instance: {e}")
+            logger.warning("Skipping merge due to deserialization error, using current object only")
+            return obj_instance
+
+        # Merge items - prioritize existing ones in current object
+        merged_items: list[QueryMonitorItem] = []
+        seen_ids = set()
+
+        # First, add all items from current queue (higher priority)
+        for item in obj_instance.get_queue_content_without_pop():
+            if item.item_id not in seen_ids:
+                merged_items.append(item)
+                seen_ids.add(item.item_id)
+
+        # Then, add items from database queue that aren't in current queue
+        for item in db_instance.get_queue_content_without_pop():
+            if item.item_id not in seen_ids:
+                merged_items.append(item)
+                seen_ids.add(item.item_id)
+
+        # Apply size limit if specified (keep most recent items)
+        if size_limit is not None and size_limit > 0:
+            # Sort by timestamp descending (newest first) and take top N
+            merged_items = sorted(merged_items, key=lambda x: x.timestamp, reverse=True)[
+                :size_limit
+            ]
+
+        # Update the queue with merged items
+        obj_instance.clear()  # Clear existing items
+        for item in merged_items:
+            obj_instance.put(item)  # Add merged items back
+
+        logger.info(
+            f"Merged {len(merged_items)} query items for {obj_instance} (size_limit: {size_limit})"
+        )
+
+        return obj_instance

memos/mem_scheduler/scheduler_factory.py

@@ -3,6 +3,7 @@ from typing import Any, ClassVar
 from memos.configs.mem_scheduler import SchedulerConfigFactory
 from memos.mem_scheduler.base_scheduler import BaseScheduler
 from memos.mem_scheduler.general_scheduler import GeneralScheduler
+from memos.mem_scheduler.optimized_scheduler import OptimizedScheduler
 
 
 class SchedulerFactory(BaseScheduler):
@@ -10,6 +11,7 @@ class SchedulerFactory(BaseScheduler):
 
     backend_to_class: ClassVar[dict[str, Any]] = {
         "general_scheduler": GeneralScheduler,
+        "optimized_scheduler": OptimizedScheduler,
     }
 
     @classmethod

memos/mem_scheduler/schemas/monitor_schemas.py

@@ -1,3 +1,4 @@
+import json
 import threading
 
 from collections import Counter
@@ -30,6 +31,8 @@ class QueryMonitorItem(BaseModel, DictConversionMixin):
     item_id: str = Field(
         description="Unique identifier for the query item", default_factory=lambda: str(uuid4())
     )
+    user_id: str = Field(..., description="Required user identifier", min_length=1)
+    mem_cube_id: str = Field(..., description="Required memory cube identifier", min_length=1)
     query_text: str = Field(
         ...,
         description="The actual user query text content",
@@ -111,7 +114,8 @@ class QueryMonitorQueue(AutoDroppingQueue[QueryMonitorItem]):
         """
         with self.mutex:
             logger.debug(f"Thread {threading.get_ident()} acquired mutex.")
-            all_keywords = [kw for item in self.queue for kw in item.keywords]
+            # Fix: Handle None keywords safely
+            all_keywords = [kw for item in self.queue if item.keywords for kw in item.keywords]
             return Counter(all_keywords)
 
     def get_queries_with_timesort(self, reverse: bool = True) -> list[str]:
@@ -132,9 +136,62 @@ class QueryMonitorQueue(AutoDroppingQueue[QueryMonitorItem]):
                 for monitor in sorted(self.queue, key=lambda x: x.timestamp, reverse=reverse)
             ]
 
+    def to_json(self) -> str:
+        """Serialize the queue to a JSON string.
+
+        Args:
+            item_serializer: Optional function to serialize individual items.
+                             If not provided, items must be JSON-serializable.
+
+        Returns:
+            A JSON string representing the queue's content and maxsize.
+        """
+        with self.mutex:
+            serialized_items = [item.to_json() for item in self.queue]
+
+        data = {"maxsize": self.maxsize, "items": serialized_items}
+        return json.dumps(data, ensure_ascii=False, indent=2)
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "QueryMonitorQueue":
+        """Create a new AutoDroppingQueue from a JSON string.
+
+        Args:
+            json_str: JSON string created by to_json()
+            item_deserializer: Optional function to reconstruct items from dicts.
+                               If not provided, items are used as-is.
+
+        Returns:
+            A new AutoDroppingQueue instance with deserialized data.
+        """
+        data = json.loads(json_str)
+        maxsize = data.get("maxsize", 0)
+        item_strs = data.get("items", [])
+
+        queue = cls(maxsize=maxsize)
+
+        items = [QueryMonitorItem.from_json(json_str=item_str) for item_str in item_strs]
+
+        # Fix: Add error handling for put operations
+        for item in items:
+            try:
+                queue.put(item)  # Use put() to respect maxsize and auto-drop behavior
+            except Exception as e:
+                logger.error(f"Failed to add item to queue: {e}")
+                # Continue with other items instead of failing completely
+
+        return queue
+
 
 # ============== Memories ==============
 class MemoryMonitorItem(BaseModel, DictConversionMixin):
+    """
+    Represents a memory item in the monitoring system.
+
+    Note: This class does NOT have a timestamp field, unlike QueryMonitorItem.
+    For sorting by recency, use sorting_score or importance_score instead.
+    """
+
     item_id: str = Field(
         description="Unique identifier for the memory item", default_factory=lambda: str(uuid4())
     )
@@ -167,7 +224,7 @@ class MemoryMonitorItem(BaseModel, DictConversionMixin):
     recording_count: int = Field(
         default=1,
         description="How many times this memory has been recorded",
-        ge=1,  # Greater than or equal to 1
+        ge=1,
     )
 
     @field_validator("tree_memory_item_mapping_key", mode="before")
@@ -177,27 +234,28 @@ class MemoryMonitorItem(BaseModel, DictConversionMixin):
         return v
 
     def get_importance_score(self, weight_vector: list[float] | None = None) -> float:
-        """
-        Calculate the effective score for the memory item.
+        return self._get_complex_importance_score(weight_vector=weight_vector)
 
-        Returns:
-            float: The importance_score if it has been initialized (>=0),
-                   otherwise the recording_count converted to float.
-
-        Note:
-            This method provides a unified way to retrieve a comparable score
-            for memory items, regardless of whether their importance has been explicitly set.
-        """
+    def _get_complex_importance_score(self, weight_vector: list[float] | None = None) -> float:
+        """Calculate traditional importance score using existing logic"""
         if weight_vector is None:
-            logger.warning("weight_vector of get_importance_score is None.")
+            logger.warning("weight_vector of get_complex_score is None.")
             weight_vector = DEFAULT_WEIGHT_VECTOR_FOR_RANKING
-        assert sum(weight_vector) == 1
-        normalized_keywords_score = min(self.keywords_score * weight_vector[1], 5)
+
+        # Fix: Add proper validation for weight_vector
+        if not weight_vector or len(weight_vector) != 3 or abs(sum(weight_vector) - 1.0) > 1e-6:
+            raise ValueError("weight_vector must be provided, have length 3, and sum to 1.0")
+
+        # Fix: Handle uninitialized scores safely
+        sorting_score = self.sorting_score if self.sorting_score != NOT_INITIALIZED else 0.0
+        keywords_score = self.keywords_score if self.keywords_score != NOT_INITIALIZED else 0.0
+
+        normalized_keywords_score = min(keywords_score * weight_vector[1], 5)
         normalized_recording_count_score = min(self.recording_count * weight_vector[2], 2)
         self.importance_score = (
-            self.sorting_score * weight_vector[0]
-            + normalized_keywords_score
-            + normalized_recording_count_score
+            sorting_score * weight_vector[0]
+            + normalized_keywords_score * weight_vector[1]
+            + normalized_recording_count_score * weight_vector[2]
         )
         return self.importance_score
 
@@ -258,7 +316,7 @@ class MemoryMonitorManager(BaseModel, DictConversionMixin):
 
     def update_memories(
         self, new_memory_monitors: list[MemoryMonitorItem], partial_retention_number: int
-    ) -> MemoryMonitorItem:
+    ) -> list[MemoryMonitorItem]:  # Fix: Correct return type
         """
         Update memories based on monitor_working_memories.
         """
@@ -302,6 +360,13 @@ class MemoryMonitorManager(BaseModel, DictConversionMixin):
             reverse=True,
         )
 
+        # Fix: Add bounds checking to prevent IndexError
+        if partial_retention_number > len(sorted_old_mem_monitors):
+            partial_retention_number = len(sorted_old_mem_monitors)
+            logger.info(
+                f"partial_retention_number adjusted to {partial_retention_number} to match available old memories"
+            )
+
         # Keep the top N old memories
         memories_to_remove = sorted_old_mem_monitors[partial_retention_number:]
         memories_to_change_score = sorted_old_mem_monitors[:partial_retention_number]
@@ -312,19 +377,21 @@ class MemoryMonitorManager(BaseModel, DictConversionMixin):
 
         for memory in memories_to_change_score:
             memory.sorting_score = 0
-            memory.recording_count = 0
+            memory.recording_count = 1
             memory.keywords_score = 0
 
         # Step 4: Enforce max_capacity if set
-        sorted_memories = sorted(
-            self.memories,
-            key=lambda item: item.get_importance_score(
-                weight_vector=DEFAULT_WEIGHT_VECTOR_FOR_RANKING
-            ),
-            reverse=True,
-        )
-        # Keep only the top max_capacity memories
-        self.memories = sorted_memories[: self.max_capacity]
+        # Fix: Handle max_capacity safely
+        if self.max_capacity is not None:
+            sorted_memories = sorted(
+                self.memories,
+                key=lambda item: item.get_importance_score(
+                    weight_vector=DEFAULT_WEIGHT_VECTOR_FOR_RANKING
+                ),
+                reverse=True,
+            )
+            # Keep only the top max_capacity memories
+            self.memories = sorted_memories[: self.max_capacity]
 
         # Log the update result
         logger.info(

memos/mem_scheduler/utils/config_utils.py

@@ -0,0 +1,100 @@
+import json
+import os
+
+from typing import Any
+
+import yaml
+
+
+def flatten_dict(
+    data: dict[str, Any], parent_keys: list[str] | None = None, prefix: str = ""
+) -> dict[str, str]:
+    """
+    Recursively flattens a nested dictionary to generate environment variable keys following the specified format.
+    Combines nested keys with underscores, converts to uppercase, and prepends a custom prefix if provided.
+
+    Args:
+        data: Nested dictionary to be flattened (parsed from JSON/YAML)
+        parent_keys: List to track nested keys during recursion
+        prefix: Custom prefix to be added to all generated keys
+
+    Returns:
+        Flattened dictionary with keys in PREFIX_KEY1_KEY2... format and string values
+    """
+    parent_keys = parent_keys or []
+    flat_data = {}
+
+    for key, value in data.items():
+        # Clean and standardize key: convert to uppercase, replace spaces/hyphens with underscores
+        clean_key = key.upper().replace(" ", "_").replace("-", "_")
+        current_keys = [*parent_keys, clean_key]
+
+        if isinstance(value, dict):
+            # Recursively process nested dictionaries
+            nested_flat = flatten_dict(value, current_keys, prefix)
+            flat_data.update(nested_flat)
+        else:
+            # Construct full key name with prefix (if provided) and nested keys
+            if prefix:
+                full_key = f"{prefix.upper()}_{'_'.join(current_keys)}"
+            else:
+                full_key = "_".join(current_keys)
+
+            # Process value: ensure string type, convert None to empty string
+            flat_value = "" if value is None else str(value).strip()
+
+            flat_data[full_key] = flat_value
+
+    return flat_data
+
+
+def convert_config_to_env(input_file: str, output_file: str = ".env", prefix: str = "") -> None:
+    """
+    Converts a JSON or YAML configuration file to a .env file with standardized environment variables.
+    Uses the flatten_dict function to generate keys in PREFIX_KEY1_KEY2... format.
+
+    Args:
+        input_file: Path to input configuration file (.json, .yaml, or .yml)
+        output_file: Path to output .env file (default: .env)
+        prefix: Custom prefix for all environment variable keys
+
+    Raises:
+        FileNotFoundError: If input file does not exist
+        ValueError: If file format is unsupported or parsing fails
+    """
+    # Check if input file exists
+    if not os.path.exists(input_file):
+        raise FileNotFoundError(f"Input file not found: {input_file}")
+
+    # Parse input file based on extension
+    file_ext = os.path.splitext(input_file)[1].lower()
+    config_data: dict[str, Any] = {}
+
+    try:
+        with open(input_file, encoding="utf-8") as f:
+            if file_ext in (".json",):
+                config_data = json.load(f)
+            elif file_ext in (".yaml", ".yml"):
+                config_data = yaml.safe_load(f)
+            else:
+                raise ValueError(
+                    f"Unsupported file format: {file_ext}. Supported formats: .json, .yaml, .yml"
+                )
+    except (json.JSONDecodeError, yaml.YAMLError) as e:
+        raise ValueError(f"Error parsing file: {e!s}") from e
+
+    # Flatten configuration and generate environment variable key-value pairs
+    flat_config = flatten_dict(config_data, prefix=prefix)
+
+    # Write to .env file
+    with open(output_file, "w", encoding="utf-8") as f:
+        for key, value in flat_config.items():
+            # Handle values containing double quotes (use no surrounding quotes)
+            if '"' in value:
+                f.write(f"{key}={value}\n")
+            else:
+                f.write(f'{key}="{value}"\n')  # Enclose regular values in double quotes
+
+    print(
+        f"Conversion complete! Generated {output_file} with {len(flat_config)} environment variables"
+    )

memos/mem_scheduler/utils/db_utils.py

@@ -0,0 +1,33 @@
+import os
+import sqlite3
+
+
+def print_db_tables(db_path: str):
+    """Print all table names and structures in the SQLite database"""
+    print(f"\n🔍 Checking database file: {db_path}")
+
+    if not os.path.exists(db_path):
+        print(f"❌ File does not exist! Path: {db_path}")
+        return
+
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # List all tables
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = cursor.fetchall()
+    if not tables:
+        print("❌ Database is empty, no tables created")
+    else:
+        print(f"✅ Database contains {len(tables)} table(s):")
+        for (table_name,) in tables:
+            print(f"  📂 Table name: {table_name}")
+
+            # Print table structure
+            cursor.execute(f"PRAGMA table_info({table_name});")
+            columns = cursor.fetchall()
+            print("    🧩 Structure:")
+            for col in columns:
+                print(f"      {col[1]} ({col[2]}) {'(PK)' if col[5] else ''}")
+
+    conn.close()

memos/mem_scheduler/utils/filter_utils.py

@@ -60,7 +60,7 @@ def is_all_chinese(input_string: str) -> bool:
     install_command="pip install scikit-learn",
     install_link="https://scikit-learn.org/stable/install.html",
 )
-def filter_similar_memories(
+def filter_vector_based_similar_memories(
     text_memories: list[str], similarity_threshold: float = 0.75
 ) -> list[str]:
     """

memos/memories/activation/kv.py

@@ -1,9 +1,10 @@
 import os
 import pickle
+
 from datetime import datetime
 from importlib.metadata import version
-from packaging.version import Version
 
+from packaging.version import Version
 from transformers import DynamicCache
 
 from memos.configs.memory import KVCacheMemoryConfig