MemoryOS 0.2.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

memos/mem_scheduler/utils/filter_utils.py

@@ -0,0 +1,176 @@
+import re
+
+from memos.dependency import require_python_package
+from memos.log import get_logger
+
+
+logger = get_logger(__name__)
+
+
+def transform_name_to_key(name):
+    """
+    Normalize text by removing all punctuation marks, keeping only letters, numbers, and word characters.
+
+    Args:
+        name (str): Input text to be processed
+
+    Returns:
+        str: Processed text with all punctuation removed
+    """
+    # Match all characters that are NOT:
+    # \w - word characters (letters, digits, underscore)
+    # \u4e00-\u9fff - Chinese/Japanese/Korean characters
+    # \s - whitespace
+    pattern = r"[^\w\u4e00-\u9fff\s]"
+
+    # Substitute all matched punctuation marks with empty string
+    # re.UNICODE flag ensures proper handling of Unicode characters
+    normalized = re.sub(pattern, "", name, flags=re.UNICODE)
+
+    # Optional: Collapse multiple whitespaces into single space
+    normalized = "_".join(normalized.split())
+
+    normalized = normalized.lower()
+
+    return normalized
+
+
+def is_all_english(input_string: str) -> bool:
+    """Determine if the string consists entirely of English characters (including spaces)"""
+    return all(char.isascii() or char.isspace() for char in input_string)
+
+
+def is_all_chinese(input_string: str) -> bool:
+    """Determine if the string consists entirely of Chinese characters (including Chinese punctuation and spaces)"""
+    return all(
+        ("\u4e00" <= char <= "\u9fff")  # Basic Chinese characters
+        or ("\u3400" <= char <= "\u4dbf")  # Extension A
+        or ("\u20000" <= char <= "\u2a6df")  # Extension B
+        or ("\u2a700" <= char <= "\u2b73f")  # Extension C
+        or ("\u2b740" <= char <= "\u2b81f")  # Extension D
+        or ("\u2b820" <= char <= "\u2ceaf")  # Extension E
+        or ("\u2f800" <= char <= "\u2fa1f")  # Extension F
+        or char.isspace()  # Spaces
+        for char in input_string
+    )
+
+
+@require_python_package(
+    import_name="sklearn",
+    install_command="pip install scikit-learn",
+    install_link="https://scikit-learn.org/stable/install.html",
+)
+def filter_similar_memories(
+    text_memories: list[str], similarity_threshold: float = 0.75
+) -> list[str]:
+    """
+    Filters out low-quality or duplicate memories based on text similarity.
+
+    Args:
+        text_memories: List of text memories to filter
+        similarity_threshold: Threshold for considering memories duplicates (0.0-1.0)
+                            Higher values mean stricter filtering
+
+    Returns:
+        List of filtered memories with duplicates removed
+    """
+    from sklearn.feature_extraction.text import TfidfVectorizer
+    from sklearn.metrics.pairwise import cosine_similarity
+
+    if not text_memories:
+        logger.warning("Received empty memories list - nothing to filter")
+        return []
+
+    for idx in range(len(text_memories)):
+        if not isinstance(text_memories[idx], str):
+            logger.error(
+                f"{text_memories[idx]} in memories is not a string,"
+                f" and now has been transformed to be a string."
+            )
+            text_memories[idx] = str(text_memories[idx])
+
+    try:
+        # Step 1: Vectorize texts using TF-IDF
+        vectorizer = TfidfVectorizer()
+        tfidf_matrix = vectorizer.fit_transform(text_memories)
+
+        # Step 2: Calculate pairwise similarity matrix
+        similarity_matrix = cosine_similarity(tfidf_matrix)
+
+        # Step 3: Identify duplicates
+        to_keep = set(range(len(text_memories)))  # Start with all indices
+        for i in range(len(similarity_matrix)):
+            if i not in to_keep:
+                continue  # Already marked for removal
+
+            # Find all similar items to this one (excluding self and already removed)
+            similar_indices = [
+                j
+                for j in range(i + 1, len(similarity_matrix))
+                if similarity_matrix[i][j] >= similarity_threshold and j in to_keep
+            ]
+            similar_indices = set(similar_indices)
+
+            # Remove all similar items (keeping the first one - i)
+            to_keep -= similar_indices
+
+        # Return filtered memories
+        filtered_memories = [text_memories[i] for i in sorted(to_keep)]
+        logger.debug(f"filtered_memories: {filtered_memories}")
+        return filtered_memories
+
+    except Exception as e:
+        logger.error(f"Error filtering memories: {e!s}")
+        return text_memories  # Return original list if error occurs
+
+
+def filter_too_short_memories(
+    text_memories: list[str], min_length_threshold: int = 20
+) -> list[str]:
+    """
+    Filters out text memories that fall below the minimum length requirement.
+    Handles both English (word count) and Chinese (character count) differently.
+
+    Args:
+        text_memories: List of text memories to be filtered
+        min_length_threshold: Minimum length required to keep a memory.
+                            For English: word count, for Chinese: character count.
+
+    Returns:
+        List of filtered memories meeting the length requirement
+    """
+    if not text_memories:
+        logger.debug("Empty memories list received in short memory filter")
+        return []
+
+    filtered_memories = []
+    removed_count = 0
+
+    for memory in text_memories:
+        stripped_memory = memory.strip()
+        if not stripped_memory:  # Skip empty/whitespace memories
+            removed_count += 1
+            continue
+
+        # Determine measurement method based on language
+        if is_all_english(stripped_memory):
+            length = len(stripped_memory.split())  # Word count for English
+        elif is_all_chinese(stripped_memory):
+            length = len(stripped_memory)  # Character count for Chinese
+        else:
+            logger.debug(f"Mixed-language memory, using character count: {stripped_memory[:50]}...")
+            length = len(stripped_memory)  # Default to character count
+
+        if length >= min_length_threshold:
+            filtered_memories.append(memory)
+        else:
+            removed_count += 1
+
+    if removed_count > 0:
+        logger.info(
+            f"Filtered out {removed_count} short memories "
+            f"(below {min_length_threshold} units). "
+            f"Total remaining: {len(filtered_memories)}"
+        )
+
+    return filtered_memories

memos/mem_scheduler/utils/misc_utils.py

@@ -0,0 +1,102 @@
+import json
+import re
+
+from functools import wraps
+from pathlib import Path
+
+import yaml
+
+from memos.log import get_logger
+
+
+logger = get_logger(__name__)
+
+
+def extract_json_dict(text: str):
+    """
+    Safely extracts JSON from LLM response text with robust error handling.
+
+    Args:
+        text: Raw text response from LLM that may contain JSON
+
+    Returns:
+        Parsed JSON data (dict or list)
+
+    Raises:
+        ValueError: If no valid JSON can be extracted
+    """
+    if not text:
+        raise ValueError("Empty input text")
+
+    # Normalize the text
+    text = text.strip()
+
+    # Remove common code block markers
+    patterns_to_remove = ["json```", "```python", "```json", "latex```", "```latex", "```"]
+    for pattern in patterns_to_remove:
+        text = text.replace(pattern, "")
+
+    # Try: direct JSON parse first
+    try:
+        return json.loads(text.strip())
+    except json.JSONDecodeError as e:
+        logger.error(f"Failed to parse JSON from text: {text}. Error: {e!s}", exc_info=True)
+
+    # Fallback 1: Extract JSON using regex
+    json_pattern = r"\{[\s\S]*\}|\[[\s\S]*\]"
+    matches = re.findall(json_pattern, text)
+    if matches:
+        try:
+            return json.loads(matches[0])
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to parse JSON from text: {text}. Error: {e!s}", exc_info=True)
+
+    # Fallback 2: Handle malformed JSON (common LLM issues)
+    try:
+        # Try adding missing quotes around keys
+        text = re.sub(r"([\{\s,])(\w+)(:)", r'\1"\2"\3', text)
+        return json.loads(text)
+    except json.JSONDecodeError as e:
+        logger.error(f"Failed to parse JSON from text: {text}. Error: {e!s}", exc_info=True)
+        raise ValueError(text) from e
+
+
+def parse_yaml(yaml_file: str | Path):
+    yaml_path = Path(yaml_file)
+    if not yaml_path.is_file():
+        raise FileNotFoundError(f"No such file: {yaml_file}")
+
+    with yaml_path.open("r", encoding="utf-8") as fr:
+        data = yaml.safe_load(fr)
+
+    return data
+
+
+def log_exceptions(logger=logger):
+    """
+    Exception-catching decorator that automatically logs errors (including stack traces)
+
+    Args:
+        logger: Optional logger object (default: module-level logger)
+
+    Example:
+        @log_exceptions()
+        def risky_function():
+            raise ValueError("Oops!")
+
+        @log_exceptions(logger=custom_logger)
+        def another_risky_function():
+            might_fail()
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                logger.error(f"Error in {func.__name__}: {e}", exc_info=True)
+
+        return wrapper
+
+    return decorator

memos/mem_user/factory.py

@@ -0,0 +1,94 @@
+from typing import Any, ClassVar
+
+from memos.configs.mem_user import UserManagerConfigFactory
+from memos.mem_user.mysql_user_manager import MySQLUserManager
+from memos.mem_user.user_manager import UserManager
+
+
+class UserManagerFactory:
+    """Factory class for creating user manager instances."""
+
+    backend_to_class: ClassVar[dict[str, Any]] = {
+        "sqlite": UserManager,
+        "mysql": MySQLUserManager,
+    }
+
+    @classmethod
+    def from_config(
+        cls, config_factory: UserManagerConfigFactory
+    ) -> UserManager | MySQLUserManager:
+        """Create a user manager instance from configuration.
+
+        Args:
+            config_factory: Configuration factory containing backend and config
+
+        Returns:
+            User manager instance
+
+        Raises:
+            ValueError: If backend is not supported
+        """
+        backend = config_factory.backend
+        if backend not in cls.backend_to_class:
+            raise ValueError(f"Invalid user manager backend: {backend}")
+
+        user_manager_class = cls.backend_to_class[backend]
+        config = config_factory.config
+
+        # Use model_dump() to convert Pydantic model to dict and unpack as kwargs
+        return user_manager_class(**config.model_dump())
+
+    @classmethod
+    def create_sqlite(cls, db_path: str | None = None, user_id: str = "root") -> UserManager:
+        """Create SQLite user manager with default configuration.
+
+        Args:
+            db_path: Path to SQLite database file
+            user_id: Default user ID for initialization
+
+        Returns:
+            SQLite user manager instance
+        """
+        config_factory = UserManagerConfigFactory(
+            backend="sqlite", config={"db_path": db_path, "user_id": user_id}
+        )
+        return cls.from_config(config_factory)
+
+    @classmethod
+    def create_mysql(
+        cls,
+        user_id: str = "root",
+        host: str = "localhost",
+        port: int = 3306,
+        username: str = "root",
+        password: str = "",
+        database: str = "memos_users",
+        charset: str = "utf8mb4",
+    ) -> MySQLUserManager:
+        """Create MySQL user manager with specified configuration.
+
+        Args:
+            user_id: Default user ID for initialization
+            host: MySQL server host
+            port: MySQL server port
+            username: MySQL username
+            password: MySQL password
+            database: MySQL database name
+            charset: MySQL charset
+
+        Returns:
+            MySQL user manager instance
+        """
+        config_factory = UserManagerConfigFactory(
+            backend="mysql",
+            config={
+                "user_id": user_id,
+                "host": host,
+                "port": port,
+                "username": username,
+                "password": password,
+                "database": database,
+                "charset": charset,
+            },
+        )
+        return cls.from_config(config_factory)

memos/mem_user/mysql_persistent_user_manager.py

@@ -0,0 +1,271 @@
+"""Persistent user management system for MemOS with configuration storage.
+
+This module extends the MySQL UserManager to provide persistent storage
+for user configurations and MOS instances.
+"""
+
+import json
+
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import Column, String, Text
+
+from memos.configs.mem_os import MOSConfig
+from memos.log import get_logger
+from memos.mem_user.mysql_user_manager import Base, MySQLUserManager
+
+
+logger = get_logger(__name__)
+
+
+class UserConfig(Base):
+    """User configuration model for the database."""
+
+    __tablename__ = "user_configs"
+
+    user_id = Column(String(255), primary_key=True)
+    config_data = Column(Text, nullable=False)  # JSON string of MOSConfig
+    created_at = Column(String(50), nullable=False)  # ISO format timestamp
+    updated_at = Column(String(50), nullable=False)  # ISO format timestamp
+
+    def __repr__(self):
+        return f"<UserConfig(user_id='{self.user_id}')>"
+
+
+class MySQLPersistentUserManager(MySQLUserManager):
+    """Extended MySQLUserManager with configuration persistence."""
+
+    def __init__(
+        self,
+        user_id: str = "root",
+        host: str = "localhost",
+        port: int = 3306,
+        username: str = "root",
+        password: str = "",
+        database: str = "memos_users",
+        charset: str = "utf8mb4",
+    ):
+        """Initialize the persistent user manager.
+
+        Args:
+            user_id (str, optional): User ID. If None, uses default user ID.
+            host (str): MySQL server host. Defaults to "localhost".
+            port (int): MySQL server port. Defaults to 3306.
+            username (str): MySQL username. Defaults to "root".
+            password (str): MySQL password. Defaults to "".
+            database (str): MySQL database name. Defaults to "memos_users".
+            charset (str): MySQL charset. Defaults to "utf8mb4".
+        """
+        super().__init__(user_id, host, port, username, password, database, charset)
+
+        # Create user_configs table
+        Base.metadata.create_all(bind=self.engine)
+        logger.info("MySQLPersistentUserManager initialized with configuration storage")
+
+    def _convert_datetime_strings(self, obj: Any) -> Any:
+        """Recursively convert datetime strings back to datetime objects in config dict.
+
+        Args:
+            obj: The object to process (dict, list, or primitive type)
+
+        Returns:
+            The object with datetime strings converted to datetime objects
+        """
+        if isinstance(obj, dict):
+            result = {}
+            for key, value in obj.items():
+                if key == "created_at" and isinstance(value, str):
+                    try:
+                        result[key] = datetime.fromisoformat(value)
+                    except ValueError:
+                        # If parsing fails, keep the original string
+                        result[key] = value
+                else:
+                    result[key] = self._convert_datetime_strings(value)
+            return result
+        elif isinstance(obj, list):
+            return [self._convert_datetime_strings(item) for item in obj]
+        else:
+            return obj
+
+    def save_user_config(self, user_id: str, config: MOSConfig) -> bool:
+        """Save user configuration to database.
+
+        Args:
+            user_id (str): The user ID.
+            config (MOSConfig): The user's MOS configuration.
+
+        Returns:
+            bool: True if successful, False otherwise.
+        """
+        session = self._get_session()
+        try:
+            # Convert config to JSON string with proper datetime handling
+            config_dict = config.model_dump(mode="json")
+            config_json = json.dumps(config_dict, indent=2)
+
+            now = datetime.now().isoformat()
+
+            # Check if config already exists
+            existing_config = (
+                session.query(UserConfig).filter(UserConfig.user_id == user_id).first()
+            )
+
+            if existing_config:
+                # Update existing config
+                existing_config.config_data = config_json
+                existing_config.updated_at = now
+                logger.info(f"Updated configuration for user {user_id}")
+            else:
+                # Create new config
+                user_config = UserConfig(
+                    user_id=user_id, config_data=config_json, created_at=now, updated_at=now
+                )
+                session.add(user_config)
+                logger.info(f"Saved new configuration for user {user_id}")
+
+            session.commit()
+            return True
+
+        except Exception as e:
+            session.rollback()
+            logger.error(f"Error saving user config for {user_id}: {e}")
+            return False
+        finally:
+            session.close()
+
+    def get_user_config(self, user_id: str) -> MOSConfig | None:
+        """Get user configuration from database.
+
+        Args:
+            user_id (str): The user ID.
+
+        Returns:
+            MOSConfig | None: The user's configuration or None if not found.
+        """
+        session = self._get_session()
+        try:
+            user_config = session.query(UserConfig).filter(UserConfig.user_id == user_id).first()
+
+            if user_config:
+                config_dict = json.loads(user_config.config_data)
+                # Convert datetime strings back to datetime objects
+                config_dict = self._convert_datetime_strings(config_dict)
+                return MOSConfig(**config_dict)
+            return None
+
+        except Exception as e:
+            logger.error(f"Error loading user config for {user_id}: {e}")
+            return None
+        finally:
+            session.close()
+
+    def delete_user_config(self, user_id: str) -> bool:
+        """Delete user configuration from database.
+
+        Args:
+            user_id (str): The user ID.
+
+        Returns:
+            bool: True if successful, False otherwise.
+        """
+        session = self._get_session()
+        try:
+            user_config = session.query(UserConfig).filter(UserConfig.user_id == user_id).first()
+
+            if user_config:
+                session.delete(user_config)
+                session.commit()
+                logger.info(f"Deleted configuration for user {user_id}")
+                return True
+            return False
+
+        except Exception as e:
+            session.rollback()
+            logger.error(f"Error deleting user config for {user_id}: {e}")
+            return False
+        finally:
+            session.close()
+
+    def list_user_configs(self) -> dict[str, MOSConfig]:
+        """List all user configurations.
+
+        Returns:
+            Dict[str, MOSConfig]: Dictionary mapping user_id to MOSConfig.
+        """
+        session = self._get_session()
+        try:
+            user_configs = session.query(UserConfig).all()
+            result = {}
+
+            for user_config in user_configs:
+                try:
+                    config_dict = json.loads(user_config.config_data)
+                    # Convert datetime strings back to datetime objects
+                    config_dict = self._convert_datetime_strings(config_dict)
+                    result[user_config.user_id] = MOSConfig(**config_dict)
+                except Exception as e:
+                    logger.error(f"Error parsing config for user {user_config.user_id}: {e}")
+                    continue
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Error listing user configs: {e}")
+            return {}
+        finally:
+            session.close()
+
+    def create_user_with_config(
+        self, user_name: str, config: MOSConfig, role=None, user_id: str | None = None
+    ) -> str:
+        """Create a new user with configuration.
+
+        Args:
+            user_name (str): Name of the user.
+            config (MOSConfig): The user's configuration.
+            role: User role (optional, uses default from UserManager).
+            user_id (str, optional): Custom user ID.
+
+        Returns:
+            str: The created user ID.
+
+        Raises:
+            ValueError: If user_name already exists.
+        """
+        # Create user using parent method
+        created_user_id = self.create_user(user_name, role, user_id)
+
+        # Save configuration
+        if not self.save_user_config(created_user_id, config):
+            logger.error(f"Failed to save configuration for user {created_user_id}")
+
+        return created_user_id
+
+    def delete_user(self, user_id: str) -> bool:
+        """Delete a user and their configuration.
+
+        Args:
+            user_id (str): The user ID.
+
+        Returns:
+            bool: True if successful, False otherwise.
+        """
+        # Delete configuration first
+        self.delete_user_config(user_id)
+
+        # Delete user using parent method
+        return super().delete_user(user_id)
+
+    def get_user_cube_access(self, user_id: str) -> list[str]:
+        """Get list of cube IDs that a user has access to.
+
+        Args:
+            user_id (str): The user ID.
+
+        Returns:
+            list[str]: List of cube IDs the user can access.
+        """
+        cubes = self.get_user_cubes(user_id)
+        return [cube.cube_id for cube in cubes]