MemoryOS 1.0.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

memos/mem_scheduler/memory_manage_modules/memory_filter.py

@@ -0,0 +1,308 @@
+from memos.configs.mem_scheduler import BaseSchedulerConfig
+from memos.llms.base import BaseLLM
+from memos.log import get_logger
+from memos.mem_scheduler.general_modules.base import BaseSchedulerModule
+from memos.mem_scheduler.utils.misc_utils import extract_json_dict
+from memos.memories.textual.tree import TextualMemoryItem
+
+
+logger = get_logger(__name__)
+
+
+class MemoryFilter(BaseSchedulerModule):
+    def __init__(self, process_llm: BaseLLM, config: BaseSchedulerConfig):
+        super().__init__()
+        self.config: BaseSchedulerConfig = config
+        self.process_llm = process_llm
+
+    def filter_unrelated_memories(
+        self,
+        query_history: list[str],
+        memories: list[TextualMemoryItem],
+    ) -> (list[TextualMemoryItem], bool):
+        """
+        Filter out memories that are completely unrelated to the query history using LLM.
+
+        Args:
+            query_history: List of query strings to determine relevance
+            memories: List of TextualMemoryItem objects to be filtered
+
+        Returns:
+            Tuple of (filtered_memories, success_flag)
+            - filtered_memories: List of TextualMemoryItem objects that are relevant to queries
+            - success_flag: Boolean indicating if LLM filtering was successful
+
+        Note:
+            If LLM filtering fails, returns all memories (conservative approach)
+        """
+        success_flag = False
+
+        if not memories:
+            logger.info("No memories to filter - returning empty list")
+            return [], True
+
+        if not query_history:
+            logger.info("No query history provided - keeping all memories")
+            return memories, True
+
+        logger.info(
+            f"Starting memory filtering for {len(memories)} memories against {len(query_history)} queries"
+        )
+
+        # Extract memory texts for LLM processing
+        memory_texts = [mem.memory for mem in memories]
+
+        # Build LLM prompt for memory filtering
+        prompt = self.build_prompt(
+            "memory_filtering",
+            query_history=[f"[{i}] {query}" for i, query in enumerate(query_history)],
+            memories=[f"[{i}] {mem}" for i, mem in enumerate(memory_texts)],
+        )
+        logger.debug(f"Generated filtering prompt: {prompt[:200]}...")  # Log first 200 chars
+
+        # Get LLM response
+        response = self.process_llm.generate([{"role": "user", "content": prompt}])
+        logger.debug(f"Received LLM filtering response: {response[:200]}...")  # Log first 200 chars
+
+        try:
+            # Parse JSON response
+            response = extract_json_dict(response)
+            logger.debug(f"Parsed JSON response: {response}")
+            relevant_indices = response["relevant_memories"]
+            filtered_count = response["filtered_count"]
+            reasoning = response["reasoning"]
+
+            # Validate indices
+            if not isinstance(relevant_indices, list):
+                raise ValueError("relevant_memories must be a list")
+
+            # Filter memories based on relevant indices
+            filtered_memories = []
+            for idx in relevant_indices:
+                if isinstance(idx, int) and 0 <= idx < len(memories):
+                    filtered_memories.append(memories[idx])
+                else:
+                    logger.warning(f"Invalid memory index {idx} - skipping")
+
+            logger.info(
+                f"Successfully filtered memories. Kept {len(filtered_memories)} out of {len(memories)} memories. "
+                f"Filtered out {filtered_count} unrelated memories. "
+                f"Filtering reasoning: {reasoning}"
+            )
+            success_flag = True
+
+        except Exception as e:
+            logger.error(
+                f"Failed to filter memories with LLM. Exception: {e}. Raw response: {response}",
+                exc_info=True,
+            )
+            # Conservative approach: keep all memories if filtering fails
+            filtered_memories = memories
+            success_flag = False
+
+        return filtered_memories, success_flag
+
+    def filter_redundant_memories(
+        self,
+        query_history: list[str],
+        memories: list[TextualMemoryItem],
+    ) -> (list[TextualMemoryItem], bool):
+        """
+        Filter out redundant memories using LLM analysis.
+
+        This function removes redundant memories by keeping the most informative
+        version when multiple memories contain similar information relevant to queries.
+
+        Args:
+            query_history: List of query strings to determine relevance and value
+            memories: List of TextualMemoryItem objects to be filtered
+
+        Returns:
+            Tuple of (filtered_memories, success_flag)
+            - filtered_memories: List of TextualMemoryItem objects after redundancy filtering
+            - success_flag: Boolean indicating if LLM filtering was successful
+
+        Note:
+            If LLM filtering fails, returns all memories (conservative approach)
+        """
+        success_flag = False
+
+        if not memories:
+            logger.info("No memories to filter for redundancy - returning empty list")
+            return [], True
+
+        if not query_history:
+            logger.info("No query history provided - keeping all memories")
+            return memories, True
+
+        if len(memories) <= 1:
+            logger.info("Only one memory - no redundancy to filter")
+            return memories, True
+
+        logger.info(
+            f"Starting redundancy filtering for {len(memories)} memories against {len(query_history)} queries"
+        )
+
+        # Extract memory texts for LLM processing
+        memory_texts = [mem.memory for mem in memories]
+
+        # Build LLM prompt for redundancy filtering
+        prompt = self.build_prompt(
+            "memory_redundancy_filtering",
+            query_history=[f"[{i}] {query}" for i, query in enumerate(query_history)],
+            memories=[f"[{i}] {mem}" for i, mem in enumerate(memory_texts)],
+        )
+        logger.debug(
+            f"Generated redundancy filtering prompt: {prompt[:200]}..."
+        )  # Log first 200 chars
+
+        # Get LLM response
+        response = self.process_llm.generate([{"role": "user", "content": prompt}])
+        logger.debug(
+            f"Received LLM redundancy filtering response: {response[:200]}..."
+        )  # Log first 200 chars
+
+        try:
+            # Parse JSON response
+            response = extract_json_dict(response)
+            logger.debug(f"Parsed JSON response: {response}")
+            kept_indices = response["kept_memories"]
+            redundant_groups = response.get("redundant_groups", [])
+            reasoning = response["reasoning"]
+
+            # Validate indices
+            if not isinstance(kept_indices, list):
+                raise ValueError("kept_memories must be a list")
+
+            # Filter memories based on kept indices
+            filtered_memories = []
+            for idx in kept_indices:
+                if isinstance(idx, int) and 0 <= idx < len(memories):
+                    filtered_memories.append(memories[idx])
+                else:
+                    logger.warning(f"Invalid memory index {idx} - skipping")
+
+            logger.info(
+                f"Successfully filtered redundant memories. "
+                f"Kept {len(filtered_memories)} out of {len(memories)} memories. "
+                f"Removed {len(memories) - len(filtered_memories)} redundant memories. "
+                f"Redundant groups identified: {len(redundant_groups)}. "
+                f"Filtering reasoning: {reasoning}"
+            )
+            success_flag = True
+
+        except Exception as e:
+            logger.error(
+                f"Failed to filter redundant memories with LLM. Exception: {e}. Raw response: {response}",
+                exc_info=True,
+            )
+            # Conservative approach: keep all memories if filtering fails
+            filtered_memories = memories
+            success_flag = False
+
+        return filtered_memories, success_flag
+
+    def filter_unrelated_and_redundant_memories(
+        self,
+        query_history: list[str],
+        memories: list[TextualMemoryItem],
+    ) -> (list[TextualMemoryItem], bool):
+        """
+        Filter out both unrelated and redundant memories using LLM analysis.
+
+        This function performs two types of filtering in sequence:
+        1. Remove memories that are completely unrelated to the query history
+        2. Remove redundant memories by keeping the most informative version
+
+        Args:
+            query_history: List of query strings to determine relevance and value
+            memories: List of TextualMemoryItem objects to be filtered
+
+        Returns:
+            Tuple of (filtered_memories, success_flag)
+            - filtered_memories: List of TextualMemoryItem objects after both filtering steps
+            - success_flag: Boolean indicating if LLM filtering was successful
+
+        Note:
+            If LLM filtering fails, returns all memories (conservative approach)
+        """
+        success_flag = False
+
+        if not memories:
+            logger.info("No memories to filter for unrelated and redundant - returning empty list")
+            return [], True
+
+        if not query_history:
+            logger.info("No query history provided - keeping all memories")
+            return memories, True
+
+        if len(memories) <= 1:
+            logger.info("Only one memory - no filtering needed")
+            return memories, True
+
+        logger.info(
+            f"Starting combined unrelated and redundant filtering for {len(memories)} memories against {len(query_history)} queries"
+        )
+
+        # Extract memory texts for LLM processing
+        memory_texts = [mem.memory for mem in memories]
+
+        # Build LLM prompt for combined filtering
+        prompt = self.build_prompt(
+            "memory_combined_filtering",
+            query_history=[f"[{i}] {query}" for i, query in enumerate(query_history)],
+            memories=[f"[{i}] {mem}" for i, mem in enumerate(memory_texts)],
+        )
+        logger.debug(
+            f"Generated combined filtering prompt: {prompt[:200]}..."
+        )  # Log first 200 chars
+
+        # Get LLM response
+        response = self.process_llm.generate([{"role": "user", "content": prompt}])
+        logger.debug(
+            f"Received LLM combined filtering response: {response[:200]}..."
+        )  # Log first 200 chars
+
+        try:
+            # Parse JSON response
+            response = extract_json_dict(response)
+            logger.debug(f"Parsed JSON response: {response}")
+            kept_indices = response["kept_memories"]
+            unrelated_removed_count = response.get("unrelated_removed_count", 0)
+            redundant_removed_count = response.get("redundant_removed_count", 0)
+            redundant_groups = response.get("redundant_groups", [])
+            reasoning = response["reasoning"]
+
+            # Validate indices
+            if not isinstance(kept_indices, list):
+                raise ValueError("kept_memories must be a list")
+
+            # Filter memories based on kept indices
+            filtered_memories = []
+            for idx in kept_indices:
+                if isinstance(idx, int) and 0 <= idx < len(memories):
+                    filtered_memories.append(memories[idx])
+                else:
+                    logger.warning(f"Invalid memory index {idx} - skipping")
+
+            logger.info(
+                f"Successfully filtered unrelated and redundant memories. "
+                f"Kept {len(filtered_memories)} out of {len(memories)} memories. "
+                f"Removed {len(memories) - len(filtered_memories)} memories total. "
+                f"Unrelated removed: {unrelated_removed_count}. "
+                f"Redundant removed: {redundant_removed_count}. "
+                f"Redundant groups identified: {len(redundant_groups)}. "
+                f"Filtering reasoning: {reasoning}"
+            )
+            success_flag = True
+
+        except Exception as e:
+            logger.error(
+                f"Failed to filter unrelated and redundant memories with LLM. Exception: {e}. Raw response: {response}",
+                exc_info=True,
+            )
+            # Conservative approach: keep all memories if filtering fails
+            filtered_memories = memories
+            success_flag = False
+
+        return filtered_memories, success_flag

memos/mem_scheduler/{general_modules → memory_manage_modules}/retriever.py

@@ -8,8 +8,8 @@ from memos.mem_scheduler.schemas.general_schemas import (
     TreeTextMemory_SEARCH_METHOD,
 )
 from memos.mem_scheduler.utils.filter_utils import (
-    filter_similar_memories,
     filter_too_short_memories,
+    filter_vector_based_similar_memories,
     transform_name_to_key,
 )
 from memos.mem_scheduler.utils.misc_utils import (
@@ -17,6 +17,8 @@ from memos.mem_scheduler.utils.misc_utils import (
 )
 from memos.memories.textual.tree import TextualMemoryItem, TreeTextMemory
 
+from .memory_filter import MemoryFilter
+
 
 logger = get_logger(__name__)
 
@@ -32,6 +34,9 @@ class SchedulerRetriever(BaseSchedulerModule):
         self.config: BaseSchedulerConfig = config
         self.process_llm = process_llm
 
+        # Initialize memory filter
+        self.memory_filter = MemoryFilter(process_llm=process_llm, config=config)
+
     def search(
         self,
         query: str,
@@ -77,10 +82,7 @@ class SchedulerRetriever(BaseSchedulerModule):
         return results
 
     def rerank_memories(
-        self,
-        queries: list[str],
-        original_memories: list[str],
-        top_k: int,
+        self, queries: list[str], original_memories: list[str], top_k: int
     ) -> (list[str], bool):
         """
         Rerank memories based on relevance to given queries using LLM.
@@ -96,7 +98,6 @@ class SchedulerRetriever(BaseSchedulerModule):
         Note:
             If LLM reranking fails, falls back to original order (truncated to top_k)
         """
-        success_flag = False
 
         logger.info(f"Starting memory reranking for {len(original_memories)} memories")
 
@@ -163,7 +164,7 @@ class SchedulerRetriever(BaseSchedulerModule):
         combined_text_memory = [m.memory for m in combined_memory]
 
         # Apply similarity filter to remove overly similar memories
-        filtered_combined_text_memory = filter_similar_memories(
+        filtered_combined_text_memory = filter_vector_based_similar_memories(
             text_memories=combined_text_memory,
             similarity_threshold=self.filter_similarity_threshold,
         )
@@ -197,3 +198,29 @@ class SchedulerRetriever(BaseSchedulerModule):
                 )
 
         return memories_with_new_order, success_flag
+
+    def filter_unrelated_memories(
+        self,
+        query_history: list[str],
+        memories: list[TextualMemoryItem],
+    ) -> (list[TextualMemoryItem], bool):
+        return self.memory_filter.filter_unrelated_memories(query_history, memories)
+
+    def filter_redundant_memories(
+        self,
+        query_history: list[str],
+        memories: list[TextualMemoryItem],
+    ) -> (list[TextualMemoryItem], bool):
+        return self.memory_filter.filter_redundant_memories(query_history, memories)
+
+    def filter_unrelated_and_redundant_memories(
+        self,
+        query_history: list[str],
+        memories: list[TextualMemoryItem],
+    ) -> (list[TextualMemoryItem], bool):
+        """
+        Filter out both unrelated and redundant memories using LLM analysis.
+
+        This method delegates to the MemoryFilter class.
+        """
+        return self.memory_filter.filter_unrelated_and_redundant_memories(query_history, memories)

memos/mem_scheduler/monitors/dispatcher_monitor.py

@@ -1,11 +1,11 @@
 import threading
 import time
 
-from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 from time import perf_counter
 
 from memos.configs.mem_scheduler import BaseSchedulerConfig
+from memos.context.context import ContextThreadPoolExecutor
 from memos.log import get_logger
 from memos.mem_scheduler.general_modules.base import BaseSchedulerModule
 from memos.mem_scheduler.general_modules.dispatcher import SchedulerDispatcher
@@ -21,7 +21,7 @@ class SchedulerDispatcherMonitor(BaseSchedulerModule):
         super().__init__()
         self.config: BaseSchedulerConfig = config
 
-        self.check_interval = self.config.get("dispatcher_monitor_check_interval", 60)
+        self.check_interval = self.config.get("dispatcher_monitor_check_interval", 300)
         self.max_failures = self.config.get("dispatcher_monitor_max_failures", 2)
 
         # Registry of monitored thread pools
@@ -49,7 +49,7 @@ class SchedulerDispatcherMonitor(BaseSchedulerModule):
     def register_pool(
         self,
         name: str,
-        executor: ThreadPoolExecutor,
+        executor: ContextThreadPoolExecutor,
         max_workers: int,
         restart_on_failure: bool = True,
     ) -> bool:
@@ -177,10 +177,11 @@ class SchedulerDispatcherMonitor(BaseSchedulerModule):
                 else:
                     pool_info["failure_count"] += 1
                     pool_info["healthy"] = False
-                    logger.warning(
-                        f"Pool '{name}' unhealthy ({pool_info['failure_count']}/{self.max_failures}): {reason}"
+                    logger.info(
+                        f"Pool '{name}' unhealthy ({pool_info['failure_count']}/{self.max_failures}): {reason}."
+                        f" Note: This status does not necessarily indicate a problem with the pool itself - "
+                        f"it may also be considered unhealthy if no tasks have been scheduled for an extended period"
                     )
-
             if (
                 pool_info["failure_count"] >= self.max_failures
                 and pool_info["restart"]
@@ -236,14 +237,14 @@ class SchedulerDispatcherMonitor(BaseSchedulerModule):
             return
 
         self._restart_in_progress = True
-        logger.warning(f"Attempting to restart thread pool '{name}'")
+        logger.info(f"Attempting to restart thread pool '{name}'")
 
         try:
             old_executor = pool_info["executor"]
             self.dispatcher.shutdown()
 
             # Create new executor with same parameters
-            new_executor = ThreadPoolExecutor(
+            new_executor = ContextThreadPoolExecutor(
                 max_workers=pool_info["max_workers"],
                 thread_name_prefix=self.dispatcher.thread_name_prefix,  # pylint: disable=protected-access
             )