MemoryOS 0.1.13__py3-none-any.whl → 0.2.1__py3-none-any.whl

memos/memories/textual/tree_text_memory/organize/conflict.py

@@ -0,0 +1,198 @@
+import json
+import re
+
+from datetime import datetime
+
+from memos.embedders.base import BaseEmbedder
+from memos.graph_dbs.neo4j import Neo4jGraphDB
+from memos.llms.base import BaseLLM
+from memos.log import get_logger
+from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata
+from memos.templates.tree_reorganize_prompts import (
+    CONFLICT_DETECTOR_PROMPT,
+    CONFLICT_RESOLVER_PROMPT,
+)
+
+
+logger = get_logger(__name__)
+
+
+class ConflictHandler:
+    EMBEDDING_THRESHOLD: float = 0.8  # Threshold for embedding similarity to consider conflict
+
+    def __init__(self, graph_store: Neo4jGraphDB, llm: BaseLLM, embedder: BaseEmbedder):
+        self.graph_store = graph_store
+        self.llm = llm
+        self.embedder = embedder
+
+    def detect(
+        self, memory: TextualMemoryItem, top_k: int = 5, scope: str | None = None
+    ) -> list[tuple[TextualMemoryItem, TextualMemoryItem]]:
+        """
+        Detect conflicts by finding the most similar items in the graph database based on embedding, then use LLM to judge conflict.
+        Args:
+            memory: The memory item (should have an embedding attribute or field).
+            top_k: Number of top similar nodes to retrieve.
+            scope: Optional memory type filter.
+        Returns:
+            List of conflict pairs (each pair is a tuple: (memory, candidate)).
+        """
+        # 1. Search for similar memories based on embedding
+        embedding = memory.metadata.embedding
+        embedding_candidates_info = self.graph_store.search_by_embedding(
+            embedding, top_k=top_k, scope=scope
+        )
+        # 2. Filter based on similarity threshold
+        embedding_candidates_ids = [
+            info["id"]
+            for info in embedding_candidates_info
+            if info["score"] >= self.EMBEDDING_THRESHOLD and info["id"] != memory.id
+        ]
+        # 3. Judge conflicts using LLM
+        embedding_candidates = self.graph_store.get_nodes(embedding_candidates_ids)
+        conflict_pairs = []
+        for embedding_candidate in embedding_candidates:
+            embedding_candidate = TextualMemoryItem.from_dict(embedding_candidate)
+            prompt = [
+                {
+                    "role": "system",
+                    "content": "You are a conflict detector for memory items.",
+                },
+                {
+                    "role": "user",
+                    "content": CONFLICT_DETECTOR_PROMPT.format(
+                        statement_1=memory.memory,
+                        statement_2=embedding_candidate.memory,
+                    ),
+                },
+            ]
+            result = self.llm.generate(prompt).strip()
+            if "yes" in result.lower():
+                conflict_pairs.append([memory, embedding_candidate])
+        if len(conflict_pairs):
+            conflict_text = "\n".join(
+                f'"{pair[0].memory!s}" <==CONFLICT==> "{pair[1].memory!s}"'
+                for pair in conflict_pairs
+            )
+            logger.warning(
+                f"Detected {len(conflict_pairs)} conflicts for memory {memory.id}\n {conflict_text}"
+            )
+        return conflict_pairs
+
+    def resolve(self, memory_a: TextualMemoryItem, memory_b: TextualMemoryItem) -> None:
+        """
+        Resolve detected conflicts between two memory items using LLM fusion.
+        Args:
+            memory_a: The first conflicting memory item.
+            memory_b: The second conflicting memory item.
+        Returns:
+            A fused TextualMemoryItem representing the resolved memory.
+        """
+
+        # ———————————— 1. LLM generate fused memory ————————————
+        metadata_for_resolve = ["key", "background", "confidence", "updated_at"]
+        metadata_1 = memory_a.metadata.model_dump_json(include=metadata_for_resolve)
+        metadata_2 = memory_b.metadata.model_dump_json(include=metadata_for_resolve)
+        prompt = [
+            {
+                "role": "system",
+                "content": "",
+            },
+            {
+                "role": "user",
+                "content": CONFLICT_RESOLVER_PROMPT.format(
+                    statement_1=memory_a.memory,
+                    metadata_1=metadata_1,
+                    statement_2=memory_b.memory,
+                    metadata_2=metadata_2,
+                ),
+            },
+        ]
+        response = self.llm.generate(prompt).strip()
+
+        # ———————————— 2. Parse the response ————————————
+        try:
+            answer = re.search(r"<answer>(.*?)</answer>", response, re.DOTALL)
+            answer = answer.group(1).strip()
+            # —————— 2.1 Can't resolve conflict, hard update by comparing timestamp ————
+            if len(answer) <= 10 and "no" in answer.lower():
+                logger.warning(
+                    f"Conflict between {memory_a.id} and {memory_b.id} could not be resolved. "
+                )
+                self._hard_update(memory_a, memory_b)
+            # —————— 2.2 Conflict resolved, update metadata and memory ————
+            else:
+                fixed_metadata = self._merge_metadata(answer, memory_a.metadata, memory_b.metadata)
+                merged_memory = TextualMemoryItem(memory=answer, metadata=fixed_metadata)
+                logger.info(f"Resolved result: {merged_memory}")
+                self._resolve_in_graph(memory_a, memory_b, merged_memory)
+        except json.decoder.JSONDecodeError:
+            logger.error(f"Failed to parse LLM response: {response}")
+
+    def _hard_update(self, memory_a: TextualMemoryItem, memory_b: TextualMemoryItem):
+        """
+        Hard update: compare updated_at, keep the newer one, overwrite the older one's metadata.
+        """
+        time_a = datetime.fromisoformat(memory_a.metadata.updated_at)
+        time_b = datetime.fromisoformat(memory_b.metadata.updated_at)
+
+        newer_mem = memory_a if time_a >= time_b else memory_b
+        older_mem = memory_b if time_a >= time_b else memory_a
+
+        self.graph_store.delete_node(older_mem.id)
+        logger.warning(
+            f"Delete older memory {older_mem.id}: <{older_mem.memory}> due to conflict with {newer_mem.id}: <{newer_mem.memory}>"
+        )
+
+    def _resolve_in_graph(
+        self,
+        conflict_a: TextualMemoryItem,
+        conflict_b: TextualMemoryItem,
+        merged: TextualMemoryItem,
+    ):
+        edges_a = self.graph_store.get_edges(conflict_a.id, type="ANY", direction="ANY")
+        edges_b = self.graph_store.get_edges(conflict_b.id, type="ANY", direction="ANY")
+        all_edges = edges_a + edges_b
+
+        self.graph_store.add_node(
+            merged.id, merged.memory, merged.metadata.model_dump(exclude_none=True)
+        )
+
+        for edge in all_edges:
+            new_from = merged.id if edge["from"] in (conflict_a.id, conflict_b.id) else edge["from"]
+            new_to = merged.id if edge["to"] in (conflict_a.id, conflict_b.id) else edge["to"]
+            if new_from == new_to:
+                continue
+            # Check if the edge already exists before adding
+            if not self.graph_store.edge_exists(new_from, new_to, edge["type"], direction="ANY"):
+                self.graph_store.add_edge(new_from, new_to, edge["type"])
+
+        self.graph_store.update_node(conflict_a.id, {"status": "archived"})
+        self.graph_store.update_node(conflict_b.id, {"status": "archived"})
+        self.graph_store.add_edge(conflict_a.id, merged.id, type="MERGED_TO")
+        self.graph_store.add_edge(conflict_b.id, merged.id, type="MERGED_TO")
+        logger.debug(
+            f"Archive {conflict_a.id} and {conflict_b.id}, and inherit their edges to {merged.id}."
+        )
+
+    def _merge_metadata(
+        self,
+        memory: str,
+        metadata_a: TreeNodeTextualMemoryMetadata,
+        metadata_b: TreeNodeTextualMemoryMetadata,
+    ) -> TreeNodeTextualMemoryMetadata:
+        metadata_1 = metadata_a.model_dump()
+        metadata_2 = metadata_b.model_dump()
+        merged_metadata = {
+            "sources": (metadata_1["sources"] or []) + (metadata_2["sources"] or []),
+            "embedding": self.embedder.embed([memory])[0],
+            "update_at": datetime.now().isoformat(),
+            "created_at": datetime.now().isoformat(),
+        }
+        for key in metadata_1:
+            if key in merged_metadata:
+                continue
+            merged_metadata[key] = (
+                metadata_1[key] if metadata_1[key] is not None else metadata_2[key]
+            )
+        return TreeNodeTextualMemoryMetadata.model_validate(merged_metadata)

memos/memories/textual/tree_text_memory/organize/manager.py

@@ -5,8 +5,13 @@ from datetime import datetime
 
 from memos.embedders.factory import OllamaEmbedder
 from memos.graph_dbs.neo4j import Neo4jGraphDB
+from memos.llms.factory import AzureLLM, OllamaLLM, OpenAILLM
 from memos.log import get_logger
 from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata
+from memos.memories.textual.tree_text_memory.organize.reorganizer import (
+    GraphStructureReorganizer,
+    QueueMessage,
+)
 
 
 logger = get_logger(__name__)
@@ -17,9 +22,11 @@ class MemoryManager:
         self,
         graph_store: Neo4jGraphDB,
         embedder: OllamaEmbedder,
+        llm: OpenAILLM | OllamaLLM | AzureLLM,
         memory_size: dict | None = None,
         threshold: float | None = 0.80,
         merged_threshold: float | None = 0.92,
+        is_reorganize: bool = False,
     ):
         self.graph_store = graph_store
         self.embedder = embedder
@@ -36,17 +43,24 @@ class MemoryManager:
                 "UserMemory": 10000,
             }
         self._threshold = threshold
+        self.is_reorganize = is_reorganize
+        self.reorganizer = GraphStructureReorganizer(
+            graph_store, llm, embedder, is_reorganize=is_reorganize
+        )
         self._merged_threshold = merged_threshold
 
-    def add(self, memories: list[TextualMemoryItem]) -> None:
+    def add(self, memories: list[TextualMemoryItem]) -> list[str]:
         """
         Add new memories in parallel to different memory types (WorkingMemory, LongTermMemory, UserMemory).
         """
+        added_ids: list[str] = []
+
         with ThreadPoolExecutor(max_workers=8) as executor:
-            futures = [executor.submit(self._process_memory, memory) for memory in memories]
+            futures = {executor.submit(self._process_memory, m): m for m in memories}
             for future in as_completed(futures):
                 try:
-                    future.result()
+                    ids = future.result()
+                    added_ids.extend(ids)
                 except Exception as e:
                     logger.exception("Memory processing error: ", exc_info=e)
 
@@ -61,6 +75,7 @@ class MemoryManager:
         )
 
         self._refresh_memory_size()
+        return added_ids
 
     def replace_working_memory(self, memories: list[TextualMemoryItem]) -> None:
         """
@@ -102,17 +117,23 @@ class MemoryManager:
         Process and add memory to different memory types (WorkingMemory, LongTermMemory, UserMemory).
         This method runs asynchronously to process each memory item.
         """
+        ids = []
+
         # Add to WorkingMemory
-        self._add_memory_to_db(memory, "WorkingMemory")
+        working_id = self._add_memory_to_db(memory, "WorkingMemory")
+        ids.append(working_id)
 
         # Add to LongTermMemory and UserMemory
         if memory.metadata.memory_type in ["LongTermMemory", "UserMemory"]:
-            self._add_to_graph_memory(
+            added_id = self._add_to_graph_memory(
                 memory=memory,
                 memory_type=memory.metadata.memory_type,
             )
+            ids.append(added_id)
+
+        return ids
 
-    def _add_memory_to_db(self, memory: TextualMemoryItem, memory_type: str):
+    def _add_memory_to_db(self, memory: TextualMemoryItem, memory_type: str) -> str:
         """
         Add a single memory item to the graph store, with FIFO logic for WorkingMemory.
         """
@@ -124,6 +145,7 @@ class MemoryManager:
 
         # Insert node into graph
         self.graph_store.add_node(working_memory.id, working_memory.memory, metadata)
+        return working_memory.id
 
     def _add_to_graph_memory(self, memory: TextualMemoryItem, memory_type: str):
         """
@@ -148,23 +170,22 @@ class MemoryManager:
         )
 
         if similar_nodes and similar_nodes[0]["score"] > self._merged_threshold:
-            self._merge(memory, similar_nodes)
+            return self._merge(memory, similar_nodes)
         else:
             node_id = str(uuid.uuid4())
             # Step 2: Add new node to graph
             self.graph_store.add_node(
                 node_id, memory.memory, memory.metadata.model_dump(exclude_none=True)
             )
-
-            # Step 3: Optionally link to a summary node based on topic
-            if memory.metadata.tags:
-                parent_id = self._ensure_structure_path(
-                    memory_type=memory_type, metadata=memory.metadata
+            self.reorganizer.add_message(
+                QueueMessage(
+                    op="add",
+                    after_node=[node_id],
                 )
-                if parent_id:
-                    self.graph_store.add_edge(parent_id, node_id, "PARENT")
+            )
+            return node_id
 
-    def _merge(self, source_node: TextualMemoryItem, similar_nodes: list[dict]) -> None:
+    def _merge(self, source_node: TextualMemoryItem, similar_nodes: list[dict]) -> str:
         """
         TODO: Add node traceability support by optionally preserving source nodes and linking them with MERGED_FROM edges.
 
@@ -191,7 +212,9 @@ class MemoryManager:
         merged_background = f"{original_meta.background}\n⟵MERGED⟶\n{source_meta.background}"
         merged_embedding = self.embedder.embed([merged_text])[0]
 
-        merged_confidence = float((original_meta.confidence + source_meta.confidence) / 2)
+        original_conf = original_meta.confidence or 0.0
+        source_conf = source_meta.confidence or 0.0
+        merged_confidence = float((original_conf + source_conf) / 2)
         merged_usage = list(set((original_meta.usage or []) + (source_meta.usage or [])))
 
         # Create new merged node
@@ -223,12 +246,18 @@ class MemoryManager:
         # After creating merged node and tracing lineage
         self._inherit_edges(original_id, merged_id)
 
-        # Relate other similar nodes to merged if needed
-        for related_node in similar_nodes[1:]:
-            if not self.graph_store.edge_exists(
-                merged_id, related_node["id"], type="ANY", direction="ANY"
-            ):
-                self.graph_store.add_edge(merged_id, related_node["id"], type="RELATE")
+        # log to reorganizer before updating the graph
+        self.reorganizer.add_message(
+            QueueMessage(
+                op="merge",
+                before_node=[
+                    original_id,
+                    source_node.id,
+                ],
+                after_node=[merged_id],
+            )
+        )
+        return merged_id
 
     def _inherit_edges(self, from_id: str, to_id: str) -> None:
         """
@@ -293,13 +322,33 @@ class MemoryManager:
                     background="",
                 ),
             )
-
             self.graph_store.add_node(
                 id=new_node.id,
                 memory=new_node.memory,
                 metadata=new_node.metadata.model_dump(exclude_none=True),
             )
+            self.reorganizer.add_message(
+                QueueMessage(
+                    op="add",
+                    after_node=[new_node.id],
+                )
+            )
+
             node_id = new_node.id
 
         # Step 3: Return this structure node ID as the parent_id
         return node_id
+
+    def wait_reorganizer(self):
+        """
+        Wait for the reorganizer to finish processing all messages.
+        """
+        logger.debug("Waiting for reorganizer to finish processing messages...")
+        self.reorganizer.wait_until_current_task_done()
+
+    def close(self):
+        self.wait_reorganizer()
+        self.reorganizer.stop()
+
+    def __del__(self):
+        self.close()

memos/memories/textual/tree_text_memory/organize/redundancy.py

@@ -0,0 +1,193 @@
+import json
+import re
+
+from datetime import datetime
+
+from memos.embedders.base import BaseEmbedder
+from memos.graph_dbs.neo4j import Neo4jGraphDB
+from memos.llms.base import BaseLLM
+from memos.log import get_logger
+from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata
+from memos.templates.tree_reorganize_prompts import (
+    REDUNDANCY_DETECTOR_PROMPT,
+    REDUNDANCY_MERGE_PROMPT,
+    REDUNDANCY_RESOLVER_PROMPT,
+)
+
+
+logger = get_logger(__name__)
+
+
+class RedundancyHandler:
+    EMBEDDING_THRESHOLD: float = 0.8  # Threshold for embedding similarity to consider redundancy
+
+    def __init__(self, graph_store: Neo4jGraphDB, llm: BaseLLM, embedder: BaseEmbedder):
+        self.graph_store = graph_store
+        self.llm = llm
+        self.embedder = embedder
+
+    def detect(
+        self, memory: TextualMemoryItem, top_k: int = 5, scope: str | None = None
+    ) -> list[tuple[TextualMemoryItem, TextualMemoryItem]]:
+        """
+        Detect redundancy by finding the most similar items in the graph database based on embedding, then use LLM to judge redundancy.
+        Args:
+            memory: The memory item (should have an embedding attribute or field).
+            top_k: Number of top similar nodes to retrieve.
+            scope: Optional memory type filter.
+        Returns:
+            List of redundancy pairs (each pair is a tuple: (memory, candidate)).
+        """
+        # 1. Search for similar memories based on embedding
+        embedding = memory.metadata.embedding
+        embedding_candidates_info = self.graph_store.search_by_embedding(
+            embedding, top_k=top_k, scope=scope
+        )
+        # 2. Filter based on similarity threshold
+        embedding_candidates_ids = [
+            info["id"]
+            for info in embedding_candidates_info
+            if info["score"] >= self.EMBEDDING_THRESHOLD and info["id"] != memory.id
+        ]
+        # 3. Judge redundancys using LLM
+        embedding_candidates = self.graph_store.get_nodes(embedding_candidates_ids)
+        redundant_pairs = []
+        for embedding_candidate in embedding_candidates:
+            embedding_candidate = TextualMemoryItem.from_dict(embedding_candidate)
+            prompt = [
+                {
+                    "role": "system",
+                    "content": "You are a redundancy detector for memory items.",
+                },
+                {
+                    "role": "user",
+                    "content": REDUNDANCY_DETECTOR_PROMPT.format(
+                        statement_1=memory.memory,
+                        statement_2=embedding_candidate.memory,
+                    ),
+                },
+            ]
+            result = self.llm.generate(prompt).strip()
+            if "yes" in result.lower():
+                redundant_pairs.append([memory, embedding_candidate])
+        if len(redundant_pairs):
+            redundant_text = "\n".join(
+                f'"{pair[0].memory!s}" <==REDUNDANCY==> "{pair[1].memory!s}"'
+                for pair in redundant_pairs
+            )
+            logger.warning(
+                f"Detected {len(redundant_pairs)} redundancies for memory {memory.id}\n {redundant_text}"
+            )
+        return redundant_pairs
+
+    def resolve_two_nodes(self, memory_a: TextualMemoryItem, memory_b: TextualMemoryItem) -> None:
+        """
+        Resolve detected redundancies between two memory items using LLM fusion.
+        Args:
+            memory_a: The first redundant memory item.
+            memory_b: The second redundant memory item.
+        Returns:
+            A fused TextualMemoryItem representing the resolved memory.
+        """
+        return  # waiting for implementation
+        # ———————————— 1. LLM generate fused memory ————————————
+        metadata_for_resolve = ["key", "background", "confidence", "updated_at"]
+        metadata_1 = memory_a.metadata.model_dump_json(include=metadata_for_resolve)
+        metadata_2 = memory_b.metadata.model_dump_json(include=metadata_for_resolve)
+        prompt = [
+            {
+                "role": "system",
+                "content": "",
+            },
+            {
+                "role": "user",
+                "content": REDUNDANCY_RESOLVER_PROMPT.format(
+                    statement_1=memory_a.memory,
+                    metadata_1=metadata_1,
+                    statement_2=memory_b.memory,
+                    metadata_2=metadata_2,
+                ),
+            },
+        ]
+        response = self.llm.generate(prompt).strip()
+
+        # ———————————— 2. Parse the response ————————————
+        try:
+            answer = re.search(r"<answer>(.*?)</answer>", response, re.DOTALL)
+            answer = answer.group(1).strip()
+            fixed_metadata = self._merge_metadata(answer, memory_a.metadata, memory_b.metadata)
+            merged_memory = TextualMemoryItem(memory=answer, metadata=fixed_metadata)
+            logger.info(f"Resolved result: {merged_memory}")
+            self._resolve_in_graph(memory_a, memory_b, merged_memory)
+        except json.decoder.JSONDecodeError:
+            logger.error(f"Failed to parse LLM response: {response}")
+
+    def resolve_one_node(self, memory: TextualMemoryItem) -> None:
+        prompt = [
+            {
+                "role": "user",
+                "content": REDUNDANCY_MERGE_PROMPT.format(merged_text=memory.memory),
+            },
+        ]
+        response = self.llm.generate(prompt)
+        memory.memory = response.strip()
+        self.graph_store.update_node(
+            memory.id,
+            {"memory": memory.memory, **memory.metadata.model_dump(exclude_none=True)},
+        )
+        logger.debug(f"Merged memory: {memory.memory}")
+
+    def _resolve_in_graph(
+        self,
+        redundant_a: TextualMemoryItem,
+        redundant_b: TextualMemoryItem,
+        merged: TextualMemoryItem,
+    ):
+        edges_a = self.graph_store.get_edges(redundant_a.id, type="ANY", direction="ANY")
+        edges_b = self.graph_store.get_edges(redundant_b.id, type="ANY", direction="ANY")
+        all_edges = edges_a + edges_b
+
+        self.graph_store.add_node(
+            merged.id, merged.memory, merged.metadata.model_dump(exclude_none=True)
+        )
+
+        for edge in all_edges:
+            new_from = (
+                merged.id if edge["from"] in (redundant_a.id, redundant_b.id) else edge["from"]
+            )
+            new_to = merged.id if edge["to"] in (redundant_a.id, redundant_b.id) else edge["to"]
+            if new_from == new_to:
+                continue
+            # Check if the edge already exists before adding
+            if not self.graph_store.edge_exists(new_from, new_to, edge["type"], direction="ANY"):
+                self.graph_store.add_edge(new_from, new_to, edge["type"])
+
+        self.graph_store.update_node(redundant_a.id, {"status": "archived"})
+        self.graph_store.update_node(redundant_b.id, {"status": "archived"})
+        self.graph_store.add_edge(redundant_a.id, merged.id, type="MERGED_TO")
+        self.graph_store.add_edge(redundant_b.id, merged.id, type="MERGED_TO")
+        logger.debug(
+            f"Archive {redundant_a.id} and {redundant_b.id}, and inherit their edges to {merged.id}."
+        )
+
+    def _merge_metadata(
+        self,
+        memory: str,
+        metadata_a: TreeNodeTextualMemoryMetadata,
+        metadata_b: TreeNodeTextualMemoryMetadata,
+    ) -> TreeNodeTextualMemoryMetadata:
+        metadata_1 = metadata_a.model_dump()
+        metadata_2 = metadata_b.model_dump()
+        merged_metadata = {
+            "sources": (metadata_1["sources"] or []) + (metadata_2["sources"] or []),
+            "embedding": self.embedder.embed([memory])[0],
+            "update_at": datetime.now().isoformat(),
+            "created_at": datetime.now().isoformat(),
+        }
+        for key in metadata_1:
+            if key in merged_metadata:
+                continue
+            merged_metadata[key] = (
+                metadata_1[key] if metadata_1[key] is not None else metadata_2[key]
+            )
+        return TreeNodeTextualMemoryMetadata.model_validate(merged_metadata)