MemoryOS 2.0.3__py3-none-any.whl

memos/reranker/strategies/base.py

@@ -0,0 +1,61 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from memos.memories.textual.item import TextualMemoryItem
+
+from .dialogue_common import DialogueRankingTracker
+
+
+class BaseRerankerStrategy(ABC):
+    """Abstract interface for memory rerankers with concatenation strategy."""
+
+    @abstractmethod
+    def prepare_documents(
+        self,
+        query: str,
+        graph_results: list[TextualMemoryItem],
+        top_k: int,
+        **kwargs,
+    ) -> tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+        """
+        Prepare documents for ranking based on the strategy.
+
+        Args:
+            query: The search query
+            graph_results: List of TextualMemoryItem objects to process
+            top_k: Maximum number of items to return
+            **kwargs: Additional strategy-specific parameters
+
+        Returns:
+            tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+            - Tracker: DialogueRankingTracker instance
+            - original_items: Dict mapping memory_id to original TextualMemoryItem
+            - documents: List of text documents ready for ranking
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def reconstruct_items(
+        self,
+        ranked_indices: list[int],
+        scores: list[float],
+        tracker: DialogueRankingTracker,
+        original_items: dict[str, Any],
+        top_k: int,
+        **kwargs,
+    ) -> list[tuple[TextualMemoryItem, float]]:
+        """
+        Reconstruct TextualMemoryItem objects from ranked results.
+
+        Args:
+            ranked_indices: List of indices sorted by relevance
+            scores: Corresponding relevance scores
+            tracker: DialogueRankingTracker instance
+            original_items: Dict mapping memory_id to original TextualMemoryItem
+            top_k: Maximum number of items to return
+            **kwargs: Additional strategy-specific parameters
+
+        Returns:
+            List of (reconstructed_memory_item, aggregated_score) tuples
+        """
+        raise NotImplementedError

memos/reranker/strategies/concat_background.py

@@ -0,0 +1,94 @@
+# memos/reranker/strategies/single_turn.py
+from __future__ import annotations
+
+import re
+
+from typing import Any
+
+from .base import BaseRerankerStrategy
+from .dialogue_common import DialogueRankingTracker
+
+
+_TAG1 = re.compile(r"^\s*\[[^\]]*\]\s*")
+
+
+class ConcatBackgroundStrategy(BaseRerankerStrategy):
+    """
+    Concat background strategy.
+
+    This strategy processes dialogue pairs by concatenating background and
+    user and assistant messages into single strings for ranking. Each dialogue pair becomes a
+    separate document for ranking.
+    """
+
+    def prepare_documents(
+        self,
+        query: str,
+        graph_results: list,
+        top_k: int,
+        **kwargs,
+    ) -> tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+        """
+        Prepare documents based on single turn concatenation strategy.
+
+        Args:
+            query: The search query
+            graph_results: List of graph results
+            top_k: Maximum number of items to return
+
+        Returns:
+            tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+            - Tracker: DialogueRankingTracker instance
+            - original_items: Dict mapping memory_id to original TextualMemoryItem
+            - documents: List of text documents ready for ranking
+        """
+
+        original_items = {}
+        tracker = DialogueRankingTracker()
+        documents = []
+        for item in graph_results:
+            memory = getattr(item, "memory", None)
+            if isinstance(memory, str):
+                memory = _TAG1.sub("", memory)
+
+            background = ""
+            if hasattr(item, "metadata") and hasattr(item.metadata, "background"):
+                background = getattr(item.metadata, "background", "")
+                if not isinstance(background, str):
+                    background = ""
+
+            documents.append(f"{memory}\n{background}")
+        return tracker, original_items, documents
+
+    def reconstruct_items(
+        self,
+        ranked_indices: list[int],
+        scores: list[float],
+        tracker: DialogueRankingTracker,
+        original_items: dict[str, Any],
+        top_k: int,
+        **kwargs,
+    ) -> list[tuple[Any, float]]:
+        """
+        Reconstruct TextualMemoryItem objects from ranked dialogue pairs.
+
+        Args:
+            ranked_indices: List of dialogue pair indices sorted by relevance
+            scores: Corresponding relevance scores
+            tracker: DialogueRankingTracker instance
+            original_items: Dict mapping memory_id to original TextualMemoryItem
+            top_k: Maximum number of items to return
+
+        Returns:
+            List of (reconstructed_memory_item, aggregated_score) tuples
+        """
+        graph_results = kwargs.get("graph_results")
+        documents = kwargs.get("documents")
+        reconstructed_items = []
+        for idx in ranked_indices:
+            item = graph_results[idx]
+            item.memory = f"{item.memory}\n{documents[idx]}"
+            reconstructed_items.append((item, scores[idx]))
+
+        reconstructed_items.sort(key=lambda x: x[1], reverse=True)
+        return reconstructed_items[:top_k]

memos/reranker/strategies/concat_docsource.py

@@ -0,0 +1,110 @@
+# memos/reranker/strategies/single_turn.py
+from __future__ import annotations
+
+import re
+
+from typing import Any
+
+from .base import BaseRerankerStrategy
+from .dialogue_common import DialogueRankingTracker
+
+
+_TAG1 = re.compile(r"^\s*\[[^\]]*\]\s*")
+
+
+class ConcatDocSourceStrategy(BaseRerankerStrategy):
+    """
+    Concat background strategy.
+
+    This strategy processes dialogue pairs by concatenating background and
+    user and assistant messages into single strings for ranking. Each dialogue pair becomes a
+    separate document for ranking.
+    """
+
+    """
+    Concat background strategy.
+
+    This strategy processes dialogue pairs by concatenating background and
+    user and assistant messages into single strings for ranking. Each dialogue pair becomes a
+    separate document for ranking.
+    """
+
+    def prepare_documents(
+        self,
+        query: str,
+        graph_results: list,
+        top_k: int,
+        **kwargs,
+    ) -> tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+        """
+        Prepare documents based on single turn concatenation strategy.
+
+        Args:
+            query: The search query
+            graph_results: List of graph results
+            top_k: Maximum number of items to return
+
+        Returns:
+            tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+            - Tracker: DialogueRankingTracker instance
+            - original_items: Dict mapping memory_id to original TextualMemoryItem
+            - documents: List of text documents ready for ranking
+        """
+
+        original_items = {}
+        tracker = DialogueRankingTracker()
+        documents = []
+        documents_set = set()
+        for item in graph_results:
+            memory = getattr(item, "memory", None)
+            if isinstance(memory, str):
+                memory = _TAG1.sub("", memory)
+
+            chunk_text = ""
+            if hasattr(item, "metadata") and hasattr(item.metadata, "sources"):
+                sources = getattr(item.metadata, "sources", [])
+                for source in sources:
+                    if source.type == "file":
+                        chunk_text += source.content
+            if chunk_text:
+                if chunk_text in documents_set:
+                    continue
+                else:
+                    documents_set.add(chunk_text)
+                    documents.append(f"{memory}\n\n[Sources]:\n{chunk_text}")
+            else:
+                documents.append(memory)
+        return tracker, original_items, documents
+
+    def reconstruct_items(
+        self,
+        ranked_indices: list[int],
+        scores: list[float],
+        tracker: DialogueRankingTracker,
+        original_items: dict[str, Any],
+        top_k: int,
+        **kwargs,
+    ) -> list[tuple[Any, float]]:
+        """
+        Reconstruct TextualMemoryItem objects from ranked dialogue pairs.
+
+        Args:
+            ranked_indices: List of dialogue pair indices sorted by relevance
+            scores: Corresponding relevance scores
+            tracker: DialogueRankingTracker instance
+            original_items: Dict mapping memory_id to original TextualMemoryItem
+            top_k: Maximum number of items to return
+
+        Returns:
+            List of (reconstructed_memory_item, aggregated_score) tuples
+        """
+        graph_results = kwargs.get("graph_results")
+        documents = kwargs.get("documents")
+        reconstructed_items = []
+        for idx in ranked_indices:
+            item = graph_results[idx]
+            item.memory = f"{documents[idx]}"
+            reconstructed_items.append((item, scores[idx]))
+
+        reconstructed_items.sort(key=lambda x: x[1], reverse=True)
+        return reconstructed_items[:top_k]

memos/reranker/strategies/dialogue_common.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import re
+
+from typing import Any, Literal
+
+from pydantic import BaseModel
+
+from memos.memories.textual.item import SourceMessage, TextualMemoryItem
+
+
+# Strip a leading "[...]" tag (e.g., "[2025-09-01] ..." or "[meta] ...")
+# before sending text to the reranker. This keeps inputs clean and
+# avoids misleading the model with bracketed prefixes.
+_TAG1 = re.compile(r"^\s*\[[^\]]*\]\s*")
+
+
+def strip_memory_tags(item: TextualMemoryItem) -> str:
+    """Strip leading tags from memory text."""
+    memory = _TAG1.sub("", m) if isinstance((m := getattr(item, "memory", None)), str) else m
+    return memory
+
+
+def extract_content(msg: dict[str, Any] | str) -> str:
+    """Extract content from message, handling both string and dict formats."""
+    if isinstance(msg, dict):
+        return msg.get("content", str(msg))
+    if isinstance(msg, SourceMessage):
+        return msg.content
+    return str(msg)
+
+
+class DialoguePair(BaseModel):
+    """Represents a single dialogue pair extracted from sources."""
+
+    pair_id: str  # Unique identifier for this dialogue pair
+    memory_id: str  # ID of the source TextualMemoryItem
+    memory: str
+    pair_index: int  # Index of this pair within the source memory's dialogue
+    user_msg: str | dict[str, Any] | SourceMessage  # User message content
+    assistant_msg: str | dict[str, Any] | SourceMessage  # Assistant message content
+    combined_text: str  # The concatenated text used for ranking
+    chat_time: str | None = None
+
+    @property
+    def user_content(self) -> str:
+        """Get user message content as string."""
+        return extract_content(self.user_msg)
+
+    @property
+    def assistant_content(self) -> str:
+        """Get assistant message content as string."""
+        return extract_content(self.assistant_msg)
+
+
+class DialogueRankingTracker:
+    """Tracks dialogue pairs and their rankings for memory reconstruction."""
+
+    def __init__(self):
+        self.dialogue_pairs: list[DialoguePair] = []
+
+    def add_dialogue_pair(
+        self,
+        memory_id: str,
+        pair_index: int,
+        user_msg: str | dict[str, Any],
+        assistant_msg: str | dict[str, Any],
+        memory: str,
+        chat_time: str | None = None,
+        concat_format: Literal["user_assistant", "user_only"] = "user_assistant",
+    ) -> str:
+        """Add a dialogue pair and return its unique ID."""
+        user_content = extract_content(user_msg)
+        assistant_content = extract_content(assistant_msg)
+        if concat_format == "user_assistant":
+            combined_text = f"[{chat_time}]: \nuser: {user_content}\nassistant: {assistant_content}"
+        elif concat_format == "user_only":
+            combined_text = f"[{chat_time}]: \nuser: {user_content}"
+        else:
+            raise ValueError(f"Invalid concat format: {concat_format}")
+
+        pair_id = f"{memory_id}_{pair_index}"
+
+        dialogue_pair = DialoguePair(
+            pair_id=pair_id,
+            memory_id=memory_id,
+            pair_index=pair_index,
+            user_msg=user_msg,
+            assistant_msg=assistant_msg,
+            combined_text=combined_text,
+            memory=memory,
+            chat_time=chat_time,
+        )
+
+        self.dialogue_pairs.append(dialogue_pair)
+        return pair_id
+
+    def get_documents_for_ranking(self, concat_memory: bool = True) -> list[str]:
+        """Get the combined text documents for ranking."""
+        if concat_memory:
+            return [(pair.memory + "\n\n" + pair.combined_text) for pair in self.dialogue_pairs]
+        else:
+            return [pair.combined_text for pair in self.dialogue_pairs]
+
+    def get_dialogue_pair_by_index(self, index: int) -> DialoguePair | None:
+        """Get dialogue pair by its index in the ranking results."""
+        if 0 <= index < len(self.dialogue_pairs):
+            return self.dialogue_pairs[index]
+        return None

memos/reranker/strategies/factory.py

@@ -0,0 +1,31 @@
+# memos/reranker/factory.py
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from .concat_background import ConcatBackgroundStrategy
+from .concat_docsource import ConcatDocSourceStrategy
+from .single_turn import SingleTurnStrategy
+from .singleturn_outmem import SingleTurnOutMemStrategy
+
+
+if TYPE_CHECKING:
+    from .base import BaseRerankerStrategy
+
+
+class RerankerStrategyFactory:
+    """Factory class for creating reranker strategy instances."""
+
+    backend_to_class: ClassVar[dict[str, Any]] = {
+        "single_turn": SingleTurnStrategy,
+        "concat_background": ConcatBackgroundStrategy,
+        "singleturn_outmem": SingleTurnOutMemStrategy,
+        "concat_docsource": ConcatDocSourceStrategy,
+    }
+
+    @classmethod
+    def from_config(cls, config_factory: str = "single_turn") -> BaseRerankerStrategy:
+        if config_factory not in cls.backend_to_class:
+            raise ValueError(f"Invalid backend: {config_factory}")
+        strategy_class = cls.backend_to_class[config_factory]
+        return strategy_class()

memos/reranker/strategies/single_turn.py

@@ -0,0 +1,107 @@
+# memos/reranker/strategies/single_turn.py
+from __future__ import annotations
+
+from copy import deepcopy
+from typing import Any
+
+from .base import BaseRerankerStrategy
+from .dialogue_common import DialogueRankingTracker, extract_content, strip_memory_tags
+
+
+class SingleTurnStrategy(BaseRerankerStrategy):
+    """
+    Single turn dialogue strategy.
+
+    This strategy processes dialogue pairs by concatenating user and assistant
+    messages into single strings for ranking. Each dialogue pair becomes a
+    separate document for ranking.
+    example:
+        >>> documents = ["chat_time: 2025-01-01 12:00:00\nuser: hello\nassistant: hi there"]
+        >>> output memory item: ["Memory:xxx \n\n chat_time: 2025-01-01 12:00:00\nuser: hello\nassistant: hi there"]
+    """
+
+    def prepare_documents(
+        self,
+        query: str,
+        graph_results: list,
+        top_k: int,
+        **kwargs,
+    ) -> tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+        """
+        Prepare documents based on single turn concatenation strategy.
+
+        Args:
+            query: The search query
+            graph_results: List of graph results
+            top_k: Maximum number of items to return
+
+        Returns:
+            tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+            - Tracker: DialogueRankingTracker instance
+            - original_items: Dict mapping memory_id to original TextualMemoryItem
+            - documents: List of text documents ready for ranking
+        """
+
+        original_items = {}
+        tracker = DialogueRankingTracker()
+        for item in graph_results:
+            memory = strip_memory_tags(item)
+            sources = getattr(item.metadata, "sources", [])
+            original_items[item.id] = item
+
+            # Group messages into pairs and concatenate
+            for i in range(0, len(sources), 2):
+                user_msg = sources[i] if i < len(sources) else {}
+                assistant_msg = sources[i + 1] if i + 1 < len(sources) else {}
+
+                user_content = extract_content(user_msg)
+                assistant_content = extract_content(assistant_msg)
+                chat_time = getattr(user_msg, "chat_time", "")
+
+                if user_content or assistant_content:  # Only add non-empty pairs
+                    pair_index = i // 2
+                    tracker.add_dialogue_pair(
+                        item.id, pair_index, user_msg, assistant_msg, memory or "", chat_time
+                    )
+
+        documents = tracker.get_documents_for_ranking()
+        return tracker, original_items, documents
+
+    def reconstruct_items(
+        self,
+        ranked_indices: list[int],
+        scores: list[float],
+        tracker: DialogueRankingTracker,
+        original_items: dict[str, Any],
+        top_k: int,
+        **kwargs,
+    ) -> list[tuple[Any, float]]:
+        """
+        Reconstruct TextualMemoryItem objects from ranked dialogue pairs.
+
+        Args:
+            ranked_indices: List of dialogue pair indices sorted by relevance
+            scores: Corresponding relevance scores
+            tracker: DialogueRankingTracker instance
+            original_items: Dict mapping memory_id to original TextualMemoryItem
+            top_k: Maximum number of items to return
+
+        Returns:
+            List of (reconstructed_memory_item, aggregated_score) tuples
+        """
+        reconstructed_items = []
+        for idx, score in zip(ranked_indices, scores, strict=False):
+            dialogue_pair = tracker.get_dialogue_pair_by_index(idx)
+            if dialogue_pair and (dialogue_pair.memory_id in original_items):
+                original_item = original_items[dialogue_pair.memory_id]
+                reconstructed_item = deepcopy(original_item)
+                reconstructed_item.memory = (
+                    dialogue_pair.memory
+                    + "\n\nsources-dialogue-pairs"
+                    + dialogue_pair.combined_text
+                )
+                reconstructed_items.append((reconstructed_item, score))
+
+        # Sort by aggregated score and return top_k
+        reconstructed_items.sort(key=lambda x: x[1], reverse=True)
+        return reconstructed_items[:top_k]

memos/reranker/strategies/singleturn_outmem.py

@@ -0,0 +1,98 @@
+# memos/reranker/strategies/single_turn.py
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import TYPE_CHECKING, Any
+
+from .dialogue_common import DialogueRankingTracker
+from .single_turn import SingleTurnStrategy
+
+
+if TYPE_CHECKING:
+    from .dialogue_common import DialogueRankingTracker
+
+
+class SingleTurnOutMemStrategy(SingleTurnStrategy):
+    """
+    Single turn dialogue strategy.
+
+    This strategy processes dialogue pairs by concatenating user and assistant
+    messages into single strings for ranking. Each dialogue pair becomes a
+    separate document for ranking.
+    example:
+        >>> documents = ["chat_time: 2025-01-01 12:00:00\nuser: hello\nassistant: hi there"]
+        >>> output memory item: ["Memory:xxx \n\n chat_time: 2025-01-01 12:00:00\nuser: hello\nassistant: hi there"]
+    """
+
+    def prepare_documents(
+        self,
+        query: str,
+        graph_results: list,
+        top_k: int,
+        **kwargs,
+    ) -> tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+        """
+        Prepare documents based on single turn concatenation strategy.
+
+        Args:
+            query: The search query
+            graph_results: List of graph results
+            top_k: Maximum number of items to return
+
+        Returns:
+            tuple[DialogueRankingTracker, dict[str, Any], list[str]]:
+            - Tracker: DialogueRankingTracker instance
+            - original_items: Dict mapping memory_id to original TextualMemoryItem
+            - documents: List of text documents ready for ranking
+        """
+        return super().prepare_documents(query, graph_results, top_k, **kwargs)
+
+    def reconstruct_items(
+        self,
+        ranked_indices: list[int],
+        scores: list[float],
+        tracker: DialogueRankingTracker,
+        original_items: dict[str, Any],
+        top_k: int,
+        **kwargs,
+    ) -> list[tuple[Any, float]]:
+        """
+        Reconstruct TextualMemoryItem objects from ranked dialogue pairs.
+
+        Args:
+            ranked_indices: List of dialogue pair indices sorted by relevance
+            scores: Corresponding relevance scores
+            tracker: DialogueRankingTracker instance
+            original_items: Dict mapping memory_id to original TextualMemoryItem
+            top_k: Maximum number of items to return
+
+        Returns:
+            List of (reconstructed_memory_item, aggregated_score) tuples
+        """
+        # Group ranked pairs by memory_id
+        memory_groups = defaultdict(list)
+        memory_scores = defaultdict(list)
+
+        for idx, score in zip(ranked_indices, scores, strict=False):
+            dialogue_pair = tracker.get_dialogue_pair_by_index(idx)
+            if dialogue_pair:
+                memory_groups[dialogue_pair.memory_id].append(dialogue_pair)
+                memory_scores[dialogue_pair.memory_id].append(score)
+
+        reconstructed_items = []
+
+        for memory_id, _pairs in memory_groups.items():
+            if memory_id not in original_items:
+                continue
+            original_item = original_items[memory_id]
+
+            # Calculate aggregated score (e.g., max, mean, or weighted average)
+            pair_scores = memory_scores[memory_id]
+
+            aggregated_score = max(pair_scores) if pair_scores else 0.0
+
+            reconstructed_items.append((original_item, aggregated_score))
+
+        # Sort by aggregated score and return top_k
+        reconstructed_items.sort(key=lambda x: x[1], reverse=True)
+        return reconstructed_items[:top_k]

memos/settings.py

@@ -0,0 +1,10 @@
+import os
+
+from pathlib import Path
+
+
+MEMOS_DIR = Path(os.getenv("MEMOS_BASE_PATH", Path.cwd())) / ".memos"
+DEBUG = False
+
+# "memos" or "memos.submodules" ... to filter logs from specific packages
+LOG_FILTER_TREE_PREFIX = ""