MemoryOS 1.0.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

memos/parsers/factory.py

@@ -1,6 +1,7 @@
 from typing import Any, ClassVar
 
 from memos.configs.parser import ParserConfigFactory
+from memos.memos_tools.singleton import singleton_factory
 from memos.parsers.base import BaseParser
 from memos.parsers.markitdown import MarkItDownParser
 
@@ -11,6 +12,7 @@ class ParserFactory(BaseParser):
     backend_to_class: ClassVar[dict[str, Any]] = {"markitdown": MarkItDownParser}
 
     @classmethod
+    @singleton_factory()
     def from_config(cls, config_factory: ParserConfigFactory) -> BaseParser:
         backend = config_factory.backend
         if backend not in cls.backend_to_class:

memos/reranker/concat.py

@@ -0,0 +1,59 @@
+import re
+
+from typing import Any
+
+
+_TAG1 = re.compile(r"^\s*\[[^\]]*\]\s*")
+
+
+def process_source(
+    items: list[tuple[Any, str | dict[str, Any] | list[Any]]] | None = None, recent_num: int = 3
+) -> str:
+    """
+    Args:
+        items: List of tuples where each tuple contains (memory, source).
+               source can be str, Dict, or List.
+        recent_num: Number of recent items to concatenate.
+    Returns:
+        str: Concatenated source.
+    """
+    if items is None:
+        items = []
+    concat_data = []
+    memory = None
+    for item in items:
+        memory, source = item
+        for content in source:
+            if isinstance(content, str):
+                if "assistant:" in content:
+                    continue
+                concat_data.append(content)
+    if memory is not None:
+        concat_data = [memory, *concat_data]
+    return "\n".join(concat_data)
+
+
+def concat_original_source(
+    graph_results: list,
+    merge_field: list[str] | None = None,
+) -> list[str]:
+    """
+    Merge memory items with original dialogue.
+    Args:
+        graph_results (list[TextualMemoryItem]): List of memory items with embeddings.
+        merge_field (List[str]): List of fields to merge.
+    Returns:
+        list[str]: List of memory and concat orginal memory.
+    """
+    if merge_field is None:
+        merge_field = ["sources"]
+    documents = []
+    for item in graph_results:
+        memory = _TAG1.sub("", m) if isinstance((m := getattr(item, "memory", None)), str) else m
+        sources = []
+        for field in merge_field:
+            source = getattr(item.metadata, field, "")
+            sources.append((memory, source))
+        concat_string = process_source(sources)
+        documents.append(concat_string)
+    return documents

memos/reranker/cosine_local.py

@@ -49,6 +49,7 @@ class CosineLocalReranker(BaseReranker):
         self,
         level_weights: dict[str, float] | None = None,
         level_field: str = "background",
+        **kwargs,
     ):
         self.level_weights = level_weights or {"topic": 1.0, "concept": 1.0, "fact": 1.0}
         self.level_field = level_field

memos/reranker/factory.py

@@ -3,6 +3,9 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
+# Import singleton decorator
+from memos.memos_tools.singleton import singleton_factory
+
 from .cosine_local import CosineLocalReranker
 from .http_bge import HTTPBGEReranker
 from .noop import NoopReranker
@@ -16,6 +19,7 @@ if TYPE_CHECKING:
 
 class RerankerFactory:
     @staticmethod
+    @singleton_factory("RerankerFactory")
     def from_config(cfg: RerankerConfigFactory | None) -> BaseReranker | None:
         if not cfg:
             return None
@@ -29,6 +33,7 @@ class RerankerFactory:
                 model=c.get("model", "bge-reranker-v2-m3"),
                 timeout=int(c.get("timeout", 10)),
                 headers_extra=c.get("headers_extra"),
+                rerank_source=c.get("rerank_source"),
             )
 
         if backend in {"cosine_local", "cosine"}:

memos/reranker/http_bge.py

@@ -3,22 +3,74 @@ from __future__ import annotations
 
 import re
 
-from typing import TYPE_CHECKING
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any
 
 import requests
 
+from memos.log import get_logger
+
 from .base import BaseReranker
+from .concat import concat_original_source
+
+
+logger = get_logger(__name__)
 
 
 if TYPE_CHECKING:
     from memos.memories.textual.item import TextualMemoryItem
 
+# Strip a leading "[...]" tag (e.g., "[2025-09-01] ..." or "[meta] ...")
+# before sending text to the reranker. This keeps inputs clean and
+# avoids misleading the model with bracketed prefixes.
 _TAG1 = re.compile(r"^\s*\[[^\]]*\]\s*")
+DEFAULT_BOOST_WEIGHTS = {"user_id": 0.5, "tags": 0.2, "session_id": 0.3}
+
+
+def _value_matches(item_value: Any, wanted: Any) -> bool:
+    """
+    Generic matching:
+    - if item_value is list/tuple/set: check membership (any match if wanted is iterable)
+    - else: equality (any match if wanted is iterable)
+    """
+
+    def _iterable(x):
+        # exclude strings from "iterable"
+        return isinstance(x, Iterable) and not isinstance(x, str | bytes)
+
+    if _iterable(item_value):
+        if _iterable(wanted):
+            return any(w in item_value for w in wanted)
+        return wanted in item_value
+    else:
+        if _iterable(wanted):
+            return any(item_value == w for w in wanted)
+        return item_value == wanted
 
 
 class HTTPBGEReranker(BaseReranker):
     """
-    HTTP-based BGE reranker. Mirrors your old MemoryReranker, but configurable.
+    HTTP-based BGE reranker.
+
+    This class sends (query, documents[]) to a remote HTTP endpoint that
+    performs cross-encoder-style re-ranking (e.g., BGE reranker) and returns
+    relevance scores. It then maps those scores back onto the original
+    TextualMemoryItem list and returns (item, score) pairs sorted by score.
+
+    Notes
+    -----
+    - The endpoint is expected to accept JSON:
+        {
+          "model": "<model-name>",
+          "query": "<query text>",
+          "documents": ["doc1", "doc2", ...]
+        }
+    - Two response shapes are supported:
+        1) {"results": [{"index": <int>, "relevance_score": <float>}, ...]}
+           where "index" refers to the *position in the documents array*.
+        2) {"data": [{"score": <float>}, ...]} (aligned by list order)
+    - If the service fails or responds unexpectedly, this falls back to
+      returning the original items with 0.0 scores (best-effort).
     """
 
     def __init__(
@@ -28,7 +80,26 @@ class HTTPBGEReranker(BaseReranker):
         model: str = "bge-reranker-v2-m3",
         timeout: int = 10,
         headers_extra: dict | None = None,
+        rerank_source: list[str] | None = None,
+        boost_weights: dict[str, float] | None = None,
+        boost_default: float = 0.0,
+        warn_unknown_filter_keys: bool = True,
+        **kwargs,
     ):
+        """
+        Parameters
+        ----------
+        reranker_url : str
+            HTTP endpoint for the reranker service.
+        token : str, optional
+            Bearer token for auth. If non-empty, added to the Authorization header.
+        model : str, optional
+            Model identifier understood by the server.
+        timeout : int, optional
+            Request timeout (seconds).
+        headers_extra : dict | None, optional
+            Additional headers to merge into the request headers.
+        """
         if not reranker_url:
             raise ValueError("reranker_url must not be empty")
         self.reranker_url = reranker_url
@@ -36,22 +107,62 @@ class HTTPBGEReranker(BaseReranker):
         self.model = model
         self.timeout = timeout
         self.headers_extra = headers_extra or {}
+        self.concat_source = rerank_source
+
+        self.boost_weights = (
+            DEFAULT_BOOST_WEIGHTS.copy()
+            if boost_weights is None
+            else {k: float(v) for k, v in boost_weights.items()}
+        )
+        self.boost_default = float(boost_default)
+        self.warn_unknown_filter_keys = bool(warn_unknown_filter_keys)
+        self._warned_missing_keys: set[str] = set()
 
     def rerank(
         self,
         query: str,
-        graph_results: list,
+        graph_results: list[TextualMemoryItem],
         top_k: int,
+        search_filter: dict | None = None,
         **kwargs,
     ) -> list[tuple[TextualMemoryItem, float]]:
+        """
+        Rank candidate memories by relevance to the query.
+
+        Parameters
+        ----------
+        query : str
+            The search query.
+        graph_results : list[TextualMemoryItem]
+            Candidate items to re-rank. Each item is expected to have a
+            `.memory` str field; non-strings are ignored.
+        top_k : int
+            Return at most this many items.
+        search_filter : dict | None
+            Currently unused. Present to keep signature compatible.
+
+        Returns
+        -------
+        list[tuple[TextualMemoryItem, float]]
+            Re-ranked items with scores, sorted descending by score.
+        """
         if not graph_results:
             return []
 
-        documents = [
-            (_TAG1.sub("", m) if isinstance((m := getattr(item, "memory", None)), str) else m)
-            for item in graph_results
-        ]
-        documents = [d for d in documents if isinstance(d, str) and d]
+        # Build a mapping from "payload docs index" -> "original graph_results index"
+        # Only include items that have a non-empty string memory. This ensures that
+        # any index returned by the server can be mapped back correctly.
+        if self.concat_source:
+            documents = concat_original_source(graph_results, self.concat_source)
+        else:
+            documents = [
+                (_TAG1.sub("", m) if isinstance((m := getattr(item, "memory", None)), str) else m)
+                for item in graph_results
+            ]
+            documents = [d for d in documents if isinstance(d, str) and d]
+
+        logger.info(f"[HTTPBGERerankerSample] query: {query} , documents: {documents[:5]}...")
+
         if not documents:
             return []
 
@@ -59,6 +170,7 @@ class HTTPBGEReranker(BaseReranker):
         payload = {"model": self.model, "query": query, "documents": documents}
 
         try:
+            # Make the HTTP request to the reranker service
             resp = requests.post(
                 self.reranker_url, headers=headers, json=payload, timeout=self.timeout
             )
@@ -68,18 +180,28 @@ class HTTPBGEReranker(BaseReranker):
             scored_items: list[tuple[TextualMemoryItem, float]] = []
 
             if "results" in data:
+                # Format:
+                # dict("results": [{"index": int, "relevance_score": float},
+                # ...])
                 rows = data.get("results", [])
                 for r in rows:
                     idx = r.get("index")
+                    # The returned index refers to 'documents' (i.e., our 'pairs' order),
+                    # so we must map it back to the original graph_results index.
                     if isinstance(idx, int) and 0 <= idx < len(graph_results):
-                        score = float(r.get("relevance_score", r.get("score", 0.0)))
-                        scored_items.append((graph_results[idx], score))
+                        raw_score = float(r.get("relevance_score", r.get("score", 0.0)))
+                        item = graph_results[idx]
+                        # generic boost
+                        score = self._apply_boost_generic(item, raw_score, search_filter)
+                        scored_items.append((item, score))
 
                 scored_items.sort(key=lambda x: x[1], reverse=True)
                 return scored_items[: min(top_k, len(scored_items))]
 
             elif "data" in data:
+                # Format: {"data": [{"score": float}, ...]} aligned by list order
                 rows = data.get("data", [])
+                # Build a list of scores aligned with our 'documents' (pairs)
                 score_list = [float(r.get("score", 0.0)) for r in rows]
 
                 if len(score_list) < len(graph_results):
@@ -87,13 +209,104 @@ class HTTPBGEReranker(BaseReranker):
                 elif len(score_list) > len(graph_results):
                     score_list = score_list[: len(graph_results)]
 
-                scored_items = list(zip(graph_results, score_list, strict=False))
+                scored_items = []
+                for item, raw_score in zip(graph_results, score_list, strict=False):
+                    score = self._apply_boost_generic(item, raw_score, search_filter)
+                    scored_items.append((item, score))
+
                 scored_items.sort(key=lambda x: x[1], reverse=True)
                 return scored_items[: min(top_k, len(scored_items))]
 
             else:
+                # Unexpected response schema: return a 0.0-scored fallback of the first top_k valid docs
+                # Note: we use 'pairs' to keep alignment with valid (string) docs.
                 return [(item, 0.0) for item in graph_results[:top_k]]
 
         except Exception as e:
-            print(f"[HTTPBGEReranker] request failed: {e}")
+            # Network error, timeout, JSON decode error, etc.
+            # Degrade gracefully by returning first top_k valid docs with 0.0 score.
+            logger.error(f"[HTTPBGEReranker] request failed: {e}")
             return [(item, 0.0) for item in graph_results[:top_k]]
+
+    def _get_attr_or_key(self, obj: Any, key: str) -> Any:
+        """
+        Resolve `key` on `obj` with one-level fallback into `obj.metadata`.
+
+        Priority:
+          1) obj.<key>
+          2) obj[key]
+          3) obj.metadata.<key>
+          4) obj.metadata[key]
+        """
+        if obj is None:
+            return None
+
+        # support input like "metadata.user_id"
+        if "." in key:
+            head, tail = key.split(".", 1)
+            base = self._get_attr_or_key(obj, head)
+            return self._get_attr_or_key(base, tail)
+
+        def _resolve(o: Any, k: str):
+            if o is None:
+                return None
+            v = getattr(o, k, None)
+            if v is not None:
+                return v
+            if hasattr(o, "get"):
+                try:
+                    return o.get(k)
+                except Exception:
+                    return None
+            return None
+
+        # 1) find in obj
+        v = _resolve(obj, key)
+        if v is not None:
+            return v
+
+        # 2) find in obj.metadata
+        meta = _resolve(obj, "metadata")
+        if meta is not None:
+            return _resolve(meta, key)
+
+        return None
+
+    def _apply_boost_generic(
+        self,
+        item: TextualMemoryItem,
+        base_score: float,
+        search_filter: dict | None,
+    ) -> float:
+        """
+        Multiply base_score by (1 + weight) for each matching key in search_filter.
+        - key resolution: self._get_attr_or_key(item, key)
+        - weight = boost_weights.get(key, self.boost_default)
+        - unknown key -> one-time warning
+        """
+        if not search_filter:
+            return base_score
+
+        score = float(base_score)
+
+        for key, wanted in search_filter.items():
+            # _get_attr_or_key automatically find key in item and
+            # item.metadata ("metadata.user_id" supported)
+            resolved = self._get_attr_or_key(item, key)
+
+            if resolved is None:
+                if self.warn_unknown_filter_keys and key not in self._warned_missing_keys:
+                    logger.warning(
+                        "[HTTPBGEReranker] search_filter key '%s' not found on TextualMemoryItem or metadata",
+                        key,
+                    )
+                    self._warned_missing_keys.add(key)
+                continue
+
+            if _value_matches(resolved, wanted):
+                w = float(self.boost_weights.get(key, self.boost_default))
+                if w != 0.0:
+                    score *= 1.0 + w
+                    score = min(max(0.0, score), 1.0)
+
+        return score

memos/templates/mem_scheduler_prompts.py

@@ -151,11 +151,253 @@ You are an intelligent keyword extraction system. Your task is to identify and e
 Answer:
 """
 
+MEMORY_FILTERING_PROMPT = """
+# Memory Relevance Filtering Task
+
+## Role
+You are an intelligent memory filtering system. Your primary function is to analyze memory relevance and filter out memories that are completely unrelated to the user's query history.
+
+## Task Description
+Analyze the provided memories and determine which ones are relevant to the user's query history:
+1. Evaluate semantic relationship between each memory and the query history
+2. Identify memories that are completely unrelated or irrelevant
+3. Filter out memories that don't contribute to answering the queries
+4. Preserve memories that provide context, evidence, or relevant information
+
+## Relevance Criteria
+A memory is considered RELEVANT if it:
+- Directly answers questions from the query history
+- Provides context or background information related to the queries
+- Contains information that could be useful for understanding the queries
+- Shares semantic similarity with query topics or themes
+- Contains keywords or concepts mentioned in the queries
+
+A memory is considered IRRELEVANT if it:
+- Has no semantic connection to any query in the history
+- Discusses completely unrelated topics
+- Contains information that cannot help answer any query
+- Is too generic or vague to be useful
+
+## Input Format
+- Query History: List of user queries (chronological order)
+- Memories: List of memory texts to be evaluated
+
+## Output Format Requirements
+You MUST output a valid JSON object with EXACTLY the following structure:
+{{
+  "relevant_memories": [array_of_memory_indices],
+  "filtered_count": <number_of_filtered_memories>,
+  "reasoning": "string_explanation"
+}}
+
+## Important Notes:
+- Only output the JSON object, nothing else
+- Do not include any markdown formatting or code block notation
+- Ensure all brackets and quotes are properly closed
+- The output must be parseable by a JSON parser
+- Memory indices should correspond to the input order (0-based)
+
+## Processing Guidelines
+1. Be conservative in filtering - when in doubt, keep the memory
+2. Consider both direct and indirect relevance
+3. Look for thematic connections, not just exact keyword matches
+4. Preserve memories that provide valuable context
+
+## Current Task
+Query History: {query_history}
+Memories to Filter: {memories}
+
+Please provide your filtering analysis:
+"""
+
+MEMORY_REDUNDANCY_FILTERING_PROMPT = """
+# Memory Redundancy Filtering Task
+
+## Role
+You are an intelligent memory optimization system. Your primary function is to analyze memories and remove redundancy to improve memory quality and relevance.
+
+## Task Description
+Analyze the provided memories and identify redundant ones:
+1. **Redundancy Detection**: Find memories that contain the same core facts relevant to queries
+2. **Best Memory Selection**: Keep only the most concise and focused version of redundant information
+3. **Quality Preservation**: Ensure the final set covers all necessary information without redundancy
+
+## Redundancy Detection Criteria
+A memory is considered REDUNDANT if it:
+- Contains the same core fact as another memory that's relevant to the queries
+- Provides the same information but with additional irrelevant details
+- Repeats information that's already covered by a more concise memory
+- Has overlapping content with another memory that serves the same purpose
+
+When redundancy is found, KEEP the memory that:
+- Is more concise and focused
+- Contains less irrelevant information
+- Is more directly relevant to the queries
+- Has higher information density
+
+## Input Format
+- Query History: List of user queries (chronological order)
+- Memories: List of memory texts to be evaluated
+
+## Output Format Requirements
+You MUST output a valid JSON object with EXACTLY the following structure:
+{{
+  "kept_memories": [array_of_memory_indices_to_keep],
+  "redundant_groups": [
+    {{
+      "group_id": <number>,
+      "memories": [array_of_redundant_memory_indices],
+      "kept_memory": <index_of_best_memory_in_group>,
+      "reason": "explanation_of_why_this_memory_was_kept"
+    }}
+  ],
+  "reasoning": "string_explanation_of_filtering_decisions"
+}}
+
+## Important Notes:
+- Only output the JSON object, nothing else
+- Do not include any markdown formatting or code block notation
+- Ensure all brackets and quotes are properly closed
+- The output must be parseable by a JSON parser
+- Memory indices should correspond to the input order (0-based)
+- Be conservative in filtering - when in doubt, keep the memory
+- Focus on semantic similarity, not just exact text matches
+
+## Processing Guidelines
+1. First identify which memories are relevant to the queries
+2. Group relevant memories by semantic similarity and core facts
+3. Within each group, select the best memory (most concise, least noise)
+4. Ensure the final set covers all necessary information without redundancy
+
+## Current Task
+Query History: {query_history}
+Memories to Filter: {memories}
+
+Please provide your redundancy filtering analysis:
+"""
+
+MEMORY_COMBINED_FILTERING_PROMPT = """
+# Memory Combined Filtering Task
+
+## Role
+You are an intelligent memory optimization system. Your primary function is to analyze memories and perform two types of filtering in sequence:
+1. **Unrelated Memory Removal**: Remove memories that are completely unrelated to the user's query history
+2. **Redundancy Removal**: Remove redundant memories by keeping only the most informative version
+
+## Task Description
+Analyze the provided memories and perform comprehensive filtering:
+1. **First Step - Unrelated Filtering**: Identify and remove memories that have no semantic connection to any query
+2. **Second Step - Redundancy Filtering**: Group similar memories and keep only the best version from each group
+
+## Unrelated Memory Detection Criteria
+A memory is considered UNRELATED if it:
+- Has no semantic connection to any query in the history
+- Discusses completely unrelated topics
+- Contains information that cannot help answer any query
+- Is too generic or vague to be useful
+
+## Redundancy Detection Criteria
+A memory is considered REDUNDANT if it:
+- Contains the same core fact as another memory that's relevant to the queries
+- Provides the same information but with additional irrelevant details
+- Repeats information that's already covered by a more concise memory
+- Has overlapping content with another memory that serves the same purpose
+
+When redundancy is found, KEEP the memory that:
+- Is more concise and focused
+- Contains less irrelevant information
+- Is more directly relevant to the queries
+- Has higher information density
+
+## Input Format
+- Query History: List of user queries (chronological order)
+- Memories: List of memory texts to be evaluated
+
+## Output Format Requirements
+You MUST output a valid JSON object with EXACTLY the following structure:
+{{
+  "kept_memories": [array_of_memory_indices_to_keep],
+  "unrelated_removed_count": <number_of_unrelated_memories_removed>,
+  "redundant_removed_count": <number_of_redundant_memories_removed>,
+  "redundant_groups": [
+    {{
+      "group_id": <number>,
+      "memories": [array_of_redundant_memory_indices],
+      "kept_memory": <index_of_best_memory_in_group>,
+      "reason": "explanation_of_why_this_memory_was_kept"
+    }}
+  ],
+  "reasoning": "string_explanation_of_filtering_decisions"
+}}
+
+## Important Notes:
+- Only output the JSON object, nothing else
+- Do not include any markdown formatting or code block notation
+- Ensure all brackets and quotes are properly closed
+- The output must be parseable by a JSON parser
+- Memory indices should correspond to the input order (0-based)
+- Be conservative in filtering - when in doubt, keep the memory
+- Focus on semantic similarity, not just exact text matches
+
+## Processing Guidelines
+1. **First, identify unrelated memories** and mark them for removal
+2. **Then, group remaining memories** by semantic similarity and core facts
+3. **Within each group, select the best memory** (most concise, least noise)
+4. **Ensure the final set covers all necessary information** without redundancy
+5. **Count how many memories were removed** for each reason
+
+## Current Task
+Query History: {query_history}
+Memories to Filter: {memories}
+
+Please provide your combined filtering analysis:
+"""
+
+
+MEMORY_ANSWER_ABILITY_EVALUATION_PROMPT = """
+# Memory Answer Ability Evaluation Task
+
+## Task
+Evaluate whether the provided memories contain sufficient information to answer the user's query.
+
+## Evaluation Criteria
+Consider these factors:
+1. **Answer completeness**: Do the memories cover all aspects of the query?
+2. **Evidence relevance**: Do the memories directly support answering the query?
+3. **Detail specificity**: Do the memories contain necessary granularity?
+4. **Information gaps**: Are there obvious missing pieces of information?
+
+## Decision Rules
+- Return `True` for "result" ONLY when memories provide complete, relevant answers
+- Return `False` for "result" if memories are insufficient, irrelevant, or incomplete
+
+## User Query
+{query}
+
+## Available Memories
+{memory_list}
+
+## Required Output
+Return a JSON object with this exact structure:
+{{
+  "result": <boolean>,
+  "reason": "<brief explanation of your decision>"
+}}
+
+## Instructions
+- Only output the JSON object, nothing else
+- Be conservative: if there's any doubt about completeness, return true
+- Focus on whether the memories can fully answer the query without additional information
+"""
 
 PROMPT_MAPPING = {
     "intent_recognizing": INTENT_RECOGNIZING_PROMPT,
     "memory_reranking": MEMORY_RERANKING_PROMPT,
     "query_keywords_extraction": QUERY_KEYWORDS_EXTRACTION_PROMPT,
+    "memory_filtering": MEMORY_FILTERING_PROMPT,
+    "memory_redundancy_filtering": MEMORY_REDUNDANCY_FILTERING_PROMPT,
+    "memory_combined_filtering": MEMORY_COMBINED_FILTERING_PROMPT,
+    "memory_answer_ability_evaluation": MEMORY_ANSWER_ABILITY_EVALUATION_PROMPT,
 }
 
 MEMORY_ASSEMBLY_TEMPLATE = """The retrieved memories are listed as follows:\n\n {memory_text}"""