MemoryOS 0.2.2__py3-none-any.whl → 1.0.1__py3-none-any.whl

memos/mem_os/utils/default_config.py

@@ -112,7 +112,7 @@ def get_default_config(
                 "thread_pool_max_workers": kwargs.get("scheduler_thread_pool_max_workers", 10),
                 "consume_interval_seconds": kwargs.get("scheduler_consume_interval_seconds", 3),
                 "enable_parallel_dispatch": kwargs.get("scheduler_enable_parallel_dispatch", True),
-                "enable_act_memory_update": True,
+                "enable_activation_memory": True,
             },
         }
 

memos/mem_os/utils/format_utils.py

@@ -570,15 +570,23 @@ def convert_graph_to_tree_forworkmem(
         else:
             other_roots.append(root_id)
 
-    def build_tree(node_id: str) -> dict[str, Any]:
-        """Recursively build tree structure"""
+    def build_tree(node_id: str, visited=None) -> dict[str, Any] | None:
+        """Recursively build tree structure with cycle detection"""
+        if visited is None:
+            visited = set()
+
+        if node_id in visited:
+            logger.warning(f"[build_tree] Detected cycle at node {node_id}, skipping.")
+            return None
+        visited.add(node_id)
+
         if node_id not in node_map:
             return None
 
         children_ids = children_map.get(node_id, [])
         children = []
         for child_id in children_ids:
-            child_tree = build_tree(child_id)
+            child_tree = build_tree(child_id, visited)
             if child_tree:
                 children.append(child_tree)
 
@@ -1355,47 +1363,3 @@ def clean_json_response(response: str) -> str:
         str: Clean JSON string without markdown formatting
     """
     return response.replace("```json", "").replace("```", "").strip()
-
-
-def split_continuous_references(text: str) -> str:
-    """
-    Split continuous reference tags into individual reference tags.
-
-    Converts patterns like [1:92ff35fb, 4:bfe6f044] to [1:92ff35fb] [4:bfe6f044]
-
-    Only processes text if:
-    1. '[' appears exactly once
-    2. ']' appears exactly once
-    3. Contains commas between '[' and ']'
-
-    Args:
-        text (str): Text containing reference tags
-
-    Returns:
-        str: Text with split reference tags, or original text if conditions not met
-    """
-    # Early return if text is empty
-    if not text:
-        return text
-    # Check if '[' appears exactly once
-    if text.count("[") != 1:
-        return text
-    # Check if ']' appears exactly once
-    if text.count("]") != 1:
-        return text
-    # Find positions of brackets
-    open_bracket_pos = text.find("[")
-    close_bracket_pos = text.find("]")
-
-    # Check if brackets are in correct order
-    if open_bracket_pos >= close_bracket_pos:
-        return text
-    # Extract content between brackets
-    content_between_brackets = text[open_bracket_pos + 1 : close_bracket_pos]
-    # Check if there's a comma between brackets
-    if "," not in content_between_brackets:
-        return text
-    text = text.replace(content_between_brackets, content_between_brackets.replace(", ", "]["))
-    text = text.replace(content_between_brackets, content_between_brackets.replace(",", "]["))
-
-    return text

memos/mem_os/utils/reference_utils.py

@@ -0,0 +1,153 @@
+from memos.memories.textual.item import (
+    TextualMemoryItem,
+)
+
+
+def split_continuous_references(text: str) -> str:
+    """
+    Split continuous reference tags into individual reference tags.
+
+    Converts patterns like [1:92ff35fb, 4:bfe6f044] to [1:92ff35fb] [4:bfe6f044]
+
+    Only processes text if:
+    1. '[' appears exactly once
+    2. ']' appears exactly once
+    3. Contains commas between '[' and ']'
+
+    Args:
+        text (str): Text containing reference tags
+
+    Returns:
+        str: Text with split reference tags, or original text if conditions not met
+    """
+    # Early return if text is empty
+    if not text:
+        return text
+    # Check if '[' appears exactly once
+    if text.count("[") != 1:
+        return text
+    # Check if ']' appears exactly once
+    if text.count("]") != 1:
+        return text
+    # Find positions of brackets
+    open_bracket_pos = text.find("[")
+    close_bracket_pos = text.find("]")
+
+    # Check if brackets are in correct order
+    if open_bracket_pos >= close_bracket_pos:
+        return text
+    # Extract content between brackets
+    content_between_brackets = text[open_bracket_pos + 1 : close_bracket_pos]
+    # Check if there's a comma between brackets
+    if "," not in content_between_brackets:
+        return text
+    text = text.replace(content_between_brackets, content_between_brackets.replace(", ", "]["))
+    text = text.replace(content_between_brackets, content_between_brackets.replace(",", "]["))
+
+    return text
+
+
+def process_streaming_references_complete(text_buffer: str) -> tuple[str, str]:
+    """
+    Complete streaming reference processing to ensure reference tags are never split.
+
+    Args:
+        text_buffer (str): The accumulated text buffer.
+
+    Returns:
+        tuple[str, str]: (processed_text, remaining_buffer)
+    """
+    import re
+
+    # Pattern to match complete reference tags: [refid:memoriesID]
+    complete_pattern = r"\[\d+:[^\]]+\]"
+
+    # Find all complete reference tags
+    complete_matches = list(re.finditer(complete_pattern, text_buffer))
+
+    if complete_matches:
+        # Find the last complete tag
+        last_match = complete_matches[-1]
+        end_pos = last_match.end()
+
+        # Check if there's any incomplete reference after the last complete one
+        remaining_text = text_buffer[end_pos:]
+
+        # Look for potential incomplete reference patterns after the last complete tag
+        incomplete_pattern = r"\[\d*:?[^\]]*$"
+        if re.search(incomplete_pattern, remaining_text):
+            # There's a potential incomplete reference, find where it starts
+            incomplete_match = re.search(incomplete_pattern, remaining_text)
+            if incomplete_match:
+                incomplete_start = end_pos + incomplete_match.start()
+                processed_text = text_buffer[:incomplete_start]
+                remaining_buffer = text_buffer[incomplete_start:]
+
+                # Apply reference splitting to the processed text
+                processed_text = split_continuous_references(processed_text)
+                return processed_text, remaining_buffer
+
+        # No incomplete reference after the last complete tag, process all
+        processed_text = split_continuous_references(text_buffer)
+        return processed_text, ""
+
+    # Check for incomplete reference tags - be more specific about what constitutes a potential reference
+    # Look for opening bracket with number and colon that could be a reference tag
+    opening_pattern = r"\[\d+:"
+    opening_matches = list(re.finditer(opening_pattern, text_buffer))
+
+    if opening_matches:
+        # Find the last opening tag
+        last_opening = opening_matches[-1]
+        opening_start = last_opening.start()
+
+        # Check if this might be a complete reference tag (has closing bracket after the pattern)
+        remaining_text = text_buffer[last_opening.end() :]
+        if "]" in remaining_text:
+            # This looks like a complete reference tag, process it
+            processed_text = split_continuous_references(text_buffer)
+            return processed_text, ""
+        else:
+            # Incomplete reference tag, keep it in buffer
+            processed_text = text_buffer[:opening_start]
+            processed_text = split_continuous_references(processed_text)
+            return processed_text, text_buffer[opening_start:]
+
+    # More sophisticated check for potential reference patterns
+    # Only hold back text if we see a pattern that could be the start of a reference tag
+    potential_ref_pattern = r"\[\d*:?$"  # Matches [, [1, [12:, etc. at end of buffer
+    if re.search(potential_ref_pattern, text_buffer):
+        # Find the position of the potential reference start
+        match = re.search(potential_ref_pattern, text_buffer)
+        if match:
+            ref_start = match.start()
+            processed_text = text_buffer[:ref_start]
+            processed_text = split_continuous_references(processed_text)
+            return processed_text, text_buffer[ref_start:]
+
+    # Check for standalone [ only at the very end of the buffer
+    # This prevents cutting off mathematical expressions like [ \Delta U = Q - W ]
+    if text_buffer.endswith("["):
+        # Only hold back the single [ character
+        processed_text = text_buffer[:-1]
+        processed_text = split_continuous_references(processed_text)
+        return processed_text, "["
+
+    # No reference-like patterns found, process all text
+    processed_text = split_continuous_references(text_buffer)
+    return processed_text, ""
+
+
+def prepare_reference_data(memories_list: list[TextualMemoryItem]) -> list[dict]:
+    # Prepare reference data
+    reference = []
+    for memories in memories_list:
+        memories_json = memories.model_dump()
+        memories_json["metadata"]["ref_id"] = f"{memories.id.split('-')[0]}"
+        memories_json["metadata"]["embedding"] = []
+        memories_json["metadata"]["sources"] = []
+        memories_json["metadata"]["memory"] = memories.memory
+        memories_json["metadata"]["id"] = memories.id
+        reference.append({"metadata": memories_json["metadata"]})
+
+    return reference

memos/mem_reader/simple_struct.py

@@ -1,10 +1,14 @@
 import concurrent.futures
 import copy
 import json
+import os
+import re
 
 from abc import ABC
 from typing import Any
 
+from tqdm import tqdm
+
 from memos import log
 from memos.chunkers import ChunkerFactory
 from memos.configs.mem_reader import SimpleStructMemReaderConfig
@@ -16,12 +20,79 @@ from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemory
 from memos.parsers.factory import ParserFactory
 from memos.templates.mem_reader_prompts import (
     SIMPLE_STRUCT_DOC_READER_PROMPT,
+    SIMPLE_STRUCT_DOC_READER_PROMPT_ZH,
     SIMPLE_STRUCT_MEM_READER_EXAMPLE,
+    SIMPLE_STRUCT_MEM_READER_EXAMPLE_ZH,
     SIMPLE_STRUCT_MEM_READER_PROMPT,
+    SIMPLE_STRUCT_MEM_READER_PROMPT_ZH,
 )
 
 
 logger = log.get_logger(__name__)
+PROMPT_DICT = {
+    "chat": {
+        "en": SIMPLE_STRUCT_MEM_READER_PROMPT,
+        "zh": SIMPLE_STRUCT_MEM_READER_PROMPT_ZH,
+        "en_example": SIMPLE_STRUCT_MEM_READER_EXAMPLE,
+        "zh_example": SIMPLE_STRUCT_MEM_READER_EXAMPLE_ZH,
+    },
+    "doc": {"en": SIMPLE_STRUCT_DOC_READER_PROMPT, "zh": SIMPLE_STRUCT_DOC_READER_PROMPT_ZH},
+}
+
+
+def detect_lang(text):
+    try:
+        if not text or not isinstance(text, str):
+            return "en"
+        chinese_pattern = r"[\u4e00-\u9fff\u3400-\u4dbf\U00020000-\U0002a6df\U0002a700-\U0002b73f\U0002b740-\U0002b81f\U0002b820-\U0002ceaf\uf900-\ufaff]"
+        chinese_chars = re.findall(chinese_pattern, text)
+        if len(chinese_chars) / len(re.sub(r"[\s\d\W]", "", text)) > 0.3:
+            return "zh"
+        return "en"
+    except Exception:
+        return "en"
+
+
+def _build_node(idx, message, info, scene_file, llm, parse_json_result, embedder):
+    # generate
+    raw = llm.generate(message)
+    if not raw:
+        return None
+
+    # parse_json_result
+    chunk_res = parse_json_result(raw)
+    if not chunk_res:
+        return None
+
+    value = chunk_res.get("value")
+    if not value:
+        return None
+
+    # embed
+    embedding = embedder.embed([value])[0]
+
+    # TextualMemoryItem
+    tags = chunk_res["tags"] if isinstance(chunk_res.get("tags"), list) else []
+    key = chunk_res.get("key", None)
+
+    node_i = TextualMemoryItem(
+        memory=value,
+        metadata=TreeNodeTextualMemoryMetadata(
+            user_id=info.get("user_id"),
+            session_id=info.get("session_id"),
+            memory_type="LongTermMemory",
+            status="activated",
+            tags=tags,
+            key=key,
+            embedding=embedding,
+            usage=[],
+            sources=[f"{scene_file}_{idx}"],
+            background="",
+            confidence=0.99,
+            type="fact",
+        ),
+    )
+    return node_i
 
 
 class SimpleStructMemReader(BaseMemReader, ABC):
@@ -40,11 +111,13 @@ class SimpleStructMemReader(BaseMemReader, ABC):
         self.chunker = ChunkerFactory.from_config(config.chunker)
 
     def _process_chat_data(self, scene_data_info, info):
-        prompt = SIMPLE_STRUCT_MEM_READER_PROMPT.replace(
-            "${conversation}", "\n".join(scene_data_info)
-        )
+        lang = detect_lang("\n".join(scene_data_info))
+        template = PROMPT_DICT["chat"][lang]
+        examples = PROMPT_DICT["chat"][f"{lang}_example"]
+
+        prompt = template.replace("${conversation}", "\n".join(scene_data_info))
         if self.config.remove_prompt_example:
-            prompt = prompt.replace(SIMPLE_STRUCT_MEM_READER_EXAMPLE, "")
+            prompt = prompt.replace(examples, "")
 
         messages = [{"role": "user", "content": prompt}]
 
@@ -180,7 +253,7 @@ class SimpleStructMemReader(BaseMemReader, ABC):
         elif type == "doc":
             for item in scene_data:
                 try:
-                    if not isinstance(item, str):
+                    if os.path.exists(item):
                         parsed_text = parser.parse(item)
                         results.append({"file": "pure_text", "text": parsed_text})
                     else:
@@ -193,46 +266,42 @@ class SimpleStructMemReader(BaseMemReader, ABC):
 
     def _process_doc_data(self, scene_data_info, info):
         chunks = self.chunker.chunk(scene_data_info["text"])
-        messages = [
-            [
-                {
-                    "role": "user",
-                    "content": SIMPLE_STRUCT_DOC_READER_PROMPT.replace("{chunk_text}", chunk.text),
-                }
-            ]
-            for chunk in chunks
-        ]
-
-        processed_chunks = []
-        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
-            futures = [executor.submit(self.llm.generate, message) for message in messages]
-            for future in concurrent.futures.as_completed(futures):
-                chunk_result = future.result()
-                if chunk_result:
-                    processed_chunks.append(chunk_result)
+        messages = []
+        for chunk in chunks:
+            lang = detect_lang(chunk.text)
+            template = PROMPT_DICT["doc"][lang]
+            prompt = template.replace("{chunk_text}", chunk.text)
+            message = [{"role": "user", "content": prompt}]
+            messages.append(message)
 
-        processed_chunks = [self.parse_json_result(r) for r in processed_chunks]
         doc_nodes = []
-        for i, chunk_res in enumerate(processed_chunks):
-            if chunk_res:
-                node_i = TextualMemoryItem(
-                    memory=chunk_res["value"],
-                    metadata=TreeNodeTextualMemoryMetadata(
-                        user_id=info.get("user_id"),
-                        session_id=info.get("session_id"),
-                        memory_type="LongTermMemory",
-                        status="activated",
-                        tags=chunk_res["tags"] if type(chunk_res["tags"]) is list else [],
-                        key=chunk_res["key"],
-                        embedding=self.embedder.embed([chunk_res["value"]])[0],
-                        usage=[],
-                        sources=[f"{scene_data_info['file']}_{i}"],
-                        background="",
-                        confidence=0.99,
-                        type="fact",
-                    ),
-                )
-                doc_nodes.append(node_i)
+        scene_file = scene_data_info["file"]
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=50) as executor:
+            futures = {
+                executor.submit(
+                    _build_node,
+                    idx,
+                    msg,
+                    info,
+                    scene_file,
+                    self.llm,
+                    self.parse_json_result,
+                    self.embedder,
+                ): idx
+                for idx, msg in enumerate(messages)
+            }
+            total = len(futures)
+
+            for future in tqdm(
+                concurrent.futures.as_completed(futures), total=total, desc="Processing"
+            ):
+                try:
+                    node = future.result()
+                    if node:
+                        doc_nodes.append(node)
+                except Exception as e:
+                    tqdm.write(f"[ERROR] {e}")
         return doc_nodes
 
     def parse_json_result(self, response_text):