MemoryOS 2.0.3__py3-none-any.whl

memos/mem_reader/read_multi_modal/__init__.py

@@ -0,0 +1,43 @@
+"""Multimodal message parsers for different message types.
+
+This package provides parsers for different message types in both fast and fine modes:
+- String messages
+- System messages
+- User messages
+- Assistant messages
+- Tool messages
+- Text content parts
+- File content parts
+
+Each parser supports both "fast" mode (quick processing without LLM) and
+"fine" mode (with LLM for better understanding).
+"""
+
+from .assistant_parser import AssistantParser
+from .base import BaseMessageParser
+from .file_content_parser import FileContentParser
+from .image_parser import ImageParser
+from .multi_modal_parser import MultiModalParser
+from .string_parser import StringParser
+from .system_parser import SystemParser
+from .text_content_parser import TextContentParser
+from .tool_parser import ToolParser
+from .user_parser import UserParser
+from .utils import coerce_scene_data, detect_lang, extract_role
+
+
+__all__ = [
+    "AssistantParser",
+    "BaseMessageParser",
+    "FileContentParser",
+    "ImageParser",
+    "MultiModalParser",
+    "StringParser",
+    "SystemParser",
+    "TextContentParser",
+    "ToolParser",
+    "UserParser",
+    "coerce_scene_data",
+    "detect_lang",
+    "extract_role",
+]

memos/mem_reader/read_multi_modal/assistant_parser.py

@@ -0,0 +1,311 @@
+"""Parser for assistant messages."""
+
+import json
+
+from typing import Any
+
+from memos.embedders.base import BaseEmbedder
+from memos.llms.base import BaseLLM
+from memos.log import get_logger
+from memos.memories.textual.item import (
+    SourceMessage,
+    TextualMemoryItem,
+    TreeNodeTextualMemoryMetadata,
+)
+from memos.types.openai_chat_completion_types import ChatCompletionAssistantMessageParam
+
+from .base import BaseMessageParser, _add_lang_to_source, _derive_key, _extract_text_from_content
+from .utils import detect_lang
+
+
+logger = get_logger(__name__)
+
+
+class AssistantParser(BaseMessageParser):
+    """Parser for assistant messages.
+
+    Handles multimodal assistant messages by creating one SourceMessage per content part.
+    Supports text and refusal content parts.
+    """
+
+    def __init__(self, embedder: BaseEmbedder, llm: BaseLLM | None = None):
+        """
+        Initialize AssistantParser.
+
+        Args:
+            embedder: Embedder for generating embeddings
+            llm: Optional LLM for fine mode processing
+        """
+        super().__init__(embedder, llm)
+
+    def create_source(
+        self,
+        message: ChatCompletionAssistantMessageParam,
+        info: dict[str, Any],
+    ) -> SourceMessage | list[SourceMessage]:
+        """
+        Create SourceMessage(s) from assistant message.
+
+        Handles:
+        - content: str | list of content parts (text/refusal) | None
+        - refusal: str | None (top-level refusal message)
+        - tool_calls: list of tool calls (when content is None)
+        - audio: Audio | None (audio response data)
+
+        For multimodal messages (content is a list), creates one SourceMessage per part.
+        For simple messages (content is str), creates a single SourceMessage.
+        """
+        if not isinstance(message, dict):
+            return []
+
+        role = message.get("role", "assistant")
+        raw_content = message.get("content")
+        refusal = message.get("refusal")
+        tool_calls = message.get("tool_calls")
+        audio = message.get("audio")
+        chat_time = message.get("chat_time")
+        message_id = message.get("message_id")
+
+        sources = []
+
+        if isinstance(raw_content, list):
+            # Multimodal: first collect all text content to detect overall language
+            text_contents = []
+            for part in raw_content:
+                if isinstance(part, dict):
+                    part_type = part.get("type", "")
+                    if part_type == "text":
+                        text_contents.append(part.get("text", ""))
+                    elif part_type == "refusal":
+                        text_contents.append(part.get("refusal", ""))
+
+            # Detect overall language from all text content
+            overall_lang = "en"  # default
+            if text_contents:
+                combined_text = " ".join(text_contents)
+                overall_lang = detect_lang(combined_text)
+            # Note: Assistant messages only support "text" and "refusal" part types
+            for part in raw_content:
+                if isinstance(part, dict):
+                    part_type = part.get("type", "")
+                    if part_type == "text":
+                        text_content = part.get("text", "")
+                        source = SourceMessage(
+                            type="chat",
+                            role=role,
+                            chat_time=chat_time,
+                            message_id=message_id,
+                            content=text_content,
+                        )
+                        source.lang = overall_lang
+                        sources.append(source)
+                    elif part_type == "refusal":
+                        refusal_content = part.get("refusal", "")
+                        source = SourceMessage(
+                            type="refusal",
+                            role=role,
+                            chat_time=chat_time,
+                            message_id=message_id,
+                            content=refusal_content,
+                        )
+                        source.lang = overall_lang
+                        sources.append(source)
+                    else:
+                        # Unknown part type - log warning but still create SourceMessage
+                        logger.warning(
+                            f"[AssistantParser] Unknown part type `{part_type}`. "
+                            f"Expected `text` or `refusal`. Creating SourceMessage with placeholder content."
+                        )
+                        source = SourceMessage(
+                            type="chat",
+                            role=role,
+                            chat_time=chat_time,
+                            message_id=message_id,
+                            content=f"[{part_type}]",
+                        )
+                        source.lang = overall_lang
+                        sources.append(source)
+        elif raw_content is not None:
+            # Simple message: single SourceMessage
+            content = _extract_text_from_content(raw_content)
+            if content:
+                source = SourceMessage(
+                    type="chat",
+                    role=role,
+                    chat_time=chat_time,
+                    message_id=message_id,
+                    content=content,
+                )
+                sources.append(_add_lang_to_source(source, content))
+
+        # Handle top-level refusal field
+        if refusal:
+            source = SourceMessage(
+                type="refusal",
+                role=role,
+                chat_time=chat_time,
+                message_id=message_id,
+                content=refusal,
+            )
+            # Use overall_lang if we have sources from multimodal content, otherwise detect
+            if sources and hasattr(sources[0], "lang"):
+                source.lang = sources[0].lang
+            else:
+                source = _add_lang_to_source(source, refusal)
+            sources.append(source)
+
+        # Handle tool_calls (when content is None or empty)
+        if tool_calls:
+            tool_calls_str = (
+                json.dumps(tool_calls, ensure_ascii=False)
+                if isinstance(tool_calls, list | dict)
+                else str(tool_calls)
+            )
+            source = SourceMessage(
+                type="tool_calls",
+                role=role,
+                chat_time=chat_time,
+                message_id=message_id,
+                content=f"[tool_calls]: {tool_calls_str}",
+            )
+            # Use overall_lang if we have sources from multimodal content, otherwise default
+            if sources and hasattr(sources[0], "lang"):
+                source.lang = sources[0].lang
+            else:
+                source = _add_lang_to_source(source, None)
+            sources.append(source)
+
+        # Handle audio (optional)
+        if audio:
+            audio_id = audio.get("id", "") if isinstance(audio, dict) else str(audio)
+            source = SourceMessage(
+                type="audio",
+                role=role,
+                chat_time=chat_time,
+                message_id=message_id,
+                content=f"[audio]: {audio_id}",
+            )
+            # Use overall_lang if we have sources from multimodal content, otherwise default
+            if sources and hasattr(sources[0], "lang"):
+                source.lang = sources[0].lang
+            else:
+                source = _add_lang_to_source(source, None)
+            sources.append(source)
+
+        if not sources:
+            return _add_lang_to_source(SourceMessage(type="chat", role=role), None)
+        if len(sources) > 1:
+            return sources
+        return sources[0]
+
+    def rebuild_from_source(
+        self,
+        source: SourceMessage,
+    ) -> ChatCompletionAssistantMessageParam:
+        """We only need rebuild from specific multimodal source"""
+
+    def parse_fast(
+        self,
+        message: ChatCompletionAssistantMessageParam,
+        info: dict[str, Any],
+        **kwargs,
+    ) -> list[TextualMemoryItem]:
+        if not isinstance(message, dict):
+            logger.warning(f"[AssistantParser] Expected dict, got {type(message)}")
+            return []
+
+        role = message.get("role", "")
+        raw_content = message.get("content")
+        refusal = message.get("refusal")
+        tool_calls = message.get("tool_calls")
+        audio = message.get("audio")
+        chat_time = message.get("chat_time", None)
+
+        if role != "assistant":
+            logger.warning(f"[AssistantParser] Expected role is `assistant`, got {role}")
+            return []
+
+        # Build content string from various sources
+        content_parts = []
+
+        # Extract content (can be str, list, or None)
+        if raw_content is not None:
+            extracted_content = _extract_text_from_content(raw_content)
+            if extracted_content:
+                content_parts.append(extracted_content)
+
+        # Add top-level refusal if present
+        if refusal:
+            content_parts.append(f"[refusal]: {refusal}")
+
+        # Add tool_calls if present (when content is None or empty)
+        if tool_calls:
+            tool_calls_str = (
+                json.dumps(tool_calls, ensure_ascii=False)
+                if isinstance(tool_calls, list | dict)
+                else str(tool_calls)
+            )
+            content_parts.append(f"[tool_calls]: {tool_calls_str}")
+
+        # Add audio if present
+        if audio:
+            audio_id = audio.get("id", "") if isinstance(audio, dict) else str(audio)
+            content_parts.append(f"[audio]: {audio_id}")
+
+        # Combine all content parts
+        content = " ".join(content_parts) if content_parts else ""
+
+        # If content is empty but we have tool_calls, audio, or refusal, still create memory
+        if not content and not tool_calls and not audio and not refusal:
+            return []
+
+        parts = [f"{role}: "]
+        if chat_time:
+            parts.append(f"[{chat_time}]: ")
+        prefix = "".join(parts)
+        line = f"{prefix}{content}\n"
+        if not line.strip():
+            return []
+        memory_type = "LongTermMemory"
+
+        # Create source(s) using parser's create_source method
+        sources = self.create_source(message, info)
+        if isinstance(sources, SourceMessage):
+            sources = [sources]
+        elif not sources:
+            return []
+
+        # Extract info fields
+        info_ = info.copy()
+        user_id = info_.pop("user_id", "")
+        session_id = info_.pop("session_id", "")
+
+        # Create memory item (equivalent to _make_memory_item)
+        memory_item = TextualMemoryItem(
+            memory=line,
+            metadata=TreeNodeTextualMemoryMetadata(
+                user_id=user_id,
+                session_id=session_id,
+                memory_type=memory_type,
+                status="activated",
+                tags=["mode:fast"],
+                key=_derive_key(line),
+                embedding=self.embedder.embed([line])[0],
+                usage=[],
+                sources=sources,
+                background="",
+                confidence=0.99,
+                type="fact",
+                info=info_,
+            ),
+        )
+
+        return [memory_item]
+
+    def parse_fine(
+        self,
+        message: ChatCompletionAssistantMessageParam,
+        info: dict[str, Any],
+        **kwargs,
+    ) -> list[TextualMemoryItem]:
+        return []

memos/mem_reader/read_multi_modal/base.py

@@ -0,0 +1,273 @@
+"""Base parser interface for multi-model message parsing.
+
+This module defines the base interface for parsing different message types
+in both fast and fine modes.
+"""
+
+import re
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from memos import log
+from memos.memories.textual.item import (
+    SourceMessage,
+    TextualMemoryItem,
+    TreeNodeTextualMemoryMetadata,
+)
+
+from .utils import detect_lang, get_text_splitter
+
+
+logger = log.get_logger(__name__)
+
+
+def _derive_key(text: str, max_len: int = 80) -> str:
+    """Default key when without LLM: first max_len words."""
+    if not text:
+        return ""
+    sent = re.split(r"[。！？!?]\s*|\n", text.strip())[0]
+    return (sent[:max_len]).strip()
+
+
+def _extract_text_from_content(content: Any) -> str:
+    """
+    Extract text from message content.
+    Handles str, list of parts, or None.
+    """
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        texts = []
+        for part in content:
+            if isinstance(part, dict):
+                part_type = part.get("type", "")
+                if part_type == "text":
+                    texts.append(part.get("text", ""))
+                elif part_type == "file":
+                    file_info = part.get("file", {})
+                    texts.append(file_info.get("file_data") or file_info.get("filename", "[file]"))
+                else:
+                    texts.append(f"[{part_type}]")
+            else:
+                texts.append(str(part))
+        return " ".join(texts)
+    return str(content)
+
+
+def _add_lang_to_source(source: SourceMessage, content: str | None = None) -> SourceMessage:
+    """
+    Add lang field to SourceMessage based on content.
+
+    Args:
+        source: SourceMessage to add lang field to
+        content: Optional content text for language detection.
+                 If None, uses source.content
+
+    Returns:
+        SourceMessage with lang field added
+    """
+    if not hasattr(source, "lang") or getattr(source, "lang", None) is None:
+        text_for_detection = content or getattr(source, "content", None) or ""
+        lang = detect_lang(text_for_detection)
+        source.lang = lang
+    return source
+
+
+class BaseMessageParser(ABC):
+    """Base interface for message type parsers."""
+
+    def __init__(self, embedder, llm=None):
+        """
+        Initialize BaseMessageParser.
+
+        Args:
+            embedder: Embedder for generating embeddings
+            llm: Optional LLM for fine mode processing
+        """
+        self.embedder = embedder
+        self.llm = llm
+
+    @abstractmethod
+    def create_source(
+        self,
+        message: Any,
+        info: dict[str, Any],
+    ) -> SourceMessage | list[SourceMessage]:
+        """
+        Create SourceMessage(s) from the message.
+
+        Each parser decides how to create sources:
+        - Simple messages: return single SourceMessage
+        - Multimodal messages: return list of SourceMessage (one per part)
+
+        Args:
+            message: The message to create source from
+            info: Dictionary containing user_id and session_id
+
+        Returns:
+            SourceMessage or list of SourceMessage
+        """
+
+    @abstractmethod
+    def rebuild_from_source(
+        self,
+        source: SourceMessage,
+    ) -> Any:
+        """
+        Rebuild original message from SourceMessage.
+
+        Each parser knows how to reconstruct its own message type.
+
+        Args:
+            source: SourceMessage to rebuild from
+
+        Returns:
+            Rebuilt message in original format
+        """
+
+    def parse_fast(
+        self,
+        message: Any,
+        info: dict[str, Any],
+        **kwargs,
+    ) -> list[TextualMemoryItem]:
+        """
+        Default parse_fast implementation (equivalent to simple_struct fast mode).
+
+        Fast mode logic:
+        - Extract text content from message
+        - Determine memory_type based on role (UserMemory for user, LongTermMemory otherwise)
+        - Create TextualMemoryItem with tags=["mode:fast"]
+        - No LLM calls, quick processing
+
+        Subclasses can override this method for custom behavior.
+
+        Args:
+            message: The message to parse
+            info: Dictionary containing user_id and session_id
+            **kwargs: Additional parameters
+
+        Returns:
+            List of TextualMemoryItem objects
+        """
+        if not isinstance(message, dict):
+            logger.warning(f"[BaseParser] Expected dict, got {type(message)}")
+            return []
+
+        # Extract text content
+        content = _extract_text_from_content(message.get("content"))
+        if not content:
+            return []
+
+        # Determine memory_type based on role (equivalent to simple_struct logic)
+        role = message.get("role", "").strip().lower()
+        memory_type = "UserMemory" if role == "user" else "LongTermMemory"
+
+        # Create source(s) using parser's create_source method
+        sources = self.create_source(message, info)
+        if isinstance(sources, SourceMessage):
+            sources = [sources]
+        elif not sources:
+            return []
+
+        # Extract info fields
+        info_ = info.copy()
+        user_id = info_.pop("user_id", "")
+        session_id = info_.pop("session_id", "")
+
+        # Create memory item (equivalent to _make_memory_item)
+        memory_item = TextualMemoryItem(
+            memory=content,
+            metadata=TreeNodeTextualMemoryMetadata(
+                user_id=user_id,
+                session_id=session_id,
+                memory_type=memory_type,
+                status="activated",
+                tags=["mode:fast"],
+                key=_derive_key(content),
+                embedding=self.embedder.embed([content])[0],
+                usage=[],
+                sources=sources,
+                background="",
+                confidence=0.99,
+                type="fact",
+                info=info_,
+            ),
+        )
+
+        return [memory_item]
+
+    @abstractmethod
+    def parse_fine(
+        self,
+        message: Any,
+        info: dict[str, Any],
+        **kwargs,
+    ) -> list[TextualMemoryItem]:
+        """
+        Parse message in fine mode (with LLM calls for better understanding).
+
+        Args:
+            message: The message to parse
+            info: Dictionary containing user_id and session_id
+            **kwargs: Additional parameters (e.g., llm, embedder)
+
+        Returns:
+            List of TextualMemoryItem objects
+        """
+
+    def parse(
+        self,
+        message: Any,
+        info: dict[str, Any],
+        mode: str = "fast",
+        **kwargs,
+    ) -> list[TextualMemoryItem]:
+        """
+        Parse message in the specified mode.
+
+        Args:
+            message: The message to parse
+            info: Dictionary containing user_id and session_id
+            mode: "fast" or "fine"
+            **kwargs: Additional parameters
+
+        Returns:
+            List of TextualMemoryItem objects
+        """
+        if mode == "fast":
+            return self.parse_fast(message, info, **kwargs)
+        elif mode == "fine":
+            return self.parse_fine(message, info, **kwargs)
+        else:
+            raise ValueError(f"Unknown mode: {mode}. Must be 'fast' or 'fine'")
+
+    def _split_text(self, text: str, is_markdown: bool = False) -> list[str]:
+        """
+        Split text into chunks using text splitter from utils.
+
+        Args:
+            text: Text to split
+
+        Returns:
+            List of text chunks
+        """
+        if not text or not text.strip():
+            return []
+
+        splitter = get_text_splitter(is_markdown=is_markdown)
+        if not splitter:
+            # If text splitter is not available, return text as single chunk
+            return [text] if text.strip() else []
+
+        try:
+            chunks = splitter.chunk(text)
+            logger.debug(f"[FileContentParser] Split text into {len(chunks)} chunks")
+            return chunks
+        except Exception as e:
+            logger.error(f"[FileContentParser] Error splitting text: {e}")
+            # Fallback to single chunk
+            return [text] if text.strip() else []