MemoryOS 0.2.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

memos/mem_scheduler/modules/schemas.py

@@ -1,328 +0,0 @@
-import json
-
-from datetime import datetime
-from pathlib import Path
-from typing import ClassVar, NewType, TypeVar
-from uuid import uuid4
-
-from pydantic import BaseModel, Field, computed_field
-from typing_extensions import TypedDict
-
-from memos.log import get_logger
-from memos.mem_cube.general import GeneralMemCube
-
-
-logger = get_logger(__name__)
-
-
-FILE_PATH = Path(__file__).absolute()
-BASE_DIR = FILE_PATH.parent.parent.parent.parent.parent
-
-QUERY_LABEL = "query"
-ANSWER_LABEL = "answer"
-ADD_LABEL = "add"
-
-TreeTextMemory_SEARCH_METHOD = "tree_text_memory_search"
-TextMemory_SEARCH_METHOD = "text_memory_search"
-DIRECT_EXCHANGE_TYPE = "direct"
-FANOUT_EXCHANGE_TYPE = "fanout"
-DEFAULT_WORKING_MEM_MONITOR_SIZE_LIMIT = 20
-DEFAULT_ACTIVATION_MEM_MONITOR_SIZE_LIMIT = 5
-DEFAULT_ACT_MEM_DUMP_PATH = f"{BASE_DIR}/outputs/mem_scheduler/mem_cube_scheduler_test.kv_cache"
-DEFAULT_THREAD__POOL_MAX_WORKERS = 5
-DEFAULT_CONSUME_INTERVAL_SECONDS = 3
-NOT_INITIALIZED = -1
-BaseModelType = TypeVar("T", bound="BaseModel")
-
-# web log
-LONG_TERM_MEMORY_TYPE = "LongTermMemory"
-USER_MEMORY_TYPE = "UserMemory"
-WORKING_MEMORY_TYPE = "WorkingMemory"
-TEXT_MEMORY_TYPE = "TextMemory"
-ACTIVATION_MEMORY_TYPE = "ActivationMemory"
-PARAMETER_MEMORY_TYPE = "ParameterMemory"
-USER_INPUT_TYPE = "UserInput"
-NOT_APPLICABLE_TYPE = "NotApplicable"
-
-# monitors
-MONITOR_WORKING_MEMORY_TYPE = "MonitorWorkingMemoryType"
-MONITOR_ACTIVATION_MEMORY_TYPE = "MonitorActivationMemoryType"
-
-
-# new types
-UserID = NewType("UserID", str)
-MemCubeID = NewType("CubeID", str)
-
-
-# ************************* Public *************************
-class DictConversionMixin:
-    def to_dict(self) -> dict:
-        """Convert the instance to a dictionary."""
-        return {
-            **self.model_dump(),  # 替换 self.dict()
-            "timestamp": self.timestamp.isoformat() if hasattr(self, "timestamp") else None,
-        }
-
-    @classmethod
-    def from_dict(cls: type[BaseModelType], data: dict) -> BaseModelType:
-        """Create an instance from a dictionary."""
-        if "timestamp" in data:
-            data["timestamp"] = datetime.fromisoformat(data["timestamp"])
-        return cls(**data)
-
-    def __str__(self) -> str:
-        """Convert the instance to a JSON string with indentation of 4 spaces.
-        This will be used when str() or print() is called on the instance.
-
-        Returns:
-            str: A JSON string representation of the instance with 4-space indentation.
-        """
-        return json.dumps(
-            self.to_dict(),
-            indent=4,
-            ensure_ascii=False,
-            default=str,  # 处理无法序列化的对象
-        )
-
-    class Config:
-        json_encoders: ClassVar[dict[type, object]] = {datetime: lambda v: v.isoformat()}
-
-
-# ************************* Messages *************************
-class ScheduleMessageItem(BaseModel, DictConversionMixin):
-    item_id: str = Field(description="uuid", default_factory=lambda: str(uuid4()))
-    user_id: str = Field(..., description="user id")
-    mem_cube_id: str = Field(..., description="memcube id")
-    label: str = Field(..., description="Label of the schedule message")
-    mem_cube: GeneralMemCube | str = Field(..., description="memcube for schedule")
-    content: str = Field(..., description="Content of the schedule message")
-    timestamp: datetime = Field(
-        default_factory=datetime.now, description="submit time for schedule_messages"
-    )
-
-    class Config:
-        arbitrary_types_allowed = True
-        json_encoders: ClassVar[dict[type, object]] = {
-            datetime: lambda v: v.isoformat(),
-            GeneralMemCube: lambda v: f"<GeneralMemCube:{id(v)}>",
-        }
-
-    def to_dict(self) -> dict:
-        """Convert model to dictionary suitable for Redis Stream"""
-        return {
-            "item_id": self.item_id,
-            "user_id": self.user_id,
-            "cube_id": self.mem_cube_id,
-            "label": self.label,
-            "cube": "Not Applicable",  # Custom cube serialization
-            "content": self.content,
-            "timestamp": self.timestamp.isoformat(),
-        }
-
-    @classmethod
-    def from_dict(cls, data: dict) -> "ScheduleMessageItem":
-        """Create model from Redis Stream dictionary"""
-        return cls(
-            item_id=data.get("item_id", str(uuid4())),
-            user_id=data["user_id"],
-            cube_id=data["cube_id"],
-            label=data["label"],
-            cube="Not Applicable",  # Custom cube deserialization
-            content=data["content"],
-            timestamp=datetime.fromisoformat(data["timestamp"]),
-        )
-
-
-class MemorySizes(TypedDict):
-    long_term_memory_size: int
-    user_memory_size: int
-    working_memory_size: int
-    transformed_act_memory_size: int
-
-
-class MemoryCapacities(TypedDict):
-    long_term_memory_capacity: int
-    user_memory_capacity: int
-    working_memory_capacity: int
-    transformed_act_memory_capacity: int
-
-
-DEFAULT_MEMORY_SIZES = {
-    "long_term_memory_size": NOT_INITIALIZED,
-    "user_memory_size": NOT_INITIALIZED,
-    "working_memory_size": NOT_INITIALIZED,
-    "transformed_act_memory_size": NOT_INITIALIZED,
-    "parameter_memory_size": NOT_INITIALIZED,
-}
-
-DEFAULT_MEMORY_CAPACITIES = {
-    "long_term_memory_capacity": 10000,
-    "user_memory_capacity": 10000,
-    "working_memory_capacity": 20,
-    "transformed_act_memory_capacity": NOT_INITIALIZED,
-    "parameter_memory_capacity": NOT_INITIALIZED,
-}
-
-
-class ScheduleLogForWebItem(BaseModel, DictConversionMixin):
-    item_id: str = Field(
-        description="Unique identifier for the log entry", default_factory=lambda: str(uuid4())
-    )
-    user_id: str = Field(..., description="Identifier for the user associated with the log")
-    mem_cube_id: str = Field(
-        ..., description="Identifier for the memcube associated with this log entry"
-    )
-    label: str = Field(..., description="Label categorizing the type of log")
-    from_memory_type: str = Field(..., description="Source memory type")
-    to_memory_type: str = Field(..., description="Destination memory type")
-    log_content: str = Field(..., description="Detailed content of the log entry")
-    current_memory_sizes: MemorySizes = Field(
-        default_factory=lambda: dict(DEFAULT_MEMORY_SIZES),
-        description="Current utilization of memory partitions",
-    )
-    memory_capacities: MemoryCapacities = Field(
-        default_factory=lambda: dict(DEFAULT_MEMORY_CAPACITIES),
-        description="Maximum capacities of memory partitions",
-    )
-    timestamp: datetime = Field(
-        default_factory=datetime.now,
-        description="Timestamp indicating when the log entry was created",
-    )
-
-
-# ************************* Monitor *************************
-class MemoryMonitorItem(BaseModel, DictConversionMixin):
-    item_id: str = Field(
-        description="Unique identifier for the memory item", default_factory=lambda: str(uuid4())
-    )
-    memory_text: str = Field(
-        ...,
-        description="The actual content of the memory",
-        min_length=1,
-        max_length=10000,  # Prevent excessively large memory texts
-    )
-    importance_score: float = Field(
-        default=NOT_INITIALIZED,
-        description="Numerical score representing the memory's importance",
-        ge=NOT_INITIALIZED,  # Minimum value of 0
-    )
-    recording_count: int = Field(
-        default=1,
-        description="How many times this memory has been recorded",
-        ge=1,  # Greater than or equal to 1
-    )
-
-    def get_score(self) -> float:
-        """
-        Calculate the effective score for the memory item.
-
-        Returns:
-            float: The importance_score if it has been initialized (>=0),
-                   otherwise the recording_count converted to float.
-
-        Note:
-            This method provides a unified way to retrieve a comparable score
-            for memory items, regardless of whether their importance has been explicitly set.
-        """
-        if self.importance_score == NOT_INITIALIZED:
-            # Return recording_count as float when importance_score is not initialized
-            return float(self.recording_count)
-        else:
-            # Return the initialized importance_score
-            return self.importance_score
-
-
-class MemoryMonitorManager(BaseModel, DictConversionMixin):
-    user_id: str = Field(..., description="Required user identifier", min_length=1)
-    mem_cube_id: str = Field(..., description="Required memory cube identifier", min_length=1)
-    memories: list[MemoryMonitorItem] = Field(
-        default_factory=list, description="Collection of memory items"
-    )
-    max_capacity: int | None = Field(
-        default=None, description="Maximum number of memories allowed (None for unlimited)", ge=1
-    )
-
-    @computed_field
-    @property
-    def memory_size(self) -> int:
-        """Automatically calculated count of memory items."""
-        return len(self.memories)
-
-    def update_memories(
-        self, text_working_memories: list[str], partial_retention_number: int
-    ) -> MemoryMonitorItem:
-        """
-        Update memories based on text_working_memories.
-
-        Args:
-            text_working_memories: List of memory texts to update
-            partial_retention_number: Number of top memories to keep by recording count
-
-        Returns:
-            List of added or updated MemoryMonitorItem instances
-        """
-
-        # Validate partial_retention_number
-        if partial_retention_number < 0:
-            raise ValueError("partial_retention_number must be non-negative")
-
-        # Create text lookup set
-        working_memory_set = set(text_working_memories)
-
-        # Step 1: Update existing memories or add new ones
-        added_or_updated = []
-        memory_text_map = {item.memory_text: item for item in self.memories}
-
-        for text in text_working_memories:
-            if text in memory_text_map:
-                # Update existing memory
-                memory = memory_text_map[text]
-                memory.recording_count += 1
-                added_or_updated.append(memory)
-            else:
-                # Add new memory
-                new_memory = MemoryMonitorItem(memory_text=text, recording_count=1)
-                self.memories.append(new_memory)
-                added_or_updated.append(new_memory)
-
-        # Step 2: Identify memories to remove
-        # Sort memories by recording_count in descending order
-        sorted_memories = sorted(self.memories, key=lambda item: item.recording_count, reverse=True)
-
-        # Keep the top N memories by recording_count
-        records_to_keep = {
-            memory.memory_text for memory in sorted_memories[:partial_retention_number]
-        }
-
-        # Collect memories to remove: not in current working memory and not in top N
-        memories_to_remove = [
-            memory
-            for memory in self.memories
-            if memory.memory_text not in working_memory_set
-            and memory.memory_text not in records_to_keep
-        ]
-
-        # Step 3: Remove identified memories
-        for memory in memories_to_remove:
-            self.memories.remove(memory)
-
-        # Step 4: Enforce max_capacity if set
-        if self.max_capacity is not None and len(self.memories) > self.max_capacity:
-            # Sort by importance and then recording count
-            sorted_memories = sorted(
-                self.memories,
-                key=lambda item: (item.importance_score, item.recording_count),
-                reverse=True,
-            )
-            # Keep only the top max_capacity memories
-            self.memories = sorted_memories[: self.max_capacity]
-
-        # Log the update result
-        logger.info(
-            f"Updated monitor manager for user {self.user_id}, mem_cube {self.mem_cube_id}: "
-            f"Total memories: {len(self.memories)}, "
-            f"Added/Updated: {len(added_or_updated)}, "
-            f"Removed: {len(memories_to_remove)} (excluding top {partial_retention_number} by recording_count)"
-        )
-
-        return added_or_updated

memos/mem_scheduler/utils.py

@@ -1,75 +0,0 @@
-import json
-import re
-
-from pathlib import Path
-
-import yaml
-
-
-def extract_json_dict(text: str):
-    text = text.strip()
-    patterns_to_remove = ["json```", "```json", "latex```", "```latex", "```"]
-    for pattern in patterns_to_remove:
-        text = text.replace(pattern, "")
-    res = json.loads(text.strip())
-    return res
-
-
-def transform_name_to_key(name):
-    """
-    Normalize text by removing all punctuation marks, keeping only letters, numbers, and word characters.
-
-    Args:
-        name (str): Input text to be processed
-
-    Returns:
-        str: Processed text with all punctuation removed
-    """
-    # Match all characters that are NOT:
-    # \w - word characters (letters, digits, underscore)
-    # \u4e00-\u9fff - Chinese/Japanese/Korean characters
-    # \s - whitespace
-    pattern = r"[^\w\u4e00-\u9fff\s]"
-
-    # Substitute all matched punctuation marks with empty string
-    # re.UNICODE flag ensures proper handling of Unicode characters
-    normalized = re.sub(pattern, "", name, flags=re.UNICODE)
-
-    # Optional: Collapse multiple whitespaces into single space
-    normalized = "_".join(normalized.split())
-
-    normalized = normalized.lower()
-
-    return normalized
-
-
-def parse_yaml(yaml_file):
-    yaml_path = Path(yaml_file)
-    yaml_path = Path(yaml_file)
-    if not yaml_path.is_file():
-        raise FileNotFoundError(f"No such file: {yaml_file}")
-
-    with yaml_path.open("r", encoding="utf-8") as fr:
-        data = yaml.safe_load(fr)
-
-    return data
-
-
-def is_all_english(input_string: str) -> bool:
-    """Determine if the string consists entirely of English characters (including spaces)"""
-    return all(char.isascii() or char.isspace() for char in input_string)
-
-
-def is_all_chinese(input_string: str) -> bool:
-    """Determine if the string consists entirely of Chinese characters (including Chinese punctuation and spaces)"""
-    return all(
-        ("\u4e00" <= char <= "\u9fff")  # Basic Chinese characters
-        or ("\u3400" <= char <= "\u4dbf")  # Extension A
-        or ("\u20000" <= char <= "\u2a6df")  # Extension B
-        or ("\u2a700" <= char <= "\u2b73f")  # Extension C
-        or ("\u2b740" <= char <= "\u2b81f")  # Extension D
-        or ("\u2b820" <= char <= "\u2ceaf")  # Extension E
-        or ("\u2f800" <= char <= "\u2fa1f")  # Extension F
-        or char.isspace()  # Spaces
-        for char in input_string
-    )

memos/memories/textual/tree_text_memory/organize/redundancy.py

@@ -1,193 +0,0 @@
-import json
-import re
-
-from datetime import datetime
-
-from memos.embedders.base import BaseEmbedder
-from memos.graph_dbs.neo4j import Neo4jGraphDB
-from memos.llms.base import BaseLLM
-from memos.log import get_logger
-from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata
-from memos.templates.tree_reorganize_prompts import (
-    REDUNDANCY_DETECTOR_PROMPT,
-    REDUNDANCY_MERGE_PROMPT,
-    REDUNDANCY_RESOLVER_PROMPT,
-)
-
-
-logger = get_logger(__name__)
-
-
-class RedundancyHandler:
-    EMBEDDING_THRESHOLD: float = 0.8  # Threshold for embedding similarity to consider redundancy
-
-    def __init__(self, graph_store: Neo4jGraphDB, llm: BaseLLM, embedder: BaseEmbedder):
-        self.graph_store = graph_store
-        self.llm = llm
-        self.embedder = embedder
-
-    def detect(
-        self, memory: TextualMemoryItem, top_k: int = 5, scope: str | None = None
-    ) -> list[tuple[TextualMemoryItem, TextualMemoryItem]]:
-        """
-        Detect redundancy by finding the most similar items in the graph database based on embedding, then use LLM to judge redundancy.
-        Args:
-            memory: The memory item (should have an embedding attribute or field).
-            top_k: Number of top similar nodes to retrieve.
-            scope: Optional memory type filter.
-        Returns:
-            List of redundancy pairs (each pair is a tuple: (memory, candidate)).
-        """
-        # 1. Search for similar memories based on embedding
-        embedding = memory.metadata.embedding
-        embedding_candidates_info = self.graph_store.search_by_embedding(
-            embedding, top_k=top_k, scope=scope
-        )
-        # 2. Filter based on similarity threshold
-        embedding_candidates_ids = [
-            info["id"]
-            for info in embedding_candidates_info
-            if info["score"] >= self.EMBEDDING_THRESHOLD and info["id"] != memory.id
-        ]
-        # 3. Judge redundancys using LLM
-        embedding_candidates = self.graph_store.get_nodes(embedding_candidates_ids)
-        redundant_pairs = []
-        for embedding_candidate in embedding_candidates:
-            embedding_candidate = TextualMemoryItem.from_dict(embedding_candidate)
-            prompt = [
-                {
-                    "role": "system",
-                    "content": "You are a redundancy detector for memory items.",
-                },
-                {
-                    "role": "user",
-                    "content": REDUNDANCY_DETECTOR_PROMPT.format(
-                        statement_1=memory.memory,
-                        statement_2=embedding_candidate.memory,
-                    ),
-                },
-            ]
-            result = self.llm.generate(prompt).strip()
-            if "yes" in result.lower():
-                redundant_pairs.append([memory, embedding_candidate])
-        if len(redundant_pairs):
-            redundant_text = "\n".join(
-                f'"{pair[0].memory!s}" <==REDUNDANCY==> "{pair[1].memory!s}"'
-                for pair in redundant_pairs
-            )
-            logger.warning(
-                f"Detected {len(redundant_pairs)} redundancies for memory {memory.id}\n {redundant_text}"
-            )
-        return redundant_pairs
-
-    def resolve_two_nodes(self, memory_a: TextualMemoryItem, memory_b: TextualMemoryItem) -> None:
-        """
-        Resolve detected redundancies between two memory items using LLM fusion.
-        Args:
-            memory_a: The first redundant memory item.
-            memory_b: The second redundant memory item.
-        Returns:
-            A fused TextualMemoryItem representing the resolved memory.
-        """
-        return  # waiting for implementation
-        # ———————————— 1. LLM generate fused memory ————————————
-        metadata_for_resolve = ["key", "background", "confidence", "updated_at"]
-        metadata_1 = memory_a.metadata.model_dump_json(include=metadata_for_resolve)
-        metadata_2 = memory_b.metadata.model_dump_json(include=metadata_for_resolve)
-        prompt = [
-            {
-                "role": "system",
-                "content": "",
-            },
-            {
-                "role": "user",
-                "content": REDUNDANCY_RESOLVER_PROMPT.format(
-                    statement_1=memory_a.memory,
-                    metadata_1=metadata_1,
-                    statement_2=memory_b.memory,
-                    metadata_2=metadata_2,
-                ),
-            },
-        ]
-        response = self.llm.generate(prompt).strip()
-
-        # ———————————— 2. Parse the response ————————————
-        try:
-            answer = re.search(r"<answer>(.*?)</answer>", response, re.DOTALL)
-            answer = answer.group(1).strip()
-            fixed_metadata = self._merge_metadata(answer, memory_a.metadata, memory_b.metadata)
-            merged_memory = TextualMemoryItem(memory=answer, metadata=fixed_metadata)
-            logger.info(f"Resolved result: {merged_memory}")
-            self._resolve_in_graph(memory_a, memory_b, merged_memory)
-        except json.decoder.JSONDecodeError:
-            logger.error(f"Failed to parse LLM response: {response}")
-
-    def resolve_one_node(self, memory: TextualMemoryItem) -> None:
-        prompt = [
-            {
-                "role": "user",
-                "content": REDUNDANCY_MERGE_PROMPT.format(merged_text=memory.memory),
-            },
-        ]
-        response = self.llm.generate(prompt)
-        memory.memory = response.strip()
-        self.graph_store.update_node(
-            memory.id,
-            {"memory": memory.memory, **memory.metadata.model_dump(exclude_none=True)},
-        )
-        logger.debug(f"Merged memory: {memory.memory}")
-
-    def _resolve_in_graph(
-        self,
-        redundant_a: TextualMemoryItem,
-        redundant_b: TextualMemoryItem,
-        merged: TextualMemoryItem,
-    ):
-        edges_a = self.graph_store.get_edges(redundant_a.id, type="ANY", direction="ANY")
-        edges_b = self.graph_store.get_edges(redundant_b.id, type="ANY", direction="ANY")
-        all_edges = edges_a + edges_b
-
-        self.graph_store.add_node(
-            merged.id, merged.memory, merged.metadata.model_dump(exclude_none=True)
-        )
-
-        for edge in all_edges:
-            new_from = (
-                merged.id if edge["from"] in (redundant_a.id, redundant_b.id) else edge["from"]
-            )
-            new_to = merged.id if edge["to"] in (redundant_a.id, redundant_b.id) else edge["to"]
-            if new_from == new_to:
-                continue
-            # Check if the edge already exists before adding
-            if not self.graph_store.edge_exists(new_from, new_to, edge["type"], direction="ANY"):
-                self.graph_store.add_edge(new_from, new_to, edge["type"])
-
-        self.graph_store.update_node(redundant_a.id, {"status": "archived"})
-        self.graph_store.update_node(redundant_b.id, {"status": "archived"})
-        self.graph_store.add_edge(redundant_a.id, merged.id, type="MERGED_TO")
-        self.graph_store.add_edge(redundant_b.id, merged.id, type="MERGED_TO")
-        logger.debug(
-            f"Archive {redundant_a.id} and {redundant_b.id}, and inherit their edges to {merged.id}."
-        )
-
-    def _merge_metadata(
-        self,
-        memory: str,
-        metadata_a: TreeNodeTextualMemoryMetadata,
-        metadata_b: TreeNodeTextualMemoryMetadata,
-    ) -> TreeNodeTextualMemoryMetadata:
-        metadata_1 = metadata_a.model_dump()
-        metadata_2 = metadata_b.model_dump()
-        merged_metadata = {
-            "sources": (metadata_1["sources"] or []) + (metadata_2["sources"] or []),
-            "embedding": self.embedder.embed([memory])[0],
-            "update_at": datetime.now().isoformat(),
-            "created_at": datetime.now().isoformat(),
-        }
-        for key in metadata_1:
-            if key in merged_metadata:
-                continue
-            merged_metadata[key] = (
-                metadata_1[key] if metadata_1[key] is not None else metadata_2[key]
-            )
-        return TreeNodeTextualMemoryMetadata.model_validate(merged_metadata)