kweaver-dolphin 0.1.0__py3-none-any.whl

dolphin/lib/memory/llm_calls.py

@@ -0,0 +1,195 @@
+"""
+LLM Call abstractions for the memory system.
+
+This module provides abstract interfaces for LLM operations in the memory system,
+including prompt assembly, post-processing, and formatting.
+"""
+
+import json
+from datetime import datetime
+from typing import List, Any
+
+from dolphin.core.llm.llm_call import LLMCall
+
+from dolphin.core.common.enums import KnowledgePoint, MessageRole, Messages
+from dolphin.core.logging.logger import get_logger
+from dolphin.lib.memory.utils import validate_knowledge_point
+
+logger = get_logger("mem")
+
+
+class KnowledgeExtractionCall(LLMCall):
+    """LLM call for extracting knowledge from conversations."""
+
+    def execute(
+        self, conversation_history: Messages, user_id: str
+    ) -> List[KnowledgePoint]:
+        """Execute knowledge extraction for a specific conversation and user."""
+        if len(conversation_history.get_messages()) == 0:
+            return []
+
+        return super().execute(
+            llm_args={"knowledge_extraction": True},
+            conversation_history=conversation_history,
+            user_id=user_id,
+        )
+
+    def _log(self, time_cost: float, **kwargs) -> str:
+        """Log the execution result."""
+        logger.info(
+            f"Knowledge extraction execution user_id[{kwargs.get('user_id', 'unknown')}] time[{time_cost} seconds]"
+        )
+
+    def _build_prompt(self, conversation_history: Messages, **kwargs) -> str:
+        """Build prompt for knowledge extraction."""
+        conversation_text = ""
+        for msg in conversation_history.get_messages():
+            if msg.role == MessageRole.SYSTEM:
+                continue
+
+            conversation_text += f"{msg.role.value}: {msg.content}\n"
+
+        """You are a professional knowledge management expert. Your task is to analyze the following conversation and extract valuable knowledge. The knowledge should be categorized into one of the following three types:
+
+        1. **WorldModel**: Facts about the external world, ontologies, data sources, or user-provided correction information.
+        2. **ExperientialKnowledge**: Methodologies, strategies, or lessons learned from agent task execution.
+        3. **OtherKnowledge**: Any other information worth remembering, such as user preferences or specific details.
+
+        For each piece of knowledge, provide a concise summary, its type, and a relevance score from 0 to 100 indicating its importance for future interactions.
+
+        Notes:
+        1. Do not include information related to specific database data, as this information is volatile.
+        2. Knowledge points should have complete predicates, not just single nouns.
+
+        Please output your findings strictly in JSONL format, with one JSON object per line. Each object should contain the fields: content, type, score, metadata. Do not include any other text or explanations.
+
+        Conversation history:
+        {conversation_text}
+        """
+
+    def _post_process(
+        self, llm_output: str, user_id: str, **kwargs
+    ) -> List[KnowledgePoint]:
+        """Parse and validate LLM extraction result."""
+        knowledge_points = []
+
+        for line in llm_output.strip().split("\n"):
+            line = line.strip()
+            if not line:
+                continue
+
+            try:
+                raw_point = json.loads(line)
+                # Add user_id and timestamp to metadata
+                raw_point["user_id"] = user_id
+                if "metadata" not in raw_point:
+                    raw_point["metadata"] = {}
+                raw_point["metadata"]["extraction_time"] = datetime.now().isoformat()
+
+                # Validate and convert
+                validated_point = validate_knowledge_point(
+                    KnowledgePoint(**raw_point), user_id
+                )
+                knowledge_points.append(validated_point)
+
+            except Exception as e:
+                logger.warning(f"Failed to parse knowledge point: {line} - {e}")
+                continue
+
+        return knowledge_points
+
+
+class KnowledgeMergeCall(LLMCall):
+    """LLM call for merging and deduplicating knowledge."""
+
+    def execute(
+        self, all_knowledge_points: List[KnowledgePoint]
+    ) -> List[KnowledgePoint]:
+        """Execute knowledge merging for a list of knowledge points."""
+        return super().execute(
+            llm_args={"knowledge_extraction": True},
+            all_knowledge_points=all_knowledge_points,
+        )
+
+    def _log(self, time_cost: float, **kwargs) -> str:
+        """Log the execution result."""
+        logger.info(
+            f"Knowledge merge execution user_id[{kwargs.get('user_id', 'unknown')}] time[{time_cost} seconds]"
+        )
+
+    def _no_merge_result(self, **kwargs) -> Any:
+        """No merge result."""
+        return kwargs.get("all_knowledge_points", None)
+
+    def _build_prompt(self, all_knowledge_points: List[KnowledgePoint]) -> str:
+        """Build prompt for knowledge merging."""
+        if len(all_knowledge_points) <= 1:
+            return ""  # No need to merge if only one or zero points
+
+        knowledge_json = json.dumps(all_knowledge_points, indent=2, ensure_ascii=False)
+
+        """You are a knowledge management expert. Please analyze the following knowledge points and perform intelligent merging:
+
+        1. Identify similar, redundant, or mergeable knowledge points
+        2. Merge related knowledge points into more comprehensive content
+        3. Keep unique and valuable knowledge points unchanged
+        4. Ensure no important information is lost during merging
+        5. Update scores based on综合 importance and reliability
+
+        Current knowledge points:
+        {knowledge_json}
+
+        Return the optimized knowledge base as a JSON array, where each element follows the KnowledgePoint format.
+        Prioritize quality over quantity — fewer but higher-quality knowledge points are better.
+
+        Return only the JSON array, without any additional text.
+        """
+
+    def _post_process(
+        self, llm_output: str, all_knowledge_points: List[KnowledgePoint]
+    ) -> List[KnowledgePoint]:
+        """Parse and validate LLM merge result."""
+        if len(all_knowledge_points) <= 1:
+            return all_knowledge_points
+
+        try:
+            merged_knowledge = json.loads(llm_output)
+
+            # Validate each point and ensure user_id consistency
+            user_id = (
+                all_knowledge_points[0]["user_id"] if all_knowledge_points else None
+            )
+            validated_knowledge = []
+
+            for point in merged_knowledge:
+                try:
+                    validated_point = validate_knowledge_point(point, user_id)
+                    validated_knowledge.append(validated_point)
+                except Exception as e:
+                    logger.warning(f"Invalid merged knowledge point: {e}")
+                    continue
+
+            return validated_knowledge
+
+        except Exception as e:
+            # Fallback: return original knowledge if LLM output is invalid
+            logger.warning(f"LLM merge failed, keeping original knowledge: {e}")
+            return self._simple_deduplication(all_knowledge_points)
+
+    def _simple_deduplication(
+        self, knowledge_points: List[KnowledgePoint]
+    ) -> List[KnowledgePoint]:
+        """Simple deduplication as fallback."""
+        seen_content = set()
+        deduplicated = []
+
+        # Sort by score to keep highest scored items
+        sorted_points = sorted(knowledge_points, key=lambda p: p["score"], reverse=True)
+
+        for point in sorted_points:
+            content_key = point["content"].strip().lower()
+            if content_key not in seen_content:
+                seen_content.add(content_key)
+                deduplicated.append(point)
+
+        return deduplicated

dolphin/lib/memory/manager.py

@@ -0,0 +1,78 @@
+"""
+Memory Manager - the main orchestrator for the memory system.
+"""
+
+from typing import List, Optional
+
+from dolphin.core.common.enums import KnowledgePoint
+from .storage import MemoryFileSys
+from dolphin.core.config.global_config import GlobalConfig
+from dolphin.core.logging.logger import get_logger
+
+logger = get_logger("mem")
+
+
+class MemoryManager:
+    """The main orchestrator for the memory system - focused on complex knowledge management."""
+
+    def __init__(self, global_config: GlobalConfig):
+        """
+        Initializes the MemoryManager.
+
+        :param global_config: Global configuration instance, will create default if None.
+        """
+        self.global_config = global_config
+        self.memory_config = global_config.memory_config
+
+        self.memory_storage = MemoryFileSys(self.memory_config.storage_path)
+
+    def retrieve_relevant_memory(
+        self, context, user_id: str, query: str = None, top_k: Optional[int] = None
+    ) -> List[KnowledgePoint]:
+        """
+        Retrieves knowledge relevant to a given query for a specific user to be injected into context.
+
+        :param context: Context instance for accessing skillkits
+        :param user_id: The user whose knowledge should be retrieved
+        :param query: Query string (not currently used in this implementation)
+        :param top_k: Number of knowledge points to return
+        """
+        if top_k is None:
+            top_k = self.memory_config.default_top_k
+
+        if context.get_cur_agent() is None:
+            return []
+
+        try:
+            memory_skill_result = context.get_skillkit().exec(
+                "_read_memory",
+                agent_name=context.get_cur_agent().get_name(),
+                user_id=user_id,
+            )
+            if not memory_skill_result:
+                return []
+
+            memory_items = memory_skill_result.result
+            if not memory_items:
+                return []
+
+            converted_points = []
+            for item in memory_items:
+                try:
+                    point = KnowledgePoint(
+                        content=item.get("content", ""),
+                        score=item.get("score", 50),
+                        user_id=item.get("user_id", ""),
+                    )
+                    converted_points.append(point)
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to convert dialog log to KnowledgePoint: {e}"
+                    )
+                    continue
+
+            return converted_points[:top_k]
+
+        except Exception as e:
+            logger.error(f"Failed to retrieve knowledge for user {user_id}: {e}")
+            return []

dolphin/lib/memory/sandbox.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+class MemorySandbox:
+    """Session-scoped filesystem sandbox for memory persistence.
+
+    - Only allows relative paths under `<storage_base>/memories/<session_id>/`.
+    - Only allows `.json` files.
+    - Provides simple size checks.
+    """
+
+    MAX_SIZE = 10 * 1024 * 1024  # 10MB
+    MAX_PATH_LENGTH = 512
+
+    def __init__(self, storage_base: str | Path = "data/memory/") -> None:
+        self.root = (Path(storage_base) / "memories").resolve()
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def resolve_session_path(self, session_id: str, rel_path: str) -> Path:
+        if not session_id:
+            raise ValueError("session_id must not be empty")
+        if not rel_path or rel_path.startswith("/"):
+            raise ValueError("Only relative paths are allowed")
+        if len(rel_path) > self.MAX_PATH_LENGTH:
+            raise ValueError("Path too long")
+
+        rel = Path(rel_path)
+        if ".." in rel.parts:
+            raise ValueError("Path escapes sandbox")
+        if rel.suffix.lower() != ".json":
+            raise ValueError("Only .json is allowed")
+
+        session_dir = (self.root / session_id).resolve()
+        session_dir.mkdir(parents=True, exist_ok=True)
+        full_path = (session_dir / rel).resolve()
+        # Ensure no escape from session_dir
+        full_path.relative_to(session_dir)
+        # Ensure parent dirs exist
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+        return full_path
+
+    def check_size_bytes(self, size: int) -> None:
+        if size > self.MAX_SIZE:
+            raise ValueError("File too large")

dolphin/lib/memory/storage.py

@@ -0,0 +1,245 @@
+"""
+Storage layer for the memory management system.
+"""
+
+import abc
+import json
+import os
+import glob
+from pathlib import Path
+from typing import List, Dict, Any
+from datetime import datetime, timedelta
+
+
+class MemoryStorage(abc.ABC):
+    """Simple memory storage abstract interface - for basic memory read and write operations"""
+
+    @abc.abstractmethod
+    def write_memory(
+        self, agent_name: str, user_id: str, memory_items: List[Dict[str, Any]]
+    ) -> bool:
+        """Write memory items
+
+        Args:
+            agent_name: Agent name
+            user_id: User ID, empty string indicates agent memory
+            memory_items: List of memory items
+
+        Returns:
+            bool: Whether successful
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def read_memory(
+        self, agent_name: str, user_id: str, days_back: int = 7
+    ) -> List[Dict[str, Any]]:
+        """Read memory items
+
+        Args:
+            agent_name: Agent name
+            user_id: User ID, empty string indicates agent memory
+            days_back: Read memories from the last N days
+
+        Returns:
+            List[Dict[str, Any]]: List of memory items
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def get_dialog_logs(
+        self, agent_name: str, user_id: str = "", count: int = 30
+    ) -> List[Dict[str, Any]]:
+        """Get conversation logs
+
+        Args:
+            agent_name: Agent name
+            user_id: User ID, empty string to get logs for all users
+            count: Number of entries to return
+
+        Returns:
+            List[Dict[str, Any]]: List of conversation logs
+        """
+        raise NotImplementedError
+
+
+class MemoryFileSys(MemoryStorage):
+    """A simple memory storage for filesystem implementation - for MemorySkillkit"""
+
+    TimeFormat = "%Y%m%d%H%M"
+
+    def __init__(self, base_path: str):
+        """
+        Initialize the simple memory storage.
+
+        :param base_path: Base directory path for storing memory data
+        """
+        self.base_path = Path(base_path)
+        self.base_path.mkdir(parents=True, exist_ok=True)
+
+    def write_memory(
+        self, agent_name: str, user_id: str, memory_items: List[Dict[str, Any]]
+    ) -> bool:
+        """Write memory items to date-organized files
+
+        Args:
+            agent_name: Agent name
+            user_id: User ID, empty string indicates agent memory (using "_agent")
+            memory_items: List of memory items
+
+        Returns:
+            bool: Whether successful
+        """
+        try:
+            # If user_id is empty, use "_agent" for agent memory
+            actual_user_id = user_id if user_id else "_agent"
+
+            current_date = datetime.now().strftime(self.TimeFormat)
+            memory_dir = self.base_path / agent_name / f"user_{actual_user_id}"
+            memory_file = memory_dir / f"memory_{current_date}.jsonl"
+
+            # Create directory if not exists
+            memory_dir.mkdir(parents=True, exist_ok=True)
+
+            # Write memory items to JSONL file
+            with open(memory_file, "a", encoding="utf-8") as f:
+                for item in memory_items:
+                    if isinstance(item, dict):
+                        # Add timestamp if not present
+                        if "timestamp" not in item:
+                            item["timestamp"] = datetime.now().isoformat()
+                        f.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+            return True
+
+        except Exception:
+            return False
+
+    def read_memory(
+        self, agent_name: str, user_id: str, days_back: int = 7
+    ) -> List[Dict[str, Any]]:
+        """Read memory items from the past few days
+
+        Args:
+            agent_name: Agent name
+            user_id: User ID, empty string indicates agent memory
+            days_back: Number of days to look back
+
+        Returns:
+            List[Dict[str, Any]]: List of memory items
+        """
+        try:
+            # If user_id is empty, use "_agent" for agent memory
+            actual_user_id = user_id if user_id else "_agent"
+
+            memory_dir = self.base_path / agent_name / f"user_{actual_user_id}"
+
+            if not memory_dir.exists():
+                return []
+
+            # Find memory files from recent days
+            memory_items = []
+            for file in memory_dir.iterdir():
+                if not file.is_file() or not file.name.startswith("memory_"):
+                    continue
+
+                date_str = file.name.split("_")[1].split(".")[0]
+                date = datetime.strptime(date_str, self.TimeFormat)
+                if date < (datetime.now() - timedelta(days=days_back)):
+                    continue
+
+                with open(file, "r", encoding="utf-8") as f:
+                    for line in f:
+                        if line.strip():
+                            try:
+                                item = json.loads(line.strip())
+                                memory_items.append(item)
+                            except json.JSONDecodeError:
+                                continue
+
+            # Sort by timestamp if available
+            memory_items.sort(key=lambda x: x.get("timestamp", ""), reverse=True)
+            return memory_items
+
+        except Exception:
+            return []
+
+    def get_dialog_logs(
+        self, agent_name: str, user_id: str = "", count: int = 30
+    ) -> List[Dict[str, Any]]:
+        """Get conversation log information
+
+        Args:
+            agent_name: Agent name
+            user_id: User ID, empty string to get logs for all users
+            count: Number of entries to return
+
+        Returns:
+            List[Dict[str, Any]]: List of conversation logs
+        """
+        try:
+            logs = []
+
+            # Dialog path structure: dialog_base_path/{agent_name}/user_{user_id}/dialog_*.jsonl
+            dialog_base_path = self.base_path.parent / "dialog"
+
+            if user_id:
+                # Get logs for a specific user
+                dialog_dir = dialog_base_path / agent_name / f"user_{user_id}"
+                if dialog_dir.exists():
+                    # Get the latest dialog file
+                    dialog_files = sorted(
+                        glob.glob(str(dialog_dir / "dialog_*.jsonl")), reverse=True
+                    )
+                    for file_path in dialog_files:
+                        with open(file_path, "r", encoding="utf-8") as f:
+                            for line in f:
+                                if line.strip():
+                                    try:
+                                        msg = json.loads(line.strip())
+                                        # Filter out system messages
+                                        if msg.get("role") != "system":
+                                            logs.append(msg)
+                                        if len(logs) >= count:
+                                            break
+                                    except json.JSONDecodeError:
+                                        continue
+                        if len(logs) >= count:
+                            break
+            else:
+                # Get logs for all users
+                dialog_base_dir = dialog_base_path / agent_name
+                if dialog_base_dir.exists():
+                    all_files = []
+                    for user_dir in dialog_base_dir.iterdir():
+                        if user_dir.is_dir():
+                            dialog_files = glob.glob(str(user_dir / "dialog_*.jsonl"))
+                            for file_path in dialog_files:
+                                # Get file modification time for sorting
+                                mtime = os.path.getmtime(file_path)
+                                all_files.append((mtime, file_path))
+
+                    # Sort by modification time (newest first)
+                    all_files.sort(reverse=True)
+
+                    for _, file_path in all_files:
+                        with open(file_path, "r", encoding="utf-8") as f:
+                            for line in f:
+                                if line.strip():
+                                    try:
+                                        msg = json.loads(line.strip())
+                                        # Filter out system messages
+                                        if msg.get("role") != "system":
+                                            logs.append(msg)
+                                        if len(logs) >= count:
+                                            break
+                                    except json.JSONDecodeError:
+                                        continue
+                        if len(logs) >= count:
+                            break
+
+            # Return the most recent count messages
+            return logs[:count]
+
+        except Exception:
+            return []

dolphin/lib/memory/utils.py

@@ -0,0 +1,51 @@
+"""
+Utility functions for the memory management system.
+"""
+
+from dolphin.core.common.enums import KnowledgePoint
+
+
+def validate_knowledge_point(
+    data: KnowledgePoint, expected_user_id: str = None
+) -> KnowledgePoint:
+    """
+    Validate and convert raw data to KnowledgePoint.
+
+    :param data: Raw dictionary data
+    :param expected_user_id: Expected user_id for validation (optional)
+    :return: Validated KnowledgePoint
+    :raises KnowledgeValidationError: If validation fails
+    """
+    if data.type not in ["WorldModel", "ExperientialKnowledge", "OtherKnowledge"]:
+        raise Exception(f"Invalid knowledge type: {data.type}")
+
+    if not isinstance(data.score, int) or not (0 <= data.score <= 100):
+        raise Exception(f"Score must be integer between 0-100: {data['score']}")
+
+    if expected_user_id and data.user_id != expected_user_id:
+        raise Exception(
+            f"User ID mismatch: expected {expected_user_id}, got {data.user_id}"
+        )
+
+    return KnowledgePoint(
+        content=str(data.content),
+        type=data.type,
+        score=int(data.score),
+        user_id=str(data.user_id),
+        metadata=data.metadata,
+    )
+
+
+def sanitize_user_id(user_id: str) -> str:
+    """
+    Sanitize user_id for safe filesystem usage.
+
+    :param user_id: Raw user ID
+    :return: Sanitized user ID safe for filesystem
+    """
+    # Remove or replace potentially problematic characters
+    # Allow only word characters, hyphens, and underscores
+    import re
+
+    sanitized = re.sub(r"[^\w\-_]", "_", user_id)
+    return sanitized[:50]  # Limit length to avoid filesystem issues

dolphin/lib/ontology/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""Ontology 模块 - 本体管理系统"""
+
+from dolphin.lib.ontology.ontology import Ontology
+from dolphin.lib.ontology.ontology_manager import OntologyManager
+from dolphin.lib.ontology.ontology_context import OntologyContext
+
+__all__ = [
+    "Ontology",
+    "OntologyManager",
+    "OntologyContext",
+]