kweaver-dolphin 0.1.0__py3-none-any.whl

dolphin/core/utils/cache_kv.py

@@ -0,0 +1,212 @@
+import fcntl
+import json
+import os
+import threading
+import time
+from typing import List, Dict, Any, Optional
+import uuid
+from dolphin.core.logging.logger import get_logger
+
+logger = get_logger("utils.cache_kv")
+
+
+class CacheKV:
+    def __init__(
+        self, filePath: str, dumpInterval: int = 5, expireTimeByDay: float = 1
+    ):
+        self.filePath = filePath
+        self.cache = {}
+        self.lock = threading.Lock()
+        self.dumpInterval = dumpInterval
+        self.lastDumpSec = 0
+        self.expireTimeByDay = expireTimeByDay
+        self.loadCache()
+
+    def loadCache(self):
+        if not os.path.exists(self.filePath):
+            return  # No file, nothing to load
+
+        try:
+            with open(self.filePath, "r", encoding="utf-8") as f:
+                fd = f.fileno()
+                fcntl.flock(fd, fcntl.LOCK_SH)  # Shared lock for reading
+                try:
+                    loaded_cache = json.load(f)
+                    for key, value in loaded_cache.items():
+                        if self._cacheItemExpired(value):
+                            continue
+
+                        if (
+                            isinstance(value, dict)
+                            and "value" in value
+                            and "timestamp" in value
+                        ):
+                            self.cache[key] = value
+                        else:
+                            # Compatible with old formats
+                            self.cache[key] = {"value": value, "timestamp": time.time()}
+                finally:
+                    fcntl.flock(fd, fcntl.LOCK_UN)
+        except (json.JSONDecodeError, IOError) as e:
+            logger.error(
+                f"Error loading cache file {self.filePath}: {e} try to backup and reset cache"
+            )
+            backup_path = self.filePath + ".err"
+            self.cache = {}
+            try:
+                os.rename(self.filePath, backup_path)
+            except OSError as rename_err:
+                logger.error(f"Failed to backup cache file: {rename_err}")
+
+    def dumpCache(self):
+        curTime = time.time()
+        if curTime - self.lastDumpSec < self.dumpInterval:
+            return
+
+        with self.lock:
+            # Ensure the directory exists
+            os.makedirs(os.path.dirname(self.filePath), exist_ok=True)
+            # Clear expired cache
+            self.cache = {
+                key: value
+                for key, value in self.cache.items()
+                if not self._cacheItemExpired(value)
+            }
+
+            # Use unique temporary filename (PID + UUID)
+            temp_path = f"{self.filePath}.tmp.{os.getpid()}.{uuid.uuid4().hex}"
+            try:
+                with open(temp_path, "w", encoding="utf-8") as f:
+                    fd = f.fileno()
+                    fcntl.flock(fd, fcntl.LOCK_EX)  # Exclusive lock for writing
+                    try:
+                        json.dump(self.cache, f, ensure_ascii=False)
+                        f.flush()
+                        os.fsync(fd)  # Ensure data is written to disk
+                    finally:
+                        fcntl.flock(fd, fcntl.LOCK_UN)
+
+                # Atomic rename
+                os.rename(temp_path, self.filePath)
+                self.lastDumpSec = curTime
+            except Exception as e:
+                logger.error(f"Error dumping cache to {self.filePath}: {e}")
+                if os.path.exists(temp_path):
+                    try:
+                        os.remove(temp_path)
+                    except OSError as remove_err:
+                        logger.error(
+                            f"Failed to remove temp file {temp_path}: {remove_err}"
+                        )
+
+    def _keyToStr(self, key: List[Dict]) -> str:
+        return json.dumps(key, sort_keys=True, ensure_ascii=False)
+
+    def get(self, key: List[Dict]) -> Optional[Any]:
+        keyStr = self._keyToStr(key)
+        with self.lock:
+            cached_item = self.cache.get(keyStr)
+            if cached_item:
+                if not self._cacheItemExpired(cached_item):
+                    return cached_item["value"]
+                else:
+                    del self.cache[keyStr]
+        return None
+
+    def set(self, key: List[Dict], value: Any):
+        keyStr = self._keyToStr(key)
+        with self.lock:
+            self.cache[keyStr] = {"value": value, "timestamp": time.time()}
+        self.dumpCache()
+
+    def remove(self, key: List[Dict]):
+        keyStr = self._keyToStr(key)
+        with self.lock:
+            if keyStr in self.cache:
+                del self.cache[keyStr]
+        self.dumpCache()
+
+    def _cacheItemExpired(self, cacheItem: dict) -> bool:
+        return time.time() - cacheItem["timestamp"] > self.expireTimeByDay * 86400
+
+
+class CacheKVMgr:
+    FilePrefix = "cache_"
+
+    def __init__(
+        self,
+        cacheDir: str,
+        category: str,
+        dumpInterval: int = 5,
+        expireTimeByDay: float = 1,
+    ):
+        self.prefix = f"{CacheKVMgr.FilePrefix}{category}_"
+        self.cacheDir = cacheDir
+        self.category = category
+        self.dumpInterval = dumpInterval
+        self.expireTimeByDay = expireTimeByDay
+        self.caches = {}
+        self.loadCaches()
+
+    def loadCaches(self):
+        if not os.path.exists(self.cacheDir):
+            os.makedirs(self.cacheDir)
+
+        for fileName in os.listdir(self.cacheDir):
+            if fileName.startswith(self.prefix) and fileName.endswith(".json"):
+                modelName = fileName[len(self.prefix) : -5]
+                filePath = os.path.join(self.cacheDir, fileName)
+                try:
+                    self.caches[modelName] = CacheKV(
+                        filePath, self.dumpInterval, self.expireTimeByDay
+                    )
+                except Exception as e:
+                    logger.error(f"Error initializing cache for {modelName}: {e}")
+                    raise Exception(f"Error initializing cache for {modelName}: {e}")
+
+    def getCache(self, modelName: str) -> CacheKV:
+        return self.caches.get(modelName)
+
+    def getValue(self, modelName: str, key: List[Dict]) -> Optional[Any]:
+        cache = self.getCache(modelName)
+        if cache:
+            result = cache.get(key)
+            if result:
+                logger.debug(f"cache hit: {modelName} {key}")
+            return result
+        return None
+
+    def setValue(self, modelName: str, key: List[Dict], value: Any):
+        cache = self.getCache(modelName)
+        if not cache:
+            filePath = os.path.join(self.cacheDir, f"{self.prefix}{modelName}.json")
+            cache = CacheKV(filePath, self.dumpInterval, self.expireTimeByDay)
+            self.caches[modelName] = cache
+        cache.set(key, value)
+
+    def removeValue(self, modelName: str, key: List[Dict]):
+        cache = self.getCache(modelName)
+        if cache:
+            cache.remove(key)
+
+
+class CacheKVCenter:
+    def __init__(self):
+        self.repos = {}
+
+    def getCacheMgr(
+        self,
+        cacheDir: str,
+        category: str,
+        dumpInterval: int = 5,
+        expireTimeByDay: float = 1,
+    ) -> CacheKVMgr:
+        key = f"{cacheDir}_{category}"
+        if key not in self.repos:
+            self.repos[key] = CacheKVMgr(
+                cacheDir, category, dumpInterval, expireTimeByDay
+            )
+        return self.repos[key]
+
+
+GlobalCacheKVCenter = CacheKVCenter()

dolphin/core/utils/tools.py

@@ -0,0 +1,340 @@
+from typing import Any, Dict, Iterator, List, Optional
+import ast
+import json
+
+from dolphin.core.logging.logger import get_logger
+
+logger = get_logger("utils.tools")
+
+
+def safe_json_loads(json_str: str, strict: bool = True) -> Dict[str, Any]:
+    """A safe JSON parsing function that supports multiple parsing strategies.
+
+        Args:
+            json_str: The JSON string to be parsed.
+            strict: Whether to use strict mode.
+
+        Returns:
+            The parsed dictionary object.
+
+        Raises:
+            ValueError: When all parsing strategies fail.
+    """
+    if not json_str or not json_str.strip():
+        return {}
+
+    json_str = json_str.strip()
+
+    # Strategy 1: Use ast.literal_eval for safer parsing
+    try:
+        value = ast.literal_eval(json_str)
+        if isinstance(value, dict) or isinstance(value, list):
+            return value
+    except Exception:
+        pass
+
+    # Strategy 2: Use standard json.loads with strict mode
+    try:
+        return json.loads(json_str, strict=strict)
+    except Exception:
+        pass
+
+    # Strategy 3: Replace single quotes with double quotes then parse
+    try:
+        normalized_content = json_str.replace("'", '"')
+        return json.loads(normalized_content, strict=strict)
+    except Exception:
+        pass
+
+    # Strategy 4: Try non-strict mode if we were in strict mode
+    if strict:
+        try:
+            return json.loads(json_str, strict=False)
+        except Exception:
+            pass
+
+    # Strategy 6: Advanced string cleaning
+    try:
+        # Clean up common formatting issues to improve fault tolerance
+        if json_str.startswith('{\\"'):
+            cleaned = json_str.replace('\\"', '"')
+        return json.loads(cleaned, strict=False)
+    except Exception:
+        pass
+
+    # Strategy 7: Replace newlines with spaces
+    try:
+        cleaned = json_str.replace("\n", " ")
+        return json.loads(cleaned, strict=False)
+    except Exception:
+        pass
+
+    # Strategy 8: Append missing closing brace if exactly one more open brace
+    try:
+        open_brace = json_str.count("{")
+        close_brace = json_str.count("}")
+        if open_brace == close_brace + 1:
+            appended = json_str + "}"
+            return json.loads(appended, strict=False)
+    except Exception:
+        pass
+
+    # Strategy 9: Handle malformed empty object like '{"}' -> treat as {}
+    try:
+        stripped = json_str.strip()
+        if stripped == '{"}':
+            return {}
+    except Exception:
+        pass
+
+    raise ValueError(f"Failed to parse JSON: {json_str}")
+
+
+def safe_jsonl_loads(jsonl_str: str) -> List[Dict[str, Any]]:
+    """Safe JSONL parsing function
+
+        Args:
+            jsonl_str: JSONL string to be parsed
+
+        Returns:
+            List of parsed dictionaries
+
+        Raises:
+            ValueError: If parsing fails
+    """
+    jsonl_str = jsonl_str.strip()
+    if not jsonl_str:
+        return []
+
+    results = []
+    try:
+        results = json.loads(jsonl_str, strict=False)
+    except Exception:
+        try:
+            results = ast.literal_eval(jsonl_str)
+            if isinstance(results, list):
+                return results
+        except Exception:
+            pass
+
+    if results:
+        if isinstance(results, list):
+            return results
+        if isinstance(results, dict):
+            for key, value in results.items():
+                if isinstance(value, list):
+                    return value
+
+    results = []
+    for line_num, line in enumerate(jsonl_str.split("\n"), 1):
+        line = line.strip()
+        if not line:
+            continue
+
+        try:
+            obj = safe_json_loads(line, strict=False)
+            results.append(obj)
+        except ValueError as e:
+            logger.error(f"Failed to parse JSONL line {line_num}: {e}")
+            logger.error(f"Line content: {line}")
+            raise ValueError(f"Invalid JSON format in JSONL line {line_num}: {e}")
+
+    return results
+
+
+def extract_json_from_response(response_str: str) -> str:
+    """Extract JSON content from response string (handling code block markers)
+
+        Args:
+            response_str: Original response string
+
+        Returns:
+            Extracted JSON string
+    """
+    response_str = response_str.strip()
+
+    # Process ```json code block
+    if "```json" in response_str:
+        start = response_str.find("```json") + 7
+        end = response_str.find("```", start)
+        if end != -1:
+            return response_str[start:end].strip()
+
+    # Process generic code blocks
+    elif response_str.startswith("```") and response_str.endswith("```"):
+        lines = response_str.split("\n")
+        if len(lines) > 2:
+            return "\n".join(lines[1:-1])
+
+    elif response_str.startswith("{"):
+        # Find the matching } considering nested braces
+        start = 0
+        count = 1
+        end = -1
+        for i in range(1, len(response_str)):
+            if response_str[i] == "{":
+                count += 1
+            elif response_str[i] == "}":
+                count -= 1
+                if count == 0:
+                    end = i
+                    break
+        if end != -1:
+            return response_str[start : end + 1].strip()
+
+    return response_str
+
+
+def extract_jsonl_from_response(response_str: str) -> str:
+    """Extract JSONL content from response string (handling code block markers)
+
+        Args:
+            response_str: Original response string
+
+        Returns:
+            Extracted JSONL string
+    """
+    response_str = response_str.strip()
+
+    # Handle ```jsonl or ```json code blocks
+    start_markers = ["```jsonl", "```json"]
+    for marker in start_markers:
+        if marker in response_str:
+            start = response_str.find(marker) + len(marker)
+            end = response_str.find("```", start)
+            if end != -1:
+                return response_str[start:end].strip()
+            break
+
+    # Process generic code blocks
+    if response_str.startswith("```") and response_str.endswith("```"):
+        lines = response_str.split("\n")
+        if len(lines) > 2:
+            return "\n".join(lines[1:-1])
+
+    elif response_str.startswith("["):
+        # Find the matching ] considering nested braces
+        start = 0
+        count = 1
+        end = -1
+        for i in range(1, len(response_str)):
+            if response_str[i] == "[":
+                count += 1
+            elif response_str[i] == "]":
+                count -= 1
+                if count == 0:
+                    end = i
+                    break
+        if end != -1:
+            return response_str[start : end + 1].strip()
+
+    return response_str
+
+
+def extract_json(content: str) -> Dict[str, Any]:
+    """Extract JSON content from response string (handling code block markers)
+
+        Args:
+            content: Original response string
+    """
+    content = extract_json_from_response(content)
+    result = safe_json_loads(content)
+    if isinstance(result, dict) or isinstance(result, list):
+        return result
+    else:
+        raise ValueError(f"Invalid JSON format: {content}")
+
+
+def extract_jsonl(content: str) -> List[Dict[str, Any]]:
+    """Extract JSONL content from response string (handling code block markers)
+
+        Args:
+            content: Original response string
+    """
+    content = extract_jsonl_from_response(content)
+    result = safe_jsonl_loads(content)
+    if isinstance(result, list):
+        return result
+    else:
+        raise ValueError(f"Invalid JSONL format: {content}")
+
+
+class Tool:
+    # The unique name of the tool that clearly communicates its purpose
+    name: str
+    # Tool description, explaining the functions of the tool, facilitating the selection and invocation of LLM
+    description: str
+    # Tool type, eg: flow, component, api, etc
+    # type: str
+    # Input parameters of the tool
+    inputs: Dict
+    # Output format of the tool
+    outputs: Dict
+
+    result_process_strategy_cfg: list[Dict[str, str]] = None
+
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        inputs: Dict,
+        outputs: Dict,
+        props: Optional[dict] = None,
+        result_process_strategy_cfg: list[Dict[str, str]] = None,
+    ):
+        self.name = name
+        self.description = description
+        self.inputs = inputs
+        self.outputs = outputs
+        self.props = props
+        self.intervention = props.get("intervention", False)
+        self.result_process_strategy_cfg = result_process_strategy_cfg
+
+    def tool_info(self) -> dict:
+        """The tool's information."""
+        return {
+            "name": self.name,
+            "description": self.description,
+            "inputs": self.inputs,
+            "outputs": self.outputs,
+        }
+
+    def run(self, **kwargs) -> Any:
+        """
+        Run the tool
+        """
+        raise NotImplementedError
+
+    async def arun(self, **kwargs) -> Any:
+        """
+        Run the tool asynchronously.
+        """
+        raise NotImplementedError
+
+    def run_stream(self, **kwargs) -> Iterator[Any]:
+        """
+        Run the tool with streaming output.
+        """
+        raise NotImplementedError
+
+    async def arun_stream(self, **kwargs) -> Iterator[Any]:
+        """
+        Run the tool with streaming output asynchronously.
+        """
+        raise NotImplementedError
+
+
+class ToolInterrupt(Exception):
+    """Exception raised when the tool is interrupted."""
+
+    def __init__(
+        self,
+        message="The tool was interrupted.",
+        tool_name: str = None,
+        tool_args: List[Dict] = None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(message, *args, **kwargs)
+        self.tool_name = tool_name if tool_name else ""
+        self.tool_args = tool_args if tool_args else []

dolphin/lib/__init__.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+"""
+Dolphin Lib - 标准库和工具集（用户态）
+
+职责：
+- 内置 Skillkits（search、sql、memory、mcp 等）
+- Ontology 管理系统
+- VM 虚拟机（可选执行后端）
+- Memory 内存管理（知识管理）
+- 工具函数库
+- 调试可视化工具
+
+依赖规则：
+- dolphin.lib → 依赖 dolphin.core
+"""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    # Skillkits
+    from dolphin.lib.skillkits import (
+        SearchSkillkit,
+        SQLSkillkit,
+        MemorySkillkit,
+        MCPSkillkit,
+        OntologySkillkit,
+        PlanActSkillkit,
+        CognitiveSkillkit,
+        VMSkillkit,
+        NoopSkillkit,
+        ResourceSkillkit,
+        SystemFunctionsSkillKit,
+        AgentSkillKit,
+    )
+    # Ontology
+    from dolphin.lib.ontology import (
+        Ontology,
+        OntologyManager,
+        OntologyContext,
+    )
+    # VM
+    from dolphin.lib.vm import (
+        VM,
+        PythonSessionManager,
+    )
+    # Memory
+    from dolphin.lib.memory import (
+        MemoryManager,
+    )
+    # Skill Results
+    from dolphin.lib.skill_results import (
+        CacheBackend,
+        ResultProcessor,
+        ResultReference,
+    )
+
+_module_lookup = {
+    # Skillkits
+    "SearchSkillkit": "dolphin.lib.skillkits",
+    "SQLSkillkit": "dolphin.lib.skillkits",
+    "MemorySkillkit": "dolphin.lib.skillkits",
+    "MCPSkillkit": "dolphin.lib.skillkits",
+    "OntologySkillkit": "dolphin.lib.skillkits",
+    "PlanActSkillkit": "dolphin.lib.skillkits",
+    "CognitiveSkillkit": "dolphin.lib.skillkits",
+    "VMSkillkit": "dolphin.lib.skillkits",
+    "NoopSkillkit": "dolphin.lib.skillkits",
+    "ResourceSkillkit": "dolphin.lib.skillkits",
+    "SystemFunctionsSkillKit": "dolphin.lib.skillkits",
+    "AgentSkillKit": "dolphin.lib.skillkits",
+    # Ontology
+    "Ontology": "dolphin.lib.ontology",
+    "OntologyManager": "dolphin.lib.ontology",
+    "OntologyContext": "dolphin.lib.ontology",
+    # VM
+    "VM": "dolphin.lib.vm",
+    "PythonSessionManager": "dolphin.lib.vm",
+    # Memory
+    "MemoryManager": "dolphin.lib.memory",
+    # Skill Results
+    "CacheBackend": "dolphin.lib.skill_results",
+    "ResultProcessor": "dolphin.lib.skill_results",
+    "ResultReference": "dolphin.lib.skill_results",
+}
+
+def __getattr__(name):
+    if name in _module_lookup:
+        import importlib
+        module = importlib.import_module(_module_lookup[name])
+        return getattr(module, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+__all__ = list(_module_lookup.keys())

dolphin/lib/debug/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+"""Debug 模块 - 调试工具"""
+
+from dolphin.lib.debug.visualizer import TraceVisualizer
+
+__all__ = [
+    "TraceVisualizer",
+]