kweaver-dolphin 0.1.0__py3-none-any.whl

dolphin/lib/utils/data_process.py

@@ -0,0 +1,207 @@
+import re
+
+
+def escape(prompt: str):
+    # to prevent 'format' exception in get_template_vars
+    return prompt.replace("{", "{{").replace("}", "}}")
+
+
+def unescape(prompt: str):
+    return re.sub(r"\{{2,}", "{", re.sub(r"\}{2,}", "}", prompt))
+
+
+def get_nested_value(var_name, data: dict):
+    """Recursively parse nested variable names, supporting a mix of array indexing and attribute access
+        For example: var[0].field[1].subfield, user['name'], header["x-user"]
+    """
+    if not var_name:
+        return data
+
+    # If the variable name starts with a square bracket, process the content within the square brackets directly.
+    if var_name.startswith("["):
+        bracket_end = var_name.find("]")
+        if bracket_end == -1:
+            raise ValueError(f"变量名 '{var_name}' 中的方括号不匹配")
+
+        # Extract content within square brackets
+        bracket_content = var_name[1:bracket_end]
+
+        # Check if it's a string key (quoted)
+        if (bracket_content.startswith("'") and bracket_content.endswith("'")) or (
+            bracket_content.startswith('"') and bracket_content.endswith('"')
+        ):
+            # String key, remove quotes
+            key_name = bracket_content[1:-1]
+
+            # Get dictionary value
+            if not isinstance(data, dict):
+                raise ValueError("数据不是字典类型，无法使用键访问")
+
+            if key_name not in data:
+                raise ValueError(f"键 '{key_name}' 不存在于数据中")
+
+            dict_value = data[key_name]
+            remaining = var_name[bracket_end + 1 :]
+
+            if not remaining:
+                return dict_value
+            elif remaining.startswith("."):
+                # Continue recursive processing
+                if not isinstance(dict_value, dict):
+                    raise ValueError("字典值不是字典类型，无法使用点号访问")
+                return get_nested_value(remaining[1:], dict_value)
+            elif remaining.startswith("["):
+                # Continue recursive processing
+                return get_nested_value(remaining, dict_value)
+            else:
+                raise ValueError(f"变量名 '{var_name}' 格式错误，无法解析")
+
+        else:
+            # Numeric Indexing (Array Access)
+            try:
+                index = int(bracket_content)
+            except ValueError:
+                # Index is not a number
+                raise ValueError(f"数组索引 '{bracket_content}' 不是有效的数字")
+
+            # Get array elements
+            if not isinstance(data, list):
+                raise ValueError("数据不是列表类型，无法使用索引访问")
+
+            if not (0 <= index < len(data)):
+                raise ValueError(f"数组索引 {index} 超出范围 [0, {len(data)})")
+
+            array_element = data[index]
+            remaining = var_name[bracket_end + 1 :]
+
+            if not remaining:
+                return array_element
+            elif remaining.startswith("."):
+                # Continue recursive processing
+                if not isinstance(array_element, dict):
+                    raise ValueError("数组元素不是字典类型，无法使用点号访问")
+                return get_nested_value(remaining[1:], array_element)
+            elif remaining.startswith("["):
+                # Continue recursive processing
+                if not isinstance(array_element, list):
+                    raise ValueError("数组元素不是列表类型，无法使用索引访问")
+                return get_nested_value(remaining, array_element)
+            else:
+                raise ValueError(f"变量名 '{var_name}' 格式错误，无法解析")
+
+    # Find the position of the first operator
+    dot_pos = var_name.find(".")
+    bracket_start = var_name.find("[")
+
+    # If there are neither dots nor square brackets, return directly.
+    if dot_pos == -1 and bracket_start == -1:
+        if var_name not in data:
+            raise ValueError(f"变量 '{var_name}' 不存在于数据中")
+        return data.get(var_name)
+
+    # Determine the first operator
+    first_op_pos = -1
+    if dot_pos == -1:
+        first_op_pos = bracket_start
+    elif bracket_start == -1:
+        first_op_pos = dot_pos
+    else:
+        first_op_pos = min(dot_pos, bracket_start)
+
+    # Extract the name of the key currently being accessed
+    current_key = var_name[:first_op_pos]
+    rest = var_name[first_op_pos:]
+
+    # Get current value
+    if current_key not in data:
+        raise ValueError(f"变量 '{current_key}' 不存在于数据中")
+    current_value = data.get(current_key)
+
+    # If there are no remaining parts, return the current value
+    if not rest:
+        return current_value
+
+    # Process the remaining part
+    if rest.startswith("."):
+        # Dot access, recursively process the remaining part
+        if not isinstance(current_value, dict):
+            raise ValueError(f"变量 '{current_key}' 不是字典类型，无法使用点号访问")
+        return get_nested_value(rest[1:], current_value)
+    elif rest.startswith("["):
+        # Bracket Access
+        bracket_end = rest.find("]")
+        if bracket_end == -1:
+            # No matching right parenthesis found
+            raise ValueError(f"变量名 '{var_name}' 中的方括号不匹配")
+
+        # Extract content within square brackets
+        bracket_content = rest[1:bracket_end]
+
+        # Check if it's a string key (quoted)
+        if (bracket_content.startswith("'") and bracket_content.endswith("'")) or (
+            bracket_content.startswith('"') and bracket_content.endswith('"')
+        ):
+            # String key, remove quotes
+            key_name = bracket_content[1:-1]
+
+            # Get dictionary value
+            if not isinstance(current_value, dict):
+                raise ValueError(f"变量 '{current_key}' 不是字典类型，无法使用键访问")
+
+            if key_name not in current_value:
+                raise ValueError(f"键 '{key_name}' 不存在于字典 '{current_key}' 中")
+
+            dict_value = current_value[key_name]
+            remaining = rest[bracket_end + 1 :]
+
+            if not remaining:
+                return dict_value
+            elif remaining.startswith("."):
+                # Continue recursive processing
+                if not isinstance(dict_value, dict):
+                    raise ValueError(
+                        f"字典值 '{current_key}['{key_name}']' 不是字典类型，无法使用点号访问"
+                    )
+                return get_nested_value(remaining[1:], dict_value)
+            elif remaining.startswith("["):
+                # Continue recursive processing
+                return get_nested_value(remaining, dict_value)
+
+        else:
+            # Numeric Indexing (Array Access)
+            try:
+                index = int(bracket_content)
+            except ValueError:
+                # Index is not a number
+                raise ValueError(f"数组索引 '{bracket_content}' 不是有效的数字")
+
+            # Get array elements
+            if not isinstance(current_value, list):
+                raise ValueError(f"变量 '{current_key}' 不是列表类型，无法使用索引访问")
+
+            if not (0 <= index < len(current_value)):
+                raise ValueError(f"数组索引 {index} 超出范围 [0, {len(current_value)})")
+
+            array_element = current_value[index]
+            remaining = rest[bracket_end + 1 :]
+
+            if not remaining:
+                return array_element
+            elif remaining.startswith("."):
+                # Continue recursive processing
+                if not isinstance(array_element, dict):
+                    raise ValueError(
+                        f"数组元素 '{current_key}[{index}]' 不是字典类型，无法使用点号访问"
+                    )
+                return get_nested_value(remaining[1:], array_element)
+            elif remaining.startswith("["):
+                # Continue recursive processing
+                if not isinstance(array_element, list):
+                    raise ValueError(
+                        f"数组元素 '{current_key}[{index}]' 不是列表类型，无法使用索引访问"
+                    )
+                return get_nested_value(remaining, array_element)
+            else:
+                raise ValueError(f"变量名 '{var_name}' 格式错误，无法解析")
+
+    raise ValueError(f"变量名 '{var_name}' 格式错误，无法解析")

dolphin/lib/utils/handle_progress.py

@@ -0,0 +1,178 @@
+import json
+import threading
+import time
+from collections import OrderedDict
+from typing import Dict, List, Optional
+from dolphin.core.logging.logger import console
+
+
+# Thread-safe global variables
+class ProgressManager:
+    def __init__(self, max_size=1000, expire_seconds=3600):
+        self.progress_map: Dict[str, List[Dict]] = OrderedDict()
+        self.progress_set: Dict[str, Dict[str, bool]] = {}
+        self.lock = threading.RLock()
+        self.max_size = max_size
+        self.expire_seconds = expire_seconds
+        self.last_access: Dict[str, float] = {}
+
+    def _cleanup_expired(self):
+        """Clean up expired progress data"""
+        current_time = time.time()
+        expired_keys = []
+
+        for key, last_time in self.last_access.items():
+            if current_time - last_time > self.expire_seconds:
+                expired_keys.append(key)
+
+        for key in expired_keys:
+            self._remove_key(key)
+
+    def _remove_key(self, key: str):
+        """Remove data for the specified key"""
+        if key in self.progress_map:
+            del self.progress_map[key]
+        if key in self.progress_set:
+            del self.progress_set[key]
+        if key in self.last_access:
+            del self.last_access[key]
+
+    def _enforce_max_size(self):
+        """Enforce maximum size limit"""
+        while len(self.progress_map) > self.max_size:
+            # Remove the oldest entry
+            oldest_key = next(iter(self.progress_map))
+            self._remove_key(oldest_key)
+
+
+# Global progress manager instance
+_progress_manager = ProgressManager()
+
+
+def handle_progress(assistant_message_id: str, progresses: List[Dict]) -> List[Dict]:
+    """Thread-safe version for processing progress information
+
+        Args:
+            assistant_message_id: Assistant message ID
+            progresses: List of progress information (in dictionary format)
+
+        Returns:
+            Processed list of progress information
+    """
+    with _progress_manager.lock:
+        # Clean up expired data
+        _progress_manager._cleanup_expired()
+
+        # Update access time
+        _progress_manager.last_access[assistant_message_id] = time.time()
+
+        # Get the progress collection of the current message
+        progress_set_for_message = _progress_manager.progress_set.get(
+            assistant_message_id, {}
+        )
+        current_progress: Optional[Dict] = None
+
+        # Traverse all progress information
+        for progress in progresses:
+            status = progress.get("status", "")
+
+            if status in ["completed", "failed"]:
+                # Use more efficient hash values for deduplication
+                progress_hash = hash(json.dumps(progress, sort_keys=True))
+
+                # Check if already exists to avoid duplication
+                if progress_hash not in progress_set_for_message:
+                    # Ensure that there is a corresponding list in progress_map.
+                    if assistant_message_id not in _progress_manager.progress_map:
+                        _progress_manager.progress_map[assistant_message_id] = []
+
+                    # Add to history progress list
+                    _progress_manager.progress_map[assistant_message_id].append(
+                        progress
+                    )
+                    # Marked as existing
+                    progress_set_for_message[progress_hash] = True
+
+            elif status == "processing":
+                # Record the current progress being processed
+                current_progress = progress
+
+        # Update progress_set
+        _progress_manager.progress_set[assistant_message_id] = progress_set_for_message
+
+        # Enforce size limits
+        _progress_manager._enforce_max_size()
+
+        # Build return result
+        result = []
+
+        # Add historical progress (completed and failed)
+        if assistant_message_id in _progress_manager.progress_map:
+            result.extend(_progress_manager.progress_map[assistant_message_id])
+
+        # Add the progress of the currently processed item (if any)
+        if current_progress is not None:
+            result.append(current_progress)
+
+        return result
+
+
+def cleanup_progress(assistant_message_id: str) -> None:
+    """Thread-safe version to clear progress data for the specified message ID"""
+    with _progress_manager.lock:
+        _progress_manager._remove_key(assistant_message_id)
+
+
+def initialize_progress(assistant_message_id: str) -> None:
+    """Initialize progress data for the specified message ID (thread-safe version)"""
+    with _progress_manager.lock:
+        _progress_manager.progress_map[assistant_message_id] = []
+        _progress_manager.progress_set[assistant_message_id] = {}
+        _progress_manager.last_access[assistant_message_id] = time.time()
+
+
+def get_progress_stats() -> Dict:
+    """Get progress manager statistics (for monitoring)"""
+    with _progress_manager.lock:
+        return {
+            "total_sessions": len(_progress_manager.progress_map),
+            "total_progress_items": sum(
+                len(items) for items in _progress_manager.progress_map.values()
+            ),
+            "max_size": _progress_manager.max_size,
+            "expire_seconds": _progress_manager.expire_seconds,
+        }
+
+
+# Usage Examples
+if __name__ == "__main__":
+    # Initialize
+    message_id = "msg_123"
+    initialize_progress(message_id)
+
+    # Example data
+    progresses = [
+        {
+            "agent_name": "agent1",
+            "stage": "thinking",
+            "status": "completed",
+            "answer": "Thinking completed",
+        },
+        {
+            "agent_name": "agent1",
+            "stage": "executing",
+            "status": "processing",
+            "answer": "正在执行",
+        },
+    ]
+
+    # Processing Progress
+    result = handle_progress(message_id, progresses)
+    console(f"处理结果: {len(result)} 个进度")
+
+    # Cleanup
+    cleanup_progress(message_id)
+
+    # Get statistical information
+    stats = get_progress_stats()
+    console(f"统计信息: {stats}")

dolphin/lib/utils/security.py

@@ -0,0 +1,139 @@
+import base64
+import os
+from typing import Optional
+from dolphin.core.logging.logger import get_logger
+
+logger = get_logger()
+
+
+class SecurityUtils:
+    """Security utility class, providing encryption and decryption functions"""
+
+    # Default salt value; in actual applications, it should be obtained from environment variables or configurations.
+    DEFAULT_SALT = b"dolphin_default_salt_value"
+    
+    # 缓存 cryptography 模块（延迟导入）
+    _cryptography = None
+    
+    @classmethod
+    def _get_cryptography(cls):
+        """获取 cryptography 模块（延迟导入）"""
+        if cls._cryptography is None:
+            try:
+                from cryptography.fernet import Fernet
+                from cryptography.hazmat.backends import default_backend
+                from cryptography.hazmat.primitives import hashes
+                from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+                cls._cryptography = {
+                    'Fernet': Fernet,
+                    'default_backend': default_backend,
+                    'hashes': hashes,
+                    'PBKDF2HMAC': PBKDF2HMAC,
+                }
+            except ImportError:
+                logger.error("cryptography is required for SecurityUtils but not installed. Please install it: pip install cryptography")
+                raise ImportError("cryptography is required for SecurityUtils but not installed. Please install it: pip install cryptography")
+        return cls._cryptography
+
+    @staticmethod
+    def _get_key(password: str, salt: Optional[bytes] = None) -> bytes:
+        """Generate encryption key
+
+        Args:
+            password: The password used to generate the key
+            salt: Optional salt value; if not provided, a default salt value is used
+
+        Returns:
+            bytes: The generated key
+        """
+        crypto = SecurityUtils._get_cryptography()
+        PBKDF2HMAC = crypto['PBKDF2HMAC']
+        hashes = crypto['hashes']
+        default_backend = crypto['default_backend']
+        
+        if not salt:
+            salt = SecurityUtils.DEFAULT_SALT
+
+        kdf = PBKDF2HMAC(
+            algorithm=hashes.SHA256(),
+            length=32,
+            salt=salt,
+            iterations=100000,
+            backend=default_backend(),
+        )
+        return base64.urlsafe_b64encode(kdf.derive(password.encode()))
+
+    @staticmethod
+    def encrypt(text: str, password: str, salt: Optional[bytes] = None) -> str:
+        """Encrypted text
+
+        Args:
+            text: The text to be encrypted
+            password: The password used to generate the key
+            salt: Optional salt value
+
+        Returns:
+            str: The encrypted text (Base64 encoded)
+        """
+        try:
+            crypto = SecurityUtils._get_cryptography()
+            Fernet = crypto['Fernet']
+            key = SecurityUtils._get_key(password, salt)
+            f = Fernet(key)
+            encrypted_data = f.encrypt(text.encode())
+            return base64.urlsafe_b64encode(encrypted_data).decode()
+        except ImportError:
+            raise
+        except Exception as e:
+            logger.error(f"加密failed: {str(e)}")
+            raise RuntimeError(f"加密failed: {str(e)}")
+
+    @staticmethod
+    def decrypt(
+        encrypted_text: str, password: str, salt: Optional[bytes] = None
+    ) -> str:
+        """Decrypt text
+
+        Args:
+            encrypted_text: The encrypted text (Base64 encoded)
+            password: The password used to generate the key
+            salt: Optional salt value
+
+        Returns:
+            str: The decrypted text
+        """
+        try:
+            crypto = SecurityUtils._get_cryptography()
+            Fernet = crypto['Fernet']
+            key = SecurityUtils._get_key(password, salt)
+            f = Fernet(key)
+            # First, perform Base64 decoding on the encrypted text.
+            decoded_data = base64.urlsafe_b64decode(encrypted_text.encode())
+            decrypted_data = f.decrypt(decoded_data)
+            return decrypted_data.decode()
+        except ImportError:
+            raise
+        except Exception as e:
+            logger.error(f"解密failed: {str(e)}")
+            return encrypted_text  # Return the original encrypted text when decryption fails
+
+    @staticmethod
+    def generate_key() -> str:
+        """Generate a new Fernet key
+
+        Returns:
+            str: The generated key (Base64-encoded string)
+        """
+        crypto = SecurityUtils._get_cryptography()
+        Fernet = crypto['Fernet']
+        key = Fernet.generate_key()
+        return key.decode()
+
+    @staticmethod
+    def get_env_password() -> str:
+        """Get password from environment variables
+
+        Returns:
+            str: Password obtained from environment variables, or default value if not set
+        """
+        return os.environ.get("DOLPHIN_PASSWORD", "default_password_please_change_me")