kweaver-dolphin 0.1.0__py3-none-any.whl

dolphin/core/context/context_manager.py

@@ -0,0 +1,161 @@
+"""
+Context Manager module for backward compatibility.
+
+This module provides the ContextEngineer class which is an alias/wrapper
+around MessageCompressor for SDK compatibility.
+
+Note: The compression strategies (TruncationStrategy, SlidingWindowStrategy, 
+LevelStrategy) are now defined in dolphin.core.message.compressor.
+"""
+
+from typing import List, Dict, Optional, TYPE_CHECKING
+
+from dolphin.core.common.enums import Messages
+from dolphin.core.config.global_config import ContextEngineerConfig
+
+# Import compression classes from the canonical location
+from dolphin.core.message.compressor import (
+    CompressionResult,
+    CompressionStrategy,
+    TruncationStrategy,
+    SlidingWindowStrategy,
+    LevelStrategy,
+)
+
+if TYPE_CHECKING:
+    from dolphin.core.config.global_config import (
+        ContextConstraints,
+        LLMInstanceConfig,
+    )
+    from dolphin.core.context.context import Context
+
+from dolphin.core.logging.logger import get_logger
+
+logger = get_logger("context_engineer")
+
+
+# Re-export for backward compatibility
+__all__ = [
+    "CompressionResult",
+    "CompressionStrategy", 
+    "TruncationStrategy",
+    "SlidingWindowStrategy",
+    "LevelStrategy",
+    "ContextEngineer",
+]
+
+
+class ContextEngineer:
+    """Context Engineer: Responsible for optimizing message context to meet model constraints.
+    
+    This class is maintained for backward compatibility with the SDK.
+    New code should use MessageCompressor from dolphin.core.message.compressor instead.
+    """
+
+    def __init__(self, config: "ContextEngineerConfig" = None, context: "Context" = None):
+        self.config = config or ContextEngineerConfig()
+        self.context = context
+        self.strategies = self._register_default_strategies()
+        logger.debug(
+            f"ContextEngineer initialized with strategy: {self.config.default_strategy}"
+        )
+
+    def _register_default_strategies(self) -> Dict[str, CompressionStrategy]:
+        """Register default compression strategy"""
+        strategies = {
+            "level": LevelStrategy(),
+            "truncation": TruncationStrategy(),
+            "sliding_window_5": SlidingWindowStrategy(5),
+            "sliding_window_10": SlidingWindowStrategy(10),
+            "sliding_window_20": SlidingWindowStrategy(20),
+        }
+        # Merge user-defined policy configurations
+        for name, strategy_config in self.config.strategy_configs.items():
+            # Here, corresponding strategy instances can be created based on strategy_config
+            # Skip temporarily, keep extension interface
+            pass
+        return strategies
+
+    def engineer_context(
+        self,
+        messages: Messages,
+        strategy_name: Optional[str] = None,
+        constraints: Optional["ContextConstraints"] = None,
+        model_config: Optional["LLMInstanceConfig"] = None,
+        **kwargs,
+    ) -> CompressionResult:
+        """Engineering processing of context
+
+        Args:
+            messages: List of original messages
+            strategy_name: Name of the compression strategy to use, default uses the default strategy from configuration
+            constraints: Compression constraints, default uses constraints from configuration
+            model_config: Model configuration, used to automatically adjust constraints
+
+        Returns:
+            CompressionResult: Compression result
+        """
+        # Select Strategy
+        strategy_name = strategy_name or self.config.default_strategy
+        if strategy_name not in self.strategies:
+            logger.warning(f"Strategy '{strategy_name}' not found, using 'truncation'")
+            strategy_name = "truncation"
+            if strategy_name not in self.strategies:
+                # If there is no truncation at all, create a default one.
+                self.strategies["truncation"] = TruncationStrategy()
+
+        strategy = self.strategies[strategy_name]
+
+        # Select constraint conditions; if model_config is provided, adjust constraints according to model capabilities.
+        if constraints is None:
+            constraints = self.config.constraints
+
+            # Automatically adjust constraints according to model_config
+            if model_config is not None:
+                from dolphin.core.config.global_config import ContextConstraints
+
+                # Dynamically create constraints suitable for the current model
+                adjusted_constraints = ContextConstraints(
+                    max_input_tokens=constraints.max_input_tokens,
+                    reserve_output_tokens=model_config.max_tokens,  # Use the model's max_tokens as reserved output
+                    preserve_system=constraints.preserve_system,
+                )
+                constraints = adjusted_constraints
+
+                logger.debug(
+                    f"Adjusted constraints for model {model_config.model_name}: "
+                    f"max_input={constraints.max_input_tokens}, "
+                    f"reserve_output={constraints.reserve_output_tokens}"
+                )
+
+        # Perform compression
+        result = strategy.compress(
+            context=self.context, messages=messages, constraints=constraints, **kwargs
+        )
+
+        # Log records
+        if result.compression_ratio < 1.0:
+            self.context.info(
+                f"Context compressed using {strategy_name}: "
+                f"{result.original_token_count} -> {result.compressed_token_count} tokens "
+                f"(ratio: {result.compression_ratio:.2f})"
+            )
+
+        return result
+
+    def register_strategy(self, name: str, strategy: CompressionStrategy):
+        """Register a new compression strategy"""
+        self.strategies[name] = strategy
+        logger.debug(f"Registered new compression strategy: {name}")
+
+    def get_available_strategies(self) -> List[str]:
+        """Get the list of available compression strategies"""
+        return list(self.strategies.keys())
+
+    def estimate_tokens(self, messages: Messages) -> int:
+        """Estimate the number of tokens in a message"""
+        # Token estimation method using default strategy
+        default_strategy = self.strategies.get(
+            self.config.default_strategy, TruncationStrategy()
+        )
+        return default_strategy.estimate_tokens(messages)

dolphin/core/context/var_output.py

@@ -0,0 +1,82 @@
+from dolphin.core.common.types import SourceType, Var
+from dolphin.core.common.enums import SkillInfo
+
+
+class VarOutput(Var):
+    def __init__(self, name, value, source_type=SourceType.OTHER, skill_info=None):
+        super().__init__(value)
+
+        self.name = name
+        self.source_type = source_type
+        self.skill_info = skill_info
+
+    def add(self, var: Var):
+        if self.source_type == SourceType.LIST:
+            self.val.append(var)
+            return self
+
+        return VarOutput(name=self.name, value=[self, var], source_type=SourceType.LIST)
+
+    def set_last(self, var: Var):
+        if self.source_type == SourceType.LIST:
+            self.val[-1] = var
+            return self
+
+    def to_dict(self):
+        # Processing the serialization of value
+        if self.source_type == SourceType.LIST:
+            # LIST type: recursively serialize each VarOutput item
+            value = [item.to_dict() for item in self.val]
+        elif self.source_type == SourceType.EXPLORE:
+            # EXPLORE type: value is already a list of dictionaries, use directly
+            value = self.val
+        else:
+            value = self.val
+
+        return {
+            "__type__": "VarOutput",  # Type identifier for deserialization
+            "name": self.name,
+            "value": value,
+            "source_type": self.source_type.value,
+            "skill_info": self.skill_info.to_dict() if self.skill_info else {},
+        }
+
+    @staticmethod
+    def is_serialized_dict(data) -> bool:
+        """Check if data is a serialized VarOutput dictionary"""
+        return isinstance(data, dict) and data.get("__type__") == "VarOutput"
+
+    @staticmethod
+    def from_dict(dict_data: dict) -> "VarOutput":
+        source_type = SourceType(dict_data.get("source_type"))
+        if source_type == SourceType.LIST:
+            # LIST type: recursively deserialize each VarOutput item
+            value = [VarOutput.from_dict(item) for item in dict_data["value"]]
+        elif source_type == SourceType.EXPLORE:
+            # EXPLORE type: value is a regular dictionary list (exploration results), no recursive deserialization needed
+            value = dict_data["value"]
+        else:
+            value = dict_data["value"]
+
+        return VarOutput(
+            name=dict_data["name"],
+            value=value,
+            source_type=source_type,
+            skill_info=(
+                SkillInfo.from_dict(dict_data["skill_info"])
+                if dict_data["skill_info"]
+                else None
+            ),
+        )
+
+    @property
+    def value(self):
+        if self.source_type == SourceType.LIST:
+            return [item.value for item in self.val]
+        return self.val
+
+    def __str__(self):
+        return f"{self.name}: {self.val}"
+
+    def __repr__(self):
+        return self.__str__()

dolphin/core/context/variable_pool.py

@@ -0,0 +1,356 @@
+"""Dolphin Language SDK - Variable Pool Module
+
+Variable Classification Description:
+==============
+
+1. User Variables
+   - Variables defined by user scripts
+   - Returned when retrieved via the get_user_variables() method
+   - Example: result, user_input, calculation, etc.
+
+2. Internal Variables
+   - Automatically recognized: All variables starting with an underscore (_) are automatically treated as internal variables
+   - Special internal variables: props, usage
+   - Automatically excluded when retrieved via the get_user_variables() method
+
+   List of internal variables:
+   - _progress: Execution progress details
+   - _user_id: User ID (optionally included)
+   - _session_id: Session ID (optionally included)
+   - _max_answer_len: Maximum answer length (optionally included)
+   - _status: Current execution status
+   - _previous_status: Previous execution status
+   - props: Execution properties
+   - usage: Usage statistics
+
+Methods to retrieve variables:
+- get_user_variables(): Get user variables, excluding all internal variables
+- get_user_variables(include_system_context_vars=True): Get user variables, including system context variables
+- get_all_variables(): Get all variables (including internal variables)
+"""
+
+import re
+
+from dolphin.core.common.constants import KEY_STATUS, KEY_PREVIOUS_STATUS
+from dolphin.core.common.types import SourceType, Var
+from dolphin.core.context.var_output import VarOutput
+
+
+class VariablePool:
+    def __init__(self):
+        self.variable_pool = {}
+
+    def copy(self):
+        new_variable_pool = VariablePool()
+        new_variable_pool.variable_pool = self.variable_pool.copy()
+        return new_variable_pool
+
+    def init_variables(self, variables):
+        self.variable_pool = {name: Var(value) for name, value in variables.items()}
+
+    def contain_var(self, name):
+        return name in self.variable_pool
+
+    def get_var(self, name):
+        return self.variable_pool.get(name)
+
+    def get_var_value(self, name, default_value=None):
+        var = self.variable_pool.get(name)
+        if not var:
+            return default_value
+
+        val = var.value
+        # Compatibility with dictionary structure after restoration from snapshot (VarOutput.to_dict format)
+        if isinstance(val, dict) and "value" in val and "source_type" in val:
+            inner = val.get("value")
+            # List scenario: extract the value of each element
+            if isinstance(inner, list):
+                return [
+                    (
+                        item.get("value")
+                        if isinstance(item, dict) and "value" in item
+                        else item
+                    )
+                    for item in inner
+                ]
+            # Normal scenario: return the value field directly
+            return inner
+
+        return val
+
+    def get_var_path_value(self, varpath, default_value=None):
+        """
+        Get value from variable pool using dot notation path
+        Example: get_var_path_value('user.profile.name')
+        :param varpath: Variable path with dot notation
+        :return: Value at the specified path
+        :raises: AttributeError if variable or path not found
+        """
+        if not varpath:
+            return default_value
+
+        parts = varpath.split(".")
+        base_var = parts[0]
+
+        # Get base variable
+        var = self.get_var(base_var)
+        if var is None:
+            return default_value
+
+        value = var.value
+
+        # Navigate through the path
+        for part in parts[1:]:
+            if isinstance(value, dict):
+                if part not in value:
+                    return default_value
+                value = value[part]
+            else:
+                return default_value
+
+        return value
+
+    def get_all_variables(self):
+        return {name: var.to_dict() for name, var in self.variable_pool.items()}
+
+    def get_user_variables(self, include_system_context_vars=False):
+        """Get user-defined variables, excluding internal variables.
+
+        Args:
+            include_system_context_vars: Whether to include system context variables (e.g., _user_id, _session_id, etc.)
+                                                 Default is False for backward compatibility
+
+                Internal variable rules:
+                - All variables starting with an underscore (_) are automatically considered internal variables (unless explicitly included)
+                - Special internal variables: props, usage
+
+                If include_system_context_vars=True, the following system context variables will be included:
+                - _user_id: User ID
+                - _session_id: Session ID
+                - _max_answer_len: Maximum answer length
+        """
+        # Automatically recognize all variables starting with an underscore as built-in variables
+        underscore_vars = {
+            name for name in self.variable_pool.keys() if name.startswith("_")
+        }
+
+        # Extra internal variables (not starting with underscore)
+        additional_internal_vars = {"props", KEY_PREVIOUS_STATUS, KEY_STATUS, "usage"}
+
+        # System context variables (if requested by the user, these variables are not excluded)
+        system_context_vars = {"_user_id", "_session_id", "_max_answer_len"}
+
+        # Set of variables that need to be excluded in the end
+        if include_system_context_vars:
+            # Include system context variables: exclude all variables starting with underscores and additional internal variables, except for system context variables
+            internal_vars = (
+                underscore_vars - system_context_vars
+            ) | additional_internal_vars
+        else:
+            # Default behavior: exclude all underscore variables and additional internal variables
+            internal_vars = underscore_vars | additional_internal_vars
+
+        return {
+            name: var.to_dict()
+            for name, var in self.variable_pool.items()
+            if name not in internal_vars
+        }
+
+    def get_all_variables_values(self):
+        result = {}
+        for name, var in self.variable_pool.items():
+            val = var.value
+            if isinstance(val, dict) and "value" in val and "source_type" in val:
+                inner = val.get("value")
+                if isinstance(inner, list):
+                    result[name] = [
+                        (
+                            item.get("value")
+                            if isinstance(item, dict) and "value" in item
+                            else item
+                        )
+                        for item in inner
+                    ]
+                else:
+                    result[name] = inner
+            else:
+                result[name] = val
+        return result
+
+    def keys(self):
+        return self.variable_pool.keys()
+
+    def set_var(self, name, value):
+        if isinstance(value, Var):
+            self.variable_pool[name] = value
+        else:
+            self.variable_pool[name] = Var(value)
+
+    def set_var_output(
+        self, name, value, source_type=SourceType.OTHER, skill_info=None
+    ):
+        # Create VarOutput with all parameters
+        self.variable_pool[name] = VarOutput(name, value, source_type, skill_info)
+
+    def delete_var(self, name):
+        if name in self.variable_pool:
+            del self.variable_pool[name]
+
+    def sync_variables(self, variable_pool: "VariablePool"):
+        for name, var in variable_pool.variable_pool.items():
+            self.variable_pool[name] = var
+
+    def clear(self):
+        self.variable_pool.clear()
+
+    def get_variable_type(self, variable_str):
+        """Get variable value from string, including simple variables, array indices, and nested attributes
+                :param variable_str: Variable string, for example '$var', '$arr[0]', '$obj.attr'
+                :return: Variable value
+        """
+        variable_name = variable_str[1:]
+        variable_name = convert_object_to_dict_access(variable_name)
+        if "." not in variable_str and "[" not in variable_str:
+            var = self.get_var(variable_name)
+            if var is None:
+                raise AttributeError(
+                    f"Variable '{variable_name}' not found in variable pool"
+                )
+            value = var.value
+        else:
+            while "[0" in variable_name and "[0]" not in variable_name:
+                variable_name = variable_name.replace("[0", "[")
+
+            variable_first_name = (
+                variable_name.split(".")[0].strip().split("[")[0].strip()
+            )
+            var = self.get_var(variable_first_name)
+            if var is None:
+                raise AttributeError(
+                    f"Variable '{variable_first_name}' not found in variable pool"
+                )
+            variable_value = var.value
+            variable_full_index = variable_name.replace(
+                variable_first_name, str(variable_value), 1
+            )
+            try:
+                value = eval(variable_full_index)
+            except Exception:
+                raise AttributeError(f"failed to eval '{variable_full_index}'")
+
+        if (
+            isinstance(value, list)
+            and len(value) > 0
+            and isinstance(value[0], dict)
+            and "agent_name" in value[0]
+        ):
+            result = ""
+            for item in value:
+                if (
+                    isinstance(item, dict)
+                    and item.get("agent_name") == "main"
+                    and "answer" in item
+                ):
+                    result += item["answer"]
+            value = result
+        return value
+
+    @staticmethod
+    def find_substring_positions(main_string, substring):
+        positions = []
+        start = 0
+
+        while True:
+            # Find the position of a substring
+            pos = main_string.find(substring, start)
+            if pos == -1:
+                break
+            positions.append(pos)
+            # Update the starting position to avoid infinite loops
+            start = pos + 1
+        return positions
+
+    def recognize_variable(self, dolphin_str):
+        # Identify the positions of all variables in the string - Simple variables: `$variableName` - Array indices: `$variableName[index]` - Nested properties: `$variableName.key1.key2`
+        """Identify the positions of all variables in a string
+                :param dolphin_str: The string to be identified
+                :return: A list of tuples containing variable names and their positions [('variable name', (start position, end position)), ...]
+        """
+        prompt_variable_pattern = re.compile(
+            r"\$[a-zA-Z_][a-zA-Z0-9_]*(?:(?:\.[a-zA-Z_][a-zA-Z0-9_]*)|(?:\[\d+\]))*"
+        )
+        matches = prompt_variable_pattern.findall(dolphin_str)
+
+        result = []
+
+        # Sort by length, with the longest variable names first
+        matches.sort(key=len, reverse=True)
+
+        # Record processed positions to avoid overlap
+        processed_positions = set()
+
+        for match in matches:
+            index_list = VariablePool.find_substring_positions(dolphin_str, match)
+
+            for start_pos in index_list:
+                end_pos = start_pos + len(match)
+
+                # Check whether this position has already been processed
+                if any(
+                    start_pos >= existing_start and end_pos <= existing_end
+                    for existing_start, existing_end in processed_positions
+                ):
+                    continue
+
+                # Check whether a variable exists in the variable pool
+                variable_base_name = match[1:].split(".")[0].split("[")[0].strip()
+                if variable_base_name in self.variable_pool.keys():
+                    result.append((match, (start_pos, end_pos)))
+                    processed_positions.add((start_pos, end_pos))
+
+        return result
+
+
+def convert_object_to_dict_access(access_str):
+    """
+    Convert object attribute access to dictionary access format
+    Example: "result.text[0].content[0].answer" -> "result['text'][0]['content'][0]['answer']"
+    """
+    parts = []
+    current = ""
+    i = 0
+    while i < len(access_str):
+        if access_str[i] == ".":
+            if current:
+                parts.append(current)
+                current = ""
+            i += 1
+        elif access_str[i] == "[":
+            if current:
+                parts.append(current)
+                current = ""
+            # Find matching closing bracket
+            bracket_count = 1
+            j = i + 1
+            while j < len(access_str) and bracket_count > 0:
+                if access_str[j] == "[":
+                    bracket_count += 1
+                elif access_str[j] == "]":
+                    bracket_count -= 1
+                j += 1
+            parts.append(access_str[i:j])
+            i = j
+        else:
+            current += access_str[i]
+            i += 1
+    if current:
+        parts.append(current)
+
+    result = parts[0]
+    for i, part in enumerate(parts[1:], 1):
+        if part.startswith("["):
+            result += part
+        else:
+            result += f"['{part}']"
+
+    return result

dolphin/core/context_engineer/__init__.py

@@ -0,0 +1,41 @@
+"""
+ContextEngineer: A system for optimizing context management in large language models.
+
+This package provides tools for token budgeting, context optimization, and dynamic
+information retrieval to maximize signal density while minimizing noise in LLM contexts.
+"""
+
+__version__ = "0.1.0"
+__author__ = "ContextEngineer Team"
+
+from .core.budget_manager import BudgetManager
+from .core.tokenizer_service import TokenizerService
+from .core.context_assembler import ContextAssembler
+from .services.compressor import Compressor
+from .config.settings import ContextConfig, BucketConfig
+
+__all__ = [
+    "BudgetManager",
+    "TokenizerService",
+    "ContextAssembler",
+    "Compressor",
+    "ContextConfig",
+    "BucketConfig",
+    "ContextEngineer",
+]
+
+
+def __getattr__(name: str):
+    """Lazy import for ContextEngineer to avoid naming conflict with package."""
+    if name == "ContextEngineer":
+        import importlib.util
+        from pathlib import Path
+        
+        module_file = Path(__file__).resolve().parent.parent / "context_engineer.py"
+        if module_file.exists():
+            spec = importlib.util.spec_from_file_location("_context_engineer_module", module_file)
+            if spec and spec.loader:
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)
+                return module.ContextEngineer
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

dolphin/core/context_engineer/config/__init__.py

@@ -0,0 +1,5 @@
+"""Configuration management for context engineering."""
+
+from .settings import ContextConfig
+
+__all__ = ["ContextConfig"]