kweaver-dolphin 0.1.0__py3-none-any.whl

dolphin/core/context_engineer/utils/token_utils.py

@@ -0,0 +1,139 @@
+"""Token utility functions."""
+
+from typing import Union, Dict, List, Optional
+
+from dolphin.core.common.enums import Messages
+from ..core.tokenizer_service import TokenizerService
+
+
+def estimate_tokens(
+    text: Union[str, List[str], Dict[str, str]], avg_chars_per_token: float = 4.0
+) -> int:
+    """
+    Estimate token count based on character length.
+
+    Args:
+        text: Input text (string, list, or dict)
+        avg_chars_per_token: Average characters per token
+
+    Returns:
+        Estimated token count
+    """
+    if isinstance(text, str):
+        return int(len(text) / avg_chars_per_token)
+    elif isinstance(text, list):
+        return sum(int(len(item) / avg_chars_per_token) for item in text)
+    elif isinstance(text, dict):
+        total = 0
+        for key, value in text.items():
+            total += int(len(str(key)) / avg_chars_per_token)
+            total += int(len(str(value)) / avg_chars_per_token)
+        return total
+    else:
+        return int(len(str(text)) / avg_chars_per_token)
+
+
+def count_tokens(
+    text: Union[str, List[str], Dict[str, str]],
+    tokenizer_service: Optional[TokenizerService] = None,
+) -> int:
+    """
+    Count tokens using tokenizer service.
+
+    Args:
+        text: Input text (string, list, or dict)
+        tokenizer_service: TokenizerService instance (creates default if None)
+
+    Returns:
+        Token count
+    """
+    if tokenizer_service is None:
+        tokenizer_service = TokenizerService()
+
+    return tokenizer_service.count_tokens(text)
+
+
+def get_text_chunks(
+    text: str, max_tokens: int, tokenizer_service: Optional[TokenizerService] = None
+) -> List[str]:
+    """
+    Split text into chunks based on token limit.
+
+    Args:
+        text: Input text
+        max_tokens: Maximum tokens per chunk
+        tokenizer_service: TokenizerService instance
+
+    Returns:
+        List of text chunks
+    """
+    if tokenizer_service is None:
+        tokenizer_service = TokenizerService()
+
+    words = text.split()
+    chunks = []
+    current_chunk = []
+    current_tokens = 0
+
+    for word in words:
+        word_tokens = tokenizer_service.count_tokens(word)
+
+        if current_tokens + word_tokens > max_tokens and current_chunk:
+            chunks.append(" ".join(current_chunk))
+            current_chunk = [word]
+            current_tokens = word_tokens
+        else:
+            current_chunk.append(word)
+            current_tokens += word_tokens
+
+    if current_chunk:
+        chunks.append(" ".join(current_chunk))
+
+    return chunks
+
+
+def truncate_to_tokens(
+    text: Union[str, Messages],
+    max_tokens: int,
+    tokenizer_service: Optional[TokenizerService] = None,
+) -> str:
+    """
+    Truncate text to fit within token limit.
+
+    Args:
+        text: Input text (string or Messages)
+        max_tokens: Maximum tokens allowed
+        tokenizer_service: TokenizerService instance
+
+    Returns:
+        Truncated text
+    """
+    if tokenizer_service is None:
+        tokenizer_service = TokenizerService()
+
+    # If it is a Messages type, extract the content of all messages and concatenate them.
+    if isinstance(text, Messages):
+        contents = []
+        for msg in text.messages:
+            if hasattr(msg, "content") and msg.content:
+                contents.append(str(msg.content))
+        text_content = "\n".join(contents)
+    else:
+        text_content = str(text)
+
+    if tokenizer_service.count_tokens(text_content) <= max_tokens:
+        return text_content
+
+    words = text_content.split()
+    result = []
+    current_tokens = 0
+
+    for word in words:
+        word_tokens = tokenizer_service.count_tokens(word)
+
+        if current_tokens + word_tokens > max_tokens:
+            break
+        result.append(word)
+        current_tokens += word_tokens
+
+    return " ".join(result)

dolphin/core/coroutine/__init__.py

@@ -0,0 +1,15 @@
+# Coroutine Execution System
+from .execution_frame import ExecutionFrame, FrameStatus
+from .context_snapshot import ContextSnapshot
+from .resume_handle import ResumeHandle
+from .execution_state_registry import ExecutionStateRegistry
+from .context_snapshot_store import ContextSnapshotStore
+
+__all__ = [
+    "ExecutionFrame",
+    "FrameStatus",
+    "ContextSnapshot",
+    "ResumeHandle",
+    "ExecutionStateRegistry",
+    "ContextSnapshotStore",
+]

dolphin/core/coroutine/context_snapshot.py

@@ -0,0 +1,154 @@
+import time
+import uuid
+from dataclasses import dataclass, asdict
+from typing import Dict, List, Any
+
+
+@dataclass
+class ContextSnapshot:
+    """Context Snapshot - Saves runtime data at execution time"""
+
+    snapshot_id: str
+    frame_id: str
+    timestamp: float
+    schema_version: str = "2.0"  # Upgrade version number to support context_manager_state
+    variables: Dict[str, Any] = None
+    messages: List[Dict] = None
+    runtime_state: Dict = None
+    skillkit_state: Dict = None
+    context_manager_state: Dict = None  # Added: Save the complete bucket structure
+
+    def __post_init__(self):
+        if self.variables is None:
+            self.variables = {}
+        if self.messages is None:
+            self.messages = []
+        if self.runtime_state is None:
+            self.runtime_state = {}
+        if self.skillkit_state is None:
+            self.skillkit_state = {}
+        if self.context_manager_state is None:
+            self.context_manager_state = {}
+
+    @classmethod
+    def create_snapshot(
+        cls,
+        frame_id: str,
+        variables: Dict = None,
+        messages: List = None,
+        runtime_state: Dict = None,
+        skillkit_state: Dict = None,
+        context_manager_state: Dict = None,
+    ) -> "ContextSnapshot":
+        """Create Snapshot"""
+        return cls(
+            snapshot_id=str(uuid.uuid4()),
+            frame_id=frame_id,
+            timestamp=time.time(),
+            variables=variables or {},
+            messages=messages or [],
+            runtime_state=runtime_state or {},
+            skillkit_state=skillkit_state or {},
+            context_manager_state=context_manager_state or {},
+        )
+
+    def encode(self) -> Dict:
+        """Encode as dictionary format for serialization"""
+        return asdict(self)
+
+    @classmethod
+    def decode(cls, data: Dict) -> "ContextSnapshot":
+        """Create a snapshot from dictionary decoding"""
+        return cls(**data)
+
+    def get_variable(self, name: str, default_value=None):
+        """Get variable value"""
+        return self.variables.get(name, default_value)
+
+    def set_variable(self, name: str, value: Any):
+        """Set variable value"""
+        self.variables[name] = value
+
+    def update_variables(self, new_variables: Dict[str, Any]):
+        """Batch update variables"""
+        self.variables.update(new_variables)
+
+    def get_message_count(self) -> int:
+        """Get message count"""
+        return len(self.messages)
+
+    def add_message(self, message: Dict):
+        """Add message"""
+        self.messages.append(message)
+
+    def get_size_info(self) -> Dict[str, int]:
+        """Get snapshot size information"""
+        import json
+
+        encoded = self.encode()
+        return {
+            "variables_count": len(self.variables),
+            "messages_count": len(self.messages),
+            "total_size_bytes": len(json.dumps(encoded, ensure_ascii=False)),
+        }
+
+    def profile(
+        self,
+        format: str = 'markdown',
+        title: str = None,
+        options: Dict = None
+    ):
+        """Visual analysis report for generating snapshots
+
+        Args:
+            format: Output format, 'markdown' or 'json'
+            title: Custom title (used only for markdown)
+            options: Configuration options dictionary
+
+        Returns:
+            str (markdown) or Dict (json)
+
+        Examples:
+            # Basic usage
+            profile_md = snapshot.profile()
+            print(profile_md)
+
+                    # JSON output
+            profile_json = snapshot.profile(format='json')
+
+                    # Custom configuration
+            profile = snapshot.profile(
+                format='markdown',
+                title='Step 5: After Processing',
+                options={
+                    'size_thresholds': {
+                        'message_bytes': [1000, 10000],
+                        'variable_bytes': [1000, 10000]
+                    }
+                }
+            )
+        """
+        from .context_snapshot_profile import (
+            SnapshotProfileAnalyzer,
+            ProfileOptions,
+            MarkdownFormatter,
+            JSONFormatter
+        )
+
+        # Parse options
+        if options is None:
+            profile_options = ProfileOptions()
+        else:
+            profile_options = ProfileOptions(**options)
+
+        # Analyze Snapshot
+        analyzer = SnapshotProfileAnalyzer(self, profile_options)
+        snapshot_profile = analyzer.analyze()
+
+        # Formatted output
+        if format == 'json':
+            formatter = JSONFormatter(snapshot_profile)
+            return formatter.format()
+        else:  # markdown
+            formatter = MarkdownFormatter(snapshot_profile, title=title, options=profile_options)
+            return formatter.format()