kweaver-dolphin 0.1.0__py3-none-any.whl

dolphin/core/hook/hook_dispatcher.py

@@ -0,0 +1,380 @@
+"""Hook Dispatcher Module
+
+This module provides the core Hook dispatching logic.
+
+Key Responsibilities:
+- Dispatch to different handler types (expression, agent)
+- Build evaluation context
+- Standardize HookResult
+- Handle errors gracefully
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import asdict
+from typing import Any, Dict, Optional, TYPE_CHECKING
+
+from dolphin.core.hook.hook_types import (
+    HookConfig,
+    HookResult,
+    HookError,
+    HookValidationError,
+    HookContextProtocol,
+    AgentRef,
+)
+from dolphin.core.hook.expression_evaluator import ExpressionEvaluator, ExpressionError
+from dolphin.core.hook.isolated_variable_pool import IsolatedVariablePool
+from dolphin.core.logging.logger import get_logger
+
+if TYPE_CHECKING:
+    from dolphin.core.context.variable_pool import VariablePool
+
+logger = get_logger("hook.dispatcher")
+
+
+class FeedbackGenerator:
+    """Generate feedback messages based on verification results."""
+
+    def __init__(
+        self,
+        handler: str,
+        score: float,
+        threshold: float,
+    ):
+        self.handler = handler
+        self.score = score
+        self.threshold = threshold
+
+    def generate(self) -> str:
+        """Generate feedback message for failed verification.
+
+        Returns:
+            Formatted feedback string
+        """
+        feedback_parts = [
+            f"[Score: {self.score:.2f}, Target: {self.threshold:.2f}]",
+            "Please improve your answer based on the following feedback:",
+        ]
+
+        # Analyze expression to provide specific feedback
+        if isinstance(self.handler, str):
+            specific_feedback = self._analyze_expression()
+            if specific_feedback:
+                feedback_parts.append(specific_feedback)
+
+        return "\n".join(feedback_parts)
+
+    def _analyze_expression(self) -> str:
+        """Analyze expression to provide specific feedback."""
+        handler = self.handler
+
+        if 'len($answer)' in handler or 'len(answer)' in handler:
+            return "- Your answer may be too short. Please provide more detailed content."
+
+        if '$tool_calls' in handler or 'tool_calls' in handler:
+            return "- Please use tools to gather real data instead of making assumptions."
+
+        if '$steps' in handler or 'steps' in handler:
+            return "- Please show more reasoning steps in your analysis."
+
+        return ""
+
+
+class HookDispatcher:
+    """Hook Dispatcher - handles different types of Hook handlers.
+
+    Supports:
+    - Expression handlers: Evaluate Python-like expressions
+    - Agent handlers: Execute verification agents (.dph files)
+
+    Example:
+        ```python
+        dispatcher = HookDispatcher(
+            config=HookConfig(handler="len($answer) > 100", threshold=0.7),
+            context=OnStopContext(answer="Hello", attempt=1),
+            variable_pool=context.variable_pool
+        )
+        result = await dispatcher.dispatch()
+        ```
+    """
+
+    def __init__(
+        self,
+        config: HookConfig,
+        context: HookContextProtocol,
+        variable_pool: VariablePool,
+        runtime: Any = None,
+    ):
+        """Initialize dispatcher.
+
+        Args:
+            config: Hook configuration
+            context: Hook context (OnStopContext for on_stop hooks)
+            variable_pool: Variable pool for variable access
+            runtime: Optional runtime for agent execution
+        """
+        self.config = config
+        self.context = context
+        self.variable_pool = variable_pool
+        self.runtime = runtime
+
+    async def dispatch(self) -> HookResult:
+        """Dispatch to appropriate handler and return standardized result.
+
+        Returns:
+            HookResult with score, passed status, and optional feedback
+
+        Raises:
+            HookValidationError: For configuration errors (should stop execution)
+        """
+        handler = self.config.handler
+
+        try:
+            # 1. Determine handler type and execute
+            if isinstance(handler, str):
+                # String expression
+                score = await self._eval_expression(handler)
+                return self._build_result(score)
+
+            elif isinstance(handler, AgentRef):
+                # Agent reference - execute verification agent
+                return await self._call_agent(handler)
+
+            else:
+                raise HookValidationError(f"Unknown handler type: {type(handler)}")
+
+        except asyncio.TimeoutError as e:
+            # Timeout error
+            logger.error(f"Hook handler timeout: {handler}")
+            return HookResult(
+                score=0.0,
+                passed=False,
+                feedback=None,
+                retry=False,  # Don't retry on system error
+                error=f"Handler execution timeout after {self.config.agent_timeout}s",
+                error_type=HookError.TIMEOUT,
+                execution_status="timeout",
+            )
+
+        except ExpressionError as e:
+            # Expression syntax/evaluation error - raise to stop execution
+            raise HookValidationError(
+                f"Invalid hook expression: {handler}",
+                original_error=e,
+            ) from e
+
+        except HookValidationError:
+            # Re-raise validation errors
+            raise
+
+        except FileNotFoundError as e:
+            # Agent file not found - raise to stop execution
+            raise HookValidationError(
+                f"Verifier agent not found: {handler}",
+                original_error=e,
+            ) from e
+
+        except Exception as e:
+            # Other errors - degrade gracefully
+            logger.error(
+                f"Hook handler failed: {handler}",
+                exc_info=True,
+            )
+            return HookResult(
+                score=0.0,
+                passed=False,
+                feedback=None,
+                retry=False,  # Don't retry on validator error
+                error=str(e),
+                error_type=HookError.VALIDATOR_ERROR,
+                execution_status="validator_error",
+            )
+
+    async def _eval_expression(self, expr: str) -> float:
+        """Evaluate expression handler.
+
+        Args:
+            expr: Expression string
+
+        Returns:
+            Score between 0.0 and 1.0
+        """
+        eval_context = self._build_eval_context()
+        evaluator = ExpressionEvaluator(
+            expr=expr,
+            context=eval_context,
+            model=self.config.model,
+        )
+        return await evaluator.evaluate()
+
+    async def _call_agent(self, agent_ref: AgentRef) -> HookResult:
+        """Execute verification agent.
+
+        Args:
+            agent_ref: Reference to the verification agent
+
+        Returns:
+            HookResult from agent execution
+        """
+        if self.runtime is None:
+            raise HookValidationError(
+                "Runtime is required for agent-based verification"
+            )
+
+        try:
+            # Load .dph file
+            agent = await self.runtime.load_agent(agent_ref.path)
+        except FileNotFoundError:
+            raise  # Re-raise for dispatch() to handle
+        except Exception as e:
+            raise HookValidationError(
+                f"Failed to load verifier agent: {agent_ref.path}"
+            ) from e
+
+        try:
+            # Create isolated variable pool
+            isolated_pool = self._create_isolated_variable_pool()
+
+            # Inject hook context
+            isolated_pool.set('_hook_context', asdict(self.context))
+
+            # Set agent's variable pool
+            agent.variable_pool = isolated_pool
+
+            # Execute agent with timeout
+            result = await asyncio.wait_for(
+                agent.execute(),
+                timeout=self.config.agent_timeout,
+            )
+
+            # Parse agent result
+            return self._parse_agent_result(result)
+
+        except asyncio.TimeoutError:
+            raise  # Re-raise for dispatch() to handle
+
+        except Exception as e:
+            logger.error(
+                f"Verifier agent {agent_ref.path} failed",
+                exc_info=True,
+            )
+            raise
+
+        finally:
+            # Cleanup agent resources
+            if 'agent' in dir() and hasattr(agent, 'cleanup'):
+                await agent.cleanup()
+
+    def _create_isolated_variable_pool(self) -> IsolatedVariablePool:
+        """Create isolated variable pool for agent execution.
+
+        Returns:
+            IsolatedVariablePool with read-only access
+        """
+        exposed_vars = self.config.exposed_variables
+
+        return IsolatedVariablePool(
+            parent=self.variable_pool,
+            read_only=True,
+            exposed_variables=exposed_vars if exposed_vars else None,
+        )
+
+    def _build_eval_context(self) -> Dict[str, Any]:
+        """Build context dictionary for expression evaluation.
+
+        Returns:
+            Dictionary with context fields accessible in expressions
+        """
+        # Get context as dict
+        ctx_dict = self.context.to_dict()
+
+        # Build evaluation context
+        eval_ctx = {
+            'answer': ctx_dict.get('answer', ''),
+            'think': ctx_dict.get('think', ''),
+            'steps': ctx_dict.get('steps', 0),
+            'tool_calls': len(ctx_dict.get('tool_calls', [])),
+            'attempt': ctx_dict.get('attempt', 1),
+        }
+
+        return eval_ctx
+
+    def _build_result(self, score: float) -> HookResult:
+        """Build standardized HookResult from score.
+
+        Args:
+            score: Evaluation score (0.0 to 1.0)
+
+        Returns:
+            HookResult with passed status and optional feedback
+        """
+        passed = score >= self.config.threshold
+
+        # Generate feedback if not passed
+        feedback = None
+        if not passed:
+            generator = FeedbackGenerator(
+                handler=self.config.handler if isinstance(self.config.handler, str) else str(self.config.handler),
+                score=score,
+                threshold=self.config.threshold,
+            )
+            feedback = generator.generate()
+
+        return HookResult(
+            score=score,
+            passed=passed,
+            feedback=feedback,
+            retry=not passed,
+            breakdown=None,
+        )
+
+    def _parse_agent_result(self, result: Any) -> HookResult:
+        """Parse verification agent result into HookResult.
+
+        Supports:
+        - Full HookResult dict: {"score": 0.85, "passed": true, ...}
+        - Simple numeric: 0.85
+
+        Args:
+            result: Agent execution result
+
+        Returns:
+            Parsed HookResult
+        """
+        # Handle dict result
+        if isinstance(result, dict):
+            # Check for score field
+            if 'score' in result:
+                score = float(result['score'])
+                passed = result.get('passed', score >= self.config.threshold)
+
+                return HookResult(
+                    score=score,
+                    passed=passed,
+                    feedback=result.get('feedback'),
+                    retry=result.get('retry', not passed),
+                    breakdown=result.get('breakdown'),
+                )
+
+            # Check for nested answer structure (from explore block)
+            if 'answer' in result:
+                answer = result['answer']
+                if isinstance(answer, dict) and 'score' in answer:
+                    return self._parse_agent_result(answer)
+
+        # Handle numeric result
+        if isinstance(result, (int, float)):
+            return self._build_result(float(result))
+
+        # Invalid result format
+        logger.warning(f"Invalid agent result format: {type(result)}")
+        return HookResult(
+            score=0.0,
+            passed=False,
+            feedback="Verification agent returned invalid result format",
+            retry=False,
+            error="Invalid result format",
+            error_type=HookError.INVALID_RESULT,
+            execution_status="validator_error",
+        )

dolphin/core/hook/hook_types.py

@@ -0,0 +1,248 @@
+"""Hook Types Module
+
+This module defines the data structures and protocols for the Hook-based verify system.
+
+Key Components:
+- HookConfig: Configuration for Hook behavior
+- OnStopContext: Context passed to on_stop Hook handlers
+- HookResult: Standardized result from Hook execution
+- HookContextProtocol: Protocol for extensibility
+- AgentRef: Reference to a verification agent (.dph file)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, asdict
+from typing import Optional, List, Dict, Any, Union, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class HookContextProtocol(Protocol):
+    """Protocol for Hook Context, enabling future extensibility.
+
+    Any Hook Context (on_stop, on_start, on_error, etc.) must satisfy this protocol.
+    """
+    attempt: int
+    stage: str
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert context to dictionary representation."""
+        ...
+
+
+@dataclass
+class AgentRef:
+    """Reference to a verification agent (.dph file).
+
+    Attributes:
+        path: Path to the .dph file (without extension)
+    """
+    path: str
+
+    def __post_init__(self):
+        # Remove @ prefix if present
+        if self.path.startswith('@'):
+            self.path = self.path[1:]
+        # Add .dph extension if not present
+        if not self.path.endswith('.dph'):
+            self.path = f"{self.path}.dph"
+
+
+@dataclass
+class HookConfig:
+    """Configuration for Hook behavior.
+
+    Attributes:
+        handler: The handler for the Hook - can be an expression string or AgentRef
+        threshold: Score threshold for passing (0.0 to 1.0)
+        max_retries: Maximum number of retry attempts
+        model: Optional model to use for verification
+        context: Optional context configuration (e.g., exposed_variables)
+        llm_timeout: Timeout for LLM calls in seconds
+        agent_timeout: Timeout for agent execution in seconds
+    """
+    handler: Union[str, AgentRef]
+    threshold: float = 0.5
+    max_retries: int = 0
+    model: Optional[str] = None
+    context: Optional[Dict[str, Any]] = None
+    llm_timeout: int = 30
+    agent_timeout: int = 60
+
+    def __post_init__(self):
+        # Validate threshold range
+        if not 0.0 <= self.threshold <= 1.0:
+            raise ValueError(f"threshold must be between 0.0 and 1.0, got {self.threshold}")
+
+        # Validate max_retries is non-negative
+        if self.max_retries < 0:
+            raise ValueError(f"max_retries must be non-negative, got {self.max_retries}")
+
+        # Limit max_retries to prevent infinite loops
+        if self.max_retries > 10:
+            raise ValueError(f"max_retries cannot exceed 10, got {self.max_retries}")
+
+    @property
+    def exposed_variables(self) -> List[str]:
+        """Get list of exposed variables from context config."""
+        if self.context and 'exposed_variables' in self.context:
+            return self.context['exposed_variables']
+        return []
+
+
+@dataclass
+class OnStopContext:
+    """Context for on_stop Hook - passed to handlers.
+
+    This is the specialized context for on_stop hooks, containing
+    all information about the explore block execution.
+
+    Attributes:
+        attempt: Current attempt number (1-indexed)
+        stage: Execution stage (always "explore" for on_stop)
+        answer: The generated answer from explore
+        think: The thinking/reasoning process
+        steps: Number of execution steps
+        tool_calls: List of tool calls made during exploration
+    """
+    # Common fields (satisfies HookContextProtocol)
+    attempt: int
+    stage: str = "explore"
+
+    # on_stop specific fields
+    answer: str = ""
+    think: str = ""
+    steps: int = 0
+    tool_calls: List[Dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert context to dictionary representation."""
+        return asdict(self)
+
+
+@dataclass
+class HookResult:
+    """Result from Hook execution.
+
+    Attributes:
+        score: Quality score between 0.0 and 1.0
+        passed: Whether the verification passed (score >= threshold)
+        feedback: Optional improvement suggestions
+        retry: Whether to retry (defaults to not passed)
+        breakdown: Optional score breakdown for debugging
+        error: Error message if validation failed
+        error_type: Type of error that occurred
+        execution_status: Overall execution status
+    """
+    score: float
+    passed: bool
+    feedback: Optional[str] = None
+    retry: bool = True
+    breakdown: Optional[Dict[str, Any]] = None
+
+    # Error handling fields
+    error: Optional[str] = None
+    error_type: Optional[str] = None
+    execution_status: str = "success"
+
+    def __post_init__(self):
+        # Normalize score to 0-1 range
+        self.score = max(0.0, min(1.0, self.score))
+
+        # Default retry to not passed
+        if self.retry is None:
+            self.retry = not self.passed
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert result to dictionary representation."""
+        result = {
+            'score': self.score,
+            'passed': self.passed,
+            'retry': self.retry,
+            'execution_status': self.execution_status,
+        }
+        if self.feedback:
+            result['feedback'] = self.feedback
+        if self.breakdown:
+            result['breakdown'] = self.breakdown
+        if self.error:
+            result['error'] = self.error
+            result['error_type'] = self.error_type
+        return result
+
+
+class HookError:
+    """Hook error type constants."""
+    VALIDATOR_ERROR = "validator_error"
+    TIMEOUT = "timeout"
+    INVALID_RESULT = "invalid_result"
+    EXPRESSION_ERROR = "expression_error"
+    AGENT_LOAD_ERROR = "agent_load_error"
+
+
+class HookValidationError(Exception):
+    """Exception raised when hook validation configuration is invalid.
+
+    This error is raised for configuration errors that should stop execution,
+    not for validation failures during runtime.
+    """
+    def __init__(self, message: str, original_error: Optional[Exception] = None):
+        super().__init__(message)
+        self.original_error = original_error
+
+
+def parse_hook_config(on_stop_value: Any) -> Optional[HookConfig]:
+    """Parse on_stop parameter value into HookConfig.
+
+    Supports multiple formats:
+    - None: No hook configured
+    - str: Simple expression handler
+    - dict with 'handler' key: Full configuration
+    - AgentRef: Agent reference
+
+    Args:
+        on_stop_value: The raw on_stop parameter value
+
+    Returns:
+        HookConfig if configured, None otherwise
+
+    Raises:
+        HookValidationError: If configuration is invalid
+    """
+    if on_stop_value is None:
+        return None
+
+    # Simple expression string
+    if isinstance(on_stop_value, str):
+        # Check if it's an agent reference
+        if on_stop_value.startswith('@'):
+            return HookConfig(handler=AgentRef(on_stop_value))
+        return HookConfig(handler=on_stop_value)
+
+    # AgentRef object
+    if isinstance(on_stop_value, AgentRef):
+        return HookConfig(handler=on_stop_value)
+
+    # Full configuration dictionary
+    if isinstance(on_stop_value, dict):
+        handler = on_stop_value.get('handler')
+        if handler is None:
+            raise HookValidationError("HookConfig requires 'handler' field")
+
+        # Parse handler
+        if isinstance(handler, str) and handler.startswith('@'):
+            handler = AgentRef(handler)
+        elif isinstance(handler, dict) and 'path' in handler:
+            handler = AgentRef(handler['path'])
+
+        return HookConfig(
+            handler=handler,
+            threshold=on_stop_value.get('threshold', 0.5),
+            max_retries=on_stop_value.get('max_retries', 0),
+            model=on_stop_value.get('model'),
+            context=on_stop_value.get('context'),
+            llm_timeout=on_stop_value.get('llm_timeout', 30),
+            agent_timeout=on_stop_value.get('agent_timeout', 60),
+        )
+
+    raise HookValidationError(f"Invalid on_stop configuration type: {type(on_stop_value)}")