agent_os_kernel 3.1.0__py3-none-any.whl

agent_kernel/nudge_mechanism.py

@@ -0,0 +1,263 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""
+Nudge Mechanism - Automatic retry with targeted prompting.
+
+When an agent gives up (GIVE_UP outcome), this module automatically
+injects a "nudge" prompt to encourage the agent to try harder.
+
+This implements "The Nudge" pattern from industry best practices:
+- Automatic intervention without human involvement
+- Targeted prompting based on the give-up type
+- Tracking of nudge effectiveness
+"""
+
+import logging
+import uuid
+from typing import Optional, List
+from datetime import datetime
+
+from .models import AgentOutcome, NudgeResult, GiveUpSignal
+
+logger = logging.getLogger(__name__)
+
+
+class NudgeMechanism:
+    """
+    Implements automatic nudging when agents give up.
+    
+    The "nudge" is a system prompt injection that asks the agent
+    to confirm it executed the task correctly and encourages
+    a more thorough attempt.
+    
+    Example nudge:
+    "You claimed no data was found. Please confirm you executed the 
+    search tool with the correct parameters and checked all data sources."
+    """
+    
+    def __init__(self):
+        """Initialize the nudge mechanism."""
+        self.nudge_history: List[NudgeResult] = []
+        self.nudge_templates = self._load_nudge_templates()
+        
+    def _load_nudge_templates(self) -> dict:
+        """Load nudge prompt templates for different give-up signals."""
+        return {
+            GiveUpSignal.NO_DATA_FOUND: (
+                "You claimed no data was found. Please confirm you:\n"
+                "1. Executed the search/query tool with the correct parameters\n"
+                "2. Checked all relevant data sources including archives\n"
+                "3. Used appropriate time ranges and filters\n"
+                "Please retry with a more comprehensive search strategy."
+            ),
+            GiveUpSignal.CANNOT_ANSWER: (
+                "You indicated you cannot answer this question. Please confirm you:\n"
+                "1. Have access to all necessary tools and resources\n"
+                "2. Attempted to use available tools to gather information\n"
+                "3. Considered alternative approaches to the problem\n"
+                "Please retry with a different strategy."
+            ),
+            GiveUpSignal.NO_RESULTS: (
+                "You reported no results. Please confirm you:\n"
+                "1. Used the correct query syntax and parameters\n"
+                "2. Checked for typos or incorrect field names\n"
+                "3. Tried alternative search terms or filters\n"
+                "Please retry with validated parameters."
+            ),
+            GiveUpSignal.NOT_AVAILABLE: (
+                "You indicated the resource is not available. Please confirm you:\n"
+                "1. Checked the resource location and accessibility\n"
+                "2. Verified you have the correct permissions\n"
+                "3. Checked for alternative access methods\n"
+                "Please retry with proper access verification."
+            ),
+            GiveUpSignal.INSUFFICIENT_INFO: (
+                "You claimed insufficient information. Please confirm you:\n"
+                "1. Attempted to gather additional context from available sources\n"
+                "2. Used all available tools to retrieve more information\n"
+                "3. Considered what information is actually required vs. nice-to-have\n"
+                "Please retry with available information."
+            ),
+            GiveUpSignal.UNKNOWN: (
+                "Your response suggests you may have given up. Please:\n"
+                "1. Re-read the user's request carefully\n"
+                "2. Use all available tools to attempt the task\n"
+                "3. Provide a specific explanation if truly impossible\n"
+                "Please retry with full effort."
+            )
+        }
+    
+    def generate_nudge(
+        self,
+        outcome: AgentOutcome,
+        include_tool_reminder: bool = True
+    ) -> str:
+        """
+        Generate a nudge prompt for the given outcome.
+        
+        Args:
+            outcome: The agent outcome that triggered the nudge
+            include_tool_reminder: Whether to include tool usage reminder
+            
+        Returns:
+            Nudge prompt string
+        """
+        signal = outcome.give_up_signal or GiveUpSignal.UNKNOWN
+        template = self.nudge_templates.get(signal, self.nudge_templates[GiveUpSignal.UNKNOWN])
+        
+        nudge_prompt = template
+        
+        # Add context-specific enhancements
+        if include_tool_reminder and outcome.tool_telemetry:
+            called_tools = [t.tool_name for t in outcome.tool_telemetry]
+            if called_tools:
+                # Tools were called
+                nudge_prompt += f"\n\nNote: You previously used tools: {', '.join(called_tools)}. Consider using additional tools or different parameters."
+            else:
+                # Telemetry exists but no tools were called
+                nudge_prompt += "\n\nNote: It appears no tools were called. Please use available tools to complete the task."
+        elif include_tool_reminder:
+            # No telemetry at all
+            nudge_prompt += "\n\nNote: It appears no tools were called. Please use available tools to complete the task."
+        
+        # Add original prompt reminder
+        nudge_prompt += f"\n\nOriginal request: {outcome.user_prompt}"
+        
+        return nudge_prompt
+    
+    def should_nudge(
+        self,
+        outcome: AgentOutcome,
+        max_nudges: int = 1
+    ) -> bool:
+        """
+        Determine if we should nudge for this outcome.
+        
+        Args:
+            outcome: The agent outcome
+            max_nudges: Maximum number of nudges per agent/task
+            
+        Returns:
+            True if nudge should be applied
+        """
+        # Check if outcome is a give-up
+        from .models import OutcomeType
+        if outcome.outcome_type != OutcomeType.GIVE_UP:
+            return False
+        
+        # Check if we've already nudged this agent recently
+        recent_nudges = [
+            n for n in self.nudge_history
+            if n.original_outcome.agent_id == outcome.agent_id
+            and (datetime.utcnow() - n.original_outcome.timestamp).total_seconds() < 300  # 5 min
+        ]
+        
+        if len(recent_nudges) >= max_nudges:
+            logger.info(f"Max nudges ({max_nudges}) reached for agent {outcome.agent_id}")
+            return False
+        
+        return True
+    
+    def record_nudge_result(
+        self,
+        outcome: AgentOutcome,
+        nudge_prompt: str,
+        retry_response: str,
+        retry_successful: bool
+    ) -> NudgeResult:
+        """
+        Record the result of a nudge attempt.
+        
+        Args:
+            outcome: Original outcome that triggered nudge
+            nudge_prompt: The nudge prompt that was used
+            retry_response: Agent's response after nudge
+            retry_successful: Whether the retry was successful
+            
+        Returns:
+            NudgeResult object
+        """
+        nudge_id = f"nudge-{uuid.uuid4().hex[:8]}"
+        
+        # Detect improvement
+        improvement = self._detect_improvement(
+            original_response=outcome.agent_response,
+            retry_response=retry_response
+        )
+        
+        result = NudgeResult(
+            nudge_id=nudge_id,
+            original_outcome=outcome,
+            nudge_prompt=nudge_prompt,
+            retry_response=retry_response,
+            retry_successful=retry_successful,
+            improvement_detected=improvement
+        )
+        
+        self.nudge_history.append(result)
+        
+        logger.info(f"Nudge {nudge_id}: success={retry_successful}, improvement={improvement}")
+        
+        return result
+    
+    def _detect_improvement(
+        self,
+        original_response: str,
+        retry_response: str
+    ) -> bool:
+        """
+        Detect if retry response shows improvement over original.
+        
+        Simple heuristic: longer response with less refusal language.
+        """
+        # Length improvement
+        length_improved = len(retry_response) > len(original_response) * 1.2
+        
+        # Refusal language reduction
+        refusal_words = ["no data", "cannot", "can't", "unable", "not found"]
+        original_refusals = sum(1 for word in refusal_words if word in original_response.lower())
+        retry_refusals = sum(1 for word in refusal_words if word in retry_response.lower())
+        refusal_reduced = retry_refusals < original_refusals
+        
+        # Check for data/results mention
+        data_indicators = ["found", "results", "data shows", "here is", "here are"]
+        has_data_now = any(indicator in retry_response.lower() for indicator in data_indicators)
+        
+        return length_improved or refusal_reduced or has_data_now
+    
+    def get_nudge_stats(self) -> dict:
+        """Get statistics about nudge effectiveness."""
+        if not self.nudge_history:
+            return {
+                "total_nudges": 0,
+                "successful_nudges": 0,
+                "success_rate": 0.0,
+                "improvements": 0,
+                "improvement_rate": 0.0
+            }
+        
+        successful = sum(1 for n in self.nudge_history if n.retry_successful)
+        improvements = sum(1 for n in self.nudge_history if n.improvement_detected)
+        
+        return {
+            "total_nudges": len(self.nudge_history),
+            "successful_nudges": successful,
+            "success_rate": successful / len(self.nudge_history),
+            "improvements": improvements,
+            "improvement_rate": improvements / len(self.nudge_history),
+            "recent_nudges": self.nudge_history[-10:]  # Last 10 nudges
+        }
+    
+    def get_nudge_history(
+        self,
+        agent_id: Optional[str] = None,
+        limit: int = 100
+    ) -> List[NudgeResult]:
+        """Get nudge history with optional filtering."""
+        history = self.nudge_history[-limit:]
+        
+        if agent_id:
+            history = [n for n in history if n.original_outcome.agent_id == agent_id]
+        
+        return history

agent_kernel/outcome_analyzer.py

@@ -0,0 +1,338 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""
+Outcome Analyzer - Filters agent outcomes for competence issues.
+
+This is part of Loop 2 (Alignment Engine) that identifies when agents
+"give up" with negative results instead of delivering value.
+
+Enhanced with:
+- Tool execution telemetry to distinguish valid empty results from laziness
+- Semantic analysis for detecting subtle forms of refusal
+"""
+
+import logging
+import re
+from typing import Optional, List
+from datetime import datetime
+
+from .models import (
+    AgentOutcome,
+    OutcomeType,
+    GiveUpSignal,
+    ToolExecutionTelemetry,
+    ToolExecutionStatus
+)
+from .semantic_analyzer import SemanticAnalyzer
+
+logger = logging.getLogger(__name__)
+
+
+class OutcomeAnalyzer:
+    """
+    Analyzes agent outcomes to detect "Give-Up Signals" (Laziness).
+    
+    This filters for competence issues - when agents comply with safety rules
+    but fail to deliver value (e.g., "No data found" is safe, but wrong if data exists).
+    
+    Enhanced Features:
+    1. Tool Execution Telemetry - Correlates give-up signals with tool usage
+    2. Semantic Analysis - Goes beyond regex for subtle refusal detection
+    3. False Positive Prevention - Distinguishes valid empty results from laziness
+    """
+    
+    def __init__(self, use_semantic_analysis: bool = True):
+        self.give_up_patterns = self._load_give_up_patterns()
+        self.outcome_history: List[AgentOutcome] = []
+        self.use_semantic_analysis = use_semantic_analysis
+        self.semantic_analyzer = SemanticAnalyzer() if use_semantic_analysis else None
+    
+    def _load_give_up_patterns(self) -> dict:
+        """Load patterns that indicate agent is giving up."""
+        return {
+            GiveUpSignal.NO_DATA_FOUND: [
+                r"no (?:data|results|logs|records|information) (?:found|available)",
+                r"could(?:n't| not) find (?:any |the )?(?:data|logs|records|information)",
+                r"(?:data|logs|records) (?:not found|unavailable|missing)",
+                r"no matching (?:data|logs|records|results)"
+            ],
+            GiveUpSignal.CANNOT_ANSWER: [
+                r"(?:i )?cannot answer",
+                r"(?:i )?(?:can't|cannot) (?:help|assist|answer) (?:with|you)",
+                r"unable to (?:answer|respond|help)",
+                r"(?:i )?don't have (?:enough|sufficient) information"
+            ],
+            GiveUpSignal.NO_RESULTS: [
+                r"no results",
+                r"0 results",
+                r"zero results",
+                r"empty result set",
+                r"query returned (?:no|zero) results"
+            ],
+            GiveUpSignal.NOT_AVAILABLE: [
+                r"(?:not|isn't) (?:currently )?available",
+                r"(?:is|are) unavailable",
+                r"(?:service|resource|data) (?:not|isn't) available"
+            ],
+            GiveUpSignal.INSUFFICIENT_INFO: [
+                r"insufficient (?:data|information)",
+                r"not enough (?:data|information|context)",
+                r"incomplete (?:data|information)",
+                r"missing (?:required|necessary) (?:data|information)"
+            ]
+        }
+    
+    def analyze_outcome(
+        self,
+        agent_id: str,
+        user_prompt: str,
+        agent_response: str,
+        context: Optional[dict] = None,
+        tool_telemetry: Optional[List[ToolExecutionTelemetry]] = None
+    ) -> AgentOutcome:
+        """
+        Analyze an agent's outcome to determine if it gave up.
+        
+        Enhanced with tool telemetry correlation and semantic analysis.
+        
+        Args:
+            agent_id: ID of the agent
+            user_prompt: Original user request
+            agent_response: Agent's response
+            context: Additional context
+            tool_telemetry: Tool execution telemetry data
+            
+        Returns:
+            AgentOutcome with classification and analysis
+        """
+        logger.info(f"Analyzing outcome for agent {agent_id}")
+        
+        # Check if this is a give-up signal (regex-based)
+        give_up_signal = self._detect_give_up_signal(agent_response)
+        
+        # Perform semantic analysis if enabled
+        semantic_analysis = None
+        if self.use_semantic_analysis:
+            semantic_analysis = self.semantic_analyzer.analyze(
+                agent_response=agent_response,
+                user_prompt=user_prompt,
+                tool_telemetry=tool_telemetry
+            )
+            logger.debug(f"Semantic analysis: {semantic_analysis.semantic_category} "
+                        f"(confidence: {semantic_analysis.refusal_confidence:.2f})")
+        
+        # Determine outcome type with enhanced logic
+        outcome_type = self._determine_outcome_type(
+            agent_response=agent_response,
+            give_up_signal=give_up_signal,
+            tool_telemetry=tool_telemetry,
+            semantic_analysis=semantic_analysis
+        )
+        
+        if outcome_type == OutcomeType.GIVE_UP:
+            logger.warning(f"Give-up detected: signal={give_up_signal.value if give_up_signal else 'semantic'}")
+        
+        outcome = AgentOutcome(
+            agent_id=agent_id,
+            outcome_type=outcome_type,
+            user_prompt=user_prompt,
+            agent_response=agent_response,
+            give_up_signal=give_up_signal,
+            context=context or {},
+            tool_telemetry=tool_telemetry or [],
+            semantic_analysis=semantic_analysis
+        )
+        
+        self.outcome_history.append(outcome)
+        
+        return outcome
+    
+    def _detect_give_up_signal(self, response: str) -> Optional[GiveUpSignal]:
+        """
+        Detect if the response contains a give-up signal.
+        
+        These are "Negative Results" that trigger the Completeness Auditor.
+        """
+        response_lower = response.lower()
+        
+        # Check each pattern category
+        for signal_type, patterns in self.give_up_patterns.items():
+            for pattern in patterns:
+                if re.search(pattern, response_lower):
+                    logger.debug(f"Matched pattern '{pattern}' for signal {signal_type.value}")
+                    return signal_type
+        
+        return None
+    
+    def _determine_outcome_type(
+        self,
+        agent_response: str,
+        give_up_signal: Optional[GiveUpSignal],
+        tool_telemetry: Optional[List[ToolExecutionTelemetry]],
+        semantic_analysis: Optional[any]
+    ) -> OutcomeType:
+        """
+        Determine outcome type with enhanced logic.
+        
+        Considers:
+        1. Regex-based give-up signal
+        2. Tool execution telemetry
+        3. Semantic analysis
+        
+        Key Enhancement: Correlation with tool execution to avoid false positives
+        """
+        # Check regex signal
+        has_regex_signal = give_up_signal is not None
+        
+        # Check semantic signal
+        has_semantic_signal = (
+            semantic_analysis is not None and
+            semantic_analysis.is_refusal and
+            semantic_analysis.refusal_confidence > 0.6
+        )
+        
+        # Analyze tool telemetry
+        tool_analysis = self._analyze_tool_execution(tool_telemetry)
+        
+        # Decision logic with false positive prevention
+        if has_regex_signal or has_semantic_signal:
+            # Agent said "no data found" or similar
+            
+            # Check if tools were actually called and returned empty
+            if tool_analysis["tools_called"] and tool_analysis["all_empty_results"]:
+                # Valid empty result: Tools called, returned empty -> SUCCESS
+                logger.info("Give-up signal present but tools returned empty results - valid empty set")
+                return OutcomeType.SUCCESS
+            
+            elif tool_analysis["tools_called"] and tool_analysis["has_errors"]:
+                # Tools called but errored -> Potential laziness (didn't handle errors)
+                logger.warning("Give-up with tool errors - potential laziness or error handling issue")
+                return OutcomeType.GIVE_UP
+            
+            elif not tool_analysis["tools_called"]:
+                # No tools called -> Clear laziness
+                logger.warning("Give-up signal without tool execution - clear laziness")
+                return OutcomeType.GIVE_UP
+            
+            else:
+                # Mixed results or unclear -> Default to GIVE_UP for audit
+                logger.warning("Give-up signal with unclear tool usage - flagging for audit")
+                return OutcomeType.GIVE_UP
+        
+        else:
+            # No give-up signal detected
+            if len(agent_response.strip()) < 20:
+                return OutcomeType.FAILURE
+            else:
+                return OutcomeType.SUCCESS
+    
+    def _analyze_tool_execution(
+        self,
+        tool_telemetry: Optional[List[ToolExecutionTelemetry]]
+    ) -> dict:
+        """
+        Analyze tool execution telemetry.
+        
+        Returns a dict with:
+        - tools_called: bool - Were any tools called?
+        - all_empty_results: bool - Did all tools return empty results?
+        - has_errors: bool - Did any tools error?
+        - tool_count: int - Number of tools called
+        """
+        if not tool_telemetry:
+            return {
+                "tools_called": False,
+                "all_empty_results": False,
+                "has_errors": False,
+                "tool_count": 0
+            }
+        
+        called_tools = [
+            t for t in tool_telemetry
+            if t.tool_status != ToolExecutionStatus.NOT_CALLED
+        ]
+        
+        if not called_tools:
+            return {
+                "tools_called": False,
+                "all_empty_results": False,
+                "has_errors": False,
+                "tool_count": 0
+            }
+        
+        empty_results = [
+            t for t in called_tools
+            if t.tool_status == ToolExecutionStatus.EMPTY_RESULT
+        ]
+        
+        errored_tools = [
+            t for t in called_tools
+            if t.tool_status == ToolExecutionStatus.ERROR
+        ]
+        
+        return {
+            "tools_called": True,
+            "all_empty_results": len(empty_results) == len(called_tools),
+            "has_errors": len(errored_tools) > 0,
+            "tool_count": len(called_tools),
+            "empty_count": len(empty_results),
+            "error_count": len(errored_tools)
+        }
+    
+    def should_trigger_audit(self, outcome: AgentOutcome) -> bool:
+        """
+        Determine if this outcome should trigger a Completeness Audit.
+        
+        The Completeness Auditor is only triggered on "Give-Up Signals"
+        to avoid expensive auditing of every interaction.
+        
+        Args:
+            outcome: The agent outcome
+            
+        Returns:
+            True if Completeness Auditor should be triggered
+        """
+        return outcome.outcome_type == OutcomeType.GIVE_UP
+    
+    def get_give_up_rate(self, agent_id: Optional[str] = None, recent_n: int = 100) -> float:
+        """
+        Calculate the give-up rate for an agent.
+        
+        This metric helps identify agents that are consistently lazy.
+        
+        Args:
+            agent_id: Optional agent ID to filter by
+            recent_n: Number of recent outcomes to analyze
+            
+        Returns:
+            Give-up rate as a float between 0 and 1
+        """
+        outcomes = self.outcome_history[-recent_n:]
+        
+        if agent_id:
+            outcomes = [o for o in outcomes if o.agent_id == agent_id]
+        
+        if not outcomes:
+            return 0.0
+        
+        give_ups = sum(1 for o in outcomes if o.outcome_type == OutcomeType.GIVE_UP)
+        
+        return give_ups / len(outcomes)
+    
+    def get_outcome_history(
+        self,
+        agent_id: Optional[str] = None,
+        outcome_type: Optional[OutcomeType] = None,
+        limit: int = 100
+    ) -> List[AgentOutcome]:
+        """Get outcome history with optional filters."""
+        outcomes = self.outcome_history[-limit:]
+        
+        if agent_id:
+            outcomes = [o for o in outcomes if o.agent_id == agent_id]
+        
+        if outcome_type:
+            outcomes = [o for o in outcomes if o.outcome_type == outcome_type]
+        
+        return outcomes