aiecs 1.2.1__py3-none-any.whl → 1.3.1__py3-none-any.whl

aiecs/tools/search_tool/deduplicator.py

@@ -0,0 +1,123 @@
+"""
+Result Deduplication
+
+This module handles detection and removal of duplicate and highly similar
+search results.
+"""
+
+import hashlib
+from typing import Any, Dict, List
+from urllib.parse import urlparse, urlunparse
+
+
+class ResultDeduplicator:
+    """Removes duplicate and similar search results"""
+    
+    def deduplicate_results(
+        self, 
+        results: List[Dict[str, Any]],
+        similarity_threshold: float = 0.85
+    ) -> List[Dict[str, Any]]:
+        """
+        Remove duplicate and highly similar results.
+        
+        Args:
+            results: List of search results
+            similarity_threshold: Similarity threshold (0-1) for considering results as duplicates
+            
+        Returns:
+            Deduplicated list of results
+        """
+        if not results:
+            return []
+        
+        unique_results = []
+        seen_urls = set()
+        seen_content_hashes = set()
+        
+        for result in results:
+            url = result.get('link', '')
+            
+            # 1. URL deduplication (normalized)
+            normalized_url = self._normalize_url(url)
+            if normalized_url in seen_urls:
+                continue
+            
+            # 2. Content similarity deduplication
+            content_hash = self._calculate_content_hash(
+                result.get('title', ''),
+                result.get('snippet', '')
+            )
+            
+            # Check for high similarity with existing results
+            is_duplicate = False
+            for seen_hash in seen_content_hashes:
+                similarity = self._calculate_similarity(content_hash, seen_hash)
+                if similarity > similarity_threshold:
+                    is_duplicate = True
+                    break
+            
+            if is_duplicate:
+                continue
+            
+            # Add to unique results
+            unique_results.append(result)
+            seen_urls.add(normalized_url)
+            seen_content_hashes.add(content_hash)
+        
+        return unique_results
+    
+    def _normalize_url(self, url: str) -> str:
+        """
+        Normalize URL by removing query parameters and fragments.
+        
+        Args:
+            url: URL to normalize
+            
+        Returns:
+            Normalized URL
+        """
+        try:
+            parsed = urlparse(url)
+            # Keep only scheme, netloc, and path
+            normalized = urlunparse((
+                parsed.scheme,
+                parsed.netloc.lower(),
+                parsed.path.rstrip('/'),
+                '', '', ''  # Remove params, query, fragment
+            ))
+            return normalized
+        except Exception:
+            return url.lower()
+    
+    def _calculate_content_hash(self, title: str, snippet: str) -> str:
+        """
+        Calculate content hash for similarity detection.
+        
+        Args:
+            title: Result title
+            snippet: Result snippet
+            
+        Returns:
+            Content hash string
+        """
+        content = f"{title.lower()} {snippet.lower()}"
+        # Remove punctuation and normalize whitespace
+        content = ''.join(c for c in content if c.isalnum() or c.isspace())
+        content = ' '.join(content.split())
+        return hashlib.md5(content.encode()).hexdigest()
+    
+    def _calculate_similarity(self, hash1: str, hash2: str) -> float:
+        """
+        Calculate similarity between two content hashes.
+        
+        Args:
+            hash1: First content hash
+            hash2: Second content hash
+            
+        Returns:
+            Similarity score (0-1)
+        """
+        # Exact hash match
+        return 1.0 if hash1 == hash2 else 0.0
+

aiecs/tools/search_tool/error_handler.py

@@ -0,0 +1,257 @@
+"""
+Agent-Friendly Error Handling
+
+This module formats errors in an agent-friendly way with clear messages,
+suggested actions, and alternative approaches.
+"""
+
+from typing import Any, Dict, List
+
+from .constants import (
+    QuotaExceededError,
+    AuthenticationError,
+    RateLimitError,
+    CircuitBreakerOpenError,
+    ValidationError,
+    SearchAPIError
+)
+
+
+class AgentFriendlyErrorHandler:
+    """Formats errors for agent consumption with actionable suggestions"""
+    
+    def format_error_for_agent(
+        self,
+        error: Exception,
+        context: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        Format error for agent-friendly consumption.
+        
+        Args:
+            error: The exception that occurred
+            context: Context information (circuit breaker timeout, etc.)
+            
+        Returns:
+            Structured error information dictionary
+        """
+        error_response = {
+            'error_type': 'unknown',
+            'severity': 'medium',
+            'user_message': '',
+            'technical_details': str(error),
+            'suggested_actions': [],
+            'alternative_approaches': [],
+            'can_retry': False,
+            'estimated_recovery_time': None
+        }
+        
+        error_str = str(error).lower()
+        error_type = type(error).__name__
+        
+        # Handle specific error types
+        if isinstance(error, QuotaExceededError) or 'quota' in error_str or 'rate limit' in error_str:
+            self._handle_quota_exceeded(error_response)
+        
+        elif isinstance(error, AuthenticationError) or 'auth' in error_str or 'credential' in error_str:
+            self._handle_authentication_error(error_response)
+        
+        elif isinstance(error, RateLimitError):
+            self._handle_rate_limit_error(error_response)
+        
+        elif isinstance(error, CircuitBreakerOpenError) or 'circuit breaker' in error_str:
+            self._handle_circuit_breaker_error(error_response, context)
+        
+        elif isinstance(error, ValidationError) or 'invalid' in error_str or 'validation' in error_str:
+            self._handle_validation_error(error_response)
+        
+        elif 'timeout' in error_str or 'connection' in error_str or 'network' in error_str:
+            self._handle_network_error(error_response)
+        
+        elif 'no results' in error_str or 'not found' in error_str:
+            self._handle_no_results(error_response)
+        
+        else:
+            # Generic error handling
+            error_response.update({
+                'error_type': error_type,
+                'severity': 'medium',
+                'user_message': f'An unexpected error occurred: {str(error)}',
+                'suggested_actions': [
+                    'Check your query parameters',
+                    'Try simplifying the query',
+                    'Retry the operation'
+                ],
+                'can_retry': True
+            })
+        
+        return error_response
+    
+    def _handle_quota_exceeded(self, response: Dict[str, Any]):
+        """Handle quota exceeded errors"""
+        response.update({
+            'error_type': 'quota_exceeded',
+            'severity': 'high',
+            'user_message': (
+                'Search API quota has been exceeded. '
+                'The service has temporarily reached its usage limit.'
+            ),
+            'suggested_actions': [
+                'Wait 60-120 seconds before retrying',
+                'Reduce the number of results requested',
+                'Use more specific queries to get better results with fewer searches',
+                'Check if cached results are available'
+            ],
+            'alternative_approaches': [
+                'Use the scraper tool to extract information from known URLs',
+                'Query specific authoritative domains using site: operator',
+                'Defer non-urgent searches to later'
+            ],
+            'can_retry': True,
+            'estimated_recovery_time': '1-2 minutes'
+        })
+    
+    def _handle_authentication_error(self, response: Dict[str, Any]):
+        """Handle authentication errors"""
+        response.update({
+            'error_type': 'authentication_failed',
+            'severity': 'high',
+            'user_message': (
+                'Search API authentication failed. '
+                'The API credentials may be invalid or expired.'
+            ),
+            'suggested_actions': [
+                'Verify that GOOGLE_API_KEY is set correctly in environment',
+                'Check that GOOGLE_CSE_ID is valid',
+                'Ensure API key has not expired',
+                'Verify API key has Custom Search API enabled'
+            ],
+            'alternative_approaches': [
+                'Use alternative data sources (apisource_tool)',
+                'Request manual search from user'
+            ],
+            'can_retry': False,
+            'estimated_recovery_time': None
+        })
+    
+    def _handle_rate_limit_error(self, response: Dict[str, Any]):
+        """Handle rate limit errors"""
+        response.update({
+            'error_type': 'rate_limit_exceeded',
+            'severity': 'medium',
+            'user_message': (
+                'Rate limit has been exceeded. '
+                'Too many requests in a short time period.'
+            ),
+            'suggested_actions': [
+                'Wait for the suggested time before retrying',
+                'Reduce request frequency',
+                'Use cached results when available',
+                'Batch similar queries together'
+            ],
+            'alternative_approaches': [
+                'Use cached or historical data',
+                'Prioritize critical searches'
+            ],
+            'can_retry': True,
+            'estimated_recovery_time': 'As indicated in error message'
+        })
+    
+    def _handle_circuit_breaker_error(
+        self,
+        response: Dict[str, Any],
+        context: Dict[str, Any]
+    ):
+        """Handle circuit breaker open errors"""
+        timeout = context.get('circuit_breaker_timeout', 60)
+        
+        response.update({
+            'error_type': 'circuit_breaker_open',
+            'severity': 'high',
+            'user_message': (
+                'Search service is temporarily unavailable due to repeated failures. '
+                'The circuit breaker has been triggered for protection.'
+            ),
+            'suggested_actions': [
+                f'Wait {timeout} seconds for circuit to reset',
+                'Check search service status',
+                'Review recent error logs'
+            ],
+            'alternative_approaches': [
+                'Use alternative data sources',
+                'Defer search to later',
+                'Use cached or historical data'
+            ],
+            'can_retry': True,
+            'estimated_recovery_time': f'{timeout} seconds'
+        })
+    
+    def _handle_validation_error(self, response: Dict[str, Any]):
+        """Handle validation errors"""
+        response.update({
+            'error_type': 'invalid_query',
+            'severity': 'low',
+            'user_message': (
+                'The search query or parameters are invalid. '
+                'Please check the query format.'
+            ),
+            'suggested_actions': [
+                'Simplify the query - remove special characters',
+                'Check that all parameters are within valid ranges',
+                'Ensure query is not empty',
+                'Review query syntax for search operators'
+            ],
+            'alternative_approaches': [
+                'Break complex query into simpler parts',
+                'Use basic search without advanced operators'
+            ],
+            'can_retry': True,
+            'estimated_recovery_time': 'immediate (after fixing query)'
+        })
+    
+    def _handle_network_error(self, response: Dict[str, Any]):
+        """Handle network-related errors"""
+        response.update({
+            'error_type': 'network_error',
+            'severity': 'medium',
+            'user_message': (
+                'Network connection to search API failed. '
+                'This is usually a temporary issue.'
+            ),
+            'suggested_actions': [
+                'Retry the search in 5-10 seconds',
+                'Check internet connectivity',
+                'Try with a shorter timeout if query is complex'
+            ],
+            'alternative_approaches': [
+                'Use cached results if available',
+                'Try alternative search parameters'
+            ],
+            'can_retry': True,
+            'estimated_recovery_time': '10-30 seconds'
+        })
+    
+    def _handle_no_results(self, response: Dict[str, Any]):
+        """Handle no results found"""
+        response.update({
+            'error_type': 'no_results',
+            'severity': 'low',
+            'user_message': (
+                'No search results found for the query. '
+                'Try broadening your search terms.'
+            ),
+            'suggested_actions': [
+                'Remove overly specific terms',
+                'Try synonyms or related terms',
+                'Remove date restrictions',
+                'Broaden the search scope'
+            ],
+            'alternative_approaches': [
+                'Search for related topics',
+                'Try different search engines or sources',
+                'Break down into sub-queries'
+            ],
+            'can_retry': True,
+            'estimated_recovery_time': 'immediate (with modified query)'
+        })
+