aiecs 1.2.1__py3-none-any.whl → 1.3.1__py3-none-any.whl

aiecs/tools/apisource/utils/__init__.py

@@ -0,0 +1,12 @@
+"""
+Utils Module
+
+Contains shared validation and utility functions.
+"""
+
+from aiecs.tools.apisource.utils.validators import DataValidator
+
+__all__ = [
+    'DataValidator'
+]
+

aiecs/tools/apisource/utils/validators.py

@@ -0,0 +1,343 @@
+"""
+Shared Validation Utilities for API Providers
+
+Common validation functions for data quality assessment:
+- Detect outliers in numeric data
+- Find gaps in time series
+- Check data completeness
+- Validate data types and ranges
+"""
+
+import logging
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class DataValidator:
+    """
+    Provides common data validation methods for API providers.
+    """
+    
+    @staticmethod
+    def detect_outliers(
+        values: List[float],
+        method: str = 'iqr',
+        threshold: float = 1.5
+    ) -> List[int]:
+        """
+        Detect outliers in numeric data.
+        
+        Args:
+            values: List of numeric values
+            method: Detection method ('iqr' or 'zscore')
+            threshold: Threshold for outlier detection
+                      - For IQR: typically 1.5 or 3.0
+                      - For Z-score: typically 2.0 or 3.0
+        
+        Returns:
+            List of indices where outliers were detected
+        """
+        if not values or len(values) < 4:
+            return []
+        
+        outlier_indices = []
+        
+        if method == 'iqr':
+            # Interquartile Range method
+            sorted_values = sorted(values)
+            n = len(sorted_values)
+            
+            q1_idx = n // 4
+            q3_idx = 3 * n // 4
+            
+            q1 = sorted_values[q1_idx]
+            q3 = sorted_values[q3_idx]
+            iqr = q3 - q1
+            
+            lower_bound = q1 - threshold * iqr
+            upper_bound = q3 + threshold * iqr
+            
+            for i, value in enumerate(values):
+                if value < lower_bound or value > upper_bound:
+                    outlier_indices.append(i)
+        
+        elif method == 'zscore':
+            # Z-score method
+            mean = sum(values) / len(values)
+            variance = sum((x - mean) ** 2 for x in values) / len(values)
+            std_dev = variance ** 0.5
+            
+            if std_dev == 0:
+                return []
+            
+            for i, value in enumerate(values):
+                z_score = abs((value - mean) / std_dev)
+                if z_score > threshold:
+                    outlier_indices.append(i)
+        
+        return outlier_indices
+    
+    @staticmethod
+    def detect_time_gaps(
+        data: List[Dict[str, Any]],
+        date_field: str = 'date',
+        expected_frequency: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Detect gaps in time series data.
+        
+        Args:
+            data: List of data items with date fields
+            date_field: Name of the date field
+            expected_frequency: Expected frequency ('daily', 'weekly', 'monthly', 'quarterly', 'annual')
+        
+        Returns:
+            List of gap information dictionaries
+        """
+        if len(data) < 2:
+            return []
+        
+        gaps = []
+        
+        # Parse dates
+        dates = []
+        for i, item in enumerate(data):
+            if date_field in item:
+                try:
+                    date_str = str(item[date_field])
+                    if 'T' in date_str:
+                        date_obj = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
+                    else:
+                        date_obj = datetime.strptime(date_str[:10], '%Y-%m-%d')
+                    dates.append((i, date_obj))
+                except (ValueError, TypeError):
+                    continue
+        
+        if len(dates) < 2:
+            return []
+        
+        # Sort by date
+        dates.sort(key=lambda x: x[1])
+        
+        # Determine expected gap if not specified
+        if expected_frequency is None:
+            # Estimate from first few intervals
+            if len(dates) >= 3:
+                intervals = [
+                    (dates[i+1][1] - dates[i][1]).days
+                    for i in range(min(3, len(dates) - 1))
+                ]
+                avg_interval = sum(intervals) / len(intervals)
+                
+                if avg_interval <= 2:
+                    expected_frequency = 'daily'
+                elif avg_interval <= 10:
+                    expected_frequency = 'weekly'
+                elif avg_interval <= 40:
+                    expected_frequency = 'monthly'
+                elif avg_interval <= 120:
+                    expected_frequency = 'quarterly'
+                else:
+                    expected_frequency = 'annual'
+        
+        # Define expected gaps in days
+        frequency_gaps = {
+            'daily': 1,
+            'weekly': 7,
+            'monthly': 31,
+            'quarterly': 92,
+            'annual': 365
+        }
+        
+        expected_gap_days = frequency_gaps.get(expected_frequency, 31)
+        tolerance = expected_gap_days * 0.5  # 50% tolerance
+        
+        # Check for gaps
+        for i in range(len(dates) - 1):
+            idx1, date1 = dates[i]
+            idx2, date2 = dates[i + 1]
+            
+            gap_days = (date2 - date1).days
+            
+            if gap_days > expected_gap_days + tolerance:
+                gaps.append({
+                    'start_index': idx1,
+                    'end_index': idx2,
+                    'start_date': date1.isoformat(),
+                    'end_date': date2.isoformat(),
+                    'gap_days': gap_days,
+                    'expected_days': expected_gap_days
+                })
+        
+        return gaps
+    
+    @staticmethod
+    def check_data_completeness(
+        data: List[Dict[str, Any]],
+        value_field: str = 'value',
+        missing_indicators: Optional[List[Any]] = None
+    ) -> Dict[str, Any]:
+        """
+        Check completeness of data.
+        
+        Args:
+            data: List of data items
+            value_field: Name of the value field to check
+            missing_indicators: Values that indicate missing data (e.g., ['.', None, 'NA'])
+        
+        Returns:
+            Completeness statistics dictionary
+        """
+        if missing_indicators is None:
+            missing_indicators = ['.', None, 'NA', 'N/A', '', 'null']
+        
+        total_records = len(data)
+        if total_records == 0:
+            return {
+                'total_records': 0,
+                'missing_count': 0,
+                'completeness': 1.0,
+                'missing_indices': []
+            }
+        
+        missing_count = 0
+        missing_indices = []
+        
+        for i, item in enumerate(data):
+            if value_field in item:
+                value = item[value_field]
+                # Check if value is missing
+                if value in missing_indicators:
+                    missing_count += 1
+                    missing_indices.append(i)
+                elif isinstance(value, str) and value.strip() in missing_indicators:
+                    missing_count += 1
+                    missing_indices.append(i)
+            else:
+                # Field doesn't exist
+                missing_count += 1
+                missing_indices.append(i)
+        
+        completeness = (total_records - missing_count) / total_records
+        
+        return {
+            'total_records': total_records,
+            'missing_count': missing_count,
+            'present_count': total_records - missing_count,
+            'completeness': round(completeness, 4),
+            'missing_indices': missing_indices[:10]  # Limit to first 10
+        }
+    
+    @staticmethod
+    def calculate_value_range(
+        data: List[Dict[str, Any]],
+        value_field: str = 'value',
+        missing_indicators: Optional[List[Any]] = None
+    ) -> Optional[Dict[str, float]]:
+        """
+        Calculate min, max, mean of numeric values.
+        
+        Args:
+            data: List of data items
+            value_field: Name of the value field
+            missing_indicators: Values to skip
+        
+        Returns:
+            Dictionary with min, max, mean, or None if no valid data
+        """
+        if missing_indicators is None:
+            missing_indicators = ['.', None, 'NA', 'N/A', '', 'null']
+        
+        numeric_values = []
+        
+        for item in data:
+            if value_field in item:
+                value = item[value_field]
+                
+                # Skip missing indicators
+                if value in missing_indicators:
+                    continue
+                
+                # Try to convert to float
+                try:
+                    if isinstance(value, (int, float)):
+                        numeric_values.append(float(value))
+                    elif isinstance(value, str):
+                        # Clean string (remove commas, etc.)
+                        cleaned = value.strip().replace(',', '')
+                        if cleaned and cleaned not in missing_indicators:
+                            numeric_values.append(float(cleaned))
+                except (ValueError, TypeError):
+                    continue
+        
+        if not numeric_values:
+            return None
+        
+        return {
+            'min': min(numeric_values),
+            'max': max(numeric_values),
+            'mean': sum(numeric_values) / len(numeric_values),
+            'count': len(numeric_values)
+        }
+    
+    @staticmethod
+    def infer_data_frequency(
+        data: List[Dict[str, Any]],
+        date_field: str = 'date'
+    ) -> Optional[str]:
+        """
+        Infer the frequency of time series data.
+        
+        Args:
+            data: List of data items with dates
+            date_field: Name of the date field
+        
+        Returns:
+            Frequency string or None
+        """
+        if len(data) < 3:
+            return None
+        
+        # Parse dates
+        dates = []
+        for item in data:
+            if date_field in item:
+                try:
+                    date_str = str(item[date_field])
+                    if 'T' in date_str:
+                        date_obj = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
+                    else:
+                        date_obj = datetime.strptime(date_str[:10], '%Y-%m-%d')
+                    dates.append(date_obj)
+                except (ValueError, TypeError):
+                    continue
+        
+        if len(dates) < 3:
+            return None
+        
+        # Sort dates
+        dates.sort()
+        
+        # Calculate intervals
+        intervals = [(dates[i+1] - dates[i]).days for i in range(len(dates) - 1)]
+        
+        # Calculate median interval
+        intervals.sort()
+        median_interval = intervals[len(intervals) // 2]
+        
+        # Classify frequency
+        if median_interval <= 2:
+            return 'daily'
+        elif median_interval <= 10:
+            return 'weekly'
+        elif median_interval <= 40:
+            return 'monthly'
+        elif median_interval <= 120:
+            return 'quarterly'
+        elif median_interval <= 400:
+            return 'annual'
+        else:
+            return 'irregular'
+

aiecs/tools/langchain_adapter.py

@@ -37,7 +37,8 @@ class LangchainToolAdapter(LangchainBaseTool):
     """
     Langchain tool adapter for single operation
 
-    Wraps one operation method of BaseTool as an independent Langchain tool
+    Wraps one operation method of BaseTool as an independent Langchain tool.
+    Supports both tool-level operations and provider-level operations.
     """
 
     # Define class attributes
@@ -46,13 +47,19 @@ class LangchainToolAdapter(LangchainBaseTool):
     base_tool_name: str = ""
     operation_name: str = ""
     operation_schema: Optional[Type[BaseModel]] = None
+    is_provider_operation: bool = False
+    provider_name: Optional[str] = None
+    method_name: Optional[str] = None
 
     def __init__(
         self,
         base_tool_name: str,
         operation_name: str,
         operation_schema: Optional[Type[BaseModel]] = None,
-        description: Optional[str] = None
+        description: Optional[str] = None,
+        is_provider_operation: bool = False,
+        provider_name: Optional[str] = None,
+        method_name: Optional[str] = None
     ):
         """
         Initialize adapter
@@ -62,6 +69,9 @@ class LangchainToolAdapter(LangchainBaseTool):
             operation_name: Operation name
             operation_schema: Pydantic Schema for the operation
             description: Tool description
+            is_provider_operation: Whether this is a provider-level operation
+            provider_name: Provider name (for provider operations)
+            method_name: Original method name (for provider operations)
         """
         # Construct tool name and description
         tool_name = f"{base_tool_name}_{operation_name}"
@@ -74,7 +84,10 @@ class LangchainToolAdapter(LangchainBaseTool):
             base_tool_name=base_tool_name,
             operation_name=operation_name,
             operation_schema=operation_schema,
-            args_schema=operation_schema
+            args_schema=operation_schema,
+            is_provider_operation=is_provider_operation,
+            provider_name=provider_name,
+            method_name=method_name
         )
     
     def _run(
@@ -87,8 +100,18 @@ class LangchainToolAdapter(LangchainBaseTool):
             # Get original tool instance
             base_tool = get_tool(self.base_tool_name)
 
-            # Execute operation
-            result = base_tool.run(self.operation_name, **kwargs)
+            # Handle provider operations differently
+            if self.is_provider_operation:
+                # For provider operations, call the query method with provider and operation
+                result = base_tool.run(
+                    'query',
+                    provider=self.provider_name,
+                    operation=self.method_name,
+                    params=kwargs
+                )
+            else:
+                # For tool-level operations, call directly
+                result = base_tool.run(self.operation_name, **kwargs)
 
             logger.info(f"Successfully executed {self.name} with result type: {type(result)}")
             return result
@@ -125,7 +148,10 @@ class ToolRegistry:
     
     def discover_operations(self, base_tool_class: Type[BaseTool]) -> List[Dict[str, Any]]:
         """
-        Discover all operation methods and Schemas of BaseTool class
+        Discover all operation methods and Schemas of BaseTool class.
+
+        Enhanced to support provider-level operations for tools like APISourceTool
+        that expose fine-grained operations from underlying providers.
 
         Args:
             base_tool_class: BaseTool subclass
@@ -135,6 +161,49 @@ class ToolRegistry:
         """
         operations = []
 
+        # 1. Discover tool-level operations (existing logic)
+        tool_operations = self._discover_tool_operations(base_tool_class)
+        operations.extend(tool_operations)
+
+        # 2. Discover provider-level operations (new logic)
+        if hasattr(base_tool_class, '_discover_provider_operations'):
+            try:
+                provider_operations = base_tool_class._discover_provider_operations()
+
+                # Convert provider operations to the expected format
+                for provider_op in provider_operations:
+                    operation_info = {
+                        'name': provider_op['name'],
+                        'method': None,  # Will be handled specially in create_langchain_tools
+                        'schema': provider_op['schema'],
+                        'description': provider_op['description'],
+                        'is_async': False,
+                        'is_provider_operation': True,  # Mark as provider operation
+                        'provider_name': provider_op.get('provider_name'),
+                        'method_name': provider_op.get('method_name')
+                    }
+                    operations.append(operation_info)
+                    logger.debug(f"Added provider operation: {provider_op['name']}")
+
+                logger.info(f"Discovered {len(provider_operations)} provider operations for {base_tool_class.__name__}")
+
+            except Exception as e:
+                logger.warning(f"Error discovering provider operations for {base_tool_class.__name__}: {e}")
+
+        return operations
+
+    def _discover_tool_operations(self, base_tool_class: Type[BaseTool]) -> List[Dict[str, Any]]:
+        """
+        Discover tool-level operations (original logic extracted to separate method).
+
+        Args:
+            base_tool_class: BaseTool subclass
+
+        Returns:
+            List of tool-level operation information
+        """
+        operations = []
+
         # Get all Schema classes
         # Build a mapping from normalized names to Schema classes
         # Check both class-level and module-level schemas
@@ -205,7 +274,8 @@ class ToolRegistry:
                 'method': method,
                 'schema': matching_schema,
                 'description': inspect.getdoc(method) or f"Execute {method_name} operation",
-                'is_async': inspect.iscoroutinefunction(method)
+                'is_async': inspect.iscoroutinefunction(method),
+                'is_provider_operation': False  # Mark as tool-level operation
             }
 
             operations.append(operation_info)
@@ -280,23 +350,31 @@ class ToolRegistry:
         langchain_tools = []
         for op_info in operations:
             # Generate enhanced description
-            enhanced_description = self._extract_description(
-                op_info['method'], 
-                tool_name, 
-                op_info['name'], 
-                op_info['schema']
-            )
-            
+            # For provider operations, use the description directly
+            if op_info.get('is_provider_operation', False):
+                enhanced_description = op_info['description']
+            else:
+                enhanced_description = self._extract_description(
+                    op_info['method'],
+                    tool_name,
+                    op_info['name'],
+                    op_info['schema']
+                )
+
+            # Create adapter with provider operation support
             adapter = LangchainToolAdapter(
                 base_tool_name=tool_name,
                 operation_name=op_info['name'],
                 operation_schema=op_info['schema'],
-                description=enhanced_description
+                description=enhanced_description,
+                is_provider_operation=op_info.get('is_provider_operation', False),
+                provider_name=op_info.get('provider_name'),
+                method_name=op_info.get('method_name')
             )
-            
+
             langchain_tools.append(adapter)
             self._langchain_tools[adapter.name] = adapter
-        
+
         logger.info(f"Created {len(langchain_tools)} Langchain tools for {tool_name}")
         return langchain_tools
     

aiecs/tools/search_tool/__init__.py

@@ -0,0 +1,102 @@
+"""
+Enhanced Search Tool Package
+
+A comprehensive, production-ready web search tool that integrates Google Custom Search API
+with advanced features including:
+
+- Result quality scoring and ranking
+- Query intent analysis and optimization
+- Result deduplication
+- Context-aware search with history tracking
+- Intelligent Redis caching with intent-aware TTL
+- Comprehensive metrics and monitoring
+- Agent-friendly error handling
+
+Features:
+- Multiple search types: web, image, news, video
+- Dual authentication: API key and service account
+- Rate limiting with token bucket algorithm
+- Circuit breaker pattern for API resilience
+- Intelligent caching with Redis backend
+- Quality analysis with authority, relevance, and freshness scoring
+- Query enhancement based on detected intent
+- Structured result summaries
+- Search context tracking and preference learning
+- Enhanced metrics and health scoring
+- Agent-optimized error messages with actionable suggestions
+
+Usage:
+    from aiecs.tools.search_tool import SearchTool
+    
+    # Create search tool instance
+    search_tool = SearchTool()
+    
+    # Perform enhanced web search
+    results = search_tool.search_web(
+        query="machine learning tutorial",
+        auto_enhance=True,
+        return_summary=True
+    )
+    
+    # Access results and quality analysis
+    for result in results['results']:
+        print(f"Title: {result['title']}")
+        print(f"Quality: {result['_quality_summary']['score']:.2f}")
+        print(f"Credibility: {result['_quality_summary']['level']}")
+    
+    # Check metrics
+    print(search_tool.get_metrics_report())
+"""
+
+from aiecs.tools import register_tool
+from .core import SearchTool
+from .constants import (
+    SearchType,
+    SafeSearch,
+    ImageSize,
+    ImageType,
+    ImageColorType,
+    QueryIntentType,
+    CredibilityLevel,
+    CircuitState,
+    # Exceptions
+    SearchToolError,
+    AuthenticationError,
+    QuotaExceededError,
+    RateLimitError,
+    CircuitBreakerOpenError,
+    SearchAPIError,
+    ValidationError,
+    CacheError
+)
+
+# Register the tool with the AIECS tool registry
+register_tool("search")(SearchTool)
+
+__all__ = [
+    # Main class
+    'SearchTool',
+    
+    # Enums
+    'SearchType',
+    'SafeSearch',
+    'ImageSize',
+    'ImageType',
+    'ImageColorType',
+    'QueryIntentType',
+    'CredibilityLevel',
+    'CircuitState',
+    
+    # Exceptions
+    'SearchToolError',
+    'AuthenticationError',
+    'QuotaExceededError',
+    'RateLimitError',
+    'CircuitBreakerOpenError',
+    'SearchAPIError',
+    'ValidationError',
+    'CacheError',
+]
+
+__version__ = '2.0.0'
+