aiecs 1.2.1__py3-none-any.whl → 1.3.1__py3-none-any.whl

aiecs/tools/apisource/intelligence/search_enhancer.py

@@ -0,0 +1,384 @@
+"""
+Search Result Enhancement and Ranking
+
+Intelligently scores and filters search results:
+- Calculate relevance scores using keyword matching
+- Compute popularity scores
+- Calculate recency/freshness scores
+- Apply composite scoring with configurable weights
+- Filter by quality, relevance, and date ranges
+"""
+
+import logging
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class SearchEnhancer:
+    """
+    Enhances search results with relevance scoring and intelligent filtering.
+    """
+    
+    def __init__(
+        self,
+        relevance_weight: float = 0.5,
+        popularity_weight: float = 0.3,
+        recency_weight: float = 0.2
+    ):
+        """
+        Initialize search enhancer.
+        
+        Args:
+            relevance_weight: Weight for relevance score in composite score
+            popularity_weight: Weight for popularity score in composite score
+            recency_weight: Weight for recency score in composite score
+        """
+        self.relevance_weight = relevance_weight
+        self.popularity_weight = popularity_weight
+        self.recency_weight = recency_weight
+        
+        # Normalize weights
+        total_weight = relevance_weight + popularity_weight + recency_weight
+        self.relevance_weight /= total_weight
+        self.popularity_weight /= total_weight
+        self.recency_weight /= total_weight
+    
+    def enhance_search_results(
+        self,
+        query: str,
+        results: List[Dict[str, Any]],
+        options: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Enhance search results with scoring and filtering.
+        
+        Args:
+            query: Original search query
+            results: Raw search results
+            options: Enhancement options:
+                - relevance_threshold: Minimum composite score (0-1)
+                - sort_by: Sort method ('relevance', 'popularity', 'recency', 'composite')
+                - date_range: {'start': 'YYYY-MM-DD', 'end': 'YYYY-MM-DD'}
+                - min_quality_score: Minimum quality score (0-1)
+                - max_results: Maximum number of results to return
+        
+        Returns:
+            Enhanced and filtered results
+        """
+        if not results:
+            return []
+        
+        options = options or {}
+        enhanced = []
+        
+        for result in results:
+            # Calculate scores
+            relevance = self._calculate_relevance(query, result)
+            popularity = self._get_popularity_score(result)
+            recency = self._calculate_recency(result)
+            
+            # Calculate composite score
+            composite_score = (
+                relevance * self.relevance_weight +
+                popularity * self.popularity_weight +
+                recency * self.recency_weight
+            )
+            
+            # Add search metadata
+            result_copy = result.copy()
+            result_copy['_search_metadata'] = {
+                'relevance_score': round(relevance, 3),
+                'popularity_score': round(popularity, 3),
+                'recency_score': round(recency, 3),
+                'composite_score': round(composite_score, 3),
+                'match_type': self._get_match_type(query, result)
+            }
+            
+            # Apply filters
+            if self._passes_filters(result_copy, options):
+                enhanced.append(result_copy)
+        
+        # Sort results
+        enhanced = self._sort_results(enhanced, options.get('sort_by', 'composite'))
+        
+        # Apply max results limit
+        max_results = options.get('max_results')
+        if max_results and max_results > 0:
+            enhanced = enhanced[:max_results]
+        
+        return enhanced
+    
+    def _calculate_relevance(self, query: str, result: Dict[str, Any]) -> float:
+        """
+        Calculate relevance score using keyword matching.
+        
+        Args:
+            query: Search query
+            result: Result item
+        
+        Returns:
+            Relevance score (0-1)
+        """
+        query_terms = set(query.lower().split())
+        if not query_terms:
+            return 0.0
+        
+        # Extract searchable text from result
+        searchable_fields = ['title', 'name', 'description', 'notes', 'sourceNote']
+        
+        title_text = ""
+        description_text = ""
+        
+        for field in ['title', 'name']:
+            if field in result:
+                title_text += " " + str(result[field]).lower()
+        
+        for field in ['description', 'notes', 'sourceNote']:
+            if field in result:
+                description_text += " " + str(result[field]).lower()
+        
+        # Count matches in title (weighted higher)
+        title_matches = sum(1 for term in query_terms if term in title_text)
+        title_score = min(title_matches / len(query_terms), 1.0)
+        
+        # Count matches in description
+        desc_matches = sum(1 for term in query_terms if term in description_text)
+        desc_score = min(desc_matches / len(query_terms), 1.0)
+        
+        # Weight title matches more heavily
+        relevance = title_score * 0.7 + desc_score * 0.3
+        
+        # Boost for exact phrase match
+        query_lower = query.lower()
+        if query_lower in title_text:
+            relevance = min(relevance * 1.5, 1.0)
+        
+        return relevance
+    
+    def _get_popularity_score(self, result: Dict[str, Any]) -> float:
+        """
+        Calculate popularity score based on usage indicators.
+        
+        Args:
+            result: Result item
+        
+        Returns:
+            Popularity score (0-1)
+        """
+        # Look for popularity indicators
+        popularity_fields = ['popularity', 'usage_count', 'frequency', 'popularity_rank']
+        
+        for field in popularity_fields:
+            if field in result:
+                value = result[field]
+                if isinstance(value, (int, float)):
+                    # Normalize to 0-1 range (assumes max popularity of 100)
+                    return min(value / 100, 1.0)
+        
+        # Check for "popular" or "commonly used" in metadata
+        if result.get('frequency') in ['Daily', 'Weekly', 'Monthly']:
+            # More frequent updates = more popular
+            frequency_scores = {'Daily': 1.0, 'Weekly': 0.8, 'Monthly': 0.6}
+            return frequency_scores.get(result.get('frequency'), 0.5)
+        
+        # Default: medium popularity
+        return 0.5
+    
+    def _calculate_recency(self, result: Dict[str, Any]) -> float:
+        """
+        Calculate recency/freshness score.
+        
+        Args:
+            result: Result item
+        
+        Returns:
+            Recency score (0-1)
+        """
+        # Look for date fields
+        date_fields = [
+            'updated', 'last_updated', 'observation_end',
+            'date', 'publishedAt', 'last_modified'
+        ]
+        
+        latest_date = None
+        
+        for field in date_fields:
+            if field in result:
+                date_str = result[field]
+                try:
+                    # Parse date
+                    if 'T' in str(date_str):
+                        # ISO format
+                        date_obj = datetime.fromisoformat(str(date_str).replace('Z', '+00:00'))
+                    else:
+                        # Simple date format
+                        date_obj = datetime.strptime(str(date_str)[:10], '%Y-%m-%d')
+                    
+                    if latest_date is None or date_obj > latest_date:
+                        latest_date = date_obj
+                except (ValueError, TypeError):
+                    continue
+        
+        if latest_date is None:
+            # No date found, assume moderate recency
+            return 0.5
+        
+        # Calculate age in days
+        now = datetime.utcnow()
+        age_days = (now - latest_date).days
+        
+        # Score based on age
+        if age_days < 7:
+            return 1.0  # Very recent
+        elif age_days < 30:
+            return 0.9  # Recent
+        elif age_days < 90:
+            return 0.7  # Somewhat recent
+        elif age_days < 365:
+            return 0.5  # This year
+        elif age_days < 365 * 2:
+            return 0.3  # Last 2 years
+        else:
+            # Older data, score decreases slowly
+            return max(0.1, 0.3 - (age_days - 365 * 2) / (365 * 10))
+    
+    def _get_match_type(self, query: str, result: Dict[str, Any]) -> str:
+        """
+        Determine the type of match.
+        
+        Args:
+            query: Search query
+            result: Result item
+        
+        Returns:
+            Match type string ('exact', 'partial', 'fuzzy')
+        """
+        query_lower = query.lower()
+        
+        # Check title/name fields
+        for field in ['title', 'name', 'id', 'series_id']:
+            if field in result:
+                value = str(result[field]).lower()
+                
+                if value == query_lower:
+                    return 'exact'
+                elif query_lower in value or value in query_lower:
+                    return 'partial'
+        
+        return 'fuzzy'
+    
+    def _passes_filters(
+        self,
+        result: Dict[str, Any],
+        options: Dict[str, Any]
+    ) -> bool:
+        """
+        Check if result passes filter criteria.
+        
+        Args:
+            result: Result with _search_metadata
+            options: Filter options
+        
+        Returns:
+            True if result passes all filters
+        """
+        # Relevance threshold
+        threshold = options.get('relevance_threshold', 0.0)
+        composite_score = result['_search_metadata']['composite_score']
+        if composite_score < threshold:
+            return False
+        
+        # Quality score threshold
+        min_quality = options.get('min_quality_score')
+        if min_quality is not None:
+            # Check if result has quality metadata
+            quality_score = result.get('_quality', {}).get('score')
+            if quality_score is None:
+                quality_score = result.get('metadata', {}).get('quality', {}).get('score')
+            
+            if quality_score is not None and quality_score < min_quality:
+                return False
+        
+        # Date range filter
+        date_range = options.get('date_range')
+        if date_range:
+            # Check if result falls within date range
+            result_date = self._extract_date(result)
+            if result_date:
+                start = date_range.get('start')
+                end = date_range.get('end')
+                
+                try:
+                    if start:
+                        start_date = datetime.strptime(start, '%Y-%m-%d')
+                        if result_date < start_date:
+                            return False
+                    
+                    if end:
+                        end_date = datetime.strptime(end, '%Y-%m-%d')
+                        if result_date > end_date:
+                            return False
+                except ValueError:
+                    logger.warning(f"Invalid date range format: {date_range}")
+        
+        return True
+    
+    def _extract_date(self, result: Dict[str, Any]) -> Optional[datetime]:
+        """Extract date from result"""
+        date_fields = ['date', 'observation_end', 'last_updated', 'publishedAt']
+        
+        for field in date_fields:
+            if field in result:
+                try:
+                    date_str = str(result[field])
+                    if 'T' in date_str:
+                        return datetime.fromisoformat(date_str.replace('Z', '+00:00'))
+                    else:
+                        return datetime.strptime(date_str[:10], '%Y-%m-%d')
+                except (ValueError, TypeError):
+                    continue
+        
+        return None
+    
+    def _sort_results(
+        self,
+        results: List[Dict[str, Any]],
+        sort_by: str
+    ) -> List[Dict[str, Any]]:
+        """
+        Sort results by specified criteria.
+        
+        Args:
+            results: Results with _search_metadata
+            sort_by: Sort method
+        
+        Returns:
+            Sorted results
+        """
+        if sort_by == 'relevance':
+            return sorted(
+                results,
+                key=lambda x: x['_search_metadata']['relevance_score'],
+                reverse=True
+            )
+        elif sort_by == 'popularity':
+            return sorted(
+                results,
+                key=lambda x: x['_search_metadata']['popularity_score'],
+                reverse=True
+            )
+        elif sort_by == 'recency':
+            return sorted(
+                results,
+                key=lambda x: x['_search_metadata']['recency_score'],
+                reverse=True
+            )
+        else:  # composite (default)
+            return sorted(
+                results,
+                key=lambda x: x['_search_metadata']['composite_score'],
+                reverse=True
+            )
+

aiecs/tools/apisource/monitoring/__init__.py

@@ -0,0 +1,12 @@
+"""
+Monitoring Module
+
+Contains metrics and health monitoring components.
+"""
+
+from aiecs.tools.apisource.monitoring.metrics import DetailedMetrics
+
+__all__ = [
+    'DetailedMetrics'
+]
+

aiecs/tools/apisource/monitoring/metrics.py

@@ -0,0 +1,308 @@
+"""
+Detailed Metrics and Health Monitoring for API Providers
+
+This module provides comprehensive performance tracking including:
+- Response time percentiles
+- Data volume statistics
+- Error type distribution
+- Rate limiting events
+- Cache hit rates
+- Overall health scoring
+"""
+
+import logging
+import time
+from collections import defaultdict
+from datetime import datetime
+from threading import Lock
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class DetailedMetrics:
+    """
+    Tracks detailed performance metrics for API providers.
+    
+    Provides comprehensive monitoring including response times, data volumes,
+    error patterns, and overall health scoring.
+    """
+    
+    def __init__(self, max_response_times: int = 100):
+        """
+        Initialize metrics tracker.
+        
+        Args:
+            max_response_times: Maximum number of response times to keep in memory
+        """
+        self.max_response_times = max_response_times
+        self.lock = Lock()
+        
+        # Request metrics
+        self.metrics = {
+            'requests': {
+                'total': 0,
+                'successful': 0,
+                'failed': 0,
+                'cached': 0
+            },
+            'performance': {
+                'response_times': [],  # Last N response times
+                'avg_response_time_ms': 0.0,
+                'p50_response_time_ms': 0.0,
+                'p95_response_time_ms': 0.0,
+                'p99_response_time_ms': 0.0,
+                'min_response_time_ms': 0.0,
+                'max_response_time_ms': 0.0
+            },
+            'data_volume': {
+                'total_records_fetched': 0,
+                'total_bytes_transferred': 0,
+                'avg_records_per_request': 0.0,
+                'avg_bytes_per_request': 0.0
+            },
+            'errors': {
+                'by_type': defaultdict(int),  # {error_type: count}
+                'recent_errors': []  # Last 10 errors with details
+            },
+            'rate_limiting': {
+                'throttled_requests': 0,
+                'total_wait_time_ms': 0.0,
+                'avg_wait_time_ms': 0.0
+            },
+            'timestamps': {
+                'first_request': None,
+                'last_request': None,
+                'last_success': None,
+                'last_failure': None
+            }
+        }
+    
+    def record_request(
+        self,
+        success: bool,
+        response_time_ms: float,
+        record_count: int = 0,
+        bytes_transferred: int = 0,
+        cached: bool = False,
+        error_type: Optional[str] = None,
+        error_message: Optional[str] = None
+    ):
+        """
+        Record a request with its metrics.
+        
+        Args:
+            success: Whether the request was successful
+            response_time_ms: Response time in milliseconds
+            record_count: Number of records returned
+            bytes_transferred: Bytes transferred in the response
+            cached: Whether the response was cached
+            error_type: Type of error if failed (e.g., 'timeout', 'auth', 'rate_limit')
+            error_message: Error message if failed
+        """
+        with self.lock:
+            now = datetime.utcnow().isoformat()
+            
+            # Update request counts
+            self.metrics['requests']['total'] += 1
+            if success:
+                self.metrics['requests']['successful'] += 1
+                self.metrics['timestamps']['last_success'] = now
+            else:
+                self.metrics['requests']['failed'] += 1
+                self.metrics['timestamps']['last_failure'] = now
+            
+            if cached:
+                self.metrics['requests']['cached'] += 1
+            
+            # Update timestamps
+            if self.metrics['timestamps']['first_request'] is None:
+                self.metrics['timestamps']['first_request'] = now
+            self.metrics['timestamps']['last_request'] = now
+            
+            # Update performance metrics
+            self.metrics['performance']['response_times'].append(response_time_ms)
+            if len(self.metrics['performance']['response_times']) > self.max_response_times:
+                self.metrics['performance']['response_times'].pop(0)
+            
+            # Calculate percentiles
+            self._calculate_percentiles()
+            
+            # Update data volume metrics
+            self.metrics['data_volume']['total_records_fetched'] += record_count
+            self.metrics['data_volume']['total_bytes_transferred'] += bytes_transferred
+            
+            total_requests = self.metrics['requests']['total']
+            if total_requests > 0:
+                self.metrics['data_volume']['avg_records_per_request'] = (
+                    self.metrics['data_volume']['total_records_fetched'] / total_requests
+                )
+                self.metrics['data_volume']['avg_bytes_per_request'] = (
+                    self.metrics['data_volume']['total_bytes_transferred'] / total_requests
+                )
+            
+            # Record errors
+            if not success and error_type:
+                self.metrics['errors']['by_type'][error_type] += 1
+                
+                error_entry = {
+                    'type': error_type,
+                    'message': error_message or 'Unknown error',
+                    'timestamp': now,
+                    'response_time_ms': response_time_ms
+                }
+                
+                self.metrics['errors']['recent_errors'].append(error_entry)
+                if len(self.metrics['errors']['recent_errors']) > 10:
+                    self.metrics['errors']['recent_errors'].pop(0)
+    
+    def record_rate_limit_wait(self, wait_time_ms: float):
+        """
+        Record a rate limit wait event.
+        
+        Args:
+            wait_time_ms: Time waited in milliseconds
+        """
+        with self.lock:
+            self.metrics['rate_limiting']['throttled_requests'] += 1
+            self.metrics['rate_limiting']['total_wait_time_ms'] += wait_time_ms
+            
+            throttled = self.metrics['rate_limiting']['throttled_requests']
+            if throttled > 0:
+                self.metrics['rate_limiting']['avg_wait_time_ms'] = (
+                    self.metrics['rate_limiting']['total_wait_time_ms'] / throttled
+                )
+    
+    def _calculate_percentiles(self):
+        """Calculate response time percentiles"""
+        times = sorted(self.metrics['performance']['response_times'])
+        if not times:
+            return
+        
+        n = len(times)
+        self.metrics['performance']['avg_response_time_ms'] = sum(times) / n
+        self.metrics['performance']['min_response_time_ms'] = times[0]
+        self.metrics['performance']['max_response_time_ms'] = times[-1]
+        self.metrics['performance']['p50_response_time_ms'] = times[n // 2]
+        self.metrics['performance']['p95_response_time_ms'] = times[int(n * 0.95)]
+        self.metrics['performance']['p99_response_time_ms'] = times[min(int(n * 0.99), n - 1)]
+    
+    def _calculate_health_score_unlocked(self) -> float:
+        """
+        Calculate health score without acquiring lock (internal use only).
+        Must be called while holding self.lock.
+        """
+        total = self.metrics['requests']['total']
+        if total == 0:
+            return 1.0
+
+        # Success rate score (40%)
+        success_rate = self.metrics['requests']['successful'] / total
+        success_score = success_rate * 0.4
+
+        # Performance score (30%)
+        avg_time = self.metrics['performance']['avg_response_time_ms']
+        # Assume < 200ms is excellent, > 2000ms is poor
+        if avg_time < 200:
+            performance_score = 0.3
+        elif avg_time > 2000:
+            performance_score = 0.0
+        else:
+            performance_score = max(0, min(1, (2000 - avg_time) / 1800)) * 0.3
+
+        # Cache hit rate score (20%)
+        cache_rate = self.metrics['requests']['cached'] / total
+        cache_score = cache_rate * 0.2
+
+        # Error diversity score (10%) - fewer error types is better
+        error_types = len(self.metrics['errors']['by_type'])
+        error_score = max(0, (5 - error_types) / 5) * 0.1
+
+        return success_score + performance_score + cache_score + error_score
+
+    def get_health_score(self) -> float:
+        """
+        Calculate overall health score (0-1).
+
+        The health score considers:
+        - Success rate (40%)
+        - Performance (30%)
+        - Cache hit rate (20%)
+        - Error diversity (10%)
+
+        Returns:
+            Health score between 0 and 1
+        """
+        with self.lock:
+            return self._calculate_health_score_unlocked()
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        Get all metrics as a dictionary.
+        
+        Returns:
+            Complete metrics dictionary
+        """
+        with self.lock:
+            # Convert defaultdict to regular dict for JSON serialization
+            stats = {
+                'requests': dict(self.metrics['requests']),
+                'performance': dict(self.metrics['performance']),
+                'data_volume': dict(self.metrics['data_volume']),
+                'errors': {
+                    'by_type': dict(self.metrics['errors']['by_type']),
+                    'recent_errors': list(self.metrics['errors']['recent_errors'])
+                },
+                'rate_limiting': dict(self.metrics['rate_limiting']),
+                'timestamps': dict(self.metrics['timestamps']),
+                'health_score': self.get_health_score()
+            }
+            
+            # Remove response_times array to keep output clean
+            stats['performance'] = {
+                k: v for k, v in stats['performance'].items()
+                if k != 'response_times'
+            }
+            
+            return stats
+    
+    def get_summary(self) -> Dict[str, Any]:
+        """
+        Get a concise summary of key metrics.
+
+        Returns:
+            Summary dictionary with key metrics
+        """
+        with self.lock:
+            total = self.metrics['requests']['total']
+            if total == 0:
+                return {
+                    'status': 'no_activity',
+                    'health_score': 1.0
+                }
+
+            success_rate = self.metrics['requests']['successful'] / total
+            cache_hit_rate = self.metrics['requests']['cached'] / total
+            health_score = self._calculate_health_score_unlocked()  # Use unlocked version to avoid deadlock
+
+            return {
+                'status': 'healthy' if health_score > 0.7 else 'degraded',
+                'health_score': round(health_score, 3),
+                'total_requests': total,
+                'success_rate': round(success_rate, 3),
+                'cache_hit_rate': round(cache_hit_rate, 3),
+                'avg_response_time_ms': round(
+                    self.metrics['performance']['avg_response_time_ms'], 2
+                ),
+                'p95_response_time_ms': round(
+                    self.metrics['performance']['p95_response_time_ms'], 2
+                ),
+                'total_errors': self.metrics['requests']['failed'],
+                'error_types': len(self.metrics['errors']['by_type'])
+            }
+    
+    def reset(self):
+        """Reset all metrics"""
+        with self.lock:
+            self.__init__(self.max_response_times)
+