aiecs 1.5.1__py3-none-any.whl

aiecs/tools/apisource/intelligence/search_enhancer.py

@@ -0,0 +1,388 @@
+"""
+Search Result Enhancement and Ranking
+
+Intelligently scores and filters search results:
+- Calculate relevance scores using keyword matching
+- Compute popularity scores
+- Calculate recency/freshness scores
+- Apply composite scoring with configurable weights
+- Filter by quality, relevance, and date ranges
+"""
+
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class SearchEnhancer:
+    """
+    Enhances search results with relevance scoring and intelligent filtering.
+    """
+
+    def __init__(
+        self,
+        relevance_weight: float = 0.5,
+        popularity_weight: float = 0.3,
+        recency_weight: float = 0.2,
+    ):
+        """
+        Initialize search enhancer.
+
+        Args:
+            relevance_weight: Weight for relevance score in composite score
+            popularity_weight: Weight for popularity score in composite score
+            recency_weight: Weight for recency score in composite score
+        """
+        self.relevance_weight = relevance_weight
+        self.popularity_weight = popularity_weight
+        self.recency_weight = recency_weight
+
+        # Normalize weights
+        total_weight = relevance_weight + popularity_weight + recency_weight
+        self.relevance_weight /= total_weight
+        self.popularity_weight /= total_weight
+        self.recency_weight /= total_weight
+
+    def enhance_search_results(
+        self,
+        query: str,
+        results: List[Dict[str, Any]],
+        options: Optional[Dict[str, Any]] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        Enhance search results with scoring and filtering.
+
+        Args:
+            query: Original search query
+            results: Raw search results
+            options: Enhancement options:
+                - relevance_threshold: Minimum composite score (0-1)
+                - sort_by: Sort method ('relevance', 'popularity', 'recency', 'composite')
+                - date_range: {'start': 'YYYY-MM-DD', 'end': 'YYYY-MM-DD'}
+                - min_quality_score: Minimum quality score (0-1)
+                - max_results: Maximum number of results to return
+
+        Returns:
+            Enhanced and filtered results
+        """
+        if not results:
+            return []
+
+        options = options or {}
+        enhanced = []
+
+        for result in results:
+            # Calculate scores
+            relevance = self._calculate_relevance(query, result)
+            popularity = self._get_popularity_score(result)
+            recency = self._calculate_recency(result)
+
+            # Calculate composite score
+            composite_score = (
+                relevance * self.relevance_weight
+                + popularity * self.popularity_weight
+                + recency * self.recency_weight
+            )
+
+            # Add search metadata
+            result_copy = result.copy()
+            result_copy["_search_metadata"] = {
+                "relevance_score": round(relevance, 3),
+                "popularity_score": round(popularity, 3),
+                "recency_score": round(recency, 3),
+                "composite_score": round(composite_score, 3),
+                "match_type": self._get_match_type(query, result),
+            }
+
+            # Apply filters
+            if self._passes_filters(result_copy, options):
+                enhanced.append(result_copy)
+
+        # Sort results
+        enhanced = self._sort_results(enhanced, options.get("sort_by", "composite"))
+
+        # Apply max results limit
+        max_results = options.get("max_results")
+        if max_results and max_results > 0:
+            enhanced = enhanced[:max_results]
+
+        return enhanced
+
+    def _calculate_relevance(self, query: str, result: Dict[str, Any]) -> float:
+        """
+        Calculate relevance score using keyword matching.
+
+        Args:
+            query: Search query
+            result: Result item
+
+        Returns:
+            Relevance score (0-1)
+        """
+        query_terms = set(query.lower().split())
+        if not query_terms:
+            return 0.0
+
+        # Extract searchable text from result
+
+        title_text = ""
+        description_text = ""
+
+        for field in ["title", "name"]:
+            if field in result:
+                title_text += " " + str(result[field]).lower()
+
+        for field in ["description", "notes", "sourceNote"]:
+            if field in result:
+                description_text += " " + str(result[field]).lower()
+
+        # Count matches in title (weighted higher)
+        title_matches = sum(1 for term in query_terms if term in title_text)
+        title_score = min(title_matches / len(query_terms), 1.0)
+
+        # Count matches in description
+        desc_matches = sum(1 for term in query_terms if term in description_text)
+        desc_score = min(desc_matches / len(query_terms), 1.0)
+
+        # Weight title matches more heavily
+        relevance = title_score * 0.7 + desc_score * 0.3
+
+        # Boost for exact phrase match
+        query_lower = query.lower()
+        if query_lower in title_text:
+            relevance = min(relevance * 1.5, 1.0)
+
+        return relevance
+
+    def _get_popularity_score(self, result: Dict[str, Any]) -> float:
+        """
+        Calculate popularity score based on usage indicators.
+
+        Args:
+            result: Result item
+
+        Returns:
+            Popularity score (0-1)
+        """
+        # Look for popularity indicators
+        popularity_fields = [
+            "popularity",
+            "usage_count",
+            "frequency",
+            "popularity_rank",
+        ]
+
+        for field in popularity_fields:
+            if field in result:
+                value = result[field]
+                if isinstance(value, (int, float)):
+                    # Normalize to 0-1 range (assumes max popularity of 100)
+                    return min(value / 100, 1.0)
+
+        # Check for "popular" or "commonly used" in metadata
+        if result.get("frequency") in ["Daily", "Weekly", "Monthly"]:
+            # More frequent updates = more popular
+            frequency_scores = {"Daily": 1.0, "Weekly": 0.8, "Monthly": 0.6}
+            return frequency_scores.get(result.get("frequency"), 0.5)
+
+        # Default: medium popularity
+        return 0.5
+
+    def _calculate_recency(self, result: Dict[str, Any]) -> float:
+        """
+        Calculate recency/freshness score.
+
+        Args:
+            result: Result item
+
+        Returns:
+            Recency score (0-1)
+        """
+        # Look for date fields
+        date_fields = [
+            "updated",
+            "last_updated",
+            "observation_end",
+            "date",
+            "publishedAt",
+            "last_modified",
+        ]
+
+        latest_date = None
+
+        for field in date_fields:
+            if field in result:
+                date_str = result[field]
+                try:
+                    # Parse date
+                    if "T" in str(date_str):
+                        # ISO format
+                        date_obj = datetime.fromisoformat(str(date_str).replace("Z", "+00:00"))
+                    else:
+                        # Simple date format
+                        date_obj = datetime.strptime(str(date_str)[:10], "%Y-%m-%d")
+
+                    if latest_date is None or date_obj > latest_date:
+                        latest_date = date_obj
+                except (ValueError, TypeError):
+                    continue
+
+        if latest_date is None:
+            # No date found, assume moderate recency
+            return 0.5
+
+        # Calculate age in days
+        now = datetime.utcnow()
+        age_days = (now - latest_date).days
+
+        # Score based on age
+        if age_days < 7:
+            return 1.0  # Very recent
+        elif age_days < 30:
+            return 0.9  # Recent
+        elif age_days < 90:
+            return 0.7  # Somewhat recent
+        elif age_days < 365:
+            return 0.5  # This year
+        elif age_days < 365 * 2:
+            return 0.3  # Last 2 years
+        else:
+            # Older data, score decreases slowly
+            return max(0.1, 0.3 - (age_days - 365 * 2) / (365 * 10))
+
+    def _get_match_type(self, query: str, result: Dict[str, Any]) -> str:
+        """
+        Determine the type of match.
+
+        Args:
+            query: Search query
+            result: Result item
+
+        Returns:
+            Match type string ('exact', 'partial', 'fuzzy')
+        """
+        query_lower = query.lower()
+
+        # Check title/name fields
+        for field in ["title", "name", "id", "series_id"]:
+            if field in result:
+                value = str(result[field]).lower()
+
+                if value == query_lower:
+                    return "exact"
+                elif query_lower in value or value in query_lower:
+                    return "partial"
+
+        return "fuzzy"
+
+    def _passes_filters(self, result: Dict[str, Any], options: Dict[str, Any]) -> bool:
+        """
+        Check if result passes filter criteria.
+
+        Args:
+            result: Result with _search_metadata
+            options: Filter options
+
+        Returns:
+            True if result passes all filters
+        """
+        # Relevance threshold
+        threshold = options.get("relevance_threshold", 0.0)
+        composite_score = result["_search_metadata"]["composite_score"]
+        if composite_score < threshold:
+            return False
+
+        # Quality score threshold
+        min_quality = options.get("min_quality_score")
+        if min_quality is not None:
+            # Check if result has quality metadata
+            quality_score = result.get("_quality", {}).get("score")
+            if quality_score is None:
+                quality_score = result.get("metadata", {}).get("quality", {}).get("score")
+
+            if quality_score is not None and quality_score < min_quality:
+                return False
+
+        # Date range filter
+        date_range = options.get("date_range")
+        if date_range:
+            # Check if result falls within date range
+            result_date = self._extract_date(result)
+            if result_date:
+                start = date_range.get("start")
+                end = date_range.get("end")
+
+                try:
+                    if start:
+                        start_date = datetime.strptime(start, "%Y-%m-%d")
+                        if result_date < start_date:
+                            return False
+
+                    if end:
+                        end_date = datetime.strptime(end, "%Y-%m-%d")
+                        if result_date > end_date:
+                            return False
+                except ValueError:
+                    logger.warning(f"Invalid date range format: {date_range}")
+
+        return True
+
+    def _extract_date(self, result: Dict[str, Any]) -> Optional[datetime]:
+        """Extract date from result"""
+        date_fields = [
+            "date",
+            "observation_end",
+            "last_updated",
+            "publishedAt",
+        ]
+
+        for field in date_fields:
+            if field in result:
+                try:
+                    date_str = str(result[field])
+                    if "T" in date_str:
+                        return datetime.fromisoformat(date_str.replace("Z", "+00:00"))
+                    else:
+                        return datetime.strptime(date_str[:10], "%Y-%m-%d")
+                except (ValueError, TypeError):
+                    continue
+
+        return None
+
+    def _sort_results(self, results: List[Dict[str, Any]], sort_by: str) -> List[Dict[str, Any]]:
+        """
+        Sort results by specified criteria.
+
+        Args:
+            results: Results with _search_metadata
+            sort_by: Sort method
+
+        Returns:
+            Sorted results
+        """
+        if sort_by == "relevance":
+            return sorted(
+                results,
+                key=lambda x: x["_search_metadata"]["relevance_score"],
+                reverse=True,
+            )
+        elif sort_by == "popularity":
+            return sorted(
+                results,
+                key=lambda x: x["_search_metadata"]["popularity_score"],
+                reverse=True,
+            )
+        elif sort_by == "recency":
+            return sorted(
+                results,
+                key=lambda x: x["_search_metadata"]["recency_score"],
+                reverse=True,
+            )
+        else:  # composite (default)
+            return sorted(
+                results,
+                key=lambda x: x["_search_metadata"]["composite_score"],
+                reverse=True,
+            )

aiecs/tools/apisource/monitoring/__init__.py

@@ -0,0 +1,9 @@
+"""
+Monitoring Module
+
+Contains metrics and health monitoring components.
+"""
+
+from aiecs.tools.apisource.monitoring.metrics import DetailedMetrics
+
+__all__ = ["DetailedMetrics"]

aiecs/tools/apisource/monitoring/metrics.py

@@ -0,0 +1,303 @@
+"""
+Detailed Metrics and Health Monitoring for API Providers
+
+This module provides comprehensive performance tracking including:
+- Response time percentiles
+- Data volume statistics
+- Error type distribution
+- Rate limiting events
+- Cache hit rates
+- Overall health scoring
+"""
+
+import logging
+from collections import defaultdict
+from datetime import datetime
+from threading import Lock
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class DetailedMetrics:
+    """
+    Tracks detailed performance metrics for API providers.
+
+    Provides comprehensive monitoring including response times, data volumes,
+    error patterns, and overall health scoring.
+    """
+
+    def __init__(self, max_response_times: int = 100):
+        """
+        Initialize metrics tracker.
+
+        Args:
+            max_response_times: Maximum number of response times to keep in memory
+        """
+        self.max_response_times = max_response_times
+        self.lock = Lock()
+
+        # Request metrics
+        self.metrics = {
+            "requests": {
+                "total": 0,
+                "successful": 0,
+                "failed": 0,
+                "cached": 0,
+            },
+            "performance": {
+                "response_times": [],  # Last N response times
+                "avg_response_time_ms": 0.0,
+                "p50_response_time_ms": 0.0,
+                "p95_response_time_ms": 0.0,
+                "p99_response_time_ms": 0.0,
+                "min_response_time_ms": 0.0,
+                "max_response_time_ms": 0.0,
+            },
+            "data_volume": {
+                "total_records_fetched": 0,
+                "total_bytes_transferred": 0,
+                "avg_records_per_request": 0.0,
+                "avg_bytes_per_request": 0.0,
+            },
+            "errors": {
+                "by_type": defaultdict(int),  # {error_type: count}
+                "recent_errors": [],  # Last 10 errors with details
+            },
+            "rate_limiting": {
+                "throttled_requests": 0,
+                "total_wait_time_ms": 0.0,
+                "avg_wait_time_ms": 0.0,
+            },
+            "timestamps": {
+                "first_request": None,
+                "last_request": None,
+                "last_success": None,
+                "last_failure": None,
+            },
+        }
+
+    def record_request(
+        self,
+        success: bool,
+        response_time_ms: float,
+        record_count: int = 0,
+        bytes_transferred: int = 0,
+        cached: bool = False,
+        error_type: Optional[str] = None,
+        error_message: Optional[str] = None,
+    ):
+        """
+        Record a request with its metrics.
+
+        Args:
+            success: Whether the request was successful
+            response_time_ms: Response time in milliseconds
+            record_count: Number of records returned
+            bytes_transferred: Bytes transferred in the response
+            cached: Whether the response was cached
+            error_type: Type of error if failed (e.g., 'timeout', 'auth', 'rate_limit')
+            error_message: Error message if failed
+        """
+        with self.lock:
+            now = datetime.utcnow().isoformat()
+
+            # Update request counts
+            self.metrics["requests"]["total"] += 1
+            if success:
+                self.metrics["requests"]["successful"] += 1
+                self.metrics["timestamps"]["last_success"] = now
+            else:
+                self.metrics["requests"]["failed"] += 1
+                self.metrics["timestamps"]["last_failure"] = now
+
+            if cached:
+                self.metrics["requests"]["cached"] += 1
+
+            # Update timestamps
+            if self.metrics["timestamps"]["first_request"] is None:
+                self.metrics["timestamps"]["first_request"] = now
+            self.metrics["timestamps"]["last_request"] = now
+
+            # Update performance metrics
+            self.metrics["performance"]["response_times"].append(response_time_ms)
+            if len(self.metrics["performance"]["response_times"]) > self.max_response_times:
+                self.metrics["performance"]["response_times"].pop(0)
+
+            # Calculate percentiles
+            self._calculate_percentiles()
+
+            # Update data volume metrics
+            self.metrics["data_volume"]["total_records_fetched"] += record_count
+            self.metrics["data_volume"]["total_bytes_transferred"] += bytes_transferred
+
+            total_requests = self.metrics["requests"]["total"]
+            if total_requests > 0:
+                self.metrics["data_volume"]["avg_records_per_request"] = (
+                    self.metrics["data_volume"]["total_records_fetched"] / total_requests
+                )
+                self.metrics["data_volume"]["avg_bytes_per_request"] = (
+                    self.metrics["data_volume"]["total_bytes_transferred"] / total_requests
+                )
+
+            # Record errors
+            if not success and error_type:
+                self.metrics["errors"]["by_type"][error_type] += 1
+
+                error_entry = {
+                    "type": error_type,
+                    "message": error_message or "Unknown error",
+                    "timestamp": now,
+                    "response_time_ms": response_time_ms,
+                }
+
+                self.metrics["errors"]["recent_errors"].append(error_entry)
+                if len(self.metrics["errors"]["recent_errors"]) > 10:
+                    self.metrics["errors"]["recent_errors"].pop(0)
+
+    def record_rate_limit_wait(self, wait_time_ms: float):
+        """
+        Record a rate limit wait event.
+
+        Args:
+            wait_time_ms: Time waited in milliseconds
+        """
+        with self.lock:
+            self.metrics["rate_limiting"]["throttled_requests"] += 1
+            self.metrics["rate_limiting"]["total_wait_time_ms"] += wait_time_ms
+
+            throttled = self.metrics["rate_limiting"]["throttled_requests"]
+            if throttled > 0:
+                self.metrics["rate_limiting"]["avg_wait_time_ms"] = (
+                    self.metrics["rate_limiting"]["total_wait_time_ms"] / throttled
+                )
+
+    def _calculate_percentiles(self):
+        """Calculate response time percentiles"""
+        times = sorted(self.metrics["performance"]["response_times"])
+        if not times:
+            return
+
+        n = len(times)
+        self.metrics["performance"]["avg_response_time_ms"] = sum(times) / n
+        self.metrics["performance"]["min_response_time_ms"] = times[0]
+        self.metrics["performance"]["max_response_time_ms"] = times[-1]
+        self.metrics["performance"]["p50_response_time_ms"] = times[n // 2]
+        self.metrics["performance"]["p95_response_time_ms"] = times[int(n * 0.95)]
+        self.metrics["performance"]["p99_response_time_ms"] = times[min(int(n * 0.99), n - 1)]
+
+    def _calculate_health_score_unlocked(self) -> float:
+        """
+        Calculate health score without acquiring lock (internal use only).
+        Must be called while holding self.lock.
+        """
+        total = self.metrics["requests"]["total"]
+        if total == 0:
+            return 1.0
+
+        # Success rate score (40%)
+        success_rate = self.metrics["requests"]["successful"] / total
+        success_score = success_rate * 0.4
+
+        # Performance score (30%)
+        avg_time = self.metrics["performance"]["avg_response_time_ms"]
+        # Assume < 200ms is excellent, > 2000ms is poor
+        if avg_time < 200:
+            performance_score = 0.3
+        elif avg_time > 2000:
+            performance_score = 0.0
+        else:
+            performance_score = max(0, min(1, (2000 - avg_time) / 1800)) * 0.3
+
+        # Cache hit rate score (20%)
+        cache_rate = self.metrics["requests"]["cached"] / total
+        cache_score = cache_rate * 0.2
+
+        # Error diversity score (10%) - fewer error types is better
+        error_types = len(self.metrics["errors"]["by_type"])
+        error_score = max(0, (5 - error_types) / 5) * 0.1
+
+        return success_score + performance_score + cache_score + error_score
+
+    def get_health_score(self) -> float:
+        """
+        Calculate overall health score (0-1).
+
+        The health score considers:
+        - Success rate (40%)
+        - Performance (30%)
+        - Cache hit rate (20%)
+        - Error diversity (10%)
+
+        Returns:
+            Health score between 0 and 1
+        """
+        with self.lock:
+            return self._calculate_health_score_unlocked()
+
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        Get all metrics as a dictionary.
+
+        Returns:
+            Complete metrics dictionary
+        """
+        with self.lock:
+            # Convert defaultdict to regular dict for JSON serialization
+            stats = {
+                "requests": dict(self.metrics["requests"]),
+                "performance": dict(self.metrics["performance"]),
+                "data_volume": dict(self.metrics["data_volume"]),
+                "errors": {
+                    "by_type": dict(self.metrics["errors"]["by_type"]),
+                    "recent_errors": list(self.metrics["errors"]["recent_errors"]),
+                },
+                "rate_limiting": dict(self.metrics["rate_limiting"]),
+                "timestamps": dict(self.metrics["timestamps"]),
+                "health_score": self.get_health_score(),
+            }
+
+            # Remove response_times array to keep output clean
+            stats["performance"] = {
+                k: v for k, v in stats["performance"].items() if k != "response_times"
+            }
+
+            return stats
+
+    def get_summary(self) -> Dict[str, Any]:
+        """
+        Get a concise summary of key metrics.
+
+        Returns:
+            Summary dictionary with key metrics
+        """
+        with self.lock:
+            total = self.metrics["requests"]["total"]
+            if total == 0:
+                return {"status": "no_activity", "health_score": 1.0}
+
+            success_rate = self.metrics["requests"]["successful"] / total
+            cache_hit_rate = self.metrics["requests"]["cached"] / total
+            # Use unlocked version to avoid deadlock
+            health_score = self._calculate_health_score_unlocked()
+
+            return {
+                "status": "healthy" if health_score > 0.7 else "degraded",
+                "health_score": round(health_score, 3),
+                "total_requests": total,
+                "success_rate": round(success_rate, 3),
+                "cache_hit_rate": round(cache_hit_rate, 3),
+                "avg_response_time_ms": round(
+                    self.metrics["performance"]["avg_response_time_ms"], 2
+                ),
+                "p95_response_time_ms": round(
+                    self.metrics["performance"]["p95_response_time_ms"], 2
+                ),
+                "total_errors": self.metrics["requests"]["failed"],
+                "error_types": len(self.metrics["errors"]["by_type"]),
+            }
+
+    def reset(self):
+        """Reset all metrics"""
+        with self.lock:
+            self.__init__(self.max_response_times)