empathy-framework 5.3.0__py3-none-any.whl → 5.4.0__py3-none-any.whl

empathy_os/telemetry/feedback_loop.py

@@ -1,567 +0,0 @@
-"""Agent-to-LLM Feedback Loop for Quality-Based Learning.
-
-Pattern 6 from Agent Coordination Architecture - Collect quality ratings
-on LLM responses and use feedback to inform routing decisions.
-
-Usage:
-    # Record feedback after LLM response
-    feedback = FeedbackLoop()
-    feedback.record_feedback(
-        workflow_name="code-review",
-        stage_name="analysis",
-        tier=ModelTier.CHEAP,
-        quality_score=0.8,
-        metadata={
-            "response_length": 500,
-            "tokens": 150,
-            "latency_ms": 1200
-        }
-    )
-
-    # Get tier recommendation based on historical performance
-    recommendation = feedback.recommend_tier(
-        workflow_name="code-review",
-        stage_name="analysis"
-    )
-    if recommendation.recommended_tier == ModelTier.CAPABLE:
-        print(f"Upgrade to CAPABLE tier (confidence: {recommendation.confidence})")
-
-    # Get quality stats for analysis
-    stats = feedback.get_quality_stats(
-        workflow_name="code-review",
-        stage_name="analysis"
-    )
-    print(f"Average quality: {stats.avg_quality}")
-
-Copyright 2025 Smart-AI-Memory
-Licensed under Fair Source License 0.9
-"""
-
-from __future__ import annotations
-
-import logging
-from dataclasses import dataclass, field
-from datetime import datetime
-from enum import Enum
-from typing import Any
-from uuid import uuid4
-
-logger = logging.getLogger(__name__)
-
-
-class ModelTier(str, Enum):
-    """Model tier enum matching workflows.base.ModelTier."""
-
-    CHEAP = "cheap"
-    CAPABLE = "capable"
-    PREMIUM = "premium"
-
-
-@dataclass
-class FeedbackEntry:
-    """Quality feedback for an LLM response.
-
-    Represents a single quality rating for a workflow stage execution.
-    """
-
-    feedback_id: str
-    workflow_name: str
-    stage_name: str
-    tier: str  # ModelTier value
-    quality_score: float  # 0.0 (bad) to 1.0 (excellent)
-    timestamp: datetime
-    metadata: dict[str, Any] = field(default_factory=dict)
-
-    def to_dict(self) -> dict[str, Any]:
-        """Convert to dictionary for serialization."""
-        return {
-            "feedback_id": self.feedback_id,
-            "workflow_name": self.workflow_name,
-            "stage_name": self.stage_name,
-            "tier": self.tier,
-            "quality_score": self.quality_score,
-            "timestamp": self.timestamp.isoformat() if isinstance(self.timestamp, datetime) else self.timestamp,
-            "metadata": self.metadata,
-        }
-
-    @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> FeedbackEntry:
-        """Create from dictionary."""
-        timestamp = data.get("timestamp")
-        if isinstance(timestamp, str):
-            timestamp = datetime.fromisoformat(timestamp)
-        elif not isinstance(timestamp, datetime):
-            timestamp = datetime.utcnow()
-
-        # Handle missing feedback_id (legacy entries)
-        feedback_id = data.get("feedback_id")
-        if not feedback_id:
-            feedback_id = f"fb-{int(timestamp.timestamp()*1000)}"
-
-        return cls(
-            feedback_id=feedback_id,
-            workflow_name=data["workflow_name"],
-            stage_name=data["stage_name"],
-            tier=data["tier"],
-            quality_score=data["quality_score"],
-            timestamp=timestamp,
-            metadata=data.get("metadata", {}),
-        )
-
-
-@dataclass
-class QualityStats:
-    """Quality statistics for a workflow stage."""
-
-    workflow_name: str
-    stage_name: str
-    tier: str
-    avg_quality: float
-    min_quality: float
-    max_quality: float
-    sample_count: int
-    recent_trend: float  # -1.0 (declining) to 1.0 (improving)
-
-
-@dataclass
-class TierRecommendation:
-    """Tier recommendation based on quality feedback."""
-
-    current_tier: str
-    recommended_tier: str
-    confidence: float  # 0.0 (low) to 1.0 (high)
-    reason: str
-    stats: dict[str, QualityStats]  # Stats by tier
-
-
-class FeedbackLoop:
-    """Agent-to-LLM feedback loop for quality-based learning.
-
-    Collects quality ratings on LLM responses and uses feedback to:
-    - Recommend tier upgrades/downgrades
-    - Track quality trends over time
-    - Identify underperforming stages
-    - Optimize routing based on historical performance
-
-    Attributes:
-        FEEDBACK_TTL: Feedback entry TTL (7 days)
-        MIN_SAMPLES: Minimum samples for recommendation (10)
-        QUALITY_THRESHOLD: Quality threshold for tier upgrade (0.7)
-    """
-
-    FEEDBACK_TTL = 604800  # 7 days (60*60*24*7)
-    MIN_SAMPLES = 10  # Minimum samples for recommendation
-    QUALITY_THRESHOLD = 0.7  # Quality below this triggers upgrade recommendation
-
-    def __init__(self, memory=None):
-        """Initialize feedback loop.
-
-        Args:
-            memory: Memory instance for storing feedback
-        """
-        self.memory = memory
-
-        if self.memory is None:
-            try:
-                from empathy_os.telemetry import UsageTracker
-
-                tracker = UsageTracker.get_instance()
-                if hasattr(tracker, "_memory"):
-                    self.memory = tracker._memory
-            except (ImportError, AttributeError):
-                pass
-
-        if self.memory is None:
-            logger.warning("No memory backend available for feedback loop")
-
-    def record_feedback(
-        self,
-        workflow_name: str,
-        stage_name: str,
-        tier: str | ModelTier,
-        quality_score: float,
-        metadata: dict[str, Any] | None = None,
-    ) -> str:
-        """Record quality feedback for a workflow stage execution.
-
-        Args:
-            workflow_name: Name of workflow
-            stage_name: Name of stage within workflow
-            tier: Model tier used (CHEAP, CAPABLE, PREMIUM)
-            quality_score: Quality rating 0.0-1.0 (0=bad, 1=excellent)
-            metadata: Optional metadata (tokens, latency, etc.)
-
-        Returns:
-            Feedback ID if stored, empty string otherwise
-
-        Example:
-            >>> feedback = FeedbackLoop()
-            >>> feedback.record_feedback(
-            ...     workflow_name="code-review",
-            ...     stage_name="analysis",
-            ...     tier=ModelTier.CHEAP,
-            ...     quality_score=0.85,
-            ...     metadata={"tokens": 150, "latency_ms": 1200}
-            ... )
-        """
-        if not self.memory:
-            logger.debug("Cannot record feedback: no memory backend")
-            return ""
-
-        # Validate quality score
-        if not 0.0 <= quality_score <= 1.0:
-            logger.warning(f"Invalid quality score: {quality_score} (must be 0.0-1.0)")
-            return ""
-
-        # Convert tier to string if ModelTier enum
-        if isinstance(tier, ModelTier):
-            tier = tier.value
-
-        feedback_id = f"feedback_{uuid4().hex[:8]}"
-
-        entry = FeedbackEntry(
-            feedback_id=feedback_id,
-            workflow_name=workflow_name,
-            stage_name=stage_name,
-            tier=tier,
-            quality_score=quality_score,
-            timestamp=datetime.utcnow(),
-            metadata=metadata or {},
-        )
-
-        # Store feedback
-        # Key format: feedback:{workflow}:{stage}:{tier}:{id}
-        key = f"feedback:{workflow_name}:{stage_name}:{tier}:{feedback_id}"
-
-        try:
-            # Use direct Redis access for custom TTL
-            if hasattr(self.memory, "_client") and self.memory._client:
-                import json
-
-                self.memory._client.setex(key, self.FEEDBACK_TTL, json.dumps(entry.to_dict()))
-            else:
-                logger.warning("Cannot store feedback: no Redis backend available")
-                return ""
-        except Exception as e:
-            logger.error(f"Failed to store feedback: {e}")
-            return ""
-
-        logger.debug(
-            f"Recorded feedback: {workflow_name}/{stage_name} tier={tier} quality={quality_score:.2f}"
-        )
-        return feedback_id
-
-    def get_feedback_history(
-        self, workflow_name: str, stage_name: str, tier: str | ModelTier | None = None, limit: int = 100
-    ) -> list[FeedbackEntry]:
-        """Get feedback history for a workflow stage.
-
-        Args:
-            workflow_name: Name of workflow
-            stage_name: Name of stage
-            tier: Optional filter by tier
-            limit: Maximum number of entries to return
-
-        Returns:
-            List of feedback entries (newest first)
-        """
-        if not self.memory or not hasattr(self.memory, "_client"):
-            return []
-
-        # Convert tier to string if ModelTier enum
-        if isinstance(tier, ModelTier):
-            tier = tier.value
-
-        try:
-            # Build search pattern
-            if tier:
-                pattern = f"feedback:{workflow_name}:{stage_name}:{tier}:*"
-            else:
-                pattern = f"feedback:{workflow_name}:{stage_name}:*"
-
-            keys = self.memory._client.keys(pattern)
-
-            entries = []
-            for key in keys:
-                if isinstance(key, bytes):
-                    key = key.decode("utf-8")
-
-                # Retrieve entry
-                data = self._retrieve_feedback(key)
-                if data:
-                    try:
-                        entries.append(FeedbackEntry.from_dict(data))
-                    except Exception as e:
-                        logger.error(f"Failed to parse feedback entry {key}: {e}, data={data}")
-                        continue
-
-                if len(entries) >= limit:
-                    break
-
-            # Sort by timestamp (newest first)
-            entries.sort(key=lambda e: e.timestamp, reverse=True)
-
-            return entries[:limit]
-        except Exception as e:
-            logger.error(f"Failed to get feedback history: {e}")
-            return []
-
-    def _retrieve_feedback(self, key: str) -> dict[str, Any] | None:
-        """Retrieve feedback entry from memory."""
-        if not self.memory:
-            return None
-
-        try:
-            # Use direct Redis access (feedback keys are stored without prefix)
-            if hasattr(self.memory, "_client"):
-                import json
-
-                data = self.memory._client.get(key)
-                if data:
-                    if isinstance(data, bytes):
-                        data = data.decode("utf-8")
-                    return json.loads(data)
-            return None
-        except Exception as e:
-            logger.debug(f"Failed to retrieve feedback: {e}")
-            return None
-
-    def get_quality_stats(
-        self, workflow_name: str, stage_name: str, tier: str | ModelTier | None = None
-    ) -> QualityStats | None:
-        """Get quality statistics for a workflow stage.
-
-        Args:
-            workflow_name: Name of workflow
-            stage_name: Name of stage
-            tier: Optional filter by tier
-
-        Returns:
-            Quality statistics or None if insufficient data
-        """
-        history = self.get_feedback_history(workflow_name, stage_name, tier=tier)
-
-        if not history:
-            return None
-
-        # Calculate statistics
-        quality_scores = [entry.quality_score for entry in history]
-
-        avg_quality = sum(quality_scores) / len(quality_scores)
-        min_quality = min(quality_scores)
-        max_quality = max(quality_scores)
-
-        # Calculate trend (recent vs older feedback)
-        if len(history) >= 4:
-            recent = quality_scores[: len(quality_scores) // 2]
-            older = quality_scores[len(quality_scores) // 2 :]
-            recent_avg = sum(recent) / len(recent)
-            older_avg = sum(older) / len(older)
-            recent_trend = (recent_avg - older_avg) / max(older_avg, 0.1)  # Normalized difference
-        else:
-            recent_trend = 0.0
-
-        tier_str = tier.value if isinstance(tier, ModelTier) else (tier or "all")
-
-        return QualityStats(
-            workflow_name=workflow_name,
-            stage_name=stage_name,
-            tier=tier_str,
-            avg_quality=avg_quality,
-            min_quality=min_quality,
-            max_quality=max_quality,
-            sample_count=len(history),
-            recent_trend=recent_trend,
-        )
-
-    def recommend_tier(
-        self, workflow_name: str, stage_name: str, current_tier: str | ModelTier | None = None
-    ) -> TierRecommendation:
-        """Recommend optimal tier based on quality feedback.
-
-        Analyzes historical quality data and recommends:
-        - Downgrade if current tier consistently delivers high quality (cost optimization)
-        - Upgrade if current tier delivers poor quality (quality optimization)
-        - Keep current if quality is acceptable
-
-        Args:
-            workflow_name: Name of workflow
-            stage_name: Name of stage
-            current_tier: Current tier in use (if known)
-
-        Returns:
-            Tier recommendation with confidence and reasoning
-        """
-        # Convert tier to string if ModelTier enum
-        if isinstance(current_tier, ModelTier):
-            current_tier = current_tier.value
-
-        # Get stats for all tiers
-        stats_by_tier = {}
-        for tier in ["cheap", "capable", "premium"]:
-            stats = self.get_quality_stats(workflow_name, stage_name, tier=tier)
-            if stats:
-                stats_by_tier[tier] = stats
-
-        # No data - default recommendation
-        if not stats_by_tier:
-            return TierRecommendation(
-                current_tier=current_tier or "unknown",
-                recommended_tier=current_tier or "cheap",
-                confidence=0.0,
-                reason="No feedback data available",
-                stats={},
-            )
-
-        # Determine current tier if not provided
-        if not current_tier:
-            # Use tier with most recent feedback
-            all_history = self.get_feedback_history(workflow_name, stage_name, tier=None, limit=1)
-            if all_history:
-                current_tier = all_history[0].tier
-            else:
-                current_tier = "cheap"
-
-        current_stats = stats_by_tier.get(current_tier)
-
-        # Insufficient data for current tier
-        if not current_stats or current_stats.sample_count < self.MIN_SAMPLES:
-            return TierRecommendation(
-                current_tier=current_tier,
-                recommended_tier=current_tier,
-                confidence=0.0,
-                reason=f"Insufficient data (need {self.MIN_SAMPLES} samples, have {current_stats.sample_count if current_stats else 0})",
-                stats=stats_by_tier,
-            )
-
-        # Analyze quality
-        avg_quality = current_stats.avg_quality
-        confidence = min(current_stats.sample_count / (self.MIN_SAMPLES * 2), 1.0)
-
-        # Decision logic
-        if avg_quality < self.QUALITY_THRESHOLD:
-            # Poor quality - recommend upgrade
-            if current_tier == "cheap":
-                recommended = "capable"
-                reason = f"Low quality ({avg_quality:.2f}) - upgrade for better results"
-            elif current_tier == "capable":
-                recommended = "premium"
-                reason = f"Low quality ({avg_quality:.2f}) - upgrade to premium tier"
-            else:  # premium
-                recommended = "premium"
-                reason = f"Already using premium tier (quality: {avg_quality:.2f})"
-                confidence = 1.0
-        elif avg_quality > 0.9 and current_tier != "cheap":
-            # Excellent quality - consider downgrade for cost optimization
-            if current_tier == "premium":
-                # Check if capable tier also has good quality
-                capable_stats = stats_by_tier.get("capable")
-                if capable_stats and capable_stats.avg_quality > 0.85:
-                    recommended = "capable"
-                    reason = f"Excellent quality ({avg_quality:.2f}) - downgrade to save cost"
-                else:
-                    recommended = "premium"
-                    reason = f"Excellent quality ({avg_quality:.2f}) - keep premium for consistency"
-            elif current_tier == "capable":
-                # Check if cheap tier also has good quality
-                cheap_stats = stats_by_tier.get("cheap")
-                if cheap_stats and cheap_stats.avg_quality > 0.85:
-                    recommended = "cheap"
-                    reason = f"Excellent quality ({avg_quality:.2f}) - downgrade to save cost"
-                else:
-                    recommended = "capable"
-                    reason = f"Excellent quality ({avg_quality:.2f}) - keep capable tier"
-            else:
-                recommended = current_tier
-                reason = f"Excellent quality ({avg_quality:.2f}) - maintain current tier"
-        else:
-            # Acceptable quality - keep current tier
-            recommended = current_tier
-            reason = f"Acceptable quality ({avg_quality:.2f}) - maintain current tier"
-
-        return TierRecommendation(
-            current_tier=current_tier,
-            recommended_tier=recommended,
-            confidence=confidence,
-            reason=reason,
-            stats=stats_by_tier,
-        )
-
-    def get_underperforming_stages(
-        self, workflow_name: str, quality_threshold: float = 0.7
-    ) -> list[tuple[str, QualityStats]]:
-        """Get workflow stages/tiers with poor quality scores.
-
-        Args:
-            workflow_name: Name of workflow
-            quality_threshold: Threshold below which stage/tier is considered underperforming
-
-        Returns:
-            List of (stage_name, stats) tuples for underperforming stage/tier combinations
-            The stage_name includes the tier for clarity (e.g., "analysis/cheap")
-        """
-        if not self.memory or not hasattr(self.memory, "_client"):
-            return []
-
-        try:
-            # Find all feedback keys for this workflow
-            pattern = f"feedback:{workflow_name}:*"
-            keys = self.memory._client.keys(pattern)
-
-            # Extract unique stage/tier combinations
-            stage_tier_combos = set()
-            for key in keys:
-                if isinstance(key, bytes):
-                    key = key.decode("utf-8")
-                # Parse key: feedback:{workflow}:{stage}:{tier}:{id}
-                parts = key.split(":")
-                if len(parts) >= 4:
-                    stage_name = parts[2]
-                    tier = parts[3]
-                    stage_tier_combos.add((stage_name, tier))
-
-            # Get stats for each stage/tier combination
-            underperforming = []
-            for stage_name, tier in stage_tier_combos:
-                stats = self.get_quality_stats(workflow_name, stage_name, tier=tier)
-                if stats and stats.avg_quality < quality_threshold:
-                    # Include tier in the stage name for clarity
-                    stage_label = f"{stage_name}/{tier}"
-                    underperforming.append((stage_label, stats))
-
-            # Sort by quality (worst first)
-            underperforming.sort(key=lambda x: x[1].avg_quality)
-
-            return underperforming
-        except Exception as e:
-            logger.error(f"Failed to get underperforming stages: {e}")
-            return []
-
-    def clear_feedback(self, workflow_name: str, stage_name: str | None = None) -> int:
-        """Clear feedback history for a workflow or stage.
-
-        Args:
-            workflow_name: Name of workflow
-            stage_name: Optional stage name (clears all stages if None)
-
-        Returns:
-            Number of feedback entries cleared
-        """
-        if not self.memory or not hasattr(self.memory, "_client"):
-            return 0
-
-        try:
-            if stage_name:
-                pattern = f"feedback:{workflow_name}:{stage_name}:*"
-            else:
-                pattern = f"feedback:{workflow_name}:*"
-
-            keys = self.memory._client.keys(pattern)
-            if not keys:
-                return 0
-
-            deleted = self.memory._client.delete(*keys)
-            return deleted
-        except Exception as e:
-            logger.error(f"Failed to clear feedback: {e}")
-            return 0