attune-ai 2.0.0__py3-none-any.whl

attune/telemetry/feedback_loop.py

@@ -0,0 +1,567 @@
+"""Agent-to-LLM Feedback Loop for Quality-Based Learning.
+
+Pattern 6 from Agent Coordination Architecture - Collect quality ratings
+on LLM responses and use feedback to inform routing decisions.
+
+Usage:
+    # Record feedback after LLM response
+    feedback = FeedbackLoop()
+    feedback.record_feedback(
+        workflow_name="code-review",
+        stage_name="analysis",
+        tier=ModelTier.CHEAP,
+        quality_score=0.8,
+        metadata={
+            "response_length": 500,
+            "tokens": 150,
+            "latency_ms": 1200
+        }
+    )
+
+    # Get tier recommendation based on historical performance
+    recommendation = feedback.recommend_tier(
+        workflow_name="code-review",
+        stage_name="analysis"
+    )
+    if recommendation.recommended_tier == ModelTier.CAPABLE:
+        print(f"Upgrade to CAPABLE tier (confidence: {recommendation.confidence})")
+
+    # Get quality stats for analysis
+    stats = feedback.get_quality_stats(
+        workflow_name="code-review",
+        stage_name="analysis"
+    )
+    print(f"Average quality: {stats.avg_quality}")
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any
+from uuid import uuid4
+
+logger = logging.getLogger(__name__)
+
+
+class ModelTier(str, Enum):
+    """Model tier enum matching workflows.base.ModelTier."""
+
+    CHEAP = "cheap"
+    CAPABLE = "capable"
+    PREMIUM = "premium"
+
+
+@dataclass
+class FeedbackEntry:
+    """Quality feedback for an LLM response.
+
+    Represents a single quality rating for a workflow stage execution.
+    """
+
+    feedback_id: str
+    workflow_name: str
+    stage_name: str
+    tier: str  # ModelTier value
+    quality_score: float  # 0.0 (bad) to 1.0 (excellent)
+    timestamp: datetime
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "feedback_id": self.feedback_id,
+            "workflow_name": self.workflow_name,
+            "stage_name": self.stage_name,
+            "tier": self.tier,
+            "quality_score": self.quality_score,
+            "timestamp": self.timestamp.isoformat() if isinstance(self.timestamp, datetime) else self.timestamp,
+            "metadata": self.metadata,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> FeedbackEntry:
+        """Create from dictionary."""
+        timestamp = data.get("timestamp")
+        if isinstance(timestamp, str):
+            timestamp = datetime.fromisoformat(timestamp)
+        elif not isinstance(timestamp, datetime):
+            timestamp = datetime.utcnow()
+
+        # Handle missing feedback_id (legacy entries)
+        feedback_id = data.get("feedback_id")
+        if not feedback_id:
+            feedback_id = f"fb-{int(timestamp.timestamp()*1000)}"
+
+        return cls(
+            feedback_id=feedback_id,
+            workflow_name=data["workflow_name"],
+            stage_name=data["stage_name"],
+            tier=data["tier"],
+            quality_score=data["quality_score"],
+            timestamp=timestamp,
+            metadata=data.get("metadata", {}),
+        )
+
+
+@dataclass
+class QualityStats:
+    """Quality statistics for a workflow stage."""
+
+    workflow_name: str
+    stage_name: str
+    tier: str
+    avg_quality: float
+    min_quality: float
+    max_quality: float
+    sample_count: int
+    recent_trend: float  # -1.0 (declining) to 1.0 (improving)
+
+
+@dataclass
+class TierRecommendation:
+    """Tier recommendation based on quality feedback."""
+
+    current_tier: str
+    recommended_tier: str
+    confidence: float  # 0.0 (low) to 1.0 (high)
+    reason: str
+    stats: dict[str, QualityStats]  # Stats by tier
+
+
+class FeedbackLoop:
+    """Agent-to-LLM feedback loop for quality-based learning.
+
+    Collects quality ratings on LLM responses and uses feedback to:
+    - Recommend tier upgrades/downgrades
+    - Track quality trends over time
+    - Identify underperforming stages
+    - Optimize routing based on historical performance
+
+    Attributes:
+        FEEDBACK_TTL: Feedback entry TTL (7 days)
+        MIN_SAMPLES: Minimum samples for recommendation (10)
+        QUALITY_THRESHOLD: Quality threshold for tier upgrade (0.7)
+    """
+
+    FEEDBACK_TTL = 604800  # 7 days (60*60*24*7)
+    MIN_SAMPLES = 10  # Minimum samples for recommendation
+    QUALITY_THRESHOLD = 0.7  # Quality below this triggers upgrade recommendation
+
+    def __init__(self, memory=None):
+        """Initialize feedback loop.
+
+        Args:
+            memory: Memory instance for storing feedback
+        """
+        self.memory = memory
+
+        if self.memory is None:
+            try:
+                from attune.telemetry import UsageTracker
+
+                tracker = UsageTracker.get_instance()
+                if hasattr(tracker, "_memory"):
+                    self.memory = tracker._memory
+            except (ImportError, AttributeError):
+                pass
+
+        if self.memory is None:
+            logger.warning("No memory backend available for feedback loop")
+
+    def record_feedback(
+        self,
+        workflow_name: str,
+        stage_name: str,
+        tier: str | ModelTier,
+        quality_score: float,
+        metadata: dict[str, Any] | None = None,
+    ) -> str:
+        """Record quality feedback for a workflow stage execution.
+
+        Args:
+            workflow_name: Name of workflow
+            stage_name: Name of stage within workflow
+            tier: Model tier used (CHEAP, CAPABLE, PREMIUM)
+            quality_score: Quality rating 0.0-1.0 (0=bad, 1=excellent)
+            metadata: Optional metadata (tokens, latency, etc.)
+
+        Returns:
+            Feedback ID if stored, empty string otherwise
+
+        Example:
+            >>> feedback = FeedbackLoop()
+            >>> feedback.record_feedback(
+            ...     workflow_name="code-review",
+            ...     stage_name="analysis",
+            ...     tier=ModelTier.CHEAP,
+            ...     quality_score=0.85,
+            ...     metadata={"tokens": 150, "latency_ms": 1200}
+            ... )
+        """
+        if not self.memory:
+            logger.debug("Cannot record feedback: no memory backend")
+            return ""
+
+        # Validate quality score
+        if not 0.0 <= quality_score <= 1.0:
+            logger.warning(f"Invalid quality score: {quality_score} (must be 0.0-1.0)")
+            return ""
+
+        # Convert tier to string if ModelTier enum
+        if isinstance(tier, ModelTier):
+            tier = tier.value
+
+        feedback_id = f"feedback_{uuid4().hex[:8]}"
+
+        entry = FeedbackEntry(
+            feedback_id=feedback_id,
+            workflow_name=workflow_name,
+            stage_name=stage_name,
+            tier=tier,
+            quality_score=quality_score,
+            timestamp=datetime.utcnow(),
+            metadata=metadata or {},
+        )
+
+        # Store feedback
+        # Key format: feedback:{workflow}:{stage}:{tier}:{id}
+        key = f"feedback:{workflow_name}:{stage_name}:{tier}:{feedback_id}"
+
+        try:
+            # Use direct Redis access for custom TTL
+            if hasattr(self.memory, "_client") and self.memory._client:
+                import json
+
+                self.memory._client.setex(key, self.FEEDBACK_TTL, json.dumps(entry.to_dict()))
+            else:
+                logger.warning("Cannot store feedback: no Redis backend available")
+                return ""
+        except Exception as e:
+            logger.error(f"Failed to store feedback: {e}")
+            return ""
+
+        logger.debug(
+            f"Recorded feedback: {workflow_name}/{stage_name} tier={tier} quality={quality_score:.2f}"
+        )
+        return feedback_id
+
+    def get_feedback_history(
+        self, workflow_name: str, stage_name: str, tier: str | ModelTier | None = None, limit: int = 100
+    ) -> list[FeedbackEntry]:
+        """Get feedback history for a workflow stage.
+
+        Args:
+            workflow_name: Name of workflow
+            stage_name: Name of stage
+            tier: Optional filter by tier
+            limit: Maximum number of entries to return
+
+        Returns:
+            List of feedback entries (newest first)
+        """
+        if not self.memory or not hasattr(self.memory, "_client"):
+            return []
+
+        # Convert tier to string if ModelTier enum
+        if isinstance(tier, ModelTier):
+            tier = tier.value
+
+        try:
+            # Build search pattern
+            if tier:
+                pattern = f"feedback:{workflow_name}:{stage_name}:{tier}:*"
+            else:
+                pattern = f"feedback:{workflow_name}:{stage_name}:*"
+
+            keys = self.memory._client.keys(pattern)
+
+            entries = []
+            for key in keys:
+                if isinstance(key, bytes):
+                    key = key.decode("utf-8")
+
+                # Retrieve entry
+                data = self._retrieve_feedback(key)
+                if data:
+                    try:
+                        entries.append(FeedbackEntry.from_dict(data))
+                    except Exception as e:
+                        logger.error(f"Failed to parse feedback entry {key}: {e}, data={data}")
+                        continue
+
+                if len(entries) >= limit:
+                    break
+
+            # Sort by timestamp (newest first)
+            entries.sort(key=lambda e: e.timestamp, reverse=True)
+
+            return entries[:limit]
+        except Exception as e:
+            logger.error(f"Failed to get feedback history: {e}")
+            return []
+
+    def _retrieve_feedback(self, key: str) -> dict[str, Any] | None:
+        """Retrieve feedback entry from memory."""
+        if not self.memory:
+            return None
+
+        try:
+            # Use direct Redis access (feedback keys are stored without prefix)
+            if hasattr(self.memory, "_client"):
+                import json
+
+                data = self.memory._client.get(key)
+                if data:
+                    if isinstance(data, bytes):
+                        data = data.decode("utf-8")
+                    return json.loads(data)
+            return None
+        except Exception as e:
+            logger.debug(f"Failed to retrieve feedback: {e}")
+            return None
+
+    def get_quality_stats(
+        self, workflow_name: str, stage_name: str, tier: str | ModelTier | None = None
+    ) -> QualityStats | None:
+        """Get quality statistics for a workflow stage.
+
+        Args:
+            workflow_name: Name of workflow
+            stage_name: Name of stage
+            tier: Optional filter by tier
+
+        Returns:
+            Quality statistics or None if insufficient data
+        """
+        history = self.get_feedback_history(workflow_name, stage_name, tier=tier)
+
+        if not history:
+            return None
+
+        # Calculate statistics
+        quality_scores = [entry.quality_score for entry in history]
+
+        avg_quality = sum(quality_scores) / len(quality_scores)
+        min_quality = min(quality_scores)
+        max_quality = max(quality_scores)
+
+        # Calculate trend (recent vs older feedback)
+        if len(history) >= 4:
+            recent = quality_scores[: len(quality_scores) // 2]
+            older = quality_scores[len(quality_scores) // 2 :]
+            recent_avg = sum(recent) / len(recent)
+            older_avg = sum(older) / len(older)
+            recent_trend = (recent_avg - older_avg) / max(older_avg, 0.1)  # Normalized difference
+        else:
+            recent_trend = 0.0
+
+        tier_str = tier.value if isinstance(tier, ModelTier) else (tier or "all")
+
+        return QualityStats(
+            workflow_name=workflow_name,
+            stage_name=stage_name,
+            tier=tier_str,
+            avg_quality=avg_quality,
+            min_quality=min_quality,
+            max_quality=max_quality,
+            sample_count=len(history),
+            recent_trend=recent_trend,
+        )
+
+    def recommend_tier(
+        self, workflow_name: str, stage_name: str, current_tier: str | ModelTier | None = None
+    ) -> TierRecommendation:
+        """Recommend optimal tier based on quality feedback.
+
+        Analyzes historical quality data and recommends:
+        - Downgrade if current tier consistently delivers high quality (cost optimization)
+        - Upgrade if current tier delivers poor quality (quality optimization)
+        - Keep current if quality is acceptable
+
+        Args:
+            workflow_name: Name of workflow
+            stage_name: Name of stage
+            current_tier: Current tier in use (if known)
+
+        Returns:
+            Tier recommendation with confidence and reasoning
+        """
+        # Convert tier to string if ModelTier enum
+        if isinstance(current_tier, ModelTier):
+            current_tier = current_tier.value
+
+        # Get stats for all tiers
+        stats_by_tier = {}
+        for tier in ["cheap", "capable", "premium"]:
+            stats = self.get_quality_stats(workflow_name, stage_name, tier=tier)
+            if stats:
+                stats_by_tier[tier] = stats
+
+        # No data - default recommendation
+        if not stats_by_tier:
+            return TierRecommendation(
+                current_tier=current_tier or "unknown",
+                recommended_tier=current_tier or "cheap",
+                confidence=0.0,
+                reason="No feedback data available",
+                stats={},
+            )
+
+        # Determine current tier if not provided
+        if not current_tier:
+            # Use tier with most recent feedback
+            all_history = self.get_feedback_history(workflow_name, stage_name, tier=None, limit=1)
+            if all_history:
+                current_tier = all_history[0].tier
+            else:
+                current_tier = "cheap"
+
+        current_stats = stats_by_tier.get(current_tier)
+
+        # Insufficient data for current tier
+        if not current_stats or current_stats.sample_count < self.MIN_SAMPLES:
+            return TierRecommendation(
+                current_tier=current_tier,
+                recommended_tier=current_tier,
+                confidence=0.0,
+                reason=f"Insufficient data (need {self.MIN_SAMPLES} samples, have {current_stats.sample_count if current_stats else 0})",
+                stats=stats_by_tier,
+            )
+
+        # Analyze quality
+        avg_quality = current_stats.avg_quality
+        confidence = min(current_stats.sample_count / (self.MIN_SAMPLES * 2), 1.0)
+
+        # Decision logic
+        if avg_quality < self.QUALITY_THRESHOLD:
+            # Poor quality - recommend upgrade
+            if current_tier == "cheap":
+                recommended = "capable"
+                reason = f"Low quality ({avg_quality:.2f}) - upgrade for better results"
+            elif current_tier == "capable":
+                recommended = "premium"
+                reason = f"Low quality ({avg_quality:.2f}) - upgrade to premium tier"
+            else:  # premium
+                recommended = "premium"
+                reason = f"Already using premium tier (quality: {avg_quality:.2f})"
+                confidence = 1.0
+        elif avg_quality > 0.9 and current_tier != "cheap":
+            # Excellent quality - consider downgrade for cost optimization
+            if current_tier == "premium":
+                # Check if capable tier also has good quality
+                capable_stats = stats_by_tier.get("capable")
+                if capable_stats and capable_stats.avg_quality > 0.85:
+                    recommended = "capable"
+                    reason = f"Excellent quality ({avg_quality:.2f}) - downgrade to save cost"
+                else:
+                    recommended = "premium"
+                    reason = f"Excellent quality ({avg_quality:.2f}) - keep premium for consistency"
+            elif current_tier == "capable":
+                # Check if cheap tier also has good quality
+                cheap_stats = stats_by_tier.get("cheap")
+                if cheap_stats and cheap_stats.avg_quality > 0.85:
+                    recommended = "cheap"
+                    reason = f"Excellent quality ({avg_quality:.2f}) - downgrade to save cost"
+                else:
+                    recommended = "capable"
+                    reason = f"Excellent quality ({avg_quality:.2f}) - keep capable tier"
+            else:
+                recommended = current_tier
+                reason = f"Excellent quality ({avg_quality:.2f}) - maintain current tier"
+        else:
+            # Acceptable quality - keep current tier
+            recommended = current_tier
+            reason = f"Acceptable quality ({avg_quality:.2f}) - maintain current tier"
+
+        return TierRecommendation(
+            current_tier=current_tier,
+            recommended_tier=recommended,
+            confidence=confidence,
+            reason=reason,
+            stats=stats_by_tier,
+        )
+
+    def get_underperforming_stages(
+        self, workflow_name: str, quality_threshold: float = 0.7
+    ) -> list[tuple[str, QualityStats]]:
+        """Get workflow stages/tiers with poor quality scores.
+
+        Args:
+            workflow_name: Name of workflow
+            quality_threshold: Threshold below which stage/tier is considered underperforming
+
+        Returns:
+            List of (stage_name, stats) tuples for underperforming stage/tier combinations
+            The stage_name includes the tier for clarity (e.g., "analysis/cheap")
+        """
+        if not self.memory or not hasattr(self.memory, "_client"):
+            return []
+
+        try:
+            # Find all feedback keys for this workflow
+            pattern = f"feedback:{workflow_name}:*"
+            keys = self.memory._client.keys(pattern)
+
+            # Extract unique stage/tier combinations
+            stage_tier_combos = set()
+            for key in keys:
+                if isinstance(key, bytes):
+                    key = key.decode("utf-8")
+                # Parse key: feedback:{workflow}:{stage}:{tier}:{id}
+                parts = key.split(":")
+                if len(parts) >= 4:
+                    stage_name = parts[2]
+                    tier = parts[3]
+                    stage_tier_combos.add((stage_name, tier))
+
+            # Get stats for each stage/tier combination
+            underperforming = []
+            for stage_name, tier in stage_tier_combos:
+                stats = self.get_quality_stats(workflow_name, stage_name, tier=tier)
+                if stats and stats.avg_quality < quality_threshold:
+                    # Include tier in the stage name for clarity
+                    stage_label = f"{stage_name}/{tier}"
+                    underperforming.append((stage_label, stats))
+
+            # Sort by quality (worst first)
+            underperforming.sort(key=lambda x: x[1].avg_quality)
+
+            return underperforming
+        except Exception as e:
+            logger.error(f"Failed to get underperforming stages: {e}")
+            return []
+
+    def clear_feedback(self, workflow_name: str, stage_name: str | None = None) -> int:
+        """Clear feedback history for a workflow or stage.
+
+        Args:
+            workflow_name: Name of workflow
+            stage_name: Optional stage name (clears all stages if None)
+
+        Returns:
+            Number of feedback entries cleared
+        """
+        if not self.memory or not hasattr(self.memory, "_client"):
+            return 0
+
+        try:
+            if stage_name:
+                pattern = f"feedback:{workflow_name}:{stage_name}:*"
+            else:
+                pattern = f"feedback:{workflow_name}:*"
+
+            keys = self.memory._client.keys(pattern)
+            if not keys:
+                return 0
+
+            deleted = self.memory._client.delete(*keys)
+            return deleted
+        except Exception as e:
+            logger.error(f"Failed to clear feedback: {e}")
+            return 0