empathy-framework 3.8.3__py3-none-any.whl → 3.9.0__py3-none-any.whl

empathy_os/workflows/tier_tracking.py

@@ -0,0 +1,408 @@
+"""
+Automatic tier recommendation and progression tracking for workflows.
+
+Integrates TierRecommender into workflows to:
+1. Auto-suggest optimal tier at workflow start
+2. Track tier progression during execution
+3. Save tier progression data automatically
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+import json
+import uuid
+from dataclasses import asdict, dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from empathy_os.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class TierAttempt:
+    """Record of a single tier attempt."""
+
+    tier: str
+    attempt: int
+    success: bool
+    quality_gate_failed: Optional[str] = None
+    quality_gates_passed: Optional[List[str]] = None
+
+
+@dataclass
+class WorkflowTierProgression:
+    """Track tier progression for a workflow run."""
+
+    workflow_name: str
+    workflow_id: str
+    bug_description: str
+    files_affected: List[str]
+    bug_type: str
+
+    # Tier progression
+    recommended_tier: str
+    starting_tier: str
+    successful_tier: str
+    total_attempts: int
+    tier_history: List[Dict[str, Any]]
+
+    # Costs
+    total_cost: float
+    cost_if_always_premium: float
+    savings_percent: float
+
+    # Quality
+    tests_passed: bool
+    error_occurred: bool
+
+    # Metadata
+    started_at: str
+    completed_at: str
+    duration_seconds: float
+
+    # Optional fields must come last
+    error_message: Optional[str] = None
+
+
+class WorkflowTierTracker:
+    """
+    Automatically track tier progression for workflow runs.
+
+    Usage in BaseWorkflow:
+        tracker = WorkflowTierTracker(workflow_name, description)
+        tracker.show_recommendation(files_affected)
+        # ... run workflow ...
+        tracker.save_progression(result)
+    """
+
+    TIER_COSTS = {
+        "cheap": 0.030,
+        "capable": 0.090,
+        "premium": 0.450,
+    }
+
+    def __init__(
+        self,
+        workflow_name: str,
+        workflow_description: str,
+        patterns_dir: Optional[Path] = None,
+    ):
+        """
+        Initialize tier tracker for a workflow.
+
+        Args:
+            workflow_name: Name of the workflow
+            workflow_description: Description/purpose of workflow
+            patterns_dir: Directory to save tier progression patterns
+        """
+        self.workflow_name = workflow_name
+        self.workflow_description = workflow_description
+        self.workflow_id = str(uuid.uuid4())
+        self.started_at = datetime.now()
+
+        if patterns_dir is None:
+            patterns_dir = Path.cwd() / "patterns" / "debugging"
+        self.patterns_dir = Path(patterns_dir)
+        self.patterns_dir.mkdir(parents=True, exist_ok=True)
+
+        self.recommended_tier: Optional[str] = None
+        self.starting_tier: Optional[str] = None
+        self.tier_attempts: List[TierAttempt] = []
+
+    def show_recommendation(
+        self,
+        files_affected: Optional[List[str]] = None,
+        show_ui: bool = True,
+    ) -> str:
+        """
+        Show tier recommendation at workflow start.
+
+        Args:
+            files_affected: Files involved in this workflow run
+            show_ui: Whether to print recommendation to console
+
+        Returns:
+            Recommended tier (CHEAP, CAPABLE, or PREMIUM)
+        """
+        try:
+            from empathy_os.tier_recommender import TierRecommender
+
+            recommender = TierRecommender()
+            result = recommender.recommend(
+                bug_description=self.workflow_description,
+                files_affected=files_affected or [],
+            )
+
+            self.recommended_tier = result.tier
+
+            if show_ui:
+                self._print_recommendation(result)
+
+            return result.tier
+
+        except Exception as e:
+            logger.debug(f"Could not get tier recommendation: {e}")
+            # Fallback to CHEAP if recommendation fails
+            self.recommended_tier = "CHEAP"
+            return "CHEAP"
+
+    def _print_recommendation(self, result):
+        """Print tier recommendation to console."""
+        from rich.console import Console
+        from rich.panel import Panel
+
+        console = Console()
+
+        confidence_color = "green" if result.confidence > 0.7 else "yellow"
+
+        message = f"""[bold]Workflow:[/bold] {self.workflow_name}
+[bold]Description:[/bold] {self.workflow_description}
+
+[bold cyan]💡 Tier Recommendation[/bold cyan]
+📍 Recommended: [bold]{result.tier}[/bold]
+🎯 Confidence: [{confidence_color}]{result.confidence * 100:.0f}%[/{confidence_color}]
+💰 Expected Cost: ${result.expected_cost:.3f}
+🔄 Expected Attempts: {result.expected_attempts:.1f}
+
+[dim]Reasoning: {result.reasoning}[/dim]"""
+
+        if result.fallback_used:
+            message += "\n\n[yellow]⚠️  Using default - limited historical data[/yellow]"
+        else:
+            message += f"\n\n[green]✅ Based on {result.similar_patterns_count} similar patterns[/green]"
+
+        console.print(Panel(message, title="🎯 Auto Tier Recommendation", border_style="cyan"))
+
+    def record_tier_attempt(
+        self,
+        tier: str,
+        attempt: int,
+        success: bool,
+        quality_gate_failed: Optional[str] = None,
+        quality_gates_passed: Optional[List[str]] = None,
+    ):
+        """Record a tier attempt during workflow execution."""
+        self.tier_attempts.append(
+            TierAttempt(
+                tier=tier,
+                attempt=attempt,
+                success=success,
+                quality_gate_failed=quality_gate_failed,
+                quality_gates_passed=quality_gates_passed,
+            )
+        )
+
+    def save_progression(
+        self,
+        workflow_result: Any,
+        files_affected: Optional[List[str]] = None,
+        bug_type: str = "workflow_run",
+    ) -> Optional[Path]:
+        """
+        Save tier progression data after workflow completion.
+
+        Args:
+            workflow_result: WorkflowResult from workflow execution
+            files_affected: Files processed by workflow
+            bug_type: Type of issue being addressed
+
+        Returns:
+            Path to saved pattern file, or None if save failed
+        """
+        try:
+            completed_at = datetime.now()
+            duration = (completed_at - self.started_at).total_seconds()
+
+            # Determine successful tier from workflow result
+            successful_tier = self._determine_successful_tier(workflow_result)
+            self.starting_tier = self.starting_tier or successful_tier
+
+            # Build tier history from stages
+            tier_history = self._build_tier_history(workflow_result)
+
+            # Calculate costs
+            total_cost = workflow_result.cost_report.get("total", 0) if isinstance(workflow_result.cost_report, dict) else sum(stage.cost for stage in workflow_result.stages)
+            cost_if_premium = self._estimate_premium_cost(workflow_result)
+            savings_percent = ((cost_if_premium - total_cost) / cost_if_premium * 100) if cost_if_premium > 0 else 0
+
+            # Create progression record
+            progression = {
+                "pattern_id": f"workflow_{datetime.now().strftime('%Y%m%d')}_{self.workflow_id[:8]}",
+                "bug_type": bug_type,
+                "status": "resolved" if workflow_result.error is None else "failed",
+                "root_cause": f"Workflow: {self.workflow_name} - {self.workflow_description}",
+                "fix": f"Completed via {self.workflow_name} workflow",
+                "resolved_by": "@empathy_framework",
+                "resolved_at": completed_at.strftime("%Y-%m-%d"),
+                "files_affected": files_affected or [],
+                "source": "workflow_tracking",
+
+                "tier_progression": {
+                    "methodology": "AI-ADDIE",
+                    "recommended_tier": self.recommended_tier or self.starting_tier,
+                    "starting_tier": self.starting_tier,
+                    "successful_tier": successful_tier,
+                    "total_attempts": len(tier_history),
+                    "tier_history": tier_history,
+                    "cost_breakdown": {
+                        "total_cost": round(total_cost, 3),
+                        "cost_if_always_premium": round(cost_if_premium, 3),
+                        "savings_percent": round(savings_percent, 1),
+                    },
+                    "quality_metrics": {
+                        "tests_passed": workflow_result.error is None,
+                        "health_score_before": 73,  # Default
+                        "health_score_after": 73,
+                    },
+                    "xml_protocol_compliance": {
+                        "prompt_used_xml": True,
+                        "response_used_xml": True,
+                        "all_sections_present": True,
+                        "test_evidence_provided": True,
+                        "false_complete_avoided": workflow_result.error is None,
+                    },
+                },
+
+                "workflow_metadata": {
+                    "workflow_name": self.workflow_name,
+                    "workflow_id": self.workflow_id,
+                    "duration_seconds": round(duration, 2),
+                    "started_at": self.started_at.isoformat(),
+                    "completed_at": completed_at.isoformat(),
+                },
+            }
+
+            # Save to individual pattern file
+            pattern_file = self.patterns_dir / f"{progression['pattern_id']}.json"
+            with open(pattern_file, "w") as f:
+                json.dump(progression, f, indent=2)
+
+            logger.info(f"💾 Saved tier progression: {pattern_file}")
+
+            # Also update consolidated patterns file
+            self._update_consolidated_patterns(progression)
+
+            return pattern_file
+
+        except Exception as e:
+            logger.warning(f"Failed to save tier progression: {e}")
+            return None
+
+    def _determine_successful_tier(self, workflow_result: Any) -> str:
+        """Determine which tier successfully completed the workflow."""
+        if not workflow_result.stages:
+            return "CHEAP"
+
+        # Use the highest tier that was actually used
+        tiers_used = [stage.tier.value if hasattr(stage.tier, 'value') else str(stage.tier).lower()
+                      for stage in workflow_result.stages]
+
+        if "premium" in tiers_used:
+            return "PREMIUM"
+        elif "capable" in tiers_used:
+            return "CAPABLE"
+        else:
+            return "CHEAP"
+
+    def _build_tier_history(self, workflow_result: Any) -> List[Dict[str, Any]]:
+        """Build tier history from workflow stages."""
+        tier_groups: Dict[str, List[Any]] = {}
+
+        # Group stages by tier
+        for stage in workflow_result.stages:
+            tier = stage.tier.value if hasattr(stage.tier, 'value') else str(stage.tier).lower()
+            tier_upper = tier.upper()
+            if tier_upper not in tier_groups:
+                tier_groups[tier_upper] = []
+            tier_groups[tier_upper].append(stage)
+
+        # Build history entries
+        history = []
+        for tier, stages in tier_groups.items():
+            # Check if any stage failed
+            failures = []
+            success_stage = None
+
+            for i, stage in enumerate(stages, 1):
+                if hasattr(stage, 'error') and stage.error:
+                    failures.append({"attempt": i, "quality_gate_failed": "execution"})
+                else:
+                    success_stage = i
+
+            entry = {
+                "tier": tier,
+                "attempts": len(stages),
+            }
+
+            if failures:
+                entry["failures"] = failures
+
+            if success_stage:
+                entry["success"] = {
+                    "attempt": success_stage,
+                    "quality_gates_passed": ["execution", "output"],
+                }
+
+            history.append(entry)
+
+        return history
+
+    def _estimate_premium_cost(self, workflow_result: Any) -> float:
+        """Estimate what the cost would be if all stages used PREMIUM tier."""
+        _total_tokens = sum(
+            (stage.input_tokens or 0) + (stage.output_tokens or 0)
+            for stage in workflow_result.stages
+        )
+
+        # Calculate actual cost from stages
+        actual_cost = sum(stage.cost for stage in workflow_result.stages)
+
+        # Rough estimate: PREMIUM tier is ~15x more expensive than CHEAP
+        return actual_cost * 5  # Conservative multiplier
+
+    def _update_consolidated_patterns(self, progression: Dict[str, Any]):
+        """Update the consolidated patterns.json file."""
+        consolidated_file = self.patterns_dir / "all_patterns.json"
+
+        try:
+            if consolidated_file.exists():
+                with open(consolidated_file) as f:
+                    data = json.load(f)
+                if "patterns" not in data:
+                    data = {"patterns": []}
+            else:
+                data = {"patterns": []}
+
+            # Add new progression
+            data["patterns"].append(progression)
+
+            # Save updated file
+            with open(consolidated_file, "w") as f:
+                json.dump(data, f, indent=2)
+
+        except Exception as e:
+            logger.warning(f"Could not update consolidated patterns: {e}")
+
+
+def auto_recommend_tier(
+    workflow_name: str,
+    workflow_description: str,
+    files_affected: Optional[List[str]] = None,
+) -> str:
+    """
+    Quick helper to get tier recommendation without tracker.
+
+    Args:
+        workflow_name: Name of workflow
+        workflow_description: What the workflow does
+        files_affected: Files involved
+
+    Returns:
+        Recommended tier
+    """
+    tracker = WorkflowTierTracker(workflow_name, workflow_description)
+    return tracker.show_recommendation(files_affected, show_ui=False)