empathy-framework 4.9.1__py3-none-any.whl → 5.0.1__py3-none-any.whl

empathy_os/memory/types.py

@@ -42,13 +42,13 @@ class TTLStrategy(Enum):
     Per EMPATHY_PHILOSOPHY.md Section 9.3:
     - Working results: 1 hour
     - Staged patterns: 24 hours
-    - Coordination signals: 5 minutes
+    - Coordination signals: 5 minutes (REMOVED in v5.0 - see CoordinationSignals)
     - Conflict context: Until resolution
     """
 
     WORKING_RESULTS = 3600  # 1 hour
     STAGED_PATTERNS = 86400  # 24 hours
-    COORDINATION = 300  # 5 minutes
+    # COORDINATION removed in v5.0 - use CoordinationSignals with custom TTLs
     CONFLICT_CONTEXT = 604800  # 7 days (fallback for unresolved)
     SESSION = 1800  # 30 minutes
     STREAM_ENTRY = 86400 * 7  # 7 days for audit stream entries

empathy_os/models/__init__.py

@@ -9,6 +9,7 @@ Copyright 2025 Smart-AI-Memory
 Licensed under Fair Source License 0.9
 """
 
+from .adaptive_routing import AdaptiveModelRouter, ModelPerformance
 from .empathy_executor import EmpathyLLMExecutor
 from .executor import ExecutionContext, LLMExecutor, LLMResponse, MockLLMExecutor
 from .fallback import (
@@ -87,6 +88,7 @@ __all__ = [
     "MODEL_REGISTRY",
     "PREMIUM_TASKS",
     "TASK_TIER_MAP",
+    "AdaptiveModelRouter",
     "AgentAssignmentRecord",
     "CircuitBreaker",
     "CircuitBreakerState",
@@ -106,6 +108,7 @@ __all__ = [
     "LLMResponse",
     "MockLLMExecutor",
     "ModelInfo",
+    "ModelPerformance",
     "ModelProvider",
     # Registry exports
     "ModelRegistry",

empathy_os/models/adaptive_routing.py

@@ -0,0 +1,437 @@
+"""Adaptive Model Routing based on historical telemetry.
+
+This module implements Pattern 3 from AGENT_COORDINATION_ARCHITECTURE.md:
+Using telemetry history to learn which models work best for each workflow/stage.
+
+Key Features:
+- Analyzes historical performance per model/workflow/stage
+- Recommends best model based on success rate and cost
+- Auto-detects when tier upgrades are needed (>20% failure rate)
+- Respects cost and latency constraints
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Lazy import to avoid circular dependency
+_model_registry = None
+
+
+def _get_registry():
+    """Get ModelRegistry instance (lazy load to avoid circular import)."""
+    global _model_registry
+    if _model_registry is None:
+        from .registry import MODEL_REGISTRY
+        _model_registry = MODEL_REGISTRY
+    return _model_registry
+
+
+@dataclass
+class ModelPerformance:
+    """Performance metrics for a model on a specific task.
+
+    Attributes:
+        model_id: Model identifier (e.g., "claude-sonnet-4.5")
+        tier: Model tier (CHEAP, CAPABLE, PREMIUM)
+        success_rate: Percentage of successful calls (0.0 - 1.0)
+        avg_latency_ms: Average response time in milliseconds
+        avg_cost: Average cost per call in USD
+        sample_size: Number of calls analyzed
+        recent_failures: Number of failures in last 20 calls
+    """
+
+    model_id: str
+    tier: str
+    success_rate: float
+    avg_latency_ms: float
+    avg_cost: float
+    sample_size: int
+    recent_failures: int = 0
+
+    @property
+    def quality_score(self) -> float:
+        """Calculate quality score for ranking models.
+
+        Score prioritizes:
+        1. Success rate (most important)
+        2. Cost (secondary)
+
+        Returns:
+            Quality score (higher is better)
+        """
+        # Success rate contributes 100 points max
+        # Lower cost adds bonus points
+        return (self.success_rate * 100) - (self.avg_cost * 10)
+
+
+class AdaptiveModelRouter:
+    """Route tasks to models based on historical telemetry performance.
+
+    Uses telemetry data to learn which models work best for each workflow/stage
+    combination. Automatically recommends tier upgrades when failure rates are high.
+
+    Example:
+        >>> from empathy_os.telemetry import UsageTracker
+        >>> router = AdaptiveModelRouter(UsageTracker.get_instance())
+        >>>
+        >>> # Get best model for this workflow stage
+        >>> model = router.get_best_model(
+        ...     workflow="code-review",
+        ...     stage="analysis",
+        ...     max_cost=0.01
+        ... )
+        >>> print(f"Using {model}")
+        Using claude-3-5-haiku-20241022
+        >>>
+        >>> # Check if we should upgrade tier
+        >>> should_upgrade, reason = router.recommend_tier_upgrade(
+        ...     workflow="code-review",
+        ...     stage="analysis"
+        ... )
+        >>> if should_upgrade:
+        ...     print(f"⚠️ {reason}")
+        ⚠️ High failure rate: 25.0% in last 20 calls
+    """
+
+    # Minimum sample size for making routing decisions
+    MIN_SAMPLE_SIZE = 10
+
+    # Failure rate threshold for tier upgrade recommendation
+    FAILURE_RATE_THRESHOLD = 0.2  # 20%
+
+    # Recent window size for failure detection
+    RECENT_WINDOW_SIZE = 20
+
+    def __init__(self, telemetry: Any):
+        """Initialize adaptive router.
+
+        Args:
+            telemetry: UsageTracker instance for telemetry data access
+        """
+        self.telemetry = telemetry
+
+    def _get_default_model(self, tier: str = "CHEAP") -> str:
+        """Get default Anthropic model for a tier from registry.
+
+        This dynamically fetches the current Anthropic model for each tier,
+        so when new models are released (e.g., Claude 5), they're automatically used.
+
+        Args:
+            tier: Tier name (CHEAP, CAPABLE, or PREMIUM)
+
+        Returns:
+            Model ID from registry (e.g., "claude-3-5-haiku-20241022")
+        """
+        registry = _get_registry()
+
+        # Get Anthropic model for this tier
+        tier_lower = tier.lower()
+        if tier_lower in registry.get("anthropic", {}):
+            return registry["anthropic"][tier_lower].id
+
+        # Fallback to known models if registry lookup fails
+        fallbacks = {
+            "cheap": "claude-3-5-haiku-20241022",
+            "capable": "claude-sonnet-4-5",
+            "premium": "claude-opus-4-5-20251101",
+        }
+        return fallbacks.get(tier_lower, "claude-3-5-haiku-20241022")
+
+    def get_best_model(
+        self,
+        workflow: str,
+        stage: str,
+        max_cost: float | None = None,
+        max_latency_ms: int | None = None,
+        min_success_rate: float = 0.8,
+    ) -> str:
+        """Get best model for workflow/stage based on historical performance.
+
+        Analyzes recent telemetry to find the model with the best quality score
+        (success rate + cost efficiency) that meets the specified constraints.
+
+        Args:
+            workflow: Workflow name (e.g., "code-review", "bug-predict")
+            stage: Stage name (e.g., "analysis", "synthesis")
+            max_cost: Maximum acceptable cost per call (USD)
+            max_latency_ms: Maximum acceptable latency (milliseconds)
+            min_success_rate: Minimum acceptable success rate (0.0 - 1.0)
+
+        Returns:
+            Model ID to use (e.g., "claude-3-5-haiku-20241022")
+
+        Example:
+            >>> model = router.get_best_model(
+            ...     workflow="code-review",
+            ...     stage="analysis",
+            ...     max_cost=0.01,
+            ...     min_success_rate=0.9
+            ... )
+            >>> print(model)
+            claude-3-5-haiku-20241022
+        """
+        # Get performance data for all models on this workflow/stage
+        performances = self._analyze_model_performance(workflow, stage)
+
+        if not performances:
+            # No historical data, use default Anthropic cheap model from registry
+            default_model = self._get_default_model("CHEAP")
+            logger.info(
+                "adaptive_routing_no_history",
+                workflow=workflow,
+                stage=stage,
+                fallback=default_model,
+            )
+            return default_model
+
+        # Filter by constraints
+        candidates = []
+        for perf in performances:
+            # Skip if insufficient data
+            if perf.sample_size < self.MIN_SAMPLE_SIZE:
+                continue
+
+            # Skip if doesn't meet minimum success rate
+            if perf.success_rate < min_success_rate:
+                continue
+
+            # Skip if exceeds cost constraint
+            if max_cost is not None and perf.avg_cost > max_cost:
+                continue
+
+            # Skip if exceeds latency constraint
+            if max_latency_ms is not None and perf.avg_latency_ms > max_latency_ms:
+                continue
+
+            candidates.append(perf)
+
+        if not candidates:
+            # All models filtered out, fall back to default Anthropic model
+            default_model = self._get_default_model("CHEAP")
+            logger.warning(
+                "adaptive_routing_no_candidates",
+                workflow=workflow,
+                stage=stage,
+                constraints={"max_cost": max_cost, "max_latency_ms": max_latency_ms},
+                fallback=default_model,
+            )
+            return default_model
+
+        # Sort by quality score (success rate + cost efficiency)
+        candidates.sort(key=lambda p: p.quality_score, reverse=True)
+        best = candidates[0]
+
+        logger.info(
+            "adaptive_routing_selected",
+            workflow=workflow,
+            stage=stage,
+            model=best.model_id,
+            success_rate=f"{best.success_rate:.1%}",
+            avg_cost=f"${best.avg_cost:.4f}",
+            sample_size=best.sample_size,
+        )
+
+        return best.model_id
+
+    def recommend_tier_upgrade(
+        self, workflow: str, stage: str
+    ) -> tuple[bool, str]:
+        """Check if tier should be upgraded based on failure rate.
+
+        Analyzes recent telemetry (last 20 calls) for this workflow/stage.
+        If failure rate exceeds threshold (20%), recommends tier upgrade.
+
+        Args:
+            workflow: Workflow name
+            stage: Stage name
+
+        Returns:
+            Tuple of (should_upgrade: bool, reason: str)
+
+        Example:
+            >>> should_upgrade, reason = router.recommend_tier_upgrade(
+            ...     workflow="code-review",
+            ...     stage="analysis"
+            ... )
+            >>> if should_upgrade:
+            ...     print(f"⚠️ Upgrading tier: {reason}")
+            ⚠️ Upgrading tier: High failure rate: 25.0% in last 20 calls
+        """
+        # Get recent entries for this workflow/stage
+        entries = self._get_workflow_stage_entries(workflow, stage, days=7)
+
+        if len(entries) < self.MIN_SAMPLE_SIZE:
+            return False, f"Insufficient data ({len(entries)} calls, need {self.MIN_SAMPLE_SIZE})"
+
+        # Analyze recent window (last 20 calls)
+        recent = entries[-self.RECENT_WINDOW_SIZE :]
+        failures = sum(1 for e in recent if not e.get("success", True))
+        failure_rate = failures / len(recent)
+
+        if failure_rate > self.FAILURE_RATE_THRESHOLD:
+            return (
+                True,
+                f"High failure rate: {failure_rate:.1%} ({failures}/{len(recent)} failed in recent calls)",
+            )
+
+        return False, f"Performance acceptable: {failure_rate:.1%} failure rate"
+
+    def get_routing_stats(
+        self, workflow: str, stage: str | None = None, days: int = 7
+    ) -> dict[str, Any]:
+        """Get routing statistics for a workflow (or specific stage).
+
+        Args:
+            workflow: Workflow name
+            stage: Optional stage name (None for all stages)
+            days: Number of days to analyze
+
+        Returns:
+            Dictionary with routing statistics:
+            - models_used: List of models used
+            - performance_by_model: Performance metrics per model
+            - total_calls: Total number of calls
+            - avg_cost: Average cost per call
+            - avg_success_rate: Average success rate
+
+        Example:
+            >>> stats = router.get_routing_stats("code-review", days=7)
+            >>> print(f"Models used: {stats['models_used']}")
+            Models used: ['claude-haiku-3.5', 'claude-sonnet-4.5']
+            >>> print(f"Average cost: ${stats['avg_cost']:.4f}")
+            Average cost: $0.0023
+        """
+        entries = self._get_workflow_stage_entries(workflow, stage, days=days)
+
+        if not entries:
+            return {
+                "models_used": [],
+                "performance_by_model": {},
+                "total_calls": 0,
+                "avg_cost": 0.0,
+                "avg_success_rate": 0.0,
+            }
+
+        # Calculate stats
+        models_used = list({e["model"] for e in entries})
+        total_calls = len(entries)
+        total_cost = sum(e.get("cost", 0.0) for e in entries)
+        successes = sum(1 for e in entries if e.get("success", True))
+
+        # Per-model performance
+        performance_by_model = {}
+        for model in models_used:
+            model_entries = [e for e in entries if e["model"] == model]
+            model_successes = sum(1 for e in model_entries if e.get("success", True))
+
+            performance_by_model[model] = {
+                "calls": len(model_entries),
+                "success_rate": model_successes / len(model_entries),
+                "avg_cost": sum(e.get("cost", 0.0) for e in model_entries)
+                / len(model_entries),
+                "avg_latency_ms": sum(e.get("duration_ms", 0) for e in model_entries)
+                / len(model_entries),
+            }
+
+        return {
+            "workflow": workflow,
+            "stage": stage or "all",
+            "days_analyzed": days,
+            "models_used": models_used,
+            "performance_by_model": performance_by_model,
+            "total_calls": total_calls,
+            "avg_cost": total_cost / total_calls if total_calls > 0 else 0.0,
+            "avg_success_rate": successes / total_calls if total_calls > 0 else 0.0,
+        }
+
+    def _analyze_model_performance(
+        self, workflow: str, stage: str, days: int = 7
+    ) -> list[ModelPerformance]:
+        """Analyze performance of all models for this workflow/stage.
+
+        Args:
+            workflow: Workflow name
+            stage: Stage name
+            days: Number of days to analyze
+
+        Returns:
+            List of ModelPerformance objects, one per model
+        """
+        entries = self._get_workflow_stage_entries(workflow, stage, days=days)
+
+        if not entries:
+            return []
+
+        # Group by model
+        by_model: dict[str, list[dict]] = {}
+        for entry in entries:
+            model = entry["model"]
+            if model not in by_model:
+                by_model[model] = []
+            by_model[model].append(entry)
+
+        # Calculate performance metrics per model
+        performances = []
+        for model, model_entries in by_model.items():
+            total = len(model_entries)
+            successes = sum(1 for e in model_entries if e.get("success", True))
+            success_rate = successes / total
+
+            avg_latency = (
+                sum(e.get("duration_ms", 0) for e in model_entries) / total
+            )
+            avg_cost = sum(e.get("cost", 0.0) for e in model_entries) / total
+
+            # Analyze recent failures (last 20 calls)
+            recent = model_entries[-self.RECENT_WINDOW_SIZE :]
+            recent_failures = sum(1 for e in recent if not e.get("success", True))
+
+            performances.append(
+                ModelPerformance(
+                    model_id=model,
+                    tier=model_entries[0].get("tier", "unknown"),
+                    success_rate=success_rate,
+                    avg_latency_ms=avg_latency,
+                    avg_cost=avg_cost,
+                    sample_size=total,
+                    recent_failures=recent_failures,
+                )
+            )
+
+        return performances
+
+    def _get_workflow_stage_entries(
+        self, workflow: str, stage: str | None, days: int
+    ) -> list[dict[str, Any]]:
+        """Get telemetry entries for a workflow/stage.
+
+        Args:
+            workflow: Workflow name
+            stage: Stage name (None for all stages)
+            days: Number of days to retrieve
+
+        Returns:
+            List of telemetry entries
+        """
+        # Get recent entries from telemetry tracker
+        all_entries = self.telemetry.get_recent_entries(limit=10000, days=days)
+
+        # Filter to this workflow
+        workflow_entries = [e for e in all_entries if e.get("workflow") == workflow]
+
+        # Filter to this stage if specified
+        if stage is not None:
+            workflow_entries = [e for e in workflow_entries if e.get("stage") == stage]
+
+        return workflow_entries
+
+
+__all__ = ["AdaptiveModelRouter", "ModelPerformance"]

empathy_os/models/registry.py

@@ -209,11 +209,11 @@ class ModelRegistry:
         """Build tier and model ID caches for O(1) lookups."""
         # Cache for get_models_by_tier (tier -> list[ModelInfo])
         self._tier_cache: dict[str, list[ModelInfo]] = {}
-        for tier_value in [t.value for t in ModelTier]:
-            self._tier_cache[tier_value] = [
-                provider_models[tier_value]
+        for tier in ModelTier:
+            self._tier_cache[tier.value] = [
+                provider_models[tier.value]
                 for provider_models in self._registry.values()
-                if tier_value in provider_models
+                if tier.value in provider_models
             ]
 
         # Cache for get_model_by_id (model_id -> ModelInfo)

empathy_os/socratic/ab_testing.py

@@ -20,7 +20,7 @@ import hashlib
 import json
 import logging
 import math
-import random
+import random  # Security Note: For A/B test simulation data, not cryptographic use
 import time
 from dataclasses import dataclass, field
 from datetime import datetime

empathy_os/telemetry/__init__.py

@@ -2,10 +2,38 @@
 
 Privacy-first, local-only usage tracking to measure actual cost savings.
 
+Includes:
+- UsageTracker: Track LLM usage and costs
+- HeartbeatCoordinator: Monitor agent liveness via TTL heartbeats
+- CoordinationSignals: Inter-agent communication via TTL signals
+- EventStreamer: Real-time event streaming via Redis Streams
+- ApprovalGate: Human approval gates for workflow control
+- FeedbackLoop: Agent-to-LLM quality feedback for adaptive routing
+
 Copyright 2025 Smart-AI-Memory
 Licensed under Fair Source License 0.9
 """
 
+from .agent_coordination import CoordinationSignal, CoordinationSignals
+from .agent_tracking import AgentHeartbeat, HeartbeatCoordinator
+from .approval_gates import ApprovalGate, ApprovalRequest, ApprovalResponse
+from .event_streaming import EventStreamer, StreamEvent
+from .feedback_loop import FeedbackEntry, FeedbackLoop, QualityStats, TierRecommendation
 from .usage_tracker import UsageTracker
 
-__all__ = ["UsageTracker"]
+__all__ = [
+    "UsageTracker",
+    "HeartbeatCoordinator",
+    "AgentHeartbeat",
+    "CoordinationSignals",
+    "CoordinationSignal",
+    "EventStreamer",
+    "StreamEvent",
+    "ApprovalGate",
+    "ApprovalRequest",
+    "ApprovalResponse",
+    "FeedbackLoop",
+    "FeedbackEntry",
+    "QualityStats",
+    "TierRecommendation",
+]