empathy-framework 5.0.1__py3-none-any.whl → 5.0.3__py3-none-any.whl

empathy_os/cli/commands/cache.py

@@ -0,0 +1,248 @@
+"""Cache monitoring and statistics commands for the CLI.
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+import json
+import re
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+
+def cmd_cache_stats(args):
+    """Display prompt caching statistics and savings.
+
+    Analyzes logs and telemetry to show cache performance metrics:
+    - Cache hit rate
+    - Total cost savings
+    - Cache read/write tokens
+    - Recommendations for optimization
+
+    Args:
+        args: Namespace object from argparse with attributes:
+            - days (int): Number of days to analyze (default: 7)
+            - format (str): Output format ('table' or 'json')
+            - verbose (bool): Show detailed breakdown
+
+    Returns:
+        None: Prints cache statistics report or JSON output
+    """
+    print(f"\n🔍 Analyzing cache performance (last {args.days} days)...\n")
+
+    # Collect cache metrics from logs
+    stats = _collect_cache_stats(days=args.days)
+
+    if args.format == "json":
+        print(json.dumps(stats, indent=2))
+        return
+
+    # Display formatted report
+    _display_cache_report(stats, verbose=args.verbose)
+
+
+def _collect_cache_stats(days: int = 7) -> dict[str, Any]:
+    """Collect cache statistics from logs and telemetry.
+
+    Args:
+        days: Number of days to analyze
+
+    Returns:
+        Dictionary with cache statistics
+    """
+    # Try to find log files
+    log_paths = [
+        Path.cwd() / "empathy_os.log",
+        Path.home() / ".empathy" / "logs" / "empathy_os.log",
+        Path("/tmp/empathy_os.log"),
+    ]
+
+    log_file = None
+    for path in log_paths:
+        if path.exists():
+            log_file = path
+            break
+
+    if not log_file:
+        return {
+            "error": "No log file found",
+            "message": "Enable logging to track cache performance",
+            "total_requests": 0,
+            "cache_hits": 0,
+            "cache_writes": 0,
+            "total_savings": 0.0,
+        }
+
+    # Parse log file for cache metrics
+    cutoff_date = datetime.now() - timedelta(days=days)
+
+    cache_hits = 0
+    cache_writes = 0
+    total_cache_read_tokens = 0
+    total_cache_write_tokens = 0
+    total_savings = 0.0
+    total_requests = 0
+
+    # Regex patterns for log parsing
+    cache_hit_pattern = re.compile(r"Cache HIT: ([\d,]+) tokens read.*saved \$([\d.]+)")
+    cache_write_pattern = re.compile(r"Cache WRITE: ([\d,]+) tokens written.*cost \$([\d.]+)")
+
+    try:
+        with open(log_file) as f:
+            for line in f:
+                # Try to extract timestamp
+                # Common format: 2026-01-27 21:30:45,123
+                timestamp_match = re.match(
+                    r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})", line
+                )
+                if timestamp_match:
+                    try:
+                        log_time = datetime.strptime(
+                            timestamp_match.group(1), "%Y-%m-%d %H:%M:%S"
+                        )
+                        if log_time < cutoff_date:
+                            continue  # Skip old entries
+                    except ValueError:
+                        pass  # Continue if timestamp parsing fails
+
+                # Count API requests (approximate)
+                if "anthropic.AsyncAnthropic" in line or "messages.create" in line:
+                    total_requests += 1
+
+                # Parse cache hit
+                hit_match = cache_hit_pattern.search(line)
+                if hit_match:
+                    tokens_str = hit_match.group(1).replace(",", "")
+                    tokens = int(tokens_str)
+                    savings = float(hit_match.group(2))
+
+                    cache_hits += 1
+                    total_cache_read_tokens += tokens
+                    total_savings += savings
+
+                # Parse cache write
+                write_match = cache_write_pattern.search(line)
+                if write_match:
+                    tokens_str = write_match.group(1).replace(",", "")
+                    tokens = int(tokens_str)
+
+                    cache_writes += 1
+                    total_cache_write_tokens += tokens
+
+    except Exception as e:
+        return {
+            "error": f"Failed to parse log file: {e}",
+            "total_requests": 0,
+            "cache_hits": 0,
+            "cache_writes": 0,
+            "total_savings": 0.0,
+        }
+
+    # Calculate metrics
+    cache_hit_rate = (
+        (cache_hits / total_requests * 100) if total_requests > 0 else 0.0
+    )
+
+    return {
+        "days_analyzed": days,
+        "log_file": str(log_file),
+        "total_requests": total_requests,
+        "cache_hits": cache_hits,
+        "cache_writes": cache_writes,
+        "cache_hit_rate": round(cache_hit_rate, 1),
+        "total_cache_read_tokens": total_cache_read_tokens,
+        "total_cache_write_tokens": total_cache_write_tokens,
+        "total_savings": round(total_savings, 4),
+        "avg_savings_per_hit": (
+            round(total_savings / cache_hits, 4) if cache_hits > 0 else 0.0
+        ),
+    }
+
+
+def _display_cache_report(stats: dict[str, Any], verbose: bool = False):
+    """Display formatted cache statistics report.
+
+    Args:
+        stats: Cache statistics dictionary
+        verbose: Show detailed breakdown
+    """
+    # Handle error cases
+    if "error" in stats:
+        print(f"⚠️  {stats['error']}")
+        if "message" in stats:
+            print(f"   {stats['message']}")
+        return
+
+    # Summary section
+    print("=" * 60)
+    print("PROMPT CACHING PERFORMANCE SUMMARY")
+    print("=" * 60)
+    print(f"Analysis Period: Last {stats['days_analyzed']} days")
+    print(f"Log File: {stats['log_file']}")
+    print()
+
+    # Key metrics
+    print("📊 Key Metrics:")
+    print(f"  Total API Requests: {stats['total_requests']:,}")
+    print(f"  Cache Hits: {stats['cache_hits']:,}")
+    print(f"  Cache Writes: {stats['cache_writes']:,}")
+    print(f"  Cache Hit Rate: {stats['cache_hit_rate']}%")
+    print()
+
+    # Cost savings
+    print("💰 Cost Savings:")
+    print(f"  Total Saved: ${stats['total_savings']:.4f}")
+    if stats['cache_hits'] > 0:
+        print(f"  Avg Savings per Hit: ${stats['avg_savings_per_hit']:.4f}")
+    print()
+
+    # Token metrics (verbose mode)
+    if verbose:
+        print("🔢 Token Metrics:")
+        print(f"  Cache Read Tokens: {stats['total_cache_read_tokens']:,}")
+        print(f"  Cache Write Tokens: {stats['total_cache_write_tokens']:,}")
+        print()
+
+    # Performance assessment
+    hit_rate = stats['cache_hit_rate']
+    print("📈 Performance Assessment:")
+    if hit_rate >= 50:
+        print("  ✅ EXCELLENT - Cache is working effectively")
+        print("     Your workflows are benefiting from prompt caching")
+    elif hit_rate >= 30:
+        print("  ✓ GOOD - Cache is providing moderate benefits")
+        print("     Consider structuring prompts for better cache reuse")
+    elif hit_rate >= 10:
+        print("  ⚠️  LOW - Cache hit rate could be improved")
+        print("     Review your workflow patterns for optimization")
+    else:
+        print("  ❌ VERY LOW - Cache is not being utilized effectively")
+        print("     Consider enabling prompt caching or restructuring prompts")
+    print()
+
+    # Recommendations
+    if stats['total_requests'] < 10:
+        print("ℹ️  Note: Limited data available. Run more workflows for accurate stats.")
+    elif hit_rate < 30:
+        print("💡 Recommendations:")
+        print("  1. Reuse system prompts across workflow steps")
+        print("  2. Structure large context (docs, code) for caching")
+        print("  3. Cache TTL is 5 minutes - batch related requests")
+        print("  4. Enable use_prompt_caching=True in AnthropicProvider")
+
+    print("=" * 60)
+
+
+def cmd_cache_clear(args):
+    """Clear cached data (placeholder for future implementation).
+
+    Args:
+        args: Namespace object from argparse
+
+    Returns:
+        None: Prints status message
+    """
+    print("\n⚠️  Cache clearing not implemented.")
+    print("Anthropic's cache has a 5-minute TTL and is server-side.")
+    print("Wait 5 minutes for cache to expire naturally.\n")

empathy_os/cli/commands/inspect.py

@@ -13,8 +13,7 @@ from empathy_os.core import EmpathyOS
 from empathy_os.logging_config import get_logger
 from empathy_os.metrics.collector import MetricsCollector
 from empathy_os.pattern_library import PatternLibrary
-from empathy_os.persistence import PatternPersistence
-from empathy_os.state_manager import StateManager
+from empathy_os.persistence import PatternPersistence, StateManager
 
 logger = get_logger(__name__)
 

empathy_os/cli/commands/metrics.py

@@ -8,7 +8,7 @@ import sys
 
 from empathy_os.logging_config import get_logger
 from empathy_os.metrics.collector import MetricsCollector
-from empathy_os.state_manager import StateManager
+from empathy_os.persistence import StateManager
 
 logger = get_logger(__name__)
 

empathy_os/cli/commands/routing.py

@@ -0,0 +1,285 @@
+"""CLI commands for adaptive model routing statistics.
+
+Provides commands to analyze model routing performance and get tier upgrade
+recommendations based on historical telemetry data.
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+import logging
+from typing import Any
+
+from empathy_os.models import AdaptiveModelRouter
+from empathy_os.telemetry import UsageTracker
+
+logger = logging.getLogger(__name__)
+
+
+def cmd_routing_stats(args: Any) -> int:
+    """Show routing statistics for a workflow.
+
+    Args:
+        args: Arguments with workflow, stage (optional), days
+
+    Returns:
+        0 on success, 1 on error
+    """
+    try:
+        # Get telemetry and router
+        tracker = UsageTracker.get_instance()
+        router = AdaptiveModelRouter(telemetry=tracker)
+
+        # Get routing stats
+        stats = router.get_routing_stats(
+            workflow=args.workflow,
+            stage=args.stage if hasattr(args, "stage") and args.stage else None,
+            days=args.days,
+        )
+
+        if stats["total_calls"] == 0:
+            print(f"❌ No data found for workflow '{args.workflow}'")
+            print(f"   (searched last {args.days} days)")
+            return 1
+
+        # Display stats
+        print("\n" + "=" * 70)
+        print(f"ADAPTIVE ROUTING STATISTICS - {stats['workflow']}")
+        if stats["stage"] != "all":
+            print(f"Stage: {stats['stage']}")
+        print("=" * 70)
+
+        print(f"\n📊 Overview (Last {stats['days_analyzed']} days)")
+        print(f"  Total calls: {stats['total_calls']:,}")
+        print(f"  Average cost: ${stats['avg_cost']:.4f}")
+        print(f"  Average success rate: {stats['avg_success_rate']:.1%}")
+        print(f"  Models used: {len(stats['models_used'])}")
+
+        # Per-model performance
+        print("\n📈 Per-Model Performance")
+        print("-" * 70)
+
+        for model in stats["models_used"]:
+            perf = stats["performance_by_model"][model]
+            print(f"\n  {model}:")
+            print(f"    Calls: {perf['calls']:,}")
+            print(f"    Success rate: {perf['success_rate']:.1%}")
+            print(f"    Avg cost: ${perf['avg_cost']:.4f}")
+            print(f"    Avg latency: {perf['avg_latency_ms']:.0f}ms")
+
+            # Quality score calculation (from AdaptiveModelRouter)
+            quality_score = (perf["success_rate"] * 100) - (perf["avg_cost"] * 10)
+            print(f"    Quality score: {quality_score:.2f}")
+
+        # Recommendations
+        print("\n💡 Recommendations")
+        print("-" * 70)
+
+        # Find best model
+        best_model = max(
+            stats["performance_by_model"].items(),
+            key=lambda x: (x[1]["success_rate"] * 100) - (x[1]["avg_cost"] * 10),
+        )
+
+        print(f"  Best model: {best_model[0]}")
+        print(f"    ({best_model[1]['success_rate']:.1%} success, ${best_model[1]['avg_cost']:.4f}/call)")
+
+        # Cost savings potential
+        if len(stats["models_used"]) > 1:
+            cheapest = min(
+                stats["performance_by_model"].items(),
+                key=lambda x: x[1]["avg_cost"],
+            )
+            most_expensive = max(
+                stats["performance_by_model"].items(),
+                key=lambda x: x[1]["avg_cost"],
+            )
+
+            if cheapest[0] != most_expensive[0]:
+                savings_per_call = most_expensive[1]["avg_cost"] - cheapest[1]["avg_cost"]
+                print(f"\n  💰 Potential savings:")
+                print(f"    Using {cheapest[0]} instead of {most_expensive[0]}")
+                print(f"    ${savings_per_call:.4f} per call")
+                if stats["total_calls"] > 0:
+                    weekly_calls = (stats["total_calls"] / stats["days_analyzed"]) * 7
+                    weekly_savings = savings_per_call * weekly_calls
+                    print(f"    ~${weekly_savings:.2f}/week potential")
+
+        return 0
+
+    except Exception as e:
+        logger.exception("Failed to get routing stats")
+        print(f"❌ Error: {e}")
+        return 1
+
+
+def cmd_routing_check(args: Any) -> int:
+    """Check if tier upgrades are recommended for workflows.
+
+    Args:
+        args: Arguments with workflow (or --all), stage (optional)
+
+    Returns:
+        0 on success, 1 on error
+    """
+    try:
+        # Get telemetry and router
+        tracker = UsageTracker.get_instance()
+        router = AdaptiveModelRouter(telemetry=tracker)
+
+        print("\n" + "=" * 70)
+        print("ADAPTIVE ROUTING - TIER UPGRADE RECOMMENDATIONS")
+        print("=" * 70)
+
+        if hasattr(args, "all") and args.all:
+            # Check all workflows
+            stats = tracker.get_stats(days=args.days)
+            workflows = list(stats["by_workflow"].keys())
+
+            if not workflows:
+                print("\n❌ No workflow data found")
+                return 1
+
+            print(f"\nChecking {len(workflows)} workflows (last {args.days} days)...\n")
+
+            upgrades_needed = []
+            upgrades_ok = []
+
+            for workflow_name in workflows:
+                should_upgrade, reason = router.recommend_tier_upgrade(
+                    workflow=workflow_name, stage=None
+                )
+
+                if should_upgrade:
+                    upgrades_needed.append((workflow_name, reason))
+                else:
+                    upgrades_ok.append((workflow_name, reason))
+
+            # Show workflows needing upgrades
+            if upgrades_needed:
+                print(f"⚠️  {len(upgrades_needed)} workflow(s) need tier upgrade:")
+                print("-" * 70)
+                for workflow_name, reason in upgrades_needed:
+                    print(f"  • {workflow_name}")
+                    print(f"    {reason}")
+                print()
+
+            # Show workflows performing well
+            if upgrades_ok:
+                print(f"✓ {len(upgrades_ok)} workflow(s) performing well:")
+                print("-" * 70)
+                for workflow_name, reason in upgrades_ok:
+                    print(f"  • {workflow_name}: {reason}")
+                print()
+
+            # Summary
+            if upgrades_needed:
+                print("💡 Recommendation:")
+                print("   Enable adaptive routing to automatically upgrade tiers:")
+                print("   workflow = MyWorkflow(enable_adaptive_routing=True)")
+                return 0
+            else:
+                print("✓ All workflows performing well - no upgrades needed")
+                return 0
+
+        else:
+            # Check specific workflow
+            workflow_name = args.workflow
+
+            should_upgrade, reason = router.recommend_tier_upgrade(
+                workflow=workflow_name,
+                stage=args.stage if hasattr(args, "stage") and args.stage else None,
+            )
+
+            print(f"\nWorkflow: {workflow_name}")
+            if hasattr(args, "stage") and args.stage:
+                print(f"Stage: {args.stage}")
+            print(f"Analysis period: Last {args.days} days")
+            print()
+
+            if should_upgrade:
+                print("⚠️  TIER UPGRADE RECOMMENDED")
+                print(f"   {reason}")
+                print()
+                print("💡 Action:")
+                print("   1. Enable adaptive routing:")
+                print("      workflow = MyWorkflow(enable_adaptive_routing=True)")
+                print("   2. Or manually upgrade tier in workflow config")
+                return 0
+            else:
+                print("✓ NO UPGRADE NEEDED")
+                print(f"   {reason}")
+                return 0
+
+    except Exception as e:
+        logger.exception("Failed to check routing recommendations")
+        print(f"❌ Error: {e}")
+        return 1
+
+
+def cmd_routing_models(args: Any) -> int:
+    """Show model performance comparison.
+
+    Args:
+        args: Arguments with provider, days
+
+    Returns:
+        0 on success, 1 on error
+    """
+    try:
+        # Get telemetry
+        tracker = UsageTracker.get_instance()
+
+        # Get recent entries
+        entries = tracker.get_recent_entries(limit=100000, days=args.days)
+
+        if args.provider:
+            entries = [e for e in entries if e.get("provider") == args.provider]
+
+        if not entries:
+            print(f"❌ No data found for provider '{args.provider}'")
+            return 1
+
+        # Group by model
+        by_model: dict[str, list] = {}
+        for entry in entries:
+            model = entry["model"]
+            if model not in by_model:
+                by_model[model] = []
+            by_model[model].append(entry)
+
+        print("\n" + "=" * 70)
+        print(f"MODEL PERFORMANCE COMPARISON - {args.provider.upper()}")
+        print(f"Last {args.days} days")
+        print("=" * 70)
+
+        # Sort by total calls
+        models_sorted = sorted(by_model.items(), key=lambda x: len(x[1]), reverse=True)
+
+        print(f"\n📊 {len(models_sorted)} model(s) used\n")
+
+        for model, model_entries in models_sorted:
+            total = len(model_entries)
+            successes = sum(1 for e in model_entries if e.get("success", True))
+            success_rate = successes / total
+
+            avg_cost = sum(e.get("cost", 0.0) for e in model_entries) / total
+            avg_latency = sum(e.get("duration_ms", 0) for e in model_entries) / total
+
+            # Quality score
+            quality_score = (success_rate * 100) - (avg_cost * 10)
+
+            print(f"  {model}")
+            print(f"    Calls: {total:,}")
+            print(f"    Success rate: {success_rate:.1%}")
+            print(f"    Avg cost: ${avg_cost:.4f}")
+            print(f"    Avg latency: {avg_latency:.0f}ms")
+            print(f"    Quality score: {quality_score:.2f}")
+            print()
+
+        return 0
+
+    except Exception as e:
+        logger.exception("Failed to get model performance")
+        print(f"❌ Error: {e}")
+        return 1

empathy_os/cli/commands/workflow.py

@@ -11,8 +11,8 @@ from pathlib import Path
 
 from empathy_os.config import _validate_file_path
 from empathy_os.logging_config import get_logger
-from empathy_os.workflows import list_workflows as get_workflow_list
-from empathy_os.workflows.config import WorkflowConfig, create_example_config, get_workflow
+from empathy_os.workflows import get_workflow, list_workflows as get_workflow_list
+from empathy_os.workflows.config import WorkflowConfig, create_example_config
 
 logger = get_logger(__name__)
 

empathy_os/cli/parsers/__init__.py

@@ -7,6 +7,8 @@ Licensed under Fair Source License 0.9
 """
 
 from . import (
+    batch,
+    cache,
     help,
     info,
     inspect,
@@ -14,6 +16,7 @@ from . import (
     orchestrate,
     patterns,
     provider,
+    routing,
     setup,
     status,
     sync,
@@ -57,6 +60,9 @@ def register_all_parsers(subparsers):
 
     # Metrics and state
     metrics.register_parsers(subparsers)
+    cache.register_parsers(subparsers)  # Cache monitoring
+    batch.register_parsers(subparsers)  # Batch processing (50% cost savings)
+    routing.register_parsers(subparsers)  # Adaptive routing statistics
 
     # Setup and initialization
     setup.register_parsers(subparsers)