empathy-framework 5.0.1__py3-none-any.whl → 5.1.0__py3-none-any.whl

empathy_os/cli/commands/routing.py

@@ -0,0 +1,285 @@
+"""CLI commands for adaptive model routing statistics.
+
+Provides commands to analyze model routing performance and get tier upgrade
+recommendations based on historical telemetry data.
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+import logging
+from typing import Any
+
+from empathy_os.models import AdaptiveModelRouter
+from empathy_os.telemetry import UsageTracker
+
+logger = logging.getLogger(__name__)
+
+
+def cmd_routing_stats(args: Any) -> int:
+    """Show routing statistics for a workflow.
+
+    Args:
+        args: Arguments with workflow, stage (optional), days
+
+    Returns:
+        0 on success, 1 on error
+    """
+    try:
+        # Get telemetry and router
+        tracker = UsageTracker.get_instance()
+        router = AdaptiveModelRouter(telemetry=tracker)
+
+        # Get routing stats
+        stats = router.get_routing_stats(
+            workflow=args.workflow,
+            stage=args.stage if hasattr(args, "stage") and args.stage else None,
+            days=args.days,
+        )
+
+        if stats["total_calls"] == 0:
+            print(f"❌ No data found for workflow '{args.workflow}'")
+            print(f"   (searched last {args.days} days)")
+            return 1
+
+        # Display stats
+        print("\n" + "=" * 70)
+        print(f"ADAPTIVE ROUTING STATISTICS - {stats['workflow']}")
+        if stats["stage"] != "all":
+            print(f"Stage: {stats['stage']}")
+        print("=" * 70)
+
+        print(f"\n📊 Overview (Last {stats['days_analyzed']} days)")
+        print(f"  Total calls: {stats['total_calls']:,}")
+        print(f"  Average cost: ${stats['avg_cost']:.4f}")
+        print(f"  Average success rate: {stats['avg_success_rate']:.1%}")
+        print(f"  Models used: {len(stats['models_used'])}")
+
+        # Per-model performance
+        print("\n📈 Per-Model Performance")
+        print("-" * 70)
+
+        for model in stats["models_used"]:
+            perf = stats["performance_by_model"][model]
+            print(f"\n  {model}:")
+            print(f"    Calls: {perf['calls']:,}")
+            print(f"    Success rate: {perf['success_rate']:.1%}")
+            print(f"    Avg cost: ${perf['avg_cost']:.4f}")
+            print(f"    Avg latency: {perf['avg_latency_ms']:.0f}ms")
+
+            # Quality score calculation (from AdaptiveModelRouter)
+            quality_score = (perf["success_rate"] * 100) - (perf["avg_cost"] * 10)
+            print(f"    Quality score: {quality_score:.2f}")
+
+        # Recommendations
+        print("\n💡 Recommendations")
+        print("-" * 70)
+
+        # Find best model
+        best_model = max(
+            stats["performance_by_model"].items(),
+            key=lambda x: (x[1]["success_rate"] * 100) - (x[1]["avg_cost"] * 10),
+        )
+
+        print(f"  Best model: {best_model[0]}")
+        print(f"    ({best_model[1]['success_rate']:.1%} success, ${best_model[1]['avg_cost']:.4f}/call)")
+
+        # Cost savings potential
+        if len(stats["models_used"]) > 1:
+            cheapest = min(
+                stats["performance_by_model"].items(),
+                key=lambda x: x[1]["avg_cost"],
+            )
+            most_expensive = max(
+                stats["performance_by_model"].items(),
+                key=lambda x: x[1]["avg_cost"],
+            )
+
+            if cheapest[0] != most_expensive[0]:
+                savings_per_call = most_expensive[1]["avg_cost"] - cheapest[1]["avg_cost"]
+                print("\n  💰 Potential savings:")
+                print(f"    Using {cheapest[0]} instead of {most_expensive[0]}")
+                print(f"    ${savings_per_call:.4f} per call")
+                if stats["total_calls"] > 0:
+                    weekly_calls = (stats["total_calls"] / stats["days_analyzed"]) * 7
+                    weekly_savings = savings_per_call * weekly_calls
+                    print(f"    ~${weekly_savings:.2f}/week potential")
+
+        return 0
+
+    except Exception as e:
+        logger.exception("Failed to get routing stats")
+        print(f"❌ Error: {e}")
+        return 1
+
+
+def cmd_routing_check(args: Any) -> int:
+    """Check if tier upgrades are recommended for workflows.
+
+    Args:
+        args: Arguments with workflow (or --all), stage (optional)
+
+    Returns:
+        0 on success, 1 on error
+    """
+    try:
+        # Get telemetry and router
+        tracker = UsageTracker.get_instance()
+        router = AdaptiveModelRouter(telemetry=tracker)
+
+        print("\n" + "=" * 70)
+        print("ADAPTIVE ROUTING - TIER UPGRADE RECOMMENDATIONS")
+        print("=" * 70)
+
+        if hasattr(args, "all") and args.all:
+            # Check all workflows
+            stats = tracker.get_stats(days=args.days)
+            workflows = list(stats["by_workflow"].keys())
+
+            if not workflows:
+                print("\n❌ No workflow data found")
+                return 1
+
+            print(f"\nChecking {len(workflows)} workflows (last {args.days} days)...\n")
+
+            upgrades_needed = []
+            upgrades_ok = []
+
+            for workflow_name in workflows:
+                should_upgrade, reason = router.recommend_tier_upgrade(
+                    workflow=workflow_name, stage=None
+                )
+
+                if should_upgrade:
+                    upgrades_needed.append((workflow_name, reason))
+                else:
+                    upgrades_ok.append((workflow_name, reason))
+
+            # Show workflows needing upgrades
+            if upgrades_needed:
+                print(f"⚠️  {len(upgrades_needed)} workflow(s) need tier upgrade:")
+                print("-" * 70)
+                for workflow_name, reason in upgrades_needed:
+                    print(f"  • {workflow_name}")
+                    print(f"    {reason}")
+                print()
+
+            # Show workflows performing well
+            if upgrades_ok:
+                print(f"✓ {len(upgrades_ok)} workflow(s) performing well:")
+                print("-" * 70)
+                for workflow_name, reason in upgrades_ok:
+                    print(f"  • {workflow_name}: {reason}")
+                print()
+
+            # Summary
+            if upgrades_needed:
+                print("💡 Recommendation:")
+                print("   Enable adaptive routing to automatically upgrade tiers:")
+                print("   workflow = MyWorkflow(enable_adaptive_routing=True)")
+                return 0
+            else:
+                print("✓ All workflows performing well - no upgrades needed")
+                return 0
+
+        else:
+            # Check specific workflow
+            workflow_name = args.workflow
+
+            should_upgrade, reason = router.recommend_tier_upgrade(
+                workflow=workflow_name,
+                stage=args.stage if hasattr(args, "stage") and args.stage else None,
+            )
+
+            print(f"\nWorkflow: {workflow_name}")
+            if hasattr(args, "stage") and args.stage:
+                print(f"Stage: {args.stage}")
+            print(f"Analysis period: Last {args.days} days")
+            print()
+
+            if should_upgrade:
+                print("⚠️  TIER UPGRADE RECOMMENDED")
+                print(f"   {reason}")
+                print()
+                print("💡 Action:")
+                print("   1. Enable adaptive routing:")
+                print("      workflow = MyWorkflow(enable_adaptive_routing=True)")
+                print("   2. Or manually upgrade tier in workflow config")
+                return 0
+            else:
+                print("✓ NO UPGRADE NEEDED")
+                print(f"   {reason}")
+                return 0
+
+    except Exception as e:
+        logger.exception("Failed to check routing recommendations")
+        print(f"❌ Error: {e}")
+        return 1
+
+
+def cmd_routing_models(args: Any) -> int:
+    """Show model performance comparison.
+
+    Args:
+        args: Arguments with provider, days
+
+    Returns:
+        0 on success, 1 on error
+    """
+    try:
+        # Get telemetry
+        tracker = UsageTracker.get_instance()
+
+        # Get recent entries
+        entries = tracker.get_recent_entries(limit=100000, days=args.days)
+
+        if args.provider:
+            entries = [e for e in entries if e.get("provider") == args.provider]
+
+        if not entries:
+            print(f"❌ No data found for provider '{args.provider}'")
+            return 1
+
+        # Group by model
+        by_model: dict[str, list] = {}
+        for entry in entries:
+            model = entry["model"]
+            if model not in by_model:
+                by_model[model] = []
+            by_model[model].append(entry)
+
+        print("\n" + "=" * 70)
+        print(f"MODEL PERFORMANCE COMPARISON - {args.provider.upper()}")
+        print(f"Last {args.days} days")
+        print("=" * 70)
+
+        # Sort by total calls
+        models_sorted = sorted(by_model.items(), key=lambda x: len(x[1]), reverse=True)
+
+        print(f"\n📊 {len(models_sorted)} model(s) used\n")
+
+        for model, model_entries in models_sorted:
+            total = len(model_entries)
+            successes = sum(1 for e in model_entries if e.get("success", True))
+            success_rate = successes / total
+
+            avg_cost = sum(e.get("cost", 0.0) for e in model_entries) / total
+            avg_latency = sum(e.get("duration_ms", 0) for e in model_entries) / total
+
+            # Quality score
+            quality_score = (success_rate * 100) - (avg_cost * 10)
+
+            print(f"  {model}")
+            print(f"    Calls: {total:,}")
+            print(f"    Success rate: {success_rate:.1%}")
+            print(f"    Avg cost: ${avg_cost:.4f}")
+            print(f"    Avg latency: {avg_latency:.0f}ms")
+            print(f"    Quality score: {quality_score:.2f}")
+            print()
+
+        return 0
+
+    except Exception as e:
+        logger.exception("Failed to get model performance")
+        print(f"❌ Error: {e}")
+        return 1

empathy_os/cli/commands/workflow.py

@@ -11,8 +11,9 @@ from pathlib import Path
 
 from empathy_os.config import _validate_file_path
 from empathy_os.logging_config import get_logger
+from empathy_os.workflows import get_workflow
 from empathy_os.workflows import list_workflows as get_workflow_list
-from empathy_os.workflows.config import WorkflowConfig, create_example_config, get_workflow
+from empathy_os.workflows.config import WorkflowConfig, create_example_config
 
 logger = get_logger(__name__)
 

empathy_os/cli/parsers/__init__.py

@@ -7,6 +7,8 @@ Licensed under Fair Source License 0.9
 """
 
 from . import (
+    batch,
+    cache,
     help,
     info,
     inspect,
@@ -14,6 +16,7 @@ from . import (
     orchestrate,
     patterns,
     provider,
+    routing,
     setup,
     status,
     sync,
@@ -57,6 +60,9 @@ def register_all_parsers(subparsers):
 
     # Metrics and state
     metrics.register_parsers(subparsers)
+    cache.register_parsers(subparsers)  # Cache monitoring
+    batch.register_parsers(subparsers)  # Batch processing (50% cost savings)
+    routing.register_parsers(subparsers)  # Adaptive routing statistics
 
     # Setup and initialization
     setup.register_parsers(subparsers)

empathy_os/cli/parsers/batch.py

@@ -0,0 +1,118 @@
+"""Argument parser for batch processing commands.
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+
+def register_parsers(subparsers):
+    """Register batch command parsers.
+
+    Args:
+        subparsers: Subparser object from main argument parser
+
+    Returns:
+        None: Adds batch subparser with submit, status, results, wait subcommands
+    """
+    from ..commands.batch import (
+        cmd_batch_results,
+        cmd_batch_status,
+        cmd_batch_submit,
+        cmd_batch_wait,
+    )
+
+    # Main batch command
+    batch_parser = subparsers.add_parser(
+        "batch",
+        help="Batch processing via Anthropic Batch API (50% cost savings)",
+        description="Submit and manage batch processing jobs for non-urgent tasks",
+    )
+
+    # Batch subcommands
+    batch_subparsers = batch_parser.add_subparsers(dest="batch_command", required=True)
+
+    # batch submit command
+    submit_parser = batch_subparsers.add_parser(
+        "submit",
+        help="Submit a batch processing job from JSON file",
+        description="Submit batch requests for asynchronous processing (50% cost savings)",
+    )
+
+    submit_parser.add_argument(
+        "input_file",
+        help='JSON file with batch requests. Format: [{"task_id": "...", "task_type": "...", "input_data": {...}}]',
+    )
+
+    submit_parser.set_defaults(func=cmd_batch_submit)
+
+    # batch status command
+    status_parser = batch_subparsers.add_parser(
+        "status",
+        help="Check status of a batch processing job",
+        description="Display current status and request counts for a batch",
+    )
+
+    status_parser.add_argument(
+        "batch_id",
+        help="Batch ID (e.g., msgbatch_abc123)",
+    )
+
+    status_parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output raw JSON status",
+    )
+
+    status_parser.set_defaults(func=cmd_batch_status)
+
+    # batch results command
+    results_parser = batch_subparsers.add_parser(
+        "results",
+        help="Retrieve results from completed batch",
+        description="Download and save batch results to JSON file",
+    )
+
+    results_parser.add_argument(
+        "batch_id",
+        help="Batch ID (e.g., msgbatch_abc123)",
+    )
+
+    results_parser.add_argument(
+        "output_file",
+        help="Path to output JSON file",
+    )
+
+    results_parser.set_defaults(func=cmd_batch_results)
+
+    # batch wait command
+    wait_parser = batch_subparsers.add_parser(
+        "wait",
+        help="Wait for batch to complete and retrieve results",
+        description="Poll batch status until completion, then download results",
+    )
+
+    wait_parser.add_argument(
+        "batch_id",
+        help="Batch ID (e.g., msgbatch_abc123)",
+    )
+
+    wait_parser.add_argument(
+        "output_file",
+        help="Path to output JSON file",
+    )
+
+    wait_parser.add_argument(
+        "--poll-interval",
+        type=int,
+        default=300,
+        help="Seconds between status checks (default: 300 = 5 minutes)",
+    )
+
+    wait_parser.add_argument(
+        "--timeout",
+        type=int,
+        default=86400,
+        help="Maximum wait time in seconds (default: 86400 = 24 hours)",
+    )
+
+    wait_parser.set_defaults(func=cmd_batch_wait)

empathy_os/cli/parsers/cache 2.py

@@ -0,0 +1,65 @@
+"""Argument parser for cache commands.
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+
+def register_parsers(subparsers):
+    """Register cache command parsers.
+
+    Args:
+        subparsers: Subparser object from main argument parser
+
+    Returns:
+        None: Adds cache subparser with stats and clear subcommands
+    """
+    from ..commands.cache import cmd_cache_clear, cmd_cache_stats
+    # Main cache command
+    cache_parser = subparsers.add_parser(
+        "cache",
+        help="Cache monitoring and management",
+        description="Monitor prompt caching performance and cost savings",
+    )
+
+    # Cache subcommands
+    cache_subparsers = cache_parser.add_subparsers(dest="cache_command", required=True)
+
+    # cache stats command
+    stats_parser = cache_subparsers.add_parser(
+        "stats",
+        help="Show cache performance statistics",
+        description="Display prompt caching metrics including hit rate and cost savings",
+    )
+
+    stats_parser.add_argument(
+        "--days",
+        type=int,
+        default=7,
+        help="Number of days to analyze (default: 7)",
+    )
+
+    stats_parser.add_argument(
+        "--format",
+        choices=["table", "json"],
+        default="table",
+        help="Output format (default: table)",
+    )
+
+    stats_parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Show detailed token metrics",
+    )
+
+    stats_parser.set_defaults(func=cmd_cache_stats)
+
+    # cache clear command (placeholder)
+    clear_parser = cache_subparsers.add_parser(
+        "clear",
+        help="Clear cache (note: Anthropic cache is server-side with 5min TTL)",
+        description="Information about cache clearing",
+    )
+
+    clear_parser.set_defaults(func=cmd_cache_clear)

empathy_os/cli/parsers/cache.py

@@ -0,0 +1,65 @@
+"""Argument parser for cache commands.
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+
+def register_parsers(subparsers):
+    """Register cache command parsers.
+
+    Args:
+        subparsers: Subparser object from main argument parser
+
+    Returns:
+        None: Adds cache subparser with stats and clear subcommands
+    """
+    from ..commands.cache import cmd_cache_clear, cmd_cache_stats
+    # Main cache command
+    cache_parser = subparsers.add_parser(
+        "cache",
+        help="Cache monitoring and management",
+        description="Monitor prompt caching performance and cost savings",
+    )
+
+    # Cache subcommands
+    cache_subparsers = cache_parser.add_subparsers(dest="cache_command", required=True)
+
+    # cache stats command
+    stats_parser = cache_subparsers.add_parser(
+        "stats",
+        help="Show cache performance statistics",
+        description="Display prompt caching metrics including hit rate and cost savings",
+    )
+
+    stats_parser.add_argument(
+        "--days",
+        type=int,
+        default=7,
+        help="Number of days to analyze (default: 7)",
+    )
+
+    stats_parser.add_argument(
+        "--format",
+        choices=["table", "json"],
+        default="table",
+        help="Output format (default: table)",
+    )
+
+    stats_parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Show detailed token metrics",
+    )
+
+    stats_parser.set_defaults(func=cmd_cache_stats)
+
+    # cache clear command (placeholder)
+    clear_parser = cache_subparsers.add_parser(
+        "clear",
+        help="Clear cache (note: Anthropic cache is server-side with 5min TTL)",
+        description="Information about cache clearing",
+    )
+
+    clear_parser.set_defaults(func=cmd_cache_clear)

empathy_os/cli/parsers/routing.py

@@ -0,0 +1,110 @@
+"""Argument parser for adaptive routing commands.
+
+Copyright 2025 Smart-AI-Memory
+Licensed under Fair Source License 0.9
+"""
+
+
+def register_parsers(subparsers):
+    """Register routing command parsers.
+
+    Args:
+        subparsers: Subparser object from main argument parser
+
+    Returns:
+        None: Adds routing subparser with stats, check, models subcommands
+    """
+    from ..commands.routing import (
+        cmd_routing_check,
+        cmd_routing_models,
+        cmd_routing_stats,
+    )
+
+    # Main routing command
+    routing_parser = subparsers.add_parser(
+        "routing",
+        help="Adaptive model routing statistics and recommendations",
+        description="Analyze model routing performance based on historical telemetry",
+    )
+
+    # Routing subcommands
+    routing_subparsers = routing_parser.add_subparsers(
+        dest="routing_command", required=True
+    )
+
+    # routing stats command
+    stats_parser = routing_subparsers.add_parser(
+        "stats",
+        help="Show routing statistics for a workflow",
+        description="Display model performance metrics and recommendations",
+    )
+
+    stats_parser.add_argument("workflow", help="Workflow name (e.g., 'code-review')")
+
+    stats_parser.add_argument(
+        "--stage",
+        help="Stage name (optional, shows all stages if not specified)",
+    )
+
+    stats_parser.add_argument(
+        "--days",
+        type=int,
+        default=7,
+        help="Number of days to analyze (default: 7)",
+    )
+
+    stats_parser.set_defaults(func=cmd_routing_stats)
+
+    # routing check command
+    check_parser = routing_subparsers.add_parser(
+        "check",
+        help="Check for tier upgrade recommendations",
+        description="Analyze failure rates and recommend tier upgrades",
+    )
+
+    check_parser.add_argument(
+        "--workflow",
+        help="Workflow name (required unless --all is used)",
+    )
+
+    check_parser.add_argument(
+        "--stage",
+        help="Stage name (optional)",
+    )
+
+    check_parser.add_argument(
+        "--all",
+        action="store_true",
+        help="Check all workflows",
+    )
+
+    check_parser.add_argument(
+        "--days",
+        type=int,
+        default=7,
+        help="Number of days to analyze (default: 7)",
+    )
+
+    check_parser.set_defaults(func=cmd_routing_check)
+
+    # routing models command
+    models_parser = routing_subparsers.add_parser(
+        "models",
+        help="Compare model performance",
+        description="Show performance metrics for all models from a provider",
+    )
+
+    models_parser.add_argument(
+        "--provider",
+        default="anthropic",
+        help="Provider name (default: anthropic)",
+    )
+
+    models_parser.add_argument(
+        "--days",
+        type=int,
+        default=7,
+        help="Number of days to analyze (default: 7)",
+    )
+
+    models_parser.set_defaults(func=cmd_routing_models)