attune-ai 2.0.0__py3-none-any.whl

attune/workflows/progressive/cli.py

@@ -0,0 +1,219 @@
+"""CLI commands for progressive workflow management.
+
+Provides commands for:
+- Listing saved results
+- Viewing detailed reports
+- Generating analytics
+- Cleaning up old results
+"""
+
+import argparse
+import sys
+
+from attune.workflows.progressive.reports import (
+    cleanup_old_results,
+    format_cost_analytics_report,
+    generate_cost_analytics,
+    list_saved_results,
+    load_result_from_disk,
+)
+
+
+def cmd_list_results(args: argparse.Namespace) -> int:
+    """List all saved progressive workflow results.
+
+    Args:
+        args: Parsed command line arguments
+
+    Returns:
+        Exit code (0 for success)
+    """
+    storage_path = args.storage_path or ".attune/progressive_runs"
+    results = list_saved_results(storage_path)
+
+    if not results:
+        print(f"No results found in {storage_path}")
+        return 0
+
+    print(f"\n📋 Found {len(results)} progressive workflow results:\n")
+    print(f"{'Task ID':<40} {'Workflow':<15} {'Cost':<10} {'Savings':<12} {'Success'}")
+    print("─" * 90)
+
+    for result in results:
+        task_id = result.get("task_id", "unknown")
+        workflow = result.get("workflow", "unknown")[:14]
+        cost = result.get("total_cost", 0.0)
+        savings = result.get("cost_savings_percent", 0.0)
+        success = "✅" if result.get("success", False) else "❌"
+
+        print(f"{task_id:<40} {workflow:<15} ${cost:<9.2f} {savings:>6.1f}%      {success}")
+
+    print()
+    return 0
+
+
+def cmd_show_report(args: argparse.Namespace) -> int:
+    """Show detailed report for a specific task.
+
+    Args:
+        args: Parsed command line arguments
+
+    Returns:
+        Exit code (0 for success, 1 for error)
+    """
+    task_id = args.task_id
+    storage_path = args.storage_path or ".attune/progressive_runs"
+
+    try:
+        result_data = load_result_from_disk(task_id, storage_path)
+
+        if args.json:
+            import json
+
+            print(json.dumps(result_data, indent=2))
+        else:
+            # Show human-readable report
+            report = result_data.get("report", "")
+            if report:
+                print(report)
+            else:
+                print("No report found for this task")
+
+        return 0
+
+    except FileNotFoundError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+
+def cmd_analytics(args: argparse.Namespace) -> int:
+    """Show cost optimization analytics.
+
+    Args:
+        args: Parsed command line arguments
+
+    Returns:
+        Exit code (0 for success)
+    """
+    storage_path = args.storage_path or ".attune/progressive_runs"
+    analytics = generate_cost_analytics(storage_path)
+
+    if analytics["total_runs"] == 0:
+        print(f"No results found in {storage_path}")
+        return 0
+
+    if args.json:
+        import json
+
+        print(json.dumps(analytics, indent=2))
+    else:
+        report = format_cost_analytics_report(analytics)
+        print(report)
+
+    return 0
+
+
+def cmd_cleanup(args: argparse.Namespace) -> int:
+    """Clean up old progressive workflow results.
+
+    Args:
+        args: Parsed command line arguments
+
+    Returns:
+        Exit code (0 for success)
+    """
+    storage_path = args.storage_path or ".attune/progressive_runs"
+    retention_days = args.retention_days
+    dry_run = args.dry_run
+
+    deleted, retained = cleanup_old_results(
+        storage_path=storage_path, retention_days=retention_days, dry_run=dry_run
+    )
+
+    if dry_run:
+        print("\n🔍 Dry run mode - no files deleted\n")
+        print(f"Would delete: {deleted} results older than {retention_days} days")
+        print(f"Would retain: {retained} recent results")
+    else:
+        print("\n🗑️  Cleanup complete\n")
+        print(f"Deleted: {deleted} results older than {retention_days} days")
+        print(f"Retained: {retained} recent results")
+
+    return 0
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """Create argument parser for progressive CLI.
+
+    Returns:
+        Configured argument parser
+    """
+    parser = argparse.ArgumentParser(
+        prog="empathy progressive", description="Manage progressive tier escalation workflows"
+    )
+
+    parser.add_argument(
+        "--storage-path",
+        type=str,
+        default=None,
+        help="Custom storage path (default: .attune/progressive_runs)",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # List command
+    list_parser = subparsers.add_parser("list", help="List all saved progressive workflow results")
+    list_parser.set_defaults(func=cmd_list_results)
+
+    # Show command
+    show_parser = subparsers.add_parser("show", help="Show detailed report for a specific task")
+    show_parser.add_argument("task_id", type=str, help="Task ID to display")
+    show_parser.add_argument("--json", action="store_true", help="Output in JSON format")
+    show_parser.set_defaults(func=cmd_show_report)
+
+    # Analytics command
+    analytics_parser = subparsers.add_parser("analytics", help="Show cost optimization analytics")
+    analytics_parser.add_argument("--json", action="store_true", help="Output in JSON format")
+    analytics_parser.set_defaults(func=cmd_analytics)
+
+    # Cleanup command
+    cleanup_parser = subparsers.add_parser(
+        "cleanup", help="Clean up old progressive workflow results"
+    )
+    cleanup_parser.add_argument(
+        "--retention-days",
+        type=int,
+        default=30,
+        help="Number of days to retain results (default: 30)",
+    )
+    cleanup_parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show what would be deleted without actually deleting",
+    )
+    cleanup_parser.set_defaults(func=cmd_cleanup)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Main entry point for progressive CLI.
+
+    Args:
+        argv: Command line arguments (defaults to sys.argv[1:])
+
+    Returns:
+        Exit code
+    """
+    parser = create_parser()
+    args = parser.parse_args(argv)
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        return 1
+
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

attune/workflows/progressive/core.py

@@ -0,0 +1,488 @@
+"""Core data structures for progressive tier escalation.
+
+This module defines the fundamental data structures used throughout the
+progressive escalation system, including failure analysis, quality metrics,
+tier results, and configuration.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+
+class Tier(Enum):
+    """Model tier levels for progressive escalation.
+
+    Attributes:
+        CHEAP: Low-cost models (e.g., gpt-4o-mini, claude-3-haiku)
+        CAPABLE: Mid-tier models (e.g., claude-3-5-sonnet, gpt-4o)
+        PREMIUM: High-end models (e.g., claude-opus-4, o1)
+    """
+
+    CHEAP = "cheap"
+    CAPABLE = "capable"
+    PREMIUM = "premium"
+
+    def __lt__(self, other: "Tier") -> bool:
+        """Compare tiers for ordering (CHEAP < CAPABLE < PREMIUM)."""
+        order = {Tier.CHEAP: 0, Tier.CAPABLE: 1, Tier.PREMIUM: 2}
+        return order[self] < order[other]
+
+
+@dataclass
+class FailureAnalysis:
+    """Multi-signal failure detection and quality analysis.
+
+    Combines multiple signals to provide robust failure detection:
+    1. Syntax errors in generated code
+    2. Execution failures (test pass rate)
+    3. Quality metrics (coverage, assertion depth)
+    4. LLM confidence signals
+
+    The composite quality score (CQS) provides an objective measure
+    that combines all signals with appropriate weighting.
+
+    Attributes:
+        syntax_errors: List of syntax errors found in generated code
+        test_failures: List of test execution failures
+        test_pass_rate: Percentage of tests that passed (0.0-1.0)
+        coverage_percent: Code coverage percentage (0.0-100.0)
+        assertion_depth: Average number of assertions per test
+        confidence_score: LLM confidence level (0.0-1.0)
+        llm_uncertainty_signals: Uncertainty phrases detected in LLM response
+
+    Example:
+        >>> analysis = FailureAnalysis(
+        ...     test_pass_rate=0.85,
+        ...     coverage_percent=78.0,
+        ...     assertion_depth=5.2,
+        ...     confidence_score=0.92
+        ... )
+        >>> analysis.calculate_quality_score()
+        87.7
+        >>> analysis.should_escalate
+        False
+    """
+
+    syntax_errors: list[SyntaxError] = field(default_factory=list)
+    test_failures: list[dict[str, Any]] = field(default_factory=list)
+    test_pass_rate: float = 0.0
+    coverage_percent: float = 0.0
+    assertion_depth: float = 0.0
+    confidence_score: float = 0.0
+    llm_uncertainty_signals: list[str] = field(default_factory=list)
+
+    def calculate_quality_score(self) -> float:
+        """Calculate composite quality score (CQS) from 0-100.
+
+        Formula:
+            CQS = (
+                0.40 × test_pass_rate +
+                0.25 × code_coverage +
+                0.20 × assertion_quality +
+                0.15 × llm_confidence
+            ) × syntax_error_penalty
+
+        Weights:
+            - Test pass rate: 40% (most important - functionality must work)
+            - Code coverage: 25% (thoroughness matters)
+            - Assertion quality: 20% (test depth is important)
+            - LLM confidence: 15% (signals potential brittleness)
+
+        Penalties:
+            - Syntax errors: 50% penalty (halves the score)
+
+        Returns:
+            Quality score from 0.0 (worst) to 100.0 (perfect)
+
+        Example:
+            >>> analysis = FailureAnalysis(
+            ...     test_pass_rate=0.90,
+            ...     coverage_percent=85.0,
+            ...     assertion_depth=6.0,
+            ...     confidence_score=0.95
+            ... )
+            >>> analysis.calculate_quality_score()
+            91.25
+        """
+        # Component scores (convert to 0-100 scale)
+        pass_rate_score = self.test_pass_rate * 100
+        coverage_score = self.coverage_percent
+
+        # Assertion quality: cap at 100% (10 assertions = 100%)
+        assertion_quality_score = min(self.assertion_depth * 10, 100)
+
+        confidence_score_scaled = self.confidence_score * 100
+
+        # Weighted composite
+        cqs = (
+            0.40 * pass_rate_score
+            + 0.25 * coverage_score
+            + 0.20 * assertion_quality_score
+            + 0.15 * confidence_score_scaled
+        )
+
+        # Apply syntax error penalty
+        if len(self.syntax_errors) > 0:
+            cqs *= 0.5  # Halve score for any syntax errors
+
+        return min(cqs, 100.0)
+
+    @property
+    def should_escalate(self) -> bool:
+        """Determine if this result should trigger escalation.
+
+        Multi-criteria decision based on:
+        - Low CQS (<70)
+        - Multiple syntax errors (>3)
+        - Low test pass rate (<70%)
+        - Low coverage (<60%)
+
+        Returns:
+            True if escalation is recommended, False otherwise
+
+        Example:
+            >>> analysis = FailureAnalysis(test_pass_rate=0.50)
+            >>> analysis.should_escalate
+            True
+        """
+        cqs = self.calculate_quality_score()
+        return (
+            cqs < 70
+            or len(self.syntax_errors) > 3
+            or self.test_pass_rate < 0.7
+            or self.coverage_percent < 60
+        )
+
+    @property
+    def failure_severity(self) -> str:
+        """Determine severity level of failures.
+
+        Returns:
+            "CRITICAL": Severe failures, consider skipping to Premium
+            "HIGH": Significant failures, escalate to next tier
+            "MODERATE": Minor failures, retry at current tier
+            "LOW": Acceptable quality, no escalation needed
+
+        Example:
+            >>> analysis = FailureAnalysis(test_pass_rate=0.25)
+            >>> analysis.failure_severity
+            'CRITICAL'
+        """
+        cqs = self.calculate_quality_score()
+
+        if len(self.syntax_errors) > 5 or self.test_pass_rate < 0.3:
+            return "CRITICAL"
+        elif cqs < 70 or self.test_pass_rate < 0.5:
+            return "HIGH"
+        elif cqs < 80 or self.test_pass_rate < 0.7:
+            return "MODERATE"
+        else:
+            return "LOW"
+
+
+@dataclass
+class TierResult:
+    """Results from a single tier execution attempt.
+
+    Captures all information about a tier's execution including
+    generated artifacts, quality analysis, cost, and escalation decision.
+
+    Attributes:
+        tier: Which tier executed (CHEAP, CAPABLE, or PREMIUM)
+        model: Specific model used (e.g., "gpt-4o-mini")
+        attempt: Attempt number at this tier (1-based)
+        timestamp: When this execution occurred
+        generated_items: Generated artifacts (tests, code, etc.)
+        failure_analysis: Quality and failure analysis
+        cost: Cost in USD for this execution
+        duration: Execution time in seconds
+        escalated: Whether this result triggered escalation
+        escalation_reason: Human-readable reason for escalation
+
+    Example:
+        >>> result = TierResult(
+        ...     tier=Tier.CHEAP,
+        ...     model="gpt-4o-mini",
+        ...     attempt=1,
+        ...     timestamp=datetime.now(),
+        ...     generated_items=[{"code": "test_foo()"}],
+        ...     failure_analysis=FailureAnalysis(test_pass_rate=0.65),
+        ...     cost=0.15,
+        ...     duration=12.5
+        ... )
+        >>> result.quality_score
+        65.0
+    """
+
+    tier: Tier
+    model: str
+    attempt: int
+    timestamp: datetime
+
+    # Generated artifacts
+    generated_items: list[dict[str, Any]] = field(default_factory=list)
+
+    # Analysis
+    failure_analysis: FailureAnalysis = field(default_factory=FailureAnalysis)
+    cost: float = 0.0
+    duration: float = 0.0
+    tokens_used: dict[str, int] = field(default_factory=dict)
+
+    # Decision
+    escalated: bool = False
+    escalation_reason: str = ""
+
+    @property
+    def quality_score(self) -> float:
+        """Get composite quality score for this tier result.
+
+        Returns:
+            CQS from 0.0 to 100.0
+        """
+        return self.failure_analysis.calculate_quality_score()
+
+    @property
+    def success_count(self) -> int:
+        """Count of successfully generated items (CQS >= 80).
+
+        Returns:
+            Number of items meeting quality threshold
+        """
+        return sum(1 for item in self.generated_items if item.get("quality_score", 0) >= 80)
+
+    @property
+    def success_rate(self) -> float:
+        """Percentage of items successfully generated.
+
+        Returns:
+            Success rate from 0.0 to 1.0
+        """
+        if not self.generated_items:
+            return 0.0
+        return self.success_count / len(self.generated_items)
+
+
+@dataclass
+class ProgressiveWorkflowResult:
+    """Complete results from a progressive workflow execution.
+
+    Captures the full progression history across all tiers, including
+    costs, quality metrics, and escalation decisions.
+
+    Attributes:
+        workflow_name: Name of the workflow (e.g., "test-gen")
+        task_id: Unique identifier for this execution
+        tier_results: Chronological list of tier execution results
+        final_result: The last tier result (may be successful or failed)
+        total_cost: Total cost in USD across all tiers
+        total_duration: Total execution time in seconds
+        success: Whether the workflow completed successfully
+
+    Example:
+        >>> result = ProgressiveWorkflowResult(
+        ...     workflow_name="test-gen",
+        ...     task_id="test-gen-20260117-143022",
+        ...     tier_results=[cheap_result, capable_result],
+        ...     final_result=capable_result,
+        ...     total_cost=0.75,
+        ...     total_duration=45.2,
+        ...     success=True
+        ... )
+        >>> print(result.generate_report())
+        🎯 PROGRESSIVE ESCALATION REPORT
+        ...
+    """
+
+    workflow_name: str
+    task_id: str
+    tier_results: list[TierResult]
+
+    final_result: TierResult
+    total_cost: float
+    total_duration: float
+    success: bool
+
+    def generate_report(self) -> str:
+        """Generate human-readable progression report.
+
+        Creates a detailed report showing:
+        - Tier-by-tier breakdown
+        - Quality scores and success rates
+        - Cost analysis and savings
+        - Escalation decisions
+
+        Returns:
+            Formatted report string
+        """
+        # Implementation will be in reports.py module
+        from attune.workflows.progressive.reports import generate_progression_report
+
+        return generate_progression_report(self)
+
+    def save_to_disk(self, storage_path: str) -> None:
+        """Save detailed results to disk.
+
+        Creates a directory with:
+        - summary.json: High-level metrics
+        - tier_N_<tier_name>.json: Detailed tier results
+        - report.txt: Human-readable report
+
+        Args:
+            storage_path: Base path for saving results
+        """
+        from attune.workflows.progressive.reports import save_results_to_disk
+
+        save_results_to_disk(self, storage_path)
+
+    @property
+    def cost_savings(self) -> float:
+        """Calculate cost savings vs running all items at Premium tier.
+
+        Returns:
+            Dollar amount saved by using progressive escalation
+        """
+        # Estimate what it would cost if all items were Premium
+        total_items = sum(len(r.generated_items) for r in self.tier_results)
+
+        # Assume Premium costs ~$0.05 per item (conservative estimate)
+        all_premium_cost = total_items * 0.05
+
+        savings = all_premium_cost - self.total_cost
+        return max(savings, 0.0)
+
+    @property
+    def cost_savings_percent(self) -> float:
+        """Calculate percentage of cost saved.
+
+        Returns:
+            Savings percentage (0-100)
+        """
+        total_items = sum(len(r.generated_items) for r in self.tier_results)
+        all_premium_cost = total_items * 0.05
+
+        if all_premium_cost == 0:
+            return 0.0
+
+        return (self.cost_savings / all_premium_cost) * 100
+
+
+@dataclass
+class EscalationConfig:
+    """Configuration for progressive tier escalation.
+
+    Controls all aspects of the escalation system including retry logic,
+    thresholds, cost management, and storage.
+
+    Attributes:
+        enabled: Whether progressive escalation is active
+        tiers: Ordered list of tiers to use (default: all three)
+
+        Retry configuration:
+            cheap_min_attempts: Minimum attempts at cheap tier
+            cheap_max_attempts: Maximum attempts at cheap tier
+            capable_min_attempts: Minimum attempts at capable tier
+            capable_max_attempts: Maximum attempts at capable tier
+            premium_max_attempts: Maximum attempts at premium tier
+
+        Thresholds (Cheap → Capable):
+            cheap_to_capable_failure_rate: Max failure rate before escalation
+            cheap_to_capable_min_cqs: Min quality score to avoid escalation
+            cheap_to_capable_max_syntax_errors: Max syntax errors allowed
+
+        Thresholds (Capable → Premium):
+            capable_to_premium_failure_rate: Max failure rate before escalation
+            capable_to_premium_min_cqs: Min quality score to avoid escalation
+            capable_to_premium_max_syntax_errors: Max syntax errors allowed
+
+        Stagnation detection:
+            improvement_threshold: Min CQS improvement to avoid stagnation (%)
+            consecutive_stagnation_limit: Consecutive stagnations before escalation
+
+        Cost management:
+            max_cost: Maximum total cost in USD
+            auto_approve_under: Auto-approve escalations under this cost
+            warn_on_budget_exceeded: Print warning if budget exceeded
+            abort_on_budget_exceeded: Abort execution if budget exceeded
+
+        Storage:
+            save_tier_results: Whether to save tier results to disk
+            storage_path: Directory for saving results
+
+    Example:
+        >>> config = EscalationConfig(
+        ...     enabled=True,
+        ...     max_cost=10.00,
+        ...     auto_approve_under=5.00,
+        ...     cheap_min_attempts=2,
+        ...     capable_max_attempts=6
+        ... )
+    """
+
+    # Global settings
+    enabled: bool = False
+    tiers: list[Tier] = field(default_factory=lambda: [Tier.CHEAP, Tier.CAPABLE, Tier.PREMIUM])
+
+    # Retry configuration
+    cheap_min_attempts: int = 2
+    cheap_max_attempts: int = 3
+    capable_min_attempts: int = 2
+    capable_max_attempts: int = 6
+    premium_max_attempts: int = 1
+
+    # Thresholds: Cheap → Capable
+    cheap_to_capable_failure_rate: float = 0.30
+    cheap_to_capable_min_cqs: float = 70.0
+    cheap_to_capable_max_syntax_errors: int = 3
+
+    # Thresholds: Capable → Premium
+    capable_to_premium_failure_rate: float = 0.20
+    capable_to_premium_min_cqs: float = 80.0
+    capable_to_premium_max_syntax_errors: int = 1
+
+    # Stagnation detection
+    improvement_threshold: float = 5.0  # 5% CQS improvement required
+    consecutive_stagnation_limit: int = 2
+
+    # Cost management
+    max_cost: float = 5.00
+    auto_approve_under: float | None = None
+    warn_on_budget_exceeded: bool = True
+    abort_on_budget_exceeded: bool = False
+
+    # Storage
+    save_tier_results: bool = True
+    storage_path: str = ".attune/progressive_runs"
+
+    def get_max_attempts(self, tier: Tier) -> int:
+        """Get maximum attempts for a specific tier.
+
+        Args:
+            tier: The tier to query
+
+        Returns:
+            Maximum number of attempts allowed
+        """
+        if tier == Tier.CHEAP:
+            return self.cheap_max_attempts
+        elif tier == Tier.CAPABLE:
+            return self.capable_max_attempts
+        else:  # PREMIUM
+            return self.premium_max_attempts
+
+    def get_min_attempts(self, tier: Tier) -> int:
+        """Get minimum attempts for a specific tier.
+
+        Args:
+            tier: The tier to query
+
+        Returns:
+            Minimum number of attempts required
+        """
+        if tier == Tier.CHEAP:
+            return self.cheap_min_attempts
+        elif tier == Tier.CAPABLE:
+            return self.capable_min_attempts
+        else:  # PREMIUM
+            return 1  # Premium always gets exactly 1 attempt