mcal-ai 0.1.0__py3-none-any.whl

mcal/core/retry.py

@@ -0,0 +1,188 @@
+"""
+Retry utilities for LLM API calls (Issue #38).
+
+Provides exponential backoff retry decorators for handling transient
+LLM API failures including rate limits, server errors, and timeouts.
+"""
+
+import logging
+import functools
+from typing import Type, Tuple, Callable, Any
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential_jitter,
+    retry_if_exception_type,
+    before_sleep_log,
+    RetryError,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Retryable Exception Types
+# =============================================================================
+
+class LLMRetryableError(Exception):
+    """Base class for retryable LLM errors."""
+    pass
+
+
+class LLMRateLimitError(LLMRetryableError):
+    """Rate limit exceeded (HTTP 429)."""
+    pass
+
+
+class LLMServerError(LLMRetryableError):
+    """Server-side error (HTTP 5xx)."""
+    pass
+
+
+class LLMTimeoutError(LLMRetryableError):
+    """Connection or request timeout."""
+    pass
+
+
+class LLMThrottlingError(LLMRetryableError):
+    """AWS/Cloud provider throttling."""
+    pass
+
+
+# Default retryable exceptions tuple
+RETRYABLE_EXCEPTIONS: Tuple[Type[Exception], ...] = (
+    LLMRateLimitError,
+    LLMServerError,
+    LLMTimeoutError,
+    LLMThrottlingError,
+)
+
+
+# =============================================================================
+# Retry Decorator Factory
+# =============================================================================
+
+def llm_retry(
+    max_attempts: int = 3,
+    min_wait: float = 1.0,
+    max_wait: float = 10.0,
+    jitter: float = 1.0,
+    retryable_exceptions: Tuple[Type[Exception], ...] = RETRYABLE_EXCEPTIONS,
+):
+    """
+    Create a retry decorator for LLM API calls.
+    
+    Uses exponential backoff with jitter to handle transient failures:
+    - HTTP 429 (Rate Limited)
+    - HTTP 500, 502, 503, 504 (Server Errors)  
+    - Connection timeouts
+    - AWS throttling exceptions
+    
+    Args:
+        max_attempts: Maximum number of retry attempts (default: 3)
+        min_wait: Minimum wait time in seconds (default: 1.0)
+        max_wait: Maximum wait time in seconds (default: 10.0)
+        jitter: Random jitter added to wait time (default: 1.0)
+        retryable_exceptions: Tuple of exception types to retry
+        
+    Returns:
+        Decorator function
+        
+    Example:
+        @llm_retry(max_attempts=3)
+        async def call_llm(prompt: str) -> str:
+            ...
+    """
+    return retry(
+        stop=stop_after_attempt(max_attempts),
+        wait=wait_exponential_jitter(initial=min_wait, max=max_wait, jitter=jitter),
+        retry=retry_if_exception_type(retryable_exceptions),
+        before_sleep=before_sleep_log(logger, logging.WARNING),
+        reraise=True,
+    )
+
+
+def llm_retry_sync(
+    max_attempts: int = 3,
+    min_wait: float = 1.0,
+    max_wait: float = 10.0,
+    jitter: float = 1.0,
+    retryable_exceptions: Tuple[Type[Exception], ...] = RETRYABLE_EXCEPTIONS,
+):
+    """
+    Synchronous version of llm_retry for sync functions.
+    
+    Same parameters as llm_retry().
+    """
+    return retry(
+        stop=stop_after_attempt(max_attempts),
+        wait=wait_exponential_jitter(initial=min_wait, max=max_wait, jitter=jitter),
+        retry=retry_if_exception_type(retryable_exceptions),
+        before_sleep=before_sleep_log(logger, logging.WARNING),
+        reraise=True,
+    )
+
+
+# =============================================================================
+# Error Classification Helpers
+# =============================================================================
+
+def classify_http_error(status_code: int, message: str = "") -> Exception:
+    """
+    Classify an HTTP error into the appropriate exception type.
+    
+    Args:
+        status_code: HTTP status code
+        message: Error message
+        
+    Returns:
+        Appropriate exception instance
+    """
+    if status_code == 429:
+        return LLMRateLimitError(f"Rate limited: {message}")
+    elif status_code in (500, 502, 503, 504):
+        return LLMServerError(f"Server error ({status_code}): {message}")
+    else:
+        # Non-retryable client error (4xx except 429)
+        return RuntimeError(f"HTTP {status_code}: {message}")
+
+
+def classify_boto_error(error_code: str, message: str = "") -> Exception:
+    """
+    Classify a boto3/botocore error into the appropriate exception type.
+    
+    Args:
+        error_code: AWS error code (e.g., 'ThrottlingException')
+        message: Error message
+        
+    Returns:
+        Appropriate exception instance
+    """
+    throttling_codes = {
+        'ThrottlingException',
+        'Throttling',
+        'TooManyRequestsException',
+        'RequestThrottled',
+        'ProvisionedThroughputExceededException',
+        'ServiceUnavailableException',
+        'ModelStreamErrorException',
+    }
+    
+    server_error_codes = {
+        'InternalServerException',
+        'ServiceException',
+        'InternalFailure',
+    }
+    
+    if error_code in throttling_codes:
+        return LLMThrottlingError(f"AWS throttling ({error_code}): {message}")
+    elif error_code in server_error_codes:
+        return LLMServerError(f"AWS server error ({error_code}): {message}")
+    else:
+        # Non-retryable AWS error
+        return RuntimeError(f"AWS error ({error_code}): {message}")
+
+
+def is_retryable_exception(exc: Exception) -> bool:
+    """Check if an exception is retryable."""
+    return isinstance(exc, RETRYABLE_EXCEPTIONS)

mcal/core/storage.py

@@ -0,0 +1,456 @@
+"""
+MCAL Storage Layer
+
+Provides persistence for intent graphs and decision trails across sessions.
+This enables cross-session reasoning preservation - the core value proposition.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Any, List
+from datetime import datetime, timezone
+
+from .models import (
+    IntentGraph,
+    IntentNode,
+    IntentEdge,
+    IntentType,
+    IntentStatus,
+    EdgeRelation,
+    DecisionTrail,
+    Alternative,
+    Evidence,
+    EvidenceSource,
+    TradeOff,
+)
+from .unified_extractor import UnifiedGraph
+
+
+def _utc_now() -> datetime:
+    """Return current UTC time (timezone-aware)."""
+    return datetime.now(timezone.utc)
+
+
+logger = logging.getLogger(__name__)
+
+
+class MCALStorage:
+    """
+    Persistent storage for MCAL data structures.
+    
+    Stores:
+    - Intent graphs per user (with versioning)
+    - Decision trails per user
+    - Session metadata
+    
+    File structure:
+    ~/.mcal/
+      users/
+        {user_id}/
+          intent_graph.json      # Legacy 3-Pillar
+          decisions.json          # Legacy 3-Pillar
+          unified_graph.json      # Unified Deep (Issue #25)
+          sessions.json
+    """
+    
+    def __init__(self, base_path: Optional[Path] = None):
+        """
+        Initialize storage.
+        
+        Args:
+            base_path: Base directory for storage (default: ~/.mcal)
+        """
+        if base_path is None:
+            base_path = Path.home() / ".mcal"
+        
+        self.base_path = Path(base_path)
+        self.users_path = self.base_path / "users"
+        self.users_path.mkdir(parents=True, exist_ok=True)
+        logger.info(f"MCAL storage initialized at {self.base_path}")
+    
+    def _get_user_path(self, user_id: str) -> Path:
+        """Get path for user's data directory."""
+        user_path = self.users_path / user_id
+        user_path.mkdir(parents=True, exist_ok=True)
+        return user_path
+    
+    # =========================================================================
+    # Intent Graph Persistence
+    # =========================================================================
+    
+    def save_intent_graph(self, user_id: str, graph: IntentGraph) -> None:
+        """
+        Save intent graph for user.
+        
+        Args:
+            user_id: User identifier
+            graph: IntentGraph to save
+        """
+        user_path = self._get_user_path(user_id)
+        graph_path = user_path / "intent_graph.json"
+        
+        # Serialize graph
+        data = self._serialize_intent_graph(graph)
+        data["_metadata"] = {
+            "user_id": user_id,
+            "saved_at": _utc_now().isoformat(),
+            "node_count": len(graph.nodes),
+            "edge_count": len(graph.edges)
+        }
+        
+        with open(graph_path, 'w') as f:
+            json.dump(data, f, indent=2)
+        
+        logger.info(f"Saved intent graph for {user_id}: {len(graph.nodes)} nodes, {len(graph.edges)} edges")
+    
+    def load_intent_graph(self, user_id: str) -> Optional[IntentGraph]:
+        """
+        Load intent graph for user.
+        
+        Args:
+            user_id: User identifier
+            
+        Returns:
+            IntentGraph or None if not found
+        """
+        user_path = self._get_user_path(user_id)
+        graph_path = user_path / "intent_graph.json"
+        
+        if not graph_path.exists():
+            logger.debug(f"No existing intent graph for {user_id}")
+            return None
+        
+        try:
+            with open(graph_path, 'r') as f:
+                data = json.load(f)
+            
+            graph = self._deserialize_intent_graph(data)
+            logger.info(f"Loaded intent graph for {user_id}: {len(graph.nodes)} nodes")
+            return graph
+            
+        except Exception as e:
+            logger.error(f"Failed to load intent graph for {user_id}: {e}")
+            return None
+    
+    def _serialize_intent_graph(self, graph: IntentGraph) -> dict:
+        """Serialize IntentGraph to JSON-compatible dict."""
+        return {
+            "session_id": graph.session_id,
+            "nodes": {
+                node_id: {
+                    "id": node.id,
+                    "type": node.type.value,
+                    "content": node.content,
+                    "status": node.status.value,
+                    "confidence": node.confidence,
+                    "evidence": node.evidence,
+                    "created_at": node.created_at.isoformat() if node.created_at else None,
+                    "updated_at": node.updated_at.isoformat() if node.updated_at else None,
+                }
+                for node_id, node in graph.nodes.items()
+            },
+            "edges": [
+                {
+                    "id": edge.id,
+                    "source": edge.source,
+                    "target": edge.target,
+                    "relation": edge.relation.value,
+                }
+                for edge in graph.edges
+            ]
+        }
+    
+    def _deserialize_intent_graph(self, data: dict) -> IntentGraph:
+        """Deserialize JSON dict to IntentGraph."""
+        graph = IntentGraph(session_id=data.get("session_id"))
+        
+        # Reconstruct nodes
+        for node_id, node_data in data.get("nodes", {}).items():
+            node = IntentNode(
+                type=IntentType(node_data["type"]),
+                content=node_data["content"],
+                status=IntentStatus(node_data["status"]),
+                confidence=node_data.get("confidence", 0.8),
+                evidence=node_data.get("evidence", [])
+            )
+            # Override auto-generated ID with stored ID
+            node.id = node_data["id"]
+            graph.nodes[node.id] = node
+        
+        # Reconstruct edges
+        for edge_data in data.get("edges", []):
+            edge = IntentEdge(
+                source=edge_data["source"],
+                target=edge_data["target"],
+                relation=EdgeRelation(edge_data["relation"])
+            )
+            edge.id = edge_data.get("id", edge.id)
+            graph.edges.append(edge)
+        
+        return graph
+    
+    # =========================================================================
+    # Unified Graph Persistence (Issue #25)
+    # =========================================================================
+    
+    def save_unified_graph(self, user_id: str, graph: UnifiedGraph) -> None:
+        """
+        Save unified graph for user.
+        
+        Args:
+            user_id: User identifier
+            graph: UnifiedGraph to save
+        """
+        user_path = self._get_user_path(user_id)
+        graph_path = user_path / "unified_graph.json"
+        
+        # Use existing to_dict() method
+        data = graph.to_dict()
+        data["_metadata"] = {
+            "user_id": user_id,
+            "saved_at": _utc_now().isoformat(),
+            "node_count": len(graph.nodes),
+            "edge_count": len(graph.edges),
+            "version": "unified_deep_v1"
+        }
+        
+        with open(graph_path, 'w') as f:
+            json.dump(data, f, indent=2)
+        
+        logger.info(f"Saved unified graph for {user_id}: {len(graph.nodes)} nodes, {len(graph.edges)} edges")
+    
+    def load_unified_graph(self, user_id: str) -> Optional[UnifiedGraph]:
+        """
+        Load unified graph for user.
+        
+        Args:
+            user_id: User identifier
+            
+        Returns:
+            UnifiedGraph or None if not found
+        """
+        user_path = self._get_user_path(user_id)
+        graph_path = user_path / "unified_graph.json"
+        
+        if not graph_path.exists():
+            logger.debug(f"No existing unified graph for {user_id}")
+            return None
+        
+        try:
+            with open(graph_path, 'r') as f:
+                data = json.load(f)
+            
+            # Use existing from_dict() method
+            graph = UnifiedGraph.from_dict(data)
+            logger.info(f"Loaded unified graph for {user_id}: {len(graph.nodes)} nodes")
+            return graph
+            
+        except Exception as e:
+            logger.error(f"Failed to load unified graph for {user_id}: {e}")
+            return None
+    
+    def delete_unified_graph(self, user_id: str) -> bool:
+        """
+        Delete unified graph for user.
+        
+        Args:
+            user_id: User identifier
+            
+        Returns:
+            True if deleted, False if not found
+        """
+        user_path = self._get_user_path(user_id)
+        graph_path = user_path / "unified_graph.json"
+        
+        if graph_path.exists():
+            graph_path.unlink()
+            logger.info(f"Deleted unified graph for {user_id}")
+            return True
+        return False
+    
+    # =========================================================================
+    # Decision Trail Persistence
+    # =========================================================================
+    
+    def save_decisions(self, user_id: str, decisions: List[DecisionTrail]) -> None:
+        """
+        Save decision trails for user.
+        
+        Args:
+            user_id: User identifier
+            decisions: List of DecisionTrail objects
+        """
+        user_path = self._get_user_path(user_id)
+        decisions_path = user_path / "decisions.json"
+        
+        data = {
+            "decisions": [
+                self._serialize_decision(decision)
+                for decision in decisions
+            ],
+            "_metadata": {
+                "user_id": user_id,
+                "saved_at": _utc_now().isoformat(),
+                "decision_count": len(decisions)
+            }
+        }
+        
+        with open(decisions_path, 'w') as f:
+            json.dump(data, f, indent=2)
+        
+        logger.info(f"Saved {len(decisions)} decisions for {user_id}")
+    
+    def load_decisions(self, user_id: str) -> List[DecisionTrail]:
+        """
+        Load decision trails for user.
+        
+        Args:
+            user_id: User identifier
+            
+        Returns:
+            List of DecisionTrail objects
+        """
+        user_path = self._get_user_path(user_id)
+        decisions_path = user_path / "decisions.json"
+        
+        if not decisions_path.exists():
+            logger.debug(f"No existing decisions for {user_id}")
+            return []
+        
+        try:
+            with open(decisions_path, 'r') as f:
+                data = json.load(f)
+            
+            decisions = [
+                self._deserialize_decision(decision_data)
+                for decision_data in data.get("decisions", [])
+            ]
+            
+            logger.info(f"Loaded {len(decisions)} decisions for {user_id}")
+            return decisions
+            
+        except Exception as e:
+            logger.error(f"Failed to load decisions for {user_id}: {e}")
+            return []
+    
+    def _serialize_decision(self, decision: DecisionTrail) -> dict:
+        """Serialize DecisionTrail to JSON-compatible dict."""
+        return {
+            "id": decision.id,
+            "decision": decision.decision,
+            "context": decision.context,
+            "rationale": decision.rationale,
+            "alternatives": [
+                {
+                    "option": alt.option,
+                    "pros": alt.pros,
+                    "cons": alt.cons,
+                    "rejection_reason": alt.rejection_reason
+                }
+                for alt in decision.alternatives
+            ],
+            "evidence": [
+                {
+                    "claim": ev.claim,
+                    "source": ev.source.value,
+                    "turn_id": ev.turn_id
+                }
+                for ev in decision.evidence
+            ],
+            "trade_offs": [
+                {
+                    "gained": to.gained,
+                    "sacrificed": to.sacrificed,
+                    "justification": to.justification
+                }
+                for to in decision.trade_offs
+            ],
+            "confidence": decision.confidence,
+            "related_goals": decision.related_goals,
+            "dependencies": decision.dependencies,
+            "created_at": decision.created_at.isoformat() if decision.created_at else None,
+            "is_valid": decision.is_valid,
+            "invalidated_by": decision.invalidated_by
+        }
+    
+    def _deserialize_decision(self, data: dict) -> DecisionTrail:
+        """Deserialize JSON dict to DecisionTrail."""
+        # Reconstruct alternatives
+        alternatives = [
+            Alternative(
+                option=alt["option"],
+                pros=alt.get("pros", []),
+                cons=alt.get("cons", []),
+                rejection_reason=alt.get("rejection_reason", "")
+            )
+            for alt in data.get("alternatives", [])
+        ]
+        
+        # Reconstruct evidence
+        evidence = [
+            Evidence(
+                claim=ev["claim"],
+                source=EvidenceSource(ev.get("source", "inferred")),
+                turn_id=ev.get("turn_id")
+            )
+            for ev in data.get("evidence", [])
+        ]
+        
+        # Reconstruct trade-offs
+        trade_offs = [
+            TradeOff(
+                gained=to["gained"],
+                sacrificed=to["sacrificed"],
+                justification=to.get("justification", "")
+            )
+            for to in data.get("trade_offs", [])
+        ]
+        
+        # Create decision with all fields
+        decision = DecisionTrail(
+            id=data["id"],
+            decision=data["decision"],
+            context=data.get("context", ""),
+            rationale=data.get("rationale", ""),
+            alternatives=alternatives,
+            evidence=evidence,
+            trade_offs=trade_offs,
+            confidence=data.get("confidence", 0.8),
+            related_goals=data.get("related_goals", []),
+            dependencies=data.get("dependencies", []),
+            invalidated_by=data.get("invalidated_by")
+        )
+        
+        return decision
+    
+    # =========================================================================
+    # Utility Methods
+    # =========================================================================
+    
+    def clear_user_data(self, user_id: str) -> None:
+        """Clear all stored data for a user."""
+        import shutil
+        user_path = self._get_user_path(user_id)
+        if user_path.exists():
+            shutil.rmtree(user_path)
+            logger.info(f"Cleared all data for {user_id}")
+    
+    def list_users(self) -> list[str]:
+        """List all users with stored data."""
+        if not self.users_path.exists():
+            return []
+        return [d.name for d in self.users_path.iterdir() if d.is_dir()]
+    
+    def get_user_summary(self, user_id: str) -> dict:
+        """Get summary of stored data for a user."""
+        graph = self.load_intent_graph(user_id)
+        decisions = self.load_decisions(user_id)
+        
+        return {
+            "user_id": user_id,
+            "has_intent_graph": graph is not None,
+            "node_count": len(graph.nodes) if graph else 0,
+            "decision_count": len(decisions),
+            "active_goals": len(graph.get_active_goals()) if graph else 0
+        }