mcal-ai 0.1.0__py3-none-any.whl

mcal/mcal.py

@@ -0,0 +1,1697 @@
+"""
+MCAL: Memory-Context Alignment Layer
+
+Main interface providing a standalone memory architecture for AI agents.
+
+Architecture:
+    ┌─────────────────────────────────────────────┐
+    │                   MCAL                       │
+    │  ┌─────────────────────────────────────┐    │
+    │  │  UnifiedExtractor (Single-pass)     │    │
+    │  │  - Entities, Goals, Decisions       │    │
+    │  │  - Relationships, Embeddings        │    │
+    │  └─────────────────────────────────────┘    │
+    │                    │                         │
+    │                    ▼                         │
+    │  ┌─────────────────────────────────────┐    │
+    │  │      UnifiedGraph + VectorIndex     │    │
+    │  │  (Graph storage + semantic search)  │    │
+    │  └─────────────────────────────────────┘    │
+    └─────────────────────────────────────────────┘
+
+Usage:
+    from mcal import MCAL
+    
+    # Initialize (no external dependencies!)
+    mcal = MCAL(
+        openai_api_key="..."  # or anthropic_api_key, or use bedrock
+    )
+    
+    # Add conversation (extracts facts + intents + reasoning)
+    result = await mcal.add(
+        messages=[
+            {"role": "user", "content": "Let's build a fraud detection system"},
+            {"role": "assistant", "content": "Great! What's your data source?"}
+        ],
+        user_id="user_123"
+    )
+    
+    # Search with goal-awareness
+    context = await mcal.search(
+        query="What database should we use?",
+        user_id="user_123"
+    )
+    
+    # Get assembled context for LLM
+    prompt_context = mcal.get_context(
+        query="What's our next step?",
+        user_id="user_123",
+        max_tokens=4000
+    )
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any, Optional, AsyncIterator
+from dataclasses import dataclass, field
+
+from anthropic import Anthropic
+from openai import OpenAI, RateLimitError as OpenAIRateLimitError, APIError as OpenAIAPIError
+
+from pathlib import Path
+
+from .backends import MemoryBackend, MemoryEntry  # MemoryEntry still needed for compatibility
+from .core.models import (
+    IntentGraph,
+    IntentNode,
+    IntentType,
+    IntentStatus,
+    DecisionTrail,
+    Turn,
+    Session,
+    RetrievalConfig,
+)
+from .core.intent_tracker import IntentTracker
+from .core.reasoning_store import ReasoningStore
+from .core.goal_retriever import ContextAssembler
+from .core.storage import MCALStorage
+from .core.extraction_cache import ExtractionCache, CacheStats
+from .core.unified_extractor import (
+    UnifiedExtractor,
+    UnifiedGraph,
+    NodeType,
+    EdgeType,
+    graph_to_memories,
+    memories_to_context_string,
+)
+from .core.retry import (
+    llm_retry,
+    classify_http_error,
+    LLMRateLimitError,
+    LLMServerError,
+)
+from .providers.bedrock import BedrockProvider
+from .core.streaming import (
+    StreamEvent,
+    StreamEventType,
+    ExtractionPhase,
+    StreamProgress,
+    event_started,
+    event_phase_started,
+    event_phase_complete,
+    event_fact_extracted,
+    event_intent_extracted,
+    event_decision_extracted,
+    event_cache_hit,
+    event_error,
+    event_complete,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# LLM Client Wrapper
+# =============================================================================
+
+class AnthropicClient:
+    """Wrapper around Anthropic client for MCAL components."""
+    
+    def __init__(self, api_key: str, model: str = "claude-sonnet-4-20250514"):
+        self.client = Anthropic(api_key=api_key)
+        self.model = model
+    
+    @llm_retry(max_attempts=3, min_wait=1.0, max_wait=10.0)
+    async def complete(self, prompt: str, system: Optional[str] = None) -> str:
+        """
+        Generate completion with automatic retry on transient failures.
+        
+        Retries on rate limits and server errors with exponential backoff.
+        """
+        messages = [{"role": "user", "content": prompt}]
+        
+        try:
+            response = self.client.messages.create(
+                model=self.model,
+                max_tokens=4096,
+                system=system or "You are a helpful assistant.",
+                messages=messages
+            )
+            return response.content[0].text
+        except Exception as e:
+            # Anthropic SDK raises specific exceptions we can classify
+            error_str = str(e).lower()
+            if 'rate' in error_str or '429' in error_str:
+                raise LLMRateLimitError(f"Anthropic rate limit: {e}")
+            elif '500' in error_str or '502' in error_str or '503' in error_str or '504' in error_str:
+                raise LLMServerError(f"Anthropic server error: {e}")
+            raise
+
+
+class OpenAIClient:
+    """Wrapper around OpenAI client for MCAL components."""
+    
+    def __init__(self, api_key: str, model: str = "gpt-4o"):
+        self.client = OpenAI(api_key=api_key)
+        self.model = model
+        # Token tracking
+        self.total_prompt_tokens = 0
+        self.total_completion_tokens = 0
+    
+    @llm_retry(max_attempts=3, min_wait=1.0, max_wait=10.0)
+    async def complete(self, prompt: str, system: Optional[str] = None) -> str:
+        """
+        Generate completion with automatic retry on transient failures.
+        
+        Retries on rate limits (429) and server errors (5xx) with exponential backoff.
+        """
+        messages = [{"role": "user", "content": prompt}]
+        if system:
+            messages.insert(0, {"role": "system", "content": system})
+        
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                max_tokens=4096,
+                messages=messages
+            )
+            
+            # Track token usage
+            if response.usage:
+                self.total_prompt_tokens += response.usage.prompt_tokens
+                self.total_completion_tokens += response.usage.completion_tokens
+            
+            return response.choices[0].message.content
+        except OpenAIRateLimitError as e:
+            raise LLMRateLimitError(f"OpenAI rate limit: {e}")
+        except OpenAIAPIError as e:
+            # Check for server errors
+            if hasattr(e, 'status_code') and e.status_code in (500, 502, 503, 504):
+                raise LLMServerError(f"OpenAI server error ({e.status_code}): {e}")
+            raise
+    
+    def get_token_usage(self) -> dict:
+        """Get cumulative token usage."""
+        return {
+            "prompt_tokens": self.total_prompt_tokens,
+            "completion_tokens": self.total_completion_tokens,
+            "total_tokens": self.total_prompt_tokens + self.total_completion_tokens
+        }
+    
+    def reset_token_usage(self):
+        """Reset token counters."""
+        self.total_prompt_tokens = 0
+        self.total_completion_tokens = 0
+
+class BedrockProviderWrapper:
+    """Wrapper around Bedrock provider for MCAL components."""
+    
+    def __init__(self, model: str = "llama-3.3-70b", region: str = "us-east-1"):
+        self.provider = BedrockProvider(model=model, region=region)
+        self.model = model
+    
+    async def complete(self, prompt: str, system: Optional[str] = None) -> str:
+        """Generate completion."""
+        messages = []
+        
+        # Bedrock Converse API doesn't support system role - merge with user message
+        if system:
+            combined_prompt = f"{system}\n\n{prompt}"
+            messages = [{"role": "user", "content": combined_prompt}]
+        else:
+            messages = [{"role": "user", "content": prompt}]
+        
+        return await self.provider.generate(messages)
+
+
+class TieredBedrockProvider:
+    """
+    Tiered Bedrock provider that routes to fast/smart models based on task complexity.
+    
+    Fast model (8B): Simple tasks like intent detection, graph updates
+    Smart model (70B): Complex tasks like decision extraction, reasoning
+    """
+    
+    def __init__(
+        self, 
+        fast_model: str = "llama-3.1-8b",
+        smart_model: str = "llama-3.3-70b",
+        region: str = "us-east-1"
+    ):
+        self.fast_provider = BedrockProvider(model=fast_model, region=region)
+        self.smart_provider = BedrockProvider(model=smart_model, region=region)
+        self.fast_model = fast_model
+        self.smart_model = smart_model
+        
+        # Track usage for metrics
+        self.fast_calls = 0
+        self.smart_calls = 0
+    
+    async def complete(
+        self, 
+        prompt: str, 
+        system: Optional[str] = None,
+        use_smart: bool = False
+    ) -> str:
+        """
+        Generate completion using appropriate model tier.
+        
+        Args:
+            prompt: The prompt to send
+            system: System prompt (merged with user prompt)
+            use_smart: Force use of smart model (for complex tasks)
+        """
+        # Build messages
+        if system:
+            combined_prompt = f"{system}\n\n{prompt}"
+            messages = [{"role": "user", "content": combined_prompt}]
+        else:
+            messages = [{"role": "user", "content": prompt}]
+        
+        # Route to appropriate model
+        if use_smart:
+            self.smart_calls += 1
+            logger.debug(f"Using SMART model ({self.smart_model})")
+            return await self.smart_provider.generate(messages)
+        else:
+            self.fast_calls += 1
+            logger.debug(f"Using FAST model ({self.fast_model})")
+            return await self.fast_provider.generate(messages)
+    
+    def get_usage_stats(self) -> dict:
+        """Get model usage statistics."""
+        total = self.fast_calls + self.smart_calls
+        return {
+            "fast_calls": self.fast_calls,
+            "smart_calls": self.smart_calls,
+            "total_calls": total,
+            "fast_ratio": self.fast_calls / total if total > 0 else 0,
+        }
+    
+    def get_token_usage(self) -> dict:
+        """Get total token usage from both providers."""
+        fast_usage = self.fast_provider.get_token_usage()
+        smart_usage = self.smart_provider.get_token_usage()
+        return {
+            "prompt_tokens": fast_usage["prompt_tokens"] + smart_usage["prompt_tokens"],
+            "completion_tokens": fast_usage["completion_tokens"] + smart_usage["completion_tokens"],
+            "total_tokens": fast_usage["total_tokens"] + smart_usage["total_tokens"],
+            "fast_tokens": fast_usage["total_tokens"],
+            "smart_tokens": smart_usage["total_tokens"],
+        }
+    
+    def reset_token_usage(self):
+        """Reset token counters on both providers."""
+        self.fast_provider.reset_token_usage()
+        self.smart_provider.reset_token_usage()
+
+
+class TieredLLMAdapter:
+    """
+    Adapter that wraps TieredBedrockProvider to fix use_smart for a specific task type.
+    
+    Used to pass different "views" of the tiered provider to different components:
+    - IntentTracker gets a "fast" view (use_smart=False)
+    - ReasoningStore gets a "smart" view (use_smart=True)
+    """
+    
+    def __init__(self, provider: TieredBedrockProvider, use_smart: bool):
+        self._provider = provider
+        self._use_smart = use_smart
+    
+    async def complete(self, prompt: str, system: Optional[str] = None) -> str:
+        """Generate completion using the fixed model tier."""
+        return await self._provider.complete(prompt, system, use_smart=self._use_smart)
+
+
+
+# =============================================================================
+# MCAL Result Types
+# =============================================================================
+
+@dataclass
+class TimingMetrics:
+    """Timing metrics for performance monitoring."""
+    total_ms: int = 0
+    facts_ms: int = 0
+    intents_ms: int = 0
+    decisions_ms: int = 0
+    parallel_savings_ms: int = 0  # Time saved by parallelization
+
+
+@dataclass
+class AddResult:
+    """Result from adding messages to MCAL."""
+    # From Mem0
+    facts: list[MemoryEntry] = field(default_factory=list)
+    
+    # From MCAL (novel)
+    intent_graph: Optional[IntentGraph] = None
+    decisions: list[DecisionTrail] = field(default_factory=list)
+    
+    # From Unified Extractor (Issue #19)
+    unified_graph: Optional[UnifiedGraph] = None
+    
+    # Performance metrics
+    timing: Optional[TimingMetrics] = None
+    
+    @property
+    def summary(self) -> dict:
+        """Quick summary of what was extracted."""
+        summary = {
+            "facts_count": len(self.facts),
+            "intents_count": len(self.intent_graph.nodes) if self.intent_graph else 0,
+            "decisions_count": len(self.decisions),
+        }
+        # Add unified graph stats if present
+        if self.unified_graph:
+            graph_summary = self.unified_graph.summary()
+            summary["unified_graph"] = {
+                "nodes": graph_summary["total_nodes"],
+                "edges": graph_summary["total_edges"],
+                "decisions": graph_summary["decisions"],
+                "goals": graph_summary["goals"],
+                "actions": graph_summary["actions"],
+            }
+        if self.timing:
+            summary["timing_ms"] = {
+                "total": self.timing.total_ms,
+                "facts": self.timing.facts_ms,
+                "intents": self.timing.intents_ms,
+                "decisions": self.timing.decisions_ms,
+                "parallel_savings": self.timing.parallel_savings_ms,
+            }
+        return summary
+
+
+@dataclass 
+class SearchResult:
+    """Result from searching MCAL."""
+    # From Mem0 (re-ranked)
+    memories: list[MemoryEntry] = field(default_factory=list)
+    
+    # From MCAL (enriched)
+    active_goals: list[IntentNode] = field(default_factory=list)
+    relevant_decisions: list[DecisionTrail] = field(default_factory=list)
+    
+    # Assembled context
+    context: Optional[str] = None
+
+# =============================================================================
+# Main MCAL Class
+# =============================================================================
+
+class MCAL:
+    """
+    Memory-Context Alignment Layer.
+    
+    Adds reasoning preservation and goal-aware retrieval on top of Mem0.
+    
+    Three Pillars:
+    1. Intent Graph Preservation - Track goal hierarchies
+    2. Reasoning Chain Storage - Store WHY decisions were made
+    3. Goal-Aware Retrieval - Retrieve based on objectives, not just similarity
+    """
+    
+    def __init__(
+        self,
+        anthropic_api_key: Optional[str] = None,
+        openai_api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        llm_provider: str = "openai",
+        bedrock_model: str = "llama-3.3-70b",
+        bedrock_region: str = "us-east-1",
+        storage_path: Optional[str] = None,
+        enable_persistence: bool = True,
+        enable_tiered_models: bool = False,
+        bedrock_fast_model: str = "llama-3.1-8b",
+        bedrock_smart_model: str = "llama-3.3-70b",
+        enable_extraction_cache: bool = False,
+        cache_ttl_seconds: int = 86400,
+        # Deprecated parameters (Issue #53 - kept for backward compatibility)
+        mem0_config: Optional[dict] = None,
+        mem0_api_key: Optional[str] = None,
+        use_standalone_backend: bool = False,
+        use_unified_extractor: bool = True,
+    ):
+        """
+        Initialize MCAL.
+        
+        Args:
+            anthropic_api_key: Anthropic API key for LLM calls (optional)
+            openai_api_key: OpenAI API key for LLM calls (optional)
+            model: Model to use for extraction (defaults based on provider)
+            llm_provider: LLM provider to use ("openai", "anthropic", or "bedrock")
+            bedrock_model: Bedrock model to use (default: llama-3.3-70b)
+            bedrock_region: AWS region for Bedrock (default: us-east-1)
+            storage_path: Path for persistent storage (default: ~/.mcal)
+            enable_persistence: Enable cross-session persistence (default: True)
+            enable_tiered_models: Use fast/smart model routing (bedrock only)
+            bedrock_fast_model: Fast model for simple tasks (default: llama-3.1-8b)
+            bedrock_smart_model: Smart model for complex tasks (default: llama-3.3-70b)
+            enable_extraction_cache: Enable caching of extracted state (Issue #9)
+            cache_ttl_seconds: Time-to-live for cache entries (default: 24h)
+            
+        Deprecated Args (Issue #53 - will be removed in v1.0):
+            mem0_config: Ignored - MCAL is now standalone
+            mem0_api_key: Ignored - MCAL is now standalone
+            use_standalone_backend: Ignored - always standalone
+            use_unified_extractor: Ignored - always uses unified extractor
+        """
+        self._enable_tiered_models = enable_tiered_models
+        self._enable_extraction_cache = enable_extraction_cache
+        self._use_unified_extractor = True  # Always use unified extractor (Issue #53)
+        
+        # Issue #53: Deprecation warnings for removed parameters
+        import warnings
+        if mem0_config is not None:
+            warnings.warn(
+                "mem0_config is deprecated and ignored. MCAL v1.0 is fully standalone.",
+                DeprecationWarning,
+                stacklevel=2
+            )
+        if mem0_api_key is not None:
+            warnings.warn(
+                "mem0_api_key is deprecated and ignored. MCAL v1.0 is fully standalone.",
+                DeprecationWarning,
+                stacklevel=2
+            )
+        if use_standalone_backend:
+            warnings.warn(
+                "use_standalone_backend is deprecated and ignored. MCAL is always standalone.",
+                DeprecationWarning,
+                stacklevel=2
+            )
+        if not use_unified_extractor:
+            warnings.warn(
+                "use_unified_extractor=False is deprecated. Legacy 3-pillar extraction "
+                "has been removed. MCAL now always uses unified extraction.",
+                DeprecationWarning,
+                stacklevel=2
+            )
+        
+        # Initialize extraction cache (Issue #9)
+        if enable_extraction_cache:
+            cache_path = None
+            if storage_path:
+                cache_path = Path(storage_path) / "extraction_cache.json"
+            elif enable_persistence:
+                cache_path = Path.home() / ".mcal" / "extraction_cache.json"
+            self._extraction_cache = ExtractionCache(
+                persist_path=cache_path,
+                ttl_seconds=cache_ttl_seconds
+            )
+            logger.info(f"Extraction cache enabled (TTL: {cache_ttl_seconds}s)")
+        else:
+            self._extraction_cache = None
+        
+        # Initialize LLM client based on provider
+        if llm_provider == "openai":
+            if not openai_api_key:
+                raise ValueError("openai_api_key required when llm_provider='openai'")
+            model = model or "gpt-4o"
+            self._llm = OpenAIClient(api_key=openai_api_key, model=model)
+        elif llm_provider == "anthropic":
+            if not anthropic_api_key:
+                raise ValueError("anthropic_api_key required when llm_provider='anthropic'")
+            model = model or "claude-sonnet-4-20250514"
+            self._llm = AnthropicClient(api_key=anthropic_api_key, model=model)
+        elif llm_provider == "bedrock":
+            # Bedrock uses AWS credentials from environment
+            if enable_tiered_models:
+                self._llm = TieredBedrockProvider(
+                    fast_model=bedrock_fast_model,
+                    smart_model=bedrock_smart_model,
+                    region=bedrock_region
+                )
+                logger.info(f"Using tiered Bedrock: fast={bedrock_fast_model}, smart={bedrock_smart_model}")
+            else:
+                self._llm = BedrockProviderWrapper(
+                    model=bedrock_model,
+                    region=bedrock_region
+                )
+        else:
+            raise ValueError(f"Unknown llm_provider: {llm_provider}. Use 'openai', 'anthropic', or 'bedrock'")
+        
+        # Issue #53: No memory backend needed - MCAL is standalone
+        # UnifiedGraph handles all storage and search
+        
+        # Initialize MCAL components (for legacy 3-pillar mode compatibility)
+        # With tiered models, use fast LLM for intents, smart LLM for decisions
+        if enable_tiered_models and isinstance(self._llm, TieredBedrockProvider):
+            intent_llm = TieredLLMAdapter(self._llm, use_smart=False)  # Fast model for intents
+            decision_llm = TieredLLMAdapter(self._llm, use_smart=True)  # Smart model for decisions
+            self._intent_tracker = IntentTracker(intent_llm)
+            self._reasoning_store = ReasoningStore(decision_llm)
+            logger.info("Tiered model routing: intents→fast, decisions→smart")
+        else:
+            self._intent_tracker = IntentTracker(self._llm)
+            self._reasoning_store = ReasoningStore(self._llm)
+        self._context_assembler = ContextAssembler()
+        
+        # Initialize unified extractor (Issue #19 - 52x token reduction)
+        if use_unified_extractor:
+            self._unified_extractor = UnifiedExtractor(self._llm)
+            self._user_graphs: dict[str, UnifiedGraph] = {}  # Per-user unified graphs
+            logger.info("Unified extractor enabled (single-pass extraction)")
+        else:
+            self._unified_extractor = None
+            self._user_graphs = {}
+        
+        # Initialize persistent storage
+        self._enable_persistence = enable_persistence
+        if enable_persistence:
+            storage_base = Path(storage_path) if storage_path else None
+            self._storage = MCALStorage(base_path=storage_base)
+            logger.info(f"Persistence enabled at {self._storage.base_path}")
+        else:
+            self._storage = None
+            logger.info("Persistence disabled (in-memory only)")
+        
+        # Session management (in-memory cache)
+        self._sessions: dict[str, Session] = {}
+        self._user_intents: dict[str, IntentGraph] = {}
+    
+    # Issue #53: _extract_facts_async removed - no longer needed with unified extractor
+    
+    async def _extract_intents_async(
+        self, 
+        turns: list[Turn], 
+        user_id: str, 
+        session_id: Optional[str]
+    ) -> IntentGraph:
+        """Extract or update intent graph."""
+        # Load existing graph from persistent storage if not in memory
+        if user_id not in self._user_intents and self._enable_persistence and self._storage:
+            stored_graph = self._storage.load_intent_graph(user_id)
+            if stored_graph:
+                self._user_intents[user_id] = stored_graph
+                logger.info(f"Loaded existing intent graph for user {user_id} from storage")
+        
+        # Get or create user's intent graph
+        if user_id in self._user_intents:
+            # Update existing graph
+            for turn in turns:
+                self._user_intents[user_id] = await self._intent_tracker.update_intent(
+                    turn, self._user_intents[user_id]
+                )
+        else:
+            # Create new graph
+            self._user_intents[user_id] = await self._intent_tracker.extract_intents(
+                turns, session_id=session_id
+            )
+        
+        return self._user_intents[user_id]
+    
+    async def _extract_decisions_async(
+        self,
+        turns: list[Turn],
+        user_id: str,
+        intent_graph: Optional[IntentGraph]
+    ) -> list[DecisionTrail]:
+        """Extract decisions with active goals context."""
+        # Load existing decisions from persistent storage
+        existing_decisions = []
+        if self._enable_persistence and self._storage:
+            existing_decisions = self._storage.load_decisions(user_id)
+            if existing_decisions:
+                logger.info(f"Loaded {len(existing_decisions)} existing decisions for user {user_id}")
+        
+        # Get active goals for context
+        goal_ids = None
+        active_goals_context = None
+        if intent_graph:
+            active_goals = intent_graph.get_active_goals()
+            goal_ids = [g.id for g in active_goals]
+            
+            # Build rich goals context for decision extraction
+            if active_goals:
+                goals_lines = []
+                for i, goal in enumerate(active_goals, 1):
+                    goal_type = goal.type.name if hasattr(goal.type, 'name') else str(goal.type)
+                    goals_lines.append(f"{i}. [{goal_type}] {goal.content}")
+                active_goals_context = "\n".join(goals_lines)
+                logger.debug(f"Active goals context: {len(active_goals)} goals")
+        
+        # Extract new decisions with full context (goals + prior decisions)
+        reconciled_decisions = await self._reasoning_store.extract_decisions(
+            turns, 
+            goal_ids=goal_ids, 
+            existing_decisions=existing_decisions,
+            active_goals_context=active_goals_context
+        )
+        
+        return reconciled_decisions
+
+    async def add(
+        self,
+        messages: list[dict],
+        user_id: str,
+        session_id: Optional[str] = None,
+        extract_intents: bool = True,
+        extract_decisions: bool = True,
+    ) -> AddResult:
+        """
+        Add messages to memory with full MCAL processing.
+        
+        This is the core method that:
+        1. Sends messages to Mem0 for fact extraction (parallel with intents)
+        2. Extracts intent graph (MCAL Pillar 1)
+        3. Extracts decision trails (MCAL Pillar 2)
+        
+        Performance optimization (Issue #7):
+        - Fact extraction and intent extraction run in PARALLEL
+        - Decision extraction runs after intents (needs active goals context)
+        
+        Args:
+            messages: List of message dicts [{"role": "user", "content": "..."}]
+            user_id: User identifier
+            session_id: Optional session identifier
+            extract_intents: Whether to extract intent graph
+            extract_decisions: Whether to extract decision trails
+            
+        Returns:
+            AddResult with facts, intents, decisions, and timing metrics
+        """
+        total_start = time.time()
+        timing = TimingMetrics()
+        result = AddResult()
+        
+        # Issue #2: Validate messages before processing
+        valid_messages = []
+        for i, msg in enumerate(messages):
+            if not isinstance(msg, dict):
+                logger.warning(f"Skipping message {i}: not a dict")
+                continue
+            if not msg.get("content"):
+                logger.warning(f"Skipping message {i}: missing or empty content")
+                continue
+            if not msg.get("role"):
+                logger.warning(f"Skipping message {i}: missing role")
+                continue
+            valid_messages.append(msg)
+        
+        if not valid_messages:
+            logger.error("No valid messages to process")
+            return result
+        
+        # =========================================================================
+        # Issue #19: UNIFIED EXTRACTOR PATH (52x token reduction)
+        # Single-pass extraction replacing 6-pass system
+        # =========================================================================
+        if self._use_unified_extractor and self._unified_extractor:
+            logger.info("Using unified extractor (single-pass)")
+            unified_start = time.time()
+            
+            # Check if user has existing graph (for delta extraction)
+            existing_graph = self._user_graphs.get(user_id)
+            
+            # Issue #25: Load from disk if not in memory
+            if not existing_graph and self._enable_persistence and self._storage:
+                existing_graph = self._storage.load_unified_graph(user_id)
+                if existing_graph:
+                    self._user_graphs[user_id] = existing_graph
+                    logger.info(f"Loaded unified graph for {user_id} from storage")
+            
+            if existing_graph:
+                # Delta extraction - only process new information
+                logger.debug("Using delta extraction for existing user")
+                unified_graph = await self._unified_extractor.extract_delta(
+                    valid_messages, existing_graph
+                )
+            else:
+                # Full extraction for new user
+                unified_graph = await self._unified_extractor.extract(valid_messages)
+            
+            # Store graph for user (in memory)
+            self._user_graphs[user_id] = unified_graph
+            result.unified_graph = unified_graph
+            
+            # Issue #25: Persist to disk
+            if self._enable_persistence and self._storage:
+                self._storage.save_unified_graph(user_id, unified_graph)
+                logger.debug(f"Saved unified graph for {user_id} to storage")
+            
+            # Map to compatible structures for backward compatibility
+            # Convert graph nodes to memory entries (facts)
+            result.facts = [
+                MemoryEntry(
+                    id=node.id,
+                    content=f"{node.type.name}: {node.label}",
+                    metadata={"type": node.type.value, **node.attrs}
+                )
+                for node in unified_graph.nodes.values()
+            ]
+            
+            # Extract decisions count from graph
+            graph_summary = unified_graph.summary()
+            
+            timing.total_ms = int((time.time() - unified_start) * 1000)
+            timing.facts_ms = timing.total_ms  # All in one pass
+            timing.intents_ms = 0
+            timing.decisions_ms = 0
+            result.timing = timing
+            
+            logger.info(
+                f"Unified extraction complete: {graph_summary['total_nodes']} nodes, "
+                f"{graph_summary['total_edges']} edges, {graph_summary['decisions']} decisions "
+                f"in {timing.total_ms}ms"
+            )
+            
+            return result
+        
+        # =========================================================================
+        # Issue #9: CHECK EXTRACTION CACHE
+        # For returning users, skip extraction if messages already processed
+        # =========================================================================
+        cached_state = None
+        messages_to_process = valid_messages
+        cache_hit_type = "miss"
+        
+        if self._enable_extraction_cache and self._extraction_cache:
+            cached_state, messages_to_process = self._extraction_cache.get_state(
+                user_id, valid_messages
+            )
+            
+            if cached_state and not messages_to_process:
+                # FULL CACHE HIT - return cached results immediately
+                cache_hit_type = "full"
+                logger.info(f"CACHE HIT: Returning cached results for user {user_id}")
+                
+                # Restore from cache
+                if cached_state.intent_graph_data:
+                    result.intent_graph = self._storage._deserialize_intent_graph(
+                        cached_state.intent_graph_data
+                    ) if self._storage else None
+                    # Also restore to memory
+                    if result.intent_graph:
+                        self._user_intents[user_id] = result.intent_graph
+                
+                result.decisions = self._deserialize_decisions(cached_state.decisions_data)
+                
+                # Timing reflects cache lookup (near-zero LLM time)
+                timing.total_ms = int((time.time() - total_start) * 1000)
+                timing.facts_ms = 0
+                timing.intents_ms = 0  
+                timing.decisions_ms = 0
+                timing.parallel_savings_ms = cached_state.extraction_time_ms  # Saved this much
+                result.timing = timing
+                
+                logger.info(f"Cache hit saved ~{cached_state.extraction_time_ms}ms of extraction time")
+                return result
+            
+            elif cached_state:
+                # PARTIAL CACHE HIT - process only new messages
+                cache_hit_type = "partial"
+                logger.info(
+                    f"PARTIAL CACHE HIT: Processing {len(messages_to_process)} new messages "
+                    f"(cached {cached_state.messages_processed})"
+                )
+                
+                # Restore cached intent graph to memory for incremental update
+                if cached_state.intent_graph_data and self._storage:
+                    cached_graph = self._storage._deserialize_intent_graph(
+                        cached_state.intent_graph_data
+                    )
+                    if cached_graph:
+                        self._user_intents[user_id] = cached_graph
+        
+        # Convert messages to Turn objects for MCAL processing
+        turns_to_process = [
+            Turn(
+                role=msg.get("role", "user"),
+                content=msg.get("content", ""),
+                session_id=session_id
+            )
+            for msg in messages_to_process
+        ]
+        
+        # All turns (for decision context)
+        all_turns = [
+            Turn(
+                role=msg.get("role", "user"),
+                content=msg.get("content", ""),
+                session_id=session_id
+            )
+            for msg in valid_messages
+        ]
+        
+        # =========================================================================
+        # PHASE 1: PARALLEL EXTRACTION (Facts + Intents)
+        # Issue #7: Run independent extractions in parallel
+        # =========================================================================
+        logger.debug("Phase 1: Parallel extraction (facts + intents)...")
+        phase1_start = time.time()
+        
+        # Prepare tasks
+        tasks = []
+        task_names = []
+        
+        # Always extract facts (for new messages only if partial cache hit)
+        facts_start = time.time()
+        tasks.append(self._extract_facts_async(messages_to_process, user_id))
+        task_names.append("facts")
+        
+        # Optionally extract intents (in parallel with facts)
+        # For partial cache hit, this will incrementally update the cached graph
+        if extract_intents:
+            intents_start = time.time()
+            tasks.append(self._extract_intents_async(turns_to_process, user_id, session_id))
+            task_names.append("intents")
+        
+        # Run in parallel
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        
+        # Process results
+        for i, (name, res) in enumerate(zip(task_names, results)):
+            if isinstance(res, Exception):
+                logger.error(f"Error in {name} extraction: {res}")
+                continue
+            
+            if name == "facts":
+                result.facts = res
+                timing.facts_ms = int((time.time() - facts_start) * 1000)
+                logger.info(f"Mem0 extracted {len(result.facts)} facts in {timing.facts_ms}ms")
+            elif name == "intents":
+                result.intent_graph = res
+                timing.intents_ms = int((time.time() - intents_start) * 1000)
+                logger.info(f"Intent graph has {len(result.intent_graph.nodes)} nodes in {timing.intents_ms}ms")
+                
+                # Persist updated graph to storage
+                if self._enable_persistence and self._storage:
+                    self._storage.save_intent_graph(user_id, result.intent_graph)
+                    logger.debug(f"Saved intent graph for user {user_id} to storage")
+        
+        phase1_duration = int((time.time() - phase1_start) * 1000)
+        
+        # Calculate parallel savings (what sequential would have taken)
+        sequential_time = timing.facts_ms + timing.intents_ms
+        timing.parallel_savings_ms = sequential_time - phase1_duration
+        logger.info(f"Phase 1 complete: {phase1_duration}ms (saved {timing.parallel_savings_ms}ms via parallelization)")
+        
+        # =========================================================================
+        # PHASE 2: SEQUENTIAL EXTRACTION (Decisions - needs intent graph)
+        # =========================================================================
+        if extract_decisions:
+            logger.debug("Phase 2: Decision extraction (requires intent graph)...")
+            decisions_start = time.time()
+            
+            # For partial cache hit, we still need full context for decision extraction
+            # but ReasoningStore handles existing_decisions merging
+            reconciled_decisions = await self._extract_decisions_async(
+                all_turns, user_id, result.intent_graph
+            )
+            
+            result.decisions = reconciled_decisions
+            timing.decisions_ms = int((time.time() - decisions_start) * 1000)
+            
+            if self._enable_persistence and self._storage:
+                self._storage.save_decisions(user_id, result.decisions)
+                logger.debug(f"Saved {len(result.decisions)} total decisions for user {user_id}")
+            
+            logger.info(f"Extracted {len(result.decisions)} decisions in {timing.decisions_ms}ms")
+        
+        # =========================================================================
+        # Issue #9: UPDATE EXTRACTION CACHE
+        # =========================================================================
+        if self._enable_extraction_cache and self._extraction_cache:
+            extraction_time = timing.facts_ms + timing.intents_ms + timing.decisions_ms
+            
+            # Serialize intent graph for cache
+            intent_graph_data = None
+            if result.intent_graph and self._storage:
+                intent_graph_data = self._storage._serialize_intent_graph(result.intent_graph)
+            
+            # Serialize decisions for cache
+            decisions_data = self._serialize_decisions(result.decisions)
+            
+            self._extraction_cache.update_state(
+                user_id=user_id,
+                messages=valid_messages,
+                intent_graph_data=intent_graph_data,
+                decisions_data=decisions_data,
+                extraction_time_ms=extraction_time
+            )
+            
+            logger.info(f"Cache updated for user {user_id} ({cache_hit_type} -> updated)")
+        
+        # Final timing
+        timing.total_ms = int((time.time() - total_start) * 1000)
+        result.timing = timing
+        
+        logger.info(f"Total add() time: {timing.total_ms}ms "
+                   f"(facts={timing.facts_ms}ms, intents={timing.intents_ms}ms, "
+                   f"decisions={timing.decisions_ms}ms, parallel_savings={timing.parallel_savings_ms}ms)")
+        
+        return result
+
+    async def add_stream(
+        self,
+        messages: list[dict],
+        user_id: str,
+        session_id: Optional[str] = None,
+        extract_intents: bool = True,
+        extract_decisions: bool = True,
+    ) -> AsyncIterator[StreamEvent]:
+        """
+        Add messages to memory with streaming progress updates.
+        
+        Issue #10: Streaming Response API
+        
+        This is the streaming version of add() that yields partial results
+        as they become available, improving perceived latency.
+        
+        Instead of waiting 22s for complete results, clients see:
+        - STARTED event immediately
+        - Each intent as it's extracted
+        - Each decision as it's extracted
+        - COMPLETE event with final AddResult
+        
+        Usage:
+            async for event in mcal.add_stream(messages, user_id):
+                if event.type == StreamEventType.INTENT_EXTRACTED:
+                    display_intent(event.data)
+                elif event.type == StreamEventType.COMPLETE:
+                    final_result = event.data
+        
+        Args:
+            messages: List of message dicts [{"role": "user", "content": "..."}]
+            user_id: User identifier
+            session_id: Optional session identifier
+            extract_intents: Whether to extract intent graph
+            extract_decisions: Whether to extract decision trails
+            
+        Yields:
+            StreamEvent objects with type, data, and optional progress
+        """
+        total_start = time.time()
+        timing = TimingMetrics()
+        result = AddResult()
+        
+        # Signal extraction start
+        yield event_started()
+        
+        # Validate messages
+        valid_messages = []
+        for i, msg in enumerate(messages):
+            if not isinstance(msg, dict):
+                yield event_error(f"Skipping message {i}: not a dict")
+                continue
+            if not msg.get("content"):
+                yield event_error(f"Skipping message {i}: missing or empty content")
+                continue
+            if not msg.get("role"):
+                yield event_error(f"Skipping message {i}: missing role")
+                continue
+            valid_messages.append(msg)
+        
+        if not valid_messages:
+            yield event_error("No valid messages to process")
+            yield event_complete(result)
+            return
+        
+        # =========================================================================
+        # CHECK EXTRACTION CACHE
+        # =========================================================================
+        cached_state = None
+        messages_to_process = valid_messages
+        
+        if self._enable_extraction_cache and self._extraction_cache:
+            cached_state, messages_to_process = self._extraction_cache.get_state(
+                user_id, valid_messages
+            )
+            
+            if cached_state and not messages_to_process:
+                # FULL CACHE HIT
+                yield event_cache_hit(
+                    hit_type="full",
+                    messages_cached=cached_state.messages_processed,
+                    messages_to_process=0,
+                    saved_time_ms=cached_state.extraction_time_ms
+                )
+                
+                # Restore from cache
+                if cached_state.intent_graph_data:
+                    result.intent_graph = self._storage._deserialize_intent_graph(
+                        cached_state.intent_graph_data
+                    ) if self._storage else None
+                    if result.intent_graph:
+                        self._user_intents[user_id] = result.intent_graph
+                
+                result.decisions = self._deserialize_decisions(cached_state.decisions_data)
+                
+                timing.total_ms = int((time.time() - total_start) * 1000)
+                timing.parallel_savings_ms = cached_state.extraction_time_ms
+                result.timing = timing
+                
+                yield event_complete(result)
+                return
+            
+            elif cached_state:
+                # PARTIAL CACHE HIT
+                yield event_cache_hit(
+                    hit_type="partial",
+                    messages_cached=cached_state.messages_processed,
+                    messages_to_process=len(messages_to_process),
+                    saved_time_ms=0  # Will calculate after extraction
+                )
+                
+                # Restore cached intent graph
+                if cached_state.intent_graph_data and self._storage:
+                    cached_graph = self._storage._deserialize_intent_graph(
+                        cached_state.intent_graph_data
+                    )
+                    if cached_graph:
+                        self._user_intents[user_id] = cached_graph
+        
+        # Convert to Turn objects
+        turns_to_process = [
+            Turn(
+                role=msg.get("role", "user"),
+                content=msg.get("content", ""),
+                session_id=session_id
+            )
+            for msg in messages_to_process
+        ]
+        
+        all_turns = [
+            Turn(
+                role=msg.get("role", "user"),
+                content=msg.get("content", ""),
+                session_id=session_id
+            )
+            for msg in valid_messages
+        ]
+        
+        # =========================================================================
+        # PHASE 1: FACTS EXTRACTION (with streaming)
+        # =========================================================================
+        yield event_phase_started(ExtractionPhase.FACTS, "Extracting facts via Mem0...")
+        facts_start = time.time()
+        
+        try:
+            result.facts = await self._extract_facts_async(messages_to_process, user_id)
+            timing.facts_ms = int((time.time() - facts_start) * 1000)
+            
+            # Yield each fact as it's extracted
+            for i, fact in enumerate(result.facts):
+                yield event_fact_extracted(
+                    {"id": fact.id, "content": fact.content, "score": fact.score},
+                    progress=StreamProgress(
+                        phase=ExtractionPhase.FACTS,
+                        current=i + 1,
+                        total=len(result.facts),
+                        elapsed_ms=int((time.time() - facts_start) * 1000)
+                    )
+                )
+            
+            yield event_phase_complete(ExtractionPhase.FACTS, len(result.facts), timing.facts_ms)
+        except Exception as e:
+            yield event_error(f"Facts extraction failed: {e}")
+            timing.facts_ms = int((time.time() - facts_start) * 1000)
+        
+        # =========================================================================
+        # PHASE 2: INTENTS EXTRACTION (with streaming)
+        # =========================================================================
+        if extract_intents:
+            yield event_phase_started(ExtractionPhase.INTENTS, "Extracting intent graph...")
+            intents_start = time.time()
+            
+            try:
+                # Track nodes before extraction for delta detection
+                prev_nodes = set(self._user_intents.get(user_id, IntentGraph()).nodes.keys())
+                
+                result.intent_graph = await self._extract_intents_async(
+                    turns_to_process, user_id, session_id
+                )
+                timing.intents_ms = int((time.time() - intents_start) * 1000)
+                
+                # Yield each intent node (new ones first, then all for completeness)
+                new_nodes = [
+                    n for n_id, n in result.intent_graph.nodes.items() 
+                    if n_id not in prev_nodes
+                ]
+                
+                for i, node in enumerate(new_nodes):
+                    yield event_intent_extracted(
+                        node,
+                        progress=StreamProgress(
+                            phase=ExtractionPhase.INTENTS,
+                            current=i + 1,
+                            total=len(new_nodes),
+                            elapsed_ms=int((time.time() - intents_start) * 1000),
+                            message=f"New intent: {node.type.value}"
+                        )
+                    )
+                
+                # Persist
+                if self._enable_persistence and self._storage:
+                    self._storage.save_intent_graph(user_id, result.intent_graph)
+                
+                yield event_phase_complete(
+                    ExtractionPhase.INTENTS, 
+                    len(result.intent_graph.nodes), 
+                    timing.intents_ms
+                )
+            except Exception as e:
+                yield event_error(f"Intent extraction failed: {e}")
+                timing.intents_ms = int((time.time() - intents_start) * 1000)
+        
+        # =========================================================================
+        # PHASE 3: DECISIONS EXTRACTION (with streaming)
+        # =========================================================================
+        if extract_decisions:
+            yield event_phase_started(ExtractionPhase.DECISIONS, "Extracting decisions...")
+            decisions_start = time.time()
+            
+            try:
+                # Load existing for delta detection
+                existing_ids = set()
+                if self._enable_persistence and self._storage:
+                    existing = self._storage.load_decisions(user_id)
+                    existing_ids = {d.id for d in existing}
+                
+                reconciled_decisions = await self._extract_decisions_async(
+                    all_turns, user_id, result.intent_graph
+                )
+                result.decisions = reconciled_decisions
+                timing.decisions_ms = int((time.time() - decisions_start) * 1000)
+                
+                # Yield each decision (highlight new ones)
+                for i, decision in enumerate(result.decisions):
+                    is_new = decision.id not in existing_ids
+                    yield event_decision_extracted(
+                        decision,
+                        progress=StreamProgress(
+                            phase=ExtractionPhase.DECISIONS,
+                            current=i + 1,
+                            total=len(result.decisions),
+                            elapsed_ms=int((time.time() - decisions_start) * 1000),
+                            message="New decision" if is_new else "Existing decision"
+                        )
+                    )
+                
+                # Persist
+                if self._enable_persistence and self._storage:
+                    self._storage.save_decisions(user_id, result.decisions)
+                
+                yield event_phase_complete(
+                    ExtractionPhase.DECISIONS, 
+                    len(result.decisions), 
+                    timing.decisions_ms
+                )
+            except Exception as e:
+                yield event_error(f"Decision extraction failed: {e}")
+                timing.decisions_ms = int((time.time() - decisions_start) * 1000)
+        
+        # =========================================================================
+        # UPDATE CACHE
+        # =========================================================================
+        if self._enable_extraction_cache and self._extraction_cache:
+            extraction_time = timing.facts_ms + timing.intents_ms + timing.decisions_ms
+            
+            intent_graph_data = None
+            if result.intent_graph and self._storage:
+                intent_graph_data = self._storage._serialize_intent_graph(result.intent_graph)
+            
+            decisions_data = self._serialize_decisions(result.decisions)
+            
+            self._extraction_cache.update_state(
+                user_id=user_id,
+                messages=valid_messages,
+                intent_graph_data=intent_graph_data,
+                decisions_data=decisions_data,
+                extraction_time_ms=extraction_time
+            )
+        
+        # Final timing
+        timing.total_ms = int((time.time() - total_start) * 1000)
+        timing.parallel_savings_ms = 0  # Sequential in streaming mode
+        result.timing = timing
+        
+        # Signal completion with full result
+        yield event_complete(result)
+
+    async def search(
+        self,
+        query: str,
+        user_id: str,
+        limit: int = 10,
+        include_goals: bool = True,
+        include_decisions: bool = True,
+        retrieval_config: Optional[RetrievalConfig] = None,
+    ) -> SearchResult:
+        """
+        Search with goal-aware retrieval.
+        
+        This enhances Mem0's search with:
+        1. Goal-aware re-ranking (MCAL Pillar 3)
+        2. Relevant decision attachment
+        3. Active goal context
+        
+        When using unified extractor mode, searches the unified graph
+        directly instead of the Mem0 backend.
+        
+        Args:
+            query: Search query
+            user_id: User identifier
+            limit: Maximum results
+            include_goals: Include active goals in result
+            include_decisions: Include relevant decisions in result
+            retrieval_config: Optional retrieval configuration
+            
+        Returns:
+            SearchResult with memories, goals, decisions, and assembled context
+        """
+        result = SearchResult()
+        
+        # Check if user has a unified graph (Issue #26 fix)
+        unified_graph = self._user_graphs.get(user_id) if self._user_graphs else None
+        
+        if unified_graph:
+            # Search unified graph directly
+            logger.debug("Searching unified graph...")
+            graph_results = unified_graph.search(query, limit=limit * 2)
+            
+            # Convert graph results to MemoryEntry format
+            result.memories = [
+                MemoryEntry(
+                    id=r["id"],
+                    content=r["content"],
+                    metadata={"type": r["type"], **r.get("attributes", {})},
+                    score=r["score"]
+                )
+                for r in graph_results
+            ]
+            
+            # Get active goals from unified graph
+            if include_goals:
+                active_goal_dicts = unified_graph.get_active_goals()
+                # Convert to IntentNode format for compatibility
+                result.active_goals = [
+                    IntentNode(
+                        id=g["id"],
+                        type=self._goal_type_to_intent_type(g.get("goal_type", "goal")),
+                        content=g["content"],
+                        status=self._status_to_intent_status(g.get("status", "active"))
+                    )
+                    for g in active_goal_dicts
+                ]
+            
+            # Get decisions from unified graph
+            if include_decisions:
+                all_decisions = unified_graph.get_all_decisions_with_detail()
+                # Filter to relevant decisions (keyword match - any query word)
+                query_words = set(query.lower().split())
+                
+                def decision_matches(d: dict) -> bool:
+                    """Check if decision matches any query word."""
+                    decision_lower = d["decision"].lower()
+                    # Check if any query word appears in decision
+                    if any(w in decision_lower for w in query_words):
+                        return True
+                    # Check rationale
+                    rationale = d.get("rationale", "") or ""
+                    if any(w in rationale.lower() for w in query_words):
+                        return True
+                    # Check reasons
+                    for r in d.get("reasons", []):
+                        claim = (r.get("claim", "") or "").lower()
+                        if any(w in claim for w in query_words):
+                            return True
+                    return False
+                
+                result.relevant_decisions = [
+                    DecisionTrail(
+                        id=d["id"],
+                        decision=d["decision"],
+                        rationale=d.get("rationale", ""),
+                        alternatives=[],  # Could expand if needed
+                        context=d.get("context", "")
+                    )
+                    for d in all_decisions
+                    if decision_matches(d)
+                ][:5]  # Limit to top 5 relevant decisions
+        else:
+            # Issue #53: No fallback needed - MCAL is always standalone
+            # If no unified graph, return empty results
+            logger.debug("No unified graph found for user")
+            result.memories = []
+            result.active_goals = []
+            result.relevant_decisions = []
+        
+        # Assemble context
+        result.context = self._context_assembler.assemble(
+            retrieved=[],  # We'll pass memories differently
+            active_goals=result.active_goals,
+            decisions=result.relevant_decisions,
+            include_goals=include_goals,
+            include_decisions=include_decisions
+        )
+        
+        # Add memories to context
+        if result.memories:
+            memory_lines = ["\n### RELEVANT MEMORIES ###"]
+            for mem in result.memories:
+                score_str = f" (relevance: {mem.score:.2f})" if mem.score else ""
+                memory_lines.append(f"- {mem.content}{score_str}")
+            result.context += "\n".join(memory_lines)
+        
+        return result
+    
+    async def get_context(
+        self,
+        query: str,
+        user_id: str,
+        max_tokens: int = 4000,
+        include_goals: bool = True,
+        include_decisions: bool = True,
+        include_reasoning: bool = True,
+    ) -> str:
+        """
+        Get assembled context for LLM prompt.
+        
+        Convenience method that searches and formats context.
+        
+        Args:
+            query: The query/task at hand
+            user_id: User identifier
+            max_tokens: Maximum tokens for context
+            include_goals: Include active goals
+            include_decisions: Include relevant decisions
+            include_reasoning: Include decision rationale
+            
+        Returns:
+            Formatted context string ready for LLM prompt
+        """
+        # Search with all enhancements
+        search_result = await self.search(
+            query=query,
+            user_id=user_id,
+            include_goals=include_goals,
+            include_decisions=include_decisions,
+        )
+        
+        # Build context sections
+        sections = []
+        
+        # Active goals
+        if include_goals and search_result.active_goals:
+            goals_text = "### ACTIVE GOALS ###\n"
+            for goal in search_result.active_goals:
+                status = "🔵" if goal.status.value == "active" else "⏳"
+                goals_text += f"{status} {goal.content}\n"
+            sections.append(goals_text)
+        
+        # Relevant decisions with reasoning
+        if include_decisions and search_result.relevant_decisions:
+            decisions_text = "### KEY DECISIONS ###\n"
+            for decision in search_result.relevant_decisions[:5]:
+                decisions_text += f"DECISION: {decision.decision}\n"
+                if include_reasoning:
+                    decisions_text += f"  Rationale: {decision.rationale}\n"
+                    if decision.alternatives:
+                        alts = ", ".join(a.option for a in decision.alternatives[:3])
+                        decisions_text += f"  Alternatives considered: {alts}\n"
+                decisions_text += "\n"
+            sections.append(decisions_text)
+        
+        # Memories
+        if search_result.memories:
+            memories_text = "### RELEVANT CONTEXT ###\n"
+            for mem in search_result.memories[:10]:
+                memories_text += f"- {mem.content}\n"
+            sections.append(memories_text)
+        
+        return "\n".join(sections)
+    
+    def get_active_goals(self, user_id: str) -> list[IntentNode]:
+        """Get active goals for a user."""
+        if user_id in self._user_intents:
+            return self._user_intents[user_id].get_active_goals()
+        return []
+    
+    def get_intent_graph(self, user_id: str) -> Optional[IntentGraph]:
+        """Get full intent graph for a user."""
+        return self._user_intents.get(user_id)
+    
+    def get_decisions(
+        self,
+        user_id: str,
+        goal_id: Optional[str] = None,
+        include_invalidated: bool = False
+    ) -> list[DecisionTrail]:
+        """Get decisions, optionally filtered by goal."""
+        if goal_id:
+            return self._reasoning_store.get_decisions_for_goal(
+                goal_id, include_invalidated=include_invalidated
+            )
+        return self._reasoning_store.get_valid_decisions()
+    
+    async def _rerank_with_goals(
+        self,
+        memories: list[MemoryEntry],
+        query: str,
+        active_goals: list[IntentNode],
+    ) -> list[MemoryEntry]:
+        """
+        Re-rank memories based on goal alignment.
+        
+        This is MCAL Pillar 3: Goal-Aware Retrieval.
+        
+        Instead of pure similarity, we boost memories that
+        help achieve the user's active goals.
+        """
+        if not active_goals or not memories:
+            return memories
+        
+        # For now, simple heuristic-based re-ranking
+        # TODO: Implement LLM-based goal alignment scoring
+        
+        goal_keywords = set()
+        for goal in active_goals:
+            # Extract keywords from goal content
+            words = goal.content.lower().split()
+            goal_keywords.update(w for w in words if len(w) > 3)
+        
+        # Score each memory
+        scored = []
+        for mem in memories:
+            base_score = mem.score or 0.5
+            
+            # Boost if memory content relates to active goals
+            mem_words = set(mem.content.lower().split())
+            overlap = len(goal_keywords & mem_words)
+            goal_boost = min(0.3, overlap * 0.1)  # Max 0.3 boost
+            
+            final_score = base_score + goal_boost
+            mem.score = final_score
+            scored.append(mem)
+        
+        # Sort by final score
+        scored.sort(key=lambda m: m.score or 0, reverse=True)
+        
+        return scored
+    
+    def _get_relevant_decisions(
+        self,
+        query: str,
+        active_goals: list[IntentNode]
+    ) -> list[DecisionTrail]:
+        """Get decisions relevant to the query and active goals."""
+        relevant = []
+        
+        # Get decisions for active goals
+        for goal in active_goals:
+            goal_decisions = self._reasoning_store.get_decisions_for_goal(goal.id)
+            relevant.extend(goal_decisions)
+        
+        # Also check for keyword matches in query
+        query_lower = query.lower()
+        for decision in self._reasoning_store.get_valid_decisions():
+            if decision not in relevant:
+                if any(word in decision.decision.lower() for word in query_lower.split() if len(word) > 3):
+                    relevant.append(decision)
+        
+        # Deduplicate and limit
+        seen = set()
+        unique = []
+        for d in relevant:
+            if d.id not in seen:
+                seen.add(d.id)
+                unique.append(d)
+        
+        return unique[:10]
+    
+    def clear_user_data(self, user_id: str) -> bool:
+        """Clear all data for a user."""
+        # Clear unified graph data
+        if user_id in self._user_graphs:
+            del self._user_graphs[user_id]
+        
+        # Clear from persistent storage
+        if self._enable_persistence and self._storage:
+            self._storage.delete_unified_graph(user_id)
+        
+        # Clear MCAL data
+        if user_id in self._user_intents:
+            del self._user_intents[user_id]
+        
+        if user_id in self._sessions:
+            del self._sessions[user_id]
+        
+        # Clear extraction cache (Issue #9)
+        if self._enable_extraction_cache and self._extraction_cache:
+            self._extraction_cache.invalidate(user_id)
+        
+        return True
+    
+    def get_tiered_model_stats(self) -> Optional[dict]:
+        """
+        Get tiered model usage statistics (Issue #8).
+        
+        Returns:
+            Dict with fast_calls, smart_calls, total, fast_ratio
+            or None if tiered models not enabled
+        """
+        if self._enable_tiered_models and isinstance(self._llm, TieredBedrockProvider):
+            return self._llm.get_usage_stats()
+        return None
+    
+    def get_cache_stats(self) -> Optional[dict]:
+        """
+        Get extraction cache statistics (Issue #9).
+        
+        Returns:
+            Dict with hits, misses, partial_hits, hit_rate, etc.
+            or None if cache not enabled
+        """
+        if self._enable_extraction_cache and self._extraction_cache:
+            return self._extraction_cache.get_stats().to_dict()
+        return None
+    
+    def invalidate_cache(self, user_id: Optional[str] = None) -> int:
+        """
+        Invalidate extraction cache.
+        
+        Args:
+            user_id: Specific user to invalidate, or None for all
+            
+        Returns:
+            Number of entries invalidated
+        """
+        if not self._enable_extraction_cache or not self._extraction_cache:
+            return 0
+        
+        if user_id:
+            return 1 if self._extraction_cache.invalidate(user_id) else 0
+        else:
+            return self._extraction_cache.clear()
+    
+    def _serialize_decisions(self, decisions: list[DecisionTrail]) -> list[dict]:
+        """Serialize decisions for cache storage."""
+        result = []
+        for d in decisions:
+            result.append({
+                "id": d.id,
+                "decision": d.decision,
+                "context": d.context,
+                "rationale": d.rationale,
+                "confidence": d.confidence,
+                "related_goals": d.related_goals,
+                "dependencies": d.dependencies,
+                "invalidated_by": d.invalidated_by,
+                "turn_id": d.turn_id,
+                "alternatives": [
+                    {"option": a.option, "pros": a.pros, "cons": a.cons, "rejection_reason": a.rejection_reason}
+                    for a in (d.alternatives or [])
+                ],
+                "evidence": [
+                    {"claim": e.claim, "source": e.source.value if e.source else "inferred", "turn_id": e.turn_id, "confidence": e.confidence}
+                    for e in (d.evidence or [])
+                ],
+                "trade_offs": [
+                    {"gained": t.gained, "sacrificed": t.sacrificed, "justification": t.justification}
+                    for t in (d.trade_offs or [])
+                ],
+            })
+        return result
+    
+    def _deserialize_decisions(self, data: list[dict]) -> list[DecisionTrail]:
+        """Deserialize decisions from cache storage."""
+        from .core.models import Alternative, Evidence, EvidenceSource, TradeOff
+        
+        result = []
+        for d in data:
+            alternatives = [
+                Alternative(
+                    option=a.get("option", ""),
+                    pros=a.get("pros", []),
+                    cons=a.get("cons", []),
+                    rejection_reason=a.get("rejection_reason")
+                )
+                for a in d.get("alternatives", [])
+            ]
+            
+            evidence = []
+            for e in d.get("evidence", []):
+                try:
+                    source = EvidenceSource(e.get("source", "inferred"))
+                except ValueError:
+                    source = EvidenceSource.INFERRED
+                evidence.append(Evidence(
+                    claim=e.get("claim", ""),
+                    source=source,
+                    turn_id=e.get("turn_id"),
+                    confidence=e.get("confidence", 0.8)
+                ))
+            
+            trade_offs = [
+                TradeOff(
+                    gained=t.get("gained", ""),
+                    sacrificed=t.get("sacrificed", ""),
+                    justification=t.get("justification")
+                )
+                for t in d.get("trade_offs", [])
+            ]
+            
+            result.append(DecisionTrail(
+                id=d.get("id", ""),
+                decision=d.get("decision", ""),
+                context=d.get("context", ""),
+                rationale=d.get("rationale", ""),
+                confidence=d.get("confidence", 0.5),
+                related_goals=d.get("related_goals", []),
+                dependencies=d.get("dependencies", []),
+                invalidated_by=d.get("invalidated_by"),
+                turn_id=d.get("turn_id"),
+                alternatives=alternatives,
+                evidence=evidence,
+                trade_offs=trade_offs,
+            ))
+        
+        return result
+
+    def _goal_type_to_intent_type(self, goal_type: str) -> IntentType:
+        """Convert unified graph goal_type string to IntentType enum."""
+        mapping = {
+            "mission": IntentType.MISSION,
+            "goal": IntentType.GOAL,
+            "task": IntentType.TASK,
+            "decision": IntentType.DECISION,
+        }
+        return mapping.get(goal_type.lower(), IntentType.GOAL)
+    
+    def _status_to_intent_status(self, status: str) -> IntentStatus:
+        """Convert unified graph status string to IntentStatus enum."""
+        mapping = {
+            "active": IntentStatus.ACTIVE,
+            "completed": IntentStatus.COMPLETED,
+            "abandoned": IntentStatus.ABANDONED,
+            "pending": IntentStatus.PENDING,
+            "blocked": IntentStatus.BLOCKED,
+        }
+        return mapping.get(status.lower(), IntentStatus.ACTIVE)