mcal-ai 0.1.0__py3-none-any.whl

mcal/core/unified_extractor.py

@@ -0,0 +1,1466 @@
+"""
+Unified Graph Extractor
+
+Single-pass extraction that captures all memory types in one LLM call:
+- Entities (WHO, WHAT)
+- Relationships (CONNECTIONS)
+- Decisions (WHY)
+- Next Actions (NEXT-STEPS)
+
+Design Goals:
+- Reduce from 6 LLM calls to 1-2 calls
+- Target: 50-80x overhead vs 220x current
+- Maintain same memory quality
+
+Token Optimization Strategies:
+1. Single comprehensive extraction prompt
+2. Compact JSON schema (short keys)
+3. Reference-based relationships (use IDs, not full text)
+4. Delta extraction for multi-turn (only process new messages)
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import hashlib
+import logging
+from datetime import datetime, timezone
+from typing import Optional, Protocol
+from dataclasses import dataclass, field
+from enum import Enum
+
+from pydantic import BaseModel, Field
+
+
+def _utc_now() -> datetime:
+    """Return current UTC time (timezone-aware)."""
+    return datetime.now(timezone.utc)
+
+
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Compact Data Models (Optimized for Token Efficiency)
+# =============================================================================
+
+class NodeType(str, Enum):
+    """Node types in the unified graph."""
+    PERSON = "P"      # WHO - people, users, stakeholders
+    THING = "T"       # WHAT - products, tools, technologies
+    CONCEPT = "C"     # WHAT - ideas, approaches, methodologies
+    GOAL = "G"        # WHAT - objectives, targets
+    DECISION = "D"    # WHY - choices made with rationale
+    ACTION = "A"      # NEXT - tasks, next steps
+
+
+class DeduplicationStrategy(str, Enum):
+    """Strategy for detecting duplicate nodes."""
+    EXACT_ONLY = "exact"     # Only exact label matches (case-insensitive)
+    SEMANTIC = "semantic"    # Use embedding similarity only
+    HYBRID = "hybrid"        # Exact first, then semantic (default)
+    NONE = "none"            # No deduplication
+
+
+@dataclass
+class DeduplicationStats:
+    """Track deduplication statistics for debugging."""
+    nodes_added: int = 0
+    nodes_merged: int = 0
+    exact_matches: int = 0
+    semantic_matches: int = 0
+    
+    @property
+    def total_operations(self) -> int:
+        return self.nodes_added + self.nodes_merged
+    
+    @property
+    def dedup_ratio(self) -> float:
+        """Ratio of merged nodes to total operations."""
+        if self.total_operations == 0:
+            return 0.0
+        return self.nodes_merged / self.total_operations
+    
+    def reset(self) -> None:
+        """Reset all counters."""
+        self.nodes_added = 0
+        self.nodes_merged = 0
+        self.exact_matches = 0
+        self.semantic_matches = 0
+
+
+class EdgeType(str, Enum):
+    """Relationship types (compact)."""
+    USES = "uses"           # Person/Goal uses Thing
+    WANTS = "wants"         # Person wants Goal
+    CHOSE = "chose"         # Person/Goal chose Decision
+    OVER = "over"           # Decision over Alternative (with pros/cons attrs)
+    BECAUSE = "because"     # Decision because Reason (with source/confidence attrs)
+    ENABLES = "enables"     # Thing/Decision enables Goal
+    BLOCKS = "blocks"       # Thing blocks Goal
+    NEXT = "next"           # Current state leads to Action
+    RELATES = "relates"     # General relationship
+    PART_OF = "part_of"     # Hierarchical containment
+    # NEW in Unified Deep v2
+    DEPENDS_ON = "depends_on"   # Task/Goal A requires B to complete first
+    CONFLICTS = "conflicts"     # Goal X conflicts with Goal Y
+    SUPERSEDES = "supersedes"   # Decision D2 replaces/updates D1
+
+
+@dataclass
+class GraphNode:
+    """
+    Compact graph node.
+    
+    Memory optimization: Use short field names and IDs.
+    
+    Embedding storage (Issue #50):
+    - Stored as Float16 binary (bytes) for 8x compression
+    - Serialized to base64 in JSON for persistence
+    - Quality preserved: 0% loss in search accuracy
+    """
+    id: str                    # Short ID: "P1", "T2", "D3"
+    type: NodeType
+    label: str                 # Short label: "PostgreSQL", "fraud detection"
+    attrs: dict = field(default_factory=dict)  # Optional attributes
+    embedding: Optional[bytes] = None  # Float16 binary embedding (384 dim = 768 bytes)
+    
+    def to_dict(self) -> dict:
+        d = {"id": self.id, "t": self.type.value, "l": self.label}
+        if self.attrs:
+            d["a"] = self.attrs
+        if self.embedding:
+            # Store as base64-encoded Float16 binary (compact)
+            d["e"] = base64.b64encode(self.embedding).decode('ascii')
+        return d
+    
+    @classmethod
+    def from_dict(cls, d: dict) -> "GraphNode":
+        embedding = None
+        if "e" in d:
+            embedding = base64.b64decode(d["e"])
+        return cls(
+            id=d["id"],
+            type=NodeType(d["t"]),
+            label=d["l"],
+            attrs=d.get("a", {}),
+            embedding=embedding
+        )
+
+
+@dataclass  
+class GraphEdge:
+    """
+    Compact graph edge.
+    """
+    src: str          # Source node ID
+    dst: str          # Destination node ID
+    rel: EdgeType     # Relationship type
+    attrs: dict = field(default_factory=dict)
+    
+    def to_dict(self) -> dict:
+        d = {"s": self.src, "d": self.dst, "r": self.rel.value}
+        if self.attrs:
+            d["a"] = self.attrs
+        return d
+    
+    @classmethod
+    def from_dict(cls, d: dict) -> "GraphEdge":
+        return cls(
+            src=d["s"],
+            dst=d["d"],
+            rel=EdgeType(d["r"]),
+            attrs=d.get("a", {})
+        )
+
+
+@dataclass
+class UnifiedGraph:
+    """
+    Compact unified graph containing all memory types.
+    
+    This single structure replaces:
+    - IntentGraph (goals, tasks)
+    - DecisionTrail (decisions, alternatives, rationale)
+    - Facts (entities, relationships)
+    - NextActions (follow-ups)
+    
+    Deduplication (Issue #52):
+    - Hybrid strategy: exact match first, then semantic similarity
+    - Configurable threshold (default 0.80)
+    - Stats tracking for debugging
+    """
+    nodes: dict[str, GraphNode] = field(default_factory=dict)
+    edges: list[GraphEdge] = field(default_factory=list)
+    metadata: dict = field(default_factory=dict)
+    
+    # Deduplication configuration
+    _dedup_strategy: DeduplicationStrategy = field(default=DeduplicationStrategy.HYBRID)
+    _similarity_threshold: float = field(default=0.80)
+    _dedup_stats: DeduplicationStats = field(default_factory=DeduplicationStats)
+    
+    # Internal indexes (rebuilt on demand)
+    _label_index: dict[str, list[str]] = field(default_factory=dict)  # normalized_label -> [node_ids]
+    _vector_index: "VectorIndex" = field(default=None)  # Lazy-built
+    
+    def __post_init__(self):
+        """Initialize internal indexes."""
+        # Rebuild label index from existing nodes
+        self._rebuild_label_index()
+    
+    def _rebuild_label_index(self) -> None:
+        """Rebuild the label index from current nodes."""
+        self._label_index = {}
+        for node in self.nodes.values():
+            self._index_node_label(node)
+    
+    def _index_node_label(self, node: GraphNode) -> None:
+        """Add node to label index."""
+        normalized = self._normalize_label(node.label)
+        if normalized not in self._label_index:
+            self._label_index[normalized] = []
+        if node.id not in self._label_index[normalized]:
+            self._label_index[normalized].append(node.id)
+    
+    @staticmethod
+    def _normalize_label(label: str) -> str:
+        """Normalize label for exact matching (lowercase, stripped)."""
+        return label.lower().strip()
+    
+    # ==========================================================================
+    # Deduplication Methods (Issue #52)
+    # ==========================================================================
+    
+    def find_duplicate(
+        self, 
+        node: GraphNode, 
+        strategy: Optional[DeduplicationStrategy] = None
+    ) -> Optional[str]:
+        """
+        Find existing duplicate node.
+        
+        Args:
+            node: Node to check for duplicates
+            strategy: Override default strategy (optional)
+        
+        Returns:
+            node_id of existing duplicate, or None
+        """
+        strategy = strategy or self._dedup_strategy
+        
+        if strategy == DeduplicationStrategy.NONE:
+            return None
+        
+        # Phase 1: Exact match (O(1) hash lookup)
+        if strategy in (DeduplicationStrategy.EXACT_ONLY, DeduplicationStrategy.HYBRID):
+            normalized = self._normalize_label(node.label)
+            if normalized in self._label_index:
+                for existing_id in self._label_index[normalized]:
+                    existing = self.nodes.get(existing_id)
+                    if existing and existing.type == node.type:
+                        return existing_id
+        
+        # Phase 2: Semantic match (requires embeddings)
+        if strategy in (DeduplicationStrategy.SEMANTIC, DeduplicationStrategy.HYBRID):
+            if node.embedding and self._vector_index:
+                results = self._vector_index.search(node.embedding, k=3)
+                for existing_id, score in results:
+                    if score >= self._similarity_threshold:
+                        existing = self.nodes.get(existing_id)
+                        if existing and existing.type == node.type:
+                            return existing_id
+        
+        return None
+    
+    def _merge_node(self, existing_id: str, new_node: GraphNode) -> None:
+        """
+        Merge new node data into existing node.
+        
+        - Merges attributes (new values override existing)
+        - Updates embedding if new one is provided
+        - Preserves original ID and type
+        """
+        existing = self.nodes[existing_id]
+        
+        # Merge attributes (new values override)
+        for key, value in new_node.attrs.items():
+            existing.attrs[key] = value
+        
+        # Update embedding if new one is provided
+        if new_node.embedding:
+            existing.embedding = new_node.embedding
+            # Update vector index
+            if self._vector_index:
+                # Note: FAISS doesn't support update, so we accept stale index
+                # It will be rebuilt on next build_vector_index() call
+                pass
+    
+    def add_node(
+        self, 
+        node: GraphNode, 
+        deduplicate: bool = True,
+        generate_embedding: bool = False
+    ) -> str:
+        """
+        Add node with optional deduplication.
+        
+        Args:
+            node: Node to add
+            deduplicate: Whether to check for duplicates (default True)
+            generate_embedding: Whether to generate embedding if missing
+        
+        Returns:
+            Actual node ID used (may be existing duplicate's ID)
+        """
+        # Generate embedding if requested and missing
+        if generate_embedding and node.embedding is None:
+            from .embeddings import EmbeddingService
+            embedder = EmbeddingService()
+            node.embedding = embedder.embed_node(node)
+        
+        # Check for duplicates
+        if deduplicate and self._dedup_strategy != DeduplicationStrategy.NONE:
+            existing_id = self.find_duplicate(node)
+            if existing_id:
+                # Merge into existing node
+                self._merge_node(existing_id, node)
+                self._dedup_stats.nodes_merged += 1
+                # Track match type
+                normalized = self._normalize_label(node.label)
+                if normalized in self._label_index and existing_id in self._label_index.get(normalized, []):
+                    self._dedup_stats.exact_matches += 1
+                else:
+                    self._dedup_stats.semantic_matches += 1
+                return existing_id
+        
+        # Add as new node
+        self.nodes[node.id] = node
+        self._index_node_label(node)
+        self._dedup_stats.nodes_added += 1
+        
+        # Add to vector index if available
+        if node.embedding and self._vector_index:
+            self._vector_index.add(node.id, node.embedding)
+        
+        return node.id
+    
+    @property
+    def dedup_stats(self) -> DeduplicationStats:
+        """Get deduplication statistics."""
+        return self._dedup_stats
+    
+    def configure_dedup(
+        self,
+        strategy: DeduplicationStrategy = DeduplicationStrategy.HYBRID,
+        threshold: float = 0.80
+    ) -> None:
+        """
+        Configure deduplication settings.
+        
+        Args:
+            strategy: Deduplication strategy
+            threshold: Similarity threshold for semantic matching (0.0-1.0)
+        """
+        self._dedup_strategy = strategy
+        self._similarity_threshold = threshold
+    
+    def add_edge(self, edge: GraphEdge) -> None:
+        """Add edge."""
+        self.edges.append(edge)
+    
+    def merge(self, other: "UnifiedGraph") -> "UnifiedGraph":
+        """Merge another graph into this one, deduplicating nodes."""
+        # Merge nodes (by label similarity)
+        label_to_id = {n.label.lower(): n.id for n in self.nodes.values()}
+        id_mapping = {}  # Map other IDs to our IDs
+        
+        for node in other.nodes.values():
+            lower_label = node.label.lower()
+            if lower_label in label_to_id:
+                # Node exists, map to existing ID
+                id_mapping[node.id] = label_to_id[lower_label]
+            else:
+                # New node, generate new ID
+                new_id = self._next_id(node.type)
+                id_mapping[node.id] = new_id
+                new_node = GraphNode(
+                    id=new_id,
+                    type=node.type,
+                    label=node.label,
+                    attrs=node.attrs
+                )
+                self.add_node(new_node)
+                label_to_id[lower_label] = new_id
+        
+        # Merge edges with remapped IDs
+        existing_edges = {(e.src, e.dst, e.rel) for e in self.edges}
+        for edge in other.edges:
+            new_src = id_mapping.get(edge.src, edge.src)
+            new_dst = id_mapping.get(edge.dst, edge.dst)
+            edge_key = (new_src, new_dst, edge.rel)
+            if edge_key not in existing_edges:
+                self.edges.append(GraphEdge(
+                    src=new_src,
+                    dst=new_dst,
+                    rel=edge.rel,
+                    attrs=edge.attrs
+                ))
+                existing_edges.add(edge_key)
+        
+        return self
+    
+    def _next_id(self, node_type: NodeType) -> str:
+        """Generate next ID for node type."""
+        prefix = node_type.value
+        existing = [n.id for n in self.nodes.values() if n.id.startswith(prefix)]
+        if not existing:
+            return f"{prefix}1"
+        max_num = max(int(nid[1:]) for nid in existing if nid[1:].isdigit())
+        return f"{prefix}{max_num + 1}"
+    
+    def to_dict(self) -> dict:
+        """Serialize to compact dict."""
+        return {
+            "n": [n.to_dict() for n in self.nodes.values()],
+            "e": [e.to_dict() for e in self.edges],
+            "m": self.metadata
+        }
+    
+    @classmethod
+    def from_dict(cls, d: dict) -> "UnifiedGraph":
+        """Deserialize from dict."""
+        graph = cls()
+        for nd in d.get("n", []):
+            node = GraphNode.from_dict(nd)
+            graph.nodes[node.id] = node
+        for ed in d.get("e", []):
+            graph.edges.append(GraphEdge.from_dict(ed))
+        graph.metadata = d.get("m", {})
+        # Rebuild label index for deduplication support
+        graph._rebuild_label_index()
+        return graph
+    
+    # ==========================================================================
+    # Query Methods
+    # ==========================================================================
+    
+    def get_entities(self) -> list[GraphNode]:
+        """Get WHO and WHAT nodes."""
+        return [n for n in self.nodes.values() 
+                if n.type in (NodeType.PERSON, NodeType.THING, NodeType.CONCEPT)]
+    
+    def get_goals(self) -> list[GraphNode]:
+        """Get GOAL nodes."""
+        return [n for n in self.nodes.values() if n.type == NodeType.GOAL]
+    
+    def get_decisions(self) -> list[GraphNode]:
+        """Get WHY nodes (decisions)."""
+        return [n for n in self.nodes.values() if n.type == NodeType.DECISION]
+    
+    def get_actions(self) -> list[GraphNode]:
+        """Get NEXT-STEP nodes."""
+        return [n for n in self.nodes.values() if n.type == NodeType.ACTION]
+    
+    def get_relationships_for(self, node_id: str) -> list[GraphEdge]:
+        """Get all edges involving a node."""
+        return [e for e in self.edges if e.src == node_id or e.dst == node_id]
+    
+    def get_decision_context(self, decision_id: str) -> dict:
+        """
+        Get full context for a decision (LEGACY - use get_decision_detail for Unified Deep).
+        """
+        decision = self.nodes.get(decision_id)
+        if not decision or decision.type != NodeType.DECISION:
+            return {}
+        
+        context = {
+            "decision": decision.label,
+            "alternatives": [],
+            "reasons": [],
+            "enables": []
+        }
+        
+        for edge in self.edges:
+            if edge.src == decision_id:
+                if edge.rel == EdgeType.OVER:
+                    alt_node = self.nodes.get(edge.dst)
+                    if alt_node:
+                        context["alternatives"].append({
+                            "option": alt_node.label,
+                            "rejected_because": edge.attrs.get("reason", "")
+                        })
+                elif edge.rel == EdgeType.BECAUSE:
+                    reason_node = self.nodes.get(edge.dst)
+                    if reason_node:
+                        context["reasons"].append(reason_node.label)
+                elif edge.rel == EdgeType.ENABLES:
+                    goal_node = self.nodes.get(edge.dst)
+                    if goal_node:
+                        context["enables"].append(goal_node.label)
+        
+        return context
+    
+    # ==========================================================================
+    # Unified Deep Query Methods (Rich Attribute Access)
+    # ==========================================================================
+    
+    def get_decision_detail(self, decision_id: str) -> dict:
+        """
+        Get FULL decision detail with Unified Deep rich attributes.
+        
+        Returns complete decision context including:
+        - rationale (full WHY text)
+        - confidence score
+        - context (situation)
+        - trade_offs (list of gained/sacrificed)
+        - alternatives with pros/cons/rejection_reason
+        - evidence/reasons with source and confidence
+        
+        This is the Unified Deep version of get_decision_context().
+        """
+        decision = self.nodes.get(decision_id)
+        if not decision or decision.type != NodeType.DECISION:
+            return {}
+        
+        # Base decision info from node attrs
+        detail = {
+            "id": decision_id,
+            "decision": decision.label,
+            "rationale": decision.attrs.get("rationale", ""),
+            "confidence": decision.attrs.get("confidence", 0.0),
+            "context": decision.attrs.get("context", ""),
+            "trade_offs": decision.attrs.get("trade_offs", []),
+            "turn_ref": decision.attrs.get("turn_ref", []),
+            "alternatives": [],
+            "reasons": [],
+            "enables": [],
+            "supersedes": None,
+        }
+        
+        # Collect rich edge data
+        for edge in self.edges:
+            if edge.src == decision_id:
+                if edge.rel == EdgeType.OVER:
+                    alt_node = self.nodes.get(edge.dst)
+                    if alt_node:
+                        detail["alternatives"].append({
+                            "option": alt_node.label,
+                            "pros": edge.attrs.get("pros", []),
+                            "cons": edge.attrs.get("cons", []),
+                            "rejection_reason": edge.attrs.get("rejection_reason", "")
+                        })
+                elif edge.rel == EdgeType.BECAUSE:
+                    reason_node = self.nodes.get(edge.dst)
+                    if reason_node:
+                        detail["reasons"].append({
+                            "claim": reason_node.label,
+                            "source": edge.attrs.get("source", "inferred"),
+                            "confidence": edge.attrs.get("confidence", 0.5),
+                            "turn_ref": edge.attrs.get("turn_ref", "")
+                        })
+                elif edge.rel == EdgeType.ENABLES:
+                    goal_node = self.nodes.get(edge.dst)
+                    if goal_node:
+                        detail["enables"].append(goal_node.label)
+                elif edge.rel == EdgeType.SUPERSEDES:
+                    old_decision = self.nodes.get(edge.dst)
+                    if old_decision:
+                        detail["supersedes"] = old_decision.label
+        
+        return detail
+    
+    def get_goal_hierarchy(self) -> dict:
+        """
+        Get hierarchical goal structure with Unified Deep attributes.
+        
+        Returns goals organized by type (mission -> goals -> tasks)
+        with status tracking and confidence scores.
+        """
+        hierarchy = {
+            "missions": [],
+            "goals": [],
+            "tasks": [],
+        }
+        
+        for node in self.nodes.values():
+            if node.type != NodeType.GOAL:
+                continue
+            
+            goal_info = {
+                "id": node.id,
+                "content": node.label,
+                "goal_type": node.attrs.get("goal_type", "goal"),
+                "status": node.attrs.get("status", "active"),
+                "confidence": node.attrs.get("confidence", 0.5),
+                "turn_ref": node.attrs.get("turn_ref", []),
+                "depends_on": [],
+                "enables": [],
+                "blocks": [],
+            }
+            
+            # Collect dependencies from edges
+            for edge in self.edges:
+                if edge.src == node.id:
+                    target = self.nodes.get(edge.dst)
+                    if target:
+                        if edge.rel == EdgeType.DEPENDS_ON:
+                            goal_info["depends_on"].append(target.label)
+                        elif edge.rel == EdgeType.ENABLES:
+                            goal_info["enables"].append(target.label)
+                        elif edge.rel == EdgeType.BLOCKS:
+                            goal_info["blocks"].append(target.label)
+            
+            # Categorize by goal_type
+            goal_type = goal_info["goal_type"]
+            if goal_type == "mission":
+                hierarchy["missions"].append(goal_info)
+            elif goal_type == "task":
+                hierarchy["tasks"].append(goal_info)
+            else:
+                hierarchy["goals"].append(goal_info)
+        
+        return hierarchy
+    
+    def get_active_goals(self) -> list[dict]:
+        """Get only active/pending goals with full attributes."""
+        active_statuses = ("active", "pending")
+        return [
+            {
+                "id": node.id,
+                "content": node.label,
+                "goal_type": node.attrs.get("goal_type", "goal"),
+                "status": node.attrs.get("status", "active"),
+                "confidence": node.attrs.get("confidence", 0.5),
+            }
+            for node in self.nodes.values()
+            if node.type == NodeType.GOAL 
+            and node.attrs.get("status", "active") in active_statuses
+        ]
+    
+    def get_all_decisions_with_detail(self) -> list[dict]:
+        """Get all decisions with full Unified Deep detail."""
+        return [
+            self.get_decision_detail(node.id)
+            for node in self.nodes.values()
+            if node.type == NodeType.DECISION
+        ]
+    
+    def search(
+        self, 
+        query: str, 
+        limit: int = 10,
+        include_types: Optional[list[NodeType]] = None
+    ) -> list[dict]:
+        """
+        Search unified graph for nodes matching query.
+        
+        Uses keyword-based matching with relevance scoring:
+        - Exact match: score 1.0
+        - Partial word match: score 0.7
+        - Related via edge: score 0.5
+        
+        Args:
+            query: Search query text
+            limit: Maximum results to return
+            include_types: Optional filter for specific node types
+            
+        Returns:
+            List of dicts with node info and relevance score, sorted by score
+        """
+        if not query or not self.nodes:
+            return []
+        
+        query_lower = query.lower()
+        query_words = set(query_lower.split())
+        
+        results = []
+        matched_ids = set()
+        
+        # Score each node
+        for node in self.nodes.values():
+            # Filter by type if specified
+            if include_types and node.type not in include_types:
+                continue
+            
+            label_lower = node.label.lower()
+            label_words = set(label_lower.split())
+            
+            score = 0.0
+            
+            # Exact substring match
+            if query_lower in label_lower:
+                score = 1.0
+            # Word overlap
+            elif query_words & label_words:
+                overlap = len(query_words & label_words) / len(query_words)
+                score = 0.7 * overlap
+            # Check attributes for matches
+            elif node.attrs:
+                for key, val in node.attrs.items():
+                    if isinstance(val, str) and query_lower in val.lower():
+                        score = 0.6
+                        break
+                    elif isinstance(val, list):
+                        for item in val:
+                            if isinstance(item, str) and query_lower in item.lower():
+                                score = 0.5
+                                break
+            
+            if score > 0:
+                result = {
+                    "id": node.id,
+                    "type": node.type.name.lower(),
+                    "content": node.label,
+                    "score": score,
+                    "attributes": node.attrs,
+                }
+                results.append(result)
+                matched_ids.add(node.id)
+        
+        # Add related nodes via edges (lower score)
+        for edge in self.edges:
+            for matched_id in list(matched_ids):
+                related_id = None
+                if edge.src == matched_id and edge.dst not in matched_ids:
+                    related_id = edge.dst
+                elif edge.dst == matched_id and edge.src not in matched_ids:
+                    related_id = edge.src
+                
+                if related_id and related_id not in matched_ids:
+                    related_node = self.nodes.get(related_id)
+                    if related_node:
+                        if include_types and related_node.type not in include_types:
+                            continue
+                        results.append({
+                            "id": related_node.id,
+                            "type": related_node.type.name.lower(),
+                            "content": related_node.label,
+                            "score": 0.4,
+                            "attributes": related_node.attrs,
+                            "related_to": matched_id,
+                        })
+                        matched_ids.add(related_id)
+        
+        # Sort by score descending, take top limit
+        results.sort(key=lambda x: x["score"], reverse=True)
+        return results[:limit]
+    
+    def summary(self) -> dict:
+        """Get graph summary statistics."""
+        type_counts = {}
+        for node in self.nodes.values():
+            type_counts[node.type.value] = type_counts.get(node.type.value, 0) + 1
+        
+        return {
+            "total_nodes": len(self.nodes),
+            "total_edges": len(self.edges),
+            "by_type": type_counts,
+            "entities": len(self.get_entities()),
+            "goals": len(self.get_goals()),
+            "decisions": len(self.get_decisions()),
+            "actions": len(self.get_actions())
+        }
+
+    # =========================================================================
+    # Semantic Search (Issue #51)
+    # =========================================================================
+    
+    def build_vector_index(self) -> "VectorIndex":
+        """
+        Build FAISS vector index from node embeddings.
+        
+        Called lazily on first semantic search. Very fast (<0.2ms for 1000 nodes).
+        Also stores the index internally for deduplication semantic matching.
+        
+        Returns:
+            VectorIndex populated with all node embeddings
+        """
+        from .vector_index import VectorIndex
+        
+        index = VectorIndex()
+        nodes_with_embeddings = [
+            (node_id, node.embedding)
+            for node_id, node in self.nodes.items()
+            if node.embedding is not None
+        ]
+        
+        if nodes_with_embeddings:
+            index.add_batch(nodes_with_embeddings)
+        
+        # Store internally for deduplication
+        self._vector_index = index
+        
+        return index
+    
+    def semantic_search(
+        self,
+        query: str,
+        k: int = 10,
+        node_types: Optional[list[NodeType]] = None,
+        min_score: float = 0.0
+    ) -> list[tuple[GraphNode, float]]:
+        """
+        Search graph nodes by semantic similarity.
+        
+        Uses FAISS vector search on node embeddings for fast, relevant results.
+        
+        Args:
+            query: Natural language search query
+            k: Maximum number of results
+            node_types: Optional filter for specific node types (e.g., [NodeType.GOAL])
+            min_score: Minimum similarity score threshold (0.0 to 1.0)
+            
+        Returns:
+            List of (GraphNode, similarity_score) tuples, sorted by score descending
+            
+        Example:
+            # Find decisions about databases
+            results = graph.semantic_search("database choice", k=5, 
+                                            node_types=[NodeType.DECISION])
+            for node, score in results:
+                print(f"{node.label}: {score:.3f}")
+        """
+        from .embeddings import EmbeddingService
+        
+        # Embed query
+        service = EmbeddingService()
+        query_embedding = service.embed_text(query)
+        
+        # Build index (very fast, <0.2ms for 1000 nodes)
+        index = self.build_vector_index()
+        
+        if len(index) == 0:
+            return []
+        
+        # Over-fetch to allow for type filtering
+        fetch_k = k * 3 if node_types else k
+        raw_results = index.search(query_embedding, fetch_k)
+        
+        # Filter and collect results
+        results = []
+        for node_id, score in raw_results:
+            if score < min_score:
+                continue
+                
+            node = self.nodes.get(node_id)
+            if node is None:
+                continue
+                
+            if node_types and node.type not in node_types:
+                continue
+                
+            results.append((node, score))
+            
+            if len(results) >= k:
+                break
+        
+        return results
+    
+    def search_goals(
+        self, 
+        query: str, 
+        k: int = 5,
+        min_score: float = 0.0
+    ) -> list[tuple[GraphNode, float]]:
+        """
+        Find goals related to query.
+        
+        Args:
+            query: Search query
+            k: Max results
+            min_score: Minimum similarity threshold
+            
+        Returns:
+            List of (goal_node, score) tuples
+        """
+        return self.semantic_search(query, k, node_types=[NodeType.GOAL], min_score=min_score)
+    
+    def search_decisions(
+        self, 
+        query: str, 
+        k: int = 5,
+        min_score: float = 0.0
+    ) -> list[tuple[GraphNode, float]]:
+        """
+        Find decisions related to query.
+        
+        Useful for "why" queries - understanding past choices.
+        
+        Args:
+            query: Search query
+            k: Max results
+            min_score: Minimum similarity threshold
+            
+        Returns:
+            List of (decision_node, score) tuples
+        """
+        return self.semantic_search(query, k, node_types=[NodeType.DECISION], min_score=min_score)
+    
+    def search_things(
+        self, 
+        query: str, 
+        k: int = 5,
+        min_score: float = 0.0
+    ) -> list[tuple[GraphNode, float]]:
+        """
+        Find things (technologies, tools, products) related to query.
+        
+        Args:
+            query: Search query
+            k: Max results
+            min_score: Minimum similarity threshold
+            
+        Returns:
+            List of (thing_node, score) tuples
+        """
+        return self.semantic_search(query, k, node_types=[NodeType.THING], min_score=min_score)
+    
+    def search_all_types(
+        self, 
+        query: str, 
+        k_per_type: int = 3,
+        min_score: float = 0.0
+    ) -> dict[str, list[tuple[GraphNode, float]]]:
+        """
+        Search across all node types, returning top results for each.
+        
+        Useful for comprehensive context retrieval.
+        
+        Args:
+            query: Search query
+            k_per_type: Max results per node type
+            min_score: Minimum similarity threshold
+            
+        Returns:
+            Dict mapping type name to list of (node, score) tuples
+        """
+        return {
+            "goals": self.search_goals(query, k_per_type, min_score),
+            "decisions": self.search_decisions(query, k_per_type, min_score),
+            "things": self.search_things(query, k_per_type, min_score),
+            "concepts": self.semantic_search(query, k_per_type, [NodeType.CONCEPT], min_score),
+            "actions": self.semantic_search(query, k_per_type, [NodeType.ACTION], min_score),
+            "persons": self.semantic_search(query, k_per_type, [NodeType.PERSON], min_score),
+        }
+
+
+# =============================================================================
+# LLM Protocol
+# =============================================================================
+
+class LLMClient(Protocol):
+    """Protocol for LLM client implementations."""
+    
+    async def complete(self, prompt: str, system: Optional[str] = None) -> str:
+        """Generate a completion for the given prompt."""
+        ...
+
+
+# =============================================================================
+# Unified Extraction Prompt (Token-Optimized)
+# =============================================================================
+
+UNIFIED_EXTRACTION_SYSTEM = """You extract structured knowledge graphs with FULL REASONING DEPTH.
+
+Output a JSON graph with:
+- n: nodes (entities, goals, decisions, actions) - with rich attributes
+- e: edges (relationships) - with rich attributes
+
+Node types: P=Person, T=Thing, C=Concept, G=Goal, D=Decision, A=Action
+Edge types: uses, wants, chose, over, because, enables, blocks, next, relates, part_of, depends_on, conflicts, supersedes
+
+CRITICAL RULES FOR UNIFIED DEEP:
+1. Extract EVERY technology/tool/framework/product by exact name
+2. GOAL nodes MUST have attrs: goal_type (mission|goal|task), status (active|completed|blocked|pending), confidence (0.0-1.0)
+3. DECISION nodes MUST have attrs: rationale (full WHY text), confidence, context, trade_offs list
+4. OVER edges (alternatives) MUST have attrs: pros list, cons list, rejection_reason
+5. BECAUSE edges MUST have attrs: source (user_stated|inferred|external), confidence
+6. Capture alternatives even if rejected - use 'over' edge with full pros/cons
+7. Use depends_on for task dependencies, conflicts for incompatible goals, supersedes for updated decisions"""
+
+UNIFIED_EXTRACTION_PROMPT = """Extract knowledge graph with FULL REASONING DEPTH from this conversation:
+
+{conversation}
+
+Output JSON with RICH ATTRIBUTES:
+{{
+  "n": [
+    {{"id": "P1", "t": "P", "l": "user"}},
+    {{"id": "G1", "t": "G", "l": "build REST API", "a": {{
+      "goal_type": "goal",
+      "status": "active",
+      "confidence": 0.9,
+      "turn_ref": ["turn_1"]
+    }}}},
+    {{"id": "D1", "t": "D", "l": "use FastAPI", "a": {{
+      "rationale": "FastAPI provides better async performance and automatic OpenAPI docs",
+      "confidence": 0.85,
+      "context": "Choosing web framework for REST API",
+      "trade_offs": [{{"gained": "performance + auto-docs", "sacrificed": "Django ecosystem"}}],
+      "turn_ref": ["turn_2", "turn_3"]
+    }}}},
+    {{"id": "T1", "t": "T", "l": "FastAPI"}},
+    {{"id": "T2", "t": "T", "l": "Django"}},
+    {{"id": "C1", "t": "C", "l": "async performance"}},
+    {{"id": "C2", "t": "C", "l": "automatic OpenAPI docs"}},
+    {{"id": "A1", "t": "A", "l": "set up project"}}
+  ],
+  "e": [
+    {{"s": "P1", "d": "G1", "r": "wants"}},
+    {{"s": "D1", "d": "T1", "r": "chose"}},
+    {{"s": "D1", "d": "T2", "r": "over", "a": {{
+      "pros": ["mature ecosystem", "built-in admin", "ORM"],
+      "cons": ["sync by default", "heavier weight"],
+      "rejection_reason": "Need async performance for high-throughput API"
+    }}}},
+    {{"s": "D1", "d": "C1", "r": "because", "a": {{
+      "source": "user_stated",
+      "confidence": 0.9,
+      "turn_ref": "turn_2"
+    }}}},
+    {{"s": "D1", "d": "C2", "r": "because", "a": {{
+      "source": "inferred",
+      "confidence": 0.7,
+      "turn_ref": "turn_3"
+    }}}},
+    {{"s": "D1", "d": "G1", "r": "enables"}},
+    {{"s": "G1", "d": "A1", "r": "next"}}
+  ]
+}}
+
+EXTRACTION RULES FOR UNIFIED DEEP:
+1. Extract EVERY technology/tool/framework/language by EXACT name
+2. GOAL nodes: Always include goal_type, status, confidence in attrs
+3. DECISION nodes: Always include rationale (full text), confidence, context, trade_offs
+4. OVER edges: Always include pros, cons, rejection_reason attrs
+5. BECAUSE edges: Always include source (user_stated/inferred/external), confidence
+6. Use depends_on edge when one task requires another
+7. Use conflicts edge when goals are mutually exclusive
+8. Use supersedes edge when a decision updates/replaces a previous one
+9. Include next actions or planned steps
+10. Only output valid JSON - no extra text"""
+
+
+DELTA_EXTRACTION_PROMPT = """Analyze new conversation content and extract NEW information to add to the existing knowledge graph.
+
+EXISTING NODES (do NOT repeat these, they already exist):
+{existing_graph}
+
+NEW CONVERSATION:
+{new_messages}
+
+Extract ONLY genuinely NEW information not already in the graph above.
+Output JSON with:
+- "add_n": New nodes (entities, goals, decisions, actions) NOT in existing list
+- "add_e": New relationships between nodes  
+- "update_n": Updates to existing node attributes (e.g., status changes)
+
+Format:
+{{
+  "add_n": [{{"id": "T1", "t": "T", "l": "NewTechnology", "a": {{}}}}],
+  "add_e": [{{"s": "G1", "d": "T1", "r": "uses"}}],
+  "update_n": [{{"id": "G1", "a": {{"status": "completed"}}}}]
+}}
+
+Node types: P=Person, T=Thing, C=Concept, G=Goal, D=Decision, A=Action
+Edge types: uses, wants, chose, over, because, enables, blocks, next, relates, part_of, depends_on, conflicts, supersedes
+
+Output valid JSON only - no explanations."""
+
+
+# =============================================================================
+# Unified Extractor
+# =============================================================================
+
+class UnifiedExtractor:
+    """
+    Single-pass extractor that captures all memory types.
+    
+    Replaces separate IntentTracker + ReasoningStore with one unified extraction.
+    
+    Token Efficiency:
+    - Current: 6 calls × ~10K tokens = 60K tokens per conversation
+    - Optimized: 1-2 calls × ~5K tokens = 5-10K tokens per conversation
+    - Target: 10-20x overhead vs 220x current
+    
+    Usage:
+        extractor = UnifiedExtractor(llm_client)
+        
+        # Full extraction (new conversation)
+        graph = await extractor.extract(messages)
+        
+        # Delta extraction (continuation)
+        graph = await extractor.extract_delta(new_messages, existing_graph)
+    """
+    
+    def __init__(self, llm_client: LLMClient):
+        self.llm = llm_client
+        self._extraction_cache: dict[str, UnifiedGraph] = {}
+    
+    def _format_conversation(self, messages: list[dict]) -> str:
+        """Format messages compactly for prompt."""
+        lines = []
+        for i, msg in enumerate(messages):
+            role = msg.get("role", "user")[0].upper()  # U or A
+            content = msg.get("content", "")
+            # Truncate very long messages
+            if len(content) > 500:
+                content = content[:500] + "..."
+            lines.append(f"{role}: {content}")
+        return "\n".join(lines)
+    
+    def _cache_key(self, messages: list[dict]) -> str:
+        """Generate cache key for messages."""
+        content = json.dumps(messages, sort_keys=True)
+        return hashlib.sha256(content.encode()).hexdigest()[:12]
+    
+    async def extract(
+        self,
+        messages: list[dict],
+        use_cache: bool = True
+    ) -> UnifiedGraph:
+        """
+        Extract unified graph from conversation.
+        
+        Single LLM call captures:
+        - Entities (WHO, WHAT)
+        - Goals (objectives)
+        - Decisions with rationale (WHY)
+        - Relationships (CONNECTIONS)
+        - Next actions (NEXT-STEPS)
+        
+        Args:
+            messages: Conversation messages
+            use_cache: Whether to use extraction cache
+            
+        Returns:
+            UnifiedGraph with all extracted knowledge
+        """
+        # Check cache
+        cache_key = self._cache_key(messages)
+        if use_cache and cache_key in self._extraction_cache:
+            logger.debug(f"Cache hit for extraction: {cache_key}")
+            return self._extraction_cache[cache_key]
+        
+        # Format conversation
+        conv_text = self._format_conversation(messages)
+        
+        # Single extraction call
+        prompt = UNIFIED_EXTRACTION_PROMPT.format(conversation=conv_text)
+        
+        try:
+            response = await self.llm.complete(prompt, system=UNIFIED_EXTRACTION_SYSTEM)
+            graph = self._parse_response(response)
+        except Exception as e:
+            logger.error(f"Extraction failed: {e}")
+            graph = UnifiedGraph()
+        
+        # Cache result
+        if use_cache:
+            self._extraction_cache[cache_key] = graph
+        
+        return graph
+    
+    async def extract_delta(
+        self,
+        new_messages: list[dict],
+        existing_graph: UnifiedGraph
+    ) -> UnifiedGraph:
+        """
+        Extract only changes from new messages.
+        
+        More token-efficient for multi-turn conversations:
+        - Sends compact representation of existing graph
+        - Only extracts NEW information
+        - Merges delta into existing graph
+        - Applies attribute updates to existing nodes
+        
+        Args:
+            new_messages: New messages to process
+            existing_graph: Current graph state
+            
+        Returns:
+            Updated UnifiedGraph
+        """
+        # Compact representation of existing graph
+        existing_compact = self._compact_graph_repr(existing_graph)
+        new_conv = self._format_conversation(new_messages)
+        
+        prompt = DELTA_EXTRACTION_PROMPT.format(
+            existing_graph=existing_compact,
+            new_messages=new_conv
+        )
+        
+        try:
+            response = await self.llm.complete(prompt, system=UNIFIED_EXTRACTION_SYSTEM)
+            delta, updates = self._parse_delta_response(response)
+            
+            # Merge delta into existing graph
+            existing_graph.merge(delta)
+            
+            # Apply node updates (Issue #27 fix)
+            for update in updates:
+                node_id = update["id"]
+                if node_id in existing_graph.nodes:
+                    existing_graph.nodes[node_id].attrs.update(update["attrs"])
+                    logger.debug(f"Updated node {node_id}: {update['attrs']}")
+            
+        except Exception as e:
+            logger.error(f"Delta extraction failed: {e}")
+        
+        return existing_graph
+    
+    def _compact_graph_repr(self, graph: UnifiedGraph) -> str:
+        """Create compact string representation of graph for prompt.
+        
+        Lists all existing nodes clearly so LLM knows what NOT to add.
+        """
+        lines = []
+        
+        # Nodes by type - clear listing format
+        for node_type in NodeType:
+            nodes = [n for n in graph.nodes.values() if n.type == node_type]
+            if nodes:
+                labels = [f"{n.id}:{n.label}" for n in nodes]
+                lines.append(f"{node_type.name}: {', '.join(labels)}")
+        
+        if not lines:
+            lines.append("(empty graph - no existing nodes)")
+        
+        # Key edges (limit to most important)
+        key_edges = [e for e in graph.edges 
+                     if e.rel in (EdgeType.CHOSE, EdgeType.BECAUSE, EdgeType.ENABLES)][:10]
+        if key_edges:
+            edge_strs = [f"{e.src}->{e.rel.value}->{e.dst}" for e in key_edges]
+            lines.append(f"KEY_EDGES: {'; '.join(edge_strs)}")
+        
+        return "\n".join(lines)
+        
+        return "\n".join(lines)
+    
+    def _parse_response(self, response: str) -> UnifiedGraph:
+        """Parse LLM response into UnifiedGraph."""
+        graph = UnifiedGraph()
+        
+        # Extract JSON from response
+        try:
+            # Try to find JSON in response
+            json_start = response.find("{")
+            json_end = response.rfind("}") + 1
+            if json_start >= 0 and json_end > json_start:
+                json_str = response[json_start:json_end]
+                data = json.loads(json_str)
+            else:
+                logger.warning("No JSON found in response")
+                return graph
+        except json.JSONDecodeError as e:
+            logger.error(f"JSON parse error: {e}")
+            return graph
+        
+        # Parse nodes
+        for node_data in data.get("n", []):
+            try:
+                node = GraphNode(
+                    id=node_data["id"],
+                    type=NodeType(node_data["t"]),
+                    label=node_data["l"],
+                    attrs=node_data.get("a", {})
+                )
+                graph.add_node(node)
+            except (KeyError, ValueError) as e:
+                logger.warning(f"Skipping invalid node: {e}")
+        
+        # Parse edges
+        for edge_data in data.get("e", []):
+            try:
+                edge = GraphEdge(
+                    src=edge_data["s"],
+                    dst=edge_data["d"],
+                    rel=EdgeType(edge_data["r"]),
+                    attrs=edge_data.get("a", {})
+                )
+                graph.add_edge(edge)
+            except (KeyError, ValueError) as e:
+                logger.warning(f"Skipping invalid edge: {e}")
+        
+        graph.metadata["extracted_at"] = _utc_now().isoformat()
+        
+        return graph
+    
+    def _parse_delta_response(self, response: str) -> tuple[UnifiedGraph, list[dict]]:
+        """Parse delta extraction response.
+        
+        Returns:
+            Tuple of (graph with new nodes/edges, list of node updates)
+        """
+        graph = UnifiedGraph()
+        updates = []
+        
+        try:
+            json_start = response.find("{")
+            json_end = response.rfind("}") + 1
+            if json_start >= 0 and json_end > json_start:
+                data = json.loads(response[json_start:json_end])
+            else:
+                return graph, updates
+        except json.JSONDecodeError:
+            return graph, updates
+        
+        # Parse new nodes
+        for node_data in data.get("add_n", []):
+            try:
+                node = GraphNode(
+                    id=node_data["id"],
+                    type=NodeType(node_data["t"]),
+                    label=node_data["l"],
+                    attrs=node_data.get("a", {})
+                )
+                graph.add_node(node)
+            except (KeyError, ValueError):
+                pass
+        
+        # Parse new edges
+        for edge_data in data.get("add_e", []):
+            try:
+                edge = GraphEdge(
+                    src=edge_data["s"],
+                    dst=edge_data["d"],
+                    rel=EdgeType(edge_data["r"]),
+                    attrs=edge_data.get("a", {})
+                )
+                graph.add_edge(edge)
+            except (KeyError, ValueError):
+                pass
+        
+        # Parse node updates (Issue #27 fix)
+        for update_data in data.get("update_n", []):
+            try:
+                updates.append({
+                    "id": update_data["id"],
+                    "attrs": update_data.get("a", {})
+                })
+            except KeyError:
+                pass
+        
+        return graph, updates
+
+
+# =============================================================================
+# Extraction Statistics
+# =============================================================================
+
+@dataclass
+class ExtractionStats:
+    """Track extraction performance."""
+    total_extractions: int = 0
+    cache_hits: int = 0
+    total_tokens: int = 0
+    total_nodes: int = 0
+    total_edges: int = 0
+    
+    @property
+    def cache_hit_rate(self) -> float:
+        if self.total_extractions == 0:
+            return 0.0
+        return self.cache_hits / self.total_extractions
+    
+    @property
+    def avg_tokens_per_extraction(self) -> float:
+        effective = self.total_extractions - self.cache_hits
+        if effective == 0:
+            return 0.0
+        return self.total_tokens / effective
+
+
+# =============================================================================
+# Helper Functions
+# =============================================================================
+
+def graph_to_memories(graph: UnifiedGraph) -> list[dict]:
+    """
+    Convert UnifiedGraph to list of memory objects for compatibility.
+    
+    Each node becomes a memory, edges add relationship context.
+    """
+    memories = []
+    
+    for node in graph.nodes.values():
+        memory = {
+            "id": node.id,
+            "type": node.type.name.lower(),
+            "content": node.label,
+            "attributes": node.attrs,
+            "relationships": []
+        }
+        
+        # Add relationship context
+        for edge in graph.get_relationships_for(node.id):
+            if edge.src == node.id:
+                other = graph.nodes.get(edge.dst)
+                if other:
+                    memory["relationships"].append({
+                        "type": edge.rel.value,
+                        "target": other.label,
+                        "direction": "outgoing"
+                    })
+            else:
+                other = graph.nodes.get(edge.src)
+                if other:
+                    memory["relationships"].append({
+                        "type": edge.rel.value,
+                        "target": other.label,
+                        "direction": "incoming"
+                    })
+        
+        memories.append(memory)
+    
+    return memories
+
+
+def memories_to_context_string(graph: UnifiedGraph, max_tokens: int = 2000) -> str:
+    """
+    Convert graph to context string for LLM prompt.
+    
+    Prioritizes:
+    1. Active goals
+    2. Recent decisions with rationale
+    3. Key entities
+    4. Next actions
+    """
+    sections = []
+    
+    # Goals
+    goals = graph.get_goals()
+    if goals:
+        goal_lines = [f"- {g.label}" for g in goals[:5]]
+        sections.append("GOALS:\n" + "\n".join(goal_lines))
+    
+    # Decisions with context
+    decisions = graph.get_decisions()
+    if decisions:
+        decision_lines = []
+        for d in decisions[:5]:
+            ctx = graph.get_decision_context(d.id)
+            line = f"- {d.label}"
+            if ctx.get("reasons"):
+                line += f" (because: {', '.join(ctx['reasons'][:2])})"
+            if ctx.get("alternatives"):
+                alts = [a["option"] for a in ctx["alternatives"][:2]]
+                line += f" [over: {', '.join(alts)}]"
+            decision_lines.append(line)
+        sections.append("DECISIONS:\n" + "\n".join(decision_lines))
+    
+    # Key entities
+    entities = graph.get_entities()
+    if entities:
+        entity_lines = [f"- {e.label} ({e.type.name})" for e in entities[:10]]
+        sections.append("KEY ENTITIES:\n" + "\n".join(entity_lines))
+    
+    # Next actions
+    actions = graph.get_actions()
+    if actions:
+        action_lines = [f"- {a.label}" for a in actions[:5]]
+        sections.append("NEXT STEPS:\n" + "\n".join(action_lines))
+    
+    return "\n\n".join(sections)