clm-kernel 0.0.0__py3-none-any.whl

clm/layer3/rag/graph/community.py

@@ -0,0 +1,198 @@
+"""
+Community Detection and Summarization
+
+Label Propagation Algorithm (LPA) for community detection
+and LLM-based hierarchical summarization.
+"""
+
+import json
+import logging
+import random
+import re
+from collections import Counter, defaultdict
+
+from .store import GraphStore
+
+logger = logging.getLogger(__name__)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Community Detection
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def detect_communities(store: GraphStore, max_iter: int = 20) -> list[list[int]]:
+    """
+    Detect communities using asynchronous Label Propagation.
+
+    Args:
+        store: GraphStore instance
+        max_iter: Maximum iterations
+
+    Returns:
+        List of communities, where each community is a list of entity IDs
+    """
+    logger.info("Starting community detection (LPA)...")
+
+    # 1. Build Adjacency List
+    adj = defaultdict(list)
+    nodes = set()
+
+    cursor = store.conn.cursor()
+    cursor.execute("SELECT source_entity_id, target_entity_id FROM graph_relationships")
+
+    rows = cursor.fetchall()
+    for src, tgt in rows:
+        adj[src].append(tgt)
+        adj[tgt].append(src)  # Undirected for community detection
+        nodes.add(src)
+        nodes.add(tgt)
+
+    node_list = list(nodes)
+    if not node_list:
+        logger.warning("No nodes found for community detection")
+        return []
+
+    logger.debug(f"Graph size: {len(nodes)} nodes, {len(rows)} edges")
+
+    # 2. Initialize Labels
+    labels = {node: node for node in nodes}
+
+    # 3. Propagate Labels
+    for i in range(max_iter):
+        changes = 0
+        random.shuffle(node_list)
+
+        for node in node_list:
+            if not adj[node]:
+                continue
+
+            neighbor_labels = [labels[neighbor] for neighbor in adj[node]]
+            if not neighbor_labels:
+                continue
+
+            # Find most frequent label (ties broken randomly)
+            counts = Counter(neighbor_labels)
+            max_freq = counts.most_common(1)[0][1]
+            best_labels = [lbl for lbl, count in counts.items() if count == max_freq]
+            new_label = random.choice(best_labels)
+
+            if labels[node] != new_label:
+                labels[node] = new_label
+                changes += 1
+
+        logger.debug(f"LPA Iteration {i + 1}: {changes} changes")
+        if changes == 0:
+            logger.info(f"LPA converged after {i + 1} iterations")
+            break
+
+    # 4. Group Communities
+    communities = defaultdict(list)
+    for node, label in labels.items():
+        communities[label].append(node)
+
+    result = list(communities.values())
+    logger.info(f"Detected {len(result)} communities")
+
+    # Sort for deterministic output (by size desc, then first ID)
+    result.sort(key=lambda c: (-len(c), min(c)))
+
+    return result
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Community Summarization
+# ─────────────────────────────────────────────────────────────────────────────
+
+SUMMARIZE_SYSTEM_PROMPT = """You are an expert graph analyst.
+Your task is to summarize a "community" of related entities from a knowledge graph.
+Focus on the common themes, purposes, or technologies that connect these entities.
+Synthesize the descriptions into a cohesive whole."""
+
+SUMMARIZE_USER_PROMPT = """Analyze the following list of entities and their descriptions which form a community in a knowledge graph.
+Create a Title and valid JSON Summary.
+
+--- BEGIN ENTITY LIST ---
+{entity_text}
+--- END ENTITY LIST ---
+
+Requirement:
+- Provide a short Title.
+- Provide a detailed Summary of common themes.
+- Output MUST be valid JSON in the format:
+{{
+  "title": "Community Title",
+  "summary": "Detailed summary..."
+}}
+"""
+
+
+class CommunitySummarizer:
+    """Summarizes graph communities using LLM."""
+
+    def __init__(self, store: GraphStore, model: str = "qwen3:0.6b"):
+        self.store = store
+        self.model = model
+
+    def summarize_and_store(self, communities: list[list[int]]) -> int:
+        """
+        Summarize communities and store them in the DB.
+
+        Returns:
+            Count of summaries generated
+        """
+        count = 0
+
+        for comm_ids in communities:
+            # Prepare context
+            entity_text = self._prepare_context(comm_ids)
+            if not entity_text:
+                continue
+
+            try:
+                title, summary = self._generate_summary(entity_text)
+                self.store.add_community(title, summary, comm_ids)
+                count += 1
+                logger.info(f"Generated community: {title}")
+            except Exception as e:
+                logger.error(f"Failed to summarize community: {e}", exc_info=True)
+
+        return count
+
+    def _prepare_context(self, ids: list[int]) -> str:
+        lines = []
+        for eid in ids[:30]:  # Limit context size
+            ent = self.store.get_entity_by_id(eid)
+            if ent:
+                lines.append(f"- {ent['name']} ({ent['type']}): {ent['description']}")
+        return "\n".join(lines)
+
+    def _generate_summary(self, entity_text: str) -> tuple[str, str]:
+        from clm.layer3.agency import chat_monad
+
+        prompt = SUMMARIZE_USER_PROMPT.format(entity_text=entity_text)
+
+        result = chat_monad(
+            prompt=prompt,
+            system_prompt=SUMMARIZE_SYSTEM_PROMPT,
+            model=self.model,
+            temperature=0.3,
+            max_tokens=1000,
+        ).unsafe_run()
+
+        if result.is_left():
+            raise RuntimeError(f"LLM failure: {result.value}")
+
+        content = result.value.get("content", str(result.value))
+
+        # Parse JSON
+        json_match = re.search(r"\{[\s\S]*\}", content)
+        if json_match:
+            try:
+                data = json.loads(json_match.group())
+                return data.get("title", "Unknown Community"), data.get("summary", "")
+            except json.JSONDecodeError:
+                pass
+
+        # Fallback if no JSON
+        return "Community Summary", content[:500]

clm/layer3/rag/graph/engine.py

@@ -0,0 +1,433 @@
+"""
+GraphRAG Engine
+
+Combines vector search with knowledge graph for enhanced RAG.
+"""
+
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+from clm.layer0.model.card import MCard
+
+from ..config import DEFAULT_RAG_CONFIG, RAGConfig
+from ..embeddings import OllamaEmbeddingProvider
+from ..vector import MCardVectorStore
+from .extractor import GraphExtractor
+from .store import GraphStore, store_extraction_result
+
+logger = logging.getLogger(__name__)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# GraphRAG Response
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@dataclass
+class GraphRAGResponse:
+    """Response from GraphRAG query."""
+
+    answer: str
+    sources: list[str]  # Source MCard hashes
+    entities: list[dict[str, Any]]  # Relevant entities
+    relationships: list[dict[str, Any]]  # Relevant relationships
+    graph_context: str  # Graph traversal context
+    vector_context: str  # Vector search context
+    confidence: float
+    metadata: dict[str, Any] | None = None
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# System Prompts
+# ─────────────────────────────────────────────────────────────────────────────
+
+GRAPHRAG_SYSTEM_PROMPT = """You are a helpful assistant that answers questions using both document context and a knowledge graph.
+
+You have access to:
+1. DOCUMENT CONTEXT: Relevant text passages from the knowledge base
+2. GRAPH CONTEXT: Entities and their relationships from the knowledge graph
+
+Use both sources to provide comprehensive, accurate answers.
+- Cite sources [1], [2], etc. for document context
+- Reference entities by name when using graph information
+- If the graph shows relationships that help answer the question, describe them
+
+Be thorough but concise. Acknowledge if information is incomplete."""
+
+
+GRAPH_QUERY_PROMPT = """Question: {question}
+
+=== DOCUMENT CONTEXT ===
+{vector_context}
+
+=== KNOWLEDGE GRAPH ===
+Entities: {entities}
+Relationships: {relationships}
+
+Please provide a comprehensive answer using both the document context and knowledge graph information."""
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# GraphRAG Engine
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+class GraphRAGEngine:
+    """
+    GraphRAG Engine combining vector search with knowledge graphs.
+
+    Features:
+    - Entity extraction from MCard content
+    - Knowledge graph storage and traversal
+    - Multi-hop reasoning across entities
+    - Combined vector + graph context for LLM
+
+    Usage:
+        from clm.layer3.rag.graph import GraphRAGEngine
+
+        engine = GraphRAGEngine(vector_db_path=':memory:')
+
+        # Index with graph extraction
+        engine.index(mcard)
+
+        # Query with graph context
+        response = engine.query("How does MCard relate to PTR?")
+    """
+
+    def __init__(
+        self,
+        config: RAGConfig = None,
+        vector_db_path: str = None,
+        graph_db_path: str = None,
+        llm_model: str = "qwen3:0.6b",
+    ):
+        """
+        Initialize GraphRAG engine.
+
+        Args:
+            config: RAG configuration
+            vector_db_path: Path to vector database
+            graph_db_path: Path to graph database (default: same as vector)
+            llm_model: LLM model for generation
+        """
+        self.config = config or DEFAULT_RAG_CONFIG
+        self.llm_model = llm_model
+
+        # Initialize embedding provider
+        self.embedder = OllamaEmbeddingProvider(
+            model=self.config.embedding_model, base_url=self.config.ollama_base_url
+        )
+
+        # Initialize vector store
+        self.vector_store = MCardVectorStore(
+            db_path=vector_db_path, config=self.config, embedding_provider=self.embedder
+        )
+
+        # Initialize graph store (same DB or separate)
+        graph_path = graph_db_path or vector_db_path or ":memory:"
+        self.graph_store = GraphStore(db_path=graph_path)
+
+        # Initialize extractor
+        self.extractor = GraphExtractor(model=llm_model)
+
+        logger.info("Initialized GraphRAG engine")
+
+    def index(
+        self, mcard: MCard, extract_graph: bool = True, force: bool = False
+    ) -> dict[str, int]:
+        """
+        Index an MCard with optional graph extraction.
+
+        Args:
+            mcard: MCard to index
+            extract_graph: Extract entities and relationships
+            force: Re-index even if exists
+
+        Returns:
+            Dict with indexing stats
+        """
+        stats = {"vectors": 0, "entities": 0, "relationships": 0}
+
+        # Get content
+        content = mcard.get_content()
+        if isinstance(content, bytes):
+            try:
+                content = content.decode("utf-8")
+            except UnicodeDecodeError:
+                logger.warning(f"Cannot decode MCard {mcard.hash[:8]}")
+                return stats
+
+        # Vector indexing
+        if force or not self.vector_store.is_indexed(mcard.hash):
+            stats["vectors"] = self.vector_store.index(mcard)
+
+        # Graph extraction
+        if extract_graph and (force or not self.graph_store.is_extracted(mcard.hash)):
+            logger.debug(f"Extracting graph from {mcard.hash[:8]}...")
+
+            result = self.extractor.extract(content)
+
+            if result.success:
+                entity_count, rel_count = store_extraction_result(
+                    self.graph_store, result, mcard.hash
+                )
+                stats["entities"] = entity_count
+                stats["relationships"] = rel_count
+                logger.debug(
+                    f"Extracted {entity_count} entities, {rel_count} relationships"
+                )
+            else:
+                logger.warning(f"Graph extraction failed: {result.error}")
+
+        return stats
+
+    def detect_and_summarize_communities(self) -> int:
+        """
+        Detect communities and generate summaries.
+
+        Returns:
+            Number of communities summarized
+        """
+        from .community import CommunitySummarizer, detect_communities
+
+        logger.info("Detecting communities...")
+        communities = detect_communities(self.graph_store)
+
+        if not communities:
+            logger.info("No communities detected.")
+            return 0
+
+        logger.info(f"Summarizing {len(communities)} communities...")
+        summarizer = CommunitySummarizer(self.graph_store, model=self.llm_model)
+        count = summarizer.summarize_and_store(communities)
+
+        logger.info(f"Generated {count} community summaries.")
+        return count
+
+    def _get_vector_context(self, query: str, k: int) -> tuple[str, list[str]]:
+        """Get vector search context."""
+        results = self.vector_store.hybrid_search(query, k)
+
+        context_parts = []
+        sources = []
+
+        for i, result in enumerate(results):
+            chunk = result.chunk_text or f"[Content from {result.hash[:8]}]"
+            context_parts.append(f"[{i + 1}] {chunk}")
+            sources.append(result.hash)
+
+        return "\n\n".join(context_parts), sources
+
+    def _get_graph_context(
+        self, query: str, source_hashes: list[str] = None
+    ) -> tuple[list[dict], list[dict]]:
+        """
+        Get relevant graph context.
+
+        Searches for entities mentioned in the query and finds related entities.
+        """
+        entities = []
+        relationships = []
+        seen_entity_ids: set[int] = set()
+
+        # Search for entities matching query terms
+        for word in query.split():
+            if len(word) > 3:
+                found = self.graph_store.search_entities(word, limit=5)
+                for entity in found:
+                    if entity["id"] not in seen_entity_ids:
+                        seen_entity_ids.add(entity["id"])
+                        entities.append(entity)
+
+                        # Get relationships
+                        rels_out = self.graph_store.get_relationships_from(entity["id"])
+                        rels_in = self.graph_store.get_relationships_to(entity["id"])
+
+                        for rel in rels_out:
+                            relationships.append(
+                                {
+                                    "source": entity["name"],
+                                    "relationship": rel["relationship"],
+                                    "target": rel["target_name"],
+                                }
+                            )
+
+                        for rel in rels_in:
+                            relationships.append(
+                                {
+                                    "source": rel["source_name"],
+                                    "relationship": rel["relationship"],
+                                    "target": entity["name"],
+                                }
+                            )
+
+        # Also get entities from source documents
+        if source_hashes:
+            for hash_val in source_hashes[:3]:  # Limit
+                source_entities = self.graph_store.get_entities_by_source(hash_val)
+                for entity in source_entities:
+                    if entity["id"] not in seen_entity_ids:
+                        seen_entity_ids.add(entity["id"])
+                        entities.append(entity)
+
+        return entities, relationships
+
+    def query(
+        self, question: str, k: int = 5, use_graph: bool = True
+    ) -> GraphRAGResponse:
+        """
+        Query with GraphRAG: vector search + graph context.
+
+        Args:
+            question: User question
+            k: Number of vector results
+            use_graph: Include graph context
+
+        Returns:
+            GraphRAGResponse with answer and sources
+        """
+        # Get vector context
+        vector_context, sources = self._get_vector_context(question, k)
+
+        # Get graph context
+        entities = []
+        relationships = []
+        graph_context = ""
+
+        if use_graph:
+            entities, relationships = self._get_graph_context(question, sources)
+
+            if entities:
+                entity_strs = [f"{e['name']} ({e['type']})" for e in entities[:10]]
+                rel_strs = [
+                    f"{r['source']} --{r['relationship']}-> {r['target']}"
+                    for r in relationships[:15]
+                ]
+
+                graph_context = f"Entities: {', '.join(entity_strs)}\n"
+                graph_context += "Relationships:\n" + "\n".join(rel_strs)
+
+        # Generate answer
+        if not vector_context and not entities:
+            return GraphRAGResponse(
+                answer="I couldn't find relevant information.",
+                sources=[],
+                entities=[],
+                relationships=[],
+                graph_context="",
+                vector_context="",
+                confidence=0.0,
+            )
+
+        # Build prompt
+        entity_str = (
+            ", ".join(f"{e['name']} ({e['type']})" for e in entities)
+            if entities
+            else "None found"
+        )
+        rel_str = (
+            "\n".join(
+                f"  - {r['source']} --{r['relationship']}-> {r['target']}"
+                for r in relationships
+            )
+            if relationships
+            else "None found"
+        )
+
+        prompt = GRAPH_QUERY_PROMPT.format(
+            question=question,
+            vector_context=vector_context or "No document context available.",
+            entities=entity_str,
+            relationships=rel_str,
+        )
+
+        # Call LLM
+        try:
+            from clm.layer3.agency import chat_monad
+
+            response = chat_monad(
+                prompt=prompt,
+                system_prompt=GRAPHRAG_SYSTEM_PROMPT,
+                model=self.llm_model,
+                temperature=0.3,
+                max_tokens=1500,
+            ).unsafe_run()
+
+            if response.is_right():
+                answer = response.value.get("content", str(response.value))
+            else:
+                answer = f"Error generating response: {response.value}"
+
+        except Exception as e:
+            logger.error(f"LLM generation failed: {e}", exc_info=True)
+            answer = f"Error: {e}"
+
+        # Calculate confidence
+        vec_results = self.vector_store.search(question, k)
+        avg_score = (
+            sum(r.score for r in vec_results) / len(vec_results) if vec_results else 0
+        )
+        graph_boost = 0.1 if entities else 0
+
+        return GraphRAGResponse(
+            answer=answer,
+            sources=sources,
+            entities=entities,
+            relationships=relationships,
+            graph_context=graph_context,
+            vector_context=vector_context,
+            confidence=min(avg_score + graph_boost, 1.0),
+            metadata={
+                "model": self.llm_model,
+                "entity_count": len(entities),
+                "relationship_count": len(relationships),
+            },
+        )
+
+    def explore_entity(self, entity_name: str, hops: int = 2) -> dict[str, Any]:
+        """
+        Explore an entity and its neighborhood.
+
+        Args:
+            entity_name: Entity to explore
+            hops: Traversal depth
+
+        Returns:
+            Dict with entity info and related entities
+        """
+        # Try exact match first
+        entity = self.graph_store.get_entity_by_name(entity_name)
+
+        # If not found, try fuzzy search
+        if not entity:
+            candidates = self.graph_store.search_entities(entity_name, limit=1)
+            if candidates:
+                entity = candidates[0]
+                # Update outgoing/incoming/related to use the found entity name/id
+
+        if not entity:
+            return {"error": f"Entity '{entity_name}' not found"}
+
+        related = self.graph_store.find_related(entity["name"], hops=hops)
+
+        return {
+            "entity": entity,
+            "related": related,
+            "outgoing": self.graph_store.get_relationships_from(entity["id"]),
+            "incoming": self.graph_store.get_relationships_to(entity["id"]),
+        }
+
+    def find_connection(self, source: str, target: str) -> list[str] | None:
+        """Find path between two entities."""
+        return self.graph_store.find_path(source, target)
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get engine statistics."""
+        return {
+            "vector_count": self.vector_store.count(),
+            "unique_mcards": self.vector_store.count_unique(),
+            "graph": self.graph_store.get_stats(),
+            "llm_model": self.llm_model,
+            "embedding_model": self.config.embedding_model,
+        }