clm-kernel 0.0.0__py3-none-any.whl

clm/layer3/rag/embeddings/vision.py

@@ -0,0 +1,290 @@
+"""
+Vision Embedding Provider
+
+Multimodal embedding provider that uses vision models to describe images,
+then embeds the descriptions for vector search.
+
+This approach is recommended for image RAG since Ollama's embed API
+doesn't directly support image inputs.
+"""
+
+import base64
+import json
+import logging
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+from clm.cross_cutting.config.config_constants import (
+    RAG_VISION_DESCRIBE_TIMEOUT_SECS,
+    RAG_VISION_VALIDATE_TIMEOUT_SECS,
+)
+from clm.cross_cutting.utils.url_safety import safe_urlopen
+
+from .base import EmbeddingProvider
+from .ollama import OllamaEmbeddingProvider
+
+logger = logging.getLogger(__name__)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Vision Models Configuration
+# ─────────────────────────────────────────────────────────────────────────────
+
+VISION_MODELS = {
+    "moondream": {
+        "description": "Moondream - Tiny, high-performance vision language model",
+        "size": "1.7GB",
+    },
+    "llama3.2-vision": {
+        "description": "Llama 3.2 Vision - 11B multimodal model",
+        "size": "7.9GB",
+    },
+    "llava": {
+        "description": "LLaVA - Large Language and Vision Assistant",
+        "size": "4.7GB",
+    },
+    "minicpm-v": {
+        "description": "MiniCPM-V - Efficient vision-language model",
+        "size": "5.6GB",
+    },
+}
+
+DEFAULT_VISION_MODEL = "moondream"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Vision Embedding Provider
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+class VisionEmbeddingProvider(EmbeddingProvider):
+    """
+    Multimodal embedding provider for images.
+
+    Uses a two-stage approach:
+    1. Vision model generates a text description of the image
+    2. Text embedding model converts description to vector
+
+    This enables semantic search over images using existing vector infrastructure.
+
+    Usage:
+        from clm.layer3.rag.embeddings import VisionEmbeddingProvider
+
+        provider = VisionEmbeddingProvider()
+
+        # Embed an image
+        embedding = provider.embed_image("/path/to/image.jpg")
+
+        # Or with bytes
+        with open("image.png", "rb") as f:
+            embedding = provider.embed_image_bytes(f.read())
+    """
+
+    def __init__(
+        self,
+        vision_model: str = DEFAULT_VISION_MODEL,
+        embedding_model: str = "nomic-embed-text",
+        base_url: str = "http://localhost:11434",
+        description_prompt: str = None,
+    ):
+        """
+        Initialize vision embedding provider.
+
+        Args:
+            vision_model: Ollama vision model for describing images
+            embedding_model: Text embedding model
+            base_url: Ollama API base URL
+            description_prompt: Custom prompt for image description
+        """
+        self.vision_model = vision_model
+        self.base_url = base_url.rstrip("/")
+
+        # Text embedder for converting descriptions to vectors
+        self.text_embedder = OllamaEmbeddingProvider(
+            model=embedding_model, base_url=base_url
+        )
+
+        # Prompt for image description
+        self.description_prompt = (
+            description_prompt
+            or """Describe this image in detail for semantic search.
+Include:
+- Main subject and objects visible
+- Colors, textures, and visual elements
+- Any text visible in the image
+- Context, setting, or environment
+- Actions or relationships between elements
+
+Be comprehensive but concise. Focus on searchable details."""
+        )
+
+    @property
+    def model_name(self) -> str:
+        """Return combined model identifier."""
+        return f"vision:{self.vision_model}+{self.text_embedder.model_name}"
+
+    @property
+    def dimensions(self) -> int:
+        """Return embedding dimensions (from text embedder)."""
+        return self.text_embedder.dimensions
+
+    @property
+    def provider_name(self) -> str:
+        return "ollama-vision"
+
+    def describe_image(self, image_data: str | bytes | Path, prompt: str = None) -> str:
+        """
+        Generate text description of an image.
+
+        Args:
+            image_data: Image as file path, bytes, or base64 string
+            prompt: Optional custom prompt
+
+        Returns:
+            Text description of the image
+        """
+        # Convert image to base64
+        if isinstance(image_data, str | Path):
+            path = Path(image_data)
+            if path.exists():
+                with open(path, "rb") as f:
+                    image_bytes = f.read()
+                image_b64 = base64.b64encode(image_bytes).decode("utf-8")
+            else:
+                # Assume it's already base64
+                image_b64 = str(image_data)
+        elif isinstance(image_data, bytes):
+            image_b64 = base64.b64encode(image_data).decode("utf-8")
+        else:
+            raise ValueError(f"Unsupported image data type: {type(image_data)}")
+
+        # Call vision model
+        url = f"{self.base_url}/api/generate"
+
+        payload = {
+            "model": self.vision_model,
+            "prompt": prompt or self.description_prompt,
+            "images": [image_b64],
+            "stream": False,
+        }
+
+        data = json.dumps(payload).encode("utf-8")
+        req = urllib.request.Request(
+            url, data=data, headers={"Content-Type": "application/json"}, method="POST"
+        )
+
+        try:
+            with safe_urlopen(
+                req, timeout=RAG_VISION_DESCRIBE_TIMEOUT_SECS
+            ) as response:
+                result = json.loads(response.read().decode("utf-8"))
+                return result.get("response", "")
+        except Exception as e:
+            logger.error(f"Vision model call failed: {e}", exc_info=True)
+            raise RuntimeError(f"Failed to describe image: {e}") from e
+
+    def embed_image(
+        self, image_data: str | bytes | Path, prompt: str = None
+    ) -> list[float]:
+        """
+        Generate embedding for an image.
+
+        Args:
+            image_data: Image as file path, bytes, or base64 string
+            prompt: Optional custom prompt for description
+
+        Returns:
+            Embedding vector
+        """
+        # Get description
+        description = self.describe_image(image_data, prompt)
+
+        if not description:
+            raise ValueError("Vision model returned empty description")
+
+        logger.debug(f"Image description: {description[:100]}...")
+
+        # Embed description
+        return self.text_embedder.embed(description)
+
+    def embed_image_with_description(
+        self, image_data: str | bytes | Path, prompt: str = None
+    ) -> tuple[list[float], str]:
+        """
+        Generate embedding and return description.
+
+        Args:
+            image_data: Image as file path, bytes, or base64
+            prompt: Optional custom prompt
+
+        Returns:
+            Tuple of (embedding, description)
+        """
+        description = self.describe_image(image_data, prompt)
+        embedding = self.text_embedder.embed(description)
+        return embedding, description
+
+    # Required EmbeddingProvider methods
+
+    def embed(self, text: str) -> list[float]:
+        """Embed text (delegates to text embedder)."""
+        return self.text_embedder.embed(text)
+
+    def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """Embed multiple texts."""
+        return self.text_embedder.embed_batch(texts)
+
+    def embed_images(self, images: list[str | bytes | Path]) -> list[list[float]]:
+        """
+        Embed multiple images.
+
+        Args:
+            images: List of image paths, bytes, or base64 strings
+
+        Returns:
+            List of embedding vectors
+        """
+        embeddings = []
+        for img in images:
+            try:
+                embeddings.append(self.embed_image(img))
+            except Exception as e:
+                logger.error(f"Failed to embed image: {e}", exc_info=True)
+                # Return zero vector on failure
+                embeddings.append([0.0] * self.dimensions)
+        return embeddings
+
+    def validate_connection(self) -> bool:
+        """Validate both vision and embedding models are available."""
+        # Check text embedder
+        if not self.text_embedder.validate_connection():
+            return False
+
+        # Check vision model
+        try:
+            url = f"{self.base_url}/api/show"
+            payload = json.dumps({"name": self.vision_model}).encode("utf-8")
+            req = urllib.request.Request(
+                url,
+                data=payload,
+                headers={"Content-Type": "application/json"},
+                method="POST",
+            )
+            with safe_urlopen(
+                req, timeout=RAG_VISION_VALIDATE_TIMEOUT_SECS
+            ) as response:
+                return response.status == 200
+        except Exception as e:
+            logger.warning(f"Vision model check failed: {e}")
+            return False
+
+    def get_info(self) -> dict[str, Any]:
+        """Get provider information."""
+        return {
+            "provider": self.provider_name,
+            "vision_model": self.vision_model,
+            "embedding_model": self.text_embedder.model_name,
+            "dimensions": self.dimensions,
+            "vision_models_available": list(VISION_MODELS.keys()),
+        }

clm/layer3/rag/engine.py

@@ -0,0 +1,276 @@
+"""
+MCard RAG Engine
+
+Main interface for Retrieval-Augmented Generation over MCard collections.
+"""
+
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+from clm.layer0.model.card import MCard
+from clm.layer2.card_collection import CardCollection
+
+from .config import DEFAULT_RAG_CONFIG, RAGConfig
+from .embeddings import OllamaEmbeddingProvider
+from .vector import MCardVectorStore, VectorSearchResult
+
+logger = logging.getLogger(__name__)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# RAG Response
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@dataclass
+class RAGResponse:
+    """Response from a RAG query."""
+
+    answer: str
+    sources: list[str]  # Source MCard hashes
+    source_chunks: list[str]  # Relevant text chunks
+    confidence: float
+    metadata: dict[str, Any] | None = None
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# RAG Engine
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+class MCardRAGEngine:
+    """
+    Retrieval-Augmented Generation engine for MCard collections.
+
+    Combines semantic vector search with LLM generation to provide
+    context-aware answers grounded in MCard content.
+
+    Usage:
+        from clm.layer2.card_collection import get_default_collection
+        default_collection = get_default_collection()
+        from clm.layer3.rag import MCardRAGEngine
+
+        # Initialize
+        rag = MCardRAGEngine(default_collection)
+
+        # Index content
+        rag.index_all()
+
+        # Query
+        response = rag.query("What is the Cubical Logic Model?")
+        print(response.answer)
+        print(f"Sources: {response.sources}")
+    """
+
+    def __init__(
+        self,
+        collection: CardCollection = None,
+        config: RAGConfig = None,
+        vector_db_path: str = None,
+    ):
+        """
+        Initialize RAG engine.
+
+        Args:
+            collection: MCard collection for storage (uses default if None)
+            config: RAG configuration
+            vector_db_path: Path to vector database (None = in-memory)
+        """
+        from clm.layer2.card_collection import get_default_collection
+        _default_collection = get_default_collection()
+
+        self.collection = collection or _default_collection
+        self.config = config or DEFAULT_RAG_CONFIG
+
+        # Initialize embedding provider
+        self.embedder = OllamaEmbeddingProvider(
+            model=self.config.embedding_model, base_url=self.config.ollama_base_url
+        )
+
+        # Initialize vector store
+        self.vector_store = MCardVectorStore(
+            db_path=vector_db_path or self.config.vector_db_path,
+            config=self.config,
+            embedding_provider=self.embedder,
+        )
+
+        # LLM for generation (lazy loaded)
+        self._llm = None
+
+        logger.info(f"Initialized RAG engine with model {self.config.embedding_model}")
+
+    @property
+    def llm(self):
+        """Lazy-load LLM runtime."""
+        if self._llm is None:
+            from clm.layer3.agency import LLMRuntime
+
+            self._llm = LLMRuntime()
+        return self._llm
+
+    def index(self, mcard: MCard) -> bool:
+        """
+        Index a single MCard for semantic search.
+
+        Args:
+            mcard: MCard to index
+
+        Returns:
+            True if indexed successfully
+        """
+        try:
+            count = self.vector_store.index(mcard)
+            return count > 0
+        except Exception as e:
+            logger.error(f"Failed to index MCard {mcard.hash[:8]}: {e}", exc_info=True)
+            return False
+
+    def index_all(self, force: bool = False) -> dict[str, int]:
+        """
+        Index all MCards in the collection.
+
+        Args:
+            force: Re-index even if already indexed
+
+        Returns:
+            Dictionary with 'indexed', 'skipped', 'failed' counts
+        """
+        stats = {"indexed": 0, "skipped": 0, "failed": 0}
+
+        # Get all cards from collection
+        page = self.collection.get_all_cards()
+
+        for mcard in page.items:
+            if not force and self.vector_store.is_indexed(mcard.hash):
+                stats["skipped"] += 1
+                continue
+
+            if self.index(mcard):
+                stats["indexed"] += 1
+            else:
+                stats["failed"] += 1
+
+        logger.info(f"Indexing complete: {stats}")
+        return stats
+
+    def search(
+        self, query: str, k: int = None, hybrid: bool = True
+    ) -> list[VectorSearchResult]:
+        """
+        Search for relevant MCards.
+
+        Args:
+            query: Search query
+            k: Number of results
+            hybrid: Use hybrid (vector + FTS) search
+
+        Returns:
+            List of search results with scores
+        """
+        k = k or self.config.top_k
+
+        if hybrid and self.config.enable_hybrid_search:
+            return self.vector_store.hybrid_search(query, k)
+        else:
+            return self.vector_store.search(query, k)
+
+    def query(
+        self, question: str, k: int = None, system_prompt: str = None, model: str = None
+    ) -> RAGResponse:
+        """
+        Query with RAG: search for context and generate answer.
+
+        Args:
+            question: User question
+            k: Number of context chunks
+            system_prompt: Custom system prompt
+            model: LLM model to use
+
+        Returns:
+            RAGResponse with answer and sources
+        """
+        k = k or self.config.top_k
+
+        # Search for relevant context
+        results = self.search(question, k)
+
+        if not results:
+            return RAGResponse(
+                answer="I couldn't find relevant information in the knowledge base.",
+                sources=[],
+                source_chunks=[],
+                confidence=0.0,
+            )
+
+        # Assemble context
+        context_parts = []
+        for i, result in enumerate(results):
+            chunk = result.chunk_text or f"[Content from {result.hash[:8]}]"
+            context_parts.append(f"[{i + 1}] {chunk}")
+
+        context = "\n\n".join(context_parts)
+
+        # Generate answer
+        default_system = """You are a helpful assistant that answers questions based on the provided context.
+Use the context to provide accurate, relevant answers.
+If the context doesn't contain enough information to fully answer the question, say so.
+Always cite the source numbers [1], [2], etc. when using information from the context."""
+
+        prompt = f"""Context:
+{context}
+
+Question: {question}
+
+Please provide a comprehensive answer based on the context above."""
+
+        try:
+            from clm.cross_cutting.config.config_constants import DEFAULT_FALLBACK_MODEL
+            from clm.layer3.agency import chat_monad
+
+            response = chat_monad(
+                prompt=prompt,
+                system_prompt=system_prompt or default_system,
+                model=model or DEFAULT_FALLBACK_MODEL,
+                temperature=0.3,
+                max_tokens=1000,
+            ).unsafe_run()
+
+            if response.is_right():
+                answer = response.value.get("content", str(response.value))
+            else:
+                answer = f"Error generating response: {response.value}"
+
+        except Exception as e:
+            logger.error(f"LLM generation failed: {e}", exc_info=True)
+            answer = f"Error: {e}"
+
+        # Calculate confidence based on top scores
+        avg_score = sum(r.score for r in results) / len(results) if results else 0
+
+        from clm.cross_cutting.config.config_constants import DEFAULT_FALLBACK_MODEL
+
+        return RAGResponse(
+            answer=answer,
+            sources=[r.hash for r in results],
+            source_chunks=[r.chunk_text or "" for r in results],
+            confidence=avg_score,
+            metadata={
+                "model": model or DEFAULT_FALLBACK_MODEL,
+                "num_sources": len(results),
+                "scores": [r.score for r in results],
+            },
+        )
+
+    def get_info(self) -> dict[str, Any]:
+        """Get RAG engine information."""
+        return {
+            "config": {
+                "embedding_model": self.config.embedding_model,
+                "top_k": self.config.top_k,
+                "chunk_size": self.config.chunk_size,
+                "hybrid_search": self.config.enable_hybrid_search,
+            },
+            "vector_store": self.vector_store.get_info(),
+            "embedder_available": self.embedder.validate_connection(),
+        }

clm/layer3/rag/graph/__init__.py

@@ -0,0 +1,22 @@
+"""
+Graph Package
+
+Knowledge graph storage and extraction for GraphRAG.
+"""
+
+from .engine import GraphRAGEngine, GraphRAGResponse
+from .extractor import Entity, ExtractionResult, GraphExtractor, Relationship
+from .schema import GRAPH_SCHEMAS
+from .store import GraphStore, store_extraction_result
+
+__all__ = [
+    "GRAPH_SCHEMAS",
+    "GraphExtractor",
+    "Entity",
+    "Relationship",
+    "ExtractionResult",
+    "GraphStore",
+    "store_extraction_result",
+    "GraphRAGEngine",
+    "GraphRAGResponse",
+]