clm-kernel 0.0.0__py3-none-any.whl

clm/layer3/rag/cli.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+"""
+MCard RAG CLI
+
+Command-line interface for RAG operations.
+
+Usage:
+    mcard-rag index              # Index all MCards
+    mcard-rag search "query"     # Semantic search
+    mcard-rag query "question"   # RAG query with LLM
+    mcard-rag status             # Show indexing status
+    mcard-rag clear              # Clear vector index
+"""
+
+import argparse
+import logging
+import sys
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+logger = logging.getLogger(__name__)
+
+
+def cmd_status(args):
+    """Show RAG system status."""
+    from clm.layer3.rag import get_indexer
+    from clm.layer3.rag.embeddings import OllamaEmbeddingProvider
+
+    logger.info("=" * 60)
+    logger.info("MCard RAG System Status")
+    logger.info("=" * 60)
+
+    # Check embedding provider
+    provider = OllamaEmbeddingProvider()
+    available = provider.validate_connection()
+    logger.info(f"\n📦 Embedding Model: {provider.model_name}")
+    logger.info(f"   Status: {'✅ Available' if available else '❌ Not available'}")
+    logger.info(f"   Dimensions: {provider.dimensions}")
+
+    # Get indexer stats
+    indexer = get_indexer()
+    stats = indexer.get_stats()
+
+    logger.info("\n📊 Vector Index:")
+    logger.info(f"   Database: {stats['vector_db_path']}")
+    logger.info(f"   Indexed MCards: {stats['unique_mcards']}")
+    logger.info(f"   Total Vectors: {stats['vector_count']}")
+    logger.info(
+        f"   sqlite-vec: {'✅' if stats['has_vec_extension'] else '❌ (fallback mode)'}"
+    )
+    logger.info(f"   Hybrid Search: {'✅' if stats['hybrid_search_enabled'] else '❌'}")
+
+    # Show MCard collection count
+    from clm.layer2.card_collection import get_default_collection
+    default_collection = get_default_collection()
+
+    total_mcards = default_collection.count()
+    indexed = stats["unique_mcards"]
+    pending = total_mcards - indexed
+
+    logger.info("\n📁 Collection:")
+    logger.info(f"   Total MCards: {total_mcards}")
+    logger.info(f"   Indexed: {indexed}")
+    logger.info(f"   Pending: {pending}")
+
+    if pending > 0:
+        logger.info(f"\n💡 Run 'mcard-rag index' to index {pending} pending cards")
+
+
+def cmd_index(args):
+    """Index all MCards."""
+    from clm.layer3.rag import get_indexer
+
+    logger.info("=" * 60)
+    logger.info("Indexing MCards")
+    logger.info("=" * 60)
+
+    indexer = get_indexer()
+
+    def progress(current, total):
+        pct = (current / total) * 100 if total > 0 else 0
+        print(f"\r  Progress: {current}/{total} ({pct:.1f}%)", end="", flush=True)
+
+    logger.info("\n📊 Starting index...")
+    stats = indexer.index_all(force=args.force, progress_callback=progress)
+
+    logger.info("\n\n✅ Indexing Complete:")
+    logger.info(f"   Indexed: {stats['indexed']}")
+    logger.info(f"   Skipped: {stats['skipped']}")
+    logger.info(f"   Failed: {stats['failed']}")
+    logger.info(f"   Total: {stats['total']}")
+
+
+def cmd_search(args):
+    """Semantic search."""
+    from clm.layer3.rag import semantic_search
+
+    logger.info("=" * 60)
+    logger.info(f"Semantic Search: {args.query}")
+    logger.info("=" * 60)
+
+    results = semantic_search(args.query, k=args.k)
+
+    if not results:
+        logger.info("\n❌ No results found")
+        return
+
+    logger.info(f"\n📋 Results ({len(results)} found):\n")
+
+    for i, result in enumerate(results):
+        logger.info(f"[{i + 1}] Score: {result.score:.4f}")
+        logger.info(f"    Hash: {result.hash}")
+        if result.chunk_text:
+            preview = result.chunk_text[:200].replace("\n", " ")
+            logger.info(f"    Preview: {preview}...")
+        logger.info("")
+
+
+def cmd_query(args):
+    """RAG query with LLM."""
+    from clm.layer3.rag import MCardRAGEngine, get_indexer
+
+    logger.info("=" * 60)
+    logger.info(f"RAG Query: {args.question}")
+    logger.info("=" * 60)
+
+    # Use persistent indexer's vector store
+    indexer = get_indexer()
+
+    rag = MCardRAGEngine(vector_db_path=indexer.vector_db_path)
+    rag.vector_store = indexer.vector_store
+
+    logger.info("\n🔍 Searching for context...")
+    response = rag.query(args.question, k=args.k, model=args.model)
+
+    logger.info(f"\n💬 Answer:\n{response.answer}")
+    logger.info(f"\n📊 Confidence: {response.confidence:.2%}")
+    logger.info(f"📚 Sources: {[h[:8] for h in response.sources]}")
+
+    if args.verbose:
+        logger.info("\n📝 Source Chunks:")
+        for i, chunk in enumerate(response.source_chunks):
+            if chunk:
+                logger.info(f"  [{i + 1}] {chunk[:150]}...")
+
+
+def cmd_clear(args):
+    """Clear vector index."""
+    from clm.layer3.rag import get_indexer
+
+    if not args.yes:
+        confirm = input("Are you sure you want to clear the vector index? [y/N]: ")
+        if confirm.lower() != "y":
+            logger.info("Cancelled")
+            return
+
+    indexer = get_indexer()
+    indexer.clear()
+    logger.info("✅ Vector index cleared")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="MCard RAG CLI",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Status command
+    status_parser = subparsers.add_parser("status", help="Show RAG system status")
+    status_parser.set_defaults(func=cmd_status)
+
+    # Index command
+    index_parser = subparsers.add_parser("index", help="Index all MCards")
+    index_parser.add_argument(
+        "--force", "-f", action="store_true", help="Re-index existing cards"
+    )
+    index_parser.set_defaults(func=cmd_index)
+
+    # Search command
+    search_parser = subparsers.add_parser("search", help="Semantic search")
+    search_parser.add_argument("query", help="Search query")
+    search_parser.add_argument(
+        "-k", type=int, default=5, help="Number of results (default: 5)"
+    )
+    search_parser.set_defaults(func=cmd_search)
+
+    # Query command
+    query_parser = subparsers.add_parser("query", help="RAG query with LLM")
+    query_parser.add_argument("question", help="Question to answer")
+    query_parser.add_argument(
+        "-k", type=int, default=5, help="Number of context chunks (default: 5)"
+    )
+    query_parser.add_argument(
+        "--model",
+        "-m",
+        default="qwen3:0.6b",
+        help="LLM model (default: qwen3:0.6b)",
+    )
+    query_parser.add_argument(
+        "--verbose", "-v", action="store_true", help="Show source chunks"
+    )
+    query_parser.set_defaults(func=cmd_query)
+
+    # Clear command
+    clear_parser = subparsers.add_parser("clear", help="Clear vector index")
+    clear_parser.add_argument(
+        "--yes", "-y", action="store_true", help="Skip confirmation"
+    )
+    clear_parser.set_defaults(func=cmd_clear)
+
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        return
+
+    try:
+        args.func(args)
+    except Exception as e:
+        logger.error(f"Error: {e}", exc_info=True)
+        if "--debug" in sys.argv:
+            raise
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

clm/layer3/rag/config.py

@@ -0,0 +1,153 @@
+import json
+from dataclasses import dataclass
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Load Centralized LLM Provider Configuration
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def _load_llm_config():
+    """Load models from centralized JSON file."""
+    from pathlib import Path
+
+    local_path = Path(__file__).resolve().parent / "llm_providers.json"
+    if local_path.exists():
+        try:
+            with open(local_path, encoding="utf-8") as f:
+                return json.load(f)
+        except Exception as e:
+            print(f"Warning: Failed to load {local_path}: {e}")
+
+    try:
+        from clm.cross_cutting.config.config_constants import _find_project_root
+        root_dir = _find_project_root()
+        config_path = root_dir / "llm_providers.json"
+        if config_path.exists():
+            with open(config_path, encoding="utf-8") as f:
+                return json.load(f)
+    except Exception:
+        pass
+
+    curr = Path(__file__).resolve()
+    for parent in list(curr.parents):
+        cand = parent / "llm_providers.json"
+        if cand.exists():
+            try:
+                with open(cand, encoding="utf-8") as f:
+                    return json.load(f)
+            except Exception:
+                pass
+
+    # Fallback if file missing or error
+    return {
+        "embedding_models": {
+            "nomic-embed-text": {
+                "dimensions": 768,
+                "provider": "ollama",
+                "max_tokens": 8192,
+            }
+        },
+        "vision_models": {"moondream": {"description": "Fallback Moondream"}},
+        "default_embedding_model": "nomic-embed-text",
+        "default_vision_model": "moondream",
+    }
+
+
+_RAW_CONFIG = _load_llm_config()
+
+EMBEDDING_MODELS = _RAW_CONFIG.get("embedding_models", {})
+DEFAULT_EMBEDDING_MODEL = _RAW_CONFIG.get("default_embedding_model", "nomic-embed-text")
+
+VISION_MODELS = _RAW_CONFIG.get("vision_models", {})
+DEFAULT_VISION_MODEL = _RAW_CONFIG.get("default_vision_model", "moondream")
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Vector Search Configuration
+# ─────────────────────────────────────────────────────────────────────────────
+
+# Distance metrics supported by sqlite-vec
+DISTANCE_METRICS = ("L2", "cosine", "L1")
+DEFAULT_DISTANCE_METRIC = "cosine"
+
+# Default search parameters
+DEFAULT_TOP_K = 5
+MAX_TOP_K = 100
+
+# Chunking configuration
+DEFAULT_CHUNK_SIZE = 1000  # characters
+DEFAULT_CHUNK_OVERLAP = 200
+
+# ─────────────────────────────────────────────────────────────────────────────
+# RAG Configuration Dataclass
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@dataclass
+class RAGConfig:
+    """
+    Configuration for MCard RAG system.
+
+    Attributes:
+        embedding_model: Name of the embedding model to use
+        embedding_provider: Provider for embeddings (ollama, openai, etc.)
+        ollama_base_url: Base URL for Ollama API
+        vector_db_path: Path to vector database (None = use main MCard db)
+        distance_metric: Distance metric for similarity search
+        top_k: Default number of results to return
+        chunk_size: Maximum characters per chunk
+        chunk_overlap: Overlap between chunks
+        enable_hybrid_search: Enable FTS + vector hybrid search
+        rerank_results: Re-rank results using cross-encoder
+    """
+
+    # Embedding configuration
+    embedding_model: str = DEFAULT_EMBEDDING_MODEL
+    embedding_provider: str = "ollama"
+    ollama_base_url: str = "http://localhost:11434"
+
+    # Vector storage
+    vector_db_path: str | None = None  # None = same as MCard db
+    distance_metric: str = DEFAULT_DISTANCE_METRIC
+
+    # Search parameters
+    top_k: int = DEFAULT_TOP_K
+    similarity_threshold: float = 0.0  # Minimum similarity (0-1)
+
+    # Chunking
+    chunk_size: int = DEFAULT_CHUNK_SIZE
+    chunk_overlap: int = DEFAULT_CHUNK_OVERLAP
+
+    # Advanced features
+    enable_hybrid_search: bool = True
+    rerank_results: bool = False
+
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if self.embedding_model not in EMBEDDING_MODELS:
+            raise ValueError(
+                f"Unknown embedding model: {self.embedding_model}. "
+                f"Available: {list(EMBEDDING_MODELS.keys())}"
+            )
+
+        if self.distance_metric not in DISTANCE_METRICS:
+            raise ValueError(
+                f"Unknown distance metric: {self.distance_metric}. "
+                f"Available: {DISTANCE_METRICS}"
+            )
+
+        if self.top_k < 1 or self.top_k > MAX_TOP_K:
+            raise ValueError(f"top_k must be between 1 and {MAX_TOP_K}")
+
+    @property
+    def dimensions(self) -> int:
+        """Get embedding dimensions for the configured model."""
+        return EMBEDDING_MODELS[self.embedding_model]["dimensions"]
+
+    @property
+    def max_tokens(self) -> int:
+        """Get max tokens for the configured model."""
+        return EMBEDDING_MODELS[self.embedding_model]["max_tokens"]
+
+
+# Default configuration
+DEFAULT_RAG_CONFIG = RAGConfig()

clm/layer3/rag/embeddings/__init__.py

@@ -0,0 +1,16 @@
+"""
+Embedding Providers Package
+
+Provides embedding generation capabilities for RAG.
+"""
+
+from .base import EmbeddingProvider
+from .ollama import OllamaEmbeddingProvider
+from .vision import VISION_MODELS, VisionEmbeddingProvider
+
+__all__ = [
+    "EmbeddingProvider",
+    "OllamaEmbeddingProvider",
+    "VisionEmbeddingProvider",
+    "VISION_MODELS",
+]

clm/layer3/rag/embeddings/base.py

@@ -0,0 +1,79 @@
+"""
+Embedding Provider Base Class
+
+Abstract interface for embedding providers.
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass
+class EmbeddingResult:
+    """Result from an embedding operation."""
+
+    vector: list[float]
+    model: str
+    dimensions: int
+    token_count: int | None = None
+
+
+class EmbeddingProvider(ABC):
+    """
+    Abstract base class for embedding providers.
+
+    Implementations should handle:
+    - Connection to the embedding service
+    - Text preprocessing (truncation, etc.)
+    - Batch processing for efficiency
+    - Error handling and retries
+    """
+
+    provider_name: str = "base"
+    model_name: str
+    dimensions: int
+
+    @abstractmethod
+    def embed(self, text: str) -> list[float]:
+        """
+        Generate embedding vector for a single text.
+
+        Args:
+            text: Input text to embed
+
+        Returns:
+            List of floats representing the embedding vector
+        """
+        pass
+
+    @abstractmethod
+    def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """
+        Generate embeddings for multiple texts.
+
+        Args:
+            texts: List of input texts
+
+        Returns:
+            List of embedding vectors
+        """
+        pass
+
+    @abstractmethod
+    def validate_connection(self) -> bool:
+        """
+        Check if the embedding service is available.
+
+        Returns:
+            True if connection is valid
+        """
+        pass
+
+    def get_info(self) -> dict:
+        """Get provider information."""
+        return {
+            "provider": self.provider_name,
+            "model": self.model_name,
+            "dimensions": self.dimensions,
+            "available": self.validate_connection(),
+        }

clm/layer3/rag/embeddings/ollama.py

@@ -0,0 +1,211 @@
+"""
+Ollama Embedding Provider
+
+Generates embeddings using Ollama's local embedding models.
+"""
+
+import json
+import logging
+import urllib.error
+import urllib.request
+
+from clm.cross_cutting.config.config_constants import (
+    RAG_OLLAMA_TIMEOUT_SECS,
+    RAG_OLLAMA_VALIDATE_TIMEOUT_SECS,
+)
+from clm.cross_cutting.utils.url_safety import safe_urlopen
+
+from ..config import DEFAULT_EMBEDDING_MODEL, EMBEDDING_MODELS
+from .base import EmbeddingProvider
+
+logger = logging.getLogger(__name__)
+
+
+class OllamaEmbeddingProvider(EmbeddingProvider):
+    """
+    Embedding provider using Ollama's embedding API.
+
+    Supports models like:
+    - nomic-embed-text (768 dimensions)
+    - bge-m3 (1024 dimensions)
+    - mxbai-embed-large (1024 dimensions)
+    - all-minilm (384 dimensions)
+
+    Usage:
+        provider = OllamaEmbeddingProvider('nomic-embed-text')
+        vector = provider.embed("Hello world")
+    """
+
+    provider_name = "ollama"
+
+    def __init__(
+        self,
+        model: str = DEFAULT_EMBEDDING_MODEL,
+        base_url: str = "http://localhost:11434",
+        timeout: int = RAG_OLLAMA_TIMEOUT_SECS,
+    ):
+        """
+        Initialize Ollama embedding provider.
+
+        Args:
+            model: Ollama embedding model name
+            base_url: Ollama API base URL
+            timeout: Request timeout in seconds
+        """
+        self.model_name = model
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+
+        # Get dimensions from config or default
+        if model in EMBEDDING_MODELS:
+            self.dimensions = EMBEDDING_MODELS[model]["dimensions"]
+        else:
+            # Unknown model, will determine dimensions on first embed
+            self.dimensions = 0
+            logger.warning(
+                f"Unknown model '{model}', dimensions will be determined on first embed"
+            )
+
+    def _make_request(self, endpoint: str, data: dict) -> dict:
+        """Make HTTP request to Ollama API."""
+        url = f"{self.base_url}{endpoint}"
+
+        try:
+            request = urllib.request.Request(
+                url,
+                data=json.dumps(data).encode("utf-8"),
+                headers={"Content-Type": "application/json"},
+                method="POST",
+            )
+
+            with safe_urlopen(request, timeout=self.timeout) as response:
+                return json.loads(response.read().decode("utf-8"))
+
+        except urllib.error.HTTPError as e:
+            error_body = e.read().decode("utf-8") if e.fp else str(e)
+            raise RuntimeError(f"Ollama HTTP error {e.code}: {error_body}") from e
+        except urllib.error.URLError as e:
+            raise RuntimeError(f"Ollama connection error: {e.reason}") from e
+        except Exception as e:
+            raise RuntimeError(f"Ollama request failed: {e}") from e
+
+    def embed(self, text: str) -> list[float]:
+        """
+        Generate embedding for a single text.
+
+        Args:
+            text: Input text to embed
+
+        Returns:
+            Embedding vector as list of floats
+        """
+        if not text or not text.strip():
+            raise ValueError("Cannot embed empty text")
+
+        response = self._make_request(
+            "/api/embed",
+            {
+                "model": self.model_name,
+                "input": text,
+            },
+        )
+
+        if "embeddings" in response and response["embeddings"]:
+            embedding = response["embeddings"][0]
+
+            # Update dimensions if unknown
+            if self.dimensions == 0:
+                self.dimensions = len(embedding)
+                logger.info(
+                    f"Determined {self.model_name} dimensions: {self.dimensions}"
+                )
+
+            return embedding
+        else:
+            raise RuntimeError(f"Unexpected Ollama response: {response}")
+
+    def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """
+        Generate embeddings for multiple texts.
+
+        Ollama supports batch embedding with the 'input' parameter as a list.
+
+        Args:
+            texts: List of input texts
+
+        Returns:
+            List of embedding vectors
+        """
+        if not texts:
+            return []
+
+        # Filter empty texts
+        valid_texts = [t for t in texts if t and t.strip()]
+        if not valid_texts:
+            raise ValueError("All texts are empty")
+
+        response = self._make_request(
+            "/api/embed",
+            {
+                "model": self.model_name,
+                "input": valid_texts,
+            },
+        )
+
+        if "embeddings" in response:
+            embeddings = response["embeddings"]
+
+            # Update dimensions if unknown
+            if self.dimensions == 0 and embeddings:
+                self.dimensions = len(embeddings[0])
+
+            return embeddings
+        else:
+            raise RuntimeError(f"Unexpected Ollama batch response: {response}")
+
+    def validate_connection(self) -> bool:
+        """Check if Ollama is running and model is available."""
+        try:
+            # Check if server is running
+            request = urllib.request.Request(f"{self.base_url}/api/tags", method="GET")
+
+            with safe_urlopen(
+                request, timeout=RAG_OLLAMA_VALIDATE_TIMEOUT_SECS
+            ) as response:
+                data = json.loads(response.read().decode("utf-8"))
+
+                # Check if model is available
+                if "models" in data:
+                    available_models = [m.get("name", "") for m in data["models"]]
+
+                    # Check exact match or partial match
+                    for model in available_models:
+                        if self.model_name in model or model in self.model_name:
+                            return True
+
+                    logger.warning(
+                        f"Model '{self.model_name}' not found. "
+                        f"Available: {available_models[:5]}..."
+                    )
+                    return False
+
+                return True
+
+        except Exception as e:
+            logger.debug(f"Ollama validation failed: {e}")
+            return False
+
+    def pull_model(self) -> bool:
+        """Pull the embedding model from Ollama."""
+        try:
+            self._make_request(
+                "/api/pull",
+                {
+                    "name": self.model_name,
+                    "stream": False,
+                },
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to pull model {self.model_name}: {e}", exc_info=True)
+            return False