clm-kernel 0.0.0__py3-none-any.whl

clm/layer3/rag/indexer.py

@@ -0,0 +1,333 @@
+"""
+Persistent Vector Indexer
+
+Manages automatic indexing of MCards into the vector store,
+with persistent storage alongside the main MCard database.
+"""
+
+import logging
+import threading
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from clm.layer0.model.card import MCard
+from clm.layer2.card_collection import CardCollection
+
+from .config import DEFAULT_RAG_CONFIG, RAGConfig
+from .embeddings import OllamaEmbeddingProvider
+from .vector import MCardVectorStore
+
+logger = logging.getLogger(__name__)
+
+
+class PersistentIndexer:
+    """
+    Manages persistent vector indexing for MCard collections.
+
+    Features:
+    - Automatic indexing when MCards are added
+    - Persistent vector database alongside MCard database
+    - Background indexing for large collections
+    - Index status tracking
+
+    Usage:
+        from clm.layer3.rag import PersistentIndexer
+
+        indexer = PersistentIndexer()  # Uses default collection
+
+        # Index all existing content
+        stats = indexer.index_all()
+
+        # Search
+        results = indexer.search("query")
+    """
+
+    _instance = None
+    _lock = threading.Lock()
+
+    def __new__(cls, *args, **kwargs):
+        """Singleton pattern for global indexer."""
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(
+        self,
+        collection: CardCollection = None,
+        config: RAGConfig = None,
+        vector_db_path: str = None,
+        auto_index: bool = False,
+    ):
+        """
+        Initialize persistent indexer.
+
+        Args:
+            collection: MCard collection (uses default if None)
+            config: RAG configuration
+            vector_db_path: Path to vector database (auto-derived if None)
+            auto_index: Automatically index new MCards
+        """
+        if self._initialized:
+            return
+
+        from clm.layer2.card_collection import get_default_collection
+        default_collection = get_default_collection()
+
+        self.collection = collection or default_collection
+        self.config = config or DEFAULT_RAG_CONFIG
+        self.auto_index = auto_index
+
+        # Derive vector DB path from MCard DB path
+        if vector_db_path is None:
+            mcard_db_path = self._get_mcard_db_path()
+            if mcard_db_path and mcard_db_path != ":memory:":
+                base_path = Path(mcard_db_path)
+                vector_db_path = str(base_path.parent / f"{base_path.stem}_vectors.db")
+            else:
+                vector_db_path = ":memory:"
+
+        self.vector_db_path = vector_db_path
+
+        # Initialize embedding provider
+        self.embedder = OllamaEmbeddingProvider(
+            model=self.config.embedding_model, base_url=self.config.ollama_base_url
+        )
+
+        # Initialize vector store
+        self.vector_store = MCardVectorStore(
+            db_path=self.vector_db_path,
+            config=self.config,
+            embedding_provider=self.embedder,
+        )
+
+        # Track indexing state
+        self._indexing_in_progress = False
+        self._indexed_hashes: set[str] = set()
+        self._load_indexed_hashes()
+
+        self._initialized = True
+        logger.info(f"PersistentIndexer initialized: {self.vector_db_path}")
+
+    def _get_mcard_db_path(self) -> str | None:
+        """Get the path of the MCard database."""
+        try:
+            if hasattr(self.collection, "engine"):
+                engine = self.collection.engine
+                if hasattr(engine, "connection") and hasattr(
+                    engine.connection, "db_path"
+                ):
+                    return engine.connection.db_path
+        except Exception:
+            pass
+        return None
+
+    def _load_indexed_hashes(self):
+        """Load the set of already-indexed hashes."""
+        try:
+            cursor = self.vector_store.conn.cursor()
+            cursor.execute("SELECT DISTINCT hash FROM mcard_vector_metadata")
+            self._indexed_hashes = {row[0] for row in cursor.fetchall()}
+            logger.debug(f"Loaded {len(self._indexed_hashes)} indexed hashes")
+        except Exception as e:
+            logger.warning(f"Failed to load indexed hashes: {e}")
+            self._indexed_hashes = set()
+
+    def is_indexed(self, hash: str) -> bool:
+        """Check if an MCard is already indexed."""
+        return hash in self._indexed_hashes or self.vector_store.is_indexed(hash)
+
+    def index_mcard(self, mcard: MCard, force: bool = False) -> bool:
+        """
+        Index a single MCard.
+
+        Args:
+            mcard: MCard to index
+            force: Re-index even if already indexed
+
+        Returns:
+            True if indexed successfully
+        """
+        if not force and self.is_indexed(mcard.hash):
+            logger.debug(f"MCard {mcard.hash[:8]} already indexed, skipping")
+            return True
+
+        try:
+            count = self.vector_store.index(mcard)
+            if count > 0:
+                self._indexed_hashes.add(mcard.hash)
+                logger.debug(f"Indexed MCard {mcard.hash[:8]} ({count} vectors)")
+                return True
+            return False
+        except Exception as e:
+            logger.error(f"Failed to index MCard {mcard.hash[:8]}: {e}", exc_info=True)
+            return False
+
+    def index_all(
+        self,
+        force: bool = False,
+        progress_callback: Callable[[int, int], None] = None,
+        _batch_size: int = 50,
+    ) -> dict[str, int]:
+        """
+        Index all MCards in the collection.
+
+        Args:
+            force: Re-index even if already indexed
+            progress_callback: Optional callback(current, total)
+            batch_size: Number of cards to process at once
+
+        Returns:
+            Dictionary with 'indexed', 'skipped', 'failed', 'total' counts
+        """
+        if self._indexing_in_progress:
+            logger.warning("Indexing already in progress")
+            return {
+                "indexed": 0,
+                "skipped": 0,
+                "failed": 0,
+                "total": 0,
+                "status": "busy",
+            }
+
+        self._indexing_in_progress = True
+        stats = {"indexed": 0, "skipped": 0, "failed": 0, "total": 0}
+
+        try:
+            # Get all cards
+            all_cards = self.collection.get_all_mcards_raw()
+            stats["total"] = len(all_cards)
+
+            for i, mcard in enumerate(all_cards):
+                if not force and self.is_indexed(mcard.hash):
+                    stats["skipped"] += 1
+                elif self.index_mcard(mcard, force=force):
+                    stats["indexed"] += 1
+                else:
+                    stats["failed"] += 1
+
+                # Progress callback
+                if progress_callback and (i + 1) % 10 == 0:
+                    progress_callback(i + 1, stats["total"])
+
+            logger.info(f"Indexing complete: {stats}")
+
+        finally:
+            self._indexing_in_progress = False
+
+        return stats
+
+    def search(self, query: str, k: int = None, hybrid: bool = True) -> list[Any]:
+        """
+        Search for similar MCards.
+
+        Args:
+            query: Search query
+            k: Number of results
+            hybrid: Use hybrid (vector + FTS) search
+
+        Returns:
+            List of VectorSearchResult
+        """
+        if hybrid and self.config.enable_hybrid_search:
+            return self.vector_store.hybrid_search(query, k)
+        else:
+            return self.vector_store.search(query, k)
+
+    def delete(self, hash: str) -> bool:
+        """Delete an MCard from the index."""
+        count = self.vector_store.delete(hash)
+        if count > 0:
+            self._indexed_hashes.discard(hash)
+            return True
+        return False
+
+    def clear(self):
+        """Clear the entire vector index."""
+        cursor = self.vector_store.conn.cursor()
+        cursor.execute("DELETE FROM mcard_vector_metadata")
+        cursor.execute("DELETE FROM mcard_embeddings")
+        if self.config.enable_hybrid_search:
+            cursor.execute("DELETE FROM mcard_fts")
+        self.vector_store.conn.commit()
+        self._indexed_hashes.clear()
+        logger.info("Vector index cleared")
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get indexer statistics."""
+        return {
+            "vector_db_path": self.vector_db_path,
+            "embedding_model": self.embedder.model_name,
+            "dimensions": self.embedder.dimensions,
+            "indexed_count": len(self._indexed_hashes),
+            "vector_count": self.vector_store.count(),
+            "unique_mcards": self.vector_store.count_unique(),
+            "has_vec_extension": self.vector_store.has_vec_extension,
+            "hybrid_search_enabled": self.config.enable_hybrid_search,
+            "indexing_in_progress": self._indexing_in_progress,
+        }
+
+    @classmethod
+    def reset_instance(cls):
+        """Reset the singleton instance (for testing)."""
+        with cls._lock:
+            cls._instance = None
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Global Convenience Functions
+# ─────────────────────────────────────────────────────────────────────────────
+
+_default_indexer: PersistentIndexer | None = None
+
+
+def get_indexer(
+    collection: CardCollection = None, config: RAGConfig = None
+) -> PersistentIndexer:
+    """
+    Get or create the default persistent indexer.
+
+    Args:
+        collection: Optional collection override
+        config: Optional configuration override
+
+    Returns:
+        PersistentIndexer instance
+    """
+    global _default_indexer
+
+    if _default_indexer is None:
+        _default_indexer = PersistentIndexer(collection=collection, config=config)
+
+    return _default_indexer
+
+
+def semantic_search(query: str, k: int = 5) -> list[Any]:
+    """
+    Convenience function for semantic search.
+
+    Args:
+        query: Search query
+        k: Number of results
+
+    Returns:
+        List of VectorSearchResult
+    """
+    return get_indexer().search(query, k)
+
+
+def index_mcard(mcard: MCard, force: bool = False) -> bool:
+    """
+    Convenience function to index an MCard.
+
+    Args:
+        mcard: MCard to index
+        force: Re-index if exists
+
+    Returns:
+        True if indexed
+    """
+    return get_indexer().index_mcard(mcard, force)

clm/layer3/rag/llm_providers.json

@@ -0,0 +1,52 @@
+{
+    "embedding_models": {
+        "nomic-embed-text": {
+            "dimensions": 768,
+            "provider": "ollama",
+            "max_tokens": 8192,
+            "description": "High quality, general purpose embeddings"
+        },
+        "bge-m3": {
+            "dimensions": 1024,
+            "provider": "ollama",
+            "max_tokens": 8192,
+            "description": "Multi-lingual, multi-granularity embeddings"
+        },
+        "mxbai-embed-large": {
+            "dimensions": 1024,
+            "provider": "ollama",
+            "max_tokens": 512,
+            "description": "High quality embeddings for retrieval"
+        },
+        "all-minilm": {
+            "dimensions": 384,
+            "provider": "ollama",
+            "max_tokens": 256,
+            "description": "Fast, lightweight embeddings"
+        }
+    },
+    "vision_models": {
+        "moondream": {
+            "size": "1.7GB",
+            "description": "Moondream - Tiny, high-performance vision language model",
+            "capabilities": ["image_description", "visual_qa"]
+        },
+        "llama3.2-vision": {
+            "size": "7.9GB",
+            "description": "Llama 3.2 Vision - 11B multimodal model with strong OCR",
+            "capabilities": ["image_description", "ocr", "visual_qa"]
+        },
+        "llava": {
+            "size": "4.7GB",
+            "description": "LLaVA - Large Language and Vision Assistant",
+            "capabilities": ["image_description", "visual_qa"]
+        },
+        "minicpm-v": {
+            "size": "5.6GB",
+            "description": "MiniCPM-V - Efficient vision-language model",
+            "capabilities": ["image_description", "ocr", "multi_image"]
+        }
+    },
+    "default_embedding_model": "nomic-embed-text",
+    "default_vision_model": "moondream"
+}

clm/layer3/rag/semantic_versioning.py

@@ -0,0 +1,313 @@
+"""
+Semantic Versioning API for MCards
+
+High-level API functions for managing MCard versions linked to handles
+with semantic similarity detection.
+
+This module provides a simplified interface for:
+- Linking MCards to handles
+- Retrieving version history with semantic info
+- Comparing versions by semantic similarity
+- Searching within a handle's version history
+
+See: docs/architecture/Handle_Vector_Similarity_Design.md
+"""
+
+import logging
+
+from clm.layer0.model.card import MCard
+from clm.layer3.rag.config import DEFAULT_RAG_CONFIG, RAGConfig
+from clm.layer3.rag.vector import (
+    HandleVectorStore,
+    VersionSimilarityResult,
+)
+
+logger = logging.getLogger(__name__)
+
+# Default store instance (lazy-initialized)
+_default_store: HandleVectorStore | None = None
+
+
+def get_store(
+    db_path: str = None, config: RAGConfig = None, reinitialize: bool = False
+) -> HandleVectorStore:
+    """
+    Get the default HandleVectorStore instance.
+
+    Args:
+        db_path: Path to SQLite database (None = in-memory)
+        config: RAG configuration
+        reinitialize: Force re-initialization of the store
+
+    Returns:
+        HandleVectorStore instance
+    """
+    global _default_store
+
+    if _default_store is None or reinitialize:
+        _default_store = HandleVectorStore(
+            db_path=db_path, config=config or DEFAULT_RAG_CONFIG
+        )
+
+    return _default_store
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Core API Functions
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def link_mcard_to_handle(
+    mcard: MCard, handle: str, is_current: bool = True, store: HandleVectorStore = None
+) -> bool:
+    """
+    Link an MCard to a handle with semantic embedding.
+
+    This function:
+    1. Indexes the MCard content with vector embeddings
+    2. Associates the MCard with the specified handle
+    3. Computes semantic delta from previous version (if exists)
+    4. Classifies the upgrade type based on similarity
+
+    Args:
+        mcard: MCard to link
+        handle: Handle name (stable identifier)
+        is_current: Whether this becomes the current version
+        store: Optional custom HandleVectorStore instance
+
+    Returns:
+        True if successful, False if indexing failed
+
+    Example:
+        >>> from clm.layer0.model.card import MCard
+        >>> doc = MCard(content="Introduction to Machine Learning...")
+        >>> link_mcard_to_handle(doc, "ml_intro")
+        True
+    """
+    store = store or get_store()
+    indexed = store.index_with_handle(mcard, handle, is_current=is_current)
+    return indexed > 0
+
+
+def get_handle_version_history(
+    handle: str, store: HandleVectorStore = None
+) -> list[dict]:
+    """
+    Get version history for a handle with semantic info.
+
+    Args:
+        handle: Handle name to query
+        store: Optional custom HandleVectorStore instance
+
+    Returns:
+        List of version dicts with hash, order, timestamps,
+        embedding status, and upgrade type
+
+    Example:
+        >>> history = get_handle_version_history("ml_intro")
+        >>> for v in history:
+        ...     print(f"v{v['version_order']}: {v['hash'][:8]} ({v['upgrade_type']})")
+    """
+    store = store or get_store()
+    versions = store.get_handle_versions(handle)
+
+    return [
+        {
+            "hash": v.hash,
+            "version_order": v.version_order,
+            "is_current": v.is_current,
+            "created_at": v.created_at,
+            "has_embedding": v.has_embedding,
+            "parent_hash": v.parent_hash,
+            "semantic_delta": v.semantic_delta,
+            "upgrade_type": v.upgrade_type,
+        }
+        for v in versions
+    ]
+
+
+def compare_versions_by_similarity(
+    handle: str,
+    reference_hash: str = None,
+    metric: str = "cosine",
+    store: HandleVectorStore = None,
+) -> list[VersionSimilarityResult]:
+    """
+    Compare all versions of a handle by semantic similarity.
+
+    This is useful for:
+    - Understanding how content has evolved
+    - Finding which version is most similar to current
+    - Identifying major semantic drift points
+
+    Args:
+        handle: Handle name
+        reference_hash: Compare to this hash (default: current version)
+        metric: 'cosine' (higher = more similar) or 'euclidean' (lower = closer)
+        store: Optional custom HandleVectorStore instance
+
+    Returns:
+        Versions sorted by similarity to reference
+
+    Example:
+        >>> results = compare_versions_by_similarity("ml_intro")
+        >>> for r in results:
+        ...     print(f"v{r.version_order}: sim={r.similarity_to_current:.4f}")
+    """
+    store = store or get_store()
+    return store.get_versions_by_similarity(
+        handle=handle, reference_hash=reference_hash, metric=metric
+    )
+
+
+def search_within_handle(
+    handle: str, query: str, k: int = 10, store: HandleVectorStore = None
+) -> list[VersionSimilarityResult]:
+    """
+    Search semantically within a handle's version history.
+
+    Useful for finding which version of a document best matches
+    a specific concept, topic, or query.
+
+    Args:
+        handle: Handle name to filter by
+        query: Search query text
+        k: Number of results to return
+        store: Optional custom HandleVectorStore instance
+
+    Returns:
+        Matching versions sorted by query similarity
+
+    Example:
+        >>> results = search_within_handle("ml_intro", "neural networks")
+        >>> for r in results:
+        ...     print(f"v{r.version_order}: query_sim={r.similarity_to_query:.4f}")
+    """
+    store = store or get_store()
+    return store.search_handle_versions(handle=handle, query=query, k=k)
+
+
+def get_version_distances(
+    handle: str, cache: bool = True, store: HandleVectorStore = None
+) -> dict[tuple, float]:
+    """
+    Compute pairwise semantic distances between all versions.
+
+    Returns a dictionary mapping (hash_a, hash_b) tuples to
+    their cosine similarity scores.
+
+    Args:
+        handle: Handle name
+        cache: Whether to cache computed similarities
+        store: Optional custom HandleVectorStore instance
+
+    Returns:
+        Dict mapping (hash_a, hash_b) to cosine similarity [-1, 1]
+
+    Example:
+        >>> distances = get_version_distances("ml_intro")
+        >>> for (h1, h2), sim in distances.items():
+        ...     print(f"{h1[:8]} <-> {h2[:8]}: {sim:.4f}")
+    """
+    store = store or get_store()
+    return store.compute_version_distances(handle=handle, cache=cache)
+
+
+def find_most_similar_version(
+    handle: str, query: str, store: HandleVectorStore = None
+) -> VersionSimilarityResult | None:
+    """
+    Find the version most similar to a query.
+
+    Args:
+        handle: Handle name
+        query: Search query text
+        store: Optional custom HandleVectorStore instance
+
+    Returns:
+        Most similar version, or None if no versions exist
+    """
+    results = search_within_handle(handle, query, k=1, store=store)
+    return results[0] if results else None
+
+
+def get_semantic_evolution(handle: str, store: HandleVectorStore = None) -> list[dict]:
+    """
+    Get the semantic evolution of a handle's versions.
+
+    Returns version history annotated with semantic deltas,
+    showing how content has evolved over time.
+
+    Args:
+        handle: Handle name
+        store: Optional custom HandleVectorStore instance
+
+    Returns:
+        List of version dicts with evolution info
+    """
+    store = store or get_store()
+    versions = store.get_handle_versions(handle)
+
+    evolution = []
+    for v in versions:
+        entry = {
+            "hash": v.hash,
+            "version": v.version_order,
+            "is_current": v.is_current,
+            "created_at": v.created_at,
+        }
+
+        if v.parent_hash:
+            entry["parent_hash"] = v.parent_hash
+            entry["semantic_delta"] = v.semantic_delta
+            entry["upgrade_type"] = v.upgrade_type
+
+            # Interpret the delta
+            if v.semantic_delta:
+                if v.semantic_delta >= 0.95:
+                    entry["interpretation"] = "Nearly identical to parent"
+                elif v.semantic_delta >= 0.85:
+                    entry["interpretation"] = "Minor changes from parent"
+                elif v.semantic_delta >= 0.70:
+                    entry["interpretation"] = "Significant changes from parent"
+                else:
+                    entry["interpretation"] = "Major semantic shift from parent"
+        else:
+            entry["interpretation"] = "Initial version (no parent)"
+
+        evolution.append(entry)
+
+    return evolution
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Utility Functions
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def list_handles(store: HandleVectorStore = None) -> list[str]:
+    """
+    List all handles with indexed versions.
+
+    Args:
+        store: Optional custom HandleVectorStore instance
+
+    Returns:
+        List of handle names
+    """
+    store = store or get_store()
+    return store.list_handles()
+
+
+def get_store_info(store: HandleVectorStore = None) -> dict:
+    """
+    Get information about the vector store.
+
+    Args:
+        store: Optional custom HandleVectorStore instance
+
+    Returns:
+        Dict with store statistics
+    """
+    store = store or get_store()
+    return store.get_info()