velune-cli 1.0.0__py3-none-any.whl

velune/retrieval/keyword.py

@@ -0,0 +1,113 @@
+"""BM25 Lexical retrieval layer for exact keyword matches."""
+
+import logging
+import re
+import threading
+import time
+
+from rank_bm25 import BM25Okapi
+
+from velune.retrieval.schemas import RetrievalDocument, RetrievalHit, RetrievalSource
+
+logger = logging.getLogger("velune.retrieval.keyword")
+
+
+class BM25Retriever:
+    """Retrieves context using BM25 exact keyword lexical indexing."""
+
+    def __init__(self) -> None:
+        self.documents: list[RetrievalDocument] = []
+        self.corpus: list[list[str]] = []
+        self.bm25: BM25Okapi | None = None
+        self._dirty: bool = False  # True when corpus needs rebuild
+        self._rebuild_lock = threading.Lock()
+
+    @property
+    def index_size(self) -> int:
+        return len(self.documents)
+
+    def add_documents(self, docs: list[RetrievalDocument]) -> None:
+        """Appends new documents to the lexical index corpus and marks the index as dirty."""
+        for doc in docs:
+            self.documents.append(doc)
+            self.corpus.append(self._tokenize(doc.content))
+        self._dirty = True
+        self.bm25 = None  # Invalidate current index
+        logger.debug("BM25 index marked dirty: %d total documents", len(self.documents))
+
+    def add_documents_batch(self, docs: list[RetrievalDocument]) -> None:
+        """More efficient batch add — tokenizes new documents and appends them to corpus."""
+        new_tokens = [self._tokenize(doc.content) for doc in docs]
+        self.documents.extend(docs)
+        self.corpus.extend(new_tokens)
+        self._dirty = True
+        self.bm25 = None
+        logger.debug("BM25 index marked dirty: %d total documents", len(self.documents))
+
+    def _ensure_index(self) -> None:
+        """Rebuild BM25 index if dirty. Thread-safe."""
+        if not self._dirty or not self.corpus:
+            return
+        with self._rebuild_lock:
+            if not self._dirty:  # Double-check after acquiring lock
+                return
+            start_time = time.time()
+            self.bm25 = BM25Okapi(self.corpus)
+            self._dirty = False
+            elapsed = time.time() - start_time
+            logger.debug("BM25 index rebuilt: %d documents, %.3fs", len(self.documents), elapsed)
+
+    def retrieve(self, query: str, top_k: int = 10, namespace: str | None = None) -> list[RetrievalHit]:
+        """Queries the BM25 lexical index and scores candidates."""
+        if not self.documents:
+            return []
+
+        if self._dirty:
+            self._ensure_index()
+
+        if not self.bm25:
+            return []
+
+        tokens = self._tokenize(query)
+        scores = self.bm25.get_scores(tokens)
+
+        # Pair scores with documents and index ranks
+        hits: list[RetrievalHit] = []
+        for i, score in enumerate(scores):
+            doc = self.documents[i]
+
+            # Match namespace filter if provided
+            if namespace and doc.namespace != namespace:
+                continue
+
+            if score > 0.0:  # Only capture positive keyword matches
+                hits.append(
+                    RetrievalHit(
+                        document=doc,
+                        score=float(score),
+                        source=RetrievalSource.LEXICAL,
+                        rank=0
+                    )
+                )
+
+        # Sort and return top candidates
+        hits.sort(key=lambda x: x.score, reverse=True)
+
+        # Trim list and apply proper sequential ranks
+        final_hits = hits[:top_k]
+        for idx, h in enumerate(final_hits):
+            h.rank = idx + 1
+
+        return final_hits
+
+    def _tokenize(self, text: str) -> list[str]:
+        """Simplistic and quick alphanumeric tokenization ignoring standard case mappings."""
+        # Lowercase and split on non-alphanumeric boundaries
+        words = re.findall(r"\w+", text.lower())
+
+        # Remove extremely common short English stop words to filter noise
+        stop_words = {
+            "a", "an", "the", "and", "or", "but", "if", "then", "else",
+            "to", "of", "in", "for", "on", "with", "at", "by", "from", "is", "this", "that"
+        }
+        return [w for w in words if w not in stop_words and len(w) > 1]

velune/retrieval/module.py

@@ -0,0 +1,19 @@
+from velune.kernel.bootstrap import RuntimeEnvironment, SubsystemModule
+
+
+def _create_hybrid_retriever(env: RuntimeEnvironment):
+    from velune.retrieval.hybrid import HybridRetriever
+    velune_dir = env.workspace / ".velune"
+    vector_path = str(velune_dir / "qdrant_local_store")
+    semantic_tier = env.container.get("runtime.semantic_memory")
+    return HybridRetriever(location=vector_path, client=semantic_tier.client)
+
+RETRIEVAL_MODULES = [
+    SubsystemModule(
+        name="retrieval",
+        factory=_create_hybrid_retriever,
+        container_key="runtime.retrieval",
+        lifecycle_key="retrieval",
+        dependencies=["runtime.semantic_memory"],
+    )
+]

velune/retrieval/reranker.py

@@ -0,0 +1,68 @@
+"""Cross-encoder API reranking with lightweight fallback similarity scoring."""
+
+
+from velune.retrieval.schemas import RetrievalHit
+
+
+class ContextReranker:
+    """Re-scores candidate hits using lexical alignment and semantic keyword density."""
+
+    def __init__(self) -> None:
+        pass
+
+    def rerank(self, query: str, hits: list[RetrievalHit]) -> list[RetrievalHit]:
+        """Re-scores and re-ranks candidate hits based on similarity density."""
+        if not hits:
+            return []
+
+        scored_hits: list[RetrievalHit] = []
+        for hit in hits:
+            score = self._compute_alignment_score(query, hit.document.content)
+
+            # Combine the original retrieval score (BM25 or vector cosine) with our alignment score
+            # Vector cosine is between -1.0 and 1.0 (typically 0.4-0.9), BM25 can be larger.
+            # We scale the alignment score to a 0.0-1.0 range and merge.
+            merged_score = hit.score * 0.4 + score * 0.6
+
+            # Clone hit with new score
+            scored_hits.append(
+                RetrievalHit(
+                    document=hit.document,
+                    score=merged_score,
+                    source=hit.source,
+                    rank=0
+                )
+            )
+
+        # Sort by updated scores
+        scored_hits.sort(key=lambda x: x.score, reverse=True)
+
+        # Apply new ranks
+        for idx, h in enumerate(scored_hits):
+            h.rank = idx + 1
+
+        return scored_hits
+
+    def _compute_alignment_score(self, query: str, document_text: str) -> float:
+        """Computes lexical term alignment and density score (Jaccard + keyword overlaps)."""
+        q_tokens = set(query.lower().split())
+        doc_tokens = set(document_text.lower().split())
+
+        if not q_tokens or not doc_tokens:
+            return 0.0
+
+        # Jaccard similarity index
+        intersection = q_tokens.intersection(doc_tokens)
+        union = q_tokens.union(doc_tokens)
+        jaccard = len(intersection) / len(union)
+
+        # Coverage fraction (how many query words appear in document)
+        coverage = len(intersection) / len(q_tokens)
+
+        # Exact substring matches (bonus weight for sequential phrase matches)
+        phrase_bonus = 0.0
+        if query.lower() in document_text.lower():
+            phrase_bonus = 0.3
+
+        # Composite score
+        return jaccard * 0.2 + coverage * 0.5 + phrase_bonus

velune/retrieval/schemas.py

@@ -0,0 +1,59 @@
+"""Retrieval data contracts."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class RetrievalSource(StrEnum):
+    """Source of a retrieval hit."""
+
+    VECTOR = "vector"
+    LEXICAL = "lexical"
+    GRAPH = "graph"
+    MEMORY = "memory"
+
+
+class RetrievalDocument(BaseModel):
+    """Document stored in retrieval indexes."""
+
+    id: str
+    content: str
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    namespace: str = "default"
+    embedding: list[float] | None = None
+    created_at: datetime = Field(default_factory=lambda: datetime.now(tz=UTC))
+
+
+class RetrievalHit(BaseModel):
+    """A ranked retrieval hit."""
+
+    document: RetrievalDocument
+    score: float
+    source: RetrievalSource
+    rank: int = 0
+
+
+class RetrievalQuery(BaseModel):
+    """Query for hybrid retrieval."""
+
+    text: str
+    top_k: int = Field(default=10, ge=1, le=100)
+    namespace: str | None = None
+    filters: dict[str, Any] = Field(default_factory=dict)
+    vector_weight: float = Field(default=0.5, ge=0.0, le=1.0)
+    lexical_weight: float = Field(default=0.3, ge=0.0, le=1.0)
+    graph_weight: float = Field(default=0.2, ge=0.0, le=1.0)
+
+
+class RetrievalResult(BaseModel):
+    """Retrieved items plus provenance metadata."""
+
+    query: RetrievalQuery
+    hits: list[RetrievalHit] = Field(default_factory=list)
+    strategy: str = "hybrid"
+    metadata: dict[str, Any] = Field(default_factory=dict)

velune/retrieval/vector.py

@@ -0,0 +1,163 @@
+"""Vector retrieval layer using Qdrant client."""
+
+import logging
+
+from qdrant_client import QdrantClient
+from qdrant_client.http import models as qmodels
+
+from velune.retrieval.schemas import RetrievalDocument, RetrievalHit, RetrievalSource
+
+logger = logging.getLogger("velune.retrieval.vector")
+
+
+class VectorRetriever:
+    """Retrieves context from Qdrant vector database using dense embeddings."""
+
+    def __init__(self, collection_name: str = "velune_symbols", location: str = ".velune/qdrant_local_store", client: QdrantClient | None = None) -> None:
+        self.collection_name = collection_name
+        self.location = location
+        if client is not None:
+            self.client = client
+        elif location.startswith(".") or "/" in location or "\\" in location:
+            self.client = QdrantClient(path=location)
+        else:
+            self.client = QdrantClient(location=location)
+        self._detected_dimension: int | None = None
+
+        # Proactively detect existing collection dimension if it exists
+        try:
+            collections = self.client.get_collections().collections
+            existing = any(c.name == self.collection_name for c in collections)
+            if existing:
+                info = self.client.get_collection(self.collection_name)
+                self._detected_dimension = info.config.params.vectors.size
+                logger.info("Collection '%s' initialized with dimension %d", self.collection_name, self._detected_dimension)
+        except Exception:
+            pass
+
+    def _ensure_collection_for_dimension(self, dim: int) -> None:
+        """Create or verify collection for the given dimension."""
+        if self._detected_dimension == dim:
+            return  # Already configured
+
+        try:
+            collections = self.client.get_collections().collections
+            existing = next((c for c in collections if c.name == self.collection_name), None)
+
+            if existing:
+                # Check if dimension matches
+                info = self.client.get_collection(self.collection_name)
+                existing_dim = info.config.params.vectors.size
+                if existing_dim != dim:
+                    logger.warning(
+                        "Embedding dimension changed from %d to %d. "
+                        "Recreating collection '%s'. Existing vectors lost.",
+                        existing_dim, dim, self.collection_name
+                    )
+                    self.client.delete_collection(self.collection_name)
+                    existing = None
+
+            if not existing:
+                self.client.create_collection(
+                    collection_name=self.collection_name,
+                    vectors_config=qmodels.VectorParams(
+                        size=dim,
+                        distance=qmodels.Distance.COSINE
+                    )
+                )
+                logger.info("Collection '%s' initialized with dimension %d", self.collection_name, dim)
+
+            self._detected_dimension = dim
+        except Exception as e:
+            logger.error("Failed to ensure collection for dimension %d: %s", dim, e)
+
+    def upsert(self, doc: RetrievalDocument) -> None:
+        """Inserts or updates a document with its embedding in Qdrant."""
+        if not doc.embedding:
+            return
+
+        dim = len(doc.embedding)
+        self._ensure_collection_for_dimension(dim)
+
+        try:
+            self.client.upsert(
+                collection_name=self.collection_name,
+                points=[
+                    qmodels.PointStruct(
+                        id=hash(doc.id) % (2**63 - 1),  # Map string ID to uint64
+                        vector=doc.embedding,  # exact length, no padding
+                        payload={
+                            "doc_id": doc.id,
+                            "content": doc.content,
+                            "namespace": doc.namespace,
+                            **doc.metadata
+                        }
+                    )
+                ]
+            )
+        except Exception as e:
+            logger.error("Failed to upsert document: %s", e)
+
+    def retrieve(self, query_vector: list[float], top_k: int = 10, namespace: str | None = None) -> list[RetrievalHit]:
+        """Queries Qdrant vector spaces and returns matching document hits."""
+        if not self._detected_dimension:
+            try:
+                collections = self.client.get_collections().collections
+                existing = any(c.name == self.collection_name for c in collections)
+                if existing:
+                    info = self.client.get_collection(self.collection_name)
+                    self._detected_dimension = info.config.params.vectors.size
+                else:
+                    return []
+            except Exception:
+                return []
+
+        if self._detected_dimension and len(query_vector) != self._detected_dimension:
+            logger.error(
+                "Query dimension mismatch — skipping vector retrieval. "
+                "Query dimension %d != collection dimension %d.",
+                len(query_vector), self._detected_dimension
+            )
+            return []
+
+        hits: list[RetrievalHit] = []
+        try:
+            # Build filters
+            query_filter = None
+            if namespace:
+                query_filter = qmodels.Filter(
+                    must=[
+                        qmodels.FieldCondition(
+                            key="namespace",
+                            match=qmodels.MatchValue(value=namespace)
+                        )
+                    ]
+                )
+
+            results = self.client.query_points(
+                collection_name=self.collection_name,
+                query=query_vector,
+                query_filter=query_filter,
+                limit=top_k
+            ).points
+
+            for rank, res in enumerate(results):
+                payload = res.payload or {}
+                doc = RetrievalDocument(
+                    id=payload.get("doc_id", str(res.id)),
+                    content=payload.get("content", ""),
+                    namespace=payload.get("namespace", "default"),
+                    metadata={k: v for k, v in payload.items() if k not in ("doc_id", "content", "namespace")}
+                )
+                hits.append(
+                    RetrievalHit(
+                        document=doc,
+                        score=res.score,
+                        source=RetrievalSource.VECTOR,
+                        rank=rank + 1
+                    )
+                )
+        except Exception as e:
+            logger.error("Failed to retrieve matching documents from Qdrant: %s", e)
+
+        return hits

velune/telemetry/__init__.py

@@ -0,0 +1,7 @@
+"""Telemetry package for Velune CLI."""
+
+from __future__ import annotations
+
+from velune.telemetry.cognition import CognitivePerformanceAnalytics
+
+__all__ = ["CognitivePerformanceAnalytics"]

velune/telemetry/cognition.py

@@ -0,0 +1,260 @@
+"""Cognitive Performance Analytics and Dynamic Model Routing."""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from velune.memory.storage.sqlite_manager import SQLiteManager
+
+logger = logging.getLogger("velune.telemetry.cognition")
+
+
+class CognitivePerformanceAnalytics:
+    """Tracks and persists cognitive metrics like hallucination rates, rollbacks, and routes dynamically.
+
+    Routes all reads and writes through SQLiteManager to guarantee thread safety and eliminate lock contention.
+    """
+
+    def __init__(self, sqlite_manager: SQLiteManager | None = None, db_path: str | Path | None = None) -> None:
+        if sqlite_manager is not None:
+            self.sqlite_manager = sqlite_manager
+            self.db_path = sqlite_manager.db_path
+        else:
+            # Standalone mode (doctor command, tests)
+            if db_path is None:
+                self.db_path = Path(".velune") / "velune_cognitive_core.db"
+            else:
+                self.db_path = Path(db_path)
+
+            self.db_path.parent.mkdir(parents=True, exist_ok=True)
+            from velune.memory.storage.sqlite_manager import SQLiteManager
+            self.sqlite_manager = SQLiteManager(self.db_path)
+
+        self._init_db()
+
+    def _init_db(self) -> None:
+        script = """
+            CREATE TABLE IF NOT EXISTS cognitive_metrics (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp TEXT NOT NULL,
+                model_id TEXT NOT NULL,
+                task_type TEXT NOT NULL,
+                language TEXT,
+                directory TEXT,
+                hallucinated INTEGER NOT NULL, -- 0 or 1
+                rolled_back INTEGER NOT NULL,  -- 0 or 1
+                token_count INTEGER NOT NULL,
+                execution_time_ms INTEGER NOT NULL,
+                success INTEGER NOT NULL       -- 0 or 1
+            );
+            CREATE TABLE IF NOT EXISTS critic_performance_metrics (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                task_id TEXT NOT NULL,
+                critic_role TEXT NOT NULL,
+                vote INTEGER NOT NULL,      -- 1 for passed, 0 for objected
+                success INTEGER NOT NULL    -- 1 for success, 0 for failed
+            );
+            CREATE TABLE IF NOT EXISTS debate_telemetry (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp TEXT NOT NULL,
+                turns_required INTEGER NOT NULL,
+                initial_objection_count INTEGER NOT NULL,
+                final_objection_count INTEGER NOT NULL,
+                converged INTEGER NOT NULL,
+                time_to_converge_ms INTEGER NOT NULL
+            );
+            CREATE TABLE IF NOT EXISTS compression_metrics (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp TEXT NOT NULL,
+                original_tokens INTEGER NOT NULL,
+                compressed_tokens INTEGER NOT NULL,
+                method TEXT NOT NULL,
+                latency_ms INTEGER NOT NULL
+            );
+            CREATE TABLE IF NOT EXISTS injection_attempts (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp TEXT NOT NULL,
+                source TEXT NOT NULL,
+                pattern TEXT NOT NULL
+            );
+        """
+        try:
+            self.sqlite_manager.execute_script(script)
+        except (TimeoutError, RuntimeError) as e:
+            logger.critical("Failed to initialize database schema: %s", e)
+            raise
+
+    def record_metrics(
+        self,
+        model_id: str,
+        task_type: str,
+        hallucinated: bool,
+        rolled_back: bool,
+        token_count: int,
+        execution_time_ms: int,
+        success: bool,
+        language: str | None = None,
+        directory: str | None = None,
+    ) -> None:
+        """Records a single cognitive invocation metric."""
+        query = """
+            INSERT INTO cognitive_metrics (
+                timestamp, model_id, task_type, language, directory,
+                hallucinated, rolled_back, token_count, execution_time_ms, success
+            )
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """
+        params = (
+            datetime.now(tz=UTC).isoformat(),
+            model_id,
+            task_type,
+            language,
+            directory,
+            1 if hallucinated else 0,
+            1 if rolled_back else 0,
+            token_count,
+            execution_time_ms,
+            1 if success else 0,
+        )
+        self.sqlite_manager.execute_write(query, params)
+        logger.debug("Recorded cognitive metrics for model %s", model_id)
+
+    def get_model_performance(self, model_id: str) -> dict[str, Any]:
+        """Calculates performance aggregated KPIs for a specific model."""
+        query = """
+            SELECT
+                COUNT(*) as total_runs,
+                SUM(hallucinated) as total_hallucinations,
+                SUM(rolled_back) as total_rollbacks,
+                SUM(success) as total_successes,
+                AVG(token_count) as avg_tokens,
+                AVG(execution_time_ms) as avg_execution_time
+            FROM cognitive_metrics
+            WHERE model_id = ?
+        """
+        rows = self.sqlite_manager.execute_read(query, (model_id,))
+        if not rows or rows[0]["total_runs"] == 0:
+            return {
+                "total_runs": 0,
+                "hallucination_rate": 0.0,
+                "rollback_ratio": 0.0,
+                "success_rate": 0.0,
+                "avg_tokens": 0.0,
+                "avg_execution_time": 0.0,
+            }
+
+        row = rows[0]
+        total = row["total_runs"]
+        return {
+            "total_runs": total,
+            "hallucination_rate": round(float(row["total_hallucinations"] or 0) / total, 3),
+            "rollback_ratio": round(float(row["total_rollbacks"] or 0) / total, 3),
+            "success_rate": round(float(row["total_successes"] or 0) / total, 3),
+            "avg_tokens": round(float(row["avg_tokens"] or 0), 1),
+            "avg_execution_time": round(float(row["avg_execution_time"] or 0), 1),
+        }
+
+    def record_critic_vote(
+        self,
+        task_id: str,
+        critic_role: str,
+        vote: bool,
+        success: bool,
+    ) -> None:
+        """Records a single critic vote and the eventual execution success."""
+        query = """
+            INSERT INTO critic_performance_metrics (task_id, critic_role, vote, success)
+            VALUES (?, ?, ?, ?)
+        """
+        params = (task_id, critic_role, 1 if vote else 0, 1 if success else 0)
+        self.sqlite_manager.execute_write(query, params)
+        logger.debug("Recorded critic vote for task %s, role %s", task_id, critic_role)
+
+    def get_critic_weights(self) -> dict[str, float]:
+        """Returns a static default weight map for council critics."""
+        return {
+            "scalability": 1.0,
+            "security": 1.0,
+            "performance": 1.0,
+            "maintainability": 1.0,
+        }
+
+    def record_debate_outcome(
+        self,
+        turns_required: int,
+        initial_objection_count: int,
+        final_objection_count: int,
+        converged: bool,
+        time_to_converge_ms: int,
+    ) -> None:
+        """Records a single debate's performance and convergence telemetry."""
+        query = """
+            INSERT INTO debate_telemetry (
+                timestamp, turns_required, initial_objection_count,
+                final_objection_count, converged, time_to_converge_ms
+            )
+            VALUES (?, ?, ?, ?, ?, ?)
+        """
+        params = (
+            datetime.now(tz=UTC).isoformat(),
+            turns_required,
+            initial_objection_count,
+            final_objection_count,
+            1 if converged else 0,
+            time_to_converge_ms,
+        )
+        self.sqlite_manager.execute_write(query, params)
+        logger.debug(
+            "Recorded debate outcome: turns=%d, converged=%s, initial_objections=%d",
+            turns_required,
+            converged,
+            initial_objection_count,
+        )
+
+    def record_compression(
+        self,
+        original_tokens: int,
+        compressed_tokens: int,
+        method: str,
+        latency_ms: int,
+    ) -> None:
+        """Records a single context compression invocation's telemetry."""
+        query = """
+            INSERT INTO compression_metrics (
+                timestamp, original_tokens, compressed_tokens,
+                method, latency_ms
+            )
+            VALUES (?, ?, ?, ?, ?)
+        """
+        params = (
+            datetime.now(tz=UTC).isoformat(),
+            original_tokens,
+            compressed_tokens,
+            method,
+            latency_ms,
+        )
+        self.sqlite_manager.execute_write(query, params)
+        logger.debug(
+            "Recorded compression metrics: original_tokens=%d, compressed_tokens=%d, method=%s",
+            original_tokens,
+            compressed_tokens,
+            method,
+        )
+
+    def record_injection_attempt(self, source: str, pattern: str) -> None:
+        """Records a single blocked prompt injection attempt."""
+        query = """
+            INSERT INTO injection_attempts (timestamp, source, pattern)
+            VALUES (?, ?, ?)
+        """
+        params = (
+            datetime.now(tz=UTC).isoformat(),
+            source,
+            pattern,
+        )
+        self.sqlite_manager.execute_write(query, params)
+        logger.warning("Recorded prompt injection attempt from source '%s' matching pattern: %s", source, pattern)