velune-cli 1.0.0__py3-none-any.whl

velune/repository/tracker.py

@@ -0,0 +1,233 @@
+"""Git tracking, commit history, and code blame metrics."""
+
+import subprocess
+from pathlib import Path
+
+
+class GitTracker:
+    """Direct Git integration for capturing branch topology, blames, and commit volatility."""
+
+    def __init__(self, root_path: Path) -> None:
+        self.root_path = root_path.resolve()
+        self.is_git = (self.root_path / ".git").exists()
+
+    def get_active_branch(self) -> str:
+        """Returns the name of the currently checked out Git branch."""
+        if not self.is_git:
+            return "non-git"
+        try:
+            res = self._run_git(["rev-parse", "--abbrev-ref", "HEAD"])
+            return res.strip()
+        except Exception:
+            return "unknown"
+
+    def get_uncommitted_changes(self) -> list[str]:
+        """Lists all unstaged, staged, or untracked changes in the workspace."""
+        if not self.is_git:
+            return []
+        try:
+            res = self._run_git(["status", "--porcelain"])
+            changes = []
+            for line in res.splitlines():
+                if len(line) > 3:
+                    # Status code is first two characters, then space, then file path
+                    changes.append(line[3:].strip())
+            return changes
+        except Exception:
+            return []
+
+    def get_recent_commits(self, limit: int = 10) -> list[dict[str, str]]:
+        """Retrieves a list of recent commits with metadata."""
+        if not self.is_git:
+            return []
+        try:
+            # Format: hash | author | date | subject
+            res = self._run_git(["log", "-n", str(limit), "--pretty=format:%H|%an|%ad|%s", "--date=short"])
+            commits = []
+            for line in res.splitlines():
+                parts = line.split("|")
+                if len(parts) >= 4:
+                    commits.append({
+                        "hash": parts[0],
+                        "author": parts[1],
+                        "date": parts[2],
+                        "subject": parts[3]
+                    })
+            return commits
+        except Exception:
+            return []
+
+    def get_file_volatility(self, file_path: str, days: int = 90) -> int:
+        """Calculates commit volatility (number of times modified in Git) over a period."""
+        if not self.is_git:
+            return 0
+        try:
+            # Count commit entries modifying this file
+            res = self._run_git(["log", f"--since={days} days ago", "--oneline", "--", file_path])
+            return len(res.splitlines())
+        except Exception:
+            return 0
+
+    def get_blame(self, file_path: str) -> list[dict[str, str]]:
+        """Parses git blame details to index code line ownership and recency."""
+        if not self.is_git:
+            return []
+        try:
+            # git blame --porcelain file
+            res = self._run_git(["blame", "--porcelain", file_path])
+            blames = []
+            commit_data: dict[str, dict[str, str]] = {}
+            lines = res.splitlines()
+
+            i = 0
+            while i < len(lines):
+                line = lines[i]
+                parts = line.split()
+                if not parts:
+                    i += 1
+                    continue
+
+                sha = parts[0]
+                if sha not in commit_data:
+                    # Parse commit info block
+                    author = "unknown"
+                    date = "unknown"
+                    j = i + 1
+                    while j < len(lines) and not lines[j].startswith("\t"):
+                        if lines[j].startswith("author "):
+                            author = lines[j][7:]
+                        elif lines[j].startswith("author-time "):
+                            date = lines[j][12:]
+                        j += 1
+                    commit_data[sha] = {"author": author, "date": date}
+
+                # Find line contents
+                j = i + 1
+                while j < len(lines) and not lines[j].startswith("\t"):
+                    j += 1
+                if j < len(lines) and lines[j].startswith("\t"):
+                    content = lines[j][1:]
+                    blames.append({
+                        "commit": sha,
+                        "author": commit_data[sha]["author"],
+                        "date": commit_data[sha]["date"],
+                        "content": content
+                    })
+                i = j + 1
+            return blames
+        except Exception:
+            return []
+
+    def create_stash(self, name: str = "velune-snapshot") -> bool:
+        """Stashes current uncommitted modifications to prepare for validation or rollback."""
+        if not self.is_git:
+            return False
+        try:
+            self._run_git(["stash", "push", "-m", name, "--include-untracked"])
+            return True
+        except Exception:
+            return False
+
+    def pop_stash(self) -> bool:
+        """Pops the last stashed state, restoring uncommitted changes."""
+        if not self.is_git:
+            return False
+        try:
+            self._run_git(["stash", "pop"])
+            return True
+        except Exception:
+            return False
+
+    def apply_stash(self) -> bool:
+        """Applies the last stashed state, keeping it in the stash list."""
+        if not self.is_git:
+            return False
+        try:
+            self._run_git(["stash", "apply"])
+            return True
+        except Exception:
+            return False
+
+    def drop_stash(self) -> bool:
+        """Drops the last stashed state from the stash list."""
+        if not self.is_git:
+            return False
+        try:
+            self._run_git(["stash", "drop"])
+            return True
+        except Exception:
+            return False
+
+    def _run_git(self, args: list[str]) -> str:
+        """Helper to safely execute git subprocess commands in the repository root."""
+        cmd = ["git"] + args
+        res = subprocess.run(
+            cmd,
+            cwd=self.root_path,
+            capture_output=True,
+            text=True,
+            check=True,
+            encoding="utf-8",
+            errors="ignore"
+        )
+        return res.stdout
+
+    async def _run_git_async(self, args: list[str]) -> str:
+        """Async version of _run_git using asyncio.to_thread."""
+        import asyncio
+        import functools
+        cmd = ["git"] + args
+        try:
+            result = await asyncio.to_thread(
+                functools.partial(
+                    subprocess.run,
+                    cmd,
+                    cwd=self.root_path,
+                    capture_output=True,
+                    text=True,
+                    check=True,
+                    encoding="utf-8",
+                    errors="ignore",
+                )
+            )
+            return result.stdout
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"Git command failed: {' '.join(cmd)}: {e.stderr}") from e
+
+    def get_all_file_volatility(self, days: int = 90) -> dict[str, int]:
+        """Get commit counts for ALL files in a single git log call.
+
+        Returns:
+            dict mapping relative file path → commit count in last {days} days.
+            Empty dict if not a git repo.
+        """
+        if not self.is_git:
+            return {}
+        try:
+            result = self._run_git([
+                "log",
+                f"--since={days} days ago",
+                "--pretty=format:",
+                "--name-only",
+            ])
+
+            counts: dict[str, int] = {}
+            for line in result.splitlines():
+                line = line.strip()
+                if line:  # Skip empty lines (between commits)
+                    # Normalize paths to use forward-slashes (consistent across all platforms)
+                    normalized_line = line.replace("\\", "/")
+                    counts[normalized_line] = counts.get(normalized_line, 0) + 1
+            return counts
+        except subprocess.CalledProcessError as e:
+            import logging
+            logging.getLogger("velune.repository.tracker").warning(
+                "Git log batch volatility failed with process error: %s", e
+            )
+            return {}
+        except Exception as e:
+            import logging
+            logging.getLogger("velune.repository.tracker").error(
+                "Unexpected error in get_all_file_volatility: %s", e
+            )
+            return {}

velune/retrieval/__init__.py

@@ -0,0 +1,27 @@
+"""Retrieval infrastructure for BM25, Qdrant vector, and graph traversals."""
+
+from velune.retrieval.graph import GraphRetriever
+from velune.retrieval.hybrid import HybridRetriever
+from velune.retrieval.keyword import BM25Retriever
+from velune.retrieval.reranker import ContextReranker
+from velune.retrieval.schemas import (
+    RetrievalDocument,
+    RetrievalHit,
+    RetrievalQuery,
+    RetrievalResult,
+    RetrievalSource,
+)
+from velune.retrieval.vector import VectorRetriever
+
+__all__ = [
+    "HybridRetriever",
+    "BM25Retriever",
+    "VectorRetriever",
+    "GraphRetriever",
+    "ContextReranker",
+    "RetrievalDocument",
+    "RetrievalHit",
+    "RetrievalQuery",
+    "RetrievalResult",
+    "RetrievalSource",
+]

velune/retrieval/graph.py

@@ -0,0 +1,117 @@
+"""Knowledge and repository AST dependency graph traversal retriever."""
+
+from velune.retrieval.schemas import RetrievalDocument, RetrievalHit, RetrievalSource
+
+
+class GraphRetriever:
+    """Traverses knowledge graphs and repository AST structures to fetch contiguous context.
+
+    This retriever uses the **cached** repository snapshot (read from the
+    on-disk JSON index) instead of triggering a full ``repo_service.index()``
+    on every call.  That eliminates the O(N·files) AST re-parse that
+    previously happened on every retrieval request.
+
+    Cold-start behaviour (no cache on disk yet):
+      - ``get_snapshot()`` returns ``None``.
+      - ``retrieve()`` returns an empty list and logs a debug warning.
+      - The first ``velune run`` / ``velune chat`` call that triggers a full
+        index will populate the cache, after which graph retrieval works normally.
+    """
+
+    def retrieve(self, node_id: str, depth: int = 1, top_k: int = 10) -> list[RetrievalHit]:
+        """Traverses adjacent AST and symbol imports from the repository cognition service.
+
+        Uses the cached snapshot via ``get_snapshot()`` to avoid triggering a
+        full repository re-index on every call.
+        """
+        hits: list[RetrievalHit] = []
+
+        try:
+            # Resolve the active RepositoryCognitionService from the kernel container
+            try:
+                from velune.kernel.registry import get_container
+                repo_service = get_container().get("runtime.repository_cognition")
+            except (KeyError, Exception):
+                return []
+
+            if not repo_service:
+                return []
+
+            # Traverse neighboring nodes in the import/dependency call graph
+            neighbors = repo_service.traverse(node_id, depth=depth)
+
+            # Remove self from traversal to avoid duplicates
+            norm_node = node_id.replace("\\", "/")
+            neighbors = [n for n in neighbors if n != norm_node]
+
+            # --- KEY FIX: use get_snapshot() instead of index() ---
+            # get_snapshot() reads the on-disk JSON cache written by the last
+            # full index run.  It does NOT re-parse ASTs or run Git commands.
+            # Falls back gracefully to empty lists when no cache exists yet.
+            snapshot = repo_service.get_snapshot()
+            if snapshot is None:
+                # Cold start — no index on disk yet; skip graph retrieval silently
+                return []
+
+            file_map = {f.path: f for f in snapshot.files}
+            symbol_by_id = {s.symbol_id: s for s in snapshot.symbols if s.symbol_id}
+            symbol_by_qualified = {s.qualified_name: s for s in snapshot.symbols if s.qualified_name}
+            symbol_by_name = {s.name: s for s in snapshot.symbols}
+
+            rank = 1
+            for n in neighbors[:top_k]:
+                content = ""
+                metadata = {}
+                s = None
+
+                # Check if neighbor is a file path
+                if n in file_map:
+                    f = file_map[n]
+                    metadata = {
+                        "path": f.path,
+                        "language": f.language.value,
+                        "size_bytes": f.size_bytes,
+                        "sha256": f.sha256
+                    }
+                    content = f"File: {f.path}\nLanguage: {f.language.value}\nSymbols: " + ", ".join(sym.name for sym in f.symbols)
+                # Check if neighbor is a symbol (by symbol_id, qualified_name, or name)
+                elif n in symbol_by_id:
+                    s = symbol_by_id[n]
+                elif n in symbol_by_qualified:
+                    s = symbol_by_qualified[n]
+                elif n in symbol_by_name:
+                    s = symbol_by_name[n]
+
+                if s:
+                    metadata = {
+                        "name": s.name,
+                        "kind": s.kind.value,
+                        "file_path": s.file_path,
+                        "parent": s.parent or "",
+                        "symbol_id": s.symbol_id or "",
+                        "qualified_name": s.qualified_name or ""
+                    }
+                    content = f"Symbol: {s.name}\nKind: {s.kind.value}\nDefined in: {s.file_path}\nLine range: {s.line_start}-{s.line_end}"
+                    if s.docstring:
+                        content += f"\nDocstring: {s.docstring}"
+
+                if content:
+                    doc = RetrievalDocument(
+                        id=f"graph-{n}",
+                        content=content,
+                        namespace="repository_graph",
+                        metadata=metadata
+                    )
+                    hits.append(
+                        RetrievalHit(
+                            document=doc,
+                            score=1.0 / (depth + 0.1),  # Closer connections get higher heuristic weightings
+                            source=RetrievalSource.GRAPH,
+                            rank=rank
+                        )
+                    )
+                    rank += 1
+        except Exception:
+            pass
+
+        return hits

velune/retrieval/hybrid.py

@@ -0,0 +1,250 @@
+import logging
+import os
+from typing import Any
+
+logger = logging.getLogger("velune.retrieval.hybrid")
+
+
+from velune.retrieval.graph import GraphRetriever
+from velune.retrieval.keyword import BM25Retriever
+from velune.retrieval.reranker import ContextReranker
+from velune.retrieval.schemas import RetrievalHit, RetrievalQuery, RetrievalResult
+from velune.retrieval.vector import VectorRetriever
+
+
+class HybridRetriever:
+    """Orchestrates fusion retrieval, combining Lexical, Vector, and Graph traversals.
+
+    Primary interface: await retrieve(). search() is sync-only.
+    """
+
+    def __init__(self, location: str = ":memory:", client: Any | None = None) -> None:
+        self.vector_retriever = VectorRetriever(location=location, client=client)
+        self.lexical_retriever = BM25Retriever()
+        self.graph_retriever = GraphRetriever()
+        self.reranker = ContextReranker()
+
+    def add_documents(self, docs: list[Any]) -> None:
+        """Adds and indexes documents in both vector and lexical subsystems.
+
+        All documents must have a pre-computed embedding.
+        """
+        # Index in Lexical (BM25)
+        self.lexical_retriever.add_documents(docs)
+
+        # Index in Vector (Qdrant)
+        for doc in docs:
+            # Require embedding to be pre-computed (make embedding field required, not optional)
+            if not doc.embedding:
+                raise ValueError(
+                    f"Document {doc.id} must have a pre-computed embedding. "
+                    "All callers of add_documents() must pre-compute embeddings using await before calling."
+                )
+            self.vector_retriever.upsert(doc)
+
+    async def retrieve(self, query: RetrievalQuery) -> RetrievalResult:
+        """Performs full hybrid retrieval, merges candidate pools, and reranks."""
+        lexical_hits: list[RetrievalHit] = []
+        vector_hits: list[RetrievalHit] = []
+        graph_hits: list[RetrievalHit] = []
+
+        # 1. Execute Lexical search (BM25)
+        if query.lexical_weight > 0.0:
+            try:
+                lexical_hits = self.lexical_retriever.retrieve(
+                    query.text, top_k=query.top_k, namespace=query.namespace
+                )
+            except Exception:
+                pass
+
+        # 2. Execute Vector search (Qdrant)
+        if query.vector_weight > 0.0:
+            try:
+                # Generate embedding for the query
+                emb = await self._generate_embedding_async(query.text)
+                if emb is not None:  # Only do vector search if real embedding
+                    vector_hits = self.vector_retriever.retrieve(
+                        emb, top_k=query.top_k, namespace=query.namespace
+                    )
+                else:
+                    logger.info("Vector retrieval skipped — no embedding available")
+            except Exception:
+                pass
+
+        # 3. Execute Graph traversal search
+        # If we have hits from lexical or vector search, traverse neighboring file links
+        if query.graph_weight > 0.0:
+            seed_nodes = []
+            # Gather file path candidates
+            for hit in lexical_hits[:3] + vector_hits[:3]:
+                path = hit.document.metadata.get("path")
+                if path:
+                    seed_nodes.append(path)
+                name = hit.document.metadata.get("name")
+                if name:
+                    seed_nodes.append(name)
+
+            for node in set(seed_nodes):
+                try:
+                    gh = self.graph_retriever.retrieve(node, depth=1, top_k=5)
+                    graph_hits.extend(gh)
+                except Exception:
+                    pass
+
+        # 4. Fusion and Deduplication
+        merged_hits_map: dict[str, RetrievalHit] = {}
+
+        # Helper to blend weights into score
+        def merge_hit(hit: RetrievalHit, weight: float) -> None:
+            doc_id = hit.document.id
+            weighted_score = hit.score * weight
+
+            if doc_id in merged_hits_map:
+                # Combine scores from multiple search strategies
+                existing = merged_hits_map[doc_id]
+                existing.score += weighted_score
+            else:
+                hit.score = weighted_score
+                merged_hits_map[doc_id] = hit
+
+        for h in lexical_hits:
+            merge_hit(h, query.lexical_weight)
+        for h in vector_hits:
+            merge_hit(h, query.vector_weight)
+        for h in graph_hits:
+            merge_hit(h, query.graph_weight)
+
+        all_hits = list(merged_hits_map.values())
+
+        # 5. Rerank final combined candidates
+        reranked_hits = self.reranker.rerank(query.text, all_hits)
+
+        return RetrievalResult(
+            query=query,
+            hits=reranked_hits[:query.top_k],
+            strategy="hybrid-fusion-reranked"
+        )
+
+    def search_sync(self, query: RetrievalQuery) -> RetrievalResult:
+        """Synchronous retrieval. DEPRECATED: use await retrieve() in async contexts.
+
+        This method creates a new event loop for synchronous callers. Do NOT call
+        from within a running event loop — use 'await self.retrieve(query)' instead.
+
+        Raises:
+            RuntimeError: If called from within a running event loop.
+        """
+        import asyncio
+        import warnings
+        warnings.warn(
+            "HybridRetriever.search_sync() is deprecated and will be removed in a future version. "
+            "Use 'await retriever.retrieve(query)' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        try:
+            asyncio.get_running_loop()
+            raise RuntimeError(
+                "HybridRetriever.search_sync() cannot be called from an async context. "
+                "Use 'await retriever.retrieve(query)' instead."
+            )
+        except RuntimeError as e:
+            if "cannot be called" in str(e):
+                raise
+            # No running loop — safe to use asyncio.run()
+
+        return asyncio.run(self.retrieve(query))
+
+    def search(self, query: RetrievalQuery) -> RetrievalResult:
+        """Synchronous interface. Do NOT call from within a running event loop.
+        Use await retrieve() instead from async contexts."""
+        import asyncio
+        import warnings
+        try:
+            asyncio.get_running_loop()
+            warnings.warn(
+                "HybridRetriever.search() called from async context. "
+                "Use 'await retriever.retrieve()' instead.",
+                RuntimeWarning, stacklevel=2
+            )
+        except RuntimeError:
+            pass  # No running loop, safe to use asyncio.run()
+        return asyncio.run(self.retrieve(query))
+
+    async def check_embedding_available(self) -> bool:
+        """Returns True if a real embedding provider is available."""
+        try:
+            test_emb = await self._generate_embedding_async("test")
+            return test_emb is not None
+        except Exception:
+            return False
+
+    def _deterministic_fallback_embedding(self, text: str) -> list[float]:
+        """Sophisticated deterministic fallback embedding vector."""
+        res = [0.0] * 1536
+        for idx, char in enumerate(text[:300]):
+            res[idx % 1536] += ord(char) / 256.0
+        return res
+
+    async def _generate_embedding_async(self, text: str) -> list[float] | None:
+        """Generates embedding asynchronously using the registered ModelProvider, or falls back to a deterministic vector.
+
+        INTERNAL ONLY.
+        """
+        try:
+            from velune.kernel.registry import get_container
+            container = get_container()
+            if container.has("runtime.provider_registry"):
+                provider_registry = container.get("runtime.provider_registry")
+                config = container.get("runtime.config") if container.has("runtime.config") else None
+
+                provider_name = "openai"
+                if config and hasattr(config, "providers") and config.providers:
+                    provider_name = config.providers.default_provider
+
+                provider = provider_registry.get(provider_name)
+                if provider:
+                    try:
+                        caps = provider.get_capabilities()
+                        if not caps.supports_embeddings:
+                            logger.info(
+                                "Provider %s does not support embeddings. Skipping vector embedding generation.",
+                                provider_name
+                            )
+                            return None
+                    except Exception as e:
+                        logger.warning("Could not query capabilities for provider %s: %s", provider_name, e)
+
+                    model_id = "text-embedding-3-small"
+                    if provider_name == "ollama":
+                        model_id = "nomic-embed-text"
+
+                    res = await provider.embed([text], model_id=model_id)
+                    emb = res[0] if res else None
+                    if emb:
+                        logger.debug("Generated embedding: dim=%d, provider=%s", len(emb), provider_name)
+                        return emb
+        except Exception as e:
+            import logging
+            logging.getLogger("velune.retrieval.hybrid").warning(
+                "Failed to generate embedding using ModelProvider: %s. Falling back to deterministic embedding.", e
+            )
+
+        # No provider available
+        allow_fallback = os.environ.get("VELUNE_ALLOW_FALLBACK_EMBEDDING", "false").lower() == "true"
+        if allow_fallback:
+            logger.warning(
+                "Using character-frequency fallback embedding. "
+                "Semantic retrieval results will be degraded. "
+                "Install an embedding model (e.g., ollama pull nomic-embed-text) "
+                "or set OPENAI_API_KEY to enable real embeddings."
+            )
+            return self._deterministic_fallback_embedding(text)
+
+        logger.warning(
+            "No embedding provider available. Vector retrieval disabled. "
+            "Set VELUNE_ALLOW_FALLBACK_EMBEDDING=true to enable degraded mode."
+        )
+        return None
+
+