coderag-ai 0.3.0__py3-none-any.whl

coderag/indexing/ignore.py

@@ -0,0 +1,101 @@
+"""Which files to index.
+
+Responsibilities:
+  * skip vendored/generated/build directories (a configurable default list),
+  * skip binary files,
+  * skip secret files (delegated to ``security.secrets``),
+  * map file extensions to a language.
+
+`.gitignore` is respected automatically for git repositories because we
+enumerate via ``git ls-files`` (tracked files only) — see ``git/repo.py``.
+"""
+
+from __future__ import annotations
+
+from coderag.security.secrets import is_secret_file
+
+DEFAULT_IGNORE_DIRS: frozenset[str] = frozenset(
+    {
+        "node_modules",
+        "vendor",
+        "dist",
+        "build",
+        ".venv",
+        "venv",
+        "__pycache__",
+        "generated",
+        "migrations",
+        ".git",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".pytest_cache",
+        ".tox",
+        ".idea",
+        ".vscode",
+        "target",
+        "site-packages",
+    }
+)
+
+BINARY_EXTENSIONS: frozenset[str] = frozenset(
+    {
+        ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".svg", ".pdf",
+        ".zip", ".gz", ".tar", ".tgz", ".bz2", ".7z", ".rar",
+        ".so", ".dylib", ".dll", ".a", ".o", ".class", ".jar", ".pyc", ".pyd",
+        ".exe", ".bin", ".wasm", ".woff", ".woff2", ".ttf", ".eot",
+        ".mp3", ".mp4", ".mov", ".avi", ".wav", ".ogg", ".webp",
+        ".parquet", ".pkl", ".pt", ".onnx", ".npy", ".npz", ".db", ".sqlite",
+    }
+)
+
+# extension -> language name
+LANGUAGE_BY_EXTENSION: dict[str, str] = {
+    ".py": "python",
+}
+
+
+def language_for_path(path: str) -> str | None:
+    dot = path.rfind(".")
+    if dot == -1:
+        return None
+    return LANGUAGE_BY_EXTENSION.get(path[dot:].lower())
+
+
+def is_binary_extension(path: str) -> bool:
+    dot = path.rfind(".")
+    return dot != -1 and path[dot:].lower() in BINARY_EXTENSIONS
+
+
+def in_ignored_dir(path: str, extra: frozenset[str] | None = None) -> bool:
+    parts = path.replace("\\", "/").split("/")
+    dirs = set(parts[:-1])
+    if dirs & DEFAULT_IGNORE_DIRS:
+        return True
+    if extra:
+        for frag in extra:
+            if frag and frag.strip("/") in dirs:
+                return True
+        for frag in extra:
+            if frag and frag in path:
+                return True
+    return False
+
+
+def should_index(path: str, extra_ignore: list[str] | None = None) -> bool:
+    """Return True if the path is an indexable source file.
+
+    Skips ignored dirs, secret files, binaries, and files of unknown language.
+    """
+    extra = frozenset(extra_ignore or [])
+    if in_ignored_dir(path, extra):
+        return False
+    if is_secret_file(path):
+        return False
+    if is_binary_extension(path):
+        return False
+    return language_for_path(path) is not None
+
+
+def looks_binary(content: bytes) -> bool:
+    """Heuristic: a NUL byte in the first chunk means binary."""
+    return b"\x00" in content[:8192]

coderag/indexing/indexer.py

@@ -0,0 +1,445 @@
+"""Repository indexing.
+
+Full index: enumerate files (git-aware, .gitignore-respecting) -> filter
+ignored/secret/binary -> redact residual secrets -> parse into symbols ->
+persist. Embedding generation is layered on in Phase 3; incremental indexing in
+Phase 8.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import time
+from dataclasses import dataclass
+
+from sqlalchemy import delete, func, select
+from sqlalchemy.orm import Session
+
+from coderag.core.config import Settings, get_settings
+from coderag.core.logging import get_logger
+from coderag.core.tokens import TokenCounter, get_token_counter
+from coderag.db.models import (
+    IndexingRun,
+    Repository,
+    SourceFile,
+    Symbol,
+    SymbolRelationship,
+)
+from coderag.git.repo import GitRepo
+from coderag.indexing.ignore import should_index
+from coderag.parsing.base import CALLS, CONTAINS, TESTS
+from coderag.parsing.registry import get_parser_for_path, module_qualified_name
+from coderag.security.secrets import redact
+
+log = get_logger("indexer")
+
+
+@dataclass
+class IndexStats:
+    files_indexed: int = 0
+    files_deleted: int = 0
+    symbols_indexed: int = 0
+    symbols_deleted: int = 0
+    files_skipped: int = 0
+    secrets_redacted: int = 0
+    embeddings_created: int = 0
+    relationships_created: int = 0
+    duration_seconds: float = 0.0
+    commit_sha: str | None = None
+
+
+def _hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def get_or_create_repository(
+    session: Session, name: str, local_path: str, url: str | None = None
+) -> Repository:
+    repo = session.scalar(select(Repository).where(Repository.name == name))
+    git = GitRepo(local_path)
+    branch = git.default_branch() if git.is_git() else "main"
+    if repo is None:
+        repo = Repository(name=name, local_path=local_path, url=url, default_branch=branch)
+        session.add(repo)
+        session.flush()
+    else:
+        repo.local_path = local_path
+        if url:
+            repo.url = url
+        repo.default_branch = branch
+    return repo
+
+
+class Indexer:
+    def __init__(
+        self,
+        session: Session,
+        settings: Settings | None = None,
+        token_counter: TokenCounter | None = None,
+        embed: bool = True,
+    ) -> None:
+        self.session = session
+        self.settings = settings or get_settings()
+        self.tokens = token_counter or get_token_counter(self.settings)
+        self.extra_ignore = self.settings.extra_ignore
+        self.embed = embed
+
+    # -- full index -------------------------------------------------------
+    def full_index(self, repo: Repository) -> tuple[IndexingRun, IndexStats]:
+        started = time.perf_counter()
+        git = GitRepo(repo.local_path)
+        commit = git.current_commit()
+        run = IndexingRun(repository_id=repo.id, mode="full", status="running",
+                          to_commit=commit)
+        self.session.add(run)
+        self.session.flush()
+
+        # Full reindex: clear prior data for this repository (cascades to
+        # symbols/embeddings/relationships via FK ON DELETE CASCADE).
+        self.session.execute(
+            delete(SourceFile).where(SourceFile.repository_id == repo.id)
+        )
+        self.session.execute(delete(Symbol).where(Symbol.repository_id == repo.id))
+
+        # accumulate cross-file graph edges, resolved after all symbols exist
+        self._pending_contains: list[tuple[int, int]] = []
+        self._pending_rels: list[tuple[int, str, str, float, dict | None]] = []
+
+        stats = IndexStats(commit_sha=commit)
+        try:
+            for rel_path in git.list_files():
+                if not should_index(rel_path, self.extra_ignore):
+                    stats.files_skipped += 1
+                    continue
+                self._index_file(repo, rel_path, commit, git, stats)
+
+            stats.relationships_created = self._persist_relationships(repo)
+
+            if self.embed:
+                stats.embeddings_created = self._embed(repo)
+
+            stats.duration_seconds = time.perf_counter() - started
+            run.status = "success"
+            run.files_indexed = stats.files_indexed
+            run.symbols_indexed = stats.symbols_indexed
+            run.embeddings_created = stats.embeddings_created
+            run.duration_seconds = stats.duration_seconds
+
+            run.finished_at = func.now()
+            repo.indexed_commit_sha = commit
+            repo.updated_at = func.now()
+            log.info(
+                "full_index.done", repository=repo.name,
+                files=stats.files_indexed, symbols=stats.symbols_indexed,
+                skipped=stats.files_skipped, redactions=stats.secrets_redacted,
+                seconds=round(stats.duration_seconds, 3),
+            )
+        except Exception as exc:  # pragma: no cover - defensive
+            run.status = "failed"
+            run.error = str(exc)[:500]
+            log.error("full_index.failed", repository=repo.name, error=str(exc)[:200])
+            raise
+        return run, stats
+
+    # -- incremental index ------------------------------------------------
+    def incremental_index(self, repo: Repository) -> tuple[IndexingRun, IndexStats]:
+        """Update only what changed since ``repo.indexed_commit_sha``.
+
+        Symbols/embeddings for unchanged files are preserved (unchanged code is
+        never re-embedded). The relationship graph is rebuilt from a fast parse
+        pass so cross-file edges stay correct. Falls back to a full index when
+        there is no prior commit or the path is not a git repo.
+        """
+        git = GitRepo(repo.local_path)
+        base = repo.indexed_commit_sha
+        head = git.current_commit()
+        if not git.is_git() or not base:
+            return self.full_index(repo)
+
+        started = time.perf_counter()
+        run = IndexingRun(repository_id=repo.id, mode="incremental", status="running",
+                          from_commit=base, to_commit=head)
+        self.session.add(run)
+        self.session.flush()
+
+        self._pending_contains = []
+        self._pending_rels = []
+        stats = IndexStats(commit_sha=head)
+        try:
+            diff = git.diff(base, head or "HEAD")
+            changed = [p for p in diff.changed if should_index(p, self.extra_ignore)]
+            gone = set(changed) | set(diff.deleted)
+            for path in gone:
+                self._delete_file(repo, path, stats)
+
+            for path in changed:
+                self._index_file(repo, path, head, git, stats)
+
+            # rebuild the whole-repo graph: collect edges from unchanged files too
+            qual_to_id = self._qualified_index(repo)
+            changed_set = set(changed)
+            for path in self._indexed_paths(repo):
+                if path not in changed_set:
+                    self._collect_edges(repo, path, git, qual_to_id)
+            self.session.execute(
+                delete(SymbolRelationship).where(
+                    SymbolRelationship.repository_id == repo.id
+                )
+            )
+            stats.relationships_created = self._persist_relationships(repo)
+
+            if self.embed:
+                stats.embeddings_created = self._embed(repo)
+
+            stats.duration_seconds = time.perf_counter() - started
+
+            run.status = "success"
+            run.files_indexed = stats.files_indexed
+            run.files_deleted = stats.files_deleted
+            run.symbols_indexed = stats.symbols_indexed
+            run.symbols_deleted = stats.symbols_deleted
+            run.embeddings_created = stats.embeddings_created
+            run.duration_seconds = stats.duration_seconds
+            run.finished_at = func.now()
+            repo.indexed_commit_sha = head
+            repo.updated_at = func.now()
+            log.info(
+                "incremental_index.done", repository=repo.name,
+                changed=len(changed), deleted=len(diff.deleted),
+                symbols_added=stats.symbols_indexed, symbols_deleted=stats.symbols_deleted,
+                embedded=stats.embeddings_created, seconds=round(stats.duration_seconds, 3),
+            )
+        except Exception as exc:  # pragma: no cover - defensive
+            run.status = "failed"
+            run.error = str(exc)[:500]
+            log.error("incremental_index.failed", repository=repo.name, error=str(exc)[:200])
+            raise
+        return run, stats
+
+    def _delete_file(self, repo: Repository, path: str, stats: IndexStats) -> None:
+        n = self.session.scalar(
+            select(func.count()).select_from(Symbol).where(
+                Symbol.repository_id == repo.id, Symbol.file_path == path
+            )
+        ) or 0
+        existed = self.session.scalar(
+            select(SourceFile.id).where(
+                SourceFile.repository_id == repo.id, SourceFile.path == path
+            )
+        ) is not None
+        self.session.execute(
+            delete(Symbol).where(Symbol.repository_id == repo.id, Symbol.file_path == path)
+        )
+        self.session.execute(
+            delete(SourceFile).where(
+                SourceFile.repository_id == repo.id, SourceFile.path == path
+            )
+        )
+        if existed:
+            stats.files_deleted += 1
+        stats.symbols_deleted += int(n)
+
+    def _qualified_index(self, repo: Repository) -> dict[str, int]:
+        return {
+            qual: sid
+            for sid, qual in self.session.execute(
+                select(Symbol.id, Symbol.qualified_name).where(
+                    Symbol.repository_id == repo.id
+                )
+            ).all()
+        }
+
+    def _indexed_paths(self, repo: Repository) -> list[str]:
+        return list(
+            self.session.scalars(
+                select(SourceFile.path).where(SourceFile.repository_id == repo.id)
+            )
+        )
+
+    def _collect_edges(
+        self, repo: Repository, path: str, git: GitRepo, qual_to_id: dict[str, int]
+    ) -> None:
+        """Parse an unchanged file for its edges only, mapping to existing ids."""
+        raw = git.read_text(path)
+        parser = get_parser_for_path(path)
+        if raw is None or parser is None:
+            return
+        result = parser.parse(module_qualified_name(path), redact(raw).text)
+        local_to_db = {
+            ps.local_id: qual_to_id.get(ps.qualified_name) for ps in result.symbols
+        }
+        for ps in result.symbols:
+            db_id = local_to_db.get(ps.local_id)
+            if db_id is None:
+                continue
+            if ps.parent_local_id is not None:
+                parent_db = local_to_db.get(ps.parent_local_id)
+                if parent_db is not None:
+                    self._pending_contains.append((parent_db, db_id))
+        for pr in result.relationships:
+            src_db = local_to_db.get(pr.source_local_id)
+            if src_db is not None:
+                self._pending_rels.append(
+                    (src_db, pr.relationship_type, pr.target_name, pr.confidence,
+                     pr.metadata)
+                )
+
+    # -- relationships ----------------------------------------------------
+    def _persist_relationships(self, repo: Repository) -> int:
+        rows = self.session.execute(
+            select(
+                Symbol.id, Symbol.symbol_name, Symbol.qualified_name,
+                Symbol.parent_symbol_id, Symbol.file_path,
+            ).where(Symbol.repository_id == repo.id)
+        ).all()
+        by_qual: dict[str, int] = {}
+        by_name: dict[str, list[int]] = {}
+        info: dict[int, tuple[int | None, str, str]] = {}
+        for sid, name, qual, parent_id, path in rows:
+            by_qual[qual.lower()] = sid
+            by_name.setdefault(name.lower(), []).append(sid)
+            info[sid] = (parent_id, name, path)
+
+        def resolve(target_name: str, via: str | None, source_id: int) -> int | None:
+            t = target_name.lower()
+            if t in by_qual:
+                return by_qual[t]
+            cands = by_name.get(t.split(".")[-1], [])
+            if not cands:
+                return None
+            if via == "self":
+                parent = info[source_id][0]
+                same = [c for c in cands if info[c][0] == parent]
+                if len(same) == 1:
+                    return same[0]
+            return cands[0] if len(cands) == 1 else None
+
+        def is_test(sid: int) -> bool:
+            _parent, name, path = info[sid]
+            p = path.replace("\\", "/")
+            return (
+                name.lower().startswith("test")
+                or "/tests/" in p
+                or p.startswith("tests/")
+                or p.rsplit("/", 1)[-1].startswith("test_")
+            )
+
+        edges: dict[tuple[int, int | None, str], tuple[float, dict | None]] = {}
+
+        def add(source: int, target: int | None, rtype: str, conf: float, meta):
+            key = (source, target, rtype)
+            if key not in edges or conf > edges[key][0]:
+                edges[key] = (conf, meta)
+
+        for parent_db, child_db in self._pending_contains:
+            add(parent_db, child_db, CONTAINS, 1.0, None)
+
+        for src, rtype, target_name, conf, meta in self._pending_rels:
+            via = (meta or {}).get("via") if meta else None
+            tid = resolve(target_name, via, src)
+            if tid is None or tid == src:
+                continue  # keep the graph correct: only in-repo, non-self-loop edges
+            add(src, tid, rtype, conf, meta)
+            if rtype == CALLS and is_test(src):
+                add(src, tid, TESTS, min(conf, 0.8), None)
+
+        for (source, target, rtype), (conf, meta) in edges.items():
+            self.session.add(
+                SymbolRelationship(
+                    repository_id=repo.id, source_symbol_id=source,
+                    target_symbol_id=target, relationship_type=rtype,
+                    confidence=conf, meta=meta,
+                )
+            )
+        self.session.flush()
+        return len(edges)
+
+    def _embed(self, repo: Repository) -> int:
+        from coderag.embeddings.pipeline import EmbeddingPipeline
+        from coderag.embeddings.registry import get_embedding_provider
+
+        provider = get_embedding_provider(self.settings)
+        return EmbeddingPipeline(self.session, provider).embed_repository(repo)
+
+    # -- per-file ---------------------------------------------------------
+    def _index_file(
+        self, repo: Repository, rel_path: str, commit: str | None,
+        git: GitRepo, stats: IndexStats,
+    ) -> None:
+        raw = git.read_text(rel_path)
+        if raw is None:
+            stats.files_skipped += 1
+            return
+        red = redact(raw)
+        if red.count:
+            stats.secrets_redacted += red.count
+            log.warning("redacted_secret", path=rel_path, count=red.count,
+                        labels=red.labels)
+        content = red.text
+
+        parser = get_parser_for_path(rel_path)
+        if parser is None:
+            stats.files_skipped += 1
+            return
+
+        sf = SourceFile(
+            repository_id=repo.id, path=rel_path, language=parser.language,
+            content_hash=_hash(content), size_bytes=len(content.encode("utf-8")),
+            token_count=self.tokens.count(content),
+            indexed_commit_sha=commit,
+        )
+        self.session.add(sf)
+        self.session.flush()
+
+        result = parser.parse(module_qualified_name(rel_path), content)
+        local_to_db: dict[int, int] = {}
+        for ps in result.symbols:
+            parent_db = (
+                local_to_db.get(ps.parent_local_id)
+                if ps.parent_local_id is not None
+                else None
+            )
+            sym = Symbol(
+                repository_id=repo.id,
+                source_file_id=sf.id,
+                file_path=rel_path,
+                language=parser.language,
+                symbol_name=ps.symbol_name,
+                qualified_name=ps.qualified_name,
+                symbol_type=ps.symbol_type,
+                signature=ps.signature,
+                start_line=ps.start_line,
+                end_line=ps.end_line,
+                parent_symbol_id=parent_db,
+                source_code=ps.source_code,
+                docstring=ps.docstring,
+                source_hash=ps.source_hash,
+                commit_sha=commit,
+                token_count=self.tokens.count(ps.source_code),
+                search_document=" ".join(ps.search_terms),
+            )
+            self.session.add(sym)
+            self.session.flush()
+            local_to_db[ps.local_id] = sym.id
+            if parent_db is not None:
+                self._pending_contains.append((parent_db, sym.id))
+            stats.symbols_indexed += 1
+
+        for pr in result.relationships:
+            src_db = local_to_db.get(pr.source_local_id)
+            if src_db is not None:
+                self._pending_rels.append(
+                    (src_db, pr.relationship_type, pr.target_name, pr.confidence, pr.metadata)
+                )
+        stats.files_indexed += 1
+
+
+def index_repository(
+    session: Session, name: str, local_path: str, url: str | None = None,
+    settings: Settings | None = None,
+) -> tuple[Repository, IndexingRun, IndexStats]:
+    """Convenience: register (or update) a repository and run a full index."""
+    repo = get_or_create_repository(session, name, local_path, url)
+    indexer = Indexer(session, settings=settings)
+    run, stats = indexer.full_index(repo)
+    return repo, run, stats

coderag/llm/anthropic.py

@@ -0,0 +1,117 @@
+"""Anthropic Claude provider over the Messages API (HTTP).
+
+Because the transport is plain HTTP via httpx, the same provider adapts to the
+Anthropic API, an internal proxy (set ``anthropic_base_url``), or — with a
+subclass swapping auth/signing — AWS Bedrock. Credentials come from config and
+are never logged.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Iterator
+
+import httpx
+
+from coderag.core.config import Settings, get_settings
+from coderag.llm.base import LLMProvider, LLMRequest, LLMResponse, Usage
+
+
+class AnthropicClaudeProvider(LLMProvider):
+    name = "anthropic"
+
+    def __init__(
+        self,
+        settings: Settings | None = None,
+        client: httpx.Client | None = None,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        self.settings = settings or get_settings()
+        if not self.settings.anthropic_api_key and client is None and transport is None:
+            raise RuntimeError(
+                "CODERAG_ANTHROPIC_API_KEY is not set. Retrieval works without an LLM; "
+                "only `ask` needs credentials."
+            )
+        self._client = client or httpx.Client(
+            base_url=self.settings.anthropic_base_url,
+            timeout=self.settings.llm_timeout_seconds,
+            transport=transport,
+            headers={
+                "x-api-key": self.settings.anthropic_api_key or "",
+                "anthropic-version": self.settings.anthropic_version,
+                "content-type": "application/json",
+            },
+        )
+        self._last_usage = Usage()
+
+    def _body(self, request: LLMRequest, stream: bool) -> dict:
+        return {
+            "model": request.model or self.settings.anthropic_model,
+            "max_tokens": request.max_tokens or self.settings.llm_max_output_tokens,
+            "temperature": request.temperature,
+            "system": request.system,
+            "messages": [{"role": "user", "content": request.prompt}],
+            "stream": stream,
+        }
+
+    def generate(self, request: LLMRequest) -> LLMResponse:
+        started = time.perf_counter()
+        model = request.model or self.settings.anthropic_model
+        try:
+            resp = self._client.post("/v1/messages", json=self._body(request, stream=False))
+            resp.raise_for_status()
+            data = resp.json()
+            text = "".join(
+                block.get("text", "")
+                for block in data.get("content", [])
+                if block.get("type") == "text"
+            )
+            usage_raw = data.get("usage", {})
+            usage = Usage(
+                input_tokens=usage_raw.get("input_tokens", 0),
+                output_tokens=usage_raw.get("output_tokens", 0),
+                cached_input_tokens=usage_raw.get("cache_read_input_tokens"),
+                model=data.get("model", model),
+                latency_ms=(time.perf_counter() - started) * 1000,
+                success=True,
+            )
+            self._last_usage = usage
+            return LLMResponse(text=text, usage=usage)
+        except Exception as exc:
+            usage = Usage(
+                model=model, latency_ms=(time.perf_counter() - started) * 1000,
+                success=False, error=type(exc).__name__ + ": " + str(exc)[:200],
+            )
+            self._last_usage = usage
+            raise
+
+    def generate_stream(self, request: LLMRequest) -> Iterator[str]:
+        started = time.perf_counter()
+        model = request.model or self.settings.anthropic_model
+        usage = Usage(model=model)
+        with self._client.stream(
+            "POST", "/v1/messages", json=self._body(request, stream=True)
+        ) as resp:
+            resp.raise_for_status()
+            for line in resp.iter_lines():
+                if not line or not line.startswith("data:"):
+                    continue
+                payload = line[len("data:") :].strip()
+                if payload in ("", "[DONE]"):
+                    continue
+                event = json.loads(payload)
+                etype = event.get("type")
+                if etype == "message_start":
+                    usage.input_tokens = (
+                        event.get("message", {}).get("usage", {}).get("input_tokens", 0)
+                    )
+                elif etype == "content_block_delta":
+                    delta = event.get("delta", {})
+                    if delta.get("type") == "text_delta":
+                        yield delta.get("text", "")
+                elif etype == "message_delta":
+                    usage.output_tokens = event.get("usage", {}).get("output_tokens", 0)
+        usage.latency_ms = (time.perf_counter() - started) * 1000
+        usage.success = True
+        self._last_usage = usage