codebase-index 1.6.0__py3-none-any.whl

codebase_index/config.py

@@ -0,0 +1,110 @@
+"""Configuration loading, merging, and validation.
+
+Resolution order (later wins): built-in defaults -> .claude/cache/codebase-index/config.json ->
+environment overrides (CBX_*) -> CLI flags. A stable `config_hash` is computed over indexing-
+relevant fields; when it changes, the indexer knows to rebuild affected rows (see SCHEMA.md).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Literal, Optional, Union
+
+from pydantic import BaseModel
+
+
+class ChunkConfig(BaseModel):
+    window_lines: int = 80
+    overlap_lines: int = 10
+
+
+class RetrievalConfig(BaseModel):
+    default_mode: Literal["hybrid", "fts", "symbol", "vector"] = "hybrid"
+    rrf_k: int = 60
+    token_budget: int = 1500
+    limit: int = 10
+    compact_snippets: bool = True
+    compact_min_reduction: float = 0.25
+
+
+class EmbeddingsConfig(BaseModel):
+    backend: Literal["noop", "local", "external"] = "noop"
+    enabled: bool = False
+    model: str = "all-MiniLM-L6-v2"
+    allow_external: bool = False  # external backend refused unless this is True AND a key is present
+    endpoint: Optional[str] = None
+
+
+class GraphConfig(BaseModel):
+    max_depth: int = 2
+    node_cap: int = 40
+
+
+class Config(BaseModel):
+    root: str = "."
+    languages: Union[Literal["auto"], list[str]] = "auto"
+    max_file_bytes: int = 1_048_576
+    ignore_files: list[str] = [".gitignore", ".cursorignore", ".claudeignore", ".codeindexignore"]
+    extra_ignore: list[str] = []
+    chunk: ChunkConfig = ChunkConfig()
+    retrieval: RetrievalConfig = RetrievalConfig()
+    embeddings: EmbeddingsConfig = EmbeddingsConfig()
+    graph: GraphConfig = GraphConfig()
+    redaction: dict = {"enabled": True}
+
+    def config_hash(self) -> str:
+        """Stable hash over indexing-relevant fields; drives rebuild decisions."""
+        relevant = {
+            "root": self.root,
+            "languages": self.languages,
+            "max_file_bytes": self.max_file_bytes,
+            "ignore_files": self.ignore_files,
+            "extra_ignore": self.extra_ignore,
+            "chunk": self.chunk.model_dump(),
+            "redaction": self.redaction,
+            "embeddings": {
+                "enabled": self.embeddings.enabled,
+                "backend": self.embeddings.backend,
+                "model": self.embeddings.model,
+            },
+        }
+        blob = json.dumps(relevant, sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(blob.encode("utf-8")).hexdigest()
+
+
+_ROOT_MARKERS = (".git", ".claude")
+
+
+def find_root(start: Optional[Path] = None) -> Path:
+    """Find the nearest project root marker, or fall back to the start directory."""
+    start = (start or Path.cwd()).resolve()
+    home = Path.home().resolve()
+    for candidate in (start, *start.parents):
+        if (candidate / ".git").exists():
+            return candidate
+        if candidate != home and (candidate / ".claude").exists():
+            return candidate
+    return start
+
+
+def _config_path(root: Path) -> Path:
+    return root / ".claude" / "cache" / "codebase-index" / "config.json"
+
+
+def load(root: Optional[Path] = None) -> Config:
+    """Discover the project root and return the resolved, validated Config."""
+    resolved_root = Path(root).resolve() if root is not None else find_root()
+    data: dict = {}
+    cfg_file = _config_path(resolved_root)
+    if cfg_file.is_file():
+        data = json.loads(cfg_file.read_text(encoding="utf-8"))
+
+    if "CBX_MAX_FILE_BYTES" in os.environ:
+        data["max_file_bytes"] = int(os.environ["CBX_MAX_FILE_BYTES"])
+
+    cfg = Config(**data)
+    cfg.root = str(resolved_root)
+    return cfg

codebase_index/discovery/__init__.py

@@ -0,0 +1,10 @@
+"""File discovery + ignore rules + classification.
+
+walker.py    : walk the project root, yield candidate paths.
+ignore.py    : merge .gitignore/.cursorignore/.claudeignore/.codeindexignore + built-in denylist
+               via pathspec; expose `is_ignored(path) -> bool`.
+classify.py  : language detection, binary/size/secret gates, generated-file detection.
+
+Hard guarantee: secrets, binaries, build/dependency dirs, and oversized files never leave this
+layer as indexable candidates. See docs/SECURITY.md §2.
+"""

codebase_index/discovery/classify.py

@@ -0,0 +1,151 @@
+"""Pure file classification helpers for discovery gates."""
+
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+from typing import Optional
+
+_LANG_BY_SUFFIX = {
+    ".py": "python",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".mjs": "javascript",
+    ".cjs": "javascript",
+    ".go": "go",
+    ".java": "java",
+    ".rs": "rust",
+    ".c": "c",
+    ".h": "c",
+    ".cpp": "cpp",
+    ".cc": "cpp",
+    ".cxx": "cpp",
+    ".hpp": "cpp",
+    ".hh": "cpp",
+    ".hxx": "cpp",
+    ".cs": "csharp",
+    ".rb": "ruby",
+    ".php": "php",
+    ".kt": "kotlin",
+    ".kts": "kotlin",
+    ".lua": "lua",
+    ".md": "markdown",
+    ".json": "json",
+    ".yml": "yaml",
+    ".yaml": "yaml",
+    ".toml": "toml",
+    ".sql": "sql",
+    # Config / IaC (Tier C: line-chunk + FTS, no tree-sitter spec). These were already
+    # indexed as unknown-language text; labeling them surfaces infra files in `stats`
+    # and lets agents scope searches to config without a tree-sitter grammar.
+    ".tf": "terraform",
+    ".tfvars": "terraform",
+    ".hcl": "hcl",
+    ".ini": "ini",
+    ".cfg": "ini",
+    ".conf": "ini",
+    ".properties": "ini",
+}
+
+# Extension-less or specially-named config/IaC files, matched on the lowercased
+# filename (and a `name.suffix` form, e.g. `web.Dockerfile`). Kept separate from
+# the suffix table because these carry their identity in the name, not the suffix.
+_LANG_BY_NAME = {
+    "dockerfile": "dockerfile",
+    "containerfile": "dockerfile",
+    "makefile": "make",
+    "gnumakefile": "make",
+}
+
+# Authoritative set of *code* languages routed to tree-sitter (Guardrail 1). Every entry MUST
+# have a working extraction path — a Tier-A LangSpec or the Tier-B generic walker. This is
+# enforced by tests/test_multilang_symbols.py (registry consistency), so the two registries
+# cannot silently drift. Note: yaml/json/markdown/toml/sql have grammars too but are *data/prose*
+# (Tier C) and deliberately stay on the line-chunk + FTS floor.
+#
+# `lua` here has no Tier-A spec on purpose: it exercises the Tier-B generic path end-to-end.
+_TREE_SITTER_LANGS = {
+    "python",
+    "typescript",
+    "javascript",
+    "go",
+    "java",
+    "rust",
+    "c",
+    "cpp",
+    "csharp",
+    "ruby",
+    "php",
+    "kotlin",
+    "lua",
+}
+
+_SECRET_NAMES = {
+    ".env",
+    "id_rsa",
+    "id_ed25519",
+    "credentials.json",
+    "service-account.json",
+    "secrets.json",
+}
+
+_SECRET_SUFFIXES = (".pem", ".key", ".p12", ".pfx")
+
+
+def detect_language(path: str) -> Optional[str]:
+    pure = PurePosixPath(path)
+    suffix = pure.suffix.lower()
+    if suffix:
+        lang = _LANG_BY_SUFFIX.get(suffix)
+        if lang is not None:
+            return lang
+    name = pure.name.lower()
+    if name in _LANG_BY_NAME:
+        return _LANG_BY_NAME[name]
+    # `web.Dockerfile`, `base.dockerfile`, etc.: identity is the suffix-as-name.
+    if suffix and suffix[1:] in _LANG_BY_NAME:
+        return _LANG_BY_NAME[suffix[1:]]
+    return None
+
+
+def parser_for(lang: Optional[str]) -> str:
+    return "treesitter" if lang in _TREE_SITTER_LANGS else "line"
+
+
+def is_secret_filename(path: str) -> bool:
+    name = PurePosixPath(path).name.lower()
+    if name in _SECRET_NAMES or name.startswith(".env."):
+        return True
+    return name.endswith(_SECRET_SUFFIXES)
+
+
+def looks_binary(data: bytes) -> bool:
+    return b"\x00" in data
+
+
+def is_generated(path: str) -> bool:
+    name = PurePosixPath(path).name.lower()
+    return (
+        ".generated." in name
+        or name.endswith(".generated")
+        or name.endswith(".min.js")
+        or name.endswith(".min.css")
+    )
+
+
+# Directory names that mark a test tree, and filename patterns for test modules.
+# Matched on whole path segments / filename stems — NOT a bare substring — so
+# `contest/`, `latest.py`, or `testimonials.ts` are never mistaken for tests.
+_TEST_DIRS = {"test", "tests", "__tests__", "__test__", "testing", "spec", "specs", "e2e"}
+
+
+def is_test_path(path: str) -> bool:
+    pure = PurePosixPath(path.replace("\\", "/"))
+    if any(part.lower() in _TEST_DIRS for part in pure.parts[:-1]):
+        return True
+    name = pure.name.lower()
+    stem = name.split(".", 1)[0]
+    if stem == "test" or stem.startswith("test_") or stem.endswith("_test"):
+        return True
+    return ".test." in name or ".spec." in name

codebase_index/discovery/ignore.py

@@ -0,0 +1,58 @@
+"""Layer built-in deny rules with root-level ignore files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pathspec
+
+DEFAULT_IGNORE_FILES = [".gitignore", ".cursorignore", ".claudeignore", ".codeindexignore"]
+
+BUILTIN_DENYLIST = [
+    ".git/",
+    ".hg/",
+    ".svn/",
+    ".claude/cache/codebase-index/",
+    "node_modules/",
+    "__pycache__/",
+    ".pytest_cache/",
+    ".mypy_cache/",
+    ".ruff_cache/",
+    ".venv/",
+    "venv/",
+    "build/",
+    "dist/",
+    "target/",
+    ".next/",
+]
+
+BUILTIN_DENY_DIRS = {p.rstrip("/") for p in BUILTIN_DENYLIST if p.endswith("/")}
+
+
+class IgnoreMatcher:
+    """Gitignore-style matcher for root-level ignore files and built-in denylist."""
+
+    def __init__(self, patterns: list[str]) -> None:
+        self._spec = pathspec.PathSpec.from_lines("gitignore", patterns)
+
+    @classmethod
+    def from_root(
+        cls,
+        root: Path,
+        *,
+        ignore_files: list[str] | None = None,
+        extra_ignore: list[str] | None = None,
+    ) -> "IgnoreMatcher":
+        patterns = list(BUILTIN_DENYLIST)
+        for ignore_file in ignore_files or DEFAULT_IGNORE_FILES:
+            path = root / ignore_file
+            if path.is_file():
+                patterns.extend(path.read_text(encoding="utf-8").splitlines())
+        patterns.extend(extra_ignore or [])
+        return cls(patterns)
+
+    def is_ignored(self, rel_path: str) -> bool:
+        return self._spec.match_file(rel_path.replace("\\", "/"))
+
+    def is_ignored_dir(self, dirname: str) -> bool:
+        return dirname in BUILTIN_DENY_DIRS

codebase_index/discovery/walker.py

@@ -0,0 +1,75 @@
+"""Walk the project root and yield indexable candidates."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator, Optional
+
+from ..config import Config
+from . import classify
+from .ignore import IgnoreMatcher
+
+_BINARY_SNIFF_BYTES = 4096
+
+
+@dataclass
+class Candidate:
+    path: Path
+    rel_path: str
+    size_bytes: int
+    lang: Optional[str]
+    parser: str
+    is_generated: bool
+
+
+def walk(root: Path, config: Config) -> Iterator[Candidate]:
+    root = Path(root).resolve()
+    matcher = IgnoreMatcher.from_root(
+        root,
+        ignore_files=config.ignore_files,
+        extra_ignore=config.extra_ignore,
+    )
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [
+            d
+            for d in dirnames
+            if not matcher.is_ignored_dir(d)
+            and not matcher.is_ignored(_rel(root, Path(dirpath) / d) + "/")
+        ]
+
+        for fname in filenames:
+            abs_path = Path(dirpath) / fname
+            rel = _rel(root, abs_path)
+
+            if matcher.is_ignored(rel) or classify.is_secret_filename(rel):
+                continue
+            try:
+                size = abs_path.stat().st_size
+            except OSError:
+                continue
+            if size > config.max_file_bytes:
+                continue
+            try:
+                with abs_path.open("rb") as fh:
+                    head = fh.read(_BINARY_SNIFF_BYTES)
+            except OSError:
+                continue
+            if classify.looks_binary(head):
+                continue
+
+            lang = classify.detect_language(rel)
+            yield Candidate(
+                path=abs_path,
+                rel_path=rel,
+                size_bytes=size,
+                lang=lang,
+                parser=classify.parser_for(lang),
+                is_generated=classify.is_generated(rel),
+            )
+
+
+def _rel(root: Path, path: Path) -> str:
+    return path.resolve().relative_to(root).as_posix()

codebase_index/doctor.py

@@ -0,0 +1,138 @@
+"""Safety / health self-check (docs/SECURITY.md §6).
+
+M8 scope: report enabled `codebase-index` hooks, whether the cache is gitignored, and
+index freshness. The fuller checklist (indexed-secret leak scan, oversized/binary audit,
+permissions, allowed-tools diff) is M9.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from . import scaffold
+from .config import Config
+
+Severity = Literal["high", "medium", "info"]
+
+
+@dataclass
+class Finding:
+    id: str
+    ok: bool
+    severity: Severity
+    detail: str
+
+
+def run_doctor(root: Path, config: Config) -> list[Finding]:
+    root = Path(root)
+    findings: list[Finding] = []
+
+    # 1. Is the cache gitignored? (committing the index can leak code/secrets.)
+    gitignore = root / ".gitignore"
+    covered = (
+        gitignore.exists()
+        and scaffold._CACHE_IGNORE_LINE in gitignore.read_text(encoding="utf-8")
+    )
+    findings.append(
+        Finding(
+            id="cache_gitignored",
+            ok=covered,
+            severity="high",
+            detail=(
+                "cache is gitignored"
+                if covered
+                else f"add '{scaffold._CACHE_IGNORE_LINE}' to .gitignore (run `init`)"
+            ),
+        )
+    )
+
+    # 2. Which auto-update hooks are enabled? (informational; hooks run on every edit.)
+    hooks = scaffold.enabled_hooks(root)
+    findings.append(
+        Finding(
+            id="hooks_enabled",
+            ok=bool(hooks),
+            severity="info",
+            detail="; ".join(hooks) if hooks else "no auto-update hook (run `init --with-hooks`)",
+        )
+    )
+
+    # 3. Index freshness.
+    db_path = root / scaffold.CACHE_REL / "index.sqlite"
+    if not db_path.exists():
+        findings.append(
+            Finding("index_fresh", ok=False, severity="medium", detail="no index (run `index`)")
+        )
+    else:
+        from .indexer.freshness import compute_freshness
+        from .storage.db import Database
+
+        with Database(db_path) as db:
+            fr = compute_freshness(db.conn, root, config)
+        findings.append(
+            Finding(
+                id="index_fresh",
+                ok=not fr.stale,
+                severity="medium",
+                detail=(
+                    "index is fresh"
+                    if not fr.stale
+                    else f"{fr.files_changed_since_build} file(s) changed — run `update`"
+                ),
+            )
+        )
+
+        # 4. Symbol-extraction coverage (Guardrail 2): a tree-sitter language with many files
+        #    but ~0 symbols means extraction silently failed (the original Java bug).
+        from .storage import repo
+        from .storage.db import Database
+
+        with Database(db_path) as db:
+            coverage = repo.treesitter_coverage(db.conn)
+        dead = [r["lang"] for r in coverage if r["files"] >= _ZERO_SYMBOL_FILE_THRESHOLD
+                and (r["symbols"] or 0) == 0]
+        findings.append(
+            Finding(
+                id="symbol_extraction",
+                ok=not dead,
+                severity="medium",
+                detail=(
+                    "tree-sitter languages extract symbols"
+                    if not dead
+                    else f"no symbols extracted for tree-sitter language(s): {', '.join(dead)} "
+                    "— extraction path likely broken"
+                ),
+            )
+        )
+
+        # 5. Dependency-graph coverage: Tier-B languages (grammar but no hand-tuned spec)
+        #    yield symbols but no import/inheritance edges, so refs/impact undercount.
+        from .parsers.languages import has_full_graph
+
+        tier_b = sorted({r["lang"] for r in coverage if not has_full_graph(r["lang"])})
+        findings.append(
+            Finding(
+                id="graph_coverage",
+                ok=True,
+                severity="info",
+                detail=(
+                    "all indexed languages have full dependency-graph support"
+                    if not tier_b
+                    else f"partial dependency graph for Tier-B language(s): {', '.join(tier_b)} "
+                    "— refs/impact may undercount (confirm with Grep)"
+                ),
+            )
+        )
+
+    return findings
+
+
+# Threshold above which a tree-sitter language with zero symbols is treated as broken rather
+# than just a tiny/empty repo.
+_ZERO_SYMBOL_FILE_THRESHOLD = 3
+
+
+def has_high_severity_failure(findings: list[Finding]) -> bool:
+    return any(f.severity == "high" and not f.ok for f in findings)

codebase_index/embeddings/__init__.py

@@ -0,0 +1,2 @@
+"""Opt-in, local-first embedding backends. Nothing here is imported by the base
+install unless embeddings are explicitly enabled (see SECURITY.md §4)."""

codebase_index/embeddings/backend.py

@@ -0,0 +1,67 @@
+# src/codebase_index/embeddings/backend.py
+"""Embedding backend protocol + the single gating factory.
+
+`resolve_backend` is the ONLY place a backend is constructed. It enforces
+SECURITY.md §4: external backends are refused unless `allow_external = true`,
+an API key is present in the environment, AND a warning naming the endpoint is
+emitted. When embeddings are disabled the factory returns a NoopBackend and
+imports no optional dependency.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Callable, Protocol, runtime_checkable
+
+API_KEY_ENV = "CBX_EMBEDDINGS_API_KEY"
+
+
+class EmbeddingError(RuntimeError):
+    """Raised when embeddings are misconfigured, refused, or a backend is unusable."""
+
+
+@runtime_checkable
+class EmbeddingBackend(Protocol):
+    enabled: bool
+    name: str
+    dim: int
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Return one vector (length == `dim`) per input text."""
+        ...
+
+
+def resolve_backend(cfg, warn: Callable[[str], None] = lambda _m: None) -> "EmbeddingBackend":
+    """Construct the configured backend, applying all security gates."""
+    emb = cfg.embeddings
+    if not emb.enabled or emb.backend == "noop":
+        from .noop import NoopBackend
+
+        return NoopBackend()
+
+    if emb.backend == "local":
+        from .local import LocalBackend
+
+        return LocalBackend(model_name=emb.model)
+
+    if emb.backend == "external":
+        if not emb.allow_external:
+            raise EmbeddingError(
+                "External embeddings require embeddings.allow_external = true (SECURITY.md §4)."
+            )
+        api_key = os.environ.get(API_KEY_ENV)
+        if not api_key:
+            raise EmbeddingError(
+                f"External embeddings require an API key in ${API_KEY_ENV} (SECURITY.md §4)."
+            )
+        if not emb.endpoint:
+            raise EmbeddingError("External embeddings require embeddings.endpoint to be set.")
+        warn(
+            f"[codebase-index] EXTERNAL EMBEDDINGS ENABLED — chunk text will be sent to "
+            f"{emb.endpoint}. Disable with embeddings.backend=local|noop."
+        )
+        from .external import ExternalBackend
+
+        return ExternalBackend(endpoint=emb.endpoint, api_key=api_key, model_name=emb.model)
+
+    raise EmbeddingError(f"Unknown embeddings.backend: {emb.backend!r}")

codebase_index/embeddings/external.py

@@ -0,0 +1,56 @@
+# src/codebase_index/embeddings/external.py
+"""External embedding API backend. Constructed ONLY via resolve_backend after the
+SECURITY.md §4 gates pass. The network call is isolated in a transport callable so
+it can be tested without hitting the network and swapped per provider.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Callable, Optional
+from urllib.request import Request, urlopen
+
+from .backend import EmbeddingError
+
+Transport = Callable[[str, str, str, list[str]], list[list[float]]]
+
+
+def _http_transport(endpoint: str, api_key: str, model: str, texts: list[str]) -> list[list[float]]:
+    body = json.dumps({"model": model, "input": texts}).encode("utf-8")
+    req = Request(
+        endpoint,
+        data=body,
+        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+        method="POST",
+    )
+    with urlopen(req, timeout=30) as resp:
+        payload = json.loads(resp.read().decode("utf-8"))
+    return [item["embedding"] for item in payload["data"]]
+
+
+class ExternalBackend:
+    enabled = True
+    dim: int = 0
+
+    def __init__(
+        self,
+        *,
+        endpoint: str,
+        api_key: str,
+        model_name: str,
+        transport: Optional[Transport] = None,
+    ) -> None:
+        self.name = f"external:{model_name}"
+        self.model_name = model_name
+        self._endpoint = endpoint
+        self._api_key = api_key
+        self._transport = transport or _http_transport
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+        vecs = self._transport(self._endpoint, self._api_key, self.model_name, list(texts))
+        if not vecs or not vecs[0]:
+            raise EmbeddingError("External embedding endpoint returned no vectors.")
+        self.dim = len(vecs[0])
+        return [[float(x) for x in v] for v in vecs]