coderag-ai 0.3.0__py3-none-any.whl

coderag/core/config.py

@@ -0,0 +1,100 @@
+"""Central configuration.
+
+All tunable behaviour (ranking weights, budgets, model names, provider
+selection) lives here so there are no magic constants scattered through the
+code. Values are read from the environment / a ``.env`` file.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="CODERAG_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    # ---- Database -------------------------------------------------------
+    # A libpq URL. In dev/test the pgserver fixture overrides this.
+    database_url: str = Field(
+        default="postgresql+psycopg://coderag:coderag@localhost:5432/coderag"
+    )
+    db_echo: bool = False
+
+    # Default repository name used when a request doesn't name one (handy for the
+    # MCP server / CLI when several repositories are indexed).
+    default_repository: str | None = None
+
+    # ---- Embeddings -----------------------------------------------------
+    # "hashing"  -> deterministic, dependency-free (default; offline, great for tests)
+    # "sentence_transformer" -> local SentenceTransformers model (needs `embeddings` extra)
+    embedding_provider: str = "hashing"
+    embedding_model: str = "sentence-transformers/all-MiniLM-L6-v2"
+    embedding_device: str = "cpu"
+    embedding_batch_size: int = 32
+    # Dimension of the hashing embedder (the ST provider reports its own).
+    embedding_dimension: int = 384
+
+    # ---- Token counting -------------------------------------------------
+    # "heuristic" (offline, deterministic) or "tiktoken" (needs `tokens` extra).
+    token_counter: str = "heuristic"
+    tiktoken_encoding: str = "cl100k_base"
+
+    # ---- Retrieval / fusion --------------------------------------------
+    # Per-source candidate limits (how many each retriever returns).
+    symbol_limit: int = 10
+    lexical_limit: int = 30
+    semantic_limit: int = 30
+    # Reciprocal Rank Fusion constant and per-source weights.
+    rrf_k: int = 60
+    weight_symbol: float = 2.0
+    weight_lexical: float = 1.0
+    weight_semantic: float = 1.0
+    weight_graph: float = 0.7
+    # Max merged candidates carried into ranking.
+    max_candidates: int = 60
+
+    # ---- Graph expansion (bounded) -------------------------------------
+    graph_enabled: bool = True
+    graph_max_depth: int = 1
+    graph_max_candidates: int = 20
+    graph_max_tokens: int = 6000
+
+    # ---- Reranking (optional) ------------------------------------------
+    reranker_enabled: bool = False
+    reranker_model: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+
+    # ---- Context builder ------------------------------------------------
+    max_context_tokens: int = 12000
+    # Reserve tokens for the scaffolding/instructions of the prompt.
+    context_overhead_tokens: int = 600
+
+    # ---- LLM provider ---------------------------------------------------
+    llm_provider: str = "anthropic"  # "anthropic" | "null"
+    anthropic_api_key: str | None = None
+    anthropic_base_url: str = "https://api.anthropic.com"
+    anthropic_model: str = "claude-sonnet-5"
+    anthropic_version: str = "2023-06-01"
+    llm_max_output_tokens: int = 1024
+    llm_timeout_seconds: float = 60.0
+
+    # ---- Analyzers (Phase 10) ------------------------------------------
+    max_fix_attempts: int = 2
+
+    # ---- Security -------------------------------------------------------
+    # Extra path fragments to ignore during indexing (comma-separated env).
+    extra_ignore: list[str] = Field(default_factory=list)
+    # If True, structured logs may include short code snippets (never full files).
+    log_code_snippets: bool = False
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

coderag/core/logging.py

@@ -0,0 +1,58 @@
+"""Structured logging.
+
+Security rule (SECURITY.md): we NEVER log full source code, full prompts,
+secrets, or credentials. Log events carry identifiers (symbol id, path,
+counts) — not payloads. Helpers here make the safe path the easy path.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+
+import structlog
+
+
+class _DynamicStderrFactory:
+    """Create a PrintLogger bound to the *current* sys.stderr on each call.
+
+    Resolving the stream dynamically (with caching disabled) avoids writing to a
+    stale/closed stream — important under pytest, which swaps sys.stderr per test.
+    """
+
+    def __call__(self, *args: object) -> structlog.PrintLogger:
+        return structlog.PrintLogger(file=sys.stderr)
+
+
+def configure_logging(level: str = "INFO") -> None:
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.processors.add_log_level,
+            structlog.processors.TimeStamper(fmt="iso"),
+            structlog.processors.StackInfoRenderer(),
+            structlog.processors.format_exc_info,
+            structlog.dev.ConsoleRenderer(colors=False),
+        ],
+        wrapper_class=structlog.make_filtering_bound_logger(
+            getattr(logging, level.upper(), logging.INFO)
+        ),
+        logger_factory=_DynamicStderrFactory(),
+        cache_logger_on_first_use=False,
+    )
+
+
+def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger:
+    return structlog.get_logger(name)
+
+
+def preview(text: str, limit: int = 80) -> str:
+    """Return a short, safe single-line preview of possibly-sensitive text.
+
+    Use this instead of logging raw code/queries. Truncates and collapses
+    whitespace so full payloads never reach the logs.
+    """
+    flat = " ".join(text.split())
+    if len(flat) <= limit:
+        return flat
+    return flat[:limit] + "…"

coderag/core/text.py

@@ -0,0 +1,80 @@
+"""Identifier tokenization shared by parsing and lexical retrieval.
+
+Full-text search over code works far better if identifiers are also indexed as
+their component words: ``retry_payment`` -> ``retry_payment retry payment`` and
+``PaymentService`` -> ``PaymentService payment service``. The same expansion is
+applied to queries so ``retry payment`` matches ``retry_payment``.
+"""
+
+from __future__ import annotations
+
+import re
+
+_IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+_CAMEL_RE = re.compile(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z0-9]+|[A-Z]+|[0-9]+")
+
+# Common language keywords we don't want dominating the search document.
+_STOP = frozenset(
+    {
+        "self", "cls", "def", "class", "return", "import", "from", "as", "if",
+        "else", "elif", "for", "while", "try", "except", "finally", "with",
+        "pass", "raise", "none", "true", "false", "and", "or", "not", "in", "is",
+        "lambda", "yield", "await", "async", "global", "nonlocal", "del", "assert",
+    }
+)
+
+
+def split_identifier(identifier: str) -> list[str]:
+    """Return the identifier plus its snake/camel component words (lowercased)."""
+    out: list[str] = [identifier]
+    for part in identifier.split("_"):
+        for w in _CAMEL_RE.findall(part):
+            if w:
+                out.append(w.lower())
+    # de-dup preserving order
+    seen: set[str] = set()
+    result: list[str] = []
+    for t in out:
+        tl = t.lower()
+        if tl and tl not in seen:
+            seen.add(tl)
+            result.append(t if t == identifier else tl)
+    return result
+
+
+def extract_identifiers(code: str, limit: int = 200) -> list[str]:
+    """Unique identifiers appearing in a code snippet (order-preserving, capped)."""
+    seen: set[str] = set()
+    out: list[str] = []
+    for m in _IDENT_RE.finditer(code):
+        tok = m.group(0)
+        if tok in seen or tok.lower() in _STOP:
+            continue
+        seen.add(tok)
+        out.append(tok)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def build_search_terms(*groups: list[str], limit: int = 300) -> list[str]:
+    """Merge identifier groups into a de-duplicated, word-expanded term list."""
+    seen: set[str] = set()
+    out: list[str] = []
+    for group in groups:
+        for ident in group:
+            for term in split_identifier(ident):
+                key = term.lower()
+                if key in seen:
+                    continue
+                seen.add(key)
+                out.append(term)
+                if len(out) >= limit:
+                    return out
+    return out
+
+
+def expand_query_terms(query: str) -> list[str]:
+    """Tokenize a free-text query into word-expanded search terms."""
+    idents = _IDENT_RE.findall(query)
+    return build_search_terms(idents)

coderag/core/tokens.py

@@ -0,0 +1,66 @@
+"""Token counting.
+
+Token accounting is a primary product feature, so counting must be:
+  * deterministic and offline by default (HeuristicTokenCounter), and
+  * pluggable (TiktokenCounter) for closer estimates.
+
+IMPORTANT: these are *estimates* used for budgeting before we call the LLM.
+The authoritative token counts always come back from the LLM provider's usage
+report and are stored in ``llm_requests``. We never present an estimate as the
+real billed number.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from coderag.core.config import Settings
+
+
+class TokenCounter(Protocol):
+    def count(self, text: str) -> int: ...
+
+    @property
+    def name(self) -> str: ...
+
+
+class HeuristicTokenCounter:
+    """Offline estimator.
+
+    Uses a blend of a per-character ratio and a whitespace token count, which
+    tracks BPE tokenisation of source code reasonably well without any network
+    access or model download.
+    """
+
+    name = "heuristic"
+
+    def count(self, text: str) -> int:
+        if not text:
+            return 0
+        char_estimate = len(text) / 4.0
+        word_estimate = len(text.split()) * 1.3
+        return max(1, round((char_estimate + word_estimate) / 2))
+
+
+class TiktokenCounter:
+    """Closer estimate using tiktoken (optional dependency, may need network)."""
+
+    def __init__(self, encoding: str = "cl100k_base") -> None:
+        import tiktoken  # noqa: PLC0415 (lazy: optional dependency)
+
+        self._enc = tiktoken.get_encoding(encoding)
+        self.name = f"tiktoken:{encoding}"
+
+    def count(self, text: str) -> int:
+        if not text:
+            return 0
+        return len(self._enc.encode(text, disallowed_special=()))
+
+
+def get_token_counter(settings: Settings) -> TokenCounter:
+    if settings.token_counter == "tiktoken":
+        try:
+            return TiktokenCounter(settings.tiktoken_encoding)
+        except Exception:  # pragma: no cover - graceful offline fallback
+            return HeuristicTokenCounter()
+    return HeuristicTokenCounter()

coderag/db/base.py

@@ -0,0 +1,58 @@
+"""Engine / session management."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+from sqlalchemy import Engine, create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from coderag.core.config import Settings, get_settings
+
+_engine: Engine | None = None
+_SessionFactory: sessionmaker[Session] | None = None
+
+
+def make_engine(settings: Settings | None = None) -> Engine:
+    settings = settings or get_settings()
+    return create_engine(
+        settings.database_url,
+        echo=settings.db_echo,
+        pool_pre_ping=True,
+        future=True,
+    )
+
+
+def get_engine() -> Engine:
+    global _engine
+    if _engine is None:
+        _engine = make_engine()
+    return _engine
+
+
+def get_session_factory() -> sessionmaker[Session]:
+    global _SessionFactory
+    if _SessionFactory is None:
+        _SessionFactory = sessionmaker(bind=get_engine(), expire_on_commit=False, future=True)
+    return _SessionFactory
+
+
+def configure_engine(engine: Engine) -> None:
+    """Install a specific engine (used by tests / the pgserver fixture)."""
+    global _engine, _SessionFactory
+    _engine = engine
+    _SessionFactory = sessionmaker(bind=engine, expire_on_commit=False, future=True)
+
+
+@contextmanager
+def session_scope() -> Iterator[Session]:
+    session = get_session_factory()()
+    try:
+        yield session
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()

coderag/db/migrations/env.py

@@ -0,0 +1,77 @@
+"""Alembic environment.
+
+The database URL comes from CODERAG settings / the environment (never hardcoded),
+and the pgvector extension is created before any migration runs so that the
+``vector`` type resolves.
+"""
+
+from __future__ import annotations
+
+from alembic import context
+from sqlalchemy import text
+
+from coderag.core.config import get_settings
+from coderag.db.base import make_engine
+from coderag.db.models import Base
+
+config = context.config
+target_metadata = Base.metadata
+
+
+def render_item(type_, obj, autogen_context):
+    """Ensure pgvector's Vector type renders with its import in migrations."""
+    import pgvector.sqlalchemy
+
+    if type_ == "type" and isinstance(obj, pgvector.sqlalchemy.Vector):
+        autogen_context.imports.add("import pgvector.sqlalchemy")
+        dim = getattr(obj, "dim", None)
+        return f"pgvector.sqlalchemy.Vector({dim if dim else ''})"
+    return False
+
+
+def _url() -> str:
+    # Allow an explicit override (used by tests), else fall back to settings.
+    return config.get_main_option("sqlalchemy.url") or get_settings().database_url
+
+
+def run_migrations_offline() -> None:
+    context.configure(
+        url=_url(),
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        compare_type=True,
+        render_item=render_item,
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    engine = make_engine_from_url(_url())
+    with engine.connect() as connection:
+        connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
+        connection.commit()
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+            render_item=render_item,
+        )
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+def make_engine_from_url(url: str):
+    from sqlalchemy import create_engine
+
+    return create_engine(url, future=True)
+
+
+# Keep import used even if make_engine is unused directly.
+_ = make_engine
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

coderag/db/migrations/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision: str = ${repr(up_revision)}
+down_revision: str | None = ${repr(down_revision)}
+branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
+depends_on: str | Sequence[str] | None = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

coderag/db/migrations/versions/9a973932b94d_measure_baseline_tokens.py

@@ -0,0 +1,41 @@
+"""measure baseline tokens
+
+Revision ID: 9a973932b94d
+Revises: c8a40feeda41
+Create Date: 2026-09-01 23:20:53.372206
+"""
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision: str = '9a973932b94d'
+down_revision: str | None = 'c8a40feeda41'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # server_default backfills existing rows (these are NOT NULL); the default is
+    # then dropped so the ORM default governs new inserts. Without this, upgrading
+    # a database that already has queries/source_files rows would fail.
+    op.add_column('queries', sa.Column('baseline_tokens', sa.Integer(),
+                                       nullable=False, server_default='0'))
+    op.add_column('queries', sa.Column('baseline_files', sa.Integer(),
+                                       nullable=False, server_default='0'))
+    op.add_column('source_files', sa.Column('token_count', sa.Integer(),
+                                            nullable=False, server_default='0'))
+    op.alter_column('queries', 'baseline_tokens', server_default=None)
+    op.alter_column('queries', 'baseline_files', server_default=None)
+    op.alter_column('source_files', 'token_count', server_default=None)
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('source_files', 'token_count')
+    op.drop_column('queries', 'baseline_files')
+    op.drop_column('queries', 'baseline_tokens')
+    # ### end Alembic commands ###