SourceIndex 0.1.0__py3-none-any.whl

sourceindex/lib/llm.py

@@ -0,0 +1,298 @@
+"""LiteLLM-backed LLM adapter for sourceindex.
+
+Single touch-point for every LLM call in build + search. Accepts any model
+string LiteLLM understands:
+
+    anthropic/claude-haiku-4-5            (direct Anthropic)
+    openrouter/anthropic/claude-haiku-4-5 (Anthropic via OpenRouter)
+    openrouter/deepseek/deepseek-chat     (DeepSeek via OpenRouter)
+    deepseek/deepseek-chat                (direct DeepSeek)
+    gemini/gemini-2.5-flash               (direct Gemini)
+
+The bare ``claude-haiku-4-5`` form (no provider prefix) is also accepted —
+LiteLLM treats it as Anthropic.
+
+Prompts are sent as flat user messages — no explicit prompt caching.
+Providers that auto-cache (DeepSeek, Gemini, OpenAI) still do so server-side;
+the adapter just doesn't ask for it.
+"""
+
+import os
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+
+from .log import get_logger
+from .timing import elapsed_ms
+
+_log = get_logger(__name__)
+
+
+@contextmanager
+def auth_error_to_systemexit(*, index_dir: Path | None = None, api_key: str | None = None):
+    """Wrap a block that may raise ``litellm.AuthenticationError`` and convert
+    it to a single ``SystemExit`` with the backend-aware translated message.
+    Used by every CLI subcommand that fires LLM calls. ``api_key`` is the key
+    actually used for the call — needed so the routing diagnostic matches
+    reality when ``--api-key`` was passed without setting the env var."""
+    import litellm
+    try:
+        yield
+    except litellm.AuthenticationError as e:
+        raise SystemExit(translate_auth_error(e, index_dir=index_dir, api_key=api_key))
+
+
+def translate_auth_error(e: Exception, *, index_dir: Path | None = None, api_key: str | None = None) -> str:
+    """Build a user-facing message for an LLM auth failure.
+
+    LiteLLM relabels auth errors by the *model prefix* (``openrouter/...`` →
+    ``OpenrouterException``) regardless of who actually replied. When the call
+    was routed through the hosted backend, the rejector is our own Worker —
+    not OpenRouter — and the user needs to be pointed at SOURCEINDEX_API_KEY,
+    not their provider account. Surface that distinction here so the caller
+    can raise SystemExit with a message that matches reality.
+    """
+    from .backend import backend_base
+    base = backend_base(api_key=api_key)
+    env_hint = f" or the persisted key in {index_dir / '.env'}" if index_dir else ""
+    if base:
+        return (
+            f"[sourceindex] Hosted backend at {base} rejected the request — "
+            f"likely an invalid SOURCEINDEX_API_KEY{env_hint}. "
+            f"To talk to a provider directly with your own key instead, set "
+            f"SOURCEINDEX_BACKEND_URL=\"\" to opt out of the backend. "
+            f"(underlying: {e})"
+        )
+    return (
+        f"[sourceindex] Provider rejected credentials. Check SOURCEINDEX_API_KEY"
+        f"{env_hint}. (provider: {e})"
+    )
+
+
+def resolve_api_key(cli_arg: str | None = None) -> str | None:
+    """Pick up the API key from (in order): explicit CLI arg, or
+    SOURCEINDEX_API_KEY in the environment."""
+    return cli_arg or os.environ.get("SOURCEINDEX_API_KEY")
+
+# Strip SOCKS proxy from the environment *before* importing litellm — it
+# inherits any ALL_PROXY=socks://... via httpx, which can't speak SOCKS
+# without an extension and crashes the model-cost-map fetch at import time
+# (and later, completion calls). The previous anthropic-SDK path worked
+# around this with httpx.Client(trust_env=False); the closest LiteLLM
+# equivalent is just removing the leak from this process's env.
+for _k in ("ALL_PROXY", "all_proxy"):
+    if os.environ.get(_k, "").lower().startswith("socks"):
+        os.environ.pop(_k, None)
+
+import litellm  # noqa: E402  (must follow the env scrub above)
+
+
+@dataclass
+class LLMUsage:
+    """Per-call token + cost accounting. Aggregated by build.state into
+    build_meta.json. Search-side calls drop usage on the floor."""
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_read_tokens: int = 0
+    cache_creation_tokens: int = 0
+    cost_usd: float = 0.0
+
+    def add(self, other: "LLMUsage") -> None:
+        self.input_tokens += other.input_tokens
+        self.output_tokens += other.output_tokens
+        self.cache_read_tokens += other.cache_read_tokens
+        self.cache_creation_tokens += other.cache_creation_tokens
+        self.cost_usd += other.cost_usd
+
+
+def model_name_candidates(model: str | None) -> list[str | None]:
+    """Yield model-name forms to try against litellm's pricing JSON.
+
+    LiteLLM keys its model_cost map by canonical names (``gpt-5.4-nano``,
+    ``claude-haiku-4-5``) — but we call through router-prefixed forms like
+    ``openrouter/openai/gpt-5.4-nano``. Cost lookup needs to peel each
+    leading provider prefix until it lands on a known key. Empty/None
+    model returns ``[None]`` so the caller falls back to inferring from
+    the response object.
+    """
+    if not model:
+        return [None]
+    out: list[str | None] = [model]
+    parts = model.split("/")
+    for i in range(1, len(parts)):
+        out.append("/".join(parts[i:]))
+    return out
+
+
+def lookup_pricing(model: str | None) -> dict | None:
+    """Find the pricing entry in ``litellm.model_cost`` by trying each
+    progressively-stripped form of ``model``. Returns the first hit
+    (a dict of per-token cost fields), or ``None`` if no form matches."""
+    if not model:
+        return None
+    for candidate in model_name_candidates(model):
+        if not candidate:
+            continue
+        info = litellm.model_cost.get(candidate)
+        if info:
+            return info
+    return None
+
+
+def _extract_usage(response, *, model: str | None = None) -> LLMUsage:
+    """Best-effort token + cost extraction. Missing fields default to 0 —
+    accounting is a soft signal, not load-bearing.
+
+    Pricing is computed from ``litellm.model_cost`` directly rather than
+    via ``litellm.completion_cost``. The latter binds the lookup to the
+    response's ``custom_llm_provider`` (e.g. ``openrouter``), and
+    ``openrouter+claude-haiku-4-5`` isn't a key in the price map even
+    though bare ``claude-haiku-4-5`` is. Manual lookup with
+    prefix-stripping bypasses that binding.
+    """
+    usage = getattr(response, "usage", None)
+    if usage is None:
+        return LLMUsage()
+
+    def _g(name: str) -> int:
+        v = getattr(usage, name, None)
+        return int(v) if v is not None else 0
+
+    inp = _g("prompt_tokens")
+    out = _g("completion_tokens")
+    cr = _g("cache_read_input_tokens")
+    cc = _g("cache_creation_input_tokens")
+
+    cost = 0.0
+    info = lookup_pricing(model)
+    if info:
+        cost = (
+            inp * info.get("input_cost_per_token", 0.0)
+            + out * info.get("output_cost_per_token", 0.0)
+            + cr * info.get("cache_read_input_token_cost", 0.0)
+            + cc * info.get("cache_creation_input_token_cost", 0.0)
+        )
+    elif model:
+        _log.debug(
+            "LLM cost lookup failed",
+            extra={
+                "event": "llm.cost_unknown",
+                "model": model,
+                "tried": [c for c in model_name_candidates(model) if c],
+            },
+        )
+
+    return LLMUsage(
+        input_tokens=_g("prompt_tokens"),
+        output_tokens=_g("completion_tokens"),
+        cache_read_tokens=_g("cache_read_input_tokens"),
+        cache_creation_tokens=_g("cache_creation_input_tokens"),
+        cost_usd=cost,
+    )
+
+
+# A self-signed cert from a localhost MITM proxy (e.g. the ``inspect`` proxy
+# used during evals) would otherwise fail TLS verification. Match the gated
+# behaviour of the previous Anthropic-SDK call site.
+_PROXY_URL = (
+    os.environ.get("HTTPS_PROXY")
+    or os.environ.get("https_proxy")
+    or ""
+)
+if "127.0.0.1" in _PROXY_URL or "localhost" in _PROXY_URL:
+    litellm.ssl_verify = False
+
+# Silently drop standardized params the upstream provider doesn't support
+# (e.g. ``reasoning_effort`` on DeepSeek). Without this, LiteLLM raises
+# UnsupportedParamsError and the call fails.
+litellm.drop_params = True
+
+
+def call_llm(
+    prompt: str,
+    *,
+    model: str,
+    api_key: str,
+    max_tokens: int = 16384,
+) -> tuple[str | None, LLMUsage]:
+    """Sync LiteLLM completion. Returns (text, usage).
+
+    ``text`` is the stripped assistant text, or ``None`` if empty. ``usage``
+    is always an ``LLMUsage`` (zeroed when the provider returned no usage
+    block) so callers can sum unconditionally without None-checking.
+
+    Callers MUST pass ``api_key`` explicitly — there is no implicit env-var
+    fallback inside this function. Resolve it via ``resolve_api_key()``
+    (CLI arg → ``SOURCEINDEX_API_KEY``) and pass the result in.
+    """
+    assert api_key, (
+        "call_llm requires an explicit api_key. Use resolve_api_key() "
+        "(CLI arg or SOURCEINDEX_API_KEY) and pass the result in."
+    )
+    # Pin OpenRouter to the model's official primary provider — disables
+    # fallback to resold/quantized/rate-limited alternates. Quality stays
+    # consistent and slow detours are avoided.
+    #
+    # NOTE on reasoning: do NOT send ``reasoning.max_tokens=0``. Empirically
+    # (gpt-5.4-nano via OpenRouter, 2026-05-04) sending that flag *triggers*
+    # ~2.5K reasoning tokens per call and pushes wall time from ~4s → ~23s.
+    # The default behaviour with no reasoning flag is reasoning_tokens=0.
+    # If a future model needs a budget cap, use ``reasoning_effort="minimal"``
+    # instead — never the max_tokens form.
+    extra: dict = {}
+    if model.startswith("openrouter/"):
+        extra["extra_body"] = {"provider": {"allow_fallbacks": False}}
+    # Lazy import so the SOCKS-proxy scrub above runs first and so this
+    # module can still be imported without hitting backend.py's stdlib deps.
+    from .backend import backend_base
+    backend_url = backend_base(api_key=api_key)
+    if backend_url:
+        extra["api_base"] = backend_url
+        session_id = os.environ.get("SOURCEINDEX_SESSION_ID")
+        if session_id:
+            extra.setdefault("extra_headers", {})["x-session-id"] = session_id
+    t0 = time.perf_counter()
+    try:
+        response = litellm.completion(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=max_tokens,
+            api_key=api_key,
+            **extra,
+        )
+    except Exception as e:
+        _log.debug(
+            "LLM call failed",
+            extra={
+                "event": "llm.call",
+                "model": model,
+                "prompt_len": len(prompt),
+                "max_tokens": max_tokens,
+                "latency_ms": elapsed_ms(t0),
+                "success": False,
+                "error_type": type(e).__name__,
+                "error": str(e),
+            },
+        )
+        raise
+    text = (response.choices[0].message.content or "").strip()
+    usage = _extract_usage(response, model=model)
+    _log.debug(
+        "LLM call complete",
+        extra={
+            "event": "llm.call",
+            "model": model,
+            "prompt_len": len(prompt),
+            "response_len": len(text),
+            "max_tokens": max_tokens,
+            "latency_ms": elapsed_ms(t0),
+            "input_tokens": usage.input_tokens,
+            "output_tokens": usage.output_tokens,
+            "cache_read_tokens": usage.cache_read_tokens,
+            "cache_creation_tokens": usage.cache_creation_tokens,
+            "cost_usd": round(usage.cost_usd, 6),
+            "success": True,
+        },
+    )
+    return (text or None), usage

sourceindex/lib/log.py

@@ -0,0 +1,228 @@
+"""Centralized logging for the sourceindex package.
+
+All sourceindex modules obtain their logger via ``get_logger(__name__)``.
+The root logger is ``sourceindex``; child loggers (``sourceindex.build``,
+``sourceindex.search``, …) inherit handlers and level from it, so a single
+``configure()`` call governs the whole package.
+
+Two destinations:
+
+- **stderr** — plain text. Reproduces the pre-logging
+  ``print(f"[sourceindex] ...", file=sys.stderr)`` surface byte-for-byte at
+  INFO so log scrapers / eval harnesses keep working. Level controlled by
+  ``--log-level`` / ``SOURCEINDEX_LOG_LEVEL`` (default INFO).
+- **<repo>/.sourceindex/sourceindex.log** — JSONL, one event per line.
+  Always at DEBUG. Rotated via ``RotatingFileHandler`` (10 MB × 5 backups).
+  Attached only when ``repo_root`` is supplied to ``configure()`` — the
+  ``statusline`` / ``report-savings`` subcommands pass nothing and stay
+  file-silent.
+
+Disable the file handler with ``SOURCEINDEX_NO_LOG_FILE=1`` or pass
+``--no-log-file``. Override the path via ``--log-file PATH`` or
+``SOURCEINDEX_LOG_FILE=...``.
+
+JSONL schema: every line carries ``ts`` / ``level`` / ``logger`` /
+``message``. Callers can tag analytics-worthy events with structured
+fields by passing ``extra={"event": "build.done", "elapsed_s": 12.4, ...}``
+to the logger call — extras land at the top level of the JSON object
+alongside the standard fields.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import logging.handlers
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .env import env_truthy
+
+ROOT_NAME = "sourceindex"
+ENV_LEVEL = "SOURCEINDEX_LOG_LEVEL"
+ENV_LOG_FILE = "SOURCEINDEX_LOG_FILE"
+ENV_NO_FILE = "SOURCEINDEX_NO_LOG_FILE"
+LOG_FILENAME = "sourceindex.log"
+DEFAULT_MAX_BYTES = 10 * 1024 * 1024
+DEFAULT_BACKUP_COUNT = 5
+
+_PREFIX = "[sourceindex]"
+
+# Snapshot the built-in LogRecord field names by constructing a placeholder
+# record at import time. Anything *not* in this set was added via
+# ``extra={...}`` and gets surfaced as a top-level field in the JSONL
+# output. Snapshotting (vs. a hand-written list) tracks stdlib additions
+# automatically — e.g. ``taskName`` arrived in Python 3.12.
+_STD_RECORD_ATTRS = frozenset(
+    logging.LogRecord("", 0, "", 0, "", None, None).__dict__.keys()
+) | {"asctime", "message"}
+
+
+class _StderrFormatter(logging.Formatter):
+    """``[sourceindex] message`` for INFO, ``[sourceindex] LEVEL: message`` else."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        msg = record.getMessage()
+        if record.exc_info:
+            msg = f"{msg}\n{self.formatException(record.exc_info)}"
+        if record.levelno == logging.INFO:
+            return f"{_PREFIX} {msg}"
+        return f"{_PREFIX} {record.levelname}: {msg}"
+
+
+class _JsonlFormatter(logging.Formatter):
+    """One JSON object per record. ``ts``/``level``/``logger``/``message``
+    are always emitted; any extras supplied via ``extra={...}`` become
+    top-level fields. Non-JSON-serializable extras fall back to ``repr``
+    so a bad value can't sink the line."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        obj: dict = {
+            "ts": datetime.fromtimestamp(record.created, timezone.utc).strftime(
+                "%Y-%m-%dT%H:%M:%S.%fZ"
+            ),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        if record.exc_info:
+            obj["traceback"] = self.formatException(record.exc_info)
+        for key, val in record.__dict__.items():
+            if key not in _STD_RECORD_ATTRS and not key.startswith("_"):
+                obj[key] = val
+        return json.dumps(obj, separators=(",", ":"), ensure_ascii=False, default=repr)
+
+
+def configure(
+    level: str | int | None = None,
+    *,
+    repo_root: Path | str | None = None,
+    log_file: Path | str | None = None,
+    enable_file: bool = True,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+    backup_count: int = DEFAULT_BACKUP_COUNT,
+) -> logging.Logger:
+    """Install handlers + set the root level. Idempotent."""
+    logger = logging.getLogger(ROOT_NAME)
+    stderr_level = _resolve_level(level)
+    file_path = _resolve_log_path(log_file, repo_root) if enable_file else None
+    file_attached = _attach_file_handler(logger, file_path, max_bytes, backup_count)
+    _attach_stderr_handler(logger, stderr_level)
+
+    # Logger level is the floor — must allow DEBUG when the file is on so
+    # DEBUG records reach the file handler. The stderr handler does its own
+    # level filtering downstream.
+    logger.setLevel(logging.DEBUG if file_attached else stderr_level)
+
+    litellm_logger = logging.getLogger("LiteLLM")
+    if litellm_logger.level == 0:
+        litellm_logger.setLevel(logging.WARNING)
+
+    return logger
+
+
+def get_logger(name: str = ROOT_NAME) -> logging.Logger:
+    """Return a logger for ``name`` (typically ``__name__``)."""
+    return logging.getLogger(name)
+
+
+def log_file_error(
+    logger: logging.Logger,
+    path: str | Path,
+    exc: BaseException,
+    *,
+    phase: str,
+) -> None:
+    """Emit a ``file.read_error`` DEBUG event for a swallowed file IO failure.
+
+    Centralizes the OSError-on-read pattern (read/stat/etc. fails, caller
+    continues with the file dropped from the index) so every site reports
+    the same shape: path, phase, error type, error message.
+    """
+    logger.debug(
+        f"File operation failed ({phase}): {path}",
+        extra={
+            "event": "file.read_error",
+            "phase": phase,
+            "path": str(path),
+            "error_type": type(exc).__name__,
+            "error": str(exc),
+        },
+    )
+
+
+# ── internals ──────────────────────────────────────────────────────────────
+
+
+def _resolve_level(level: str | int | None) -> int:
+    if level is None:
+        level = os.environ.get(ENV_LEVEL) or "INFO"
+    if isinstance(level, int):
+        return level
+    return logging._nameToLevel.get(str(level).upper(), logging.INFO)
+
+
+def _resolve_log_path(
+    explicit_path: Path | str | None, repo_root: Path | str | None
+) -> Path | None:
+    if env_truthy(ENV_NO_FILE):
+        return None
+    if explicit_path:
+        return Path(explicit_path).expanduser().resolve()
+    env_path = os.environ.get(ENV_LOG_FILE)
+    if env_path:
+        return Path(env_path).expanduser().resolve()
+    if repo_root is not None:
+        # Lazy import — log.py is foundational and used by every module.
+        from ..daemon import IndexDir
+        return IndexDir.for_repo(Path(repo_root)).path / LOG_FILENAME
+    return None
+
+
+def _attach_stderr_handler(logger: logging.Logger, level: int) -> None:
+    for h in logger.handlers:
+        if getattr(h, "_si_stderr", False):
+            h.setLevel(level)
+            return
+    h = logging.StreamHandler(sys.stderr)
+    h.setFormatter(_StderrFormatter())
+    h.setLevel(level)
+    h._si_stderr = True  # type: ignore[attr-defined]
+    logger.addHandler(h)
+    logger.propagate = False
+
+
+def _attach_file_handler(
+    logger: logging.Logger,
+    path: Path | None,
+    max_bytes: int,
+    backup_count: int,
+) -> bool:
+    """Attach a JSONL rotating file handler at ``path``.
+
+    Returns True iff a handler is now attached (already-present or freshly
+    installed). False on disabled or on an OSError that prevents opening
+    the file (silently degrades to stderr-only).
+    """
+    if path is None:
+        return False
+    for h in logger.handlers:
+        if getattr(h, "_si_file", False):
+            return True
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        handler = logging.handlers.RotatingFileHandler(
+            path,
+            maxBytes=max_bytes,
+            backupCount=backup_count,
+            encoding="utf-8",
+        )
+    except OSError:
+        return False
+    handler.setFormatter(_JsonlFormatter())
+    handler.setLevel(logging.DEBUG)
+    handler._si_file = True  # type: ignore[attr-defined]
+    logger.addHandler(handler)
+    return True

sourceindex/lib/registry.py

@@ -0,0 +1,75 @@
+"""Local registry of repos this user has indexed.
+
+`sourceindex init` appends here; `sourceindex deinit` removes. The login-time
+sweep (`sourceindex deinit --all`) walks the file looking for entries whose
+`repo_path` no longer exists and tears down their per-repo state.
+
+The hash → path direction is one-way (sha256), so without this registry we
+have no way to discover orphaned keys after a reboot wipes `XDG_RUNTIME_DIR`.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+Entry = tuple[str, str]  # (repo_hash, repo_path)
+
+
+def _registry_path() -> Path:
+    xdg = os.environ.get("XDG_CONFIG_HOME")
+    base = Path(xdg).expanduser() if xdg else Path.home() / ".config"
+    return base / "sourceindex" / "repos.tsv"
+
+
+def read() -> list[Entry]:
+    path = _registry_path()
+    if not path.is_file():
+        return []
+    entries: list[Entry] = []
+    for raw in path.read_text().splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        parts = line.split("\t", 1)
+        if len(parts) != 2:
+            continue
+        entries.append((parts[0], parts[1]))
+    return entries
+
+
+def add(repo_hash: str, repo_path: str) -> None:
+    """Record (repo_hash, repo_path). If the hash already exists, update its
+    path — handles re-init after a path rename."""
+    entries = read()
+    found = False
+    updated: list[Entry] = []
+    for h, p in entries:
+        if h == repo_hash:
+            updated.append((h, repo_path))
+            found = True
+        else:
+            updated.append((h, p))
+    if not found:
+        updated.append((repo_hash, repo_path))
+    _write(updated)
+
+
+def remove(repo_hash: str) -> None:
+    """Drop the entry for repo_hash. No-op if absent."""
+    entries = [(h, p) for h, p in read() if h != repo_hash]
+    _write(entries)
+
+
+def _write(entries: list[Entry]) -> None:
+    path = _registry_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        path.parent.chmod(0o700)
+    except OSError:
+        pass
+    text = "".join(f"{h}\t{p}\n" for h, p in entries)
+    tmp = path.with_suffix(".tsv.tmp")
+    tmp.write_text(text)
+    tmp.replace(path)

sourceindex/lib/timing.py

@@ -0,0 +1,10 @@
+"""Timing helpers shared by callers that emit ``latency_ms`` log fields."""
+
+from __future__ import annotations
+
+import time
+
+
+def elapsed_ms(start: float) -> int:
+    """Whole milliseconds between ``start`` (a ``time.perf_counter()`` value) and now."""
+    return int((time.perf_counter() - start) * 1000)